bump version

bin/sau

@@ -8,6 +8,7 @@ import sys
 import time
 
 import sau
+import sau.errors
 import sau.services
 import sau.platforms
 
@@ -44,12 +45,23 @@ def init():
     log.addHandler(handler)
 
 
-def fork_and_reboot():
+def fork_and_reboot(report_success=True):
     log = logging.getLogger(sau.LOGNAME)
+    if report_success:
+        exit_code=0
+    else:
+        exit_code=1
+
+    if os.path.exists('/proc/1/comm'):
+        with open('/proc/1/comm', 'r') as f:
+            if f.readline().strip() == 'systemd':
+                os.execl('/usr/bin/systemctl', 'reboot')
+                log.error("Failed to execl?")
+                sys.exit(1)
     try:
         pid = os.fork()
         if pid != 0:
-            sys.exit(0)
+            sys.exit(exit_code)
     except OSError as err:
         log.error("Fork #1 failed when going for reboot: {}".format(err))
         sys.exit(1)
@@ -76,8 +88,6 @@ def fork_and_reboot():
     os.dup2(stdout.fileno(), sys.stdout.fileno())
     os.dup2(stderr.fileno(), sys.stderr.fileno())
 
-    log.warning("New fork!")
-
     # sleep for a short while to give parent time to exit
     time.sleep(30)
     try:
@@ -95,11 +105,15 @@ def main():
     reboot_required = False
     reboot_recommended = False
 
+    try:
         if conf.getboolean('default', 'do_system_upgrade', fallback=True):
             reboot_required = platform.system_upgrade()
 
         if conf.getboolean('default', 'do_package_upgrade', fallback=True):
-        platform.pkg_upgrade()
+            reboot_required = reboot_required or platform.pkg_upgrade()
+    except sau.errors.UpgradeError as e:
+        log.error(f'Upgrade failed: {e}')
+        return 1
 
     if conf.getboolean('default', 'do_service_restart', fallback=True):
         reboot_recommended = sau.services.restart_services()
@@ -110,7 +124,17 @@ def main():
         elif reboot_recommended:
             log.info('Rebooting because service restarts did not close all deleted files')
         if reboot_required or reboot_recommended:
-            fork_and_reboot()
+            fork_and_reboot(report_success=conf.getboolean('default', 'reboot_is_success', Fallback=True))
+
+    if reboot_required:
+        log.warning("Upgrade was success, but a reboot is required due to a system upgrade")
+        return 1
+    elif reboot_recommended:
+        log.warning("Some services still uses old, deleted, files. You probably want to reboot")
+        return 1
+
+    return 0
+
 
 def _conf_level_to_logging_level(conf_level):
     if conf_level.lower() == 'debug':

config.cfg

@@ -16,9 +16,17 @@ version_sensitivity=1
 # not close all deleted files (any platform)
 do_reboot=no
 
-# FreeBSD system update (freebsd-update fetch install, not freebsd-update upgrade)
+# Set to 0 to exit with failure code when going for reboot.
+reboot_is_success=yes
+
+# Attempt to do a system upgrade
+# FreeBSD: upgrade to latest patch version using freebsd-update fetch install
+# Gentoo: allow upgrade of sys-kernel/-packages, clean old kernels, and update grub-config
 do_system_upgrade=yes
 
+# On Gentoo kernel upgrades, remove all but the last keep_kernels kernels from /boot
+keep_kernels=4
+
 # upgrade packages
 do_package_upgrade=yes
 
@@ -46,9 +54,7 @@ syslog_loglevel=info
 # Gentoo uses the category/package naming scheme
 dev-db/postgresql=1
 
-# Gentoo kernel stuff should be updated manually
-sys-kernel/gentoo-sources=99
-sys-kernel/spl=99
+# It's safer to upgrade zfs manually
 sys-fs/zfs-kmod=99
 
 # FreeBSD uses the short package name (without category)

sau/__init__.py

@@ -1,2 +1 @@
-
 LOGNAME="sau"

sau/errors.py

@@ -5,3 +5,5 @@ class PlatformNotSupported(Exception):
 class UnknownServiceError(Exception):
     pass
 
+class UpgradeError(Exception):
+    pass

sau/freebsd.py

@@ -132,5 +132,14 @@ def pkg_upgrade():
         for line in err.splitlines():
             log.warning('stderr: {}'.format(line))
 
+    if conf.getboolean('default', 'do_depclean', fallback=False):
+        cmd = [ PKG_PATH, 'autoremove', '-yq' ]
+        ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600)
+        if ret != 0 or err:
+            log.warning('{} failed:'.format(' '.join(cmd)))
+            for line in out.splitlines():
+                log.warning('stdout: {}'.format(line))
+            for line in err.splitlines():
+                log.warning('stderr: {}'.format(line))
     return True
 

sau/gentoo.py

@@ -7,8 +7,11 @@ import sau.helpers
 
 EIX_SYNC_PATH='/usr/bin/eix-sync'
 RC_SERVICE_PATH='/sbin/rc-service'
+SYSTEMCTL='/usr/bin/systemctl'
 EMERGE_PATH='/usr/bin/emerge'
 EQUERY_PATH='/usr/bin/equery'
+EMAINT_PATH='/usr/sbin/emaint'
+GRUB_MKCONFIG='/usr/sbin/grub-mkconfig'
 
 # parsing output from eix -Ttnc
 package_re = re.compile('^\[([^\]])\] ([^ ]*) \((.*)\): .*$')
@@ -18,18 +21,17 @@ slot_re = re.compile('^(\(~\))?([^\(]+)(\([^\)]+\))$')
 def identify_service_from_bin(exe):
     log = logging.getLogger(sau.LOGNAME)
 
+    with open('/proc/1/comm', 'r') as f:
+        if f.readline().strip() == 'systemd':
+            init_script_re = re.compile(r'[^/]*(.*)\.service$')
+        else:
             init_script_re = re.compile(r'/etc/init\.d/(.*)')
 
     cmd = [ EQUERY_PATH, '-Cq', 'b', exe ]
     ret, out, err = sau.helpers.exec_cmd(cmd)
 
     if ret != 0:
-        log.warning("searching for owner of {} failed:".format(exe))
-        for line in out.splitlines():
-            log.warning("stdout: {}".format(line))
-        for line in err.splitlines():
-            log.warning("stderr: {}".format(line))
-        return None
+        raise sau.errors.UnknownServiceError("searching for owner of {} failed:".format(exe))
     
     pkg = out.strip()
     cmd = [ EQUERY_PATH, '-Cq', 'f', pkg ]
@@ -48,9 +50,9 @@ def identify_service_from_bin(exe):
         if match:
             matches.add(match.group(1))
     if len(matches) < 1:
-        log.warning('Could not find any init script in package {}'.format(pkg))
+        raise sau.errors.UnknownServiceError('Could not find any init script in package {}'.format(pkg))
     elif len(matches) > 1:
-        log.warning('Found multiple init script in package {}'.format(pkg))
+        raise sau.errors.UnknownServiceError('Found multiple init script in package {}'.format(pkg))
     else:
         return matches.pop()
     return None
@@ -58,6 +60,10 @@ def identify_service_from_bin(exe):
 
 def restart_service(service):
     log = logging.getLogger(sau.LOGNAME)
+    with open('/proc/1/comm', 'r') as f:
+        if f.readline().strip() == 'systemd':
+            cmd = [ SYSTEMCTL, 'restart', service ]
+        else:
             cmd = [ RC_SERVICE_PATH, service, 'restart' ]
     ret, out, err = sau.helpers.exec_cmd(cmd)
 
@@ -72,7 +78,7 @@ def restart_service(service):
 
 def system_upgrade():
     log = logging.getLogger(sau.LOGNAME)
-    log.debug('Gentoo has no concept of system upgrade, ignoring...')
+    log.debug('Gentoo "system_upgrade" is done at package upgrade stage; ignoring here...')
     return False
 
 def _sync_portage():
@@ -86,7 +92,18 @@ def _sync_portage():
         ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600)
 
     if ret != 0:
-        log.warning("Portage sync failed:")
+        log.error("Portage sync failed:")
+        for line in out.splitlines():
+            log.error("stdout: {}".format(line))
+        for line in err.splitlines():
+            log.error("stderr: {}".format(line))
+        raise sau.errors.UpgradeError(f'Sync command {cmd} failed')
+
+    cmd = [ EMAINT_PATH, '-f', 'all' ]
+    ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600)
+
+    if ret != 0:
+        log.warning("emaint failed:")
         for line in out.splitlines():
             log.warning("stdout: {}".format(line))
         for line in err.splitlines():
@@ -94,9 +111,11 @@ def _sync_portage():
 
 
 
+
 def pkg_upgrade():
     log = logging.getLogger(sau.LOGNAME)
     conf = sau.config
+    do_system_upgrade = conf.getboolean('default', 'do_system_upgrade', fallback=False)
 
     if conf.getboolean('default', 'do_reposync', fallback=True):
         _sync_portage()
@@ -107,7 +126,7 @@ def pkg_upgrade():
     
     default_version_sens = conf.getint('default', 'version_sensitivity', fallback=1)
 
-    cmd = [ EMERGE_PATH, '--color', 'n', '-uDNpq', '@world' ]
+    cmd = [ EMERGE_PATH, '--color', 'n', '-uDNpq', '--with-bdeps=y', '@world' ]
     ret, out, err = sau.helpers.exec_cmd(cmd)
 
     if not ret == 0:
@@ -116,9 +135,10 @@ def pkg_upgrade():
             log.error('stdout: {}'.format(line))
         for line in err.splitlines():
             log.error('stderr: {}'.format(line))
-        return False
+        raise sau.errors.UpgradeError(f'Failed to calculate upgrade path')
 
     do_rebuild = True
+    do_grub = False
     for line in out.splitlines():
         if re.match(ignore_re, line):
             continue
@@ -139,22 +159,27 @@ def pkg_upgrade():
         if sens <= common:
             log.info('{} -- {} -> {} configured level {} <= pkg level {}'.format(name, old, new, sens, common))
         else:
-            log.warning('{} -- {} -> {} configured level {} > pkg level {}'.format(name, old, new, sens, common))
+            log.error('{} -- {} -> {} configured level {} > pkg level {}'.format(name, old, new, sens, common))
             do_rebuild = False
+        if name.startswith('sys-kernel/'):
+            if do_system_upgrade:
+                do_grub = True
+            else:
+                raise sau.errors.UpgradeError(f"Kernel package {name} has an update, but system upgrade is disabled")
 
     if not do_rebuild:
-        log.warning('Some packages require manual attention, did not upgrade')
-        return False
+        raise sau.errors.UpgradeError('Some packages require manual attention, did not upgrade')
 
-    cmd = [ EMERGE_PATH, '--color', 'n', '-uDNq', '@world' ]
+    cmd = [ EMERGE_PATH, '--color', 'n', '-uDNq', '--with-bdeps=y', '@world' ]
     ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000)
 
     if ret != 0 or err:
-        log.warning('emerge returned {}'.format(ret))
+        log.error('emerge returned {}'.format(ret))
         for line in out.splitlines():
-            log.warning('stdout: {}'.format(line))
+            log.error('stdout: {}'.format(line))
         for line in err.splitlines():
-            log.warning('stderr: {}'.format(line))
+            log.error('stderr: {}'.format(line))
+        raise sau.errors.UpgradeError(f'Error during upgrade')
     else:
         log.info('upgrade complete')
         for line in out.splitlines():
@@ -165,11 +190,12 @@ def pkg_upgrade():
     ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000)
 
     if ret != 0 or err:
-        log.warning('preserved-rebuild returned {}'.format(ret))
+        log.error('preserved-rebuild returned {}'.format(ret))
         for line in out.splitlines():
-            log.warning('stdout: {}'.format(line))
+            log.error('stdout: {}'.format(line))
         for line in err.splitlines():
-            log.warning('stderr: {}'.format(line))
+            log.error('stderr: {}'.format(line))
+        raise sau.errors.UpgradeError(f'Error during preserved-rebuild')
     else:
         log.info('preserved-rebuild complete')
         for line in out.splitlines():
@@ -180,13 +206,42 @@ def pkg_upgrade():
         cmd = [ EMERGE_PATH, '--color', 'n', '-q', '--depclean' ]
         ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600)
         if ret != 0 or err:
-            log.warning('depclean returned {}'.format(ret))
+            log.error('depclean returned {}'.format(ret))
             for line in out.splitlines():
-                log.warning('stdout: {}'.format(line))
+                log.error('stdout: {}'.format(line))
             for line in err.splitlines():
-                log.warning('stderr: {}'.format(line))
+                log.error('stderr: {}'.format(line))
+            raise sau.errors.UpgradeError(f'Error during depclean')
         else:
             log.info('depclean complete')
             for line in out.splitlines():
                 if line.startswith(' * '):
                     log.warning(line)
+
+    if do_grub and os.path.exists(GRUB_MKCONFIG):
+        keep_kernels = conf.getint('default', 'keep_kernels', fallback=4)
+        if keep_kernels < 1:
+            log.error('keep_kernels cannot be less than one; falling back to default')
+            keep_kernels = 4
+        for root, dirs, files in os.walk('/boot'):
+            for sysfile in ['config', 'initramfs', 'System.map', 'vmlinuz', 'kernel']:
+                match = sorted(
+                        [f for f in files if f.startswith(f'{sysfile}-')],
+                        reverse=True)
+                for f in match[keep_kernels:]:
+                    log.debug(f"Removing old kernel file {f}")
+                    os.remove(os.path.join(root, f))
+            break
+
+        cmd = [ GRUB_MKCONFIG, '-o', '/boot/grub/grub.cfg' ]
+        ret, out, err = sau.helpers.exec_cmd(cmd)
+        if ret != 0:
+            log.error(f"grub-mkconfig returned {ret}:")
+            for line in out.splitlines():
+                log.error('stdout: {}'.format(line))
+            for line in err.splitlines():
+                log.error('stderr: {}'.format(line))
+            raise sau.errors.UpgradeError(f'Failed to reconfiugre grub')
+        else:
+            log.info("grub reconfigured")
+            return True

sau/services.py

@@ -7,6 +7,7 @@ import psutil
 
 import sau
 import sau.errors
+import sau.helpers
 import sau.platforms
 
 proc_fd_map_re = re.compile(r'^.*(/[^\(]*) \(deleted\)$')
@@ -16,7 +17,7 @@ def _warn(policy, msg):
     if not policy.startswith('silent'):
         log.warning(msg)
 
-def _get_deleted_open_files(proc):
+def get_deleted_open_files(proc):
     log = logging.getLogger(sau.LOGNAME)
     files = set()
 
@@ -62,25 +63,40 @@ def get_exe_file(name):
                     log.debug('Found binary for {} at {}'.format(name, root))
                     return os.path.join(root, name)
 
+# return all processes with open files
+def _get_processes():
+    log = logging.getLogger(sau.LOGNAME)
+    check_procs = set()
+    for proc in psutil.process_iter():
+        files = get_deleted_open_files(proc)
+        if files:
+            log.debug('{} has open deleted files'.format(proc))
+            check_procs.add(proc)
+
+    return check_procs
+
 def restart_services():
     log = logging.getLogger(sau.LOGNAME)
     platform = sau.platforms.get_platform()
     conf = sau.config
-    check_procs = set()
-    for proc in psutil.process_iter():
-        files = _get_deleted_open_files(proc)
-        if files:
-            log.info('{} has open deleted files'.format(proc))
-            check_procs.add(proc)
 
+    check_procs = _get_processes()
     # wait before the second test
-    time.sleep(1)
+    time.sleep(5)
+
+    on_systemd = False
+    try:
+        init_proc = psutil.Process(pid=1)
+        if init_proc.name() == 'systemd':
+            on_systemd = True
+    except psutil.NoSuchProcess:
+        pass
 
     # perform a second check to remove potential false positives
     service_procs = set()
     retest_procs = set()
     for proc in check_procs:
-        files = _get_deleted_open_files(proc)
+        files = get_deleted_open_files(proc)
         if not files:
             # no deleted open files for this process any longer
             continue
@@ -89,32 +105,60 @@ def restart_services():
         except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
             # either of the above exceptions means the process has quit
             continue
+        if on_systemd:
+            service_procs.add(proc)
+        else:
             parent = _get_top_parent(proc)
-
             service_procs.add(parent)
+
         retest_procs.add(proc)
 
+    recommend_restart = False
     processes = {}
     services = {}
     for proc in service_procs:
         if not proc:
             continue
+
+        service_name = None
         try:
             service_exe = proc.exe()
             proc_name = proc.name()
         except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
             log.debug('{} died before it could be restarted'.format(proc))
             continue
+
+        if on_systemd:
+            if proc.pid == 1:
+                log.info("Upgrade of systemd detected; doing daemon-reexec")
+                ret, _out, _err = sau.helpers.exec_cmd([ '/usr/bin/systemctl', 'daemon-reexec' ])
+                continue
+            ret, unit, err = sau.helpers.exec_cmd([ '/usr/bin/systemctl', 'whoami', f'{proc.pid}' ])
+            unit = unit.strip()
+            name, unit_type = unit.split('.')
+            if ret != 0:
+                log.debug(f'Non-success ({ret}) when checking unit for process: {err}')
+                continue
+            elif unit_type != 'service':
+                log.warning(f'not restarting non-service unit "{unit}"; owner of {proc}')
+            else:
+                _ret, enabled, _err = sau.helpers.exec_cmd([ '/usr/bin/systemctl', 'is-enabled', unit ])
+                enabled = enabled.strip()
+                if enabled in ('enabled', 'static'):
+                    log.warning(f'Unit {name}.service has enable status: {enabled} - will only restart "enabled" services')
+                else:
+                    service_name = name
+        else:
             service_name = _get_service_from_proc(proc)
 
         if not service_name:
-            log.debug('no service for process {}'.format(proc))
+            log.warning('no service for process {}'.format(proc))
+            recommend_restart = True
             continue
 
         services[proc_name] = service_name
         processes[service_name] = [proc]
 
-    recommend_restart = False
     for service in set([x for x in services.values() if x]):
         policy = _get_service_restart_policy(service)
         if policy == 'ignore':
@@ -131,34 +175,17 @@ def restart_services():
 
     tested_parents = set()
     for proc in retest_procs:
-        parent = _get_top_parent(proc)
-        if not parent:
+        try:
+            proc_name = proc.name()
+            if proc_name not in services:
                 continue
-        parent_name = parent.name()
-        if parent in tested_parents:
-            log.debug('{} belongs to already tested parent {}'.format(proc, parent))
+        except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
             continue
 
-        if _get_deleted_open_files(proc):
-            tested_parents.add(parent)
-            service = _get_service_from_proc(parent)
-            if not service:
-                log.warning('could not re-check process {} - failed to identify service'.format(proc))
-                recommend_restart = True
-                continue
+        if get_deleted_open_files(proc):
+            service = services[proc_name]
             policy = _get_service_restart_policy(service)
-
-            log.debug('{} is in service {}'.format(proc, service))
-            if parent_name in services and not services[parent_name]:
-                _warn(policy, '{} (parent {}) does not belong to a service and could not be restarted'.format(proc, parent))
-                recommend_restart = True
-                continue
-            elif parent_name in services:
-                policy = _get_service_restart_policy(service)
-                log.debug('service {} has policy {}'.format(service, policy))
-                if policy in ('ignore', 'warn'):
-                    continue
-            _warn(policy, '{} (parent {}) still has deleted files open'.format(proc, parent))
+            _warn(policy, '{} still has deleted files open'.format(proc, parent))
             recommend_restart = True
     return recommend_restart
 

setup.py

@@ -5,7 +5,7 @@ from setuptools import setup, find_packages
 
 setup(
         name='sau',
-        version='0.9.4',
+        version='1.2.0',
         description='Tool for auto-updating OS and packages',
         author='Feffe',
         author_email='feffe@fulh.ax',