support better rebuilds in gentoo

* run @golang-rebuild after upgrade of dev-lang/go
* rebuild dependencies to updated go modules
* run @rust-rebuild after upgrade of dev-lang/rust
* run perl-cleaner after upgrade of dev-lang/perl
* run @live-rebuild during upgrade

sau/gentoo.py

@@ -11,6 +11,7 @@ SYSTEMCTL='/usr/bin/systemctl'
 EMERGE_PATH='/usr/bin/emerge'
 EQUERY_PATH='/usr/bin/equery'
 EMAINT_PATH='/usr/sbin/emaint'
+PCLEAN_PATH='/usr/bin/perl-cleaner'
 GRUB_MKCONFIG='/usr/sbin/grub-mkconfig'
 
 # parsing output from eix -Ttnc
@@ -109,7 +110,8 @@ def _sync_portage():
         for line in err.splitlines():
             log.warning("stderr: {}".format(line))
 
-def is_system_package(atom):
+def is_system_package(atom, eclasses):
+    log = logging.getLogger(sau.LOGNAME)
     name=re.sub(r'^[<=>]*(.*?)(?:-\d)?(?:::\w+)?$', r'\1', atom)
 
     # sys-boot/ category should probably always be considered
@@ -124,13 +126,20 @@ def is_system_package(atom):
     if re.search(r'^sys-libs/(glibc|musl)', name):
         return True
 
-    # anything that requires boot-partition to be mounted should be considered
+    if any([
-    # system packages. So far I've identified these eclasses that touches
+            x in eclasses for x in [
-    # boot/kernel:
+                'dist-kernel-utils',
-    # * dist-kernel-utils
+                'linux-mod',
-    # * linux-mod-r1
+                'kernel-install' ]
-    # * kernel-install
+            ]):
-    test_re = re.compile(r'^\s*inherit\s+.*(dist-kernel-utils|linux-mod|kernel-install)')
+        return True
+    return False
+
+def get_eclasses(atom):
+    log = logging.getLogger(sau.LOGNAME)
+    eclasses = []
+    name=re.sub(r'^[<=>]*(.*?)(?:-\d)?(?:::\w+)?$', r'\1', atom)
+    test_re = re.compile(r'^\s*inherit\s+')
     cmd=[ EQUERY_PATH, 'w', name ]
     ret, out, err = sau.helpers.exec_cmd(cmd)
     if not ret == 0:
@@ -144,11 +153,23 @@ def is_system_package(atom):
         return True
     with open(path, 'r', encoding='utf-8') as f:
         for line in f.readlines():
+            if eclasses and eclasses[-1] == '\\':
+                eclasses = eclasses[:-1]
+                eclasses.extend(line.split())
             if re.search(test_re, line):
-                return True
+                if re.search(test_re, line):
+                    eclasses.extend(line.split()[1:])
 
-    return False
+    # Remove revisions from eclasses, hopefully makes it less messy if they get
+    # updated
+    eclasses = [sub(r'^(.*?)-r\d+', r'\1', x) for x in eclasses]
+    return eclasses
 
+def get_dependencies(atom):
+    cmd=[ EQUERY_PATH, '-q', 'd', '-F', '$cp', name ]
+    ret, out, err = sau.helpers.exec_cmd(cmd)
+    dependencies = [l.strip() for l in out.splitlines()]
+    return dependencies
 
 
 def pkg_upgrade():
@@ -165,6 +186,7 @@ def pkg_upgrade():
     
     default_version_sens = conf.getint('default', 'version_sensitivity', fallback=1)
 
+    ## Query upgradeable packages
     cmd = [ EMERGE_PATH, '--color', 'n', '-uDNpq', '--with-bdeps=y', '@world' ]
     ret, out, err = sau.helpers.exec_cmd(cmd)
 
@@ -178,6 +200,7 @@ def pkg_upgrade():
 
     do_rebuild = True
     do_grub = False
+    rebuild_packages = {}
     for line in out.splitlines():
         if re.match(ignore_re, line):
             continue
@@ -201,7 +224,12 @@ def pkg_upgrade():
             log.error('{} -- {} -> {} configured level {} > pkg level {}'.format(name, old, new, sens, common))
             do_rebuild = False
 
-        if is_system_package(f'{name}-{new}'):
+        nameversion = f'{name}-{new}'
+        eclasses = get_eclasses(nameversion)
+        rebuild_packages[name] = eclasses
+
+    for package,eclasses in rebuild_packages:
+        if is_system_package(package, eclasses):
             if do_system_upgrade:
                 do_grub = True
             else:
@@ -210,6 +238,11 @@ def pkg_upgrade():
     if not do_rebuild:
         raise sau.errors.UpgradeError('Some packages require manual attention, did not upgrade')
 
+    if not rebuild_packages:
+        log.info('No packages to upgrade')
+        return False
+
+    ## Actual upgrade
     cmd = [ EMERGE_PATH, '--color', 'n', '-uDNq', '--with-bdeps=y', '@world' ]
     ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000)
 
@@ -226,22 +259,113 @@ def pkg_upgrade():
             if line.startswith(' * '):
                 log.warning(line)
 
-    cmd = [ EMERGE_PATH, '--color', 'n', '-q', '@preserved-rebuild' ]
+    # from here on we shouldn't need to rebuild the upgraded packages again
-    ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000)
+    exclude_list = '--exclude '.join(rebuild_packages.keys()).split()
 
+    # Rebuild go
+    go_packages = []
+    cmd = None
+    for package,eclasses in rebuild_packages.items():
+        if 'go-module' in eclasses or package == 'dev-lang/go':
+            go_packages.append(package)
+    if 'dev-lang/go' in go_packages:
+        log.info("Running golang-rebuild due to update of dev-lang/go")
+        cmd = [
+                EMERGE_PATH,
+                '--color', 'n',
+                '-q',
+                '--usepkg-exclude', '*/*',
+                '@golang-rebuild',
+                '--exclude' ] + exclude_list
+    elif go_packages:
+        dependencies = []
+        for package in go_packages:
+            dependencies.append(get_dependencies(package))
+        dependencies = set(dependencies)
+        upgraded = set(rebuild_packages.keys())
+        not_upgraded = dependencies-upgraded
+        if not_upgraded:
+            log.info(f'Rebuilding packages dependant of go modules {", ".join(go_packages)}')
+            cmd = [
+                EMERGE_PATH,
+                '--color', 'n',
+                '-q',
+                '--usepkg-exclude', '*/*'] + not_upgraded ]
+    if cmd:
+        ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000)
+        if ret != 0 or err:
+            log.error('Rebuild of go packages returned {}'.format(ret))
+            for line in out.splitlines():
+                log.error('stdout: {}'.format(line))
+            for line in err.splitlines():
+                log.error('stderr: {}'.format(line))
+            raise sau.errors.UpgradeError(f'Error during go rebuild')
+        else:
+            log.info('go rebuild complete')
+            for line in out.splitlines():
+                if line.startswith(' * '):
+                    log.warning(line)
+
+    # rebuild rust
+    if 'dev-lang/rust' in rebuild_packages:
+        log.info("Running rust-rebuild due to update of dev-lang/rust")
+        cmd = [
+                EMERGE_PATH,
+                '--color', 'n',
+                '-q',
+                '--usepkg-exclude', '*/*',
+                '@rust-rebuild',
+                '--exclude' ] + exclude_list
+        ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000)
+        if ret != 0 or err:
+            log.error('Rebuild of rust packages returned {}'.format(ret))
+            for line in out.splitlines():
+                log.error('stdout: {}'.format(line))
+            for line in err.splitlines():
+                log.error('stderr: {}'.format(line))
+            raise sau.errors.UpgradeError(f'Error during rust rebuild')
+        else:
+            log.info('rust rebuild complete')
+            for line in out.splitlines():
+                if line.startswith(' * '):
+                    log.warning(line)
+
+    # run perl-cleaner
+    if 'dev-lang/perl' in rebuild_packages:
+        log.info("Running perl-cleaner due to perl upgrade")
+        cmd = [ PCLEAN_PATH, '--all', '--', '-q', '--usepkg-exclude', '*/*']
+        ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000)
+        if ret != 0 or err:
+            log.error('perl-cleaner failed with code {}'.format(ret))
+            for line in out.splitlines():
+                log.error('stdout: {}'.format(line))
+            for line in err.splitlines():
+                log.error('stderr: {}'.format(line))
+            raise sau.errors.UpgradeError(f'Error during perl-cleaner')
+        else:
+            log.info('perl-cleaner complete')
+            for line in out.splitlines():
+                if line.startswith(' * '):
+                    log.warning(line)
+
+    # rebuild live packages
+    cmd = [ EMERGE_PATH, '--color', 'n', '-q', '--usepkg-exclude', '*/*', '@live-rebuild' ]
+    ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600)
     if ret != 0 or err:
-        log.error('preserved-rebuild returned {}'.format(ret))
+        log.error('live-rebuild returned {}'.format(ret))
         for line in out.splitlines():
             log.error('stdout: {}'.format(line))
         for line in err.splitlines():
             log.error('stderr: {}'.format(line))
-        raise sau.errors.UpgradeError(f'Error during preserved-rebuild')
+        raise sau.errors.UpgradeError(f'Error during live-rebuild')
     else:
-        log.info('preserved-rebuild complete')
+        log.info('live-rebuild complete')
         for line in out.splitlines():
             if line.startswith(' * '):
                 log.warning(line)
 
+
+    # Depclean
     if conf.getboolean('default', 'do_depclean', fallback=False):
         cmd = [ EMERGE_PATH, '--color', 'n', '-q', '--depclean' ]
         ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600)
@@ -258,6 +382,26 @@ def pkg_upgrade():
                 if line.startswith(' * '):
                     log.warning(line)
 
+
+    # Preserved rebuild
+    cmd = [ EMERGE_PATH, '--color', 'n', '-q', '@preserved-rebuild' ]
+    ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000)
+
+    if ret != 0 or err:
+        log.error('preserved-rebuild returned {}'.format(ret))
+        for line in out.splitlines():
+            log.error('stdout: {}'.format(line))
+        for line in err.splitlines():
+            log.error('stderr: {}'.format(line))
+        raise sau.errors.UpgradeError(f'Error during preserved-rebuild')
+    else:
+        log.info('preserved-rebuild complete')
+        for line in out.splitlines():
+            if line.startswith(' * '):
+                log.warning(line)
+
+
+    # doing grub reconfig and clean old kernels
     if do_grub and os.path.exists(GRUB_MKCONFIG):
         keep_kernels = conf.getint('default', 'keep_kernels', fallback=4)
         if keep_kernels < 1: