From 8bd417de2de14d2277c8ba96df1c14d344fbd058 Mon Sep 17 00:00:00 2001 From: Fredrik Eriksson Date: Sun, 21 Jul 2024 11:10:26 +0200 Subject: [PATCH] support better rebuilds in gentoo * run @golang-rebuild after upgrade of dev-lang/go * rebuild dependencies to updated go modules * run @rust-rebuild after upgrade of dev-lang/rust * run perl-cleaner after upgrade of dev-lang/perl * run @live-rebuild during upgrade --- sau/gentoo.py | 176 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 160 insertions(+), 16 deletions(-) diff --git a/sau/gentoo.py b/sau/gentoo.py index a028855..3516859 100644 --- a/sau/gentoo.py +++ b/sau/gentoo.py @@ -11,6 +11,7 @@ SYSTEMCTL='/usr/bin/systemctl' EMERGE_PATH='/usr/bin/emerge' EQUERY_PATH='/usr/bin/equery' EMAINT_PATH='/usr/sbin/emaint' +PCLEAN_PATH='/usr/bin/perl-cleaner' GRUB_MKCONFIG='/usr/sbin/grub-mkconfig' # parsing output from eix -Ttnc @@ -109,7 +110,8 @@ def _sync_portage(): for line in err.splitlines(): log.warning("stderr: {}".format(line)) -def is_system_package(atom): +def is_system_package(atom, eclasses): + log = logging.getLogger(sau.LOGNAME) name=re.sub(r'^[<=>]*(.*?)(?:-\d)?(?:::\w+)?$', r'\1', atom) # sys-boot/ category should probably always be considered @@ -124,13 +126,20 @@ def is_system_package(atom): if re.search(r'^sys-libs/(glibc|musl)', name): return True - # anything that requires boot-partition to be mounted should be considered - # system packages. So far I've identified these eclasses that touches - # boot/kernel: - # * dist-kernel-utils - # * linux-mod-r1 - # * kernel-install - test_re = re.compile(r'^\s*inherit\s+.*(dist-kernel-utils|linux-mod|kernel-install)') + if any([ + x in eclasses for x in [ + 'dist-kernel-utils', + 'linux-mod', + 'kernel-install' ] + ]): + return True + return False + +def get_eclasses(atom): + log = logging.getLogger(sau.LOGNAME) + eclasses = [] + name=re.sub(r'^[<=>]*(.*?)(?:-\d)?(?:::\w+)?$', r'\1', atom) + test_re = re.compile(r'^\s*inherit\s+') cmd=[ EQUERY_PATH, 'w', name ] ret, out, err = sau.helpers.exec_cmd(cmd) if not ret == 0: @@ -144,11 +153,23 @@ def is_system_package(atom): return True with open(path, 'r', encoding='utf-8') as f: for line in f.readlines(): + if eclasses and eclasses[-1] == '\\': + eclasses = eclasses[:-1] + eclasses.extend(line.split()) if re.search(test_re, line): - return True + if re.search(test_re, line): + eclasses.extend(line.split()[1:]) - return False + # Remove revisions from eclasses, hopefully makes it less messy if they get + # updated + eclasses = [re.sub(r'^(.*?)-r\d+', r'\1', x) for x in eclasses] + return eclasses +def get_dependencies(atom): + cmd=[ EQUERY_PATH, '-q', 'd', '-F', '$cp', name ] + ret, out, err = sau.helpers.exec_cmd(cmd) + dependencies = [l.strip() for l in out.splitlines()] + return dependencies def pkg_upgrade(): @@ -165,6 +186,7 @@ def pkg_upgrade(): default_version_sens = conf.getint('default', 'version_sensitivity', fallback=1) + ## Query upgradeable packages cmd = [ EMERGE_PATH, '--color', 'n', '-uDNpq', '--with-bdeps=y', '@world' ] ret, out, err = sau.helpers.exec_cmd(cmd) @@ -178,6 +200,7 @@ def pkg_upgrade(): do_rebuild = True do_grub = False + rebuild_packages = {} for line in out.splitlines(): if re.match(ignore_re, line): continue @@ -201,7 +224,12 @@ def pkg_upgrade(): log.error('{} -- {} -> {} configured level {} > pkg level {}'.format(name, old, new, sens, common)) do_rebuild = False - if is_system_package(f'{name}-{new}'): + nameversion = f'{name}-{new}' + eclasses = get_eclasses(nameversion) + rebuild_packages[name] = eclasses + + for package,eclasses in rebuild_packages.items(): + if is_system_package(package, eclasses): if do_system_upgrade: do_grub = True else: @@ -210,6 +238,11 @@ def pkg_upgrade(): if not do_rebuild: raise sau.errors.UpgradeError('Some packages require manual attention, did not upgrade') + if not rebuild_packages: + log.info('No packages to upgrade') + return False + + ## Actual upgrade cmd = [ EMERGE_PATH, '--color', 'n', '-uDNq', '--with-bdeps=y', '@world' ] ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000) @@ -226,22 +259,113 @@ def pkg_upgrade(): if line.startswith(' * '): log.warning(line) - cmd = [ EMERGE_PATH, '--color', 'n', '-q', '@preserved-rebuild' ] - ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000) + # from here on we shouldn't need to rebuild the upgraded packages again + exclude_list = '--exclude '.join(rebuild_packages.keys()).split() + # Rebuild go + go_packages = [] + cmd = None + for package,eclasses in rebuild_packages.items(): + if 'go-module' in eclasses or package == 'dev-lang/go': + go_packages.append(package) + if 'dev-lang/go' in go_packages: + log.info("Running golang-rebuild due to update of dev-lang/go") + cmd = [ + EMERGE_PATH, + '--color', 'n', + '-q', + '--usepkg-exclude', '*/*', + '@golang-rebuild', + '--exclude' ] + exclude_list + elif go_packages: + dependencies = [] + for package in go_packages: + dependencies.append(get_dependencies(package)) + dependencies = set(dependencies) + upgraded = set(rebuild_packages.keys()) + not_upgraded = dependencies-upgraded + if not_upgraded: + log.info(f'Rebuilding packages dependant of go modules {", ".join(go_packages)}') + cmd = [ + EMERGE_PATH, + '--color', 'n', + '-q', + '--usepkg-exclude', '*/*'] + not_upgraded + if cmd: + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000) + if ret != 0 or err: + log.error('Rebuild of go packages returned {}'.format(ret)) + for line in out.splitlines(): + log.error('stdout: {}'.format(line)) + for line in err.splitlines(): + log.error('stderr: {}'.format(line)) + raise sau.errors.UpgradeError(f'Error during go rebuild') + else: + log.info('go rebuild complete') + for line in out.splitlines(): + if line.startswith(' * '): + log.warning(line) + + # rebuild rust + if 'dev-lang/rust' in rebuild_packages: + log.info("Running rust-rebuild due to update of dev-lang/rust") + cmd = [ + EMERGE_PATH, + '--color', 'n', + '-q', + '--usepkg-exclude', '*/*', + '@rust-rebuild', + '--exclude' ] + exclude_list + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000) + if ret != 0 or err: + log.error('Rebuild of rust packages returned {}'.format(ret)) + for line in out.splitlines(): + log.error('stdout: {}'.format(line)) + for line in err.splitlines(): + log.error('stderr: {}'.format(line)) + raise sau.errors.UpgradeError(f'Error during rust rebuild') + else: + log.info('rust rebuild complete') + for line in out.splitlines(): + if line.startswith(' * '): + log.warning(line) + + # run perl-cleaner + if 'dev-lang/perl' in rebuild_packages: + log.info("Running perl-cleaner due to perl upgrade") + cmd = [ PCLEAN_PATH, '--all', '--', '-q', '--usepkg-exclude', '*/*'] + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000) + if ret != 0 or err: + log.error('perl-cleaner failed with code {}'.format(ret)) + for line in out.splitlines(): + log.error('stdout: {}'.format(line)) + for line in err.splitlines(): + log.error('stderr: {}'.format(line)) + raise sau.errors.UpgradeError(f'Error during perl-cleaner') + else: + log.info('perl-cleaner complete') + for line in out.splitlines(): + if line.startswith(' * '): + log.warning(line) + + # rebuild live packages + cmd = [ EMERGE_PATH, '--color', 'n', '-q', '--usepkg-exclude', '*/*', '@live-rebuild' ] + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600) if ret != 0 or err: - log.error('preserved-rebuild returned {}'.format(ret)) + log.error('live-rebuild returned {}'.format(ret)) for line in out.splitlines(): log.error('stdout: {}'.format(line)) for line in err.splitlines(): log.error('stderr: {}'.format(line)) - raise sau.errors.UpgradeError(f'Error during preserved-rebuild') + raise sau.errors.UpgradeError(f'Error during live-rebuild') else: - log.info('preserved-rebuild complete') + log.info('live-rebuild complete') for line in out.splitlines(): if line.startswith(' * '): log.warning(line) + + # Depclean if conf.getboolean('default', 'do_depclean', fallback=False): cmd = [ EMERGE_PATH, '--color', 'n', '-q', '--depclean' ] ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600) @@ -258,6 +382,26 @@ def pkg_upgrade(): if line.startswith(' * '): log.warning(line) + + # Preserved rebuild + cmd = [ EMERGE_PATH, '--color', 'n', '-q', '@preserved-rebuild' ] + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=72000) + + if ret != 0 or err: + log.error('preserved-rebuild returned {}'.format(ret)) + for line in out.splitlines(): + log.error('stdout: {}'.format(line)) + for line in err.splitlines(): + log.error('stderr: {}'.format(line)) + raise sau.errors.UpgradeError(f'Error during preserved-rebuild') + else: + log.info('preserved-rebuild complete') + for line in out.splitlines(): + if line.startswith(' * '): + log.warning(line) + + + # doing grub reconfig and clean old kernels if do_grub and os.path.exists(GRUB_MKCONFIG): keep_kernels = conf.getint('default', 'keep_kernels', fallback=4) if keep_kernels < 1: