diff --git a/sau/freebsd.py b/sau/freebsd.py index 42a5ede..91de5cc 100644 --- a/sau/freebsd.py +++ b/sau/freebsd.py @@ -1,7 +1,6 @@ import logging import random import re -import subprocess import time import sau @@ -21,24 +20,14 @@ pkg_upgrade_re = re.compile(r'^\s([^\s]*): ([^\s]*) -> ([^\s]*).*$') def identify_service_from_bin(exe): log = logging.getLogger(sau.LOGNAME) cmd = [ PKG_PATH, 'which', '-q', exe ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - pkg, err = proc.communicate() + ret, pkg, err = sau.helpers.exec_cmd(cmd) if not pkg: raise sau.errors.UnknownServiceError("'{}' does not belong to any package".format(exe)) pkg = pkg.decode('utf-8').strip() log.debug('{} belongs to package {}'.format(exe, pkg)) cmd = [ PKG_PATH, 'info', '-l', pkg ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() + ret, out, err = sau.helpers.exec_cmd(cmd) rc_scripts = set() for line in out.decode('utf-8').splitlines(): match = re.match(rc_script_re, line.strip()) @@ -55,16 +44,9 @@ def identify_service_from_bin(exe): def restart_service(service): log = logging.getLogger(sau.LOGNAME) cmd = [ SERVICE_PATH, service, 'restart' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') + ret, out, err = sau.helpers.exec_cmd(cmd) - if proc.returncode != 0: + if ret != 0: log.warning("Restart of {} failed:".format(service)) for line in out.splitlines(): log.warning("stdout: {}".format(line)) @@ -79,17 +61,9 @@ def system_upgrade(): time.sleep(random.randint(0, 3600)) # now we can lie without soiling our conscience too much cmd = [ FREEBSD_UPDATE_PATH, '--not-running-from-cron', 'fetch', 'install' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - env = { 'PAGER': '/bin/cat' }) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=7200, env = { 'PAGER': '/bin/cat' }) - if proc.returncode == 0: + if ret == 0: if out.endswith('No updates are available to install.\n'): log.info('No system updates are available') else: @@ -98,7 +72,7 @@ def system_upgrade(): log.info(line) return True else: - log.warning('System upgrade failed (return code {}):'.format(proc.returncode)) + log.warning('System upgrade failed (return code {}):'.format(ret)) for line in out.splitlines(): log.warning('stdout: {}'.format(line)) for line in err.splitlines(): @@ -109,16 +83,9 @@ def pkg_upgrade(): log = logging.getLogger(sau.LOGNAME) conf = sau.config cmd = [ PKG_PATH, 'upgrade', '-nq' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') + ret, out, err = sau.helpers.exec_cmd(cmd) - if proc.returncode == 0 and not out and not err: + if ret == 0 and not out and not err: log.info('No package upgrades available') return False @@ -157,15 +124,8 @@ def pkg_upgrade(): return False cmd = [ PKG_PATH, 'upgrade', '-yq' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') - if proc.returncode != 0 or err: + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600) + if ret != 0 or err: log.warning('{} failed:'.format(' '.join(cmd))) for line in out.splitlines(): log.warning('stdout: {}'.format(line)) diff --git a/sau/gentoo.py b/sau/gentoo.py index 274acd9..59eed96 100644 --- a/sau/gentoo.py +++ b/sau/gentoo.py @@ -1,7 +1,6 @@ import logging import os import re -import subprocess import sau import sau.helpers @@ -22,16 +21,9 @@ def identify_service_from_bin(exe): init_script_re = re.compile(r'/etc/init\.d/(.*)') cmd = [ EQUERY_PATH, '-Cq', 'b', exe ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') + ret, out, err = sau.helpers.exec_cmd(cmd) - if proc.returncode != 0: + if ret != 0: log.warning("searching for owner of {} failed:".format(exe)) for line in out.splitlines(): log.warning("stdout: {}".format(line)) @@ -41,16 +33,9 @@ def identify_service_from_bin(exe): pkg = out.strip() cmd = [ EQUERY_PATH, '-Cq', 'f', pkg ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') + ret, out, err = sau.helpers.exec_cmd(cmd) - if proc.returncode != 0: + if ret != 0: log.warning("listing files for package {} failed:".format(pkg)) for line in out.splitlines(): log.warning("stdout: {}".format(line)) @@ -74,16 +59,9 @@ def identify_service_from_bin(exe): def restart_service(service): log = logging.getLogger(sau.LOGNAME) cmd = [ RC_SERVICE_PATH, service, 'restart' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') + ret, out, err = sau.helpers.exec_cmd(cmd) - if proc.returncode != 0: + if ret != 0: log.warning("Restart of {} failed:".format(service)) for line in out.splitlines(): log.warning("stdout: {}".format(line)) @@ -101,18 +79,9 @@ def _sync_portage(): log = logging.getLogger(sau.LOGNAME) cmd = [ EMERGE_PATH, '-q', '--sync' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600) - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') - - if proc.returncode != 0: + if ret != 0: log.warning("Portage sync failed:") for line in out.splitlines(): log.warning("stdout: {}".format(line)) @@ -121,18 +90,9 @@ def _sync_portage(): if os.path.exists(EIX_UPDATE_PATH): cmd = [ EIX_UPDATE_PATH, '-q' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600) - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') - - if proc.returncode != 0: + if ret != 0: log.warning("eix-update failed:") for line in out.splitlines(): log.warning("stdout: {}".format(line)) @@ -140,15 +100,6 @@ def _sync_portage(): log.warning("stderr: {}".format(line)) -def _parse_version(version): - slot_re = re.compile('^(\(~\))?([^\(]+)(\([^\)]+\))$') - match = re.match(slot_re, version) - return { - 'keyword': match.group(1), - 'version': match.group(2), - 'slot': match.group(2) - } - def pkg_upgrade(): log = logging.getLogger(sau.LOGNAME) conf = sau.config @@ -165,17 +116,10 @@ def pkg_upgrade(): default_version_diff = conf.getint('default', 'min_version_diff', fallback=2) cmd = [ EMERGE_PATH, '--color', 'n', '-uDNpq', '@world' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') + ret, out, err = sau.helpers.exec_cmd(cmd) - if not proc.returncode == 0: - log.error('emerge pretend returned {}'.format(proc.returncode)) + if not ret == 0: + log.error('emerge pretend returned {}'.format(ret)) for line in out.splitlines(): log.error('stdout: {}'.format(line)) for line in err.splitlines(): @@ -213,17 +157,10 @@ def pkg_upgrade(): return False cmd = [ EMERGE_PATH, '--color', 'n', '-uDNq', '@world' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=36000) - if proc.returncode != 0 or err: - log.warning('emerge returned {}'.format(proc.returncode)) + if ret != 0 or err: + log.warning('emerge returned {}'.format(ret)) for line in out.splitlines(): log.warning('stdout: {}'.format(line)) for line in err.splitlines(): @@ -235,17 +172,10 @@ def pkg_upgrade(): log.warning(line) cmd = [ EMERGE_PATH, '--color', 'n', '-q', '@preserved-rebuild' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=36000) - if proc.returncode != 0 or err: - log.warning('preserved-rebuild returned {}'.format(proc.returncode)) + if ret != 0 or err: + log.warning('preserved-rebuild returned {}'.format(ret)) for line in out.splitlines(): log.warning('stdout: {}'.format(line)) for line in err.splitlines(): @@ -258,16 +188,9 @@ def pkg_upgrade(): if conf.getboolean('default', 'do_depclean', fallback=False): cmd = [ EMERGE_PATH, '--color', 'n', '-q', '--depclean' ] - log.debug('Executing "{}"'.format(' '.join(cmd))) - proc = subprocess.Popen( - cmd, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - out, err = proc.communicate() - out = out.decode('utf-8') - err = err.decode('utf-8') - if proc.returncode != 0 or err: - log.warning('depclean returned {}'.format(proc.returncode)) + ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600) + if ret != 0 or err: + log.warning('depclean returned {}'.format(ret)) for line in out.splitlines(): log.warning('stdout: {}'.format(line)) for line in err.splitlines(): diff --git a/sau/helpers.py b/sau/helpers.py index 5c0c0a5..cafdd76 100644 --- a/sau/helpers.py +++ b/sau/helpers.py @@ -1,3 +1,30 @@ +import logging +import subprocess +import time + +import sau + +def exec_cmd(cmd, timeout=600, env = None): + log = logging.getLogger(sau.LOGNAME) + log.debug('Executing "{}"'.format(' '.join(cmd))) + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env = env) + try: + out, err = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired as err: + log.error('Command "{}" timed out, killing it.'.format(' '.join(cmd))) + proc.kill() + time.sleep(30) + if proc.poll() != None: + log.error('Command "{}" would not be killed, forcing a termination'.format(' '.join(cmd))) + proc.terminate() + time.sleep(5) + + return (proc.returncode, out.decode('utf-8'), err.decode('utf-8')) + def version_diff(new, old): """ This will return 100 if the versions are not properly compareable, diff --git a/sau/services.py b/sau/services.py index a049c6e..a786841 100644 --- a/sau/services.py +++ b/sau/services.py @@ -2,7 +2,6 @@ import logging import os import re -import subprocess import time import psutil