feffe/sau
1
0
Fork 0
Code Issues Pull Requests Projects Releases 20 Wiki Activity

set timeout on all subprocesses

Browse Source
This commit is contained in:
Fredrik Eriksson 2019-04-18 09:40:28 +02:00
parent 5eda3b6e85
commit f121466eaa
No known key found for this signature in database
GPG Key ID: 8825C73A0FD1502A
4 changed files with 60 additions and 151 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
sau/freebsd.py
View File

@ -1,7 +1,6 @@
import logging import logging
import random import random
import re import re
import subprocess
import time import time
import sau import sau
@ -21,24 +20,14 @@ pkg_upgrade_re = re.compile(r'^\s([^\s]*): ([^\s]*) -> ([^\s]*).*$')
def identify_service_from_bin(exe): def identify_service_from_bin(exe):
log = logging.getLogger(sau.LOGNAME) log = logging.getLogger(sau.LOGNAME)
cmd = [ PKG_PATH, 'which', '-q', exe ] cmd = [ PKG_PATH, 'which', '-q', exe ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, pkg, err = sau.helpers.exec_cmd(cmd)
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
pkg, err = proc.communicate()
if not pkg: if not pkg:
raise sau.errors.UnknownServiceError("'{}' does not belong to any package".format(exe)) raise sau.errors.UnknownServiceError("'{}' does not belong to any package".format(exe))
pkg = pkg.decode('utf-8').strip() pkg = pkg.decode('utf-8').strip()
log.debug('{} belongs to package {}'.format(exe, pkg)) log.debug('{} belongs to package {}'.format(exe, pkg))
cmd = [ PKG_PATH, 'info', '-l', pkg ] cmd = [ PKG_PATH, 'info', '-l', pkg ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd)
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
rc_scripts = set() rc_scripts = set()
for line in out.decode('utf-8').splitlines(): for line in out.decode('utf-8').splitlines():
match = re.match(rc_script_re, line.strip()) match = re.match(rc_script_re, line.strip())
@ -55,16 +44,9 @@ def identify_service_from_bin(exe):
def restart_service(service): def restart_service(service):
log = logging.getLogger(sau.LOGNAME) log = logging.getLogger(sau.LOGNAME)
cmd = [ SERVICE_PATH, service, 'restart' ] cmd = [ SERVICE_PATH, service, 'restart' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd)
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0: if ret != 0:
log.warning("Restart of {} failed:".format(service)) log.warning("Restart of {} failed:".format(service))
for line in out.splitlines(): for line in out.splitlines():
log.warning("stdout: {}".format(line)) log.warning("stdout: {}".format(line))
@ -79,17 +61,9 @@ def system_upgrade():
time.sleep(random.randint(0, 3600)) time.sleep(random.randint(0, 3600))
# now we can lie without soiling our conscience too much # now we can lie without soiling our conscience too much
cmd = [ FREEBSD_UPDATE_PATH, '--not-running-from-cron', 'fetch', 'install' ] cmd = [ FREEBSD_UPDATE_PATH, '--not-running-from-cron', 'fetch', 'install' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd, timeout=7200, env = { 'PAGER': '/bin/cat' })
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE,
env = { 'PAGER': '/bin/cat' })
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode == 0: if ret == 0:
if out.endswith('No updates are available to install.\n'): if out.endswith('No updates are available to install.\n'):
log.info('No system updates are available') log.info('No system updates are available')
else: else:
@ -98,7 +72,7 @@ def system_upgrade():
log.info(line) log.info(line)
return True return True
else: else:
log.warning('System upgrade failed (return code {}):'.format(proc.returncode)) log.warning('System upgrade failed (return code {}):'.format(ret))
for line in out.splitlines(): for line in out.splitlines():
log.warning('stdout: {}'.format(line)) log.warning('stdout: {}'.format(line))
for line in err.splitlines(): for line in err.splitlines():
@ -109,16 +83,9 @@ def pkg_upgrade():
log = logging.getLogger(sau.LOGNAME) log = logging.getLogger(sau.LOGNAME)
conf = sau.config conf = sau.config
cmd = [ PKG_PATH, 'upgrade', '-nq' ] cmd = [ PKG_PATH, 'upgrade', '-nq' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd)
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode == 0 and not out and not err: if ret == 0 and not out and not err:
log.info('No package upgrades available') log.info('No package upgrades available')
return False return False
@ -157,15 +124,8 @@ def pkg_upgrade():
return False return False
cmd = [ PKG_PATH, 'upgrade', '-yq' ] cmd = [ PKG_PATH, 'upgrade', '-yq' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600)
proc = subprocess.Popen( if ret != 0 or err:
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0 or err:
log.warning('{} failed:'.format(' '.join(cmd))) log.warning('{} failed:'.format(' '.join(cmd)))
for line in out.splitlines(): for line in out.splitlines():
log.warning('stdout: {}'.format(line)) log.warning('stdout: {}'.format(line))

121
sau/gentoo.py
View File

@ -1,7 +1,6 @@
import logging import logging
import os import os
import re import re
import subprocess
import sau import sau
import sau.helpers import sau.helpers
@ -22,16 +21,9 @@ def identify_service_from_bin(exe):
init_script_re = re.compile(r'/etc/init\.d/(.*)') init_script_re = re.compile(r'/etc/init\.d/(.*)')
cmd = [ EQUERY_PATH, '-Cq', 'b', exe ] cmd = [ EQUERY_PATH, '-Cq', 'b', exe ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd)
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0: if ret != 0:
log.warning("searching for owner of {} failed:".format(exe)) log.warning("searching for owner of {} failed:".format(exe))
for line in out.splitlines(): for line in out.splitlines():
log.warning("stdout: {}".format(line)) log.warning("stdout: {}".format(line))
@ -41,16 +33,9 @@ def identify_service_from_bin(exe):
pkg = out.strip() pkg = out.strip()
cmd = [ EQUERY_PATH, '-Cq', 'f', pkg ] cmd = [ EQUERY_PATH, '-Cq', 'f', pkg ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd)
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0: if ret != 0:
log.warning("listing files for package {} failed:".format(pkg)) log.warning("listing files for package {} failed:".format(pkg))
for line in out.splitlines(): for line in out.splitlines():
log.warning("stdout: {}".format(line)) log.warning("stdout: {}".format(line))
@ -74,16 +59,9 @@ def identify_service_from_bin(exe):
def restart_service(service): def restart_service(service):
log = logging.getLogger(sau.LOGNAME) log = logging.getLogger(sau.LOGNAME)
cmd = [ RC_SERVICE_PATH, service, 'restart' ] cmd = [ RC_SERVICE_PATH, service, 'restart' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd)
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0: if ret != 0:
log.warning("Restart of {} failed:".format(service)) log.warning("Restart of {} failed:".format(service))
for line in out.splitlines(): for line in out.splitlines():
log.warning("stdout: {}".format(line)) log.warning("stdout: {}".format(line))
@ -101,18 +79,9 @@ def _sync_portage():
log = logging.getLogger(sau.LOGNAME) log = logging.getLogger(sau.LOGNAME)
cmd = [ EMERGE_PATH, '-q', '--sync' ] cmd = [ EMERGE_PATH, '-q', '--sync' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600)
proc = subprocess.Popen( if ret != 0:
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0:
log.warning("Portage sync failed:") log.warning("Portage sync failed:")
for line in out.splitlines(): for line in out.splitlines():
log.warning("stdout: {}".format(line)) log.warning("stdout: {}".format(line))
@ -121,18 +90,9 @@ def _sync_portage():
if os.path.exists(EIX_UPDATE_PATH): if os.path.exists(EIX_UPDATE_PATH):
cmd = [ EIX_UPDATE_PATH, '-q' ] cmd = [ EIX_UPDATE_PATH, '-q' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600)
proc = subprocess.Popen( if ret != 0:
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0:
log.warning("eix-update failed:") log.warning("eix-update failed:")
for line in out.splitlines(): for line in out.splitlines():
log.warning("stdout: {}".format(line)) log.warning("stdout: {}".format(line))
@ -140,15 +100,6 @@ def _sync_portage():
log.warning("stderr: {}".format(line)) log.warning("stderr: {}".format(line))
def _parse_version(version):
slot_re = re.compile('^(\(~\))?([^\(]+)(\([^\)]+\))$')
match = re.match(slot_re, version)
return {
'keyword': match.group(1),
'version': match.group(2),
'slot': match.group(2)
}
def pkg_upgrade(): def pkg_upgrade():
log = logging.getLogger(sau.LOGNAME) log = logging.getLogger(sau.LOGNAME)
conf = sau.config conf = sau.config
@ -165,17 +116,10 @@ def pkg_upgrade():
default_version_diff = conf.getint('default', 'min_version_diff', fallback=2) default_version_diff = conf.getint('default', 'min_version_diff', fallback=2)
cmd = [ EMERGE_PATH, '--color', 'n', '-uDNpq', '@world' ] cmd = [ EMERGE_PATH, '--color', 'n', '-uDNpq', '@world' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd)
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if not proc.returncode == 0: if not ret == 0:
log.error('emerge pretend returned {}'.format(proc.returncode)) log.error('emerge pretend returned {}'.format(ret))
for line in out.splitlines(): for line in out.splitlines():
log.error('stdout: {}'.format(line)) log.error('stdout: {}'.format(line))
for line in err.splitlines(): for line in err.splitlines():
@ -213,17 +157,10 @@ def pkg_upgrade():
return False return False
cmd = [ EMERGE_PATH, '--color', 'n', '-uDNq', '@world' ] cmd = [ EMERGE_PATH, '--color', 'n', '-uDNq', '@world' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd, timeout=36000)
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0 or err: if ret != 0 or err:
log.warning('emerge returned {}'.format(proc.returncode)) log.warning('emerge returned {}'.format(ret))
for line in out.splitlines(): for line in out.splitlines():
log.warning('stdout: {}'.format(line)) log.warning('stdout: {}'.format(line))
for line in err.splitlines(): for line in err.splitlines():
@ -235,17 +172,10 @@ def pkg_upgrade():
log.warning(line) log.warning(line)
cmd = [ EMERGE_PATH, '--color', 'n', '-q', '@preserved-rebuild' ] cmd = [ EMERGE_PATH, '--color', 'n', '-q', '@preserved-rebuild' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd, timeout=36000)
proc = subprocess.Popen(
cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0 or err: if ret != 0 or err:
log.warning('preserved-rebuild returned {}'.format(proc.returncode)) log.warning('preserved-rebuild returned {}'.format(ret))
for line in out.splitlines(): for line in out.splitlines():
log.warning('stdout: {}'.format(line)) log.warning('stdout: {}'.format(line))
for line in err.splitlines(): for line in err.splitlines():
@ -258,16 +188,9 @@ def pkg_upgrade():
if conf.getboolean('default', 'do_depclean', fallback=False): if conf.getboolean('default', 'do_depclean', fallback=False):
cmd = [ EMERGE_PATH, '--color', 'n', '-q', '--depclean' ] cmd = [ EMERGE_PATH, '--color', 'n', '-q', '--depclean' ]
log.debug('Executing "{}"'.format(' '.join(cmd))) ret, out, err = sau.helpers.exec_cmd(cmd, timeout=3600)
proc = subprocess.Popen( if ret != 0 or err:
cmd, log.warning('depclean returned {}'.format(ret))
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
out, err = proc.communicate()
out = out.decode('utf-8')
err = err.decode('utf-8')
if proc.returncode != 0 or err:
log.warning('depclean returned {}'.format(proc.returncode))
for line in out.splitlines(): for line in out.splitlines():
log.warning('stdout: {}'.format(line)) log.warning('stdout: {}'.format(line))
for line in err.splitlines(): for line in err.splitlines():

27
sau/helpers.py
View File

@ -1,3 +1,30 @@
import logging
import subprocess
import time
import sau
def exec_cmd(cmd, timeout=600, env = None):
log = logging.getLogger(sau.LOGNAME)
log.debug('Executing "{}"'.format(' '.join(cmd)))
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env = env)
try:
out, err = proc.communicate(timeout=timeout)
except subprocess.TimeoutExpired as err:
log.error('Command "{}" timed out, killing it.'.format(' '.join(cmd)))
proc.kill()
time.sleep(30)
if proc.poll() != None:
log.error('Command "{}" would not be killed, forcing a termination'.format(' '.join(cmd)))
proc.terminate()
time.sleep(5)
return (proc.returncode, out.decode('utf-8'), err.decode('utf-8'))
def version_diff(new, old): def version_diff(new, old):
""" This will return 100 if the versions are not properly compareable, """ This will return 100 if the versions are not properly compareable,

1
sau/services.py
View File

@ -2,7 +2,6 @@
import logging import logging
import os import os
import re import re
import subprocess
import time import time
import psutil import psutil