sau/sau/services.py

#!/usr/bin/env python3.6
import logging
import os
import re
import time

import psutil

import sau
import sau.errors
import sau.platforms

proc_fd_map_re = re.compile(r'^.*(/[^\(]*) \(deleted\)$')

def _get_deleted_open_files(proc):
    log = logging.getLogger(sau.LOGNAME)
    files = set()
    
    # try the linux-way first
    maps_file = '/proc/{}/maps'.format(proc.pid)
    if os.path.isfile(maps_file):
        with open(maps_file, 'r') as f:
            for line in f:
                match = re.match(proc_fd_map_re, line)
                if match:
                    files.add(match.group(1))
        return files
    
    # on FreeBSD psutils open_files() helpfully returns a null path if a file
    # has been deleted.
    try:
        for f in proc.open_files():
            if f.path and os.path.exists(f.path):
                continue
            else:
                files.add(f)
    except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
        pass
    return files

def get_exe_file(name):
    log = logging.getLogger(sau.LOGNAME)
    search_paths = [
            '/bin'
            '/sbin'
            '/usr/bin',
            '/usr/sbin',
            '/usr/local/bin',
            '/usr/local/sbin',
            '/libexec',
            '/usr/libexec',
            '/usr/local/libexec'
            ]
    for path in search_paths:
        if os.path.isdir(path):
            for root, dirs, files in os.walk(path):
                if name in files:
                    log.debug('Found binary for {} at {}'.format(name, root))
                    return os.path.join(root, name)

def restart_services():
    log = logging.getLogger(sau.LOGNAME)
    platform = sau.platforms.get_platform()
    conf = sau.config
    check_procs = set()
    for proc in psutil.process_iter():
        files = _get_deleted_open_files(proc)
        if files:
            log.info('{} has open deleted files'.format(proc))
            check_procs.add(proc)

    # wait before the second test
    time.sleep(1)

    # perform a second check to remove potential false positives
    service_procs = set()
    retest_procs = set()
    for proc in check_procs:
        files = _get_deleted_open_files(proc)
        if not files:
            # no deleted open files for this process any longer
            continue
        try:
            exe = proc.exe()
            parents = proc.parents()
        except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
            # either of the above exceptions means the process has quit
            continue
        
        log.debug('will attempt to restart parent of {}'.format(proc))
        if len(parents) < 2:
            log.debug('{} is its own top parent'.format(proc))
            service_procs.add(proc)
        else:
            log.debug('{} has top parent {}'.format(proc, parents[-2]))
            service_procs.add(parents[-2])
        retest_procs.add(proc)

    processes = {}
    services = {}
    for proc in service_procs:
        try:
            service_exe = proc.exe()
            proc_name = proc.name()
        except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
            log.debug('{} died before it could be restarted'.format(proc))
            continue

        if proc_name in services:
            processes[services[proc_name]].append(proc)
            # we have already checked a process with this name
            continue

        service_name = conf.get('processes', proc_name, fallback=None)
        if service_name == '':
            log.debug('Ignoring process {}'.format(proc))
            retest_procs.remove(proc)
            continue

        if not service_name:
            # if the exe file has been deleted since started, service_exe will be empty
            # and we'll have to guess
            if not service_exe:
                log.debug('Could not get full path to executable for process {}, will attempt to guess'.format(proc))
                service_exe = get_exe_file(service_name)
            if not service_exe:
                log.error('Failed to find executable for process {}'.format(proc))
                continue

            try: 
                service_name = platform.identify_service_from_bin(service_exe)
            except sau.errors.UnknownServiceError:
                log.warning('Could not find service for process {}'.format(proc))
                continue

        services[proc_name] = service_name
        processes[service_name] = [proc]

    for service_all in [x for x in services.values() if x]:
        for service in service_all.split():
            policy = _get_service_restart_policy(service)
            if policy == 'ignore':
                log.info('Service "{}" ignored by configuration'.format(service))
                continue
            elif policy == 'warn':
                log.warning('Service "{}" has open deleted files and should be restarted'.format(service))
                continue

            if not policy.startswith('silent'):
                log.warning('Restarting service {}'.format(service))
            platform.restart_service(service)

    recommend_restart = False
    for proc in retest_procs:
        try:
            name = proc.name()
        except (psutil.NoSuchProcess, psutil.ZombieProcess, psutil.AccessDenied):
            log.debug('{} was successfully killed'.format(proc))
            continue

        if _get_deleted_open_files(proc):
            if name in services and not services[name]:
                log.warning('{} does not belong to a service and could not be restarted'.format(proc))
                recommend_restart = True
                continue
            elif name in services:
                policy = _get_service_restart_policy(services[name])
                if policy in ('ignore', 'warn'):
                    continue
            log.warning('{} still has deleted files open'.format(proc))
            recommend_restart = True
    return recommend_restart

def _get_service_restart_policy(service):
    conf = sau.config
    policy = conf.get('services', service, fallback=None)
    if policy and policy.lower() in ('restart', 'warn', 'ignore', 'silent-restart'):
        return policy.lower()
    elif policy:
        log.warning('service policy {} for {} is invalid'.format(policy, service))

    default_policy = conf.get('services', 'default_service_policy', fallback='warn')
    if default_policy.lower() in ('restart', 'warn', 'ignore', 'silent-restart'):
        return default_policy.lower()
    log.warning('default service policy {} is invalid'.format(default_policy))
    return 'warn'