import email import smtplib import socket import subprocess import systemd.journal def _get_status(name): cmd = ['/usr/bin/systemctl', 'status', name] proc = subprocess.Popen( cmd, stdout=subprocess.PIPE) out, err = proc.communicate() out = out.decode('utf-8') return out def _send_email(subject, message, config): mail = email.message.EmailMessage() hostname = socket.getfqdn() smtp_host='localhost' mail_from=f'sysalert ' mail_to=f'root ' if 'smtp_host' in config: smtp_host = config['smtp_host'] if 'mail_from' in config: mail_from = config['mail_from'] if 'mail_to' in config: mail_to = config['mail_to'] mail.set_content(message) mail['To'] = mail_to mail['From'] = mail_from mail['Subject'] = subject with smtplib.SMTP(smtp_host) as smtp: smtp.send_message(mail) def success(name, failures, config): cur_status = _get_status(name) nr_failures = len(failures) hostname = socket.getfqdn() subject=f'{hostname} - {name}: recovery' message=f""" {name} has recovered after {nr_failures} failures. Current status: {cur_status} """ _send_email(subject, message, config) def failure(name, failures, config): cur_status = _get_status(name) hostname = socket.getfqdn() latest_inv_id = failures[-1]['invocation_id'] nr_failures = len(failures) prev_failures=0 for i,f in enumerate(reversed(failures[:-1])): if f['alert_method']: prev_failures=i+1 break reader = systemd.journal.Reader() reader.add_match(MONITOR_INVOCATION_ID=latest_inv_id) reader.add_disjunction() reader.add_match(_SYSTEMD_INVOCATION_ID=lateset_inv_id) journal_txt = "\n".join([entry['MESSAGE'] for entry in reader]) if nr_failures <= 1: subject=f"{hostname} - {name}: failure" message=f""" New failure for {name}. Current status: {cur_status} Latest journal log: {journal_txt} """ else: subject=f"{hostname} - {name}: {nr_failures} failures" message=f""" {nr_failures} failures for {name} ({prev_failures} since last notification). Current status: {cur_status} Latest journal log: {journal_txt} """ _send_email(subject, message, config)