99 lines
2.3 KiB
Python
99 lines
2.3 KiB
Python
import email
|
|
import smtplib
|
|
import socket
|
|
import subprocess
|
|
|
|
import systemd.journal
|
|
|
|
def _get_status(name):
|
|
cmd = ['/usr/bin/systemctl', 'status', name]
|
|
|
|
proc = subprocess.Popen(
|
|
cmd,
|
|
stdout=subprocess.PIPE)
|
|
out, err = proc.communicate()
|
|
out = out.decode('utf-8')
|
|
return out
|
|
|
|
def _send_email(subject, message, config):
|
|
mail = email.message.EmailMessage()
|
|
hostname = socket.getfqdn()
|
|
smtp_host='localhost'
|
|
mail_from=f'sysalert <root@{hostname}>'
|
|
mail_to=f'root <root@{hostname}>'
|
|
if 'smtp_host' in config:
|
|
smtp_host = config['smtp_host']
|
|
if 'mail_from' in config:
|
|
mail_from = config['mail_from']
|
|
if 'mail_to' in config:
|
|
mail_to = config['mail_to']
|
|
|
|
|
|
mail.set_content(message)
|
|
mail['To'] = mail_to
|
|
mail['From'] = mail_from
|
|
mail['Subject'] = subject
|
|
|
|
with smtplib.SMTP(smtp_host) as smtp:
|
|
smtp.send_message(mail)
|
|
|
|
def success(name, failures, config):
|
|
cur_status = _get_status(name)
|
|
nr_failures = len(failures)
|
|
hostname = socket.getfqdn()
|
|
subject=f'{hostname} - {name}: recovery'
|
|
message=f"""
|
|
{name} has recovered after {nr_failures} failures.
|
|
Current status:
|
|
|
|
{cur_status}
|
|
"""
|
|
_send_email(subject, message, config)
|
|
|
|
|
|
def failure(name, failures, config):
|
|
cur_status = _get_status(name)
|
|
hostname = socket.getfqdn()
|
|
latest_inv_id = failures[-1]['invocation_id']
|
|
nr_failures = len(failures)
|
|
prev_failures=0
|
|
for i,f in enumerate(reversed(failures[:-1])):
|
|
if f['alert_method']:
|
|
prev_failures=i+1
|
|
break
|
|
|
|
reader = systemd.journal.Reader()
|
|
reader.add_match(MONITOR_INVOCATION_ID=latest_inv_id)
|
|
reader.add_disjunction()
|
|
reader.add_match(_SYSTEMD_INVOCATION_ID=lateset_inv_id)
|
|
journal_txt = "\n".join([entry['MESSAGE'] for entry in reader])
|
|
|
|
if nr_failures <= 1:
|
|
subject=f"{hostname} - {name}: failure"
|
|
message=f"""
|
|
New failure for {name}.
|
|
Current status:
|
|
|
|
{cur_status}
|
|
|
|
|
|
Latest journal log:
|
|
|
|
{journal_txt}
|
|
"""
|
|
else:
|
|
subject=f"{hostname} - {name}: {nr_failures} failures"
|
|
message=f"""
|
|
{nr_failures} failures for {name} ({prev_failures} since last notification).
|
|
Current status:
|
|
|
|
{cur_status}
|
|
|
|
|
|
Latest journal log:
|
|
|
|
{journal_txt}
|
|
"""
|
|
|
|
_send_email(subject, message, config)
|