diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 1e78d1f..0000000 --- a/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -FROM alpine:latest as aprsd - -ENV VERSION=1.0.0 -ENV APRS_USER=aprs -ENV HOME=/home/aprs -ENV VIRTUAL_ENV=$HOME/.venv3 - -ENV INSTALL=$HOME/install -RUN apk add --update git wget py3-pip py3-virtualenv bash fortune - -# Setup Timezone -ENV TZ=US/Eastern -#RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -#RUN apt-get install -y tzdata -#RUN dpkg-reconfigure --frontend noninteractive tzdata - - -RUN addgroup --gid 1000 $APRS_USER -RUN adduser -h $HOME -D -u 1001 -G $APRS_USER $APRS_USER - -ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 - -USER $APRS_USER -RUN pip3 install wheel -RUN python3 -m venv $VIRTUAL_ENV -ENV PATH="$VIRTUAL_ENV/bin:$PATH" -RUN echo "export PATH=\$PATH:\$HOME/.local/bin" >> $HOME/.bashrc -VOLUME ["/config", "/plugins"] - -WORKDIR $HOME -RUN pip install aprsd -USER root -RUN aprsd sample-config > /config/aprsd.yml -RUN chown -R $APRS_USER:$APRS_USER /config - -# override this to run another configuration -ENV CONF default -USER $APRS_USER - -ADD build/bin/run.sh $HOME/ -ENTRYPOINT ["/home/aprs/run.sh"] diff --git a/aprsd/email.py b/aprsd/email.py index 9012434..12d3840 100644 --- a/aprsd/email.py +++ b/aprsd/email.py @@ -7,7 +7,7 @@ import re import smtplib import time -from aprsd import messaging, stats, threads +from aprsd import messaging, stats, threads, trace import imapclient from validate_email import validate_email @@ -17,6 +17,7 @@ LOG = logging.getLogger("APRSD") CONFIG = None +@trace.trace def _imap_connect(): imap_port = CONFIG["aprsd"]["email"]["imap"].get("port", 143) use_ssl = CONFIG["aprsd"]["email"]["imap"].get("use_ssl", False) @@ -31,9 +32,10 @@ def _imap_connect(): port=imap_port, use_uid=True, ssl=use_ssl, + timeout=30, ) - except Exception: - LOG.error("Failed to connect IMAP server") + except Exception as e: + LOG.error("Failed to connect IMAP server", e) return try: @@ -47,9 +49,15 @@ def _imap_connect(): return server.select_folder("INBOX") + + server.fetch = trace.trace(server.fetch) + server.search = trace.trace(server.search) + server.remove_flags = trace.trace(server.remove_flags) + server.add_flags = trace.trace(server.add_flags) return server +@trace.trace def _smtp_connect(): host = CONFIG["aprsd"]["email"]["smtp"]["host"] smtp_port = CONFIG["aprsd"]["email"]["smtp"]["port"] @@ -64,15 +72,28 @@ def _smtp_connect(): try: if use_ssl: - server = smtplib.SMTP_SSL(host=host, port=smtp_port) + server = smtplib.SMTP_SSL( + host=host, + port=smtp_port, + timeout=30, + ) else: - server = smtplib.SMTP(host=host, port=smtp_port) + server = smtplib.SMTP( + host=host, + port=smtp_port, + timeout=30, + ) except Exception: LOG.error("Couldn't connect to SMTP Server") return LOG.debug("Connected to smtp host {}".format(msg)) + debug = CONFIG["aprsd"]["email"]["smtp"].get("debug", False) + if debug: + server.set_debuglevel(5) + server.sendmail = trace.trace(server.sendmail) + try: server.login( CONFIG["aprsd"]["email"]["smtp"]["login"], @@ -87,7 +108,7 @@ def _smtp_connect(): def validate_shortcuts(config): - shortcuts = config.get("shortcuts", None) + shortcuts = config["aprsd"]["email"].get("shortcuts", None) if not shortcuts: return @@ -120,7 +141,7 @@ def validate_shortcuts(config): for key in delete_keys: del config["aprsd"]["email"]["shortcuts"][key] - LOG.info("Available shortcuts: {}".format(config["shortcuts"])) + LOG.info("Available shortcuts: {}".format(config["aprsd"]["email"]["shortcuts"])) def get_email_from_shortcut(addr): @@ -152,6 +173,7 @@ def validate_email_config(config, disable_validation=False): return False +@trace.trace def parse_email(msgid, data, server): envelope = data[b"ENVELOPE"] # email address match @@ -162,7 +184,12 @@ def parse_email(msgid, data, server): else: from_addr = "noaddr" LOG.debug("Got a message from '{}'".format(from_addr)) - m = server.fetch([msgid], ["RFC822"]) + try: + m = server.fetch([msgid], ["RFC822"]) + except Exception as e: + LOG.exception("Couldn't fetch email from server in parse_email", e) + return + msg = email.message_from_string(m[msgid][b"RFC822"].decode(errors="ignore")) if msg.is_multipart(): text = "" @@ -238,6 +265,7 @@ def parse_email(msgid, data, server): # end parse_email +@trace.trace def send_email(to_addr, content): global check_email_delay @@ -282,6 +310,7 @@ def send_email(to_addr, content): # end send_email +@trace.trace def resend_email(count, fromcall): global check_email_delay date = datetime.datetime.now() @@ -290,7 +319,7 @@ def resend_email(count, fromcall): year = date.year today = "{}-{}-{}".format(day, month, year) - shortcuts = CONFIG["shortcuts"] + shortcuts = CONFIG["aprsd"]["email"]["shortcuts"] # swap key/value shortcuts_inverted = {v: k for k, v in shortcuts.items()} @@ -300,7 +329,12 @@ def resend_email(count, fromcall): LOG.exception("Failed to Connect to IMAP. Cannot resend email ", e) return - messages = server.search(["SINCE", today]) + try: + messages = server.search(["SINCE", today]) + except Exception as e: + LOG.exception("Couldn't search for emails in resend_email ", e) + return + # LOG.debug("%d messages received today" % len(messages)) msgexists = False @@ -308,11 +342,21 @@ def resend_email(count, fromcall): messages.sort(reverse=True) del messages[int(count) :] # only the latest "count" messages for message in messages: - for msgid, data in list(server.fetch(message, ["ENVELOPE"]).items()): + try: + parts = server.fetch(message, ["ENVELOPE"]).items() + except Exception as e: + LOG.exception("Couldn't fetch email parts in resend_email", e) + continue + + for msgid, data in list(parts): # one at a time, otherwise order is random (body, from_addr) = parse_email(msgid, data, server) # unset seen flag, will stay bold in email client - server.remove_flags(msgid, [imapclient.SEEN]) + try: + server.remove_flags(msgid, [imapclient.SEEN]) + except Exception as e: + LOG.exception("Failed to remove SEEN flag in resend_email", e) + if from_addr in shortcuts_inverted: # reverse lookup of a shortcut from_addr = shortcuts_inverted[from_addr] @@ -320,7 +364,7 @@ def resend_email(count, fromcall): reply = "-" + from_addr + " * " + body.decode(errors="ignore") # messaging.send_message(fromcall, reply) msg = messaging.TextMessage( - CONFIG["aprsd"]["email"]["aprs"]["login"], + CONFIG["aprs"]["login"], fromcall, reply, ) @@ -358,6 +402,7 @@ class APRSDEmailThread(threads.APRSDThread): self.msg_queues = msg_queues self.config = config + @trace.trace def run(self): global check_email_delay @@ -395,17 +440,30 @@ class APRSDEmailThread(threads.APRSDThread): try: server = _imap_connect() except Exception as e: - LOG.exception("Failed to get IMAP server Can't check email.", e) + LOG.exception("IMAP failed to connect.", e) if not server: continue - messages = server.search(["SINCE", today]) + try: + messages = server.search(["SINCE", today]) + except Exception as e: + LOG.exception("IMAP failed to search for messages since today.", e) + continue LOG.debug("{} messages received today".format(len(messages))) - for msgid, data in server.fetch(messages, ["ENVELOPE"]).items(): + try: + _msgs = server.fetch(messages, ["ENVELOPE"]) + except Exception as e: + LOG.exception("IMAP failed to fetch/flag messages: ", e) + continue + + for msgid, data in _msgs.items(): envelope = data[b"ENVELOPE"] - # LOG.debug('ID:%d "%s" (%s)' % (msgid, envelope.subject.decode(), envelope.date)) + LOG.debug( + 'ID:%d "%s" (%s)' + % (msgid, envelope.subject.decode(), envelope.date), + ) f = re.search( r"'([[A-a][0-9]_-]+@[[A-a][0-9]_-\.]+)", str(envelope.from_[0]), @@ -418,16 +476,31 @@ class APRSDEmailThread(threads.APRSDThread): # LOG.debug("Message flags/tags: " + str(server.get_flags(msgid)[msgid])) # if "APRS" not in server.get_flags(msgid)[msgid]: # in python3, imap tags are unicode. in py2 they're strings. so .decode them to handle both - taglist = [ - x.decode(errors="ignore") - for x in server.get_flags(msgid)[msgid] - ] + try: + taglist = [ + x.decode(errors="ignore") + for x in server.get_flags(msgid)[msgid] + ] + except Exception as e: + LOG.exception("Failed to get flags.", e) + break + if "APRS" not in taglist: # if msg not flagged as sent via aprs - server.fetch([msgid], ["RFC822"]) + try: + server.fetch([msgid], ["RFC822"]) + except Exception as e: + LOG.exception("Failed single server fetch for RFC822", e) + break + (body, from_addr) = parse_email(msgid, data, server) # unset seen flag, will stay bold in email client - server.remove_flags(msgid, [imapclient.SEEN]) + try: + server.remove_flags(msgid, [imapclient.SEEN]) + except Exception as e: + LOG.exception("Failed to remove flags SEEN", e) + # Not much we can do here, so lets try and + # send the aprs message anyway if from_addr in shortcuts_inverted: # reverse lookup of a shortcut @@ -441,14 +514,28 @@ class APRSDEmailThread(threads.APRSDThread): ) self.msg_queues["tx"].put(msg) # flag message as sent via aprs - server.add_flags(msgid, ["APRS"]) - # unset seen flag, will stay bold in email client - server.remove_flags(msgid, [imapclient.SEEN]) + try: + server.add_flags(msgid, ["APRS"]) + # unset seen flag, will stay bold in email client + except Exception as e: + LOG.exception("Couldn't add APRS flag to email", e) + + try: + server.remove_flags(msgid, [imapclient.SEEN]) + except Exception as e: + LOG.exception("Couldn't remove seen flag from email", e) + # check email more often since we just received an email check_email_delay = 60 + # reset clock + LOG.debug("Done looping over Server.fetch, logging out.") past = datetime.datetime.now() - server.logout() + try: + server.logout() + except Exception as e: + LOG.exception("IMAP failed to logout: ", e) + continue else: # We haven't hit the email delay yet. # LOG.debug("Delta({}) < {}".format(now - past, check_email_delay)) @@ -457,6 +544,3 @@ class APRSDEmailThread(threads.APRSDThread): # Remove ourselves from the global threads list threads.APRSDThreadList().remove(self) LOG.info("Exiting") - - -# end check_email() diff --git a/aprsd/flask.py b/aprsd/flask.py index 19b4839..b49c862 100644 --- a/aprsd/flask.py +++ b/aprsd/flask.py @@ -1,21 +1,65 @@ import json +import logging import aprsd from aprsd import messaging, stats import flask import flask_classful +from flask_httpauth import HTTPBasicAuth +from werkzeug.security import check_password_hash, generate_password_hash + +LOG = logging.getLogger("APRSD") + +auth = HTTPBasicAuth() +users = None + + +# HTTPBasicAuth doesn't work on a class method. +# This has to be out here. Rely on the APRSDFlask +# class to initialize the users from the config +@auth.verify_password +def verify_password(username, password): + global users + + if username in users and check_password_hash(users.get(username), password): + return username class APRSDFlask(flask_classful.FlaskView): config = None def set_config(self, config): + global users self.config = config + self.users = {} + for user in self.config["aprsd"]["web"]["users"]: + self.users[user] = generate_password_hash( + self.config["aprsd"]["web"]["users"][user], + ) + + users = self.users def index(self): return "Hello" # return flask.render_template("index.html", message=msg) + @auth.login_required + def messages(self): + track = messaging.MsgTrack() + msgs = [] + for id in track: + LOG.info(track[id].dict()) + msgs.append(track[id].dict()) + + return flask.render_template("messages.html", messages=json.dumps(msgs)) + + @auth.login_required + def save(self): + """Save the existing queue to disk.""" + track = messaging.MsgTrack() + track.save() + return json.dumps({"messages": "saved"}) + def stats(self): stats_obj = stats.APRSDStats() track = messaging.MsgTrack() @@ -30,9 +74,16 @@ class APRSDFlask(flask_classful.FlaskView): def init_flask(config): - flask_app = flask.Flask("aprsd") + flask_app = flask.Flask( + "aprsd", + static_url_path="", + static_folder="web/static", + template_folder="web/templates", + ) server = APRSDFlask() server.set_config(config) # flask_app.route('/', methods=['GET'])(server.index) flask_app.route("/stats", methods=["GET"])(server.stats) + flask_app.route("/messages", methods=["GET"])(server.messages) + flask_app.route("/save", methods=["GET"])(server.save) return flask_app diff --git a/aprsd/main.py b/aprsd/main.py index ad4df16..d0b621e 100644 --- a/aprsd/main.py +++ b/aprsd/main.py @@ -20,6 +20,7 @@ # # python included libs +import datetime import logging from logging import NullHandler from logging.handlers import RotatingFileHandler @@ -27,12 +28,11 @@ import os import queue import signal import sys -import threading import time # local imports here import aprsd -from aprsd import client, email, flask, messaging, plugin, stats, threads, utils +from aprsd import client, email, flask, messaging, plugin, stats, threads, trace, utils import aprslib from aprslib.exceptions import LoginError import click @@ -52,7 +52,9 @@ LOG_LEVELS = { CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"]) -server_event = threading.Event() +flask_enabled = False + +# server_event = threading.Event() # localization, please edit: # HOST = "noam.aprs2.net" # north america tier2 servers round robin @@ -150,20 +152,23 @@ def install(append, case_insensitive, shell, path): def signal_handler(sig, frame): - global server_vent + global flask_enabled - LOG.info( - "Ctrl+C, Sending all threads exit! Can take up to 10 seconds to exit all threads", - ) threads.APRSDThreadList().stop_all() - server_event.set() - LOG.info("EXITING STATS") - LOG.info(stats.APRSDStats()) - # time.sleep(1) - signal.signal(signal.SIGTERM, sys.exit(0)) - - -# end signal_handler + if "subprocess" not in str(frame): + LOG.info( + "Ctrl+C, Sending all threads exit! Can take up to 10 seconds {}".format( + datetime.datetime.now(), + ), + ) + time.sleep(5) + tracker = messaging.MsgTrack() + tracker.save() + LOG.info(stats.APRSDStats()) + # signal.signal(signal.SIGTERM, sys.exit(0)) + # sys.exit(0) + if flask_enabled: + signal.signal(signal.SIGTERM, sys.exit(0)) # Setup the logging faciility @@ -184,10 +189,21 @@ def setup_logging(config, loglevel, quiet): fh.setFormatter(log_formatter) LOG.addHandler(fh) + imap_logger = None + if config["aprsd"]["email"].get("enabled", False) and config["aprsd"]["email"][ + "imap" + ].get("debug", False): + + imap_logger = logging.getLogger("imapclient.imaplib") + imap_logger.setLevel(log_level) + imap_logger.addHandler(fh) + if not quiet: sh = logging.StreamHandler(sys.stdout) sh.setFormatter(log_formatter) LOG.addHandler(sh) + if imap_logger: + imap_logger.addHandler(sh) @main.command() @@ -394,9 +410,7 @@ def server( flush, ): """Start the aprsd server process.""" - global event - - event = threading.Event() + global flask_enabled signal.signal(signal.SIGINT, signal_handler) if not quiet: @@ -410,6 +424,8 @@ def server( email.CONFIG = config setup_logging(config, loglevel, quiet) + if config["aprsd"].get("trace", False): + trace.setup_tracing(["method", "api"]) LOG.info("APRSD Started version: {}".format(aprsd.__version__)) stats.APRSDStats(config) @@ -468,6 +484,7 @@ def server( web_enabled = False if web_enabled: + flask_enabled = True app = flask.init_flask(config) app.run( host=config["aprsd"]["web"]["host"], @@ -475,10 +492,8 @@ def server( ) # If there are items in the msgTracker, then save them - tracker = messaging.MsgTrack() - tracker.save() - LOG.info(stats.APRSDStats()) LOG.info("APRSD Exiting.") + return 0 if __name__ == "__main__": diff --git a/aprsd/messaging.py b/aprsd/messaging.py index f84c78a..e63a258 100644 --- a/aprsd/messaging.py +++ b/aprsd/messaging.py @@ -53,6 +53,38 @@ class MsgTrack: cls._instance.lock = threading.Lock() return cls._instance + def __getitem__(self, name): + with self.lock: + return self.track[name] + + def __iter__(self): + with self.lock: + return iter(self.track) + + def keys(self): + with self.lock: + return self.track.keys() + + def items(self): + with self.lock: + return self.track.items() + + def values(self): + with self.lock: + return self.track.values() + + def __len__(self): + with self.lock: + return len(self.track) + + def __str__(self): + with self.lock: + result = "{" + for key in self.track.keys(): + result += "{}: {}, ".format(key, str(self.track[key])) + result += "}" + return result + def add(self, msg): with self.lock: key = int(msg.id) @@ -71,24 +103,18 @@ class MsgTrack: if key in self.track.keys(): del self.track[key] - def __len__(self): - with self.lock: - return len(self.track) - - def __str__(self): - with self.lock: - result = "{" - for key in self.track.keys(): - result += "{}: {}, ".format(key, str(self.track[key])) - result += "}" - return result - def save(self): - """Save this shit to disk?""" + """Save any queued to disk?""" + LOG.debug("Save tracker to disk? {}".format(len(self))) if len(self) > 0: LOG.info("Saving {} tracking messages to disk".format(len(self))) pickle.dump(self.dump(), open(utils.DEFAULT_SAVE_FILE, "wb+")) else: + LOG.debug( + "Nothing to save, flushing old save file '{}'".format( + utils.DEFAULT_SAVE_FILE, + ), + ) self.flush() def dump(self): @@ -229,8 +255,17 @@ class RawMessage(Message): super().__init__(None, None, msg_id=None) self.message = message - def __repr__(self): - return self.message + def dict(self): + now = datetime.datetime.now() + return { + "type": "raw", + "message": self.message.rstrip("\n"), + "raw": self.message.rstrip("\n"), + "retry_count": self.retry_count, + "last_send_attempt": self.last_send_attempt, + "last_send_time": str(self.last_send_time), + "last_send_age": str(now - self.last_send_time), + } def __str__(self): return self.message @@ -246,12 +281,12 @@ class RawMessage(Message): cl = client.get_client() log_message( "Sending Message Direct", - repr(self).rstrip("\n"), + str(self).rstrip("\n"), self.message, tocall=self.tocall, fromcall=self.fromcall, ) - cl.sendall(repr(self)) + cl.sendall(str(self)) stats.APRSDStats().msgs_sent_inc() @@ -267,7 +302,22 @@ class TextMessage(Message): # an ack? Some messages we don't want to do this ever. self.allow_delay = allow_delay - def __repr__(self): + def dict(self): + now = datetime.datetime.now() + return { + "id": self.id, + "type": "text-message", + "fromcall": self.fromcall, + "tocall": self.tocall, + "message": self.message.rstrip("\n"), + "raw": str(self).rstrip("\n"), + "retry_count": self.retry_count, + "last_send_attempt": self.last_send_attempt, + "last_send_time": str(self.last_send_time), + "last_send_age": str(now - self.last_send_time), + } + + def __str__(self): """Build raw string to send over the air.""" return "{}>APZ100::{}:{}{{{}\n".format( self.fromcall, @@ -276,19 +326,6 @@ class TextMessage(Message): str(self.id), ) - def __str__(self): - delta = "Never" - if self.last_send_time: - now = datetime.datetime.now() - delta = now - self.last_send_time - return "{}>{} Msg({})({}): '{}'".format( - self.fromcall, - self.tocall, - self.id, - delta, - self.message, - ) - def _filter_for_send(self): """Filter and format message string for FCC.""" # max? ftm400 displays 64, raw msg shows 74 @@ -311,12 +348,12 @@ class TextMessage(Message): cl = client.get_client() log_message( "Sending Message Direct", - repr(self).rstrip("\n"), + str(self).rstrip("\n"), self.message, tocall=self.tocall, fromcall=self.fromcall, ) - cl.sendall(repr(self)) + cl.sendall(str(self)) stats.APRSDStats().msgs_tx_inc() @@ -370,13 +407,13 @@ class SendMessageThread(threads.APRSDThread): # tracking the time. log_message( "Sending Message", - repr(msg).rstrip("\n"), + str(msg).rstrip("\n"), msg.message, tocall=self.msg.tocall, retry_number=msg.last_send_attempt, msg_num=msg.id, ) - cl.sendall(repr(msg)) + cl.sendall(str(msg)) stats.APRSDStats().msgs_tx_inc() msg.last_send_time = datetime.datetime.now() msg.last_send_attempt += 1 @@ -392,29 +429,40 @@ class AckMessage(Message): def __init__(self, fromcall, tocall, msg_id): super().__init__(fromcall, tocall, msg_id=msg_id) - def __repr__(self): + def dict(self): + now = datetime.datetime.now() + return { + "id": self.id, + "type": "ack", + "fromcall": self.fromcall, + "tocall": self.tocall, + "raw": str(self).rstrip("\n"), + "retry_count": self.retry_count, + "last_send_attempt": self.last_send_attempt, + "last_send_time": str(self.last_send_time), + "last_send_age": str(now - self.last_send_time), + } + + def __str__(self): return "{}>APZ100::{}:ack{}\n".format( self.fromcall, self.tocall.ljust(9), self.id, ) - def __str__(self): - return "From({}) TO({}) Ack ({})".format(self.fromcall, self.tocall, self.id) - def send_thread(self): """Separate thread to send acks with retries.""" cl = client.get_client() for i in range(self.retry_count, 0, -1): log_message( "Sending ack", - repr(self).rstrip("\n"), + str(self).rstrip("\n"), None, ack=self.id, tocall=self.tocall, retry_number=i, ) - cl.sendall(repr(self)) + cl.sendall(str(self)) stats.APRSDStats().ack_tx_inc() # aprs duplicate detection is 30 secs? # (21 only sends first, 28 skips middle) @@ -433,13 +481,13 @@ class AckMessage(Message): cl = client.get_client() log_message( "Sending ack", - repr(self).rstrip("\n"), + str(self).rstrip("\n"), None, ack=self.id, tocall=self.tocall, fromcall=self.fromcall, ) - cl.sendall(repr(self)) + cl.sendall(str(self)) class SendAckThread(threads.APRSDThread): @@ -476,13 +524,13 @@ class SendAckThread(threads.APRSDThread): cl = client.get_client() log_message( "Sending ack", - repr(self.ack).rstrip("\n"), + str(self.ack).rstrip("\n"), None, ack=self.ack.id, tocall=self.ack.tocall, retry_number=self.ack.last_send_attempt, ) - cl.sendall(repr(self.ack)) + cl.sendall(str(self.ack)) stats.APRSDStats().ack_tx_inc() self.ack.last_send_attempt += 1 self.ack.last_send_time = datetime.datetime.now() diff --git a/aprsd/plugins/email.py b/aprsd/plugins/email.py index b13bffa..677d106 100644 --- a/aprsd/plugins/email.py +++ b/aprsd/plugins/email.py @@ -2,7 +2,7 @@ import logging import re import time -from aprsd import email, messaging, plugin +from aprsd import email, messaging, plugin, trace LOG = logging.getLogger("APRSD") @@ -18,6 +18,7 @@ class EmailPlugin(plugin.APRSDPluginBase): # five mins {int:int} email_sent_dict = {} + @trace.trace def command(self, fromcall, message, ack): LOG.info("Email COMMAND") reply = None @@ -79,6 +80,7 @@ class EmailPlugin(plugin.APRSDPluginBase): self.email_sent_dict.clear() self.email_sent_dict[ack] = now else: + reply = messaging.NULL_MESSAGE LOG.info( "Email for message number " + ack diff --git a/aprsd/plugins/fortune.py b/aprsd/plugins/fortune.py index a1100f9..9764234 100644 --- a/aprsd/plugins/fortune.py +++ b/aprsd/plugins/fortune.py @@ -2,7 +2,7 @@ import logging import shutil import subprocess -from aprsd import plugin +from aprsd import plugin, trace LOG = logging.getLogger("APRSD") @@ -14,6 +14,7 @@ class FortunePlugin(plugin.APRSDPluginBase): command_regex = "^[fF]" command_name = "fortune" + @trace.trace def command(self, fromcall, message, ack): LOG.info("FortunePlugin") reply = None diff --git a/aprsd/plugins/location.py b/aprsd/plugins/location.py index fd667ef..2dec600 100644 --- a/aprsd/plugins/location.py +++ b/aprsd/plugins/location.py @@ -2,7 +2,7 @@ import logging import re import time -from aprsd import plugin, plugin_utils, utils +from aprsd import plugin, plugin_utils, trace, utils LOG = logging.getLogger("APRSD") @@ -14,6 +14,7 @@ class LocationPlugin(plugin.APRSDPluginBase): command_regex = "^[lL]" command_name = "location" + @trace.trace def command(self, fromcall, message, ack): LOG.info("Location Plugin") # get last location of a callsign, get descriptive name from weather service @@ -65,6 +66,10 @@ class LocationPlugin(plugin.APRSDPluginBase): LOG.error("Couldn't fetch forecast.weather.gov '{}'".format(ex)) wx_data = {"location": {"areaDescription": "Unknown Location"}} + if "location" not in wx_data: + LOG.error("Couldn't fetch forecast.weather.gov '{}'".format(wx_data)) + wx_data = {"location": {"areaDescription": "Unknown Location"}} + reply = "{}: {} {}' {},{} {}h ago".format( searchcall, wx_data["location"]["areaDescription"], diff --git a/aprsd/plugins/ping.py b/aprsd/plugins/ping.py index 754da05..19e6ca2 100644 --- a/aprsd/plugins/ping.py +++ b/aprsd/plugins/ping.py @@ -1,7 +1,7 @@ import logging import time -from aprsd import plugin +from aprsd import plugin, trace LOG = logging.getLogger("APRSD") @@ -13,6 +13,7 @@ class PingPlugin(plugin.APRSDPluginBase): command_regex = "^[pP]" command_name = "ping" + @trace.trace def command(self, fromcall, message, ack): LOG.info("PINGPlugin") stm = time.localtime() diff --git a/aprsd/plugins/query.py b/aprsd/plugins/query.py index 759d779..35b2404 100644 --- a/aprsd/plugins/query.py +++ b/aprsd/plugins/query.py @@ -2,7 +2,7 @@ import datetime import logging import re -from aprsd import messaging, plugin +from aprsd import messaging, plugin, trace LOG = logging.getLogger("APRSD") @@ -14,6 +14,7 @@ class QueryPlugin(plugin.APRSDPluginBase): command_regex = r"^\!.*" command_name = "query" + @trace.trace def command(self, fromcall, message, ack): LOG.info("Query COMMAND") diff --git a/aprsd/plugins/time.py b/aprsd/plugins/time.py index 396d9c0..e5bc3ef 100644 --- a/aprsd/plugins/time.py +++ b/aprsd/plugins/time.py @@ -1,7 +1,7 @@ import logging import time -from aprsd import fuzzyclock, plugin, plugin_utils, utils +from aprsd import fuzzyclock, plugin, plugin_utils, trace, utils from opencage.geocoder import OpenCageGeocode import pytz @@ -38,6 +38,7 @@ class TimePlugin(plugin.APRSDPluginBase): return reply + @trace.trace def command(self, fromcall, message, ack): LOG.info("TIME COMMAND") # So we can mock this in unit tests @@ -52,6 +53,7 @@ class TimeOpenCageDataPlugin(TimePlugin): command_regex = "^[tT]" command_name = "Time" + @trace.trace def command(self, fromcall, message, ack): api_key = self.config["services"]["aprs.fi"]["apiKey"] try: @@ -92,6 +94,7 @@ class TimeOWMPlugin(TimePlugin): command_regex = "^[tT]" command_name = "Time" + @trace.trace def command(self, fromcall, message, ack): api_key = self.config["services"]["aprs.fi"]["apiKey"] try: diff --git a/aprsd/plugins/version.py b/aprsd/plugins/version.py index d037ac7..80cf766 100644 --- a/aprsd/plugins/version.py +++ b/aprsd/plugins/version.py @@ -1,7 +1,7 @@ import logging import aprsd -from aprsd import plugin +from aprsd import plugin, trace LOG = logging.getLogger("APRSD") @@ -17,6 +17,7 @@ class VersionPlugin(plugin.APRSDPluginBase): # five mins {int:int} email_sent_dict = {} + @trace.trace def command(self, fromcall, message, ack): LOG.info("Version COMMAND") return "APRSD version '{}'".format(aprsd.__version__) diff --git a/aprsd/plugins/weather.py b/aprsd/plugins/weather.py index 0dedc28..cd0a741 100644 --- a/aprsd/plugins/weather.py +++ b/aprsd/plugins/weather.py @@ -2,7 +2,7 @@ import json import logging import re -from aprsd import plugin, plugin_utils, utils +from aprsd import plugin, plugin_utils, trace, utils import requests LOG = logging.getLogger("APRSD") @@ -25,6 +25,7 @@ class USWeatherPlugin(plugin.APRSDPluginBase): command_regex = "^[wW]" command_name = "weather" + @trace.trace def command(self, fromcall, message, ack): LOG.info("Weather Plugin") try: @@ -84,6 +85,7 @@ class USMetarPlugin(plugin.APRSDPluginBase): command_regex = "^[metar]" command_name = "Metar" + @trace.trace def command(self, fromcall, message, ack): LOG.info("WX Plugin '{}'".format(message)) a = re.search(r"^.*\s+(.*)", message) @@ -175,6 +177,7 @@ class OWMWeatherPlugin(plugin.APRSDPluginBase): command_regex = "^[wW]" command_name = "Weather" + @trace.trace def command(self, fromcall, message, ack): LOG.info("OWMWeather Plugin '{}'".format(message)) a = re.search(r"^.*\s+(.*)", message) @@ -295,6 +298,7 @@ class AVWXWeatherPlugin(plugin.APRSDPluginBase): command_regex = "^[metar]" command_name = "Weather" + @trace.trace def command(self, fromcall, message, ack): LOG.info("OWMWeather Plugin '{}'".format(message)) a = re.search(r"^.*\s+(.*)", message) diff --git a/aprsd/threads.py b/aprsd/threads.py index ea595e9..736a052 100644 --- a/aprsd/threads.py +++ b/aprsd/threads.py @@ -4,8 +4,9 @@ import logging import queue import threading import time +import tracemalloc -from aprsd import client, messaging, plugin, stats +from aprsd import client, messaging, plugin, stats, trace import aprslib LOG = logging.getLogger("APRSD") @@ -69,6 +70,7 @@ class KeepAliveThread(APRSDThread): def __init__(self): super().__init__("KeepAlive") + tracemalloc.start() def loop(self): if self.cntr % 6 == 0: @@ -81,14 +83,17 @@ class KeepAliveThread(APRSDThread): else: email_thread_time = "N/A" + current, peak = tracemalloc.get_traced_memory() LOG.debug( "Uptime ({}) Tracker({}) " - "Msgs: TX:{} RX:{} EmailThread: {}".format( + "Msgs: TX:{} RX:{} EmailThread: {} RAM: Current:{} Peak:{}".format( stats_obj.uptime, len(tracker), stats_obj.msgs_tx, stats_obj.msgs_rx, email_thread_time, + current, + peak, ), ) self.cntr += 1 @@ -219,11 +224,11 @@ class APRSDRXThread(APRSDThread): self.msg_queues["tx"].put(ack) LOG.debug("Packet processing complete") + @trace.trace def process_packet(self, packet): """Process a packet recieved from aprs-is server.""" try: - LOG.info("Got message: {}".format(packet)) stats.APRSDStats().msgs_rx_inc() msg = packet.get("message_text", None) diff --git a/aprsd/trace.py b/aprsd/trace.py new file mode 100644 index 0000000..3b985e2 --- /dev/null +++ b/aprsd/trace.py @@ -0,0 +1,181 @@ +import abc +import functools +import inspect +import logging +import time +import types + +VALID_TRACE_FLAGS = {"method", "api"} +TRACE_API = False +TRACE_METHOD = False +TRACE_ENABLED = False +LOG = logging.getLogger("APRSD") + + +def trace(*dec_args, **dec_kwargs): + """Trace calls to the decorated function. + + This decorator should always be defined as the outermost decorator so it + is defined last. This is important so it does not interfere + with other decorators. + + Using this decorator on a function will cause its execution to be logged at + `DEBUG` level with arguments, return values, and exceptions. + + :returns: a function decorator + """ + + def _decorator(f): + + func_name = f.__name__ + + @functools.wraps(f) + def trace_logging_wrapper(*args, **kwargs): + filter_function = dec_kwargs.get("filter_function") + logger = LOG + + # NOTE(ameade): Don't bother going any further if DEBUG log level + # is not enabled for the logger. + if not logger.isEnabledFor(logging.DEBUG) or not TRACE_ENABLED: + return f(*args, **kwargs) + + all_args = inspect.getcallargs(f, *args, **kwargs) + + pass_filter = filter_function is None or filter_function(all_args) + + if pass_filter: + logger.debug( + "==> %(func)s: call %(all_args)r", + { + "func": func_name, + "all_args": str(all_args), + }, + ) + + start_time = time.time() * 1000 + try: + result = f(*args, **kwargs) + except Exception as exc: + total_time = int(round(time.time() * 1000)) - start_time + logger.debug( + "<== %(func)s: exception (%(time)dms) %(exc)r", + { + "func": func_name, + "time": total_time, + "exc": exc, + }, + ) + raise + total_time = int(round(time.time() * 1000)) - start_time + + if isinstance(result, dict): + mask_result = result + elif isinstance(result, str): + mask_result = result + else: + mask_result = result + + if pass_filter: + logger.debug( + "<== %(func)s: return (%(time)dms) %(result)r", + { + "func": func_name, + "time": total_time, + "result": mask_result, + }, + ) + return result + + return trace_logging_wrapper + + if len(dec_args) == 0: + # filter_function is passed and args does not contain f + return _decorator + else: + # filter_function is not passed + return _decorator(dec_args[0]) + + +def trace_api(*dec_args, **dec_kwargs): + """Decorates a function if TRACE_API is true.""" + + def _decorator(f): + @functools.wraps(f) + def trace_api_logging_wrapper(*args, **kwargs): + if TRACE_API: + return trace(f, *dec_args, **dec_kwargs)(*args, **kwargs) + return f(*args, **kwargs) + + return trace_api_logging_wrapper + + if len(dec_args) == 0: + # filter_function is passed and args does not contain f + return _decorator + else: + # filter_function is not passed + return _decorator(dec_args[0]) + + +def trace_method(f): + """Decorates a function if TRACE_METHOD is true.""" + + @functools.wraps(f) + def trace_method_logging_wrapper(*args, **kwargs): + if TRACE_METHOD: + return trace(f)(*args, **kwargs) + return f(*args, **kwargs) + + return trace_method_logging_wrapper + + +class TraceWrapperMetaclass(type): + """Metaclass that wraps all methods of a class with trace_method. + + This metaclass will cause every function inside of the class to be + decorated with the trace_method decorator. + + To use the metaclass you define a class like so: + class MyClass(object, metaclass=utils.TraceWrapperMetaclass): + """ + + def __new__(cls, classname, bases, class_dict): + new_class_dict = {} + for attribute_name, attribute in class_dict.items(): + if isinstance(attribute, types.FunctionType): + # replace it with a wrapped version + attribute = functools.update_wrapper( + trace_method(attribute), + attribute, + ) + new_class_dict[attribute_name] = attribute + + return type.__new__(cls, classname, bases, new_class_dict) + + +class TraceWrapperWithABCMetaclass(abc.ABCMeta, TraceWrapperMetaclass): + """Metaclass that wraps all methods of a class with trace.""" + + pass + + +def setup_tracing(trace_flags): + """Set global variables for each trace flag. + + Sets variables TRACE_METHOD and TRACE_API, which represent + whether to log methods or api traces. + + :param trace_flags: a list of strings + """ + global TRACE_METHOD + global TRACE_API + global TRACE_ENABLED + + try: + trace_flags = [flag.strip() for flag in trace_flags] + except TypeError: # Handle when trace_flags is None or a test mock + trace_flags = [] + for invalid_flag in set(trace_flags) - VALID_TRACE_FLAGS: + LOG.warning("Invalid trace flag: %s", invalid_flag) + TRACE_METHOD = "method" in trace_flags + TRACE_API = "api" in trace_flags + TRACE_ENABLED = True diff --git a/aprsd/utils.py b/aprsd/utils.py index 182ef89..8b84cea 100644 --- a/aprsd/utils.py +++ b/aprsd/utils.py @@ -22,6 +22,7 @@ DEFAULT_CONFIG_DICT = { }, "aprsd": { "logfile": "/tmp/aprsd.log", + "trace": False, "plugin_dir": "~/.config/aprsd/plugins", "enabled_plugins": plugin.CORE_PLUGINS, "units": "imperial", @@ -29,6 +30,9 @@ DEFAULT_CONFIG_DICT = { "enabled": True, "host": "0.0.0.0", "port": 8001, + "users": { + "admin": "aprsd", + }, }, "email": { "enabled": True, @@ -43,6 +47,7 @@ DEFAULT_CONFIG_DICT = { "host": "smtp.gmail.com", "port": 465, "use_ssl": False, + "debug": False, }, "imap": { "login": "IMAP_USERNAME", @@ -50,6 +55,7 @@ DEFAULT_CONFIG_DICT = { "host": "imap.gmail.com", "port": 993, "use_ssl": True, + "debug": False, }, }, }, @@ -297,6 +303,15 @@ def parse_config(config_file): ["aprs", "password"], default_fail=DEFAULT_CONFIG_DICT["aprs"]["password"], ) + + # Ensure they change the admin password + if config["aprsd"]["web"]["enabled"] is True: + check_option( + config, + ["aprsd", "web", "users", "admin"], + default_fail=DEFAULT_CONFIG_DICT["aprsd"]["web"]["users"]["admin"], + ) + if config["aprsd"]["email"]["enabled"] is True: # Check IMAP server settings check_option(config, ["aprsd", "email", "imap", "host"]) diff --git a/aprsd/web/static/json-viewer/jquery.json-viewer.css b/aprsd/web/static/json-viewer/jquery.json-viewer.css new file mode 100644 index 0000000..57aa450 --- /dev/null +++ b/aprsd/web/static/json-viewer/jquery.json-viewer.css @@ -0,0 +1,57 @@ +/* Root element */ +.json-document { + padding: 1em 2em; +} + +/* Syntax highlighting for JSON objects */ +ul.json-dict, ol.json-array { + list-style-type: none; + margin: 0 0 0 1px; + border-left: 1px dotted #ccc; + padding-left: 2em; +} +.json-string { + color: #0B7500; +} +.json-literal { + color: #1A01CC; + font-weight: bold; +} + +/* Toggle button */ +a.json-toggle { + position: relative; + color: inherit; + text-decoration: none; +} +a.json-toggle:focus { + outline: none; +} +a.json-toggle:before { + font-size: 1.1em; + color: #c0c0c0; + content: "\25BC"; /* down arrow */ + position: absolute; + display: inline-block; + width: 1em; + text-align: center; + line-height: 1em; + left: -1.2em; +} +a.json-toggle:hover:before { + color: #aaa; +} +a.json-toggle.collapsed:before { + /* Use rotated down arrow, prevents right arrow appearing smaller than down arrow in some browsers */ + transform: rotate(-90deg); +} + +/* Collapsable placeholder links */ +a.json-placeholder { + color: #aaa; + padding: 0 1em; + text-decoration: none; +} +a.json-placeholder:hover { + text-decoration: underline; +} diff --git a/aprsd/web/static/json-viewer/jquery.json-viewer.js b/aprsd/web/static/json-viewer/jquery.json-viewer.js new file mode 100644 index 0000000..611411b --- /dev/null +++ b/aprsd/web/static/json-viewer/jquery.json-viewer.js @@ -0,0 +1,158 @@ +/** + * jQuery json-viewer + * @author: Alexandre Bodelot + * @link: https://github.com/abodelot/jquery.json-viewer + */ +(function($) { + + /** + * Check if arg is either an array with at least 1 element, or a dict with at least 1 key + * @return boolean + */ + function isCollapsable(arg) { + return arg instanceof Object && Object.keys(arg).length > 0; + } + + /** + * Check if a string represents a valid url + * @return boolean + */ + function isUrl(string) { + var urlRegexp = /^(https?:\/\/|ftps?:\/\/)?([a-z0-9%-]+\.){1,}([a-z0-9-]+)?(:(\d{1,5}))?(\/([a-z0-9\-._~:/?#[\]@!$&'()*+,;=%]+)?)?$/i; + return urlRegexp.test(string); + } + + /** + * Transform a json object into html representation + * @return string + */ + function json2html(json, options) { + var html = ''; + if (typeof json === 'string') { + // Escape tags and quotes + json = json + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/'/g, ''') + .replace(/"/g, '"'); + + if (options.withLinks && isUrl(json)) { + html += '' + json + ''; + } else { + // Escape double quotes in the rendered non-URL string. + json = json.replace(/"/g, '\\"'); + html += '"' + json + '"'; + } + } else if (typeof json === 'number') { + html += '' + json + ''; + } else if (typeof json === 'boolean') { + html += '' + json + ''; + } else if (json === null) { + html += 'null'; + } else if (json instanceof Array) { + if (json.length > 0) { + html += '[
    '; + for (var i = 0; i < json.length; ++i) { + html += '
  1. '; + // Add toggle button if item is collapsable + if (isCollapsable(json[i])) { + html += ''; + } + html += json2html(json[i], options); + // Add comma if item is not last + if (i < json.length - 1) { + html += ','; + } + html += '
    2. '; + } + html += '
]'; + } else { + html += '[]'; + } + } else if (typeof json === 'object') { + var keyCount = Object.keys(json).length; + if (keyCount > 0) { + html += '{}'; + } else { + html += '{}'; + } + } + return html; + } + + /** + * jQuery plugin method + * @param json: a javascript object + * @param options: an optional options hash + */ + $.fn.jsonViewer = function(json, options) { + // Merge user options with default options + options = Object.assign({}, { + collapsed: false, + rootCollapsable: true, + withQuotes: false, + withLinks: true + }, options); + + // jQuery chaining + return this.each(function() { + + // Transform to HTML + var html = json2html(json, options); + if (options.rootCollapsable && isCollapsable(json)) { + html = '' + html; + } + + // Insert HTML in target DOM element + $(this).html(html); + $(this).addClass('json-document'); + + // Bind click on toggle buttons + $(this).off('click'); + $(this).on('click', 'a.json-toggle', function() { + var target = $(this).toggleClass('collapsed').siblings('ul.json-dict, ol.json-array'); + target.toggle(); + if (target.is(':visible')) { + target.siblings('.json-placeholder').remove(); + } else { + var count = target.children('li').length; + var placeholder = count + (count > 1 ? ' items' : ' item'); + target.after('' + placeholder + ''); + } + return false; + }); + + // Simulate click on toggle button when placeholder is clicked + $(this).on('click', 'a.json-placeholder', function() { + $(this).siblings('a.json-toggle').click(); + return false; + }); + + if (options.collapsed == true) { + // Trigger click to collapse all nodes + $(this).find('a.json-toggle').click(); + } + }); + }; +})(jQuery); diff --git a/aprsd/templates/index.html b/aprsd/web/templates/index.html similarity index 100% rename from aprsd/templates/index.html rename to aprsd/web/templates/index.html diff --git a/aprsd/web/templates/messages.html b/aprsd/web/templates/messages.html new file mode 100644 index 0000000..c3f6beb --- /dev/null +++ b/aprsd/web/templates/messages.html @@ -0,0 +1,15 @@ + + + + + + + + 

+
+    
+
+
diff --git a/build/build.sh b/build/build.sh
deleted file mode 100755
index 356ccdd..0000000
--- a/build/build.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-# Use this script to locally build the docker image
-docker build --no-cache -t hemna6969/aprsd:latest ..
diff --git a/Dockerfile-dev b/docker/Dockerfile
similarity index 94%
rename from Dockerfile-dev
rename to docker/Dockerfile
index cc968d3..4b5bf32 100644
--- a/Dockerfile-dev
+++ b/docker/Dockerfile
@@ -1,12 +1,12 @@
 FROM alpine:latest as aprsd
 
 # Dockerfile for building a container during aprsd development.
+ARG BRANCH
 
-ENV VERSION=1.5.1
 ENV APRS_USER=aprs
 ENV HOME=/home/aprs
 ENV APRSD=http://github.com/craigerl/aprsd.git
-ENV APRSD_BRANCH="master"
+ENV APRSD_BRANCH=$BRANCH
 ENV VIRTUAL_ENV=$HOME/.venv3
 
 ENV INSTALL=$HOME/install
@@ -46,5 +46,5 @@ RUN chown -R $APRS_USER:$APRS_USER /config
 ENV CONF default
 USER $APRS_USER
 
-ADD build/bin/run.sh $HOME/
+ADD bin/run.sh $HOME/
 ENTRYPOINT ["/home/aprs/run.sh"]
diff --git a/build/bin/run.sh b/docker/bin/run.sh
similarity index 100%
rename from build/bin/run.sh
rename to docker/bin/run.sh
diff --git a/docker/build.sh b/docker/build.sh
new file mode 100755
index 0000000..005a916
--- /dev/null
+++ b/docker/build.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Use this script to locally build the docker image
+docker build --no-cache -t hemna6969/aprsd:latest -f ./Dockerfile .
diff --git a/docker-compose.yml b/docker/docker-compose.yml
similarity index 100%
rename from docker-compose.yml
rename to docker/docker-compose.yml
diff --git a/requirements.in b/requirements.in
index b67542b..b4775ee 100644
--- a/requirements.in
+++ b/requirements.in
@@ -13,4 +13,5 @@ pre-commit
 pytz
 opencage
 flask
-flask_classful
+flask-classful
+flask-httpauth
diff --git a/requirements.txt b/requirements.txt
index ab1554d..af86952 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -37,10 +37,13 @@ filelock==3.0.12
     #   virtualenv
 flask-classful==0.14.2
     # via -r requirements.in
+flask-httpauth==4.2.0
+    # via -r requirements.in
 flask==1.1.2
     # via
     #   -r requirements.in
     #   flask-classful
+    #   flask-httpauth
 identify==1.5.13
     # via pre-commit
 idna==2.10