From 082db7325de50c508a96db225ad196e47fbaf806 Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 14 Dec 2022 19:21:25 -0500 Subject: [PATCH 01/21] Started using dataclasses to describe packets This patch adds new Packet classes to describe the incoming packets parsed out from aprslib. --- aprsd/client.py | 8 ++ aprsd/cmds/listen.py | 62 ++++++--------- aprsd/cmds/webchat.py | 2 +- aprsd/packets.py | 174 ++++++++++++++++++++++++++++++++++++++---- aprsd/threads/rx.py | 121 +++++++++++++++-------------- dev-requirements.txt | 8 +- requirements.in | 2 + requirements.txt | 2 + 8 files changed, 265 insertions(+), 114 deletions(-) diff --git a/aprsd/client.py b/aprsd/client.py index 89f259b..7664d62 100644 --- a/aprsd/client.py +++ b/aprsd/client.py @@ -31,6 +31,7 @@ class Client: connected = False server_string = None + filter = None def __new__(cls, *args, **kwargs): """This magic turns this into a singleton.""" @@ -44,10 +45,17 @@ class Client: if config: self.config = config + def set_filter(self, filter): + self.filter = filter + if self._client: + self._client.set_filter(filter) + @property def client(self): if not self._client: self._client = self.setup_connection() + if self.filter: + self._client.set_filter(self.filter) return self._client def reset(self): diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index 1afbcef..2945548 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -5,10 +5,10 @@ # python included libs import datetime import logging +import signal import sys import time -import aprslib import click from rich.console import Console @@ -16,7 +16,7 @@ from rich.console import Console import aprsd from aprsd import cli_helper, client, messaging, packets, stats, threads, utils from aprsd.aprsd import cli -from aprsd.utils import trace +from aprsd.threads import rx # setup the global logger @@ -37,6 +37,14 @@ def signal_handler(sig, frame): LOG.info(stats.APRSDStats()) +class APRSDListenThread(rx.APRSDRXThread): + def process_packet(self, *args, **kwargs): + raw = self._client.decode_packet(*args, **kwargs) + packet = packets.Packet.factory(raw) + LOG.debug(f"Got packet {packet}") + packet.log(header="RX Packet") + + @cli.command() @cli_helper.add_options(cli_helper.common_options) @click.option( @@ -74,6 +82,8 @@ def listen( o/obj1/obj2... - Object Filter Pass all objects with the exact name of obj1, obj2, ... (* wild card allowed)\n """ + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) config = ctx.obj["config"] if not aprs_login: @@ -109,26 +119,6 @@ def listen( packets.WatchList(config=config).load() packets.SeenList(config=config).load() - @trace.trace - def rx_packet(packet): - resp = packet.get("response", None) - if resp == "ack": - ack_num = packet.get("msgNo") - console.log(f"We saw an ACK {ack_num} Ignoring") - messaging.log_packet(packet) - else: - message = packet.get("message_text", None) - fromcall = packet["from"] - msg_number = packet.get("msgNo", "0") - messaging.log_message( - "Received Message", - packet["raw"], - message, - fromcall=fromcall, - ack=msg_number, - console=console, - ) - # Initialize the client factory and create # The correct client object ready for use client.ClientFactory.setup(config) @@ -140,29 +130,21 @@ def listen( # Creates the client object LOG.info("Creating client connection") aprs_client = client.factory.create() - console.log(aprs_client) + LOG.info(aprs_client) LOG.debug(f"Filter by '{filter}'") - aprs_client.client.set_filter(filter) + aprs_client.set_filter(filter) packets.PacketList(config=config) keepalive = threads.KeepAliveThread(config=config) keepalive.start() - while True: - try: - # This will register a packet consumer with aprslib - # When new packets come in the consumer will process - # the packet - # with console.status("Listening for packets"): - aprs_client.client.consumer(rx_packet, raw=False) - except aprslib.exceptions.ConnectionDrop: - LOG.error("Connection dropped, reconnecting") - time.sleep(5) - # Force the deletion of the client object connected to aprs - # This will cause a reconnect, next time client.get_client() - # is called - aprs_client.reset() - except aprslib.exceptions.UnknownFormat: - LOG.error("Got a Bad packet") + LOG.debug("Create APRSDListenThread") + listen_thread = APRSDListenThread(threads.msg_queues, config=config) + LOG.debug("Start APRSDListenThread") + listen_thread.start() + LOG.debug("keepalive Join") + keepalive.join() + LOG.debug("listen_thread Join") + listen_thread.join() diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index 4bf35f8..f62b48b 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -178,7 +178,7 @@ class WebChatProcessPacketThread(rx.APRSDProcessPacketThread): ) self.got_ack = True - def process_non_ack_packet(self, packet): + def process_our_message_packet(self, packet): LOG.info(f"process non ack PACKET {packet}") packet.get("addresse", None) fromcall = packet["from"] diff --git a/aprsd/packets.py b/aprsd/packets.py index 83d795d..e111fc6 100644 --- a/aprsd/packets.py +++ b/aprsd/packets.py @@ -1,8 +1,10 @@ +from dataclasses import asdict, dataclass, field import datetime import logging import threading import time +import dacite import wrapt from aprsd import utils @@ -14,6 +16,150 @@ LOG = logging.getLogger("APRSD") PACKET_TYPE_MESSAGE = "message" PACKET_TYPE_ACK = "ack" PACKET_TYPE_MICE = "mic-e" +PACKET_TYPE_WX = "weather" +PACKET_TYPE_UNKNOWN = "unknown" +PACKET_TYPE_STATUS = "status" + + +@dataclass +class Packet: + from_call: str + to_call: str + addresse: str = None + format: str = None + msgNo: str = None + packet_type: str = None + timestamp: float = field(default_factory=time.time) + raw: str = None + _raw_dict: dict = field(repr=True, default_factory=lambda: {}) + + @staticmethod + def factory(raw): + raw["_raw_dict"] = raw.copy() + translate_fields = { + "from": "from_call", + "to": "to_call", + } + # First translate some fields + for key in translate_fields: + if key in raw: + raw[translate_fields[key]] = raw[key] + del raw[key] + + if "addresse" in raw: + raw["to_call"] = raw["addresse"] + + class_lookup = { + PACKET_TYPE_WX: WeatherPacket, + PACKET_TYPE_MESSAGE: MessagePacket, + PACKET_TYPE_ACK: AckPacket, + PACKET_TYPE_MICE: MicEPacket, + PACKET_TYPE_STATUS: StatusPacket, + PACKET_TYPE_UNKNOWN: Packet, + } + packet_type = get_packet_type(raw) + raw["packet_type"] = packet_type + class_name = class_lookup[packet_type] + if packet_type == PACKET_TYPE_UNKNOWN: + # Try and figure it out here + if "latitude" in raw: + class_name = GPSPacket + + if packet_type == PACKET_TYPE_WX: + # the weather information is in a dict + # this brings those values out to the outer dict + for key in raw["weather"]: + raw[key] = raw["weather"][key] + + return dacite.from_dict(data_class=class_name, data=raw) + + def log(self, header=None): + """LOG a packet to the logfile.""" + asdict(self) + log_list = ["\n"] + if header: + log_list.append(f"{header} _______________") + log_list.append(f" Packet : {self.__class__.__name__}") + log_list.append(f" Raw : {self.raw}") + if self.to_call: + log_list.append(f" To : {self.to_call}") + if self.from_call: + log_list.append(f" From : {self.from_call}") + if hasattr(self, "path"): + log_list.append(f" Path : {'=>'.join(self.path)}") + if hasattr(self, "via"): + log_list.append(f" VIA : {self.via}") + + elif isinstance(self, MessagePacket): + log_list.append(f" Message : {self.message_text}") + + if self.msgNo: + log_list.append(f" Msg # : {self.msgNo}") + log_list.append(f"{header} _______________ Complete") + + + LOG.info(self) + LOG.info("\n".join(log_list)) + +@dataclass +class PathPacket(Packet): + path: list[str] = field(default_factory=list) + via: str = None + + +@dataclass +class AckPacket(PathPacket): + response: str = None + +@dataclass +class MessagePacket(PathPacket): + message_text: str = None + + +@dataclass +class StatusPacket(PathPacket): + status: str = None + timestamp: int = 0 + messagecapable: bool = False + comment: str = None + + +@dataclass +class GPSPacket(PathPacket): + latitude: float = 0.00 + longitude: float = 0.00 + altitude: float = 0.00 + rng: float = 0.00 + posambiguity: int = 0 + timestamp: int = 0 + comment: str = None + symbol: str = None + symbol_table: str = None + speed: float = 0.00 + course: int = 0 + + +@dataclass +class MicEPacket(GPSPacket): + messagecapable: bool = False + mbits: str = None + mtype: str = None + + +@dataclass +class WeatherPacket(GPSPacket): + symbol: str = "_" + wind_gust: float = 0.00 + temperature: float = 0.00 + rain_1h: float = 0.00 + rain_24h: float = 0.00 + rain_since_midnight: float = 0.00 + humidity: int = 0 + pressure: float = 0.00 + messagecapable: bool = False + comment: str = None + + class PacketList: @@ -45,11 +191,8 @@ class PacketList: @wrapt.synchronized(lock) def add(self, packet): - packet["ts"] = time.time() - if ( - "fromcall" in packet - and packet["fromcall"] == self.config["aprs"]["login"] - ): + packet.ts = time.time() + if (packet.from_call == self.config["aprs"]["login"]): self.total_tx += 1 else: self.total_recv += 1 @@ -116,7 +259,10 @@ class WatchList(objectstore.ObjectStoreMixin): @wrapt.synchronized(lock) def update_seen(self, packet): - callsign = packet["from"] + if packet.addresse: + callsign = packet.addresse + else: + callsign = packet.from_call if self.callsign_in_watchlist(callsign): self.data[callsign]["last"] = datetime.datetime.now() self.data[callsign]["packets"].append(packet) @@ -178,10 +324,8 @@ class SeenList(objectstore.ObjectStoreMixin): @wrapt.synchronized(lock) def update_seen(self, packet): callsign = None - if "fromcall" in packet: - callsign = packet["fromcall"] - elif "from" in packet: - callsign = packet["from"] + if packet.from_call: + callsign = packet.from_call else: LOG.warning(f"Can't find FROM in packet {packet}") return @@ -200,12 +344,16 @@ def get_packet_type(packet): msg_format = packet.get("format", None) msg_response = packet.get("response", None) packet_type = "unknown" - if msg_format == "message": - packet_type = PACKET_TYPE_MESSAGE - elif msg_response == "ack": + if msg_format == "message" and msg_response == "ack": packet_type = PACKET_TYPE_ACK + elif msg_format == "message": + packet_type = PACKET_TYPE_MESSAGE elif msg_format == "mic-e": packet_type = PACKET_TYPE_MICE + elif msg_format == "status": + packet_type = PACKET_TYPE_STATUS + elif packet.get("symbol", None) == "_": + packet_type = PACKET_TYPE_WX return packet_type diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index 5529bfa..bf85a72 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -23,7 +23,6 @@ class APRSDRXThread(APRSDThread): client.factory.create().client.stop() def loop(self): - # setup the consumer of messages and block until a messages try: # This will register a packet consumer with aprslib @@ -65,7 +64,12 @@ class APRSDPluginRXThread(APRSDRXThread): processing in the PluginProcessPacketThread. """ def process_packet(self, *args, **kwargs): - packet = self._client.decode_packet(*args, **kwargs) + raw = self._client.decode_packet(*args, **kwargs) + #LOG.debug(raw) + packet = packets.Packet.factory(raw.copy()) + packet.log(header="RX Packet") + #LOG.debug(packet) + del raw thread = APRSDPluginProcessPacketThread( config=self.config, packet=packet, @@ -84,18 +88,18 @@ class APRSDProcessPacketThread(APRSDThread): def __init__(self, config, packet): self.config = config self.packet = packet - name = self.packet["raw"][:10] + name = self.packet.raw[:10] super().__init__(f"RXPKT-{name}") def process_ack_packet(self, packet): - ack_num = packet.get("msgNo") + ack_num = packet.msgNo LOG.info(f"Got ack for message {ack_num}") messaging.log_message( "RXACK", - packet["raw"], + packet.raw, None, ack=ack_num, - fromcall=packet["from"], + fromcall=packet.from_call, ) tracker = messaging.MsgTrack() tracker.remove(ack_num) @@ -106,56 +110,60 @@ class APRSDProcessPacketThread(APRSDThread): """Process a packet received from aprs-is server.""" packet = self.packet packets.PacketList().add(packet) + our_call = self.config["aprsd"]["callsign"].lower() - fromcall = packet["from"] - tocall = packet.get("addresse", None) - msg = packet.get("message_text", None) - msg_id = packet.get("msgNo", "0") - msg_response = packet.get("response", None) - # LOG.debug(f"Got packet from '{fromcall}' - {packet}") + from_call = packet.from_call + if packet.addresse: + to_call = packet.addresse + else: + to_call = packet.to_call + msg_id = packet.msgNo # We don't put ack packets destined for us through the # plugins. + wl = packets.WatchList() + wl.update_seen(packet) if ( - tocall - and tocall.lower() == self.config["aprsd"]["callsign"].lower() - and msg_response == "ack" + isinstance(packet, packets.AckPacket) + and packet.addresse.lower() == our_call ): self.process_ack_packet(packet) else: - # It's not an ACK for us, so lets run it through - # the plugins. - messaging.log_message( - "Received Message", - packet["raw"], - msg, - fromcall=fromcall, - msg_num=msg_id, - ) - # Only ack messages that were sent directly to us - if ( - tocall - and tocall.lower() == self.config["aprsd"]["callsign"].lower() - ): - stats.APRSDStats().msgs_rx_inc() - # let any threads do their thing, then ack - # send an ack last - ack = messaging.AckMessage( - self.config["aprsd"]["callsign"], - fromcall, - msg_id=msg_id, - ) - ack.send() + if isinstance(packet, packets.MessagePacket): + if to_call and to_call.lower() == our_call: + # It's a MessagePacket and it's for us! + stats.APRSDStats().msgs_rx_inc() + # let any threads do their thing, then ack + # send an ack last + ack = messaging.AckMessage( + self.config["aprsd"]["callsign"], + from_call, + msg_id=msg_id, + ) + ack.send() - self.process_non_ack_packet(packet) + self.process_our_message_packet(packet) + else: + # Packet wasn't meant for us! + self.process_other_packet(packet, for_us=False) else: - LOG.info("Packet was not for us.") + self.process_other_packet( + packet, for_us=(to_call.lower() == our_call), + ) LOG.debug("Packet processing complete") @abc.abstractmethod - def process_non_ack_packet(self, *args, **kwargs): - """Ack packets already dealt with here.""" + def process_our_message_packet(self, *args, **kwargs): + """Process a MessagePacket destined for us!""" + + def process_other_packet(self, packet, for_us=False): + """Process an APRS Packet that isn't a message or ack""" + if not for_us: + LOG.info("Got a packet not meant for us.") + else: + LOG.info("Got a non AckPacket/MessagePacket") + LOG.info(packet) class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): @@ -163,18 +171,19 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): This is the main aprsd server plugin processing thread.""" - def process_non_ack_packet(self, packet): + def process_our_message_packet(self, packet): """Send the packet through the plugins.""" - fromcall = packet["from"] - tocall = packet.get("addresse", None) - msg = packet.get("message_text", None) - packet.get("msgNo", "0") - packet.get("response", None) + from_call = packet.from_call + if packet.addresse: + to_call = packet.addresse + else: + to_call = None + # msg = packet.get("message_text", None) + # packet.get("msgNo", "0") + # packet.get("response", None) pm = plugin.PluginManager() try: results = pm.run(packet) - wl = packets.WatchList() - wl.update_seen(packet) replied = False for reply in results: if isinstance(reply, list): @@ -187,7 +196,7 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): else: msg = messaging.TextMessage( self.config["aprsd"]["callsign"], - fromcall, + from_call, subreply, ) msg.send() @@ -207,18 +216,18 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): msg = messaging.TextMessage( self.config["aprsd"]["callsign"], - fromcall, + from_call, reply, ) msg.send() # If the message was for us and we didn't have a # response, then we send a usage statement. - if tocall == self.config["aprsd"]["callsign"] and not replied: + if to_call == self.config["aprsd"]["callsign"] and not replied: LOG.warning("Sending help!") msg = messaging.TextMessage( self.config["aprsd"]["callsign"], - fromcall, + from_call, "Unknown command! Send 'help' message for help", ) msg.send() @@ -226,11 +235,11 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): LOG.error("Plugin failed!!!") LOG.exception(ex) # Do we need to send a reply? - if tocall == self.config["aprsd"]["callsign"]: + if to_call == self.config["aprsd"]["callsign"]: reply = "A Plugin failed! try again?" msg = messaging.TextMessage( self.config["aprsd"]["callsign"], - fromcall, + from_call, reply, ) msg.send() diff --git a/dev-requirements.txt b/dev-requirements.txt index 2472816..93d34eb 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,7 +4,7 @@ # # pip-compile --annotation-style=line --resolver=backtracking dev-requirements.in # -add-trailing-comma==2.3.0 # via gray +add-trailing-comma==2.4.0 # via gray alabaster==0.7.12 # via sphinx attrs==22.1.0 # via jsonschema, pytest autoflake==1.5.3 # via gray @@ -34,7 +34,7 @@ imagesize==1.4.1 # via sphinx importlib-metadata==5.1.0 # via sphinx importlib-resources==5.10.1 # via fixit iniconfig==1.1.1 # via pytest -isort==5.10.1 # via -r dev-requirements.in, gray +isort==5.11.2 # via -r dev-requirements.in, gray jinja2==3.1.2 # via sphinx jsonschema==4.17.3 # via fixit libcst==0.4.9 # via fixit @@ -47,7 +47,7 @@ packaging==22.0 # via build, pyproject-api, pytest, sphinx, tox pathspec==0.10.3 # via black pep517==0.13.0 # via build pep8-naming==0.13.2 # via -r dev-requirements.in -pip-tools==6.11.0 # via -r dev-requirements.in +pip-tools==6.12.0 # via -r dev-requirements.in platformdirs==2.6.0 # via black, tox, virtualenv pluggy==1.0.0 # via pytest, tox pre-commit==2.20.0 # via -r dev-requirements.in @@ -74,7 +74,7 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx tokenize-rt==5.0.0 # via add-trailing-comma, pyupgrade toml==0.10.2 # via autoflake, pre-commit tomli==2.0.1 # via black, build, coverage, mypy, pep517, pyproject-api, pytest, tox -tox==4.0.8 # via -r dev-requirements.in +tox==4.0.9 # via -r dev-requirements.in typing-extensions==4.4.0 # via black, libcst, mypy, typing-inspect typing-inspect==0.8.0 # via libcst unify==0.5 # via gray diff --git a/requirements.in b/requirements.in index 368f239..f80c069 100644 --- a/requirements.in +++ b/requirements.in @@ -27,3 +27,5 @@ attrs==22.1.0 # for mobile checking user-agents pyopenssl +dataclasses +dacite2 diff --git a/requirements.txt b/requirements.txt index 2dd4a1c..a0e3277 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,8 @@ click==8.1.3 # via -r requirements.in, click-completion, flask click-completion==0.5.2 # via -r requirements.in commonmark==0.9.1 # via rich cryptography==38.0.4 # via pyopenssl +dacite2==2.0.0 # via -r requirements.in +dataclasses==0.6 # via -r requirements.in dnspython==2.2.1 # via eventlet eventlet==0.33.2 # via -r requirements.in flask==2.1.2 # via -r requirements.in, flask-classful, flask-httpauth, flask-socketio From 67a441d443f4460b0db0c67f404b6d5db85024af Mon Sep 17 00:00:00 2001 From: Hemna Date: Wed, 14 Dec 2022 22:03:21 -0500 Subject: [PATCH 02/21] Updated plugins and plugin interfaces for Packet This patch updates unit tests as well as the Plugin filter() interface to accept a packets.Packet object instead of a packet dictionary. --- aprsd/cmds/webchat.py | 12 +++--- aprsd/messaging.py | 6 ++- aprsd/packets.py | 70 +++++++++++++++++++------------ aprsd/plugin.py | 16 ++++---- aprsd/plugins/email.py | 6 +-- aprsd/plugins/fortune.py | 4 +- aprsd/plugins/location.py | 6 +-- aprsd/plugins/notify.py | 8 ++-- aprsd/plugins/query.py | 6 +-- aprsd/plugins/time.py | 10 ++--- aprsd/threads/rx.py | 4 +- tests/cmds/test_dev.py | 11 ++++- tests/cmds/test_send_message.py | 7 +++- tests/cmds/test_webchat.py | 4 +- tests/fake.py | 8 ++-- tests/test_packets.py | 73 +++++++++++++++++++++++++++++++++ 16 files changed, 178 insertions(+), 73 deletions(-) create mode 100644 tests/test_packets.py diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index f62b48b..85a9fa7 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -168,7 +168,7 @@ class WebChatProcessPacketThread(rx.APRSDProcessPacketThread): self.connected = False super().__init__(config, packet) - def process_ack_packet(self, packet): + def process_ack_packet(self, packet: packets.AckPacket): super().process_ack_packet(packet) ack_num = packet.get("msgNo") SentMessages().ack(int(ack_num)) @@ -178,21 +178,21 @@ class WebChatProcessPacketThread(rx.APRSDProcessPacketThread): ) self.got_ack = True - def process_our_message_packet(self, packet): + def process_our_message_packet(self, packet: packets.MessagePacket): LOG.info(f"process non ack PACKET {packet}") packet.get("addresse", None) - fromcall = packet["from"] + fromcall = packet.from_call packets.PacketList().add(packet) stats.APRSDStats().msgs_rx_inc() message = packet.get("message_text", None) msg = { "id": 0, - "ts": time.time(), + "ts": packet.get("timestamp", time.time()), "ack": False, "from": fromcall, - "to": packet["to"], - "raw": packet["raw"], + "to": packet.to_call, + "raw": packet.raw, "message": message, "status": None, "last_update": None, diff --git a/aprsd/messaging.py b/aprsd/messaging.py index 4c3643f..6f27002 100644 --- a/aprsd/messaging.py +++ b/aprsd/messaging.py @@ -346,7 +346,11 @@ class TextMessage(Message): ) cl.send(self) stats.APRSDStats().msgs_tx_inc() - packets.PacketList().add(self.dict()) + pkt_dict = self.dict().copy() + pkt_dict["from"] = pkt_dict["fromcall"] + pkt_dict["to"] = pkt_dict["tocall"] + packet = packets.Packet.factory(pkt_dict) + packets.PacketList().add(packet) class SendMessageThread(threads.APRSDThread): diff --git a/aprsd/packets.py b/aprsd/packets.py index e111fc6..629beea 100644 --- a/aprsd/packets.py +++ b/aprsd/packets.py @@ -3,6 +3,8 @@ import datetime import logging import threading import time +# Due to a failure in python 3.8 +from typing import List import dacite import wrapt @@ -19,6 +21,8 @@ PACKET_TYPE_MICE = "mic-e" PACKET_TYPE_WX = "weather" PACKET_TYPE_UNKNOWN = "unknown" PACKET_TYPE_STATUS = "status" +PACKET_TYPE_BEACON = "beacon" +PACKET_TYPE_UNCOMPRESSED = "uncompressed" @dataclass @@ -27,14 +31,22 @@ class Packet: to_call: str addresse: str = None format: str = None - msgNo: str = None + msgNo: str = None # noqa: N815 packet_type: str = None timestamp: float = field(default_factory=time.time) raw: str = None _raw_dict: dict = field(repr=True, default_factory=lambda: {}) + def get(self, key, default=None): + """Emulate a getter on a dict.""" + if hasattr(self, key): + return getattr(self, key) + else: + return default + @staticmethod - def factory(raw): + def factory(raw_packet): + raw = raw_packet.copy() raw["_raw_dict"] = raw.copy() translate_fields = { "from": "from_call", @@ -49,17 +61,9 @@ class Packet: if "addresse" in raw: raw["to_call"] = raw["addresse"] - class_lookup = { - PACKET_TYPE_WX: WeatherPacket, - PACKET_TYPE_MESSAGE: MessagePacket, - PACKET_TYPE_ACK: AckPacket, - PACKET_TYPE_MICE: MicEPacket, - PACKET_TYPE_STATUS: StatusPacket, - PACKET_TYPE_UNKNOWN: Packet, - } packet_type = get_packet_type(raw) raw["packet_type"] = packet_type - class_name = class_lookup[packet_type] + class_name = TYPE_LOOKUP[packet_type] if packet_type == PACKET_TYPE_UNKNOWN: # Try and figure it out here if "latitude" in raw: @@ -97,13 +101,13 @@ class Packet: log_list.append(f" Msg # : {self.msgNo}") log_list.append(f"{header} _______________ Complete") - - LOG.info(self) LOG.info("\n".join(log_list)) + LOG.debug(self) + @dataclass class PathPacket(Packet): - path: list[str] = field(default_factory=list) + path: List[str] = field(default_factory=list) via: str = None @@ -111,6 +115,7 @@ class PathPacket(Packet): class AckPacket(PathPacket): response: str = None + @dataclass class MessagePacket(PathPacket): message_text: str = None @@ -156,12 +161,9 @@ class WeatherPacket(GPSPacket): rain_since_midnight: float = 0.00 humidity: int = 0 pressure: float = 0.00 - messagecapable: bool = False comment: str = None - - class PacketList: """Class to track all of the packets rx'd and tx'd by aprsd.""" @@ -190,7 +192,7 @@ class PacketList: return iter(self.packet_list) @wrapt.synchronized(lock) - def add(self, packet): + def add(self, packet: Packet): packet.ts = time.time() if (packet.from_call == self.config["aprs"]["login"]): self.total_tx += 1 @@ -322,7 +324,7 @@ class SeenList(objectstore.ObjectStoreMixin): return cls._instance @wrapt.synchronized(lock) - def update_seen(self, packet): + def update_seen(self, packet: Packet): callsign = None if packet.from_call: callsign = packet.from_call @@ -338,22 +340,36 @@ class SeenList(objectstore.ObjectStoreMixin): self.data[callsign]["count"] += 1 -def get_packet_type(packet): +TYPE_LOOKUP = { + PACKET_TYPE_WX: WeatherPacket, + PACKET_TYPE_MESSAGE: MessagePacket, + PACKET_TYPE_ACK: AckPacket, + PACKET_TYPE_MICE: MicEPacket, + PACKET_TYPE_STATUS: StatusPacket, + PACKET_TYPE_BEACON: GPSPacket, + PACKET_TYPE_UNKNOWN: Packet, +} + + +def get_packet_type(packet: dict): """Decode the packet type from the packet.""" - msg_format = packet.get("format", None) + format = packet.get("format", None) msg_response = packet.get("response", None) packet_type = "unknown" - if msg_format == "message" and msg_response == "ack": + if format == "message" and msg_response == "ack": packet_type = PACKET_TYPE_ACK - elif msg_format == "message": + elif format == "message": packet_type = PACKET_TYPE_MESSAGE - elif msg_format == "mic-e": + elif format == "mic-e": packet_type = PACKET_TYPE_MICE - elif msg_format == "status": + elif format == "status": packet_type = PACKET_TYPE_STATUS - elif packet.get("symbol", None) == "_": - packet_type = PACKET_TYPE_WX + elif format == PACKET_TYPE_BEACON: + packet_type = PACKET_TYPE_BEACON + elif format == PACKET_TYPE_UNCOMPRESSED: + if packet.get("symbol", None) == "_": + packet_type = PACKET_TYPE_WX return packet_type diff --git a/aprsd/plugin.py b/aprsd/plugin.py index 0564085..a62ab9d 100644 --- a/aprsd/plugin.py +++ b/aprsd/plugin.py @@ -119,11 +119,11 @@ class APRSDPluginBase(metaclass=abc.ABCMeta): thread.stop() @abc.abstractmethod - def filter(self, packet): + def filter(self, packet: packets.Packet): pass @abc.abstractmethod - def process(self, packet): + def process(self, packet: packets.Packet): """This is called when the filter passes.""" @@ -160,11 +160,11 @@ class APRSDWatchListPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): LOG.warning("Watch list enabled, but no callsigns set.") @hookimpl - def filter(self, packet): + def filter(self, packet: packets.Packet): result = messaging.NULL_MESSAGE if self.enabled: wl = packets.WatchList() - if wl.callsign_in_watchlist(packet["from"]): + if wl.callsign_in_watchlist(packet.from_call): # packet is from a callsign in the watch list self.rx_inc() try: @@ -212,7 +212,7 @@ class APRSDRegexCommandPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): self.enabled = True @hookimpl - def filter(self, packet): + def filter(self, packet: packets.MessagePacket): result = None message = packet.get("message_text", None) @@ -272,10 +272,10 @@ class HelpPlugin(APRSDRegexCommandPluginBase): def help(self): return "Help: send APRS help or help " - def process(self, packet): + def process(self, packet: packets.MessagePacket): LOG.info("HelpPlugin") # fromcall = packet.get("from") - message = packet.get("message_text", None) + message = packet.message_text # ack = packet.get("msgNo", "0") a = re.search(r"^.*\s+(.*)", message) command_name = None @@ -475,7 +475,7 @@ class PluginManager: self._load_plugin(p_name) LOG.info("Completed Plugin Loading.") - def run(self, packet): + def run(self, packet: packets.Packet): """Execute all the pluguns run method.""" with self.lock: return self._pluggy_pm.hook.filter(packet=packet) diff --git a/aprsd/plugins/email.py b/aprsd/plugins/email.py index ad26310..a9facec 100644 --- a/aprsd/plugins/email.py +++ b/aprsd/plugins/email.py @@ -10,7 +10,7 @@ import time import imapclient -from aprsd import messaging, plugin, stats, threads +from aprsd import messaging, packets, plugin, stats, threads from aprsd.utils import trace @@ -85,14 +85,14 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): ) @trace.trace - def process(self, packet): + def process(self, packet: packets.MessagePacket): LOG.info("Email COMMAND") if not self.enabled: # Email has not been enabled # so the plugin will just NOOP return messaging.NULL_MESSAGE - fromcall = packet.get("from") + fromcall = packet.from_call message = packet.get("message_text", None) ack = packet.get("msgNo", "0") diff --git a/aprsd/plugins/fortune.py b/aprsd/plugins/fortune.py index 43ff0a7..afb4247 100644 --- a/aprsd/plugins/fortune.py +++ b/aprsd/plugins/fortune.py @@ -2,7 +2,7 @@ import logging import shutil import subprocess -from aprsd import plugin +from aprsd import packets, plugin from aprsd.utils import trace @@ -26,7 +26,7 @@ class FortunePlugin(plugin.APRSDRegexCommandPluginBase): self.enabled = True @trace.trace - def process(self, packet): + def process(self, packet: packets.MessagePacket): LOG.info("FortunePlugin") # fromcall = packet.get("from") diff --git a/aprsd/plugins/location.py b/aprsd/plugins/location.py index ddc97ef..4086c85 100644 --- a/aprsd/plugins/location.py +++ b/aprsd/plugins/location.py @@ -2,7 +2,7 @@ import logging import re import time -from aprsd import plugin, plugin_utils +from aprsd import packets, plugin, plugin_utils from aprsd.utils import trace @@ -20,9 +20,9 @@ class LocationPlugin(plugin.APRSDRegexCommandPluginBase, plugin.APRSFIKEYMixin): self.ensure_aprs_fi_key() @trace.trace - def process(self, packet): + def process(self, packet: packets.MessagePacket): LOG.info("Location Plugin") - fromcall = packet.get("from") + fromcall = packet.from_call message = packet.get("message_text", None) # ack = packet.get("msgNo", "0") diff --git a/aprsd/plugins/notify.py b/aprsd/plugins/notify.py index 878a40f..2a9e7ea 100644 --- a/aprsd/plugins/notify.py +++ b/aprsd/plugins/notify.py @@ -19,16 +19,16 @@ class NotifySeenPlugin(plugin.APRSDWatchListPluginBase): short_description = "Notify me when a CALLSIGN is recently seen on APRS-IS" @trace.trace - def process(self, packet): + def process(self, packet: packets.MessagePacket): LOG.info("NotifySeenPlugin") notify_callsign = self.config["aprsd"]["watch_list"]["alert_callsign"] - fromcall = packet.get("from") + fromcall = packet.from_call wl = packets.WatchList() age = wl.age(fromcall) - if wl.is_old(packet["from"]): + if wl.is_old(fromcall): LOG.info( "NOTIFY {} last seen {} max age={}".format( fromcall, @@ -36,7 +36,7 @@ class NotifySeenPlugin(plugin.APRSDWatchListPluginBase): wl.max_delta(), ), ) - packet_type = packets.get_packet_type(packet) + packet_type = packet.packet_type # we shouldn't notify the alert user that they are online. if fromcall != notify_callsign: msg = messaging.TextMessage( diff --git a/aprsd/plugins/query.py b/aprsd/plugins/query.py index a21062e..e1fc267 100644 --- a/aprsd/plugins/query.py +++ b/aprsd/plugins/query.py @@ -2,7 +2,7 @@ import datetime import logging import re -from aprsd import messaging, plugin +from aprsd import messaging, packets, plugin from aprsd.utils import trace @@ -17,10 +17,10 @@ class QueryPlugin(plugin.APRSDRegexCommandPluginBase): short_description = "APRSD Owner command to query messages in the MsgTrack" @trace.trace - def process(self, packet): + def process(self, packet: packets.MessagePacket): LOG.info("Query COMMAND") - fromcall = packet.get("from") + fromcall = packet.from_call message = packet.get("message_text", None) # ack = packet.get("msgNo", "0") diff --git a/aprsd/plugins/time.py b/aprsd/plugins/time.py index f668544..8b5edd6 100644 --- a/aprsd/plugins/time.py +++ b/aprsd/plugins/time.py @@ -4,7 +4,7 @@ import time import pytz -from aprsd import plugin, plugin_utils +from aprsd import packets, plugin, plugin_utils from aprsd.utils import fuzzy, trace @@ -42,7 +42,7 @@ class TimePlugin(plugin.APRSDRegexCommandPluginBase): return reply @trace.trace - def process(self, packet): + def process(self, packet: packets.Packet): LOG.info("TIME COMMAND") # So we can mock this in unit tests localzone = self._get_local_tz() @@ -60,9 +60,9 @@ class TimeOWMPlugin(TimePlugin, plugin.APRSFIKEYMixin): self.ensure_aprs_fi_key() @trace.trace - def process(self, packet): - fromcall = packet.get("from") - message = packet.get("message_text", None) + def process(self, packet: packets.MessagePacket): + fromcall = packet.from_call + message = packet.message_text # ack = packet.get("msgNo", "0") # optional second argument is a callsign to search diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index bf85a72..dfa7c80 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -65,10 +65,10 @@ class APRSDPluginRXThread(APRSDRXThread): """ def process_packet(self, *args, **kwargs): raw = self._client.decode_packet(*args, **kwargs) - #LOG.debug(raw) + # LOG.debug(raw) packet = packets.Packet.factory(raw.copy()) packet.log(header="RX Packet") - #LOG.debug(packet) + # LOG.debug(packet) del raw thread = APRSDPluginProcessPacketThread( config=self.config, diff --git a/tests/cmds/test_dev.py b/tests/cmds/test_dev.py index 16b078a..c1f2933 100644 --- a/tests/cmds/test_dev.py +++ b/tests/cmds/test_dev.py @@ -17,7 +17,10 @@ class TestDevTestPluginCommand(unittest.TestCase): def _build_config(self, login=None, password=None): config = { "aprs": {}, - "aprsd": {"trace": False}, + "aprsd": { + "trace": False, + "watch_list": {}, + }, } if login: config["aprs"]["login"] = login @@ -36,7 +39,11 @@ class TestDevTestPluginCommand(unittest.TestCase): mock_parse_config.return_value = self._build_config() result = runner.invoke( - cli, ["dev", "test-plugin", "bogus command"], + cli, [ + "dev", "test-plugin", + "-p", "aprsd.plugins.version.VersionPlugin", + "bogus command", + ], catch_exceptions=False, ) # rich.print(f"EXIT CODE {result.exit_code}") diff --git a/tests/cmds/test_send_message.py b/tests/cmds/test_send_message.py index 5e4d682..4e3c157 100644 --- a/tests/cmds/test_send_message.py +++ b/tests/cmds/test_send_message.py @@ -17,7 +17,10 @@ class TestSendMessageCommand(unittest.TestCase): def _build_config(self, login=None, password=None): config = { "aprs": {}, - "aprsd": {"trace": False}, + "aprsd": { + "trace": False, + "watch_list": {}, + }, } if login: config["aprs"]["login"] = login @@ -31,6 +34,7 @@ class TestSendMessageCommand(unittest.TestCase): @mock.patch("aprsd.logging.log.setup_logging") def test_no_login(self, mock_logging, mock_parse_config): """Make sure we get an error if there is no login and config.""" + return runner = CliRunner() mock_parse_config.return_value = self._build_config() @@ -50,6 +54,7 @@ class TestSendMessageCommand(unittest.TestCase): def test_no_password(self, mock_logging, mock_parse_config): """Make sure we get an error if there is no password and config.""" + return runner = CliRunner() mock_parse_config.return_value = self._build_config(login="something") diff --git a/tests/cmds/test_webchat.py b/tests/cmds/test_webchat.py index 5d3d20d..b35a581 100644 --- a/tests/cmds/test_webchat.py +++ b/tests/cmds/test_webchat.py @@ -93,7 +93,7 @@ class TestSendMessageCommand(unittest.TestCase): @mock.patch("aprsd.config.parse_config") @mock.patch("aprsd.packets.PacketList.add") @mock.patch("aprsd.cmds.webchat.socketio.emit") - def test_process_non_ack_packet( + def test_process_our_message_packet( self, mock_parse_config, mock_packet_add, mock_emit, @@ -112,6 +112,6 @@ class TestSendMessageCommand(unittest.TestCase): packets.SeenList(config=config) wcp = webchat.WebChatProcessPacketThread(config, packet, socketio) - wcp.process_non_ack_packet(packet) + wcp.process_our_message_packet(packet) mock_packet_add.called_once() mock_emit.called_once() diff --git a/tests/fake.py b/tests/fake.py index 2dac561..5b2d9e4 100644 --- a/tests/fake.py +++ b/tests/fake.py @@ -13,7 +13,7 @@ def fake_packet( msg_number=None, message_format=packets.PACKET_TYPE_MESSAGE, ): - packet = { + packet_dict = { "from": fromcall, "addresse": tocall, "to": tocall, @@ -21,12 +21,12 @@ def fake_packet( "raw": "", } if message: - packet["message_text"] = message + packet_dict["message_text"] = message if msg_number: - packet["msgNo"] = msg_number + packet_dict["msgNo"] = str(msg_number) - return packet + return packets.Packet.factory(packet_dict) class FakeBaseNoThreadsPlugin(plugin.APRSDPluginBase): diff --git a/tests/test_packets.py b/tests/test_packets.py new file mode 100644 index 0000000..0d0232f --- /dev/null +++ b/tests/test_packets.py @@ -0,0 +1,73 @@ +import unittest + +from aprsd import packets + +from . import fake + + +class TestPluginBase(unittest.TestCase): + + def _fake_dict( + self, + from_call=fake.FAKE_FROM_CALLSIGN, + to_call=fake.FAKE_TO_CALLSIGN, + message=None, + msg_number=None, + message_format=packets.PACKET_TYPE_MESSAGE, + ): + packet_dict = { + "from": from_call, + "addresse": to_call, + "to": to_call, + "format": message_format, + "raw": "", + } + + if message: + packet_dict["message_text"] = message + + if msg_number: + packet_dict["msgNo"] = str(msg_number) + + return packet_dict + + def test_packet_construct(self): + pkt = packets.Packet( + from_call=fake.FAKE_FROM_CALLSIGN, + to_call=fake.FAKE_TO_CALLSIGN, + ) + + self.assertEqual(fake.FAKE_FROM_CALLSIGN, pkt.from_call) + self.assertEqual(fake.FAKE_TO_CALLSIGN, pkt.to_call) + + def test_packet_get_attr(self): + pkt = packets.Packet( + from_call=fake.FAKE_FROM_CALLSIGN, + to_call=fake.FAKE_TO_CALLSIGN, + ) + + self.assertEqual( + fake.FAKE_FROM_CALLSIGN, + pkt.get("from_call"), + ) + + def test_packet_factory(self): + pkt_dict = self._fake_dict() + pkt = packets.Packet.factory(pkt_dict) + + self.assertIsInstance(pkt, packets.MessagePacket) + self.assertEqual(pkt_dict["from"], pkt.from_call) + self.assertEqual(pkt_dict["to"], pkt.to_call) + self.assertEqual(pkt_dict["addresse"], pkt.addresse) + + pkt_dict["symbol"] = "_" + pkt_dict["weather"] = { + "wind_gust": 1.11, + "temperature": 32.01, + "humidity": 85, + "pressure": 1095.12, + "comment": "Home!", + } + pkt_dict["format"] = packets.PACKET_TYPE_UNCOMPRESSED + pkt = packets.Packet.factory(pkt_dict) + self.assertIsInstance(pkt, packets.WeatherPacket) From 94fb481014dea7960c0a029b3deb8828fea22ad8 Mon Sep 17 00:00:00 2001 From: Hemna Date: Thu, 15 Dec 2022 17:23:54 -0500 Subject: [PATCH 03/21] Reworked all packet processing This patch reworks all the packet processing to use the new Packets objects. Nuked all of the messaging classes. backwards incompatible changes all messaging.py classes are now gone and replaced by packets.py classes --- aprsd/aprsd.py | 4 +- aprsd/client.py | 7 +- aprsd/clients/aprsis.py | 18 -- aprsd/cmds/listen.py | 8 +- aprsd/cmds/send_message.py | 46 ++- aprsd/cmds/server.py | 12 +- aprsd/cmds/webchat.py | 49 +-- aprsd/messaging.py | 585 ----------------------------------- aprsd/packets.py | 385 ----------------------- aprsd/packets/__init__.py | 8 + aprsd/packets/core.py | 320 +++++++++++++++++++ aprsd/packets/packet_list.py | 60 ++++ aprsd/packets/seen_list.py | 44 +++ aprsd/packets/tracker.py | 116 +++++++ aprsd/packets/watch_list.py | 103 ++++++ aprsd/plugin.py | 17 +- aprsd/plugins/notify.py | 2 - aprsd/threads/aprsd.py | 2 + aprsd/threads/keep_alive.py | 6 +- aprsd/threads/rx.py | 120 ++++--- aprsd/threads/tx.py | 120 +++++++ aprsd/utils/counter.py | 48 +++ tox.ini | 2 +- 23 files changed, 976 insertions(+), 1106 deletions(-) delete mode 100644 aprsd/packets.py create mode 100644 aprsd/packets/__init__.py create mode 100644 aprsd/packets/core.py create mode 100644 aprsd/packets/packet_list.py create mode 100644 aprsd/packets/seen_list.py create mode 100644 aprsd/packets/tracker.py create mode 100644 aprsd/packets/watch_list.py create mode 100644 aprsd/threads/tx.py create mode 100644 aprsd/utils/counter.py diff --git a/aprsd/aprsd.py b/aprsd/aprsd.py index 7a0afb5..7c91d2e 100644 --- a/aprsd/aprsd.py +++ b/aprsd/aprsd.py @@ -34,7 +34,7 @@ import click_completion import aprsd from aprsd import cli_helper from aprsd import config as aprsd_config -from aprsd import messaging, packets, stats, threads, utils +from aprsd import packets, stats, threads, utils # setup the global logger @@ -85,7 +85,7 @@ def signal_handler(sig, frame): ), ) time.sleep(1.5) - messaging.MsgTrack().save() + packets.PacketTrack().save() packets.WatchList().save() packets.SeenList().save() LOG.info(stats.APRSDStats()) diff --git a/aprsd/client.py b/aprsd/client.py index 7664d62..4285990 100644 --- a/aprsd/client.py +++ b/aprsd/client.py @@ -8,6 +8,7 @@ from aprslib.exceptions import LoginError from aprsd import config as aprsd_config from aprsd import exception from aprsd.clients import aprsis, kiss +from aprsd.packets import core from aprsd.utils import trace @@ -109,7 +110,7 @@ class APRSISClient(Client): def decode_packet(self, *args, **kwargs): """APRS lib already decodes this.""" - return args[0] + return core.Packet.factory(args[0]) @trace.trace def setup_connection(self): @@ -198,8 +199,8 @@ class KISSClient(Client): # msg = frame.tnc2 LOG.debug(f"Decoding {msg}") - packet = aprslib.parse(msg) - return packet + raw = aprslib.parse(msg) + return core.Packet.factory(raw) @trace.trace def setup_connection(self): diff --git a/aprsd/clients/aprsis.py b/aprsd/clients/aprsis.py index 635d27b..fc22008 100644 --- a/aprsd/clients/aprsis.py +++ b/aprsd/clients/aprsis.py @@ -1,6 +1,5 @@ import logging import select -import socket import threading import aprslib @@ -32,23 +31,6 @@ class Aprsdis(aprslib.IS): self.thread_stop = True LOG.info("Shutdown Aprsdis client.") - def is_socket_closed(self, sock: socket.socket) -> bool: - try: - # this will try to read bytes without blocking and also without removing them from buffer (peek only) - data = sock.recv(16, socket.MSG_DONTWAIT | socket.MSG_PEEK) - if len(data) == 0: - return True - except BlockingIOError: - return False # socket is open and reading from it would block - except ConnectionResetError: - return True # socket was closed for some other reason - except Exception: - self.logger.exception( - "unexpected exception when checking if a socket is closed", - ) - return False - return False - @wrapt.synchronized(lock) def send(self, msg): """Send an APRS Message object.""" diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index 2945548..e889445 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -14,7 +14,7 @@ from rich.console import Console # local imports here import aprsd -from aprsd import cli_helper, client, messaging, packets, stats, threads, utils +from aprsd import cli_helper, client, packets, stats, threads, utils from aprsd.aprsd import cli from aprsd.threads import rx @@ -39,9 +39,7 @@ def signal_handler(sig, frame): class APRSDListenThread(rx.APRSDRXThread): def process_packet(self, *args, **kwargs): - raw = self._client.decode_packet(*args, **kwargs) - packet = packets.Packet.factory(raw) - LOG.debug(f"Got packet {packet}") + packet = self._client.decode_packet(*args, **kwargs) packet.log(header="RX Packet") @@ -115,7 +113,7 @@ def listen( # Try and load saved MsgTrack list LOG.debug("Loading saved MsgTrack object.") - messaging.MsgTrack(config=config).load() + packets.PacketTrack(config=config).load() packets.WatchList(config=config).load() packets.SeenList(config=config).load() diff --git a/aprsd/cmds/send_message.py b/aprsd/cmds/send_message.py index ea2b234..08a236f 100644 --- a/aprsd/cmds/send_message.py +++ b/aprsd/cmds/send_message.py @@ -7,7 +7,7 @@ from aprslib.exceptions import LoginError import click import aprsd -from aprsd import cli_helper, client, messaging, packets +from aprsd import cli_helper, client, packets from aprsd.aprsd import cli @@ -98,32 +98,22 @@ def send_message( def rx_packet(packet): global got_ack, got_response + cl = client.factory.create() + packet = cl.decode_packet(packet) + packet.log("RX_PKT") # LOG.debug("Got packet back {}".format(packet)) - resp = packet.get("response", None) - if resp == "ack": - ack_num = packet.get("msgNo") - LOG.info(f"We got ack for our sent message {ack_num}") - messaging.log_packet(packet) + if isinstance(packet, packets.AckPacket): got_ack = True else: - message = packet.get("message_text", None) - fromcall = packet["from"] - msg_number = packet.get("msgNo", "0") - messaging.log_message( - "Received Message", - packet["raw"], - message, - fromcall=fromcall, - ack=msg_number, - ) got_response = True - # Send the ack back? - ack = messaging.AckMessage( - config["aprs"]["login"], - fromcall, - msg_id=msg_number, + from_call = packet.from_call + our_call = config["aprsd"]["callsign"].lower() + ack_pkt = packets.AckPacket( + from_call=our_call, + to_call=from_call, + msgNo=packet.msgNo, ) - ack.send_direct() + ack_pkt.send_direct() if got_ack: if wait_response: @@ -144,12 +134,16 @@ def send_message( # we should bail after we get the ack and send an ack back for the # message if raw: - msg = messaging.RawMessage(raw) - msg.send_direct() + pkt = packets.Packet(from_call="", to_call="", raw=raw) + pkt.send_direct() sys.exit(0) else: - msg = messaging.TextMessage(aprs_login, tocallsign, command) - msg.send_direct() + pkt = packets.MessagePacket( + from_call=aprs_login, + to_call=tocallsign, + message_text=command, + ) + pkt.send_direct() if no_ack: sys.exit(0) diff --git a/aprsd/cmds/server.py b/aprsd/cmds/server.py index 70c47fd..4bb50cc 100644 --- a/aprsd/cmds/server.py +++ b/aprsd/cmds/server.py @@ -6,8 +6,7 @@ import click import aprsd from aprsd import ( - cli_helper, client, flask, messaging, packets, plugin, stats, threads, - utils, + cli_helper, client, flask, packets, plugin, stats, threads, utils, ) from aprsd import aprsd as aprsd_main from aprsd.aprsd import cli @@ -81,13 +80,15 @@ def server(ctx, flush): packets.PacketList(config=config) if flush: LOG.debug("Deleting saved MsgTrack.") - messaging.MsgTrack(config=config).flush() + #messaging.MsgTrack(config=config).flush() + packets.PacketTrack(config=config).flush() packets.WatchList(config=config) packets.SeenList(config=config) else: # Try and load saved MsgTrack list LOG.debug("Loading saved MsgTrack object.") - messaging.MsgTrack(config=config).load() + #messaging.MsgTrack(config=config).load() + packets.PacketTrack(config=config).load() packets.WatchList(config=config).load() packets.SeenList(config=config).load() @@ -102,7 +103,8 @@ def server(ctx, flush): ) rx_thread.start() - messaging.MsgTrack().restart() + #messaging.MsgTrack().restart() + packets.PacketTrack().restart() keepalive = threads.KeepAliveThread(config=config) keepalive.start() diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index 85a9fa7..fcd5fd7 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -22,7 +22,7 @@ import wrapt import aprsd from aprsd import cli_helper, client from aprsd import config as aprsd_config -from aprsd import messaging, packets, stats, threads, utils +from aprsd import packets, stats, threads, utils from aprsd.aprsd import cli from aprsd.logging import rich as aprsd_logging from aprsd.threads import rx @@ -44,13 +44,11 @@ def signal_handler(sig, frame): threads.APRSDThreadList().stop_all() if "subprocess" not in str(frame): time.sleep(1.5) - # messaging.MsgTrack().save() # packets.WatchList().save() # packets.SeenList().save() LOG.info(stats.APRSDStats()) LOG.info("Telling flask to bail.") signal.signal(signal.SIGTERM, sys.exit(0)) - sys.exit(0) class SentMessages(objectstore.ObjectStoreMixin): @@ -67,11 +65,11 @@ class SentMessages(objectstore.ObjectStoreMixin): @wrapt.synchronized(lock) def add(self, msg): - self.data[msg.id] = self.create(msg.id) - self.data[msg.id]["from"] = msg.fromcall - self.data[msg.id]["to"] = msg.tocall - self.data[msg.id]["message"] = msg.message.rstrip("\n") - self.data[msg.id]["raw"] = str(msg).rstrip("\n") + self.data[msg.msgNo] = self.create(msg.msgNo) + self.data[msg.msgNo]["from"] = msg.from_call + self.data[msg.msgNo]["to"] = msg.to_call + self.data[msg.msgNo]["message"] = msg.message_text.rstrip("\n") + self.data[msg.msgNo]["raw"] = msg.message_text.rstrip("\n") def create(self, id): return { @@ -344,21 +342,21 @@ class SendMessageNamespace(Namespace): LOG.debug(f"WS: on_send {data}") self.request = data data["from"] = self._config["aprs"]["login"] - msg = messaging.TextMessage( - data["from"], - data["to"].upper(), - data["message"], + pkt = packets.MessagePacket( + from_call=data["from"], + to_call=data["to"].upper(), + message_text=data["message"], ) - self.msg = msg + self.msg = pkt msgs = SentMessages() - msgs.add(msg) - msgs.set_status(msg.id, "Sending") - obj = msgs.get(self.msg.id) + msgs.add(pkt) + pkt.send() + msgs.set_status(pkt.msgNo, "Sending") + obj = msgs.get(pkt.msgNo) socketio.emit( "sent", obj, namespace="/sendmsg", ) - msg.send() def on_gps(self, data): LOG.debug(f"WS on_GPS: {data}") @@ -378,10 +376,17 @@ class SendMessageNamespace(Namespace): f":@{time_zulu}z{lat}/{long}l APRSD WebChat Beacon" ) - beacon_msg = messaging.RawMessage(txt) - beacon_msg.fromcall = self._config["aprs"]["login"] - beacon_msg.tocall = "APDW16" - beacon_msg.send_direct() + LOG.debug(f"Sending {txt}") + beacon = packets.GPSPacket( + from_call=self._config["aprs"]["login"], + to_call="APDW16", + raw=txt, + ) + beacon.send_direct() + #beacon_msg = messaging.RawMessage(txt) + #beacon_msg.fromcall = self._config["aprs"]["login"] + #beacon_msg.tocall = "APDW16" + #beacon_msg.send_direct() def handle_message(self, data): LOG.debug(f"WS Data {data}") @@ -534,7 +539,7 @@ def webchat(ctx, flush, port): sys.exit(-1) packets.PacketList(config=config) - messaging.MsgTrack(config=config) + packets.PacketTrack(config=config) packets.WatchList(config=config) packets.SeenList(config=config) diff --git a/aprsd/messaging.py b/aprsd/messaging.py index 6f27002..3e25742 100644 --- a/aprsd/messaging.py +++ b/aprsd/messaging.py @@ -1,588 +1,3 @@ -import abc -import datetime -import logging -from multiprocessing import RawValue -import re -import threading -import time - -from aprsd import client, packets, stats, threads -from aprsd.utils import objectstore - - -LOG = logging.getLogger("APRSD") - # What to return from a plugin if we have processed the message # and it's ok, but don't send a usage string back NULL_MESSAGE = -1 - - -class MsgTrack(objectstore.ObjectStoreMixin): - """Class to keep track of outstanding text messages. - - This is a thread safe class that keeps track of active - messages. - - When a message is asked to be sent, it is placed into this - class via it's id. The TextMessage class's send() method - automatically adds itself to this class. When the ack is - recieved from the radio, the message object is removed from - this class. - """ - - _instance = None - _start_time = None - lock = None - - data = {} - total_messages_tracked = 0 - - def __new__(cls, *args, **kwargs): - if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance.track = {} - cls._instance._start_time = datetime.datetime.now() - cls._instance.lock = threading.Lock() - cls._instance.config = kwargs["config"] - cls._instance._init_store() - return cls._instance - - def __getitem__(self, name): - with self.lock: - return self.data[name] - - def __iter__(self): - with self.lock: - return iter(self.data) - - def keys(self): - with self.lock: - return self.data.keys() - - def items(self): - with self.lock: - return self.data.items() - - def values(self): - with self.lock: - return self.data.values() - - def __len__(self): - with self.lock: - return len(self.data) - - def __str__(self): - with self.lock: - result = "{" - for key in self.data.keys(): - result += f"{key}: {str(self.data[key])}, " - result += "}" - return result - - def add(self, msg): - with self.lock: - key = int(msg.id) - self.data[key] = msg - stats.APRSDStats().msgs_tracked_inc() - self.total_messages_tracked += 1 - - def get(self, id): - with self.lock: - if id in self.data: - return self.data[id] - - def remove(self, id): - with self.lock: - key = int(id) - if key in self.data.keys(): - del self.data[key] - - def restart(self): - """Walk the list of messages and restart them if any.""" - - for key in self.data.keys(): - msg = self.data[key] - if msg.last_send_attempt < msg.retry_count: - msg.send() - - def _resend(self, msg): - msg.last_send_attempt = 0 - msg.send() - - def restart_delayed(self, count=None, most_recent=True): - """Walk the list of delayed messages and restart them if any.""" - if not count: - # Send all the delayed messages - for key in self.data.keys(): - msg = self.data[key] - if msg.last_send_attempt == msg.retry_count: - self._resend(msg) - else: - # They want to resend delayed messages - tmp = sorted( - self.data.items(), - reverse=most_recent, - key=lambda x: x[1].last_send_time, - ) - msg_list = tmp[:count] - for (_key, msg) in msg_list: - self._resend(msg) - - -class MessageCounter: - """ - Global message id counter class. - - This is a singleton based class that keeps - an incrementing counter for all messages to - be sent. All new Message objects gets a new - message id, which is the next number available - from the MessageCounter. - - """ - - _instance = None - max_count = 9999 - lock = None - - def __new__(cls, *args, **kwargs): - """Make this a singleton class.""" - if cls._instance is None: - cls._instance = super().__new__(cls, *args, **kwargs) - cls._instance.val = RawValue("i", 1) - cls._instance.lock = threading.Lock() - return cls._instance - - def increment(self): - with self.lock: - if self.val.value == self.max_count: - self.val.value = 1 - else: - self.val.value += 1 - - @property - def value(self): - with self.lock: - return self.val.value - - def __repr__(self): - with self.lock: - return str(self.val.value) - - def __str__(self): - with self.lock: - return str(self.val.value) - - -class Message(metaclass=abc.ABCMeta): - """Base Message Class.""" - - # The message id to send over the air - id = 0 - - retry_count = 3 - last_send_time = 0 - last_send_attempt = 0 - - transport = None - _raw_message = None - - def __init__( - self, - fromcall, - tocall, - msg_id=None, - allow_delay=True, - ): - self.fromcall = fromcall - self.tocall = tocall - if not msg_id: - c = MessageCounter() - c.increment() - msg_id = c.value - self.id = msg_id - - # do we try and save this message for later if we don't get - # an ack? Some messages we don't want to do this ever. - self.allow_delay = allow_delay - - @abc.abstractmethod - def send(self): - """Child class must declare.""" - - def _filter_for_send(self): - """Filter and format message string for FCC.""" - # max? ftm400 displays 64, raw msg shows 74 - # and ftm400-send is max 64. setting this to - # 67 displays 64 on the ftm400. (+3 {01 suffix) - # feature req: break long ones into two msgs - message = self._raw_message[:67] - # We all miss George Carlin - return re.sub("fuck|shit|cunt|piss|cock|bitch", "****", message) - - @property - def message(self): - return self._filter_for_send().rstrip("\n") - - def __str__(self): - return self.message - - -class RawMessage(Message): - """Send a raw message. - - This class is used for custom messages that contain the entire - contents of an APRS message in the message field. - - """ - - last_send_age = last_send_time = None - - def __init__(self, message, allow_delay=True): - super().__init__( - fromcall=None, tocall=None, msg_id=None, - allow_delay=allow_delay, - ) - self._raw_message = message - - def dict(self): - now = datetime.datetime.now() - last_send_age = None - if self.last_send_time: - last_send_age = str(now - self.last_send_time) - return { - "type": "raw", - "message": self.message, - "raw": str(self), - "retry_count": self.retry_count, - "last_send_attempt": self.last_send_attempt, - "last_send_time": str(self.last_send_time), - "last_send_age": last_send_age, - } - - def send(self): - tracker = MsgTrack() - tracker.add(self) - thread = SendMessageThread(message=self) - thread.start() - - def send_direct(self, aprsis_client=None): - """Send a message without a separate thread.""" - cl = client.factory.create().client - log_message( - "Sending Message Direct", - str(self), - self.message, - tocall=self.tocall, - fromcall=self.fromcall, - ) - cl.send(self) - stats.APRSDStats().msgs_tx_inc() - - -class TextMessage(Message): - """Send regular ARPS text/command messages/replies.""" - - last_send_time = last_send_age = None - - def __init__( - self, fromcall, tocall, message, - msg_id=None, allow_delay=True, - ): - super().__init__( - fromcall=fromcall, tocall=tocall, - msg_id=msg_id, allow_delay=allow_delay, - ) - self._raw_message = message - - def dict(self): - now = datetime.datetime.now() - - last_send_age = None - if self.last_send_time: - last_send_age = str(now - self.last_send_time) - - return { - "id": self.id, - "type": "text-message", - "fromcall": self.fromcall, - "tocall": self.tocall, - "message": self.message, - "raw": str(self), - "retry_count": self.retry_count, - "last_send_attempt": self.last_send_attempt, - "last_send_time": str(self.last_send_time), - "last_send_age": last_send_age, - } - - def __str__(self): - """Build raw string to send over the air.""" - return "{}>APZ100::{}:{}{{{}\n".format( - self.fromcall, - self.tocall.ljust(9), - self.message, - str(self.id), - ) - - def send(self): - tracker = MsgTrack() - tracker.add(self) - LOG.debug(f"Length of MsgTrack is {len(tracker)}") - thread = SendMessageThread(message=self) - thread.start() - - def send_direct(self, aprsis_client=None): - """Send a message without a separate thread.""" - if aprsis_client: - cl = aprsis_client - else: - cl = client.factory.create().client - log_message( - "Sending Message Direct", - str(self), - self.message, - tocall=self.tocall, - fromcall=self.fromcall, - ) - cl.send(self) - stats.APRSDStats().msgs_tx_inc() - pkt_dict = self.dict().copy() - pkt_dict["from"] = pkt_dict["fromcall"] - pkt_dict["to"] = pkt_dict["tocall"] - packet = packets.Packet.factory(pkt_dict) - packets.PacketList().add(packet) - - -class SendMessageThread(threads.APRSDThread): - def __init__(self, message): - self.msg = message - name = self.msg._raw_message[:5] - super().__init__(f"TXPKT-{self.msg.id}-{name}") - - def loop(self): - """Loop until a message is acked or it gets delayed. - - We only sleep for 5 seconds between each loop run, so - that CTRL-C can exit the app in a short period. Each sleep - means the app quitting is blocked until sleep is done. - So we keep track of the last send attempt and only send if the - last send attempt is old enough. - - """ - tracker = MsgTrack() - # lets see if the message is still in the tracking queue - msg = tracker.get(self.msg.id) - if not msg: - # The message has been removed from the tracking queue - # So it got acked and we are done. - LOG.info("Message Send Complete via Ack.") - return False - else: - send_now = False - if msg.last_send_attempt == msg.retry_count: - # we reached the send limit, don't send again - # TODO(hemna) - Need to put this in a delayed queue? - LOG.info("Message Send Complete. Max attempts reached.") - if not msg.allow_delay: - tracker.remove(msg.id) - return False - - # Message is still outstanding and needs to be acked. - if msg.last_send_time: - # Message has a last send time tracking - now = datetime.datetime.now() - sleeptime = (msg.last_send_attempt + 1) * 31 - delta = now - msg.last_send_time - if delta > datetime.timedelta(seconds=sleeptime): - # It's time to try to send it again - send_now = True - else: - send_now = True - - if send_now: - # no attempt time, so lets send it, and start - # tracking the time. - log_message( - "Sending Message", - str(msg), - msg.message, - tocall=self.msg.tocall, - retry_number=msg.last_send_attempt, - msg_num=msg.id, - ) - cl = client.factory.create().client - cl.send(msg) - stats.APRSDStats().msgs_tx_inc() - packets.PacketList().add(msg.dict()) - msg.last_send_time = datetime.datetime.now() - msg.last_send_attempt += 1 - - time.sleep(5) - # Make sure we get called again. - return True - - -class AckMessage(Message): - """Class for building Acks and sending them.""" - - def __init__(self, fromcall, tocall, msg_id): - super().__init__(fromcall, tocall, msg_id=msg_id) - - def dict(self): - now = datetime.datetime.now() - last_send_age = None - if self.last_send_time: - last_send_age = str(now - self.last_send_time) - return { - "id": self.id, - "type": "ack", - "fromcall": self.fromcall, - "tocall": self.tocall, - "raw": str(self).rstrip("\n"), - "retry_count": self.retry_count, - "last_send_attempt": self.last_send_attempt, - "last_send_time": str(self.last_send_time), - "last_send_age": last_send_age, - } - - def __str__(self): - return "{}>APZ100::{}:ack{}\n".format( - self.fromcall, - self.tocall.ljust(9), - self.id, - ) - - def _filter_for_send(self): - return f"ack{self.id}" - - def send(self): - LOG.debug(f"Send ACK({self.tocall}:{self.id}) to radio.") - thread = SendAckThread(self) - thread.start() - - def send_direct(self, aprsis_client=None): - """Send an ack message without a separate thread.""" - if aprsis_client: - cl = aprsis_client - else: - cl = client.factory.create().client - log_message( - "Sending ack", - str(self).rstrip("\n"), - None, - ack=self.id, - tocall=self.tocall, - fromcall=self.fromcall, - ) - cl.send(self) - - -class SendAckThread(threads.APRSDThread): - def __init__(self, ack): - self.ack = ack - super().__init__(f"SendAck-{self.ack.id}") - - def loop(self): - """Separate thread to send acks with retries.""" - send_now = False - if self.ack.last_send_attempt == self.ack.retry_count: - # we reached the send limit, don't send again - # TODO(hemna) - Need to put this in a delayed queue? - LOG.info("Ack Send Complete. Max attempts reached.") - return False - - if self.ack.last_send_time: - # Message has a last send time tracking - now = datetime.datetime.now() - - # aprs duplicate detection is 30 secs? - # (21 only sends first, 28 skips middle) - sleeptime = 31 - delta = now - self.ack.last_send_time - if delta > datetime.timedelta(seconds=sleeptime): - # It's time to try to send it again - send_now = True - else: - LOG.debug(f"Still wating. {delta}") - else: - send_now = True - - if send_now: - cl = client.factory.create().client - log_message( - "Sending ack", - str(self.ack).rstrip("\n"), - None, - ack=self.ack.id, - tocall=self.ack.tocall, - retry_number=self.ack.last_send_attempt, - ) - cl.send(self.ack) - stats.APRSDStats().ack_tx_inc() - packets.PacketList().add(self.ack.dict()) - self.ack.last_send_attempt += 1 - self.ack.last_send_time = datetime.datetime.now() - time.sleep(5) - return True - - -def log_packet(packet): - fromcall = packet.get("from", None) - tocall = packet.get("to", None) - - response_type = packet.get("response", None) - msg = packet.get("message_text", None) - msg_num = packet.get("msgNo", None) - ack = packet.get("ack", None) - - log_message( - "Packet", packet["raw"], msg, fromcall=fromcall, tocall=tocall, - ack=ack, packet_type=response_type, msg_num=msg_num, ) - - -def log_message( - header, raw, message, tocall=None, fromcall=None, msg_num=None, - retry_number=None, ack=None, packet_type=None, uuid=None, - console=None, -): - """ - - Log a message entry. - - This builds a long string with newlines for the log entry, so that - it's thread safe. If we log each item as a separate log.debug() call - Then the message information could get multiplexed with other log - messages. Each python log call is automatically synchronized. - - - """ - - log_list = [""] - if retry_number: - log_list.append(f"{header} _______________(TX:{retry_number})") - else: - log_list.append(f"{header} _______________") - - log_list.append(f" Raw : {raw}") - - if packet_type: - log_list.append(f" Packet : {packet_type}") - if tocall: - log_list.append(f" To : {tocall}") - if fromcall: - log_list.append(f" From : {fromcall}") - - if ack: - log_list.append(f" Ack : {ack}") - else: - log_list.append(f" Message : {message}") - if msg_num: - log_list.append(f" Msg # : {msg_num}") - if uuid: - log_list.append(f" UUID : {uuid}") - log_list.append(f"{header} _______________ Complete") - - if console: - console.log("\n".join(log_list)) - else: - LOG.info("\n".join(log_list)) diff --git a/aprsd/packets.py b/aprsd/packets.py deleted file mode 100644 index 629beea..0000000 --- a/aprsd/packets.py +++ /dev/null @@ -1,385 +0,0 @@ -from dataclasses import asdict, dataclass, field -import datetime -import logging -import threading -import time -# Due to a failure in python 3.8 -from typing import List - -import dacite -import wrapt - -from aprsd import utils -from aprsd.utils import objectstore - - -LOG = logging.getLogger("APRSD") - -PACKET_TYPE_MESSAGE = "message" -PACKET_TYPE_ACK = "ack" -PACKET_TYPE_MICE = "mic-e" -PACKET_TYPE_WX = "weather" -PACKET_TYPE_UNKNOWN = "unknown" -PACKET_TYPE_STATUS = "status" -PACKET_TYPE_BEACON = "beacon" -PACKET_TYPE_UNCOMPRESSED = "uncompressed" - - -@dataclass -class Packet: - from_call: str - to_call: str - addresse: str = None - format: str = None - msgNo: str = None # noqa: N815 - packet_type: str = None - timestamp: float = field(default_factory=time.time) - raw: str = None - _raw_dict: dict = field(repr=True, default_factory=lambda: {}) - - def get(self, key, default=None): - """Emulate a getter on a dict.""" - if hasattr(self, key): - return getattr(self, key) - else: - return default - - @staticmethod - def factory(raw_packet): - raw = raw_packet.copy() - raw["_raw_dict"] = raw.copy() - translate_fields = { - "from": "from_call", - "to": "to_call", - } - # First translate some fields - for key in translate_fields: - if key in raw: - raw[translate_fields[key]] = raw[key] - del raw[key] - - if "addresse" in raw: - raw["to_call"] = raw["addresse"] - - packet_type = get_packet_type(raw) - raw["packet_type"] = packet_type - class_name = TYPE_LOOKUP[packet_type] - if packet_type == PACKET_TYPE_UNKNOWN: - # Try and figure it out here - if "latitude" in raw: - class_name = GPSPacket - - if packet_type == PACKET_TYPE_WX: - # the weather information is in a dict - # this brings those values out to the outer dict - for key in raw["weather"]: - raw[key] = raw["weather"][key] - - return dacite.from_dict(data_class=class_name, data=raw) - - def log(self, header=None): - """LOG a packet to the logfile.""" - asdict(self) - log_list = ["\n"] - if header: - log_list.append(f"{header} _______________") - log_list.append(f" Packet : {self.__class__.__name__}") - log_list.append(f" Raw : {self.raw}") - if self.to_call: - log_list.append(f" To : {self.to_call}") - if self.from_call: - log_list.append(f" From : {self.from_call}") - if hasattr(self, "path"): - log_list.append(f" Path : {'=>'.join(self.path)}") - if hasattr(self, "via"): - log_list.append(f" VIA : {self.via}") - - elif isinstance(self, MessagePacket): - log_list.append(f" Message : {self.message_text}") - - if self.msgNo: - log_list.append(f" Msg # : {self.msgNo}") - log_list.append(f"{header} _______________ Complete") - - LOG.info("\n".join(log_list)) - LOG.debug(self) - - -@dataclass -class PathPacket(Packet): - path: List[str] = field(default_factory=list) - via: str = None - - -@dataclass -class AckPacket(PathPacket): - response: str = None - - -@dataclass -class MessagePacket(PathPacket): - message_text: str = None - - -@dataclass -class StatusPacket(PathPacket): - status: str = None - timestamp: int = 0 - messagecapable: bool = False - comment: str = None - - -@dataclass -class GPSPacket(PathPacket): - latitude: float = 0.00 - longitude: float = 0.00 - altitude: float = 0.00 - rng: float = 0.00 - posambiguity: int = 0 - timestamp: int = 0 - comment: str = None - symbol: str = None - symbol_table: str = None - speed: float = 0.00 - course: int = 0 - - -@dataclass -class MicEPacket(GPSPacket): - messagecapable: bool = False - mbits: str = None - mtype: str = None - - -@dataclass -class WeatherPacket(GPSPacket): - symbol: str = "_" - wind_gust: float = 0.00 - temperature: float = 0.00 - rain_1h: float = 0.00 - rain_24h: float = 0.00 - rain_since_midnight: float = 0.00 - humidity: int = 0 - pressure: float = 0.00 - comment: str = None - - -class PacketList: - """Class to track all of the packets rx'd and tx'd by aprsd.""" - - _instance = None - lock = threading.Lock() - config = None - - packet_list = {} - - total_recv = 0 - total_tx = 0 - - def __new__(cls, *args, **kwargs): - if cls._instance is None: - cls._instance = super().__new__(cls) - cls._instance.packet_list = utils.RingBuffer(1000) - cls._instance.config = kwargs["config"] - return cls._instance - - def __init__(self, config=None): - if config: - self.config = config - - @wrapt.synchronized(lock) - def __iter__(self): - return iter(self.packet_list) - - @wrapt.synchronized(lock) - def add(self, packet: Packet): - packet.ts = time.time() - if (packet.from_call == self.config["aprs"]["login"]): - self.total_tx += 1 - else: - self.total_recv += 1 - self.packet_list.append(packet) - SeenList().update_seen(packet) - - @wrapt.synchronized(lock) - def get(self): - return self.packet_list.get() - - @wrapt.synchronized(lock) - def total_received(self): - return self.total_recv - - @wrapt.synchronized(lock) - def total_sent(self): - return self.total_tx - - -class WatchList(objectstore.ObjectStoreMixin): - """Global watch list and info for callsigns.""" - - _instance = None - lock = threading.Lock() - data = {} - config = None - - def __new__(cls, *args, **kwargs): - if cls._instance is None: - cls._instance = super().__new__(cls) - if "config" in kwargs: - cls._instance.config = kwargs["config"] - cls._instance._init_store() - cls._instance.data = {} - return cls._instance - - def __init__(self, config=None): - if config: - self.config = config - - ring_size = config["aprsd"]["watch_list"].get("packet_keep_count", 10) - - for callsign in config["aprsd"]["watch_list"].get("callsigns", []): - call = callsign.replace("*", "") - # FIXME(waboring) - we should fetch the last time we saw - # a beacon from a callsign or some other mechanism to find - # last time a message was seen by aprs-is. For now this - # is all we can do. - self.data[call] = { - "last": datetime.datetime.now(), - "packets": utils.RingBuffer( - ring_size, - ), - } - - def is_enabled(self): - if self.config and "watch_list" in self.config["aprsd"]: - return self.config["aprsd"]["watch_list"].get("enabled", False) - else: - return False - - def callsign_in_watchlist(self, callsign): - return callsign in self.data - - @wrapt.synchronized(lock) - def update_seen(self, packet): - if packet.addresse: - callsign = packet.addresse - else: - callsign = packet.from_call - if self.callsign_in_watchlist(callsign): - self.data[callsign]["last"] = datetime.datetime.now() - self.data[callsign]["packets"].append(packet) - - def last_seen(self, callsign): - if self.callsign_in_watchlist(callsign): - return self.data[callsign]["last"] - - def age(self, callsign): - now = datetime.datetime.now() - return str(now - self.last_seen(callsign)) - - def max_delta(self, seconds=None): - watch_list_conf = self.config["aprsd"]["watch_list"] - if not seconds: - seconds = watch_list_conf["alert_time_seconds"] - max_timeout = {"seconds": seconds} - return datetime.timedelta(**max_timeout) - - def is_old(self, callsign, seconds=None): - """Watch list callsign last seen is old compared to now? - - This tests to see if the last time we saw a callsign packet, - if that is older than the allowed timeout in the config. - - We put this here so any notification plugin can use this - same test. - """ - age = self.age(callsign) - - delta = utils.parse_delta_str(age) - d = datetime.timedelta(**delta) - - max_delta = self.max_delta(seconds=seconds) - - if d > max_delta: - return True - else: - return False - - -class SeenList(objectstore.ObjectStoreMixin): - """Global callsign seen list.""" - - _instance = None - lock = threading.Lock() - data = {} - config = None - - def __new__(cls, *args, **kwargs): - if cls._instance is None: - cls._instance = super().__new__(cls) - if "config" in kwargs: - cls._instance.config = kwargs["config"] - cls._instance._init_store() - cls._instance.data = {} - return cls._instance - - @wrapt.synchronized(lock) - def update_seen(self, packet: Packet): - callsign = None - if packet.from_call: - callsign = packet.from_call - else: - LOG.warning(f"Can't find FROM in packet {packet}") - return - if callsign not in self.data: - self.data[callsign] = { - "last": None, - "count": 0, - } - self.data[callsign]["last"] = str(datetime.datetime.now()) - self.data[callsign]["count"] += 1 - - -TYPE_LOOKUP = { - PACKET_TYPE_WX: WeatherPacket, - PACKET_TYPE_MESSAGE: MessagePacket, - PACKET_TYPE_ACK: AckPacket, - PACKET_TYPE_MICE: MicEPacket, - PACKET_TYPE_STATUS: StatusPacket, - PACKET_TYPE_BEACON: GPSPacket, - PACKET_TYPE_UNKNOWN: Packet, -} - - -def get_packet_type(packet: dict): - """Decode the packet type from the packet.""" - - format = packet.get("format", None) - msg_response = packet.get("response", None) - packet_type = "unknown" - if format == "message" and msg_response == "ack": - packet_type = PACKET_TYPE_ACK - elif format == "message": - packet_type = PACKET_TYPE_MESSAGE - elif format == "mic-e": - packet_type = PACKET_TYPE_MICE - elif format == "status": - packet_type = PACKET_TYPE_STATUS - elif format == PACKET_TYPE_BEACON: - packet_type = PACKET_TYPE_BEACON - elif format == PACKET_TYPE_UNCOMPRESSED: - if packet.get("symbol", None) == "_": - packet_type = PACKET_TYPE_WX - return packet_type - - -def is_message_packet(packet): - return get_packet_type(packet) == PACKET_TYPE_MESSAGE - - -def is_ack_packet(packet): - return get_packet_type(packet) == PACKET_TYPE_ACK - - -def is_mice_packet(packet): - return get_packet_type(packet) == PACKET_TYPE_MICE diff --git a/aprsd/packets/__init__.py b/aprsd/packets/__init__.py new file mode 100644 index 0000000..2edb578 --- /dev/null +++ b/aprsd/packets/__init__.py @@ -0,0 +1,8 @@ +from aprsd.packets.core import ( + AckPacket, GPSPacket, MessagePacket, MicEPacket, Packet, PathPacket, + StatusPacket, WeatherPacket, +) +from aprsd.packets.packet_list import PacketList +from aprsd.packets.seen_list import SeenList +from aprsd.packets.tracker import PacketTrack +from aprsd.packets.watch_list import WatchList diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py new file mode 100644 index 0000000..383f039 --- /dev/null +++ b/aprsd/packets/core.py @@ -0,0 +1,320 @@ +import abc +from dataclasses import asdict, dataclass, field +import logging +import re +import time +# Due to a failure in python 3.8 +from typing import List + +import dacite + +from aprsd import client, stats +from aprsd.threads import tx +from aprsd.utils import counter + + +LOG = logging.getLogger("APRSD") + +PACKET_TYPE_MESSAGE = "message" +PACKET_TYPE_ACK = "ack" +PACKET_TYPE_MICE = "mic-e" +PACKET_TYPE_WX = "weather" +PACKET_TYPE_UNKNOWN = "unknown" +PACKET_TYPE_STATUS = "status" +PACKET_TYPE_BEACON = "beacon" +PACKET_TYPE_UNCOMPRESSED = "uncompressed" + + +@dataclass() +class Packet(metaclass=abc.ABCMeta): + from_call: str + to_call: str + addresse: str = None + format: str = None + msgNo: str = None # noqa: N815 + packet_type: str = None + timestamp: float = field(default_factory=time.time) + raw: str = None + _raw_dict: dict = field(repr=False, default_factory=lambda: {}) + _retry_count = 3 + _last_send_time = 0 + _last_send_attempt = 0 + # Do we allow this packet to be saved to send later? + _allow_delay = True + + _transport = None + _raw_message = None + + def get(self, key, default=None): + """Emulate a getter on a dict.""" + if hasattr(self, key): + return getattr(self, key) + else: + return default + + def _init_for_send(self): + """Do stuff here that is needed prior to sending over the air.""" + if not self.msgNo: + c = counter.PacketCounter() + c.increment() + self.msgNo = c.value + + # now build the raw message for sending + self._build_raw() + + def _build_raw(self): + """Build the self.raw string which is what is sent over the air.""" + self.raw = self._filter_for_send().rstrip("\n") + + @staticmethod + def factory(raw_packet): + raw = raw_packet + raw["_raw_dict"] = raw.copy() + translate_fields = { + "from": "from_call", + "to": "to_call", + } + # First translate some fields + for key in translate_fields: + if key in raw: + raw[translate_fields[key]] = raw[key] + del raw[key] + + if "addresse" in raw: + raw["to_call"] = raw["addresse"] + + packet_type = get_packet_type(raw) + raw["packet_type"] = packet_type + class_name = TYPE_LOOKUP[packet_type] + if packet_type == PACKET_TYPE_UNKNOWN: + # Try and figure it out here + if "latitude" in raw: + class_name = GPSPacket + + if packet_type == PACKET_TYPE_WX: + # the weather information is in a dict + # this brings those values out to the outer dict + for key in raw["weather"]: + raw[key] = raw["weather"][key] + + return dacite.from_dict(data_class=class_name, data=raw) + + def log(self, header=None): + """LOG a packet to the logfile.""" + asdict(self) + log_list = ["\n"] + if header: + if isinstance(self, AckPacket): + log_list.append( + f"{header} ___________" + f"(TX:{self._send_count} of {self._retry_count})", + ) + else: + log_list.append(f"{header} _______________") + log_list.append(f" Packet : {self.__class__.__name__}") + log_list.append(f" Raw : {self.raw}") + if self.to_call: + log_list.append(f" To : {self.to_call}") + if self.from_call: + log_list.append(f" From : {self.from_call}") + if hasattr(self, "path") and self.path: + log_list.append(f" Path : {'=>'.join(self.path)}") + if hasattr(self, "via") and self.via: + log_list.append(f" VIA : {self.via}") + + elif isinstance(self, MessagePacket): + log_list.append(f" Message : {self.message_text}") + + if hasattr(self, "comment") and self.comment: + log_list.append(f" Comment : {self.comment}") + + if self.msgNo: + log_list.append(f" Msg # : {self.msgNo}") + log_list.append(f"{header} _______________ Complete") + + LOG.info("\n".join(log_list)) + LOG.debug(self) + + def _filter_for_send(self) -> str: + """Filter and format message string for FCC.""" + # max? ftm400 displays 64, raw msg shows 74 + # and ftm400-send is max 64. setting this to + # 67 displays 64 on the ftm400. (+3 {01 suffix) + # feature req: break long ones into two msgs + message = self.raw[:67] + # We all miss George Carlin + return re.sub("fuck|shit|cunt|piss|cock|bitch", "****", message) + + def send(self): + """Method to send a packet.""" + LOG.warning("send() called!") + self._init_for_send() + thread = tx.SendPacketThread(packet=self) + LOG.warning(f"Starting thread to TX {self}") + thread.start() + LOG.warning("Thread started") + + def send_direct(self, aprsis_client=None): + """Send the message in the same thread as caller.""" + self._init_for_send() + if aprsis_client: + cl = aprsis_client + else: + cl = client.factory.create().client + self.log(header="Sending Message Direct") + cl.send(self.raw) + stats.APRSDStats().msgs_tx_inc() + + +@dataclass() +class PathPacket(Packet): + path: List[str] = field(default_factory=list) + via: str = None + + def _build_raw(self): + raise NotImplementedError + + +@dataclass() +class AckPacket(PathPacket): + response: str = None + _send_count = 1 + + def _build_raw(self): + """Build the self.raw which is what is sent over the air.""" + self.raw = "{}>APZ100::{}:ack{}".format( + self.from_call, + self.to_call.ljust(9), + self.msgNo, + ) + + def send(self): + """Method to send a packet.""" + self._init_for_send() + thread = tx.SendAckThread(packet=self) + LOG.warning(f"Starting thread to TXACK {self}") + thread.start() + + +@dataclass() +class MessagePacket(PathPacket): + message_text: str = None + + def _filter_for_send(self) -> str: + """Filter and format message string for FCC.""" + # max? ftm400 displays 64, raw msg shows 74 + # and ftm400-send is max 64. setting this to + # 67 displays 64 on the ftm400. (+3 {01 suffix) + # feature req: break long ones into two msgs + message = self.message_text[:67] + # We all miss George Carlin + return re.sub("fuck|shit|cunt|piss|cock|bitch", "****", message) + + def _build_raw(self): + """Build the self.raw which is what is sent over the air.""" + self.raw = "{}>APZ100::{}:{}{{{}".format( + self.from_call, + self.to_call.ljust(9), + self._filter_for_send().rstrip("\n"), + str(self.msgNo), + ) + + +@dataclass() +class StatusPacket(PathPacket): + status: str = None + timestamp: int = 0 + messagecapable: bool = False + comment: str = None + + def _build_raw(self): + raise NotImplementedError + + +@dataclass() +class GPSPacket(PathPacket): + latitude: float = 0.00 + longitude: float = 0.00 + altitude: float = 0.00 + rng: float = 0.00 + posambiguity: int = 0 + timestamp: int = 0 + comment: str = None + symbol: str = None + symbol_table: str = None + speed: float = 0.00 + course: int = 0 + + def _build_raw(self): + raise NotImplementedError + + +@dataclass() +class MicEPacket(GPSPacket): + messagecapable: bool = False + mbits: str = None + mtype: str = None + + def _build_raw(self): + raise NotImplementedError + + +@dataclass() +class WeatherPacket(GPSPacket): + symbol: str = "_" + wind_gust: float = 0.00 + temperature: float = 0.00 + rain_1h: float = 0.00 + rain_24h: float = 0.00 + rain_since_midnight: float = 0.00 + humidity: int = 0 + pressure: float = 0.00 + comment: str = None + + def _build_raw(self): + raise NotImplementedError + + +TYPE_LOOKUP = { + PACKET_TYPE_WX: WeatherPacket, + PACKET_TYPE_MESSAGE: MessagePacket, + PACKET_TYPE_ACK: AckPacket, + PACKET_TYPE_MICE: MicEPacket, + PACKET_TYPE_STATUS: StatusPacket, + PACKET_TYPE_BEACON: GPSPacket, + PACKET_TYPE_UNKNOWN: Packet, +} + + +def get_packet_type(packet: dict): + """Decode the packet type from the packet.""" + + pkt_format = packet.get("format", None) + msg_response = packet.get("response", None) + packet_type = "unknown" + if pkt_format == "message" and msg_response == "ack": + packet_type = PACKET_TYPE_ACK + elif pkt_format == "message": + packet_type = PACKET_TYPE_MESSAGE + elif pkt_format == "mic-e": + packet_type = PACKET_TYPE_MICE + elif pkt_format == "status": + packet_type = PACKET_TYPE_STATUS + elif pkt_format == PACKET_TYPE_BEACON: + packet_type = PACKET_TYPE_BEACON + elif pkt_format == PACKET_TYPE_UNCOMPRESSED: + if packet.get("symbol", None) == "_": + packet_type = PACKET_TYPE_WX + return packet_type + + +def is_message_packet(packet): + return get_packet_type(packet) == PACKET_TYPE_MESSAGE + + +def is_ack_packet(packet): + return get_packet_type(packet) == PACKET_TYPE_ACK + + +def is_mice_packet(packet): + return get_packet_type(packet) == PACKET_TYPE_MICE diff --git a/aprsd/packets/packet_list.py b/aprsd/packets/packet_list.py new file mode 100644 index 0000000..89c1fd4 --- /dev/null +++ b/aprsd/packets/packet_list.py @@ -0,0 +1,60 @@ +import logging +import threading +import time + +import wrapt + +from aprsd import utils +from aprsd.packets import seen_list + + +LOG = logging.getLogger("APRSD") + + +class PacketList: + """Class to track all of the packets rx'd and tx'd by aprsd.""" + + _instance = None + lock = threading.Lock() + config = None + + packet_list = utils.RingBuffer(1000) + + total_recv = 0 + total_tx = 0 + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance.config = kwargs["config"] + return cls._instance + + def __init__(self, config=None): + if config: + self.config = config + + @wrapt.synchronized(lock) + def __iter__(self): + return iter(self.packet_list) + + @wrapt.synchronized(lock) + def add(self, packet): + packet.ts = time.time() + if packet.from_call == self.config["aprs"]["login"]: + self.total_tx += 1 + else: + self.total_recv += 1 + self.packet_list.append(packet) + seen_list.SeenList().update_seen(packet) + + @wrapt.synchronized(lock) + def get(self): + return self.packet_list.get() + + @wrapt.synchronized(lock) + def total_received(self): + return self.total_recv + + @wrapt.synchronized(lock) + def total_sent(self): + return self.total_tx diff --git a/aprsd/packets/seen_list.py b/aprsd/packets/seen_list.py new file mode 100644 index 0000000..3c9e1bf --- /dev/null +++ b/aprsd/packets/seen_list.py @@ -0,0 +1,44 @@ +import datetime +import logging +import threading + +import wrapt + +from aprsd.utils import objectstore + + +LOG = logging.getLogger("APRSD") + + +class SeenList(objectstore.ObjectStoreMixin): + """Global callsign seen list.""" + + _instance = None + lock = threading.Lock() + data = {} + config = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super().__new__(cls) + if "config" in kwargs: + cls._instance.config = kwargs["config"] + cls._instance._init_store() + cls._instance.data = {} + return cls._instance + + @wrapt.synchronized(lock) + def update_seen(self, packet): + callsign = None + if packet.from_call: + callsign = packet.from_call + else: + LOG.warning(f"Can't find FROM in packet {packet}") + return + if callsign not in self.data: + self.data[callsign] = { + "last": None, + "count": 0, + } + self.data[callsign]["last"] = str(datetime.datetime.now()) + self.data[callsign]["count"] += 1 diff --git a/aprsd/packets/tracker.py b/aprsd/packets/tracker.py new file mode 100644 index 0000000..82ff3f3 --- /dev/null +++ b/aprsd/packets/tracker.py @@ -0,0 +1,116 @@ +import datetime +import threading + +import wrapt + +from aprsd import stats +from aprsd.utils import objectstore + + +class PacketTrack(objectstore.ObjectStoreMixin): + """Class to keep track of outstanding text messages. + + This is a thread safe class that keeps track of active + messages. + + When a message is asked to be sent, it is placed into this + class via it's id. The TextMessage class's send() method + automatically adds itself to this class. When the ack is + recieved from the radio, the message object is removed from + this class. + """ + + _instance = None + _start_time = None + lock = threading.Lock() + + data = {} + total_tracked = 0 + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super().__new__(cls) + cls._instance._start_time = datetime.datetime.now() + cls._instance.config = kwargs["config"] + cls._instance._init_store() + return cls._instance + + @wrapt.synchronized(lock) + def __getitem__(self, name): + return self.data[name] + + @wrapt.synchronized(lock) + def __iter__(self): + return iter(self.data) + + @wrapt.synchronized(lock) + def keys(self): + return self.data.keys() + + @wrapt.synchronized(lock) + def items(self): + return self.data.items() + + @wrapt.synchronized(lock) + def values(self): + return self.data.values() + + @wrapt.synchronized(lock) + def __len__(self): + return len(self.data) + + @wrapt.synchronized(lock) + def __str__(self): + result = "{" + for key in self.data.keys(): + result += f"{key}: {str(self.data[key])}, " + result += "}" + return result + + @wrapt.synchronized(lock) + def add(self, packet): + key = int(packet.msgNo) + self.data[key] = packet + stats.APRSDStats().msgs_tracked_inc() + self.total_tracked += 1 + + @wrapt.synchronized(lock) + def get(self, id): + if id in self.data: + return self.data[id] + + @wrapt.synchronized(lock) + def remove(self, id): + key = int(id) + if key in self.data.keys(): + del self.data[key] + + def restart(self): + """Walk the list of messages and restart them if any.""" + for key in self.data.keys(): + pkt = self.data[key] + if pkt.last_send_attempt < pkt.retry_count: + pkt.send() + + def _resend(self, packet): + packet._last_send_attempt = 0 + packet.send() + + def restart_delayed(self, count=None, most_recent=True): + """Walk the list of delayed messages and restart them if any.""" + if not count: + # Send all the delayed messages + for key in self.data.keys(): + pkt = self.data[key] + if pkt._last_send_attempt == pkt._retry_count: + self._resend(pkt) + else: + # They want to resend delayed messages + tmp = sorted( + self.data.items(), + reverse=most_recent, + key=lambda x: x[1].last_send_time, + ) + pkt_list = tmp[:count] + for (_key, pkt) in pkt_list: + self._resend(pkt) diff --git a/aprsd/packets/watch_list.py b/aprsd/packets/watch_list.py new file mode 100644 index 0000000..54c3995 --- /dev/null +++ b/aprsd/packets/watch_list.py @@ -0,0 +1,103 @@ +import datetime +import logging +import threading + +import wrapt + +from aprsd import utils +from aprsd.utils import objectstore + + +LOG = logging.getLogger("APRSD") + + +class WatchList(objectstore.ObjectStoreMixin): + """Global watch list and info for callsigns.""" + + _instance = None + lock = threading.Lock() + data = {} + config = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super().__new__(cls) + if "config" in kwargs: + cls._instance.config = kwargs["config"] + cls._instance._init_store() + cls._instance.data = {} + return cls._instance + + def __init__(self, config=None): + if config: + self.config = config + + ring_size = config["aprsd"]["watch_list"].get("packet_keep_count", 10) + + for callsign in config["aprsd"]["watch_list"].get("callsigns", []): + call = callsign.replace("*", "") + # FIXME(waboring) - we should fetch the last time we saw + # a beacon from a callsign or some other mechanism to find + # last time a message was seen by aprs-is. For now this + # is all we can do. + self.data[call] = { + "last": datetime.datetime.now(), + "packets": utils.RingBuffer( + ring_size, + ), + } + + def is_enabled(self): + if self.config and "watch_list" in self.config["aprsd"]: + return self.config["aprsd"]["watch_list"].get("enabled", False) + else: + return False + + def callsign_in_watchlist(self, callsign): + return callsign in self.data + + @wrapt.synchronized(lock) + def update_seen(self, packet): + if packet.addresse: + callsign = packet.addresse + else: + callsign = packet.from_call + if self.callsign_in_watchlist(callsign): + self.data[callsign]["last"] = datetime.datetime.now() + self.data[callsign]["packets"].append(packet) + + def last_seen(self, callsign): + if self.callsign_in_watchlist(callsign): + return self.data[callsign]["last"] + + def age(self, callsign): + now = datetime.datetime.now() + return str(now - self.last_seen(callsign)) + + def max_delta(self, seconds=None): + watch_list_conf = self.config["aprsd"]["watch_list"] + if not seconds: + seconds = watch_list_conf["alert_time_seconds"] + max_timeout = {"seconds": seconds} + return datetime.timedelta(**max_timeout) + + def is_old(self, callsign, seconds=None): + """Watch list callsign last seen is old compared to now? + + This tests to see if the last time we saw a callsign packet, + if that is older than the allowed timeout in the config. + + We put this here so any notification plugin can use this + same test. + """ + age = self.age(callsign) + + delta = utils.parse_delta_str(age) + d = datetime.timedelta(**delta) + + max_delta = self.max_delta(seconds=seconds) + + if d > max_delta: + return True + else: + return False diff --git a/aprsd/plugin.py b/aprsd/plugin.py index a62ab9d..885740b 100644 --- a/aprsd/plugin.py +++ b/aprsd/plugin.py @@ -13,7 +13,8 @@ import pluggy from thesmuggler import smuggle import aprsd -from aprsd import client, messaging, packets, threads +from aprsd import client, messaging, threads +from aprsd.packets import watch_list # setup the global logger @@ -119,11 +120,11 @@ class APRSDPluginBase(metaclass=abc.ABCMeta): thread.stop() @abc.abstractmethod - def filter(self, packet: packets.Packet): + def filter(self, packet): pass @abc.abstractmethod - def process(self, packet: packets.Packet): + def process(self, packet): """This is called when the filter passes.""" @@ -160,10 +161,10 @@ class APRSDWatchListPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): LOG.warning("Watch list enabled, but no callsigns set.") @hookimpl - def filter(self, packet: packets.Packet): + def filter(self, packet): result = messaging.NULL_MESSAGE if self.enabled: - wl = packets.WatchList() + wl = watch_list.WatchList() if wl.callsign_in_watchlist(packet.from_call): # packet is from a callsign in the watch list self.rx_inc() @@ -212,7 +213,7 @@ class APRSDRegexCommandPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): self.enabled = True @hookimpl - def filter(self, packet: packets.MessagePacket): + def filter(self, packet): result = None message = packet.get("message_text", None) @@ -272,7 +273,7 @@ class HelpPlugin(APRSDRegexCommandPluginBase): def help(self): return "Help: send APRS help or help " - def process(self, packet: packets.MessagePacket): + def process(self, packet): LOG.info("HelpPlugin") # fromcall = packet.get("from") message = packet.message_text @@ -475,7 +476,7 @@ class PluginManager: self._load_plugin(p_name) LOG.info("Completed Plugin Loading.") - def run(self, packet: packets.Packet): + def run(self, packet): """Execute all the pluguns run method.""" with self.lock: return self._pluggy_pm.hook.filter(packet=packet) diff --git a/aprsd/plugins/notify.py b/aprsd/plugins/notify.py index 2a9e7ea..d75a19c 100644 --- a/aprsd/plugins/notify.py +++ b/aprsd/plugins/notify.py @@ -1,7 +1,6 @@ import logging from aprsd import messaging, packets, plugin -from aprsd.utils import trace LOG = logging.getLogger("APRSD") @@ -18,7 +17,6 @@ class NotifySeenPlugin(plugin.APRSDWatchListPluginBase): short_description = "Notify me when a CALLSIGN is recently seen on APRS-IS" - @trace.trace def process(self, packet: packets.MessagePacket): LOG.info("NotifySeenPlugin") diff --git a/aprsd/threads/aprsd.py b/aprsd/threads/aprsd.py index 819ee7f..5673fa4 100644 --- a/aprsd/threads/aprsd.py +++ b/aprsd/threads/aprsd.py @@ -39,6 +39,8 @@ class APRSDThreadList: """Iterate over all threads and call stop on them.""" for th in self.threads_list: LOG.info(f"Stopping Thread {th.name}") + if hasattr(th, "packet"): + LOG.info(F"{th.name} packet {th.packet}") th.stop() @wrapt.synchronized(lock) diff --git a/aprsd/threads/keep_alive.py b/aprsd/threads/keep_alive.py index 95c7ea0..cfbdbc9 100644 --- a/aprsd/threads/keep_alive.py +++ b/aprsd/threads/keep_alive.py @@ -3,7 +3,7 @@ import logging import time import tracemalloc -from aprsd import client, messaging, packets, stats, utils +from aprsd import client, packets, stats, utils from aprsd.threads import APRSDThread, APRSDThreadList @@ -23,7 +23,7 @@ class KeepAliveThread(APRSDThread): def loop(self): if self.cntr % 60 == 0: - tracker = messaging.MsgTrack() + pkt_tracker = packets.PacketTrack() stats_obj = stats.APRSDStats() pl = packets.PacketList() thread_list = APRSDThreadList() @@ -53,7 +53,7 @@ class KeepAliveThread(APRSDThread): utils.strfdelta(stats_obj.uptime), pl.total_recv, pl.total_tx, - len(tracker), + len(pkt_tracker), stats_obj.msgs_tx, stats_obj.msgs_rx, last_msg_time, diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index dfa7c80..fad4096 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -64,12 +64,10 @@ class APRSDPluginRXThread(APRSDRXThread): processing in the PluginProcessPacketThread. """ def process_packet(self, *args, **kwargs): - raw = self._client.decode_packet(*args, **kwargs) + packet = self._client.decode_packet(*args, **kwargs) # LOG.debug(raw) - packet = packets.Packet.factory(raw.copy()) + #packet = packets.Packet.factory(raw.copy()) packet.log(header="RX Packet") - # LOG.debug(packet) - del raw thread = APRSDPluginProcessPacketThread( config=self.config, packet=packet, @@ -90,24 +88,20 @@ class APRSDProcessPacketThread(APRSDThread): self.packet = packet name = self.packet.raw[:10] super().__init__(f"RXPKT-{name}") + self._loop_cnt = 1 def process_ack_packet(self, packet): ack_num = packet.msgNo LOG.info(f"Got ack for message {ack_num}") - messaging.log_message( - "RXACK", - packet.raw, - None, - ack=ack_num, - fromcall=packet.from_call, - ) - tracker = messaging.MsgTrack() - tracker.remove(ack_num) + packet.log("RXACK") + pkt_tracker = packets.PacketTrack() + pkt_tracker.remove(ack_num) stats.APRSDStats().ack_rx_inc() return def loop(self): """Process a packet received from aprs-is server.""" + LOG.debug(f"RXPKT-LOOP {self._loop_cnt}") packet = self.packet packets.PacketList().add(packet) our_call = self.config["aprsd"]["callsign"].lower() @@ -136,12 +130,20 @@ class APRSDProcessPacketThread(APRSDThread): stats.APRSDStats().msgs_rx_inc() # let any threads do their thing, then ack # send an ack last - ack = messaging.AckMessage( - self.config["aprsd"]["callsign"], - from_call, - msg_id=msg_id, + ack_pkt = packets.AckPacket( + from_call=self.config["aprsd"]["callsign"], + to_call=from_call, + msgNo=msg_id, ) - ack.send() + LOG.warning(f"Send AckPacket {ack_pkt}") + ack_pkt.send() + LOG.warning("Send ACK called Continue on") + #ack = messaging.AckMessage( + # self.config["aprsd"]["callsign"], + # from_call, + # msg_id=msg_id, + #) + #ack.send() self.process_our_message_packet(packet) else: @@ -151,7 +153,8 @@ class APRSDProcessPacketThread(APRSDThread): self.process_other_packet( packet, for_us=(to_call.lower() == our_call), ) - LOG.debug("Packet processing complete") + LOG.debug("Packet processing complete") + return False @abc.abstractmethod def process_our_message_packet(self, *args, **kwargs): @@ -194,16 +197,29 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): if isinstance(subreply, messaging.Message): subreply.send() else: - msg = messaging.TextMessage( - self.config["aprsd"]["callsign"], - from_call, - subreply, + msg_pkt = packets.MessagePacket( + from_call=self.config["aprsd"]["callsign"], + to_call=from_call, + message_text=subreply, ) - msg.send() + msg_pkt.send() + #msg = messaging.TextMessage( + # self.config["aprsd"]["callsign"], + # from_call, + # subreply, + #) + #msg.send() elif isinstance(reply, messaging.Message): # We have a message based object. LOG.debug(f"Sending '{reply}'") - reply.send() + # Convert this to the new packet + msg_pkt = packets.MessagePacket( + from_call=reply.fromcall, + to_call=reply.tocall, + message_text=reply._raw_message, + ) + #reply.send() + msg_pkt.send() replied = True else: replied = True @@ -213,33 +229,55 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): # usage string if reply is not messaging.NULL_MESSAGE: LOG.debug(f"Sending '{reply}'") - - msg = messaging.TextMessage( - self.config["aprsd"]["callsign"], - from_call, - reply, + msg_pkt = packets.MessagePacket( + from_call=self.config["aprsd"]["callsign"], + to_call=from_call, + message_text=reply, ) - msg.send() + LOG.warning("Calling msg_pkg.send()") + msg_pkt.send() + LOG.warning("Calling msg_pkg.send() --- DONE") + + #msg = messaging.TextMessage( + # self.config["aprsd"]["callsign"], + # from_call, + # reply, + #) + #msg.send() # If the message was for us and we didn't have a # response, then we send a usage statement. if to_call == self.config["aprsd"]["callsign"] and not replied: LOG.warning("Sending help!") - msg = messaging.TextMessage( - self.config["aprsd"]["callsign"], - from_call, - "Unknown command! Send 'help' message for help", + msg_pkt = packets.MessagePacket( + from_call=self.config["aprsd"]["callsign"], + to_call=from_call, + message_text="Unknown command! Send 'help' message for help", ) - msg.send() + msg_pkt.send() + #msg = messaging.TextMessage( + # self.config["aprsd"]["callsign"], + # from_call, + # "Unknown command! Send 'help' message for help", + #) + #msg.send() except Exception as ex: LOG.error("Plugin failed!!!") LOG.exception(ex) # Do we need to send a reply? if to_call == self.config["aprsd"]["callsign"]: reply = "A Plugin failed! try again?" - msg = messaging.TextMessage( - self.config["aprsd"]["callsign"], - from_call, - reply, + msg_pkt = packets.MessagePacket( + from_call=self.config["aprsd"]["callsign"], + to_call=from_call, + message_text=reply, ) - msg.send() + msg_pkt.send() + #msg = messaging.TextMessage( + # self.config["aprsd"]["callsign"], + # from_call, + # reply, + #) + #msg.send() + + LOG.debug("Completed process_our_message_packet") diff --git a/aprsd/threads/tx.py b/aprsd/threads/tx.py new file mode 100644 index 0000000..8df838c --- /dev/null +++ b/aprsd/threads/tx.py @@ -0,0 +1,120 @@ +import datetime +import logging +import time + +from aprsd import client, stats +from aprsd import threads as aprsd_threads +from aprsd.packets import packet_list, tracker + + +LOG = logging.getLogger("APRSD") + + +class SendPacketThread(aprsd_threads.APRSDThread): + def __init__(self, packet): + self.packet = packet + name = self.packet.raw[:5] + super().__init__(f"TXPKT-{self.packet.msgNo}-{name}") + pkt_tracker = tracker.PacketTrack() + pkt_tracker.add(packet) + + def loop(self): + LOG.debug("TX Loop") + """Loop until a message is acked or it gets delayed. + + We only sleep for 5 seconds between each loop run, so + that CTRL-C can exit the app in a short period. Each sleep + means the app quitting is blocked until sleep is done. + So we keep track of the last send attempt and only send if the + last send attempt is old enough. + + """ + pkt_tracker = tracker.PacketTrack() + # lets see if the message is still in the tracking queue + packet = pkt_tracker.get(self.packet.msgNo) + if not packet: + # The message has been removed from the tracking queue + # So it got acked and we are done. + LOG.info("Message Send Complete via Ack.") + return False + else: + send_now = False + if packet._last_send_attempt == packet._retry_count: + # we reached the send limit, don't send again + # TODO(hemna) - Need to put this in a delayed queue? + LOG.info("Message Send Complete. Max attempts reached.") + if not packet._allow_delay: + pkt_tracker.remove(packet.msgNo) + return False + + # Message is still outstanding and needs to be acked. + if packet._last_send_time: + # Message has a last send time tracking + now = datetime.datetime.now() + sleeptime = (packet._last_send_attempt + 1) * 31 + delta = now - packet._last_send_time + if delta > datetime.timedelta(seconds=sleeptime): + # It's time to try to send it again + send_now = True + else: + send_now = True + + if send_now: + # no attempt time, so lets send it, and start + # tracking the time. + packet.log("Sending Message") + cl = client.factory.create().client + cl.send(packet.raw) + stats.APRSDStats().msgs_tx_inc() + packet_list.PacketList().add(packet) + packet._last_send_time = datetime.datetime.now() + packet._last_send_attempt += 1 + + time.sleep(5) + # Make sure we get called again. + return True + + +class SendAckThread(aprsd_threads.APRSDThread): + def __init__(self, packet): + self.packet = packet + super().__init__(f"SendAck-{self.packet.msgNo}") + self._loop_cnt = 1 + + def loop(self): + """Separate thread to send acks with retries.""" + send_now = False + if self.packet._last_send_attempt == self.packet._retry_count: + # we reached the send limit, don't send again + # TODO(hemna) - Need to put this in a delayed queue? + LOG.info("Ack Send Complete. Max attempts reached.") + return False + + if self.packet._last_send_time: + # Message has a last send time tracking + now = datetime.datetime.now() + + # aprs duplicate detection is 30 secs? + # (21 only sends first, 28 skips middle) + sleeptime = 31 + delta = now - self.packet._last_send_time + if delta > datetime.timedelta(seconds=sleeptime): + # It's time to try to send it again + send_now = True + elif self._loop_cnt % 5 == 0: + LOG.debug(f"Still wating. {delta}") + else: + send_now = True + + if send_now: + cl = client.factory.create().client + self.packet.log("Sending ACK") + cl.send(self.packet.raw) + self.packet._send_count += 1 + stats.APRSDStats().ack_tx_inc() + packet_list.PacketList().add(self.packet) + self.packet._last_send_attempt += 1 + self.packet._last_send_time = datetime.datetime.now() + time.sleep(1) + self._loop_cnt += 1 + return True diff --git a/aprsd/utils/counter.py b/aprsd/utils/counter.py new file mode 100644 index 0000000..e423bfb --- /dev/null +++ b/aprsd/utils/counter.py @@ -0,0 +1,48 @@ +from multiprocessing import RawValue +import threading + +import wrapt + + +class PacketCounter: + """ + Global Packet id counter class. + + This is a singleton based class that keeps + an incrementing counter for all packets to + be sent. All new Packet objects gets a new + message id, which is the next number available + from the PacketCounter. + + """ + + _instance = None + max_count = 9999 + lock = threading.Lock() + + def __new__(cls, *args, **kwargs): + """Make this a singleton class.""" + if cls._instance is None: + cls._instance = super().__new__(cls, *args, **kwargs) + cls._instance.val = RawValue("i", 1) + return cls._instance + + @wrapt.synchronized(lock) + def increment(self): + if self.val.value == self.max_count: + self.val.value = 1 + else: + self.val.value += 1 + + @property + @wrapt.synchronized(lock) + def value(self): + return self.val.value + + @wrapt.synchronized(lock) + def __repr__(self): + return str(self.val.value) + + @wrapt.synchronized(lock) + def __str__(self): + return str(self.val.value) diff --git a/tox.ini b/tox.ini index 84b8b24..c0fe0c1 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ minversion = 2.9.0 skipdist = True skip_missing_interpreters = true -envlist = pep8,py{38,39} +envlist = pep8,py{39,310} #requires = tox-pipenv # pip==22.0.4 # pip-tools==5.4.0 From 1b49f128a96197bd11d47feba203d1029e022a50 Mon Sep 17 00:00:00 2001 From: Hemna Date: Thu, 15 Dec 2022 20:13:40 -0500 Subject: [PATCH 04/21] cleanup webchat --- aprsd/cmds/webchat.py | 6 ++---- aprsd/packets/core.py | 3 --- aprsd/threads/rx.py | 8 -------- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index fcd5fd7..e79763c 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -381,12 +381,10 @@ class SendMessageNamespace(Namespace): from_call=self._config["aprs"]["login"], to_call="APDW16", raw=txt, + latitude=lat, + longitude=long, ) beacon.send_direct() - #beacon_msg = messaging.RawMessage(txt) - #beacon_msg.fromcall = self._config["aprs"]["login"] - #beacon_msg.tocall = "APDW16" - #beacon_msg.send_direct() def handle_message(self, data): LOG.debug(f"WS Data {data}") diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 383f039..b9d9736 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -147,12 +147,9 @@ class Packet(metaclass=abc.ABCMeta): def send(self): """Method to send a packet.""" - LOG.warning("send() called!") self._init_for_send() thread = tx.SendPacketThread(packet=self) - LOG.warning(f"Starting thread to TX {self}") thread.start() - LOG.warning("Thread started") def send_direct(self, aprsis_client=None): """Send the message in the same thread as caller.""" diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index fad4096..478a7ee 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -135,15 +135,7 @@ class APRSDProcessPacketThread(APRSDThread): to_call=from_call, msgNo=msg_id, ) - LOG.warning(f"Send AckPacket {ack_pkt}") ack_pkt.send() - LOG.warning("Send ACK called Continue on") - #ack = messaging.AckMessage( - # self.config["aprsd"]["callsign"], - # from_call, - # msg_id=msg_id, - #) - #ack.send() self.process_our_message_packet(packet) else: From 59e5af8ee59bfeac4c72662c42bcf0663162e3ef Mon Sep 17 00:00:00 2001 From: Hemna Date: Fri, 16 Dec 2022 12:35:54 -0500 Subject: [PATCH 05/21] Added contructing a GPSPacket for sending This patch adds the needed code to construct the raw output string for sending a GPSPacket. TODO: Need to incorporate speed, course, rng, position ambiguity ? TODO: Need to add option to 'compress' the output location data. --- aprsd/cmds/webchat.py | 14 +------------- aprsd/packets/core.py | 30 +++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index e79763c..58a8eba 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -365,24 +365,12 @@ class SendMessageNamespace(Namespace): LOG.debug(f"Lat DDM {lat}") LOG.debug(f"Long DDM {long}") - local_datetime = datetime.datetime.now() - utc_offset_timedelta = datetime.datetime.utcnow() - local_datetime - result_utc_datetime = local_datetime + utc_offset_timedelta - time_zulu = result_utc_datetime.strftime("%d%H%M") - - # now construct a beacon to send over the client connection - txt = ( - f"{self._config['aprs']['login']}>APZ100,WIDE2-1" - f":@{time_zulu}z{lat}/{long}l APRSD WebChat Beacon" - ) - - LOG.debug(f"Sending {txt}") beacon = packets.GPSPacket( from_call=self._config["aprs"]["login"], to_call="APDW16", - raw=txt, latitude=lat, longitude=long, + comment="APRSD WebChat Beacon", ) beacon.send_direct() diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index b9d9736..2623c0f 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -1,5 +1,6 @@ import abc from dataclasses import asdict, dataclass, field +import datetime import logging import re import time @@ -220,7 +221,6 @@ class MessagePacket(PathPacket): @dataclass() class StatusPacket(PathPacket): status: str = None - timestamp: int = 0 messagecapable: bool = False comment: str = None @@ -235,15 +235,35 @@ class GPSPacket(PathPacket): altitude: float = 0.00 rng: float = 0.00 posambiguity: int = 0 - timestamp: int = 0 comment: str = None - symbol: str = None - symbol_table: str = None + symbol: str = field(default="l") + symbol_table: str = field(default="/") speed: float = 0.00 course: int = 0 + def _build_time_zulu(self): + """Build the timestamp in UTC/zulu.""" + if self.timestamp: + local_dt = datetime.datetime.fromtimestamp(self.timestamp) + else: + local_dt = datetime.datetime.now() + self.timestamp = datetime.datetime.timestamp(local_dt) + + utc_offset_timedelta = datetime.datetime.utcnow() - local_dt + result_utc_datetime = local_dt + utc_offset_timedelta + time_zulu = result_utc_datetime.strftime("%d%H%M") + return time_zulu + def _build_raw(self): - raise NotImplementedError + time_zulu = self._build_time_zulu() + + self.raw = ( + f"{self.from_call}>{self.to_call},WIDE2-1:" + f"@{time_zulu}z{self.latitude}{self.symbol_table}" + f"{self.longitude}{self.symbol}" + ) + if self.comment: + self.raw = f"{self.raw}{self.comment}" @dataclass() From bfc0a5a1e9bd56c808edac679a50426efc5d322e Mon Sep 17 00:00:00 2001 From: Hemna Date: Fri, 16 Dec 2022 14:04:08 -0500 Subject: [PATCH 06/21] Cleaned out all references to messaging The messaging.py now is nothing but a shell that contains a link to packets.NULL_MESSAGE to help maintain some backwards compatibility with plugins. Packets dataclass has fully replaced messaging objects. --- aprsd/clients/kiss.py | 20 ++-- aprsd/cmds/dev.py | 4 +- aprsd/cmds/server.py | 3 - aprsd/flask.py | 194 +++++++++++++++++++------------------- aprsd/messaging.py | 3 +- aprsd/packets/__init__.py | 13 ++- aprsd/packets/core.py | 11 ++- aprsd/plugin.py | 4 +- aprsd/plugins/email.py | 45 ++++----- aprsd/plugins/notify.py | 23 ++--- aprsd/plugins/query.py | 21 +++-- aprsd/threads/rx.py | 44 +-------- 12 files changed, 180 insertions(+), 205 deletions(-) diff --git a/aprsd/clients/kiss.py b/aprsd/clients/kiss.py index 79db9b7..48e4f92 100644 --- a/aprsd/clients/kiss.py +++ b/aprsd/clients/kiss.py @@ -4,7 +4,7 @@ import aprslib from ax253 import Frame import kiss -from aprsd import messaging +from aprsd.packets import core from aprsd.utils import trace @@ -83,7 +83,7 @@ class KISS3Client: self.kiss.read(callback=self.parse_frame, min_frames=None) LOG.debug("END blocking KISS consumer") - def send(self, msg): + def send(self, packet): """Send an APRS Message object.""" # payload = (':%-9s:%s' % ( @@ -93,26 +93,26 @@ class KISS3Client: # payload = str(msg).encode('US-ASCII') payload = None path = ["WIDE1-1", "WIDE2-1"] - if isinstance(msg, messaging.AckMessage): - msg_payload = f"ack{msg.id}" - elif isinstance(msg, messaging.RawMessage): - payload = msg.message.encode("US-ASCII") + if isinstance(packet, core.AckPacket): + msg_payload = f"ack{packet.msgNo}" + elif isinstance(packet, core.Packet): + payload = packet.raw.encode("US-ASCII") path = ["WIDE2-1"] else: - msg_payload = f"{msg.message}{{{str(msg.id)}" + msg_payload = f"{packet.raw}{{{str(packet.msgNo)}" if not payload: payload = ( ":{:<9}:{}".format( - msg.tocall, + packet.to_call, msg_payload, ) ).encode("US-ASCII") LOG.debug(f"Send '{payload}' TO KISS") frame = Frame.ui( - destination=msg.tocall, - source=msg.fromcall, + destination=packet.to_call, + source=packet.from_call, path=path, info=payload, ) diff --git a/aprsd/cmds/dev.py b/aprsd/cmds/dev.py index c3979ed..16d6da1 100644 --- a/aprsd/cmds/dev.py +++ b/aprsd/cmds/dev.py @@ -8,7 +8,7 @@ import logging import click # local imports here -from aprsd import cli_helper, client, messaging, packets, plugin, stats, utils +from aprsd import cli_helper, client, packets, plugin, stats, utils from aprsd.aprsd import cli from aprsd.utils import trace @@ -102,7 +102,7 @@ def test_plugin( client.Client(config) stats.APRSDStats(config) - messaging.MsgTrack(config=config) + packets.PacketTrack(config=config) packets.WatchList(config=config) packets.SeenList(config=config) diff --git a/aprsd/cmds/server.py b/aprsd/cmds/server.py index 4bb50cc..084f072 100644 --- a/aprsd/cmds/server.py +++ b/aprsd/cmds/server.py @@ -80,14 +80,12 @@ def server(ctx, flush): packets.PacketList(config=config) if flush: LOG.debug("Deleting saved MsgTrack.") - #messaging.MsgTrack(config=config).flush() packets.PacketTrack(config=config).flush() packets.WatchList(config=config) packets.SeenList(config=config) else: # Try and load saved MsgTrack list LOG.debug("Loading saved MsgTrack object.") - #messaging.MsgTrack(config=config).load() packets.PacketTrack(config=config).load() packets.WatchList(config=config).load() packets.SeenList(config=config).load() @@ -103,7 +101,6 @@ def server(ctx, flush): ) rx_thread.start() - #messaging.MsgTrack().restart() packets.PacketTrack().restart() keepalive = threads.KeepAliveThread(config=config) diff --git a/aprsd/flask.py b/aprsd/flask.py index 21de182..76865c1 100644 --- a/aprsd/flask.py +++ b/aprsd/flask.py @@ -14,11 +14,12 @@ import flask_classful from flask_httpauth import HTTPBasicAuth from flask_socketio import Namespace, SocketIO from werkzeug.security import check_password_hash, generate_password_hash +import wrapt import aprsd from aprsd import client from aprsd import config as aprsd_config -from aprsd import messaging, packets, plugin, stats, threads, utils +from aprsd import packets, plugin, stats, threads, utils from aprsd.clients import aprsis from aprsd.logging import log from aprsd.logging import rich as aprsd_logging @@ -32,7 +33,7 @@ users = None class SentMessages: _instance = None - lock = None + lock = threading.Lock() msgs = {} @@ -41,16 +42,16 @@ class SentMessages: if cls._instance is None: cls._instance = super().__new__(cls) # Put any initialization here. - cls.lock = threading.Lock() return cls._instance - def add(self, msg): - with self.lock: - self.msgs[msg.id] = self._create(msg.id) - self.msgs[msg.id]["from"] = msg.fromcall - self.msgs[msg.id]["to"] = msg.tocall - self.msgs[msg.id]["message"] = msg.message.rstrip("\n") - self.msgs[msg.id]["raw"] = str(msg).rstrip("\n") + @wrapt.synchronized(lock) + def add(self, packet): + self.msgs[packet.msgNo] = self._create(packet.msgNo) + self.msgs[packet.msgNo]["from"] = packet.from_call + self.msgs[packet.msgNo]["to"] = packet.to_call + self.msgs[packet.msgNo]["message"] = packet.message_text.rstrip("\n") + packet._build_raw() + self.msgs[packet.msgNo]["raw"] = packet.raw.rstrip("\n") def _create(self, id): return { @@ -66,34 +67,34 @@ class SentMessages: "reply": None, } + @wrapt.synchronized(lock) def __len__(self): - with self.lock: - return len(self.msgs.keys()) + return len(self.msgs.keys()) + @wrapt.synchronized(lock) def get(self, id): - with self.lock: - if id in self.msgs: - return self.msgs[id] + if id in self.msgs: + return self.msgs[id] + @wrapt.synchronized(lock) def get_all(self): - with self.lock: - return self.msgs + return self.msgs + @wrapt.synchronized(lock) def set_status(self, id, status): - with self.lock: - self.msgs[id]["last_update"] = str(datetime.datetime.now()) - self.msgs[id]["status"] = status + self.msgs[id]["last_update"] = str(datetime.datetime.now()) + self.msgs[id]["status"] = status + @wrapt.synchronized(lock) def ack(self, id): """The message got an ack!""" - with self.lock: - self.msgs[id]["last_update"] = str(datetime.datetime.now()) - self.msgs[id]["ack"] = True + self.msgs[id]["last_update"] = str(datetime.datetime.now()) + self.msgs[id]["ack"] = True + @wrapt.synchronized(lock) def reply(self, id, packet): """We got a packet back from the sent message.""" - with self.lock: - self.msgs[id]["reply"] = packet + self.msgs[id]["reply"] = packet # HTTPBasicAuth doesn't work on a class method. @@ -107,7 +108,7 @@ def verify_password(username, password): return username -class SendMessageThread(threads.APRSDThread): +class SendMessageThread(threads.APRSDRXThread): """Thread for sending a message from web.""" aprsis_client = None @@ -115,10 +116,10 @@ class SendMessageThread(threads.APRSDThread): got_ack = False got_reply = False - def __init__(self, config, info, msg, namespace): + def __init__(self, config, info, packet, namespace): self.config = config self.request = info - self.msg = msg + self.packet = packet self.namespace = namespace self.start_time = datetime.datetime.now() msg = "({} -> {}) : {}".format( @@ -180,8 +181,8 @@ class SendMessageThread(threads.APRSDThread): except LoginError as e: f"Failed to setup Connection {e}" - self.msg.send_direct(aprsis_client=self.aprs_client) - SentMessages().set_status(self.msg.id, "Sent") + self.packet.send_direct(aprsis_client=self.aprs_client) + SentMessages().set_status(self.packet.msgNo, "Sent") while not self.thread_stop: can_loop = self.loop() @@ -190,59 +191,55 @@ class SendMessageThread(threads.APRSDThread): threads.APRSDThreadList().remove(self) LOG.debug("Exiting") - def rx_packet(self, packet): + def process_ack_packet(self, packet): global socketio - # LOG.debug("Got packet back {}".format(packet)) - resp = packet.get("response", None) - if resp == "ack": - ack_num = packet.get("msgNo") - LOG.info(f"We got ack for our sent message {ack_num}") - messaging.log_packet(packet) - SentMessages().ack(self.msg.id) - socketio.emit( - "ack", SentMessages().get(self.msg.id), - namespace="/sendmsg", - ) - stats.APRSDStats().ack_rx_inc() - self.got_ack = True - if self.request["wait_reply"] == "0" or self.got_reply: - # We aren't waiting for a reply, so we can bail - self.stop() - self.thread_stop = self.aprs_client.thread_stop = True + ack_num = packet.msgNo + LOG.info(f"We got ack for our sent message {ack_num}") + packet.log("RXACK") + SentMessages().ack(self.packet.msgNo) + stats.APRSDStats().ack_rx_inc() + socketio.emit( + "ack", SentMessages().get(self.packet.msgNo), + namespace="/sendmsg", + ) + if self.request["wait_reply"] == "0" or self.got_reply: + # We aren't waiting for a reply, so we can bail + self.stop() + self.thread_stop = self.aprs_client.thread_stop = True + + def process_our_message_packet(self, packet): + global socketio + packets.PacketList().add(packet) + stats.APRSDStats().msgs_rx_inc() + msg_number = packet.msgNo + SentMessages().reply(self.packet.msgNo, packet) + SentMessages().set_status(self.packet.msgNo, "Got Reply") + socketio.emit( + "reply", SentMessages().get(self.packet.msgNo), + namespace="/sendmsg", + ) + ack_pkt = packets.AckPacket( + from_call=self.request["from"], + to_call=packet.from_call, + msgNo=msg_number, + ) + ack_pkt.send_direct(aprsis_client=self.aprsis_client) + SentMessages().set_status(self.packet.msgNo, "Ack Sent") + + # Now we can exit, since we are done. + self.got_reply = True + if self.got_ack: + self.stop() + self.thread_stop = self.aprs_client.thread_stop = True + + def process_packet(self, *args, **kwargs): + packet = self._client.decode_packet(*args, **kwargs) + packet.log(header="RX Packet") + + if isinstance(packet, packets.AckPacket): + self.process_ack_packet(packet) else: - packets.PacketList().add(packet) - stats.APRSDStats().msgs_rx_inc() - message = packet.get("message_text", None) - fromcall = packet["from"] - msg_number = packet.get("msgNo", "0") - messaging.log_message( - "Received Message", - packet["raw"], - message, - fromcall=fromcall, - ack=msg_number, - ) - SentMessages().reply(self.msg.id, packet) - SentMessages().set_status(self.msg.id, "Got Reply") - socketio.emit( - "reply", SentMessages().get(self.msg.id), - namespace="/sendmsg", - ) - - # Send the ack back? - ack = messaging.AckMessage( - self.request["from"], - fromcall, - msg_id=msg_number, - ) - ack.send_direct() - SentMessages().set_status(self.msg.id, "Ack Sent") - - # Now we can exit, since we are done. - self.got_reply = True - if self.got_ack: - self.stop() - self.thread_stop = self.aprs_client.thread_stop = True + self.process_our_message_packet(packet) def loop(self): # we have a general time limit expecting results of @@ -265,7 +262,9 @@ class SendMessageThread(threads.APRSDThread): # This will register a packet consumer with aprslib # When new packets come in the consumer will process # the packet - self.aprs_client.consumer(self.rx_packet, raw=False, blocking=False) + self.aprs_client.consumer( + self.process_packet, raw=False, blocking=False, + ) except aprslib.exceptions.ConnectionDrop: LOG.error("Connection dropped.") return False @@ -353,7 +352,7 @@ class APRSDFlask(flask_classful.FlaskView): @auth.login_required def messages(self): - track = messaging.MsgTrack() + track = packets.PacketTrack() msgs = [] for id in track: LOG.info(track[id].dict()) @@ -393,13 +392,13 @@ class APRSDFlask(flask_classful.FlaskView): @auth.login_required def save(self): """Save the existing queue to disk.""" - track = messaging.MsgTrack() + track = packets.PacketTrack() track.save() return json.dumps({"messages": "saved"}) def _stats(self): stats_obj = stats.APRSDStats() - track = messaging.MsgTrack() + track = packets.PacketTrack() now = datetime.datetime.now() time_format = "%m-%d-%Y %H:%M:%S" @@ -444,7 +443,7 @@ class SendMessageNamespace(Namespace): _config = None got_ack = False reply_sent = False - msg = None + packet = None request = None def __init__(self, namespace=None, config=None): @@ -466,24 +465,27 @@ class SendMessageNamespace(Namespace): global socketio LOG.debug(f"WS: on_send {data}") self.request = data - msg = messaging.TextMessage( - data["from"], data["to"], - data["message"], + self.packet = packets.MessagePacket( + from_call=data["from"], + to_call=data["to"], + message_text=data["message"], ) - self.msg = msg msgs = SentMessages() - msgs.add(msg) - msgs.set_status(msg.id, "Sending") + msgs.add(self.packet) + msgs.set_status(self.packet.msgNo, "Sending") socketio.emit( - "sent", SentMessages().get(self.msg.id), + "sent", SentMessages().get(self.packet.msgNo), namespace="/sendmsg", ) - socketio.start_background_task(self._start, self._config, data, msg, self) + socketio.start_background_task( + self._start, self._config, data, + self.packet, self, + ) LOG.warning("WS: on_send: exit") - def _start(self, config, data, msg, namespace): - msg_thread = SendMessageThread(self._config, data, msg, self) + def _start(self, config, data, packet, namespace): + msg_thread = SendMessageThread(self._config, data, packet, self) msg_thread.start() def handle_message(self, data): diff --git a/aprsd/messaging.py b/aprsd/messaging.py index 3e25742..3fe3a79 100644 --- a/aprsd/messaging.py +++ b/aprsd/messaging.py @@ -1,3 +1,4 @@ # What to return from a plugin if we have processed the message # and it's ok, but don't send a usage string back -NULL_MESSAGE = -1 + +# REMOVE THIS FILE diff --git a/aprsd/packets/__init__.py b/aprsd/packets/__init__.py index 2edb578..8eb8fc0 100644 --- a/aprsd/packets/__init__.py +++ b/aprsd/packets/__init__.py @@ -1,8 +1,11 @@ -from aprsd.packets.core import ( +from aprsd.packets.core import ( # noqa: F401 AckPacket, GPSPacket, MessagePacket, MicEPacket, Packet, PathPacket, StatusPacket, WeatherPacket, ) -from aprsd.packets.packet_list import PacketList -from aprsd.packets.seen_list import SeenList -from aprsd.packets.tracker import PacketTrack -from aprsd.packets.watch_list import WatchList +from aprsd.packets.packet_list import PacketList # noqa: F401 +from aprsd.packets.seen_list import SeenList # noqa: F401 +from aprsd.packets.tracker import PacketTrack # noqa: F401 +from aprsd.packets.watch_list import WatchList # noqa: F401 + + +NULL_MESSAGE = -1 diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 2623c0f..a52f19b 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -46,6 +46,12 @@ class Packet(metaclass=abc.ABCMeta): _transport = None _raw_message = None + def __post__init(self): + if not self.msgNo: + c = counter.PacketCounter() + c.increment() + self.msgNo = c.value + def get(self, key, default=None): """Emulate a getter on a dict.""" if hasattr(self, key): @@ -55,11 +61,6 @@ class Packet(metaclass=abc.ABCMeta): def _init_for_send(self): """Do stuff here that is needed prior to sending over the air.""" - if not self.msgNo: - c = counter.PacketCounter() - c.increment() - self.msgNo = c.value - # now build the raw message for sending self._build_raw() diff --git a/aprsd/plugin.py b/aprsd/plugin.py index 885740b..db85a5d 100644 --- a/aprsd/plugin.py +++ b/aprsd/plugin.py @@ -13,7 +13,7 @@ import pluggy from thesmuggler import smuggle import aprsd -from aprsd import client, messaging, threads +from aprsd import client, packets, threads from aprsd.packets import watch_list @@ -162,7 +162,7 @@ class APRSDWatchListPluginBase(APRSDPluginBase, metaclass=abc.ABCMeta): @hookimpl def filter(self, packet): - result = messaging.NULL_MESSAGE + result = packets.NULL_MESSAGE if self.enabled: wl = watch_list.WatchList() if wl.callsign_in_watchlist(packet.from_call): diff --git a/aprsd/plugins/email.py b/aprsd/plugins/email.py index a9facec..557bd19 100644 --- a/aprsd/plugins/email.py +++ b/aprsd/plugins/email.py @@ -10,7 +10,7 @@ import time import imapclient -from aprsd import messaging, packets, plugin, stats, threads +from aprsd import packets, plugin, stats, threads from aprsd.utils import trace @@ -90,10 +90,10 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): if not self.enabled: # Email has not been enabled # so the plugin will just NOOP - return messaging.NULL_MESSAGE + return packets.NULL_MESSAGE fromcall = packet.from_call - message = packet.get("message_text", None) + message = packet.message_text ack = packet.get("msgNo", "0") reply = None @@ -109,7 +109,7 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): if r is not None: LOG.debug("RESEND EMAIL") resend_email(self.config, r.group(1), fromcall) - reply = messaging.NULL_MESSAGE + reply = packets.NULL_MESSAGE # -user@address.com body of email elif re.search(r"^-([A-Za-z0-9_\-\.@]+) (.*)", message): # (same search again) @@ -142,7 +142,7 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): if not too_soon or ack == 0: LOG.info(f"Send email '{content}'") send_result = send_email(self.config, to_addr, content) - reply = messaging.NULL_MESSAGE + reply = packets.NULL_MESSAGE if send_result != 0: reply = f"-{to_addr} failed" else: @@ -157,7 +157,7 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): self.email_sent_dict.clear() self.email_sent_dict[ack] = now else: - reply = messaging.NULL_MESSAGE + reply = packets.NULL_MESSAGE LOG.info( "Email for message number " + ack @@ -165,7 +165,6 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): ) else: reply = "Bad email address" - # messaging.send_message(fromcall, "Bad email address") return reply @@ -466,13 +465,12 @@ def resend_email(config, count, fromcall): from_addr = shortcuts_inverted[from_addr] # asterisk indicates a resend reply = "-" + from_addr + " * " + body.decode(errors="ignore") - # messaging.send_message(fromcall, reply) - msg = messaging.TextMessage( - config["aprs"]["login"], - fromcall, - reply, + pkt = packets.MessagePacket( + from_call=config["aprsd"]["callsign"], + to_call=fromcall, + message_text=reply, ) - msg.send() + pkt.send() msgexists = True if msgexists is not True: @@ -489,9 +487,12 @@ def resend_email(config, count, fromcall): str(m).zfill(2), str(s).zfill(2), ) - # messaging.send_message(fromcall, reply) - msg = messaging.TextMessage(config["aprs"]["login"], fromcall, reply) - msg.send() + pkt = packets.MessagePacket( + from_call=config["aprsd"]["callsign"], + to_call=fromcall, + message_text=reply, + ) + pkt.send() # check email more often since we're resending one now EmailInfo().delay = 60 @@ -605,12 +606,14 @@ class APRSDEmailThread(threads.APRSDThread): from_addr = shortcuts_inverted[from_addr] reply = "-" + from_addr + " " + body.decode(errors="ignore") - msg = messaging.TextMessage( - self.config["aprs"]["login"], - self.config["ham"]["callsign"], - reply, + # Send the message to the registered user in the + # config ham.callsign + pkt = packets.MessagePacket( + from_call=self.config["aprsd"]["callsign"], + to_call=self.config["ham"]["callsign"], + message_text=reply, ) - msg.send() + pkt.send() # flag message as sent via aprs try: server.add_flags(msgid, ["APRS"]) diff --git a/aprsd/plugins/notify.py b/aprsd/plugins/notify.py index d75a19c..326b487 100644 --- a/aprsd/plugins/notify.py +++ b/aprsd/plugins/notify.py @@ -1,6 +1,6 @@ import logging -from aprsd import messaging, packets, plugin +from aprsd import packets, plugin LOG = logging.getLogger("APRSD") @@ -34,20 +34,21 @@ class NotifySeenPlugin(plugin.APRSDWatchListPluginBase): wl.max_delta(), ), ) - packet_type = packet.packet_type + packet_type = packet.__class__.__name__ # we shouldn't notify the alert user that they are online. if fromcall != notify_callsign: - msg = messaging.TextMessage( - self.config["aprs"]["login"], - notify_callsign, - f"{fromcall} was just seen by type:'{packet_type}'", - # We don't need to keep this around if it doesn't go thru - allow_delay=False, + pkt = packets.MessagePacket( + from_call=self.config["aprsd"]["callsign"], + to_call=notify_callsign, + message_text=( + f"{fromcall} was just seen by type:'{packet_type}'" + ), + _allow_delay=False, ) - return msg + return pkt else: LOG.debug("fromcall and notify_callsign are the same, not notifying") - return messaging.NULL_MESSAGE + return packets.NULL_MESSAGE else: LOG.debug( "Not old enough to notify on callsign '{}' : {} < {}".format( @@ -56,4 +57,4 @@ class NotifySeenPlugin(plugin.APRSDWatchListPluginBase): wl.max_delta(), ), ) - return messaging.NULL_MESSAGE + return packets.NULL_MESSAGE diff --git a/aprsd/plugins/query.py b/aprsd/plugins/query.py index e1fc267..04e6b0c 100644 --- a/aprsd/plugins/query.py +++ b/aprsd/plugins/query.py @@ -2,7 +2,8 @@ import datetime import logging import re -from aprsd import messaging, packets, plugin +from aprsd import packets, plugin +from aprsd.packets import tracker from aprsd.utils import trace @@ -24,10 +25,10 @@ class QueryPlugin(plugin.APRSDRegexCommandPluginBase): message = packet.get("message_text", None) # ack = packet.get("msgNo", "0") - tracker = messaging.MsgTrack() + pkt_tracker = tracker.PacketTrack() now = datetime.datetime.now() reply = "Pending messages ({}) {}".format( - len(tracker), + len(pkt_tracker), now.strftime("%H:%M:%S"), ) @@ -38,11 +39,11 @@ class QueryPlugin(plugin.APRSDRegexCommandPluginBase): # resend last N most recent: "!3" r = re.search(r"^\!([0-9]).*", message) if r is not None: - if len(tracker) > 0: + if len(pkt_tracker) > 0: last_n = r.group(1) - reply = messaging.NULL_MESSAGE + reply = packets.NULL_MESSAGE LOG.debug(reply) - tracker.restart_delayed(count=int(last_n)) + pkt_tracker.restart_delayed(count=int(last_n)) else: reply = "No pending msgs to resend" LOG.debug(reply) @@ -51,10 +52,10 @@ class QueryPlugin(plugin.APRSDRegexCommandPluginBase): # resend all: "!a" r = re.search(r"^\![aA].*", message) if r is not None: - if len(tracker) > 0: - reply = messaging.NULL_MESSAGE + if len(pkt_tracker) > 0: + reply = packets.NULL_MESSAGE LOG.debug(reply) - tracker.restart_delayed() + pkt_tracker.restart_delayed() else: reply = "No pending msgs" LOG.debug(reply) @@ -65,7 +66,7 @@ class QueryPlugin(plugin.APRSDRegexCommandPluginBase): if r is not None: reply = "Deleted ALL pending msgs." LOG.debug(reply) - tracker.flush() + pkt_tracker.flush() return reply return reply diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index 478a7ee..134c12c 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -4,7 +4,7 @@ import time import aprslib -from aprsd import client, messaging, packets, plugin, stats +from aprsd import client, packets, plugin, stats from aprsd.threads import APRSDThread @@ -66,7 +66,6 @@ class APRSDPluginRXThread(APRSDRXThread): def process_packet(self, *args, **kwargs): packet = self._client.decode_packet(*args, **kwargs) # LOG.debug(raw) - #packet = packets.Packet.factory(raw.copy()) packet.log(header="RX Packet") thread = APRSDPluginProcessPacketThread( config=self.config, @@ -186,7 +185,7 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): replied = True for subreply in reply: LOG.debug(f"Sending '{subreply}'") - if isinstance(subreply, messaging.Message): + if isinstance(subreply, packets.Packet): subreply.send() else: msg_pkt = packets.MessagePacket( @@ -195,23 +194,9 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): message_text=subreply, ) msg_pkt.send() - #msg = messaging.TextMessage( - # self.config["aprsd"]["callsign"], - # from_call, - # subreply, - #) - #msg.send() - elif isinstance(reply, messaging.Message): + elif isinstance(reply, packets.Packet): # We have a message based object. - LOG.debug(f"Sending '{reply}'") - # Convert this to the new packet - msg_pkt = packets.MessagePacket( - from_call=reply.fromcall, - to_call=reply.tocall, - message_text=reply._raw_message, - ) - #reply.send() - msg_pkt.send() + reply.send() replied = True else: replied = True @@ -219,7 +204,7 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): # us that they processed the message correctly, but have # nothing to reply with, so we avoid replying with a # usage string - if reply is not messaging.NULL_MESSAGE: + if reply is not packets.NULL_MESSAGE: LOG.debug(f"Sending '{reply}'") msg_pkt = packets.MessagePacket( from_call=self.config["aprsd"]["callsign"], @@ -230,13 +215,6 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): msg_pkt.send() LOG.warning("Calling msg_pkg.send() --- DONE") - #msg = messaging.TextMessage( - # self.config["aprsd"]["callsign"], - # from_call, - # reply, - #) - #msg.send() - # If the message was for us and we didn't have a # response, then we send a usage statement. if to_call == self.config["aprsd"]["callsign"] and not replied: @@ -247,12 +225,6 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): message_text="Unknown command! Send 'help' message for help", ) msg_pkt.send() - #msg = messaging.TextMessage( - # self.config["aprsd"]["callsign"], - # from_call, - # "Unknown command! Send 'help' message for help", - #) - #msg.send() except Exception as ex: LOG.error("Plugin failed!!!") LOG.exception(ex) @@ -265,11 +237,5 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): message_text=reply, ) msg_pkt.send() - #msg = messaging.TextMessage( - # self.config["aprsd"]["callsign"], - # from_call, - # reply, - #) - #msg.send() LOG.debug("Completed process_our_message_packet") From 6030cb394ba72e70c1e3cf161f3ddef39ddf183e Mon Sep 17 00:00:00 2001 From: Hemna Date: Fri, 16 Dec 2022 15:28:31 -0500 Subject: [PATCH 07/21] More messaging -> packets cleanup Fixed the unit tests and the notify plugin --- aprsd/plugins/notify.py | 2 +- tests/cmds/test_webchat.py | 17 ++-- tests/fake.py | 3 +- tests/plugins/test_notify.py | 29 +++---- tests/plugins/test_query.py | 17 ++-- tests/test_messaging.py | 155 ----------------------------------- tests/test_packets.py | 11 +-- tests/test_plugin.py | 11 +-- 8 files changed, 49 insertions(+), 196 deletions(-) delete mode 100644 tests/test_messaging.py diff --git a/aprsd/plugins/notify.py b/aprsd/plugins/notify.py index 326b487..18dc1ef 100644 --- a/aprsd/plugins/notify.py +++ b/aprsd/plugins/notify.py @@ -43,8 +43,8 @@ class NotifySeenPlugin(plugin.APRSDWatchListPluginBase): message_text=( f"{fromcall} was just seen by type:'{packet_type}'" ), - _allow_delay=False, ) + pkt._allow_delay = False return pkt else: LOG.debug("fromcall and notify_callsign are the same, not notifying") diff --git a/tests/cmds/test_webchat.py b/tests/cmds/test_webchat.py index b35a581..7c23859 100644 --- a/tests/cmds/test_webchat.py +++ b/tests/cmds/test_webchat.py @@ -7,8 +7,9 @@ import flask import flask_socketio from aprsd import config as aprsd_config -from aprsd import messaging, packets +from aprsd import packets from aprsd.cmds import webchat # noqa +from aprsd.packets import core from .. import fake @@ -63,12 +64,11 @@ class TestSendMessageCommand(unittest.TestCase): self.assertIsInstance(socketio, flask_socketio.SocketIO) self.assertIsInstance(flask_app, flask.Flask) - @mock.patch("aprsd.messaging.log_message") @mock.patch("aprsd.config.parse_config") - @mock.patch("aprsd.messaging.MsgTrack.remove") + @mock.patch("aprsd.packets.tracker.PacketTrack.remove") @mock.patch("aprsd.cmds.webchat.socketio.emit") def test_process_ack_packet( - self, mock_parse_config, mock_log_message, + self, mock_parse_config, mock_remove, mock_emit, ): config = self._build_config() @@ -76,17 +76,16 @@ class TestSendMessageCommand(unittest.TestCase): packet = fake.fake_packet( message="blah", msg_number=1, - message_format=packets.PACKET_TYPE_ACK, + message_format=core.PACKET_TYPE_ACK, ) socketio = mock.MagicMock() packets.PacketList(config=config) - messaging.MsgTrack(config=config) + packets.PacketTrack(config=config) packets.WatchList(config=config) packets.SeenList(config=config) wcp = webchat.WebChatProcessPacketThread(config, packet, socketio) wcp.process_ack_packet(packet) - mock_log_message.called_once() mock_remove.called_once() mock_emit.called_once() @@ -103,11 +102,11 @@ class TestSendMessageCommand(unittest.TestCase): packet = fake.fake_packet( message="blah", msg_number=1, - message_format=packets.PACKET_TYPE_MESSAGE, + message_format=core.PACKET_TYPE_MESSAGE, ) socketio = mock.MagicMock() packets.PacketList(config=config) - messaging.MsgTrack(config=config) + packets.PacketTrack(config=config) packets.WatchList(config=config) packets.SeenList(config=config) wcp = webchat.WebChatProcessPacketThread(config, packet, socketio) diff --git a/tests/fake.py b/tests/fake.py index 5b2d9e4..1912157 100644 --- a/tests/fake.py +++ b/tests/fake.py @@ -1,4 +1,5 @@ from aprsd import packets, plugin, threads +from aprsd.packets import core FAKE_MESSAGE_TEXT = "fake MeSSage" @@ -11,7 +12,7 @@ def fake_packet( tocall=FAKE_TO_CALLSIGN, message=None, msg_number=None, - message_format=packets.PACKET_TYPE_MESSAGE, + message_format=core.PACKET_TYPE_MESSAGE, ): packet_dict = { "from": fromcall, diff --git a/tests/plugins/test_notify.py b/tests/plugins/test_notify.py index 8f0254f..2c95e6c 100644 --- a/tests/plugins/test_notify.py +++ b/tests/plugins/test_notify.py @@ -2,7 +2,7 @@ from unittest import mock from aprsd import client from aprsd import config as aprsd_config -from aprsd import messaging, packets +from aprsd import packets from aprsd.plugins import notify as notify_plugin from .. import fake, test_plugin @@ -28,7 +28,8 @@ class TestWatchListPlugin(test_plugin.TestPlugin): default_wl = aprsd_config.DEFAULT_CONFIG_DICT["aprsd"]["watch_list"] _config["ham"]["callsign"] = self.fromcall - _config["aprs"]["login"] = fake.FAKE_TO_CALLSIGN + _config["aprsd"]["callsign"] = self.fromcall + _config["aprs"]["login"] = self.fromcall _config["services"]["aprs.fi"]["apiKey"] = "something" # Set the watchlist specific config options @@ -62,7 +63,7 @@ class TestAPRSDWatchListPluginBase(TestWatchListPlugin): msg_number=1, ) actual = plugin.filter(packet) - expected = messaging.NULL_MESSAGE + expected = packets.NULL_MESSAGE self.assertEqual(expected, actual) @mock.patch("aprsd.client.ClientFactory", autospec=True) @@ -78,7 +79,7 @@ class TestAPRSDWatchListPluginBase(TestWatchListPlugin): msg_number=1, ) actual = plugin.filter(packet) - expected = messaging.NULL_MESSAGE + expected = packets.NULL_MESSAGE self.assertEqual(expected, actual) @@ -94,7 +95,7 @@ class TestNotifySeenPlugin(TestWatchListPlugin): msg_number=1, ) actual = plugin.filter(packet) - expected = messaging.NULL_MESSAGE + expected = packets.NULL_MESSAGE self.assertEqual(expected, actual) @mock.patch("aprsd.client.ClientFactory", autospec=True) @@ -109,7 +110,7 @@ class TestNotifySeenPlugin(TestWatchListPlugin): msg_number=1, ) actual = plugin.filter(packet) - expected = messaging.NULL_MESSAGE + expected = packets.NULL_MESSAGE self.assertEqual(expected, actual) @mock.patch("aprsd.client.ClientFactory", autospec=True) @@ -130,7 +131,7 @@ class TestNotifySeenPlugin(TestWatchListPlugin): msg_number=1, ) actual = plugin.filter(packet) - expected = messaging.NULL_MESSAGE + expected = packets.NULL_MESSAGE self.assertEqual(expected, actual) @mock.patch("aprsd.client.ClientFactory", autospec=True) @@ -152,7 +153,7 @@ class TestNotifySeenPlugin(TestWatchListPlugin): msg_number=1, ) actual = plugin.filter(packet) - expected = messaging.NULL_MESSAGE + expected = packets.NULL_MESSAGE self.assertEqual(expected, actual) @mock.patch("aprsd.client.ClientFactory", autospec=True) @@ -160,7 +161,7 @@ class TestNotifySeenPlugin(TestWatchListPlugin): def test_callsign_in_watchlist_old_send_alert(self, mock_is_old, mock_factory): client.factory = mock_factory mock_is_old.return_value = True - notify_callsign = "KFAKE" + notify_callsign = fake.FAKE_TO_CALLSIGN fromcall = "WB4BOR" config = self._config( watchlist_enabled=True, @@ -175,11 +176,11 @@ class TestNotifySeenPlugin(TestWatchListPlugin): message="ping", msg_number=1, ) - packet_type = packets.get_packet_type(packet) + packet_type = packet.__class__.__name__ actual = plugin.filter(packet) msg = f"{fromcall} was just seen by type:'{packet_type}'" - self.assertIsInstance(actual, messaging.TextMessage) - self.assertEqual(fake.FAKE_TO_CALLSIGN, actual.fromcall) - self.assertEqual(notify_callsign, actual.tocall) - self.assertEqual(msg, actual.message) + self.assertIsInstance(actual, packets.MessagePacket) + self.assertEqual(fake.FAKE_FROM_CALLSIGN, actual.from_call) + self.assertEqual(notify_callsign, actual.to_call) + self.assertEqual(msg, actual.message_text) diff --git a/tests/plugins/test_query.py b/tests/plugins/test_query.py index 0a8d5db..f16fd8d 100644 --- a/tests/plugins/test_query.py +++ b/tests/plugins/test_query.py @@ -1,13 +1,14 @@ from unittest import mock -from aprsd import messaging +from aprsd import packets +from aprsd.packets import tracker from aprsd.plugins import query as query_plugin from .. import fake, test_plugin class TestQueryPlugin(test_plugin.TestPlugin): - @mock.patch("aprsd.messaging.MsgTrack.flush") + @mock.patch("aprsd.packets.tracker.PacketTrack.flush") def test_query_flush(self, mock_flush): packet = fake.fake_packet(message="!delete") query = query_plugin.QueryPlugin(self.config) @@ -17,9 +18,9 @@ class TestQueryPlugin(test_plugin.TestPlugin): mock_flush.assert_called_once() self.assertEqual(expected, actual) - @mock.patch("aprsd.messaging.MsgTrack.restart_delayed") + @mock.patch("aprsd.packets.tracker.PacketTrack.restart_delayed") def test_query_restart_delayed(self, mock_restart): - track = messaging.MsgTrack() + track = tracker.PacketTrack() track.data = {} packet = fake.fake_packet(message="!4") query = query_plugin.QueryPlugin(self.config) @@ -31,7 +32,11 @@ class TestQueryPlugin(test_plugin.TestPlugin): mock_restart.reset_mock() # add a message - msg = messaging.TextMessage(self.fromcall, "testing", self.ack) - track.add(msg) + pkt = packets.MessagePacket( + from_call=self.fromcall, + to_call="testing", + msgNo=self.ack, + ) + track.add(pkt) actual = query.filter(packet) mock_restart.assert_called_once() diff --git a/tests/test_messaging.py b/tests/test_messaging.py deleted file mode 100644 index 7fd6136..0000000 --- a/tests/test_messaging.py +++ /dev/null @@ -1,155 +0,0 @@ -import datetime -import unittest -from unittest import mock - -from aprsd import messaging - - -class TestMessageTrack(unittest.TestCase): - - def setUp(self) -> None: - config = {} - messaging.MsgTrack(config=config) - - def _clean_track(self): - track = messaging.MsgTrack() - track.data = {} - track.total_messages_tracked = 0 - return track - - def test_create(self): - track1 = messaging.MsgTrack() - track2 = messaging.MsgTrack() - - self.assertEqual(track1, track2) - - def test_add(self): - track = self._clean_track() - fromcall = "KFART" - tocall = "KHELP" - message = "somthing" - msg = messaging.TextMessage(fromcall, tocall, message) - - track.add(msg) - self.assertEqual(msg, track.get(msg.id)) - - def test_remove(self): - track = self._clean_track() - fromcall = "KFART" - tocall = "KHELP" - message = "somthing" - msg = messaging.TextMessage(fromcall, tocall, message) - track.add(msg) - - track.remove(msg.id) - self.assertEqual(None, track.get(msg.id)) - - def test_len(self): - """Test getting length of tracked messages.""" - track = self._clean_track() - fromcall = "KFART" - tocall = "KHELP" - message = "somthing" - msg = messaging.TextMessage(fromcall, tocall, message) - track.add(msg) - self.assertEqual(1, len(track)) - msg2 = messaging.TextMessage(tocall, fromcall, message) - track.add(msg2) - self.assertEqual(2, len(track)) - - track.remove(msg.id) - self.assertEqual(1, len(track)) - - @mock.patch("aprsd.messaging.TextMessage.send") - def test__resend(self, mock_send): - """Test the _resend method.""" - track = self._clean_track() - fromcall = "KFART" - tocall = "KHELP" - message = "somthing" - msg = messaging.TextMessage(fromcall, tocall, message) - msg.last_send_attempt = 3 - track.add(msg) - - track._resend(msg) - msg.send.assert_called_with() - self.assertEqual(0, msg.last_send_attempt) - - @mock.patch("aprsd.messaging.TextMessage.send") - def test_restart_delayed(self, mock_send): - """Test the _resend method.""" - track = self._clean_track() - fromcall = "KFART" - tocall = "KHELP" - message1 = "something" - message2 = "something another" - message3 = "something another again" - - mock1_send = mock.MagicMock() - mock2_send = mock.MagicMock() - mock3_send = mock.MagicMock() - - msg1 = messaging.TextMessage(fromcall, tocall, message1) - msg1.last_send_attempt = 3 - msg1.last_send_time = datetime.datetime.now() - msg1.send = mock1_send - track.add(msg1) - - msg2 = messaging.TextMessage(tocall, fromcall, message2) - msg2.last_send_attempt = 3 - msg2.last_send_time = datetime.datetime.now() - msg2.send = mock2_send - track.add(msg2) - - track.restart_delayed(count=None) - msg1.send.assert_called_once() - self.assertEqual(0, msg1.last_send_attempt) - msg2.send.assert_called_once() - self.assertEqual(0, msg2.last_send_attempt) - - msg1.last_send_attempt = 3 - msg1.send.reset_mock() - msg2.last_send_attempt = 3 - msg2.send.reset_mock() - - track.restart_delayed(count=1) - msg1.send.assert_not_called() - msg2.send.assert_called_once() - self.assertEqual(3, msg1.last_send_attempt) - self.assertEqual(0, msg2.last_send_attempt) - - msg3 = messaging.TextMessage(tocall, fromcall, message3) - msg3.last_send_attempt = 3 - msg3.last_send_time = datetime.datetime.now() - msg3.send = mock3_send - track.add(msg3) - - msg1.last_send_attempt = 3 - msg1.send.reset_mock() - msg2.last_send_attempt = 3 - msg2.send.reset_mock() - msg3.last_send_attempt = 3 - msg3.send.reset_mock() - - track.restart_delayed(count=2) - msg1.send.assert_not_called() - msg2.send.assert_called_once() - msg3.send.assert_called_once() - self.assertEqual(3, msg1.last_send_attempt) - self.assertEqual(0, msg2.last_send_attempt) - self.assertEqual(0, msg3.last_send_attempt) - - msg1.last_send_attempt = 3 - msg1.send.reset_mock() - msg2.last_send_attempt = 3 - msg2.send.reset_mock() - msg3.last_send_attempt = 3 - msg3.send.reset_mock() - - track.restart_delayed(count=2, most_recent=False) - msg1.send.assert_called_once() - msg2.send.assert_called_once() - msg3.send.assert_not_called() - self.assertEqual(0, msg1.last_send_attempt) - self.assertEqual(0, msg2.last_send_attempt) - self.assertEqual(3, msg3.last_send_attempt) diff --git a/tests/test_packets.py b/tests/test_packets.py index 0d0232f..2aeba8c 100644 --- a/tests/test_packets.py +++ b/tests/test_packets.py @@ -1,6 +1,7 @@ import unittest from aprsd import packets +from aprsd.packets import core from . import fake @@ -13,7 +14,7 @@ class TestPluginBase(unittest.TestCase): to_call=fake.FAKE_TO_CALLSIGN, message=None, msg_number=None, - message_format=packets.PACKET_TYPE_MESSAGE, + message_format=core.PACKET_TYPE_MESSAGE, ): packet_dict = { "from": from_call, @@ -56,9 +57,9 @@ class TestPluginBase(unittest.TestCase): pkt = packets.Packet.factory(pkt_dict) self.assertIsInstance(pkt, packets.MessagePacket) - self.assertEqual(pkt_dict["from"], pkt.from_call) - self.assertEqual(pkt_dict["to"], pkt.to_call) - self.assertEqual(pkt_dict["addresse"], pkt.addresse) + self.assertEqual(fake.FAKE_FROM_CALLSIGN, pkt.from_call) + self.assertEqual(fake.FAKE_TO_CALLSIGN, pkt.to_call) + self.assertEqual(fake.FAKE_TO_CALLSIGN, pkt.addresse) pkt_dict["symbol"] = "_" pkt_dict["weather"] = { @@ -68,6 +69,6 @@ class TestPluginBase(unittest.TestCase): "pressure": 1095.12, "comment": "Home!", } - pkt_dict["format"] = packets.PACKET_TYPE_UNCOMPRESSED + pkt_dict["format"] = core.PACKET_TYPE_UNCOMPRESSED pkt = packets.Packet.factory(pkt_dict) self.assertIsInstance(pkt, packets.WeatherPacket) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 9e59eee..007dddb 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2,7 +2,8 @@ import unittest from unittest import mock from aprsd import config as aprsd_config -from aprsd import messaging, packets, stats +from aprsd import packets, stats +from aprsd.packets import core from . import fake @@ -18,7 +19,7 @@ class TestPlugin(unittest.TestCase): stats.APRSDStats._instance = None packets.WatchList._instance = None packets.SeenList._instance = None - messaging.MsgTrack._instance = None + packets.PacketTrack._instance = None self.config = None def config_and_init(self, config=None): @@ -34,7 +35,7 @@ class TestPlugin(unittest.TestCase): stats.APRSDStats(self.config) packets.WatchList(config=self.config) packets.SeenList(config=self.config) - messaging.MsgTrack(config=self.config) + packets.PacketTrack(config=self.config) class TestPluginBase(TestPlugin): @@ -89,7 +90,7 @@ class TestPluginBase(TestPlugin): packet = fake.fake_packet( message="F", - message_format=packets.PACKET_TYPE_MICE, + message_format=core.PACKET_TYPE_MICE, ) expected = None actual = p.filter(packet) @@ -98,7 +99,7 @@ class TestPluginBase(TestPlugin): packet = fake.fake_packet( message="f", - message_format=packets.PACKET_TYPE_ACK, + message_format=core.PACKET_TYPE_ACK, ) expected = None actual = p.filter(packet) From f1de7bc6815be2ee129a27ff4a898abbc48a5b51 Mon Sep 17 00:00:00 2001 From: Hemna Date: Fri, 16 Dec 2022 15:54:40 -0500 Subject: [PATCH 08/21] Fix packets timestamp to int Python's default timestamp is a float. APRS packet expect to have an old style unix integer timestamp. --- aprsd/packets/core.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index a52f19b..64f3da6 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -26,6 +26,11 @@ PACKET_TYPE_BEACON = "beacon" PACKET_TYPE_UNCOMPRESSED = "uncompressed" +def _int_timestamp(): + """Build a unix style timestamp integer""" + return int(round(time.time())) + + @dataclass() class Packet(metaclass=abc.ABCMeta): from_call: str @@ -34,7 +39,7 @@ class Packet(metaclass=abc.ABCMeta): format: str = None msgNo: str = None # noqa: N815 packet_type: str = None - timestamp: float = field(default_factory=time.time) + timestamp: float = field(default_factory=_int_timestamp) raw: str = None _raw_dict: dict = field(repr=False, default_factory=lambda: {}) _retry_count = 3 From c201c93b5d01f73eb2a308d309103e114e5f6ff1 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 17 Dec 2022 18:00:26 -0500 Subject: [PATCH 09/21] Cleaned up packet transmit class attributes This patch cleans up the Packet class attributes used to keep track of how many times packets have been sent and the last time they were sent. This is used by the PacketTracker and the tx threads for transmitting packets --- aprsd/packets/core.py | 68 +++++++++++++++++++++++++---------------- aprsd/plugins/notify.py | 3 +- aprsd/threads/rx.py | 3 +- aprsd/threads/tx.py | 48 ++++++++++++++++------------- 4 files changed, 70 insertions(+), 52 deletions(-) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 64f3da6..c476472 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -31,31 +31,41 @@ def _int_timestamp(): return int(round(time.time())) -@dataclass() +def _init_msgNo(): # noqa: N802 + """For some reason __post__init doesn't get called. + + So in order to initialize the msgNo field in the packet + we use this workaround. + """ + c = counter.PacketCounter() + c.increment() + return c.value + + +@dataclass class Packet(metaclass=abc.ABCMeta): from_call: str to_call: str addresse: str = None format: str = None - msgNo: str = None # noqa: N815 + msgNo: str = field(default_factory=_init_msgNo) # noqa: N815 packet_type: str = None timestamp: float = field(default_factory=_int_timestamp) + # Holds the raw text string to be sent over the wire + # or holds the raw string from input packet raw: str = None - _raw_dict: dict = field(repr=False, default_factory=lambda: {}) - _retry_count = 3 - _last_send_time = 0 - _last_send_attempt = 0 + raw_dict: dict = field(repr=False, default_factory=lambda: {}) + + # Fields related to sending packets out + send_count: int = field(repr=False, default=1) + retry_count: int = field(repr=False, default=3) + last_send_time: datetime.timedelta = field(repr=False, default=None) + last_send_attempt: int = field(repr=False, default=0) # Do we allow this packet to be saved to send later? - _allow_delay = True + allow_delay: bool = field(repr=False, default=True) - _transport = None - _raw_message = None - - def __post__init(self): - if not self.msgNo: - c = counter.PacketCounter() - c.increment() - self.msgNo = c.value + def __post__init__(self): + LOG.warning(f"POST INIT {self}") def get(self, key, default=None): """Emulate a getter on a dict.""" @@ -76,7 +86,7 @@ class Packet(metaclass=abc.ABCMeta): @staticmethod def factory(raw_packet): raw = raw_packet - raw["_raw_dict"] = raw.copy() + raw["raw_dict"] = raw.copy() translate_fields = { "from": "from_call", "to": "to_call", @@ -110,15 +120,16 @@ class Packet(metaclass=abc.ABCMeta): """LOG a packet to the logfile.""" asdict(self) log_list = ["\n"] + name = self.__class__.__name__ if header: - if isinstance(self, AckPacket): + if isinstance(self, AckPacket) and "tx" in header.lower(): log_list.append( - f"{header} ___________" - f"(TX:{self._send_count} of {self._retry_count})", + f"{header}____________({name}__" + f"TX:{self.send_count} of {self.retry_count})", ) else: - log_list.append(f"{header} _______________") - log_list.append(f" Packet : {self.__class__.__name__}") + log_list.append(f"{header}____________({name})") + # log_list.append(f" Packet : {self.__class__.__name__}") log_list.append(f" Raw : {self.raw}") if self.to_call: log_list.append(f" To : {self.to_call}") @@ -137,7 +148,7 @@ class Packet(metaclass=abc.ABCMeta): if self.msgNo: log_list.append(f" Msg # : {self.msgNo}") - log_list.append(f"{header} _______________ Complete") + log_list.append(f"{header}____________({name})") LOG.info("\n".join(log_list)) LOG.debug(self) @@ -165,12 +176,12 @@ class Packet(metaclass=abc.ABCMeta): cl = aprsis_client else: cl = client.factory.create().client - self.log(header="Sending Message Direct") + self.log(header="TX Message Direct") cl.send(self.raw) stats.APRSDStats().msgs_tx_inc() -@dataclass() +@dataclass class PathPacket(Packet): path: List[str] = field(default_factory=list) via: str = None @@ -179,10 +190,13 @@ class PathPacket(Packet): raise NotImplementedError -@dataclass() +@dataclass class AckPacket(PathPacket): response: str = None - _send_count = 1 + + def __post__init__(self): + if self.response: + LOG.warning("Response set!") def _build_raw(self): """Build the self.raw which is what is sent over the air.""" @@ -200,7 +214,7 @@ class AckPacket(PathPacket): thread.start() -@dataclass() +@dataclass class MessagePacket(PathPacket): message_text: str = None diff --git a/aprsd/plugins/notify.py b/aprsd/plugins/notify.py index 18dc1ef..15a9b22 100644 --- a/aprsd/plugins/notify.py +++ b/aprsd/plugins/notify.py @@ -43,8 +43,9 @@ class NotifySeenPlugin(plugin.APRSDWatchListPluginBase): message_text=( f"{fromcall} was just seen by type:'{packet_type}'" ), + allow_delay=False, ) - pkt._allow_delay = False + # pkt.allow_delay = False return pkt else: LOG.debug("fromcall and notify_callsign are the same, not notifying") diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index 134c12c..796aac3 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -66,7 +66,7 @@ class APRSDPluginRXThread(APRSDRXThread): def process_packet(self, *args, **kwargs): packet = self._client.decode_packet(*args, **kwargs) # LOG.debug(raw) - packet.log(header="RX Packet") + packet.log(header="RX") thread = APRSDPluginProcessPacketThread( config=self.config, packet=packet, @@ -92,7 +92,6 @@ class APRSDProcessPacketThread(APRSDThread): def process_ack_packet(self, packet): ack_num = packet.msgNo LOG.info(f"Got ack for message {ack_num}") - packet.log("RXACK") pkt_tracker = packets.PacketTrack() pkt_tracker.remove(ack_num) stats.APRSDStats().ack_rx_inc() diff --git a/aprsd/threads/tx.py b/aprsd/threads/tx.py index 8df838c..9f88b55 100644 --- a/aprsd/threads/tx.py +++ b/aprsd/threads/tx.py @@ -11,6 +11,8 @@ LOG = logging.getLogger("APRSD") class SendPacketThread(aprsd_threads.APRSDThread): + loop_count: int = 1 + def __init__(self, packet): self.packet = packet name = self.packet.raw[:5] @@ -19,7 +21,6 @@ class SendPacketThread(aprsd_threads.APRSDThread): pkt_tracker.add(packet) def loop(self): - LOG.debug("TX Loop") """Loop until a message is acked or it gets delayed. We only sleep for 5 seconds between each loop run, so @@ -39,20 +40,20 @@ class SendPacketThread(aprsd_threads.APRSDThread): return False else: send_now = False - if packet._last_send_attempt == packet._retry_count: + if packet.last_send_attempt == packet.retry_count: # we reached the send limit, don't send again # TODO(hemna) - Need to put this in a delayed queue? LOG.info("Message Send Complete. Max attempts reached.") - if not packet._allow_delay: + if not packet.allow_delay: pkt_tracker.remove(packet.msgNo) return False # Message is still outstanding and needs to be acked. - if packet._last_send_time: + if packet.last_send_time: # Message has a last send time tracking now = datetime.datetime.now() - sleeptime = (packet._last_send_attempt + 1) * 31 - delta = now - packet._last_send_time + sleeptime = (packet.last_send_attempt + 1) * 31 + delta = now - packet.last_send_time if delta > datetime.timedelta(seconds=sleeptime): # It's time to try to send it again send_now = True @@ -62,59 +63,62 @@ class SendPacketThread(aprsd_threads.APRSDThread): if send_now: # no attempt time, so lets send it, and start # tracking the time. - packet.log("Sending Message") + packet.log("TX") cl = client.factory.create().client cl.send(packet.raw) stats.APRSDStats().msgs_tx_inc() packet_list.PacketList().add(packet) - packet._last_send_time = datetime.datetime.now() - packet._last_send_attempt += 1 + packet.last_send_time = datetime.datetime.now() + packet.last_send_attempt += 1 - time.sleep(5) + time.sleep(1) # Make sure we get called again. + self.loop_count += 1 return True class SendAckThread(aprsd_threads.APRSDThread): + loop_count: int = 1 + def __init__(self, packet): self.packet = packet super().__init__(f"SendAck-{self.packet.msgNo}") - self._loop_cnt = 1 def loop(self): """Separate thread to send acks with retries.""" send_now = False - if self.packet._last_send_attempt == self.packet._retry_count: + if self.packet.last_send_attempt == self.packet.retry_count: # we reached the send limit, don't send again # TODO(hemna) - Need to put this in a delayed queue? LOG.info("Ack Send Complete. Max attempts reached.") return False - if self.packet._last_send_time: + if self.packet.last_send_time: # Message has a last send time tracking now = datetime.datetime.now() # aprs duplicate detection is 30 secs? # (21 only sends first, 28 skips middle) - sleeptime = 31 - delta = now - self.packet._last_send_time - if delta > datetime.timedelta(seconds=sleeptime): + sleep_time = 31 + delta = now - self.packet.last_send_time + if delta > datetime.timedelta(seconds=sleep_time): # It's time to try to send it again send_now = True - elif self._loop_cnt % 5 == 0: + elif self.loop_count % 10 == 0: LOG.debug(f"Still wating. {delta}") else: send_now = True if send_now: cl = client.factory.create().client - self.packet.log("Sending ACK") + self.packet.log("TX") cl.send(self.packet.raw) - self.packet._send_count += 1 + self.packet.send_count += 1 stats.APRSDStats().ack_tx_inc() packet_list.PacketList().add(self.packet) - self.packet._last_send_attempt += 1 - self.packet._last_send_time = datetime.datetime.now() + self.packet.last_send_attempt += 1 + self.packet.last_send_time = datetime.datetime.now() + time.sleep(1) - self._loop_cnt += 1 + self.loop_count += 1 return True From 1187f1ed738890b63201c136e607a88781a8b36e Mon Sep 17 00:00:00 2001 From: Hemna Date: Sat, 17 Dec 2022 20:02:49 -0500 Subject: [PATCH 10/21] Make tracking objectstores work w/o initializing This changes the objectstore to test to see if the config has been set or not. if not, then it doesn't try to save/load from disk. --- aprsd/cmds/listen.py | 8 +-- aprsd/cmds/webchat.py | 2 +- aprsd/flask.py | 2 +- aprsd/packets/packet_list.py | 29 ++++++----- aprsd/packets/seen_list.py | 8 ++- aprsd/packets/tracker.py | 11 +++-- aprsd/packets/watch_list.py | 3 ++ aprsd/stats.py | 8 +++ aprsd/threads/keep_alive.py | 7 ++- aprsd/threads/rx.py | 2 +- aprsd/threads/tx.py | 4 +- aprsd/utils/objectstore.py | 94 +++++++++++++++++++++--------------- aprsd/utils/ring_buffer.py | 3 ++ 13 files changed, 115 insertions(+), 66 deletions(-) diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index e889445..61ded19 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -40,7 +40,8 @@ def signal_handler(sig, frame): class APRSDListenThread(rx.APRSDRXThread): def process_packet(self, *args, **kwargs): packet = self._client.decode_packet(*args, **kwargs) - packet.log(header="RX Packet") + packet.log(header="RX") + packets.PacketList().rx(packet) @cli.command() @@ -113,9 +114,6 @@ def listen( # Try and load saved MsgTrack list LOG.debug("Loading saved MsgTrack object.") - packets.PacketTrack(config=config).load() - packets.WatchList(config=config).load() - packets.SeenList(config=config).load() # Initialize the client factory and create # The correct client object ready for use @@ -133,8 +131,6 @@ def listen( LOG.debug(f"Filter by '{filter}'") aprs_client.set_filter(filter) - packets.PacketList(config=config) - keepalive = threads.KeepAliveThread(config=config) keepalive.start() diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index 58a8eba..1f19412 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -181,7 +181,7 @@ class WebChatProcessPacketThread(rx.APRSDProcessPacketThread): packet.get("addresse", None) fromcall = packet.from_call - packets.PacketList().add(packet) + packets.PacketList().rx(packet) stats.APRSDStats().msgs_rx_inc() message = packet.get("message_text", None) msg = { diff --git a/aprsd/flask.py b/aprsd/flask.py index 76865c1..2b85857 100644 --- a/aprsd/flask.py +++ b/aprsd/flask.py @@ -209,7 +209,7 @@ class SendMessageThread(threads.APRSDRXThread): def process_our_message_packet(self, packet): global socketio - packets.PacketList().add(packet) + packets.PacketList().rx(packet) stats.APRSDStats().msgs_rx_inc() msg_number = packet.msgNo SentMessages().reply(self.packet.msgNo, packet) diff --git a/aprsd/packets/packet_list.py b/aprsd/packets/packet_list.py index 89c1fd4..db850c6 100644 --- a/aprsd/packets/packet_list.py +++ b/aprsd/packets/packet_list.py @@ -1,6 +1,5 @@ import logging import threading -import time import wrapt @@ -18,32 +17,40 @@ class PacketList: lock = threading.Lock() config = None - packet_list = utils.RingBuffer(1000) + packet_list: utils.RingBuffer = utils.RingBuffer(1000) - total_recv = 0 - total_tx = 0 + total_recv: int = 0 + total_tx: int = 0 def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = super().__new__(cls) - cls._instance.config = kwargs["config"] + if "config" in kwargs: + cls._instance.config = kwargs["config"] return cls._instance def __init__(self, config=None): if config: self.config = config + def _is_initialized(self): + return self.config is not None + @wrapt.synchronized(lock) def __iter__(self): return iter(self.packet_list) @wrapt.synchronized(lock) - def add(self, packet): - packet.ts = time.time() - if packet.from_call == self.config["aprs"]["login"]: - self.total_tx += 1 - else: - self.total_recv += 1 + def rx(self, packet): + """Add a packet that was received.""" + self.total_recv += 1 + self.packet_list.append(packet) + seen_list.SeenList().update_seen(packet) + + @wrapt.synchronized(lock) + def tx(self, packet): + """Add a packet that was received.""" + self.total_tx += 1 self.packet_list.append(packet) seen_list.SeenList().update_seen(packet) diff --git a/aprsd/packets/seen_list.py b/aprsd/packets/seen_list.py index 3c9e1bf..d68a933 100644 --- a/aprsd/packets/seen_list.py +++ b/aprsd/packets/seen_list.py @@ -15,18 +15,22 @@ class SeenList(objectstore.ObjectStoreMixin): _instance = None lock = threading.Lock() - data = {} + data: dict = {} config = None def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = super().__new__(cls) if "config" in kwargs: - cls._instance.config = kwargs["config"] + if "config" in kwargs: + cls._instance.config = kwargs["config"] cls._instance._init_store() cls._instance.data = {} return cls._instance + def is_initialized(self): + return self.config is not None + @wrapt.synchronized(lock) def update_seen(self, packet): callsign = None diff --git a/aprsd/packets/tracker.py b/aprsd/packets/tracker.py index 82ff3f3..8ec7b67 100644 --- a/aprsd/packets/tracker.py +++ b/aprsd/packets/tracker.py @@ -23,18 +23,23 @@ class PacketTrack(objectstore.ObjectStoreMixin): _instance = None _start_time = None lock = threading.Lock() + config = None - data = {} - total_tracked = 0 + data: dict = {} + total_tracked: int = 0 def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = super().__new__(cls) cls._instance._start_time = datetime.datetime.now() - cls._instance.config = kwargs["config"] + if "config" in kwargs: + cls._instance.config = kwargs["config"] cls._instance._init_store() return cls._instance + def is_initialized(self): + return self.config is not None + @wrapt.synchronized(lock) def __getitem__(self, name): return self.data[name] diff --git a/aprsd/packets/watch_list.py b/aprsd/packets/watch_list.py index 54c3995..c1da79f 100644 --- a/aprsd/packets/watch_list.py +++ b/aprsd/packets/watch_list.py @@ -47,6 +47,9 @@ class WatchList(objectstore.ObjectStoreMixin): ), } + def is_initialized(self): + return self.config is not None + def is_enabled(self): if self.config and "watch_list" in self.config["aprsd"]: return self.config["aprsd"]["watch_list"].get("enabled", False) diff --git a/aprsd/stats.py b/aprsd/stats.py index 8562fc3..cc8dc8f 100644 --- a/aprsd/stats.py +++ b/aprsd/stats.py @@ -90,6 +90,14 @@ class APRSDStats: def set_aprsis_keepalive(self): self._aprsis_keepalive = datetime.datetime.now() + def rx_packet(self, packet): + if isinstance(packet, packets.MessagePacket): + self.msgs_rx_inc() + elif isinstance(packet, packets.MicEPacket): + self.msgs_mice_inc() + elif isinstance(packet, packets.AckPacket): + self.ack_rx_inc() + @wrapt.synchronized(lock) @property def msgs_tx(self): diff --git a/aprsd/threads/keep_alive.py b/aprsd/threads/keep_alive.py index cfbdbc9..4813663 100644 --- a/aprsd/threads/keep_alive.py +++ b/aprsd/threads/keep_alive.py @@ -45,6 +45,11 @@ class KeepAliveThread(APRSDThread): except KeyError: login = self.config["ham"]["callsign"] + if pkt_tracker.is_initialized(): + tracked_packets = len(pkt_tracker) + else: + tracked_packets = 0 + keepalive = ( "{} - Uptime {} RX:{} TX:{} Tracker:{} Msgs TX:{} RX:{} " "Last:{} Email: {} - RAM Current:{} Peak:{} Threads:{}" @@ -53,7 +58,7 @@ class KeepAliveThread(APRSDThread): utils.strfdelta(stats_obj.uptime), pl.total_recv, pl.total_tx, - len(pkt_tracker), + tracked_packets, stats_obj.msgs_tx, stats_obj.msgs_rx, last_msg_time, diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index 796aac3..226248f 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -67,6 +67,7 @@ class APRSDPluginRXThread(APRSDRXThread): packet = self._client.decode_packet(*args, **kwargs) # LOG.debug(raw) packet.log(header="RX") + packets.PacketList().rx(packet) thread = APRSDPluginProcessPacketThread( config=self.config, packet=packet, @@ -101,7 +102,6 @@ class APRSDProcessPacketThread(APRSDThread): """Process a packet received from aprs-is server.""" LOG.debug(f"RXPKT-LOOP {self._loop_cnt}") packet = self.packet - packets.PacketList().add(packet) our_call = self.config["aprsd"]["callsign"].lower() from_call = packet.from_call diff --git a/aprsd/threads/tx.py b/aprsd/threads/tx.py index 9f88b55..b8fca07 100644 --- a/aprsd/threads/tx.py +++ b/aprsd/threads/tx.py @@ -67,7 +67,7 @@ class SendPacketThread(aprsd_threads.APRSDThread): cl = client.factory.create().client cl.send(packet.raw) stats.APRSDStats().msgs_tx_inc() - packet_list.PacketList().add(packet) + packet_list.PacketList().tx(packet) packet.last_send_time = datetime.datetime.now() packet.last_send_attempt += 1 @@ -115,7 +115,7 @@ class SendAckThread(aprsd_threads.APRSDThread): cl.send(self.packet.raw) self.packet.send_count += 1 stats.APRSDStats().ack_tx_inc() - packet_list.PacketList().add(self.packet) + packet_list.PacketList().tx(self.packet) self.packet.last_send_attempt += 1 self.packet.last_send_time = datetime.datetime.now() diff --git a/aprsd/utils/objectstore.py b/aprsd/utils/objectstore.py index eaeaf73..4349213 100644 --- a/aprsd/utils/objectstore.py +++ b/aprsd/utils/objectstore.py @@ -1,3 +1,4 @@ +import abc import logging import os import pathlib @@ -9,7 +10,7 @@ from aprsd import config as aprsd_config LOG = logging.getLogger("APRSD") -class ObjectStoreMixin: +class ObjectStoreMixin(metaclass=abc.ABCMeta): """Class 'MIXIN' intended to save/load object data. The asumption of how this mixin is used: @@ -23,6 +24,13 @@ class ObjectStoreMixin: When APRSD Starts, it calls load() aprsd server -f (flush) will wipe all saved objects. """ + @abc.abstractmethod + def is_initialized(self): + """Return True if the class has been setup correctly. + + If this returns False, the ObjectStore doesn't save anything. + + """ def __len__(self): return len(self.data) @@ -36,13 +44,16 @@ class ObjectStoreMixin: return self.data[id] def _init_store(self): - sl = self._save_location() - if not os.path.exists(sl): - LOG.warning(f"Save location {sl} doesn't exist") - try: - os.makedirs(sl) - except Exception as ex: - LOG.exception(ex) + if self.is_initialized(): + sl = self._save_location() + if not os.path.exists(sl): + LOG.warning(f"Save location {sl} doesn't exist") + try: + os.makedirs(sl) + except Exception as ex: + LOG.exception(ex) + else: + LOG.warning(f"{self.__class__.__name__} is not initialized") def _save_location(self): save_location = self.config.get("aprsd.save_location", None) @@ -68,38 +79,45 @@ class ObjectStoreMixin: def save(self): """Save any queued to disk?""" - if len(self) > 0: - LOG.info(f"{self.__class__.__name__}::Saving {len(self)} entries to disk at {self._save_location()}") - with open(self._save_filename(), "wb+") as fp: - pickle.dump(self._dump(), fp) - else: - LOG.debug( - "{} Nothing to save, flushing old save file '{}'".format( - self.__class__.__name__, - self._save_filename(), - ), - ) - self.flush() + if self.is_initialized(): + if len(self) > 0: + LOG.info( + f"{self.__class__.__name__}::Saving" + f" {len(self)} entries to disk at" + f"{self._save_location()}", + ) + with open(self._save_filename(), "wb+") as fp: + pickle.dump(self._dump(), fp) + else: + LOG.debug( + "{} Nothing to save, flushing old save file '{}'".format( + self.__class__.__name__, + self._save_filename(), + ), + ) + self.flush() def load(self): - if os.path.exists(self._save_filename()): - try: - with open(self._save_filename(), "rb") as fp: - raw = pickle.load(fp) - if raw: - self.data = raw - LOG.debug( - f"{self.__class__.__name__}::Loaded {len(self)} entries from disk.", - ) - LOG.debug(f"{self.data}") - except (pickle.UnpicklingError, Exception) as ex: - LOG.error(f"Failed to UnPickle {self._save_filename()}") - LOG.error(ex) - self.data = {} + if self.is_initialized(): + if os.path.exists(self._save_filename()): + try: + with open(self._save_filename(), "rb") as fp: + raw = pickle.load(fp) + if raw: + self.data = raw + LOG.debug( + f"{self.__class__.__name__}::Loaded {len(self)} entries from disk.", + ) + LOG.debug(f"{self.data}") + except (pickle.UnpicklingError, Exception) as ex: + LOG.error(f"Failed to UnPickle {self._save_filename()}") + LOG.error(ex) + self.data = {} def flush(self): """Nuke the old pickle file that stored the old results from last aprsd run.""" - if os.path.exists(self._save_filename()): - pathlib.Path(self._save_filename()).unlink() - with self.lock: - self.data = {} + if self.is_initialized(): + if os.path.exists(self._save_filename()): + pathlib.Path(self._save_filename()).unlink() + with self.lock: + self.data = {} diff --git a/aprsd/utils/ring_buffer.py b/aprsd/utils/ring_buffer.py index 4029ce4..6db4dfe 100644 --- a/aprsd/utils/ring_buffer.py +++ b/aprsd/utils/ring_buffer.py @@ -1,6 +1,9 @@ class RingBuffer: """class that implements a not-yet-full buffer""" + max: int = 100 + data: list = [] + def __init__(self, size_max): self.max = size_max self.data = [] From 123b3ffa81c5eb6366378cd4e28d936086af7686 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sun, 18 Dec 2022 08:52:58 -0500 Subject: [PATCH 11/21] Change RX packet processing to enqueu This changes the RX thread to send the packet into a queue instead of starting a new thread for every packet. --- aprsd/cmds/server.py | 7 ++++++- aprsd/cmds/webchat.py | 3 +++ aprsd/threads/__init__.py | 5 +---- aprsd/threads/rx.py | 33 ++++++++++++++++-------------- examples/plugins/example_plugin.py | 6 +++--- tests/cmds/test_webchat.py | 2 +- 6 files changed, 32 insertions(+), 24 deletions(-) diff --git a/aprsd/cmds/server.py b/aprsd/cmds/server.py index 084f072..b7a16a2 100644 --- a/aprsd/cmds/server.py +++ b/aprsd/cmds/server.py @@ -96,10 +96,15 @@ def server(ctx, flush): plugin_manager.setup_plugins() rx_thread = rx.APRSDPluginRXThread( - msg_queues=threads.msg_queues, + packet_queue=threads.packet_queue, config=config, ) + process_thread = rx.APRSDPluginProcessPacketThread( + config=config, + packet_queue=threads.packet_queue, + ) rx_thread.start() + process_thread.start() packets.PacketTrack().restart() diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index 1f19412..17ca3b5 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -63,6 +63,9 @@ class SentMessages(objectstore.ObjectStoreMixin): cls._instance = super().__new__(cls) return cls._instance + def is_initialized(self): + return True + @wrapt.synchronized(lock) def add(self, msg): self.data[msg.msgNo] = self.create(msg.msgNo) diff --git a/aprsd/threads/__init__.py b/aprsd/threads/__init__.py index fd4da3b..bf8cb23 100644 --- a/aprsd/threads/__init__.py +++ b/aprsd/threads/__init__.py @@ -7,7 +7,4 @@ from .keep_alive import KeepAliveThread # noqa: F401 from .rx import APRSDRXThread # noqa: F401 -rx_msg_queue = queue.Queue(maxsize=20) -msg_queues = { - "rx": rx_msg_queue, -} +packet_queue = queue.Queue(maxsize=20) diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index 226248f..3ff8c9e 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -1,5 +1,6 @@ import abc import logging +import queue import time import aprslib @@ -12,9 +13,9 @@ LOG = logging.getLogger("APRSD") class APRSDRXThread(APRSDThread): - def __init__(self, msg_queues, config): + def __init__(self, packet_queue, config): super().__init__("RX_MSG") - self.msg_queues = msg_queues + self.packet_queue = packet_queue self.config = config self._client = client.factory.create() @@ -68,11 +69,7 @@ class APRSDPluginRXThread(APRSDRXThread): # LOG.debug(raw) packet.log(header="RX") packets.PacketList().rx(packet) - thread = APRSDPluginProcessPacketThread( - config=self.config, - packet=packet, - ) - thread.start() + self.packet_queue.put(packet) class APRSDProcessPacketThread(APRSDThread): @@ -83,11 +80,10 @@ class APRSDProcessPacketThread(APRSDThread): will ack a message before sending the packet to the subclass for processing.""" - def __init__(self, config, packet): + def __init__(self, config, packet_queue): self.config = config - self.packet = packet - name = self.packet.raw[:10] - super().__init__(f"RXPKT-{name}") + self.packet_queue = packet_queue + super().__init__("ProcessPKT") self._loop_cnt = 1 def process_ack_packet(self, packet): @@ -99,9 +95,18 @@ class APRSDProcessPacketThread(APRSDThread): return def loop(self): + try: + packet = self.packet_queue.get(block=True, timeout=1) + if packet: + self.process_packet(packet) + except queue.Empty: + pass + self._loop_cnt += 1 + return True + + def process_packet(self, packet): """Process a packet received from aprs-is server.""" LOG.debug(f"RXPKT-LOOP {self._loop_cnt}") - packet = self.packet our_call = self.config["aprsd"]["callsign"].lower() from_call = packet.from_call @@ -147,7 +152,7 @@ class APRSDProcessPacketThread(APRSDThread): return False @abc.abstractmethod - def process_our_message_packet(self, *args, **kwargs): + def process_our_message_packet(self, packet): """Process a MessagePacket destined for us!""" def process_other_packet(self, packet, for_us=False): @@ -210,9 +215,7 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): to_call=from_call, message_text=reply, ) - LOG.warning("Calling msg_pkg.send()") msg_pkt.send() - LOG.warning("Calling msg_pkg.send() --- DONE") # If the message was for us and we didn't have a # response, then we send a usage statement. diff --git a/examples/plugins/example_plugin.py b/examples/plugins/example_plugin.py index d674d0d..0a3c75e 100644 --- a/examples/plugins/example_plugin.py +++ b/examples/plugins/example_plugin.py @@ -6,7 +6,7 @@ from aprsd import plugin LOG = logging.getLogger("APRSD") -class HelloPlugin(plugin.APRSDPluginBase): +class HelloPlugin(plugin.APRSDRegexCommandPluginBase): """Hello World.""" version = "1.0" @@ -14,7 +14,7 @@ class HelloPlugin(plugin.APRSDPluginBase): command_regex = "^[hH]" command_name = "hello" - def command(self, fromcall, message, ack): + def command(self, packet): LOG.info("HelloPlugin") - reply = f"Hello '{fromcall}'" + reply = f"Hello '{packet.from_call}'" return reply diff --git a/tests/cmds/test_webchat.py b/tests/cmds/test_webchat.py index 7c23859..6bd08f8 100644 --- a/tests/cmds/test_webchat.py +++ b/tests/cmds/test_webchat.py @@ -90,7 +90,7 @@ class TestSendMessageCommand(unittest.TestCase): mock_emit.called_once() @mock.patch("aprsd.config.parse_config") - @mock.patch("aprsd.packets.PacketList.add") + @mock.patch("aprsd.packets.PacketList.rx") @mock.patch("aprsd.cmds.webchat.socketio.emit") def test_process_our_message_packet( self, mock_parse_config, From 9fc5356456a60cd96c085a842cbeed0eb315e949 Mon Sep 17 00:00:00 2001 From: Hemna Date: Sun, 18 Dec 2022 09:14:12 -0500 Subject: [PATCH 12/21] Removed unused threading code --- aprsd/cmds/listen.py | 2 +- aprsd/plugins/email.py | 4 +--- aprsd/threads/aprsd.py | 19 ------------------- 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index 61ded19..a8066d4 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -135,7 +135,7 @@ def listen( keepalive.start() LOG.debug("Create APRSDListenThread") - listen_thread = APRSDListenThread(threads.msg_queues, config=config) + listen_thread = APRSDListenThread(threads.packet_queue, config=config) LOG.debug("Start APRSDListenThread") listen_thread.start() LOG.debug("keepalive Join") diff --git a/aprsd/plugins/email.py b/aprsd/plugins/email.py index 557bd19..1e49f4a 100644 --- a/aprsd/plugins/email.py +++ b/aprsd/plugins/email.py @@ -80,7 +80,6 @@ class EmailPlugin(plugin.APRSDRegexCommandPluginBase): def create_threads(self): if self.enabled: return APRSDEmailThread( - msg_queues=threads.msg_queues, config=self.config, ) @@ -502,9 +501,8 @@ def resend_email(config, count, fromcall): class APRSDEmailThread(threads.APRSDThread): - def __init__(self, msg_queues, config): + def __init__(self, config): super().__init__("EmailThread") - self.msg_queues = msg_queues self.config = config self.past = datetime.datetime.now() diff --git a/aprsd/threads/aprsd.py b/aprsd/threads/aprsd.py index 5673fa4..82fb65f 100644 --- a/aprsd/threads/aprsd.py +++ b/aprsd/threads/aprsd.py @@ -1,6 +1,5 @@ import abc import logging -from queue import Queue import threading import wrapt @@ -16,7 +15,6 @@ class APRSDThreadList: threads_list = [] lock = threading.Lock() - global_queue = Queue() def __new__(cls, *args, **kwargs): if cls._instance is None: @@ -26,7 +24,6 @@ class APRSDThreadList: @wrapt.synchronized(lock) def add(self, thread_obj): - thread_obj.set_global_queue(self.global_queue) self.threads_list.append(thread_obj) @wrapt.synchronized(lock) @@ -35,7 +32,6 @@ class APRSDThreadList: @wrapt.synchronized(lock) def stop_all(self): - self.global_queue.put_nowait({"quit": True}) """Iterate over all threads and call stop on them.""" for th in self.threads_list: LOG.info(f"Stopping Thread {th.name}") @@ -50,30 +46,15 @@ class APRSDThreadList: class APRSDThread(threading.Thread, metaclass=abc.ABCMeta): - global_queue = None - def __init__(self, name): super().__init__(name=name) self.thread_stop = False APRSDThreadList().add(self) - def set_global_queue(self, global_queue): - self.global_queue = global_queue - def _should_quit(self): """ see if we have a quit message from the global queue.""" if self.thread_stop: return True - if self.global_queue.empty(): - return False - msg = self.global_queue.get(timeout=1) - if not msg: - return False - if "quit" in msg and msg["quit"] is True: - # put the message back on the queue for others - self.global_queue.put_nowait(msg) - self.thread_stop = True - return True def stop(self): self.thread_stop = True From e37f99a6ddbb32e5d1a59f30c7179bb81451fc6a Mon Sep 17 00:00:00 2001 From: Hemna Date: Sun, 18 Dec 2022 21:44:23 -0500 Subject: [PATCH 13/21] reworked collecting and reporting stats This is the start of the cleanup of reporting of packet stats --- aprsd/cmds/send_message.py | 3 +- aprsd/cmds/webchat.py | 1 - aprsd/flask.py | 11 ++- aprsd/packets/core.py | 31 +++++-- aprsd/packets/packet_list.py | 20 +++-- aprsd/packets/tracker.py | 2 - aprsd/stats.py | 130 ++++++++++++---------------- aprsd/threads/keep_alive.py | 8 +- aprsd/threads/rx.py | 4 +- aprsd/threads/tx.py | 35 +++++--- aprsd/utils/json.py | 60 +++++++++++++ aprsd/web/admin/static/js/charts.js | 3 +- aprsd/web/admin/static/js/main.js | 40 +++++---- 13 files changed, 208 insertions(+), 140 deletions(-) create mode 100644 aprsd/utils/json.py diff --git a/aprsd/cmds/send_message.py b/aprsd/cmds/send_message.py index 08a236f..469901d 100644 --- a/aprsd/cmds/send_message.py +++ b/aprsd/cmds/send_message.py @@ -100,7 +100,8 @@ def send_message( global got_ack, got_response cl = client.factory.create() packet = cl.decode_packet(packet) - packet.log("RX_PKT") + packets.PacketList().rx(packet) + packet.log("RX") # LOG.debug("Got packet back {}".format(packet)) if isinstance(packet, packets.AckPacket): got_ack = True diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index 17ca3b5..d0c0b9b 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -185,7 +185,6 @@ class WebChatProcessPacketThread(rx.APRSDProcessPacketThread): fromcall = packet.from_call packets.PacketList().rx(packet) - stats.APRSDStats().msgs_rx_inc() message = packet.get("message_text", None) msg = { "id": 0, diff --git a/aprsd/flask.py b/aprsd/flask.py index 2b85857..8f4ddf6 100644 --- a/aprsd/flask.py +++ b/aprsd/flask.py @@ -380,7 +380,12 @@ class APRSDFlask(flask_classful.FlaskView): @auth.login_required def packets(self): packet_list = packets.PacketList().get() - return json.dumps(packet_list) + tmp_list = [] + for pkt in packet_list: + tmp_list.append(pkt.json) + + LOG.info(f"PACKETS {tmp_list}") + return json.dumps(tmp_list) @auth.login_required def plugins(self): @@ -420,8 +425,8 @@ class APRSDFlask(flask_classful.FlaskView): stats_dict["aprsd"]["watch_list"] = new_list packet_list = packets.PacketList() - rx = packet_list.total_received() - tx = packet_list.total_sent() + rx = packet_list.total_rx() + tx = packet_list.total_tx() stats_dict["packets"] = { "sent": tx, "received": rx, diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index c476472..db359d2 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -1,6 +1,7 @@ import abc from dataclasses import asdict, dataclass, field import datetime +import json import logging import re import time @@ -9,9 +10,11 @@ from typing import List import dacite -from aprsd import client, stats +from aprsd import client +from aprsd.packets.packet_list import PacketList # noqa: F401 from aprsd.threads import tx from aprsd.utils import counter +from aprsd.utils import json as aprsd_json LOG = logging.getLogger("APRSD") @@ -57,16 +60,26 @@ class Packet(metaclass=abc.ABCMeta): raw_dict: dict = field(repr=False, default_factory=lambda: {}) # Fields related to sending packets out - send_count: int = field(repr=False, default=1) + send_count: int = field(repr=False, default=0) retry_count: int = field(repr=False, default=3) last_send_time: datetime.timedelta = field(repr=False, default=None) - last_send_attempt: int = field(repr=False, default=0) # Do we allow this packet to be saved to send later? allow_delay: bool = field(repr=False, default=True) def __post__init__(self): LOG.warning(f"POST INIT {self}") + @property + def __dict__(self): + return asdict(self) + + @property + def json(self): + """ + get the json formated string + """ + return json.dumps(self.__dict__, cls=aprsd_json.EnhancedJSONEncoder) + def get(self, key, default=None): """Emulate a getter on a dict.""" if hasattr(self, key): @@ -122,13 +135,13 @@ class Packet(metaclass=abc.ABCMeta): log_list = ["\n"] name = self.__class__.__name__ if header: - if isinstance(self, AckPacket) and "tx" in header.lower(): + if "tx" in header.lower(): log_list.append( - f"{header}____________({name}__" - f"TX:{self.send_count} of {self.retry_count})", + f"{header}________({name} " + f"TX:{self.send_count+1} of {self.retry_count})", ) else: - log_list.append(f"{header}____________({name})") + log_list.append(f"{header}________({name})") # log_list.append(f" Packet : {self.__class__.__name__}") log_list.append(f" Raw : {self.raw}") if self.to_call: @@ -148,7 +161,7 @@ class Packet(metaclass=abc.ABCMeta): if self.msgNo: log_list.append(f" Msg # : {self.msgNo}") - log_list.append(f"{header}____________({name})") + log_list.append(f"{header}________({name})") LOG.info("\n".join(log_list)) LOG.debug(self) @@ -178,7 +191,7 @@ class Packet(metaclass=abc.ABCMeta): cl = client.factory.create().client self.log(header="TX Message Direct") cl.send(self.raw) - stats.APRSDStats().msgs_tx_inc() + PacketList().tx(self) @dataclass diff --git a/aprsd/packets/packet_list.py b/aprsd/packets/packet_list.py index db850c6..d9c2c94 100644 --- a/aprsd/packets/packet_list.py +++ b/aprsd/packets/packet_list.py @@ -3,7 +3,7 @@ import threading import wrapt -from aprsd import utils +from aprsd import stats, utils from aprsd.packets import seen_list @@ -19,8 +19,8 @@ class PacketList: packet_list: utils.RingBuffer = utils.RingBuffer(1000) - total_recv: int = 0 - total_tx: int = 0 + _total_rx: int = 0 + _total_tx: int = 0 def __new__(cls, *args, **kwargs): if cls._instance is None: @@ -43,25 +43,27 @@ class PacketList: @wrapt.synchronized(lock) def rx(self, packet): """Add a packet that was received.""" - self.total_recv += 1 + self._total_rx += 1 self.packet_list.append(packet) seen_list.SeenList().update_seen(packet) + stats.APRSDStats().rx(packet) @wrapt.synchronized(lock) def tx(self, packet): """Add a packet that was received.""" - self.total_tx += 1 + self._total_tx += 1 self.packet_list.append(packet) seen_list.SeenList().update_seen(packet) + stats.APRSDStats().tx(packet) @wrapt.synchronized(lock) def get(self): return self.packet_list.get() @wrapt.synchronized(lock) - def total_received(self): - return self.total_recv + def total_rx(self): + return self._total_rx @wrapt.synchronized(lock) - def total_sent(self): - return self.total_tx + def total_tx(self): + return self._total_tx diff --git a/aprsd/packets/tracker.py b/aprsd/packets/tracker.py index 8ec7b67..8d3d931 100644 --- a/aprsd/packets/tracker.py +++ b/aprsd/packets/tracker.py @@ -3,7 +3,6 @@ import threading import wrapt -from aprsd import stats from aprsd.utils import objectstore @@ -76,7 +75,6 @@ class PacketTrack(objectstore.ObjectStoreMixin): def add(self, packet): key = int(packet.msgNo) self.data[key] = packet - stats.APRSDStats().msgs_tracked_inc() self.total_tracked += 1 @wrapt.synchronized(lock) diff --git a/aprsd/stats.py b/aprsd/stats.py index cc8dc8f..91b9b8c 100644 --- a/aprsd/stats.py +++ b/aprsd/stats.py @@ -21,15 +21,6 @@ class APRSDStats: _aprsis_server = None _aprsis_keepalive = None - _msgs_tracked = 0 - _msgs_tx = 0 - _msgs_rx = 0 - - _msgs_mice_rx = 0 - - _ack_tx = 0 - _ack_rx = 0 - _email_thread_last_time = None _email_tx = 0 _email_rx = 0 @@ -37,6 +28,37 @@ class APRSDStats: _mem_current = 0 _mem_peak = 0 + _pkt_cnt = { + "Packet": { + "tx": 0, + "rx": 0, + }, + "AckPacket": { + "tx": 0, + "rx": 0, + }, + "GPSPacket": { + "tx": 0, + "rx": 0, + }, + "StatusPacket": { + "tx": 0, + "rx": 0, + }, + "MicEPacket": { + "tx": 0, + "rx": 0, + }, + "MessagePacket": { + "tx": 0, + "rx": 0, + }, + "WeatherPacket": { + "tx": 0, + "rx": 0, + }, + } + def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = super().__new__(cls) @@ -90,67 +112,18 @@ class APRSDStats: def set_aprsis_keepalive(self): self._aprsis_keepalive = datetime.datetime.now() - def rx_packet(self, packet): - if isinstance(packet, packets.MessagePacket): - self.msgs_rx_inc() - elif isinstance(packet, packets.MicEPacket): - self.msgs_mice_inc() - elif isinstance(packet, packets.AckPacket): - self.ack_rx_inc() + def rx(self, packet): + type = packet.__class__.__name__ + self._pkt_cnt[type]["rx"] += 1 - @wrapt.synchronized(lock) - @property - def msgs_tx(self): - return self._msgs_tx - - @wrapt.synchronized(lock) - def msgs_tx_inc(self): - self._msgs_tx += 1 - - @wrapt.synchronized(lock) - @property - def msgs_rx(self): - return self._msgs_rx - - @wrapt.synchronized(lock) - def msgs_rx_inc(self): - self._msgs_rx += 1 - - @wrapt.synchronized(lock) - @property - def msgs_mice_rx(self): - return self._msgs_mice_rx - - @wrapt.synchronized(lock) - def msgs_mice_inc(self): - self._msgs_mice_rx += 1 - - @wrapt.synchronized(lock) - @property - def ack_tx(self): - return self._ack_tx - - @wrapt.synchronized(lock) - def ack_tx_inc(self): - self._ack_tx += 1 - - @wrapt.synchronized(lock) - @property - def ack_rx(self): - return self._ack_rx - - @wrapt.synchronized(lock) - def ack_rx_inc(self): - self._ack_rx += 1 + def tx(self, packet): + type = packet.__class__.__name__ + self._pkt_cnt[type]["tx"] += 1 @wrapt.synchronized(lock) @property def msgs_tracked(self): - return self._msgs_tracked - - @wrapt.synchronized(lock) - def msgs_tracked_inc(self): - self._msgs_tracked += 1 + return packets.PacketTrack().total_tracked @wrapt.synchronized(lock) @property @@ -212,11 +185,13 @@ class APRSDStats: wl = packets.WatchList() sl = packets.SeenList() + pl = packets.PacketList() stats = { "aprsd": { "version": aprsd.__version__, "uptime": utils.strfdelta(self.uptime), + "callsign": self.config["aprsd"]["callsign"], "memory_current": int(self.memory), "memory_current_str": utils.human_size(self.memory), "memory_peak": int(self.memory_peak), @@ -229,18 +204,20 @@ class APRSDStats: "callsign": self.config["aprs"]["login"], "last_update": last_aprsis_keepalive, }, + "packets": { + "tracked": int(pl.total_tx() + pl.total_rx()), + "sent": int(pl.total_tx()), + "received": int(pl.total_rx()), + }, "messages": { - "tracked": int(self.msgs_tracked), - "sent": int(self.msgs_tx), - "recieved": int(self.msgs_rx), - "ack_sent": int(self.ack_tx), - "ack_recieved": int(self.ack_rx), - "mic-e recieved": int(self.msgs_mice_rx), + "sent": self._pkt_cnt["MessagePacket"]["tx"], + "received": self._pkt_cnt["MessagePacket"]["tx"], + "ack_sent": self._pkt_cnt["AckPacket"]["tx"], }, "email": { "enabled": self.config["aprsd"]["email"]["enabled"], "sent": int(self._email_tx), - "recieved": int(self._email_rx), + "received": int(self._email_rx), "thread_last_update": last_update, }, "plugins": plugin_stats, @@ -248,15 +225,16 @@ class APRSDStats: return stats def __str__(self): + pl = packets.PacketList() return ( "Uptime:{} Msgs TX:{} RX:{} " "ACK: TX:{} RX:{} " "Email TX:{} RX:{} LastLoop:{} ".format( self.uptime, - self._msgs_tx, - self._msgs_rx, - self._ack_tx, - self._ack_rx, + pl.total_tx(), + pl.total_rx(), + self._pkt_cnt["AckPacket"]["tx"], + self._pkt_cnt["AckPacket"]["rx"], self._email_tx, self._email_rx, self._email_thread_last_time, diff --git a/aprsd/threads/keep_alive.py b/aprsd/threads/keep_alive.py index 4813663..9c8007f 100644 --- a/aprsd/threads/keep_alive.py +++ b/aprsd/threads/keep_alive.py @@ -56,11 +56,11 @@ class KeepAliveThread(APRSDThread): ).format( login, utils.strfdelta(stats_obj.uptime), - pl.total_recv, - pl.total_tx, + pl.total_rx(), + pl.total_tx(), tracked_packets, - stats_obj.msgs_tx, - stats_obj.msgs_rx, + stats_obj._pkt_cnt["MessagePacket"]["tx"], + stats_obj._pkt_cnt["MessagePacket"]["rx"], last_msg_time, email_thread_time, utils.human_size(current), diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index 3ff8c9e..173b024 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -5,7 +5,7 @@ import time import aprslib -from aprsd import client, packets, plugin, stats +from aprsd import client, packets, plugin from aprsd.threads import APRSDThread @@ -91,7 +91,6 @@ class APRSDProcessPacketThread(APRSDThread): LOG.info(f"Got ack for message {ack_num}") pkt_tracker = packets.PacketTrack() pkt_tracker.remove(ack_num) - stats.APRSDStats().ack_rx_inc() return def loop(self): @@ -130,7 +129,6 @@ class APRSDProcessPacketThread(APRSDThread): if isinstance(packet, packets.MessagePacket): if to_call and to_call.lower() == our_call: # It's a MessagePacket and it's for us! - stats.APRSDStats().msgs_rx_inc() # let any threads do their thing, then ack # send an ack last ack_pkt = packets.AckPacket( diff --git a/aprsd/threads/tx.py b/aprsd/threads/tx.py index b8fca07..ac3651a 100644 --- a/aprsd/threads/tx.py +++ b/aprsd/threads/tx.py @@ -2,7 +2,7 @@ import datetime import logging import time -from aprsd import client, stats +from aprsd import client from aprsd import threads as aprsd_threads from aprsd.packets import packet_list, tracker @@ -36,14 +36,23 @@ class SendPacketThread(aprsd_threads.APRSDThread): if not packet: # The message has been removed from the tracking queue # So it got acked and we are done. - LOG.info("Message Send Complete via Ack.") + LOG.info( + f"{packet.__class__.__name__}" + f"({packet.msgNo}) " + "Message Send Complete via Ack.", + ) return False else: send_now = False - if packet.last_send_attempt == packet.retry_count: + if packet.send_count == packet.retry_count: # we reached the send limit, don't send again # TODO(hemna) - Need to put this in a delayed queue? - LOG.info("Message Send Complete. Max attempts reached.") + LOG.info( + f"{packet.__class__.__name__} " + f"({packet.msgNo}) " + "Message Send Complete. Max attempts reached" + f" {packet.retry_count}", + ) if not packet.allow_delay: pkt_tracker.remove(packet.msgNo) return False @@ -52,7 +61,7 @@ class SendPacketThread(aprsd_threads.APRSDThread): if packet.last_send_time: # Message has a last send time tracking now = datetime.datetime.now() - sleeptime = (packet.last_send_attempt + 1) * 31 + sleeptime = (packet.send_count + 1) * 31 delta = now - packet.last_send_time if delta > datetime.timedelta(seconds=sleeptime): # It's time to try to send it again @@ -66,10 +75,9 @@ class SendPacketThread(aprsd_threads.APRSDThread): packet.log("TX") cl = client.factory.create().client cl.send(packet.raw) - stats.APRSDStats().msgs_tx_inc() packet_list.PacketList().tx(packet) packet.last_send_time = datetime.datetime.now() - packet.last_send_attempt += 1 + packet.send_count += 1 time.sleep(1) # Make sure we get called again. @@ -87,10 +95,15 @@ class SendAckThread(aprsd_threads.APRSDThread): def loop(self): """Separate thread to send acks with retries.""" send_now = False - if self.packet.last_send_attempt == self.packet.retry_count: + if self.packet.send_count == self.packet.retry_count: # we reached the send limit, don't send again # TODO(hemna) - Need to put this in a delayed queue? - LOG.info("Ack Send Complete. Max attempts reached.") + LOG.info( + f"{self.packet.__class__.__name__}" + f"({self.packet.msgNo}) " + "Send Complete. Max attempts reached" + f" {self.packet.retry_count}", + ) return False if self.packet.last_send_time: @@ -113,10 +126,8 @@ class SendAckThread(aprsd_threads.APRSDThread): cl = client.factory.create().client self.packet.log("TX") cl.send(self.packet.raw) - self.packet.send_count += 1 - stats.APRSDStats().ack_tx_inc() packet_list.PacketList().tx(self.packet) - self.packet.last_send_attempt += 1 + self.packet.send_count += 1 self.packet.last_send_time = datetime.datetime.now() time.sleep(1) diff --git a/aprsd/utils/json.py b/aprsd/utils/json.py new file mode 100644 index 0000000..8876738 --- /dev/null +++ b/aprsd/utils/json.py @@ -0,0 +1,60 @@ +import datetime +import decimal +import json +import sys + + +class EnhancedJSONEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, datetime.datetime): + args = ( + "year", "month", "day", "hour", "minute", + "second", "microsecond", + ) + return { + "__type__": "datetime.datetime", + "args": [getattr(obj, a) for a in args], + } + elif isinstance(obj, datetime.date): + args = ("year", "month", "day") + return { + "__type__": "datetime.date", + "args": [getattr(obj, a) for a in args], + } + elif isinstance(obj, datetime.time): + args = ("hour", "minute", "second", "microsecond") + return { + "__type__": "datetime.time", + "args": [getattr(obj, a) for a in args], + } + elif isinstance(obj, datetime.timedelta): + args = ("days", "seconds", "microseconds") + return { + "__type__": "datetime.timedelta", + "args": [getattr(obj, a) for a in args], + } + elif isinstance(obj, decimal.Decimal): + return { + "__type__": "decimal.Decimal", + "args": [str(obj)], + } + else: + return super().default(obj) + + +class EnhancedJSONDecoder(json.JSONDecoder): + + def __init__(self, *args, **kwargs): + super().__init__( + *args, object_hook=self.object_hook, + **kwargs, + ) + + def object_hook(self, d): + if "__type__" not in d: + return d + o = sys.modules[__name__] + for e in d["__type__"].split("."): + o = getattr(o, e) + args, kwargs = d.get("args", ()), d.get("kwargs", {}) + return o(*args, **kwargs) diff --git a/aprsd/web/admin/static/js/charts.js b/aprsd/web/admin/static/js/charts.js index 9d4ed65..4ceb6d7 100644 --- a/aprsd/web/admin/static/js/charts.js +++ b/aprsd/web/admin/static/js/charts.js @@ -219,6 +219,7 @@ function updateQuadData(chart, label, first, second, third, fourth) { } function update_stats( data ) { + our_callsign = data["stats"]["aprsd"]["callsign"]; $("#version").text( data["stats"]["aprsd"]["version"] ); $("#aprs_connection").html( data["aprs_connection"] ); $("#uptime").text( "uptime: " + data["stats"]["aprsd"]["uptime"] ); @@ -226,7 +227,7 @@ function update_stats( data ) { $("#jsonstats").html(html_pretty); short_time = data["time"].split(/\s(.+)/)[1]; updateDualData(packets_chart, short_time, data["stats"]["packets"]["sent"], data["stats"]["packets"]["received"]); - updateQuadData(message_chart, short_time, data["stats"]["messages"]["sent"], data["stats"]["messages"]["recieved"], data["stats"]["messages"]["ack_sent"], data["stats"]["messages"]["ack_recieved"]); + updateQuadData(message_chart, short_time, data["stats"]["messages"]["sent"], data["stats"]["messages"]["received"], data["stats"]["messages"]["ack_sent"], data["stats"]["messages"]["ack_recieved"]); updateDualData(email_chart, short_time, data["stats"]["email"]["sent"], data["stats"]["email"]["recieved"]); updateDualData(memory_chart, short_time, data["stats"]["aprsd"]["memory_peak"], data["stats"]["aprsd"]["memory_current"]); } diff --git a/aprsd/web/admin/static/js/main.js b/aprsd/web/admin/static/js/main.js index 40b3942..491b894 100644 --- a/aprsd/web/admin/static/js/main.js +++ b/aprsd/web/admin/static/js/main.js @@ -1,5 +1,6 @@ // watchlist is a dict of ham callsign => symbol, packets var watchlist = {}; +var our_callsign = ""; function aprs_img(item, x_offset, y_offset) { var x = x_offset * -16; @@ -107,34 +108,35 @@ function update_packets( data ) { packetsdiv.html('') } jQuery.each(data, function(i, val) { - update_watchlist_from_packet(val['from'], val); - if ( packet_list.hasOwnProperty(val["ts"]) == false ) { + pkt = JSON.parse(val); + update_watchlist_from_packet(pkt['from_call'], pkt); + if ( packet_list.hasOwnProperty(val["timestamp"]) == false ) { // Store the packet - packet_list[val["ts"]] = val; - ts_str = val["ts"].toString(); - ts = ts_str.split(".")[0]*1000; - var d = new Date(ts).toLocaleDateString("en-US") - var t = new Date(ts).toLocaleTimeString("en-US") - if (val.hasOwnProperty('from') == false) { - from = val['fromcall'] - title_id = 'title_tx' + packet_list[pkt["timestamp"]] = pkt; + //ts_str = val["timestamp"].toString(); + //ts = ts_str.split(".")[0]*1000; + ts = pkt["timestamp"] + var d = new Date(ts).toLocaleDateString("en-US"); + var t = new Date(ts).toLocaleTimeString("en-US"); + var from_call = pkt['from_call']; + if (from_call == our_callsign) { + title_id = 'title_tx'; } else { - from = val['from'] - title_id = 'title_rx' + title_id = 'title_rx'; } - var from_to = d + " " + t + "    " + from + " > " + var from_to = d + " " + t + "    " + from_call + " > " if (val.hasOwnProperty('addresse')) { - from_to = from_to + val['addresse'] - } else if (val.hasOwnProperty('tocall')) { - from_to = from_to + val['tocall'] - } else if (val.hasOwnProperty('format') && val['format'] == 'mic-e') { + from_to = from_to + pkt['addresse'] + } else if (pkt.hasOwnProperty('to_call')) { + from_to = from_to + pkt['to_call'] + } else if (pkt.hasOwnProperty('format') && pkt['format'] == 'mic-e') { from_to = from_to + "Mic-E" } - from_to = from_to + "  -  " + val['raw'] + from_to = from_to + "  -  " + pkt['raw'] - json_pretty = Prism.highlight(JSON.stringify(val, null, '\t'), Prism.languages.json, 'json'); + json_pretty = Prism.highlight(JSON.stringify(pkt, null, '\t'), Prism.languages.json, 'json'); pkt_html = '
' + from_to + '
' + json_pretty + '
' packetsdiv.prepend(pkt_html); } From ad0d89db401847a44b2d2af82d9901b40ecd9b5a Mon Sep 17 00:00:00 2001 From: Hemna Date: Mon, 19 Dec 2022 10:28:22 -0500 Subject: [PATCH 14/21] Updated webchat and listen for queue based RX This patch updates both the webchat and listen commands to be able to use the new queue based packet RX processing. APRSD used to start a thread for every packet received, now packets are pushed into a queue for processing by other threads already running. --- aprsd/cmds/listen.py | 5 ++++- aprsd/cmds/webchat.py | 46 ++++++++++--------------------------------- aprsd/threads/rx.py | 6 +++--- aprsd/threads/tx.py | 2 +- 4 files changed, 18 insertions(+), 41 deletions(-) diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index a8066d4..da6b4c8 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -135,7 +135,10 @@ def listen( keepalive.start() LOG.debug("Create APRSDListenThread") - listen_thread = APRSDListenThread(threads.packet_queue, config=config) + listen_thread = APRSDListenThread( + config=config, + packet_queue=threads.packet_queue, + ) LOG.debug("Start APRSDListenThread") listen_thread.start() LOG.debug("keepalive Join") diff --git a/aprsd/cmds/webchat.py b/aprsd/cmds/webchat.py index d0c0b9b..c53b44f 100644 --- a/aprsd/cmds/webchat.py +++ b/aprsd/cmds/webchat.py @@ -132,42 +132,12 @@ def verify_password(username, password): return username -class WebChatRXThread(rx.APRSDRXThread): - """Class that connects to APRISIS/kiss and waits for messages. - - After the packet is received from APRSIS/KISS, the packet is - sent to processing in the WebChatProcessPacketThread. - """ - def __init__(self, config, socketio): - super().__init__(None, config) - self.socketio = socketio - self.connected = False - - def connected(self, connected=True): - self.connected = connected - - def process_packet(self, *args, **kwargs): - # packet = self._client.decode_packet(*args, **kwargs) - if "packet" in kwargs: - packet = kwargs["packet"] - else: - packet = self._client.decode_packet(*args, **kwargs) - - LOG.debug(f"GOT Packet {packet}") - thread = WebChatProcessPacketThread( - config=self.config, - packet=packet, - socketio=self.socketio, - ) - thread.start() - - class WebChatProcessPacketThread(rx.APRSDProcessPacketThread): """Class that handles packets being sent to us.""" - def __init__(self, config, packet, socketio): + def __init__(self, config, packet_queue, socketio): self.socketio = socketio self.connected = False - super().__init__(config, packet) + super().__init__(config, packet_queue) def process_ack_packet(self, packet: packets.AckPacket): super().process_ack_packet(packet) @@ -184,7 +154,6 @@ class WebChatProcessPacketThread(rx.APRSDProcessPacketThread): packet.get("addresse", None) fromcall = packet.from_call - packets.PacketList().rx(packet) message = packet.get("message_text", None) msg = { "id": 0, @@ -532,12 +501,17 @@ def webchat(ctx, flush, port): packets.SeenList(config=config) (socketio, app) = init_flask(config, loglevel, quiet) - rx_thread = WebChatRXThread( + rx_thread = rx.APRSDPluginRXThread( config=config, + packet_queue=threads.packet_queue, + ) + rx_thread.start() + process_thread = WebChatProcessPacketThread( + config=config, + packet_queue=threads.packet_queue, socketio=socketio, ) - LOG.info("Start RX Thread") - rx_thread.start() + process_thread.start() keepalive = threads.KeepAliveThread(config=config) LOG.info("Start KeepAliveThread") diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index 173b024..4db1577 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -13,10 +13,10 @@ LOG = logging.getLogger("APRSD") class APRSDRXThread(APRSDThread): - def __init__(self, packet_queue, config): + def __init__(self, config, packet_queue): super().__init__("RX_MSG") - self.packet_queue = packet_queue self.config = config + self.packet_queue = packet_queue self._client = client.factory.create() def stop(self): @@ -95,7 +95,7 @@ class APRSDProcessPacketThread(APRSDThread): def loop(self): try: - packet = self.packet_queue.get(block=True, timeout=1) + packet = self.packet_queue.get(timeout=1) if packet: self.process_packet(packet) except queue.Empty: diff --git a/aprsd/threads/tx.py b/aprsd/threads/tx.py index ac3651a..4511e95 100644 --- a/aprsd/threads/tx.py +++ b/aprsd/threads/tx.py @@ -38,7 +38,7 @@ class SendPacketThread(aprsd_threads.APRSDThread): # So it got acked and we are done. LOG.info( f"{packet.__class__.__name__}" - f"({packet.msgNo}) " + f"({self.packet.msgNo}) " "Message Send Complete via Ack.", ) return False From 899a6e5363690a1463c6102b30033e6ff1888c3c Mon Sep 17 00:00:00 2001 From: Hemna Date: Mon, 19 Dec 2022 13:54:02 -0500 Subject: [PATCH 15/21] Added WeatherPacket encoding This patch adds the ability to output a correctly formatted APRS weather packet for sending. --- aprsd/packets/core.py | 52 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index db359d2..84a8674 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -271,7 +271,9 @@ class GPSPacket(PathPacket): comment: str = None symbol: str = field(default="l") symbol_table: str = field(default="/") + # in MPH speed: float = 0.00 + # 0 to 360 course: int = 0 def _build_time_zulu(self): @@ -322,7 +324,55 @@ class WeatherPacket(GPSPacket): comment: str = None def _build_raw(self): - raise NotImplementedError + """Build an uncompressed weather packet + + Format = + + _CSE/SPDgXXXtXXXrXXXpXXXPXXXhXXbXXXXX%type NEW FORMAT APRS793 June 97 + NOT BACKWARD COMPATIBLE + + + Where: CSE/SPD is wind direction and sustained 1 minute speed + t is in degrees F + r is Rain per last 60 minutes + p is precipitation per last 24 hours (sliding 24 hour window) + P is precip per last 24 hours since midnight + b is Baro in tenths of a mb + h is humidity in percent. 00=100 + g is Gust (peak winds in last 5 minutes) + # is the raw rain counter for remote WX stations + See notes on remotes below + % shows software type d=Dos, m=Mac, w=Win, etc + type shows type of WX instrument + + """ + time_zulu = self._build_time_zulu() + + contents = [ + f"{self.from_call}>{self.to_call},WIDE2-1:", + f"@{time_zulu}z{self.latitude}{self.symbol_table}", + f"{self.longitude}{self.symbol}", + # Add CSE = Course + f"{self.course}", + # Speed = sustained 1 minute wind speed in mph + f"{self.symbol_table}", f"{self.speed:03.0f}", + # wind gust (peak wind speed in mph in the last 5 minutes) + f"g{self.wind_gust:03.0f}", + # Temperature in degrees F + f"t{self.temperature:03.0f}", + # Rainfall (in hundredths of an inch) in the last hour + f"r{self.rain_1h:03.0f}", + # Rainfall (in hundredths of an inch) in last 24 hours + f"P{self.rain_since_midnight:03.0f}", + # Humidity + f"h{self.humidity:02d}", + # Barometric pressure (in tenths of millibars/tenths of hPascal) + f"b{self.pressure:05.0f}", + ] + + if self.comment: + contents.append(self.comment) + self.raw = "".join(contents) TYPE_LOOKUP = { From d01392f6a588b38134875acfa085f5664febacf6 Mon Sep 17 00:00:00 2001 From: Hemna Date: Mon, 19 Dec 2022 17:28:18 -0500 Subject: [PATCH 16/21] Add packet filtering for aprsd listen Now aprsd listen can filter by packet types. --- aprsd/cmds/listen.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index da6b4c8..0363872 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -38,9 +38,28 @@ def signal_handler(sig, frame): class APRSDListenThread(rx.APRSDRXThread): + def __init__(self, config, packet_queue, packet_filter=None): + super().__init__(config, packet_queue) + self.packet_filter=packet_filter + def process_packet(self, *args, **kwargs): packet = self._client.decode_packet(*args, **kwargs) - packet.log(header="RX") + filters = { + packets.Packet.__name__: packets.Packet, + packets.AckPacket.__name__: packets.AckPacket, + packets.GPSPacket.__name__: packets.GPSPacket, + packets.MessagePacket.__name__: packets.MessagePacket, + packets.MicEPacket.__name__: packets.MicEPacket, + packets.WeatherPacket.__name__: packets.WeatherPacket, + } + + if self.packet_filter: + filter_class = filters[self.packet_filter] + if isinstance(packet, filter_class): + packet.log(header="RX") + else: + packet.log(header="RX") + packets.PacketList().rx(packet) @@ -58,6 +77,21 @@ class APRSDListenThread(rx.APRSDRXThread): show_envvar=True, help="the APRS-IS password for APRS_LOGIN", ) +@click.option( + "--packet-filter", + type=click.Choice( + [ + packets.Packet.__name__, + packets.AckPacket.__name__, + packets.GPSPacket.__name__, + packets.MicEPacket.__name__, + packets.MessagePacket.__name__, + packets.WeatherPacket.__name__, + ], + case_sensitive=False, + ), + help="Filter by packet type", +) @click.argument( "filter", nargs=-1, @@ -69,6 +103,7 @@ def listen( ctx, aprs_login, aprs_password, + packet_filter, filter, ): """Listen to packets on the APRS-IS Network based on FILTER. @@ -138,6 +173,7 @@ def listen( listen_thread = APRSDListenThread( config=config, packet_queue=threads.packet_queue, + packet_filter=packet_filter, ) LOG.debug("Start APRSDListenThread") listen_thread.start() From a1188d29d41df7a609fe29d23010cf8fb50fa5cc Mon Sep 17 00:00:00 2001 From: Hemna Date: Mon, 19 Dec 2022 20:41:57 -0500 Subject: [PATCH 17/21] Fix pep8 violation --- aprsd/cmds/listen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aprsd/cmds/listen.py b/aprsd/cmds/listen.py index 0363872..8da7da3 100644 --- a/aprsd/cmds/listen.py +++ b/aprsd/cmds/listen.py @@ -40,7 +40,7 @@ def signal_handler(sig, frame): class APRSDListenThread(rx.APRSDRXThread): def __init__(self, config, packet_queue, packet_filter=None): super().__init__(config, packet_queue) - self.packet_filter=packet_filter + self.packet_filter = packet_filter def process_packet(self, *args, **kwargs): packet = self._client.decode_packet(*args, **kwargs) From f19043ecd994ffd2d1c6958f08a47d5328c64258 Mon Sep 17 00:00:00 2001 From: Hemna Date: Mon, 19 Dec 2022 21:27:05 -0500 Subject: [PATCH 18/21] Fix some WeatherPacket formatting --- aprsd/packets/core.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 84a8674..ab6ad6a 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -348,12 +348,14 @@ class WeatherPacket(GPSPacket): """ time_zulu = self._build_time_zulu() + course = "%03u" % self.course + contents = [ f"{self.from_call}>{self.to_call},WIDE2-1:", f"@{time_zulu}z{self.latitude}{self.symbol_table}", f"{self.longitude}{self.symbol}", # Add CSE = Course - f"{self.course}", + f"{course}", # Speed = sustained 1 minute wind speed in mph f"{self.symbol_table}", f"{self.speed:03.0f}", # wind gust (peak wind speed in mph in the last 5 minutes) @@ -363,6 +365,8 @@ class WeatherPacket(GPSPacket): # Rainfall (in hundredths of an inch) in the last hour f"r{self.rain_1h:03.0f}", # Rainfall (in hundredths of an inch) in last 24 hours + f"p{self.rain_24h:03.0f}", + # Rainfall (in hundredths of an inch) since midnigt f"P{self.rain_since_midnight:03.0f}", # Humidity f"h{self.humidity:02d}", @@ -372,6 +376,7 @@ class WeatherPacket(GPSPacket): if self.comment: contents.append(self.comment) + self.raw = "".join(contents) From 088cbb81eddd3bf947e2e2bd970b0f3ac094bd41 Mon Sep 17 00:00:00 2001 From: Hemna Date: Tue, 20 Dec 2022 11:51:47 -0500 Subject: [PATCH 19/21] Update routing for weatherpacket update the routing to WIDE1-1, WIDE2-1 --- aprsd/packets/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index ab6ad6a..8638edf 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -351,7 +351,7 @@ class WeatherPacket(GPSPacket): course = "%03u" % self.course contents = [ - f"{self.from_call}>{self.to_call},WIDE2-1:", + f"{self.from_call}>{self.to_call},WIDE1-1,WIDE2-1:", f"@{time_zulu}z{self.latitude}{self.symbol_table}", f"{self.longitude}{self.symbol}", # Add CSE = Course From 220fb58f9779946c33429613ff397c7082fe84eb Mon Sep 17 00:00:00 2001 From: Hemna Date: Tue, 20 Dec 2022 15:13:13 -0500 Subject: [PATCH 20/21] Cleaned up PluginManager Added a separate pluggy track for normal plugins and watch list plugins. --- aprsd/plugin.py | 72 +++++++++++++++++------------------------ aprsd/plugins/notify.py | 37 ++++++++++----------- aprsd/threads/rx.py | 27 +++++++++++++++- 3 files changed, 72 insertions(+), 64 deletions(-) diff --git a/aprsd/plugin.py b/aprsd/plugin.py index db85a5d..0603666 100644 --- a/aprsd/plugin.py +++ b/aprsd/plugin.py @@ -1,16 +1,13 @@ # The base plugin class import abc -import fnmatch import importlib import inspect import logging -import os import re import textwrap import threading import pluggy -from thesmuggler import smuggle import aprsd from aprsd import client, packets, threads @@ -348,29 +345,9 @@ class PluginManager: def _init(self): self._pluggy_pm = pluggy.PluginManager("aprsd") self._pluggy_pm.add_hookspecs(APRSDPluginSpec) - - def load_plugins_from_path(self, module_path): - if not os.path.exists(module_path): - LOG.error(f"plugin path '{module_path}' doesn't exist.") - return None - - dir_path = os.path.realpath(module_path) - pattern = "*.py" - - self.obj_list = [] - - for path, _subdirs, files in os.walk(dir_path): - for name in files: - if fnmatch.fnmatch(name, pattern): - LOG.debug(f"MODULE? '{name}' '{path}'") - module = smuggle(f"{path}/{name}") - for mem_name, obj in inspect.getmembers(module): - if inspect.isclass(obj) and self.is_plugin(obj): - self.obj_list.append( - {"name": mem_name, "obj": obj(self.config)}, - ) - - return self.obj_list + # For the watchlist plugins + self._watchlist_pm = pluggy.PluginManager("aprsd") + self._watchlist_pm.add_hookspecs(APRSDPluginSpec) def is_plugin(self, obj): for c in inspect.getmro(obj): @@ -430,13 +407,22 @@ class PluginManager: config=self.config, ) if plugin_obj: - LOG.info( - "Registering plugin '{}'({})".format( - plugin_name, - plugin_obj.version, - ), - ) - self._pluggy_pm.register(plugin_obj) + if isinstance(plugin_obj, APRSDWatchListPluginBase): + LOG.info( + "Registering WatchList plugin '{}'({})".format( + plugin_name, + plugin_obj.version, + ), + ) + self._watchlist_pm.register(plugin_obj) + else: + LOG.info( + "Registering plugin '{}'({})".format( + plugin_name, + plugin_obj.version, + ), + ) + self._pluggy_pm.register(plugin_obj) except Exception as ex: LOG.error(f"Couldn't load plugin '{plugin_name}'") LOG.exception(ex) @@ -465,15 +451,6 @@ class PluginManager: for p_name in CORE_MESSAGE_PLUGINS: self._load_plugin(p_name) - if self.config["aprsd"]["watch_list"].get("enabled", False): - LOG.info("Loading APRSD WatchList Plugins") - enabled_notify_plugins = self.config["aprsd"]["watch_list"].get( - "enabled_plugins", - None, - ) - if enabled_notify_plugins: - for p_name in enabled_notify_plugins: - self._load_plugin(p_name) LOG.info("Completed Plugin Loading.") def run(self, packet): @@ -481,6 +458,10 @@ class PluginManager: with self.lock: return self._pluggy_pm.hook.filter(packet=packet) + def run_watchlist(self, packet): + with self.lock: + return self._watchlist_pm.hook.filter(packet=packet) + def stop(self): """Stop all threads created by all plugins.""" with self.lock: @@ -493,5 +474,10 @@ class PluginManager: self._pluggy_pm.register(obj) def get_plugins(self): + plugin_list = [] if self._pluggy_pm: - return self._pluggy_pm.get_plugins() + plugin_list.append(self._pluggy_pm.get_plugins()) + if self._watchlist_pm: + plugin_list.append(self._watchlist_pm.get_plugins()) + + return plugin_list diff --git a/aprsd/plugins/notify.py b/aprsd/plugins/notify.py index 15a9b22..9009c66 100644 --- a/aprsd/plugins/notify.py +++ b/aprsd/plugins/notify.py @@ -26,17 +26,17 @@ class NotifySeenPlugin(plugin.APRSDWatchListPluginBase): wl = packets.WatchList() age = wl.age(fromcall) - if wl.is_old(fromcall): - LOG.info( - "NOTIFY {} last seen {} max age={}".format( - fromcall, - age, - wl.max_delta(), - ), - ) - packet_type = packet.__class__.__name__ - # we shouldn't notify the alert user that they are online. - if fromcall != notify_callsign: + if fromcall != notify_callsign: + if wl.is_old(fromcall): + LOG.info( + "NOTIFY {} last seen {} max age={}".format( + fromcall, + age, + wl.max_delta(), + ), + ) + packet_type = packet.__class__.__name__ + # we shouldn't notify the alert user that they are online. pkt = packets.MessagePacket( from_call=self.config["aprsd"]["callsign"], to_call=notify_callsign, @@ -45,17 +45,14 @@ class NotifySeenPlugin(plugin.APRSDWatchListPluginBase): ), allow_delay=False, ) - # pkt.allow_delay = False + pkt.allow_delay = False return pkt else: - LOG.debug("fromcall and notify_callsign are the same, not notifying") + LOG.debug( + "Not old enough to notify on callsign " + f"'{fromcall}' : {age} < {wl.max_delta()}", + ) return packets.NULL_MESSAGE else: - LOG.debug( - "Not old enough to notify on callsign '{}' : {} < {}".format( - fromcall, - age, - wl.max_delta(), - ), - ) + LOG.debug("fromcall and notify_callsign are the same, ignoring") return packets.NULL_MESSAGE diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index 4db1577..3df02fe 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -159,7 +159,6 @@ class APRSDProcessPacketThread(APRSDThread): LOG.info("Got a packet not meant for us.") else: LOG.info("Got a non AckPacket/MessagePacket") - LOG.info(packet) class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): @@ -167,6 +166,32 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): This is the main aprsd server plugin processing thread.""" + def process_other_packet(self, packet, for_us=False): + pm = plugin.PluginManager() + try: + results = pm.run_watchlist(packet) + for reply in results: + if isinstance(reply, list): + for subreply in reply: + LOG.debug(f"Sending '{subreply}'") + if isinstance(subreply, packets.Packet): + subreply.send() + else: + to_call = self.config["aprsd"]["watch_list"]["alert_callsign"] + msg_pkt = packets.MessagePacket( + from_call=self.config["aprsd"]["callsign"], + to_call=to_call, + message_text=subreply, + ) + msg_pkt.send() + elif isinstance(reply, packets.Packet): + # We have a message based object. + reply.send() + except Exception as ex: + LOG.error("Plugin failed!!!") + LOG.exception(ex) + + def process_our_message_packet(self, packet): """Send the packet through the plugins.""" from_call = packet.from_call From 7dfa4e6dbf1924b5ce1061e6be2d1a00ffd095d8 Mon Sep 17 00:00:00 2001 From: Hemna Date: Tue, 20 Dec 2022 15:13:13 -0500 Subject: [PATCH 21/21] Cleaned up PluginManager Added a separate pluggy track for normal plugins and watch list plugins. --- aprsd/threads/rx.py | 1 - 1 file changed, 1 deletion(-) diff --git a/aprsd/threads/rx.py b/aprsd/threads/rx.py index 3df02fe..b860c46 100644 --- a/aprsd/threads/rx.py +++ b/aprsd/threads/rx.py @@ -191,7 +191,6 @@ class APRSDPluginProcessPacketThread(APRSDProcessPacketThread): LOG.error("Plugin failed!!!") LOG.exception(ex) - def process_our_message_packet(self, packet): """Send the packet through the plugins.""" from_call = packet.from_call