diff --git a/aprsd/main.py b/aprsd/main.py index 1f5a4eb..f55be0b 100644 --- a/aprsd/main.py +++ b/aprsd/main.py @@ -145,6 +145,8 @@ def sample_config(ctx): if not sys.argv[1:]: raise SystemExit raise + LOG.warning(conf.namespace) + return generator.generate(conf) diff --git a/aprsd/packets/core.py b/aprsd/packets/core.py index 1b3dd8c..c9362d8 100644 --- a/aprsd/packets/core.py +++ b/aprsd/packets/core.py @@ -1,7 +1,6 @@ import abc from dataclasses import asdict, dataclass, field -import datetime -import json +from datetime import datetime import logging import re import time @@ -9,9 +8,9 @@ import time from typing import List import dacite +from dataclasses_json import dataclass_json from aprsd.utils import counter -from aprsd.utils import json as aprsd_json LOG = logging.getLogger("APRSD") @@ -28,12 +27,20 @@ PACKET_TYPE_BEACON = "beacon" PACKET_TYPE_THIRDPARTY = "thirdparty" PACKET_TYPE_UNCOMPRESSED = "uncompressed" +NO_DATE = datetime(1900, 10, 24) + def _init_timestamp(): """Build a unix style timestamp integer""" return int(round(time.time())) +def _init_send_time(): + # We have to use a datetime here, or the json encoder + # Fails on a NoneType. + return NO_DATE + + def _init_msgNo(): # noqa: N802 """For some reason __post__init doesn't get called. @@ -45,6 +52,20 @@ def _init_msgNo(): # noqa: N802 return c.value +def factory_from_dict(packet_dict): + pkt_type = get_packet_type(packet_dict) + if pkt_type: + cls = TYPE_LOOKUP[pkt_type] + return cls.from_dict(packet_dict) + + +def factory_from_json(packet_dict): + pkt_type = get_packet_type(packet_dict) + if pkt_type: + return TYPE_LOOKUP[pkt_type].from_json(packet_dict) + + +@dataclass_json @dataclass(unsafe_hash=True) class Packet(metaclass=abc.ABCMeta): from_call: str = field(default=None) @@ -64,7 +85,19 @@ class Packet(metaclass=abc.ABCMeta): # Fields related to sending packets out send_count: int = field(repr=False, default=0, compare=False, hash=False) retry_count: int = field(repr=False, default=3, compare=False, hash=False) - last_send_time: datetime.timedelta = field(repr=False, default=None, compare=False, hash=False) + # last_send_time: datetime = field( + # metadata=dc_json_config( + # encoder=datetime.isoformat, + # decoder=datetime.fromisoformat, + # ), + # repr=True, + # default_factory=_init_send_time, + # compare=False, + # hash=False + # ) + last_send_time: float = field(repr=False, default=0, compare=False, hash=False) + last_send_attempt: int = field(repr=False, default=0, compare=False, hash=False) + # Do we allow this packet to be saved to send later? allow_delay: bool = field(repr=False, default=True, compare=False, hash=False) path: List[str] = field(default_factory=list, compare=False, hash=False) @@ -73,16 +106,12 @@ class Packet(metaclass=abc.ABCMeta): def __post__init__(self): LOG.warning(f"POST INIT {self}") - @property - def __dict__(self): - return asdict(self) - @property def json(self): """ get the json formated string """ - return json.dumps(self.__dict__, cls=aprsd_json.EnhancedJSONEncoder) + return self.to_json() def get(self, key, default=None): """Emulate a getter on a dict.""" @@ -289,6 +318,7 @@ class RejectPacket(Packet): self.payload = f":{self.to_call.ljust(9)} :rej{self.msgNo}" +@dataclass_json @dataclass(unsafe_hash=True) class MessagePacket(Packet): message_text: str = field(default=None) @@ -406,12 +436,12 @@ class GPSPacket(Packet): def _build_time_zulu(self): """Build the timestamp in UTC/zulu.""" if self.timestamp: - local_dt = datetime.datetime.fromtimestamp(self.timestamp) + local_dt = datetime.fromtimestamp(self.timestamp) else: - local_dt = datetime.datetime.now() - self.timestamp = datetime.datetime.timestamp(local_dt) + local_dt = datetime.now() + self.timestamp = datetime.timestamp(local_dt) - utc_offset_timedelta = datetime.datetime.utcnow() - local_dt + utc_offset_timedelta = datetime.utcnow() - local_dt result_utc_datetime = local_dt + utc_offset_timedelta time_zulu = result_utc_datetime.strftime("%d%H%M") return time_zulu @@ -567,7 +597,7 @@ class WeatherPacket(GPSPacket): class ThirdParty(Packet): # Holds the encapsulated packet - subpacket: Packet = None + subpacket: Packet = field(default=None, compare=True, hash=False) def __repr__(self): """Build the repr version of the packet.""" @@ -600,7 +630,7 @@ def get_packet_type(packet: dict): pkt_format = packet.get("format", None) msg_response = packet.get("response", None) - packet_type = "unknown" + packet_type = PACKET_TYPE_UNKNOWN if pkt_format == "message" and msg_response == "ack": packet_type = PACKET_TYPE_ACK elif pkt_format == "message" and msg_response == "rej": @@ -620,6 +650,10 @@ def get_packet_type(packet: dict): packet_type = PACKET_TYPE_WX elif pkt_format == PACKET_TYPE_THIRDPARTY: packet_type = PACKET_TYPE_THIRDPARTY + + if packet_type == PACKET_TYPE_UNKNOWN: + if "latitude" in packet: + packet_type = PACKET_TYPE_BEACON return packet_type diff --git a/aprsd/packets/packet_list.py b/aprsd/packets/packet_list.py index a6dc6f7..3d75a07 100644 --- a/aprsd/packets/packet_list.py +++ b/aprsd/packets/packet_list.py @@ -19,11 +19,12 @@ class PacketList(MutableMapping): lock = threading.Lock() _total_rx: int = 0 _total_tx: int = 0 + types = {} def __new__(cls, *args, **kwargs): if cls._instance is None: cls._instance = super().__new__(cls) - cls._maxlen = 1000 + cls._maxlen = 100 cls.d = OrderedDict() return cls._instance @@ -32,6 +33,10 @@ class PacketList(MutableMapping): """Add a packet that was received.""" self._total_rx += 1 self._add(packet) + ptype = packet.__class__.__name__ + if not ptype in self.types: + self.types[ptype] = {"tx": 0, "rx": 0} + self.types[ptype]["rx"] += 1 seen_list.SeenList().update_seen(packet) stats.APRSDStats().rx(packet) @@ -40,6 +45,10 @@ class PacketList(MutableMapping): """Add a packet that was received.""" self._total_tx += 1 self._add(packet) + ptype = packet.__class__.__name__ + if not ptype in self.types: + self.types[ptype] = {"tx": 0, "rx": 0} + self.types[ptype]["tx"] += 1 seen_list.SeenList().update_seen(packet) stats.APRSDStats().tx(packet) @@ -50,6 +59,9 @@ class PacketList(MutableMapping): def _add(self, packet): self[packet.key] = packet + def copy(self): + return self.d.copy() + @property def maxlen(self): return self._maxlen diff --git a/aprsd/packets/tracker.py b/aprsd/packets/tracker.py index b512320..1246dc1 100644 --- a/aprsd/packets/tracker.py +++ b/aprsd/packets/tracker.py @@ -65,6 +65,7 @@ class PacketTrack(objectstore.ObjectStoreMixin): @wrapt.synchronized(lock) def add(self, packet): key = packet.msgNo + packet._last_send_attempt = 0 self.data[key] = packet self.total_tracked += 1 @@ -83,7 +84,7 @@ class PacketTrack(objectstore.ObjectStoreMixin): """Walk the list of messages and restart them if any.""" for key in self.data.keys(): pkt = self.data[key] - if pkt.last_send_attempt < pkt.retry_count: + if pkt._last_send_attempt < pkt.retry_count: tx.send(pkt) def _resend(self, packet): diff --git a/aprsd/threads/tx.py b/aprsd/threads/tx.py index 0ceb355..708d263 100644 --- a/aprsd/threads/tx.py +++ b/aprsd/threads/tx.py @@ -1,4 +1,3 @@ -import datetime import logging import time @@ -128,10 +127,10 @@ class SendPacketThread(aprsd_threads.APRSDThread): # Message is still outstanding and needs to be acked. if packet.last_send_time: # Message has a last send time tracking - now = datetime.datetime.now() + now = int(round(time.time())) sleeptime = (packet.send_count + 1) * 31 delta = now - packet.last_send_time - if delta > datetime.timedelta(seconds=sleeptime): + if delta > sleeptime: # It's time to try to send it again send_now = True else: @@ -140,7 +139,7 @@ class SendPacketThread(aprsd_threads.APRSDThread): if send_now: # no attempt time, so lets send it, and start # tracking the time. - packet.last_send_time = datetime.datetime.now() + packet.last_send_time = int(round(time.time())) send(packet, direct=True) packet.send_count += 1 @@ -173,13 +172,13 @@ class SendAckThread(aprsd_threads.APRSDThread): if self.packet.last_send_time: # Message has a last send time tracking - now = datetime.datetime.now() + now = int(round(time.time())) # aprs duplicate detection is 30 secs? # (21 only sends first, 28 skips middle) sleep_time = 31 delta = now - self.packet.last_send_time - if delta > datetime.timedelta(seconds=sleep_time): + if delta > sleep_time: # It's time to try to send it again send_now = True elif self.loop_count % 10 == 0: @@ -190,7 +189,7 @@ class SendAckThread(aprsd_threads.APRSDThread): if send_now: send(self.packet, direct=True) self.packet.send_count += 1 - self.packet.last_send_time = datetime.datetime.now() + self.packet.last_send_time = int(round(time.time())) time.sleep(1) self.loop_count += 1 diff --git a/aprsd/web/admin/static/js/echarts.js b/aprsd/web/admin/static/js/echarts.js new file mode 100644 index 0000000..adeb5f6 --- /dev/null +++ b/aprsd/web/admin/static/js/echarts.js @@ -0,0 +1,403 @@ +var packet_list = {}; + +var tx_data = []; +var rx_data = []; + +var packet_types_data = {}; + +var mem_current = [] +var mem_peak = [] + + +function start_charts() { + console.log("start_charts() called"); + // Initialize the echarts instance based on the prepared dom + create_packets_chart(); + create_packets_types_chart(); + create_messages_chart(); + create_ack_chart(); + create_memory_chart(); +} + + +function create_packets_chart() { + // The packets totals TX/RX chart. + pkt_c_canvas = document.getElementById('packetsChart'); + packets_chart = echarts.init(pkt_c_canvas); + + // Specify the configuration items and data for the chart + var option = { + title: { + text: 'APRS Packet totals' + }, + legend: {}, + tooltip : { + trigger: 'axis' + }, + toolbox: { + show : true, + feature : { + mark : {show: true}, + dataView : {show: true, readOnly: true}, + magicType : {show: true, type: ['line', 'bar']}, + restore : {show: true}, + saveAsImage : {show: true} + } + }, + calculable : true, + xAxis: { type: 'time' }, + yAxis: { }, + series: [ + { + name: 'tx', + type: 'line', + smooth: true, + color: 'red', + encode: { + x: 'timestamp', + y: 'tx' // refer sensor 1 value + } + },{ + name: 'rx', + type: 'line', + smooth: true, + encode: { + x: 'timestamp', + y: 'rx' + } + }] + }; + + // Display the chart using the configuration items and data just specified. + packets_chart.setOption(option); +} + + +function create_packets_types_chart() { + // The packets types chart + pkt_types_canvas = document.getElementById('packetTypesChart'); + packet_types_chart = echarts.init(pkt_types_canvas); + + // The series and data are built and updated on the fly + // as packets come in. + var option = { + title: { + text: 'Packet Types' + }, + legend: {}, + tooltip : { + trigger: 'axis' + }, + toolbox: { + show : true, + feature : { + mark : {show: true}, + dataView : {show: true, readOnly: true}, + magicType : {show: true, type: ['line', 'bar']}, + restore : {show: true}, + saveAsImage : {show: true} + } + }, + calculable : true, + xAxis: { type: 'time' }, + yAxis: { }, + } + + packet_types_chart.setOption(option); +} + + +function create_messages_chart() { + msg_c_canvas = document.getElementById('messagesChart'); + message_chart = echarts.init(msg_c_canvas); + + // Specify the configuration items and data for the chart + var option = { + title: { + text: 'Message Packets' + }, + legend: {}, + tooltip: { + trigger: 'axis' + }, + toolbox: { + show: true, + feature: { + mark : {show: true}, + dataView : {show: true, readOnly: true}, + magicType : {show: true, type: ['line', 'bar']}, + restore : {show: true}, + saveAsImage : {show: true} + } + }, + calculable: true, + xAxis: { type: 'time' }, + yAxis: { }, + series: [ + { + name: 'tx', + type: 'line', + smooth: true, + color: 'red', + encode: { + x: 'timestamp', + y: 'tx' // refer sensor 1 value + } + },{ + name: 'rx', + type: 'line', + smooth: true, + encode: { + x: 'timestamp', + y: 'rx' + } + }] + }; + + // Display the chart using the configuration items and data just specified. + message_chart.setOption(option); +} + +function create_ack_chart() { + ack_canvas = document.getElementById('acksChart'); + ack_chart = echarts.init(ack_canvas); + + // Specify the configuration items and data for the chart + var option = { + title: { + text: 'Ack Packets' + }, + legend: {}, + tooltip: { + trigger: 'axis' + }, + toolbox: { + show: true, + feature: { + mark : {show: true}, + dataView : {show: true, readOnly: false}, + magicType : {show: true, type: ['line', 'bar']}, + restore : {show: true}, + saveAsImage : {show: true} + } + }, + calculable: true, + xAxis: { type: 'time' }, + yAxis: { }, + series: [ + { + name: 'tx', + type: 'line', + smooth: true, + color: 'red', + encode: { + x: 'timestamp', + y: 'tx' // refer sensor 1 value + } + },{ + name: 'rx', + type: 'line', + smooth: true, + encode: { + x: 'timestamp', + y: 'rx' + } + }] + }; + + ack_chart.setOption(option); +} + +function create_memory_chart() { + ack_canvas = document.getElementById('memChart'); + memory_chart = echarts.init(ack_canvas); + + // Specify the configuration items and data for the chart + var option = { + title: { + text: 'Memory Usage' + }, + legend: {}, + tooltip: { + trigger: 'axis' + }, + toolbox: { + show: true, + feature: { + mark : {show: true}, + dataView : {show: true, readOnly: false}, + magicType : {show: true, type: ['line', 'bar']}, + restore : {show: true}, + saveAsImage : {show: true} + } + }, + calculable: true, + xAxis: { type: 'time' }, + yAxis: { }, + series: [ + { + name: 'current', + type: 'line', + smooth: true, + color: 'red', + encode: { + x: 'timestamp', + y: 'current' // refer sensor 1 value + } + },{ + name: 'peak', + type: 'line', + smooth: true, + encode: { + x: 'timestamp', + y: 'peak' + } + }] + }; + + memory_chart.setOption(option); +} + + + + +function updatePacketData(chart, time, first, second) { + tx_data.push([time, first]); + rx_data.push([time, second]); + option = { + series: [ + { + name: 'tx', + data: tx_data, + }, + { + name: 'rx', + data: rx_data, + } + ] + } + chart.setOption(option); +} + +function updatePacketTypesData(time, typesdata) { + //The options series is created on the fly each time based on + //the packet types we have in the data + var series = [] + + for (const k in typesdata) { + tx = [time, typesdata[k]["tx"]] + rx = [time, typesdata[k]["rx"]] + + if (packet_types_data.hasOwnProperty(k)) { + packet_types_data[k]["tx"].push(tx) + packet_types_data[k]["rx"].push(rx) + } else { + packet_types_data[k] = {'tx': [tx], 'rx': [rx]} + } + } +} + +function updatePacketTypesChart() { + series = [] + for (const k in packet_types_data) { + entry = { + name: k+"tx", + data: packet_types_data[k]["tx"], + type: 'line', + smooth: true, + encode: { + x: 'timestamp', + y: k+'tx' // refer sensor 1 value + } + } + series.push(entry) + entry = { + name: k+"rx", + data: packet_types_data[k]["rx"], + type: 'line', + smooth: true, + encode: { + x: 'timestamp', + y: k+'rx' // refer sensor 1 value + } + } + series.push(entry) + } + + option = { + series: series + } + console.log(option) + packet_types_chart.setOption(option); +} + +function updateTypeChart(chart, key) { + //Generic function to update a packet type chart + if (! packet_types_data.hasOwnProperty(key)) { + return; + } + + if (! packet_types_data[key].hasOwnProperty('tx')) { + return; + } + var option = { + series: [{ + name: "tx", + data: packet_types_data[key]["tx"], + }, + { + name: "rx", + data: packet_types_data[key]["rx"] + }] + } + + chart.setOption(option); +} + +function updateMemChart(time, current, peak) { + mem_current.push([time, current]); + mem_peak.push([time, peak]); + option = { + series: [ + { + name: 'current', + data: mem_current, + }, + { + name: 'peak', + data: mem_peak, + } + ] + } + memory_chart.setOption(option); +} + +function updateMessagesChart() { + updateTypeChart(message_chart, "MessagePacket") +} + +function updateAcksChart() { + updateTypeChart(ack_chart, "AckPacket") +} + +function update_stats( data ) { + console.log(data); + our_callsign = data["stats"]["aprsd"]["callsign"]; + $("#version").text( data["stats"]["aprsd"]["version"] ); + $("#aprs_connection").html( data["aprs_connection"] ); + $("#uptime").text( "uptime: " + data["stats"]["aprsd"]["uptime"] ); + const html_pretty = Prism.highlight(JSON.stringify(data, null, '\t'), Prism.languages.json, 'json'); + $("#jsonstats").html(html_pretty); + + t = Date.parse(data["time"]); + ts = new Date(t); + updatePacketData(packets_chart, ts, data["stats"]["packets"]["sent"], data["stats"]["packets"]["received"]); + updatePacketTypesData(ts, data["stats"]["packets"]["types"]); + updatePacketTypesChart(); + updateMessagesChart(); + updateAcksChart(); + updateMemChart(ts, data["stats"]["aprsd"]["memory_current"], data["stats"]["aprsd"]["memory_peak"]); + //updateQuadData(message_chart, short_time, data["stats"]["messages"]["sent"], data["stats"]["messages"]["received"], data["stats"]["messages"]["ack_sent"], data["stats"]["messages"]["ack_recieved"]); + //updateDualData(email_chart, short_time, data["stats"]["email"]["sent"], data["stats"]["email"]["recieved"]); + //updateDualData(memory_chart, short_time, data["stats"]["aprsd"]["memory_peak"], data["stats"]["aprsd"]["memory_current"]); +} diff --git a/aprsd/web/admin/templates/index.html b/aprsd/web/admin/templates/index.html index fe9bac1..3bd95bb 100644 --- a/aprsd/web/admin/templates/index.html +++ b/aprsd/web/admin/templates/index.html @@ -6,6 +6,7 @@ + @@ -15,7 +16,7 @@ - + @@ -83,6 +84,7 @@
{{ stats }}-
+
+
{{ stats|safe }}+
{{ stats|safe }}