From 10c43e83bd2a909f983039f4c0ed047a3f87eebb Mon Sep 17 00:00:00 2001 From: Abigail Gold Date: Tue, 15 Oct 2019 17:37:13 -0400 Subject: [PATCH 01/11] basic functionality for QRZ rich lookup still have to handle some errors (call not found, unable to login, etc) and display the error. also probably don't need to open a new session for every invocation. When this gets merged, remember to delete the old command in lookupcog. --- cogs/qrzcog.py | 129 ++++++++++++++++++++++++++++++++++ requirements.txt | 1 + templates/template_keys.py | 2 + templates/template_options.py | 2 +- 4 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 cogs/qrzcog.py diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py new file mode 100644 index 0000000..bb7931f --- /dev/null +++ b/cogs/qrzcog.py @@ -0,0 +1,129 @@ +""" +QRZ cog for qrm +--- +Copyright (C) 2019 Abigail Gold, 0x5c + +This file is part of discord-qrmbot and is released under the terms of the GNU +General Public License, version 2. +""" +from collections import OrderedDict + +import discord +import discord.ext.commands as commands + +from datetime import datetime +import aiohttp +from bs4 import BeautifulSoup + + +class QRZCog(commands.Cog): + def __init__(self, bot: commands.Bot): + self.bot = bot + self.gs = bot.get_cog("GlobalSettings") + + @commands.command(name="qrz", aliases=["call"]) + async def _qrz_lookup(self, ctx: commands.Context, call: str): + '''Links to info about a callsign from QRZ.''' + if self.gs.keys.qrz_user == '' or self.gs.keys.qrz_pass == '': + await ctx.send(f'http://qrz.com/db/{call}') + return + try: + # TODO: see if there's a key first (i.e. don't log in every time) + # TODO: maybe make it a task to generate a key? + key = await _qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + except ConnectionError as err: + print(err) + url = f'http://xmldata.qrz.com/xml/current/?s={key};callsign={call}' + async with aiohttp.ClientSession() as session: + async with session.get(url) as resp: + if resp.status != 200: + raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') + resp_xml = await resp.text() + + xml_soup = BeautifulSoup(resp_xml, "xml") + resp_data = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Callsign *')} + resp_session = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} + if 'Error' in resp_session: + raise ValueError(resp_session['Error']) + + embed = discord.Embed(title=f"QRZ Data for {resp_data['call']}", + colour=self.gs.colours.good, + url=f'http://www.qrz.com/db/{resp_data["call"]}', + timestamp=datetime.utcnow()) + embed.set_footer(text=ctx.author.name, + icon_url=str(ctx.author.avatar_url)) + if 'image' in resp_data: + embed.set_image(url=resp_data['image']) + + if 'name' in resp_data: + if 'fname' in resp_data: + name = resp_data['fname'] + ' ' + resp_data['name'] + else: + name = resp_data['name'] + else: + name = None + if 'state' in resp_data: + state = f', {resp_data["state"]}' + else: + state = '' + address = resp_data.get('addr1', '') + '\n' + resp_data.get('addr2', '') + \ + state + ' ' + resp_data.get('zip', '') + if 'eqsl' in resp_data: + eqsl = 'Yes' if resp_data['eqsl'] == 1 else 'No' + else: + eqsl = 'Unknown' + if 'mqsl' in resp_data: + mqsl = 'Yes' if resp_data['mqsl'] == 1 else 'No' + else: + mqsl = 'Unknown' + if 'lotw' in resp_data: + lotw = 'Yes' if resp_data['lotw'] == 1 else 'No' + else: + lotw = 'Unknown' + + data = OrderedDict([('Name', name), + ('Country', resp_data.get('country', None)), + ('Address', address), + ('Grid Square', resp_data.get('grid', None)), + ('County', resp_data.get('county', None)), + ('CQ Zone', resp_data.get('cqzone', None)), + ('ITU Zone', resp_data.get('ituzone', None)), + ('IOTA Designator', resp_data.get('iota', None)), + ('Expires', resp_data.get('expdate', None)), + ('Aliases', resp_data.get('aliases', None)), + ('Previous Callsign', resp_data.get('p_call', None)), + ('License Class', resp_data.get('class', None)), + ('eQSL?', eqsl), + ('Paper QSL?', mqsl), + ('LotW?', lotw), + ('QSL Info', resp_data.get('qslmgr', None)), + ('CQ Zone', resp_data.get('cqzone', None)), + ('ITU Zone', resp_data.get('ituzone', None)), + ('IOTA Designator', resp_data.get('iota', None)), + ('Born', resp_data.get('born', None)), + ]) + for title, val in data.items(): + if val is not None: + embed.add_field(name=title, value=val, inline=True) + await ctx.send(embed=embed) + + +async def _qrz_login(user: str, passwd: str): + url = f'http://xmldata.qrz.com/xml/current/?username={user};password={passwd};agent=qrmbot' + async with aiohttp.ClientSession() as session: + async with session.get(url) as resp: + if resp.status != 200: + raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') + resp_xml = await resp.text() + + xml_soup = BeautifulSoup(resp_xml, "xml") + resp_data = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} + if 'Error' in resp_data: + raise ConnectionError(resp_data['Error']) + if resp_data['SubExp'] == 'non-subscriber': + raise ConnectionError('Invalid QRZ Subscription') + return resp_data['Key'] + + +def setup(bot): + bot.add_cog(QRZCog(bot)) diff --git a/requirements.txt b/requirements.txt index 11e81ce..65d4abd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ discord.py beautifulsoup4 +lxml diff --git a/templates/template_keys.py b/templates/template_keys.py index 28ce5ef..7f6279d 100644 --- a/templates/template_keys.py +++ b/templates/template_keys.py @@ -14,3 +14,5 @@ API keys and tokens for the bot. # The Discord bot token discord_token = "" +qrz_user = "" +qrz_pass = "" diff --git a/templates/template_options.py b/templates/template_options.py index d93895d..f9a9b39 100644 --- a/templates/template_options.py +++ b/templates/template_options.py @@ -27,7 +27,7 @@ owners_uids = (200102491231092736,) # The cogs to load when running the bot. cogs = ['basecog', 'morsecog', 'funcog', 'gridcog', 'hamcog', 'imagecog', - 'studycog', 'ae7qcog'] + 'studycog', 'ae7qcog', 'qrzcog'] # The text to put in the "playing" status. game = 'with lids on 7.200' From 907065e71848df3d9462903d2157c2544d975ed8 Mon Sep 17 00:00:00 2001 From: Abigail Gold Date: Wed, 16 Oct 2019 17:23:56 -0400 Subject: [PATCH 02/11] (NOT FUNCTIONAL) progress on qrz cmd --- cogs/qrzcog.py | 145 +++++++++++++++++++++++++++++++------------------ 1 file changed, 91 insertions(+), 54 deletions(-) diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py index bb7931f..7e206cf 100644 --- a/cogs/qrzcog.py +++ b/cogs/qrzcog.py @@ -7,11 +7,11 @@ This file is part of discord-qrmbot and is released under the terms of the GNU General Public License, version 2. """ from collections import OrderedDict +from datetime import datetime import discord import discord.ext.commands as commands -from datetime import datetime import aiohttp from bs4 import BeautifulSoup @@ -20,6 +20,18 @@ class QRZCog(commands.Cog): def __init__(self, bot: commands.Bot): self.bot = bot self.gs = bot.get_cog("GlobalSettings") + try: + with open('data/qrz_session') as qrz_file: + self.key = qrz_file.readline().strip() + qrz_test_session(self.key) + except FileNotFoundError: + self.key = qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + with open('data/qrz_session', 'w') as qrz_file: + qrz_file.write(self.key) + except ConnectionError: + self.key = qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + with open('data/qrz_session', 'w') as qrz_file: + qrz_file.write(self.key) @commands.command(name="qrz", aliases=["call"]) async def _qrz_lookup(self, ctx: commands.Context, call: str): @@ -27,13 +39,15 @@ class QRZCog(commands.Cog): if self.gs.keys.qrz_user == '' or self.gs.keys.qrz_pass == '': await ctx.send(f'http://qrz.com/db/{call}') return + try: - # TODO: see if there's a key first (i.e. don't log in every time) - # TODO: maybe make it a task to generate a key? - key = await _qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) - except ConnectionError as err: - print(err) - url = f'http://xmldata.qrz.com/xml/current/?s={key};callsign={call}' + await qrz_test_session(self.key) + except ConnectionError: + self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + with open('data/qrz_session', 'w') as qrz_file: + qrz_file.write(self.key) + + url = f'http://xmldata.qrz.com/xml/current/?s={self.key};callsign={call}' async with aiohttp.ClientSession() as session: async with session.get(url) as resp: if resp.status != 200: @@ -41,6 +55,8 @@ class QRZCog(commands.Cog): resp_xml = await resp.text() xml_soup = BeautifulSoup(resp_xml, "xml") + print(xml_soup) + resp_data = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Callsign *')} resp_session = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} if 'Error' in resp_session: @@ -55,60 +71,15 @@ class QRZCog(commands.Cog): if 'image' in resp_data: embed.set_image(url=resp_data['image']) - if 'name' in resp_data: - if 'fname' in resp_data: - name = resp_data['fname'] + ' ' + resp_data['name'] - else: - name = resp_data['name'] - else: - name = None - if 'state' in resp_data: - state = f', {resp_data["state"]}' - else: - state = '' - address = resp_data.get('addr1', '') + '\n' + resp_data.get('addr2', '') + \ - state + ' ' + resp_data.get('zip', '') - if 'eqsl' in resp_data: - eqsl = 'Yes' if resp_data['eqsl'] == 1 else 'No' - else: - eqsl = 'Unknown' - if 'mqsl' in resp_data: - mqsl = 'Yes' if resp_data['mqsl'] == 1 else 'No' - else: - mqsl = 'Unknown' - if 'lotw' in resp_data: - lotw = 'Yes' if resp_data['lotw'] == 1 else 'No' - else: - lotw = 'Unknown' + data = await qrz_process_info(resp_data) - data = OrderedDict([('Name', name), - ('Country', resp_data.get('country', None)), - ('Address', address), - ('Grid Square', resp_data.get('grid', None)), - ('County', resp_data.get('county', None)), - ('CQ Zone', resp_data.get('cqzone', None)), - ('ITU Zone', resp_data.get('ituzone', None)), - ('IOTA Designator', resp_data.get('iota', None)), - ('Expires', resp_data.get('expdate', None)), - ('Aliases', resp_data.get('aliases', None)), - ('Previous Callsign', resp_data.get('p_call', None)), - ('License Class', resp_data.get('class', None)), - ('eQSL?', eqsl), - ('Paper QSL?', mqsl), - ('LotW?', lotw), - ('QSL Info', resp_data.get('qslmgr', None)), - ('CQ Zone', resp_data.get('cqzone', None)), - ('ITU Zone', resp_data.get('ituzone', None)), - ('IOTA Designator', resp_data.get('iota', None)), - ('Born', resp_data.get('born', None)), - ]) for title, val in data.items(): if val is not None: embed.add_field(name=title, value=val, inline=True) await ctx.send(embed=embed) -async def _qrz_login(user: str, passwd: str): +async def qrz_login(user: str, passwd: str): url = f'http://xmldata.qrz.com/xml/current/?username={user};password={passwd};agent=qrmbot' async with aiohttp.ClientSession() as session: async with session.get(url) as resp: @@ -125,5 +96,71 @@ async def _qrz_login(user: str, passwd: str): return resp_data['Key'] +async def qrz_test_session(key: str): + url = f'http://xmldata.qrz.com/xml/current/?s={key}' + async with aiohttp.ClientSession() as session: + async with session.get(url) as resp: + if resp.status != 200: + raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') + resp_xml = await resp.text() + + xml_soup = BeautifulSoup(resp_xml, "xml") + print(xml_soup) + + resp_session = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} + if 'Error' in resp_session: + raise ConnectionError(resp_session['Error']) + + +async def qrz_process_info(data: dict): + if 'name' in data: + if 'fname' in data: + name = data['fname'] + ' ' + data['name'] + else: + name = data['name'] + else: + name = None + if 'state' in data: + state = f', {data["state"]}' + else: + state = '' + address = data.get('addr1', '') + '\n' + data.get('addr2', '') + \ + state + ' ' + data.get('zip', '') + if 'eqsl' in data: + eqsl = 'Yes' if data['eqsl'] == 1 else 'No' + else: + eqsl = 'Unknown' + if 'mqsl' in data: + mqsl = 'Yes' if data['mqsl'] == 1 else 'No' + else: + mqsl = 'Unknown' + if 'lotw' in data: + lotw = 'Yes' if data['lotw'] == 1 else 'No' + else: + lotw = 'Unknown' + + return OrderedDict([('Name', name), + ('Country', data.get('country', None)), + ('Address', address), + ('Grid Square', data.get('grid', None)), + ('County', data.get('county', None)), + ('CQ Zone', data.get('cqzone', None)), + ('ITU Zone', data.get('ituzone', None)), + ('IOTA Designator', data.get('iota', None)), + ('Expires', data.get('expdate', None)), + ('Aliases', data.get('aliases', None)), + ('Previous Callsign', data.get('p_call', None)), + ('License Class', data.get('class', None)), + ('eQSL?', eqsl), + ('Paper QSL?', mqsl), + ('LotW?', lotw), + ('QSL Info', data.get('qslmgr', None)), + ('CQ Zone', data.get('cqzone', None)), + ('ITU Zone', data.get('ituzone', None)), + ('IOTA Designator', data.get('iota', None)), + ('Born', data.get('born', None)), + ]) + + def setup(bot): bot.add_cog(QRZCog(bot)) From 288160eb325db53d3fee410c6271a873dd750d5d Mon Sep 17 00:00:00 2001 From: Abigail Gold Date: Wed, 16 Oct 2019 17:45:11 -0400 Subject: [PATCH 03/11] (NOT FUNCTIONAL) small change to async --- cogs/qrzcog.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py index 7e206cf..c2528fd 100644 --- a/cogs/qrzcog.py +++ b/cogs/qrzcog.py @@ -23,13 +23,13 @@ class QRZCog(commands.Cog): try: with open('data/qrz_session') as qrz_file: self.key = qrz_file.readline().strip() - qrz_test_session(self.key) + await qrz_test_session(self.key) except FileNotFoundError: - self.key = qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) with open('data/qrz_session', 'w') as qrz_file: qrz_file.write(self.key) except ConnectionError: - self.key = qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) with open('data/qrz_session', 'w') as qrz_file: qrz_file.write(self.key) @@ -55,11 +55,14 @@ class QRZCog(commands.Cog): resp_xml = await resp.text() xml_soup = BeautifulSoup(resp_xml, "xml") - print(xml_soup) resp_data = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Callsign *')} resp_session = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} if 'Error' in resp_session: + if 'Session Timeout' in resp_session['Error']: + self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + with open('data/qrz_session', 'w') as qrz_file: + qrz_file.write(self.key) raise ValueError(resp_session['Error']) embed = discord.Embed(title=f"QRZ Data for {resp_data['call']}", From a0c224654b9ab37519bc3b69b98fd24f1151edeb Mon Sep 17 00:00:00 2001 From: Abigail Gold Date: Thu, 17 Oct 2019 22:29:53 -0400 Subject: [PATCH 04/11] slight changes --- cogs/qrzcog.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py index c2528fd..d213e8b 100644 --- a/cogs/qrzcog.py +++ b/cogs/qrzcog.py @@ -34,8 +34,6 @@ class QRZCog(commands.Cog): qrz_file.write(self.key) @commands.command(name="qrz", aliases=["call"]) - async def _qrz_lookup(self, ctx: commands.Context, call: str): - '''Links to info about a callsign from QRZ.''' if self.gs.keys.qrz_user == '' or self.gs.keys.qrz_pass == '': await ctx.send(f'http://qrz.com/db/{call}') return @@ -74,7 +72,7 @@ class QRZCog(commands.Cog): if 'image' in resp_data: embed.set_image(url=resp_data['image']) - data = await qrz_process_info(resp_data) + data = qrz_process_info(resp_data) for title, val in data.items(): if val is not None: @@ -115,7 +113,7 @@ async def qrz_test_session(key: str): raise ConnectionError(resp_session['Error']) -async def qrz_process_info(data: dict): +def qrz_process_info(data: dict): if 'name' in data: if 'fname' in data: name = data['fname'] + ' ' + data['name'] From daf461b6ac4c0d9a7d568938154926246e0aaeaa Mon Sep 17 00:00:00 2001 From: 0x5c <0x5c.dev@gmail.com> Date: Fri, 18 Oct 2019 11:13:24 -0400 Subject: [PATCH 05/11] now it can start --- cogs/qrzcog.py | 44 +++++++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py index d213e8b..8e06723 100644 --- a/cogs/qrzcog.py +++ b/cogs/qrzcog.py @@ -10,7 +10,7 @@ from collections import OrderedDict from datetime import datetime import discord -import discord.ext.commands as commands +from discord.ext import commands, tasks import aiohttp from bs4 import BeautifulSoup @@ -20,20 +20,10 @@ class QRZCog(commands.Cog): def __init__(self, bot: commands.Bot): self.bot = bot self.gs = bot.get_cog("GlobalSettings") - try: - with open('data/qrz_session') as qrz_file: - self.key = qrz_file.readline().strip() - await qrz_test_session(self.key) - except FileNotFoundError: - self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) - with open('data/qrz_session', 'w') as qrz_file: - qrz_file.write(self.key) - except ConnectionError: - self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) - with open('data/qrz_session', 'w') as qrz_file: - qrz_file.write(self.key) + self._qrz_session_init.start() @commands.command(name="qrz", aliases=["call"]) + async def _qrz_lookup(self, ctx: commands.Context, call: str): if self.gs.keys.qrz_user == '' or self.gs.keys.qrz_pass == '': await ctx.send(f'http://qrz.com/db/{call}') return @@ -41,9 +31,7 @@ class QRZCog(commands.Cog): try: await qrz_test_session(self.key) except ConnectionError: - self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) - with open('data/qrz_session', 'w') as qrz_file: - qrz_file.write(self.key) + await self.get_session() url = f'http://xmldata.qrz.com/xml/current/?s={self.key};callsign={call}' async with aiohttp.ClientSession() as session: @@ -58,9 +46,7 @@ class QRZCog(commands.Cog): resp_session = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} if 'Error' in resp_session: if 'Session Timeout' in resp_session['Error']: - self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) - with open('data/qrz_session', 'w') as qrz_file: - qrz_file.write(self.key) + await self.get_session() raise ValueError(resp_session['Error']) embed = discord.Embed(title=f"QRZ Data for {resp_data['call']}", @@ -79,6 +65,26 @@ class QRZCog(commands.Cog): embed.add_field(name=title, value=val, inline=True) await ctx.send(embed=embed) + async def get_session(self): + """QRZ API Session handling.""" + try: + with open('data/qrz_session') as qrz_file: + self.key = qrz_file.readline().strip() + await qrz_test_session(self.key) + except FileNotFoundError: + self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + with open('data/qrz_session', 'w') as qrz_file: + qrz_file.write(self.key) + except ConnectionError: + self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + with open('data/qrz_session', 'w') as qrz_file: + qrz_file.write(self.key) + + @tasks.loop(count=1) + async def _qrz_session_init(self): + """Helper task to allow initialisation of the session at cog instantiation.""" + await self.qrz_get_session() + async def qrz_login(user: str, passwd: str): url = f'http://xmldata.qrz.com/xml/current/?username={user};password={passwd};agent=qrmbot' From c790e856d93b38d4fcf6031fca615b480ed73e24 Mon Sep 17 00:00:00 2001 From: 0x5c <0x5c.dev@gmail.com> Date: Fri, 18 Oct 2019 11:26:06 -0400 Subject: [PATCH 06/11] should be better --- cogs/qrzcog.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py index 8e06723..d6500f7 100644 --- a/cogs/qrzcog.py +++ b/cogs/qrzcog.py @@ -66,24 +66,20 @@ class QRZCog(commands.Cog): await ctx.send(embed=embed) async def get_session(self): - """QRZ API Session handling.""" + """Session creation and caching.""" + self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + with open('data/qrz_session', 'w') as qrz_file: + qrz_file.write(self.key) + + @tasks.loop(count=) + async def _qrz_session_init(self): + """Helper task to allow obtaining a session at cog instantiation.""" try: with open('data/qrz_session') as qrz_file: self.key = qrz_file.readline().strip() await qrz_test_session(self.key) - except FileNotFoundError: - self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) - with open('data/qrz_session', 'w') as qrz_file: - qrz_file.write(self.key) - except ConnectionError: - self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) - with open('data/qrz_session', 'w') as qrz_file: - qrz_file.write(self.key) - - @tasks.loop(count=1) - async def _qrz_session_init(self): - """Helper task to allow initialisation of the session at cog instantiation.""" - await self.qrz_get_session() + except (FileNotFoundError, ConnectionError): + await self.qrz_get_session() async def qrz_login(user: str, passwd: str): From ae067acf0ef7042e856d00f63f3237f7ce3f9423 Mon Sep 17 00:00:00 2001 From: 0x5c <0x5c.dev@gmail.com> Date: Fri, 18 Oct 2019 11:37:24 -0400 Subject: [PATCH 07/11] sYnTAx eRrOR --- cogs/qrzcog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py index d6500f7..aacef40 100644 --- a/cogs/qrzcog.py +++ b/cogs/qrzcog.py @@ -71,7 +71,7 @@ class QRZCog(commands.Cog): with open('data/qrz_session', 'w') as qrz_file: qrz_file.write(self.key) - @tasks.loop(count=) + @tasks.loop(count=1) async def _qrz_session_init(self): """Helper task to allow obtaining a session at cog instantiation.""" try: From 5bc7011f7db472f7dbe10ab9a757e77d597e4630 Mon Sep 17 00:00:00 2001 From: Abigail Gold Date: Fri, 18 Oct 2019 12:45:26 -0400 Subject: [PATCH 08/11] smol fixes --- cogs/qrzcog.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py index aacef40..0b722bd 100644 --- a/cogs/qrzcog.py +++ b/cogs/qrzcog.py @@ -79,7 +79,7 @@ class QRZCog(commands.Cog): self.key = qrz_file.readline().strip() await qrz_test_session(self.key) except (FileNotFoundError, ConnectionError): - await self.qrz_get_session() + await self.get_session() async def qrz_login(user: str, passwd: str): @@ -108,7 +108,6 @@ async def qrz_test_session(key: str): resp_xml = await resp.text() xml_soup = BeautifulSoup(resp_xml, "xml") - print(xml_soup) resp_session = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} if 'Error' in resp_session: From 93b0bae2c175fd9a6229bcc47cbc01d9e277a2dd Mon Sep 17 00:00:00 2001 From: Abigail Gold Date: Fri, 18 Oct 2019 15:27:35 -0400 Subject: [PATCH 09/11] fix wasteful session usage --- cogs/qrzcog.py | 38 ++++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py index 0b722bd..2e0ddbe 100644 --- a/cogs/qrzcog.py +++ b/cogs/qrzcog.py @@ -20,6 +20,7 @@ class QRZCog(commands.Cog): def __init__(self, bot: commands.Bot): self.bot = bot self.gs = bot.get_cog("GlobalSettings") + self.session = aiohttp.ClientSession() self._qrz_session_init.start() @commands.command(name="qrz", aliases=["call"]) @@ -29,16 +30,15 @@ class QRZCog(commands.Cog): return try: - await qrz_test_session(self.key) + await qrz_test_session(self.key, self.session) except ConnectionError: await self.get_session() url = f'http://xmldata.qrz.com/xml/current/?s={self.key};callsign={call}' - async with aiohttp.ClientSession() as session: - async with session.get(url) as resp: - if resp.status != 200: - raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') - resp_xml = await resp.text() + async with self.session.get(url) as resp: + if resp.status != 200: + raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') + resp_xml = await resp.text() xml_soup = BeautifulSoup(resp_xml, "xml") @@ -67,7 +67,7 @@ class QRZCog(commands.Cog): async def get_session(self): """Session creation and caching.""" - self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass) + self.key = await qrz_login(self.gs.keys.qrz_user, self.gs.keys.qrz_pass, self.session) with open('data/qrz_session', 'w') as qrz_file: qrz_file.write(self.key) @@ -77,18 +77,17 @@ class QRZCog(commands.Cog): try: with open('data/qrz_session') as qrz_file: self.key = qrz_file.readline().strip() - await qrz_test_session(self.key) + await qrz_test_session(self.key, self.session) except (FileNotFoundError, ConnectionError): await self.get_session() -async def qrz_login(user: str, passwd: str): +async def qrz_login(user: str, passwd: str, session: aiohttp.ClientSession): url = f'http://xmldata.qrz.com/xml/current/?username={user};password={passwd};agent=qrmbot' - async with aiohttp.ClientSession() as session: - async with session.get(url) as resp: - if resp.status != 200: - raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') - resp_xml = await resp.text() + async with session.get(url) as resp: + if resp.status != 200: + raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') + resp_xml = await resp.text() xml_soup = BeautifulSoup(resp_xml, "xml") resp_data = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} @@ -99,13 +98,12 @@ async def qrz_login(user: str, passwd: str): return resp_data['Key'] -async def qrz_test_session(key: str): +async def qrz_test_session(key: str, session: aiohttp.ClientSession): url = f'http://xmldata.qrz.com/xml/current/?s={key}' - async with aiohttp.ClientSession() as session: - async with session.get(url) as resp: - if resp.status != 200: - raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') - resp_xml = await resp.text() + async with session.get(url) as resp: + if resp.status != 200: + raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') + resp_xml = await resp.text() xml_soup = BeautifulSoup(resp_xml, "xml") From f5ec8f0d8bc3a99225b58fece87e59001c698bc2 Mon Sep 17 00:00:00 2001 From: Abigail Gold Date: Sat, 19 Oct 2019 22:55:53 -0400 Subject: [PATCH 10/11] convert qrz lookup to use pure lxml instead of bs4 --- cogs/qrzcog.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py index 2e0ddbe..a1b085b 100644 --- a/cogs/qrzcog.py +++ b/cogs/qrzcog.py @@ -8,12 +8,13 @@ General Public License, version 2. """ from collections import OrderedDict from datetime import datetime +from io import BytesIO import discord from discord.ext import commands, tasks import aiohttp -from bs4 import BeautifulSoup +from lxml import etree class QRZCog(commands.Cog): @@ -38,12 +39,14 @@ class QRZCog(commands.Cog): async with self.session.get(url) as resp: if resp.status != 200: raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') - resp_xml = await resp.text() + resp_xml = etree.parse(BytesIO(await resp.read())).getroot() - xml_soup = BeautifulSoup(resp_xml, "xml") - - resp_data = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Callsign *')} - resp_session = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} + resp_xml_data = resp_xml.xpath('/x:QRZDatabase/x:Callsign', + namespaces={'x':'http://xmldata.qrz.com'}) + resp_data = {el.tag.split('}')[1]: el.text for el in resp_xml_data[0].getiterator()} + resp_xml_session = resp_xml.xpath('/x:QRZDatabase/x:Session', + namespaces={'x':'http://xmldata.qrz.com'}) + resp_session = {el.tag.split('}')[1]: el.text for el in resp_xml_session[0].getiterator()} if 'Error' in resp_session: if 'Session Timeout' in resp_session['Error']: await self.get_session() @@ -87,15 +90,16 @@ async def qrz_login(user: str, passwd: str, session: aiohttp.ClientSession): async with session.get(url) as resp: if resp.status != 200: raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') - resp_xml = await resp.text() + resp_xml = etree.parse(BytesIO(await resp.read())).getroot() - xml_soup = BeautifulSoup(resp_xml, "xml") - resp_data = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} - if 'Error' in resp_data: - raise ConnectionError(resp_data['Error']) - if resp_data['SubExp'] == 'non-subscriber': + resp_xml_session = resp_xml.xpath('/x:QRZDatabase/x:Session', + namespaces={'x':'http://xmldata.qrz.com'}) + resp_session = {el.tag.split('}')[1]: el.text for el in resp_xml_session[0].getiterator()} + if 'Error' in resp_session: + raise ConnectionError(resp_session['Error']) + if resp_session['SubExp'] == 'non-subscriber': raise ConnectionError('Invalid QRZ Subscription') - return resp_data['Key'] + return resp_session['Key'] async def qrz_test_session(key: str, session: aiohttp.ClientSession): @@ -103,11 +107,11 @@ async def qrz_test_session(key: str, session: aiohttp.ClientSession): async with session.get(url) as resp: if resp.status != 200: raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') - resp_xml = await resp.text() + resp_xml = etree.parse(BytesIO(await resp.read())).getroot() - xml_soup = BeautifulSoup(resp_xml, "xml") - - resp_session = {tag.name: tag.contents[0] for tag in xml_soup.select('QRZDatabase Session *')} + resp_xml_session = resp_xml.xpath('/x:QRZDatabase/x:Session', + namespaces={'x':'http://xmldata.qrz.com'}) + resp_session = {el.tag.split('}')[1]: el.text for el in resp_xml_session[0].getiterator()} if 'Error' in resp_session: raise ConnectionError(resp_session['Error']) From 02433947b62aa71954cb789c2405de5ed6bfd690 Mon Sep 17 00:00:00 2001 From: Abigail Gold Date: Sun, 20 Oct 2019 14:44:29 -0400 Subject: [PATCH 11/11] add error message, retry on session timeout --- cogs/qrzcog.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/cogs/qrzcog.py b/cogs/qrzcog.py index a1b085b..785bfa0 100644 --- a/cogs/qrzcog.py +++ b/cogs/qrzcog.py @@ -41,17 +41,29 @@ class QRZCog(commands.Cog): raise ConnectionError(f'Unable to connect to QRZ (HTTP Error {resp.status})') resp_xml = etree.parse(BytesIO(await resp.read())).getroot() - resp_xml_data = resp_xml.xpath('/x:QRZDatabase/x:Callsign', - namespaces={'x':'http://xmldata.qrz.com'}) - resp_data = {el.tag.split('}')[1]: el.text for el in resp_xml_data[0].getiterator()} resp_xml_session = resp_xml.xpath('/x:QRZDatabase/x:Session', - namespaces={'x':'http://xmldata.qrz.com'}) + namespaces={'x': 'http://xmldata.qrz.com'}) resp_session = {el.tag.split('}')[1]: el.text for el in resp_xml_session[0].getiterator()} if 'Error' in resp_session: if 'Session Timeout' in resp_session['Error']: await self.get_session() + await self._qrz_lookup(ctx, call) + return + if 'Not found' in resp_session['Error']: + embed = discord.Embed(title=f"QRZ Data for {call.upper()}", + colour=self.gs.colours.bad, + description='No data found!', + timestamp=datetime.utcnow()) + embed.set_footer(text=ctx.author.name, + icon_url=str(ctx.author.avatar_url)) + await ctx.send(embed=embed) + return raise ValueError(resp_session['Error']) + resp_xml_data = resp_xml.xpath('/x:QRZDatabase/x:Callsign', + namespaces={'x':'http://xmldata.qrz.com'}) + resp_data = {el.tag.split('}')[1]: el.text for el in resp_xml_data[0].getiterator()} + embed = discord.Embed(title=f"QRZ Data for {resp_data['call']}", colour=self.gs.colours.good, url=f'http://www.qrz.com/db/{resp_data["call"]}',