From 76691a5b0fcd6030b34bddd6b2ac855b1ac6043a Mon Sep 17 00:00:00 2001 From: krateng Date: Wed, 1 Nov 2023 19:01:29 +0100 Subject: [PATCH] Upgrade third party modules to use requests --- dev/releases/3.2.yml | 3 +- maloja/thirdparty/__init__.py | 58 ++++++++++++++++-------------- maloja/thirdparty/lastfm.py | 31 ++++++++++------ maloja/thirdparty/maloja.py | 6 ++-- maloja/thirdparty/musicbrainz.py | 62 ++++++++++++++++---------------- maloja/thirdparty/spotify.py | 19 +++++----- 6 files changed, 98 insertions(+), 81 deletions(-) diff --git a/dev/releases/3.2.yml b/dev/releases/3.2.yml index ca388eb7..cf75ee32 100644 --- a/dev/releases/3.2.yml +++ b/dev/releases/3.2.yml @@ -28,4 +28,5 @@ minor_release_name: "Nicole" - "[Bugfix] Fixed scrobbling of tracks when all artists have been removed by server parsing" - "[Bugfix] Fixed Spotify import of multiple files" - "[Bugfix] Fixed process control on FreeBSD" - - "[Bugfix] Fixed Spotify authentication thread blocking the process from terminating" \ No newline at end of file + - "[Bugfix] Fixed Spotify authentication thread blocking the process from terminating" + - "[Technical] Upgraded all third party modules to use requests module and send User Agent" \ No newline at end of file diff --git a/maloja/thirdparty/__init__.py b/maloja/thirdparty/__init__.py index 7c582d0b..7b45e380 100644 --- a/maloja/thirdparty/__init__.py +++ b/maloja/thirdparty/__init__.py @@ -8,7 +8,8 @@ import xml.etree.ElementTree as ElementTree import json -import urllib.parse, urllib.request +import requests +import urllib.parse import base64 import time from doreah.logging import log @@ -16,6 +17,7 @@ from ..pkg_global.conf import malojaconfig from .. import database +from ..__pkginfo__ import USER_AGENT services = { @@ -100,6 +102,8 @@ def active_metadata(self): scrobbleimport = {} metadata = {} + useragent = USER_AGENT + def __init__(self): # populate from settings file once on creation # avoid constant disk access, restart on adding services is acceptable @@ -127,16 +131,6 @@ def authorize(self): return True # per default. no authorization is necessary - # wrapper method - def request(self,url,data,responsetype): - response = urllib.request.urlopen( - url, - data=utf(data) - ) - responsedata = response.read() - if responsetype == "xml": - data = ElementTree.fromstring(responsedata) - return data # proxy scrobbler class ProxyScrobbleInterface(GenericInterface,abstract=True): @@ -155,11 +149,15 @@ def active_proxyscrobble(self): ) def scrobble(self,artists,title,timestamp): - response = urllib.request.urlopen( - self.proxyscrobble["scrobbleurl"], - data=utf(self.proxyscrobble_postdata(artists,title,timestamp))) - responsedata = response.read() + response = requests.post( + url=self.proxyscrobble["scrobbleurl"], + data=self.proxyscrobble_postdata(artists,title,timestamp), + headers={ + "User-Agent":self.useragent + } + ) if self.proxyscrobble["response_type"] == "xml": + responsedata = response.text data = ElementTree.fromstring(responsedata) return self.proxyscrobble_parse_response(data) @@ -211,13 +209,15 @@ def get_image_track(self,track): artists, title = track artiststring = urllib.parse.quote(", ".join(artists)) titlestring = urllib.parse.quote(title) - response = urllib.request.urlopen( - self.metadata["trackurl"].format(artist=artiststring,title=titlestring,**self.settings) + response = requests.get( + self.metadata["trackurl"].format(artist=artiststring,title=titlestring,**self.settings), + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() if self.metadata["response_type"] == "json": - data = json.loads(responsedata) + data = response.json() imgurl = self.metadata_parse_response_track(data) else: imgurl = None @@ -227,13 +227,15 @@ def get_image_track(self,track): def get_image_artist(self,artist): artiststring = urllib.parse.quote(artist) - response = urllib.request.urlopen( - self.metadata["artisturl"].format(artist=artiststring,**self.settings) + response = requests.get( + self.metadata["artisturl"].format(artist=artiststring,**self.settings), + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() if self.metadata["response_type"] == "json": - data = json.loads(responsedata) + data = response.json() imgurl = self.metadata_parse_response_artist(data) else: imgurl = None @@ -245,13 +247,15 @@ def get_image_album(self,album): artists, title = album artiststring = urllib.parse.quote(", ".join(artists or [])) titlestring = urllib.parse.quote(title) - response = urllib.request.urlopen( - self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings) + response = requests.get( + self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings), + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() if self.metadata["response_type"] == "json": - data = json.loads(responsedata) + data = response.json() imgurl = self.metadata_parse_response_album(data) else: imgurl = None diff --git a/maloja/thirdparty/lastfm.py b/maloja/thirdparty/lastfm.py index e565e5fd..ff2ec65d 100644 --- a/maloja/thirdparty/lastfm.py +++ b/maloja/thirdparty/lastfm.py @@ -1,6 +1,7 @@ from . import MetadataInterface, ProxyScrobbleInterface, utf import hashlib -import urllib.parse, urllib.request +import requests +import xml.etree.ElementTree as ElementTree from doreah.logging import log class LastFM(MetadataInterface, ProxyScrobbleInterface): @@ -54,27 +55,37 @@ def proxyscrobble_postdata(self,artists,title,timestamp): def authorize(self): try: - result = self.request( - self.proxyscrobble['scrobbleurl'], - self.query_compose({ + response = requests.post( + url=self.proxyscrobble['scrobbleurl'], + params=self.query_compose({ "method":"auth.getMobileSession", "username":self.settings["username"], "password":self.settings["password"], "api_key":self.settings["apikey"] }), - responsetype="xml" + headers={ + "User-Agent":self.useragent + } ) - self.settings["sk"] = result.find("session").findtext("key") + + data = ElementTree.fromstring(response.text) + self.settings["sk"] = data.find("session").findtext("key") except Exception as e: - pass - #log("Error while authenticating with LastFM: " + repr(e)) + log("Error while authenticating with LastFM: " + repr(e)) - # creates signature and returns full query string + # creates signature and returns full query def query_compose(self,parameters): m = hashlib.md5() keys = sorted(str(k) for k in parameters) m.update(utf("".join(str(k) + str(parameters[k]) for k in keys))) m.update(utf(self.settings["secret"])) sig = m.hexdigest() - return urllib.parse.urlencode(parameters) + "&api_sig=" + sig + return {**parameters,"api_sig":sig} + + def handle_json_result_error(self,result): + if "track" in result and not result.get("track").get('album',{}): + return True + + if "error" in result and result.get("error") == 6: + return True diff --git a/maloja/thirdparty/maloja.py b/maloja/thirdparty/maloja.py index 53413f89..49754009 100644 --- a/maloja/thirdparty/maloja.py +++ b/maloja/thirdparty/maloja.py @@ -1,5 +1,5 @@ from . import ProxyScrobbleInterface, ImportInterface -import urllib.request +import requests from doreah.logging import log import json @@ -32,8 +32,8 @@ def active_proxyscrobble(self): def get_remote_scrobbles(self): url = f"{self.settings['instance']}/apis/mlj_1/scrobbles" - response = urllib.request.urlopen(url) - data = json.loads(response.read().decode('utf-8')) + response = requests.get(url) + data = response.json() for scrobble in data['list']: yield scrobble diff --git a/maloja/thirdparty/musicbrainz.py b/maloja/thirdparty/musicbrainz.py index 6f1f2b1b..5dc6f720 100644 --- a/maloja/thirdparty/musicbrainz.py +++ b/maloja/thirdparty/musicbrainz.py @@ -1,9 +1,7 @@ from . import MetadataInterface -import urllib.parse, urllib.request -import json +import requests import time import threading -from ..__pkginfo__ import USER_AGENT class MusicBrainz(MetadataInterface): name = "MusicBrainz" @@ -11,7 +9,6 @@ class MusicBrainz(MetadataInterface): # musicbrainz is rate-limited lock = threading.Lock() - useragent = USER_AGENT thumbnailsize_order = ['500','large','1200','250','small'] @@ -35,30 +32,32 @@ def get_image_album(self,album): searchstr = f'release:"{title}"' for artist in artists: searchstr += f' artist:"{artist}"' - querystr = urllib.parse.urlencode({ - "fmt":"json", - "query":searchstr - }) - req = urllib.request.Request(**{ - "url":"https://musicbrainz.org/ws/2/release?" + querystr, - "method":"GET", + res = requests.get(**{ + "url":"https://musicbrainz.org/ws/2/release", + "params":{ + "fmt":"json", + "query":searchstr + }, "headers":{ "User-Agent":self.useragent } }) - response = urllib.request.urlopen(req) - responsedata = response.read() - data = json.loads(responsedata) + data = res.json() entity = data["releases"][0] coverartendpoint = "release" while True: mbid = entity["id"] try: - response = urllib.request.urlopen( - f"https://coverartarchive.org/{coverartendpoint}/{mbid}?fmt=json" + response = requests.get( + f"https://coverartarchive.org/{coverartendpoint}/{mbid}", + params={ + "fmt":"json" + }, + headers={ + "User-Agent":self.useragent + } ) - responsedata = response.read() - data = json.loads(responsedata) + data = response.json() thumbnails = data['images'][0]['thumbnails'] for size in self.thumbnailsize_order: if thumbnails.get(size) is not None: @@ -88,30 +87,29 @@ def get_image_track(self,track): searchstr = f'recording:"{title}"' for artist in artists: searchstr += f' artist:"{artist}"' - querystr = urllib.parse.urlencode({ - "fmt":"json", - "query":searchstr - }) - req = urllib.request.Request(**{ - "url":"https://musicbrainz.org/ws/2/recording?" + querystr, - "method":"GET", + res = requests.get(**{ + "url":"https://musicbrainz.org/ws/2/recording", + "params":{ + "fmt":"json", + "query":searchstr + }, "headers":{ "User-Agent":self.useragent } }) - response = urllib.request.urlopen(req) - responsedata = response.read() - data = json.loads(responsedata) + data = res.json() entity = data["recordings"][0]["releases"][0] coverartendpoint = "release" while True: mbid = entity["id"] try: - response = urllib.request.urlopen( - f"https://coverartarchive.org/{coverartendpoint}/{mbid}?fmt=json" + response = requests.get( + f"https://coverartarchive.org/{coverartendpoint}/{mbid}", + params={ + "fmt":"json" + } ) - responsedata = response.read() - data = json.loads(responsedata) + data = response.json() thumbnails = data['images'][0]['thumbnails'] for size in self.thumbnailsize_order: if thumbnails.get(size) is not None: diff --git a/maloja/thirdparty/spotify.py b/maloja/thirdparty/spotify.py index 9d9e4b97..03d614da 100644 --- a/maloja/thirdparty/spotify.py +++ b/maloja/thirdparty/spotify.py @@ -1,6 +1,5 @@ from . import MetadataInterface, utf, b64 -import urllib.parse, urllib.request -import json +import requests from threading import Timer from doreah.logging import log @@ -31,15 +30,14 @@ def authorize(self): try: keys = { "url":"https://accounts.spotify.com/api/token", - "method":"POST", "headers":{ - "Authorization":"Basic " + b64(utf(self.settings["apiid"] + ":" + self.settings["secret"])).decode("utf-8") + "Authorization":"Basic " + b64(utf(self.settings["apiid"] + ":" + self.settings["secret"])).decode("utf-8"), + "User-Agent": self.useragent }, - "data":bytes(urllib.parse.urlencode({"grant_type":"client_credentials"}),encoding="utf-8") + "data":{"grant_type":"client_credentials"} } - req = urllib.request.Request(**keys) - response = urllib.request.urlopen(req) - responsedata = json.loads(response.read()) + res = requests.post(**keys) + responsedata = res.json() if "error" in responsedata: log("Error authenticating with Spotify: " + responsedata['error_description']) expire = 3600 @@ -52,3 +50,8 @@ def authorize(self): t.start() except Exception as e: log("Error while authenticating with Spotify: " + repr(e)) + + def handle_json_result_error(self,result): + result = result.get('tracks') or result.get('albums') or result.get('artists') + if not result['items']: + return True \ No newline at end of file