Upgrade third party modules to use requests

dev/releases/3.2.yml

@@ -28,4 +28,5 @@ minor_release_name: "Nicole"
     - "[Bugfix] Fixed scrobbling of tracks when all artists have been removed by server parsing"
     - "[Bugfix] Fixed Spotify import of multiple files"
     - "[Bugfix] Fixed process control on FreeBSD"
-    - "[Bugfix] Fixed Spotify authentication thread blocking the process from terminating"
+    - "[Bugfix] Fixed Spotify authentication thread blocking the process from terminating"
+    - "[Technical] Upgraded all third party modules to use requests module and send User Agent"

maloja/thirdparty/__init__.py

@@ -8,14 +8,16 @@
 
 import xml.etree.ElementTree as ElementTree
 import json
-import urllib.parse, urllib.request
+import requests
+import urllib.parse
 import base64
 import time
 from doreah.logging import log
 from threading import BoundedSemaphore
 
 from ..pkg_global.conf import malojaconfig
 from .. import database
+from ..__pkginfo__ import USER_AGENT
 
 
 services = {
@@ -100,6 +102,8 @@ def active_metadata(self):
 	scrobbleimport = {}
 	metadata = {}
 
+	useragent = USER_AGENT
+
 	def __init__(self):
 		# populate from settings file once on creation
 		# avoid constant disk access, restart on adding services is acceptable
@@ -127,16 +131,6 @@ def authorize(self):
 		return True
 		# per default. no authorization is necessary
 
-	# wrapper method
-	def request(self,url,data,responsetype):
-		response = urllib.request.urlopen(
-			url,
-			data=utf(data)
-		)
-		responsedata = response.read()
-		if responsetype == "xml":
-			data = ElementTree.fromstring(responsedata)
-			return data
 
 # proxy scrobbler
 class ProxyScrobbleInterface(GenericInterface,abstract=True):
@@ -155,11 +149,15 @@ def active_proxyscrobble(self):
 		)
 
 	def scrobble(self,artists,title,timestamp):
-		response = urllib.request.urlopen(
-			self.proxyscrobble["scrobbleurl"],
-			data=utf(self.proxyscrobble_postdata(artists,title,timestamp)))
-		responsedata = response.read()
+		response = requests.post(
+			url=self.proxyscrobble["scrobbleurl"],
+			data=self.proxyscrobble_postdata(artists,title,timestamp),
+			headers={
+				"User-Agent":self.useragent
+			}
+		)
 		if self.proxyscrobble["response_type"] == "xml":
+			responsedata = response.text
 			data = ElementTree.fromstring(responsedata)
 			return self.proxyscrobble_parse_response(data)
 
@@ -211,13 +209,15 @@ def get_image_track(self,track):
 		artists, title = track
 		artiststring = urllib.parse.quote(", ".join(artists))
 		titlestring = urllib.parse.quote(title)
-		response = urllib.request.urlopen(
-			self.metadata["trackurl"].format(artist=artiststring,title=titlestring,**self.settings)
+		response = requests.get(
+			self.metadata["trackurl"].format(artist=artiststring,title=titlestring,**self.settings),
+			headers={
+				"User-Agent":self.useragent
+			}
 		)
 
-		responsedata = response.read()
 		if self.metadata["response_type"] == "json":
-			data = json.loads(responsedata)
+			data = response.json()
 			imgurl = self.metadata_parse_response_track(data)
 		else:
 			imgurl = None
@@ -227,13 +227,15 @@ def get_image_track(self,track):
 
 	def get_image_artist(self,artist):
 		artiststring = urllib.parse.quote(artist)
-		response = urllib.request.urlopen(
-			self.metadata["artisturl"].format(artist=artiststring,**self.settings)
+		response = requests.get(
+			self.metadata["artisturl"].format(artist=artiststring,**self.settings),
+			headers={
+				"User-Agent":self.useragent
+			}
 		)
 
-		responsedata = response.read()
 		if self.metadata["response_type"] == "json":
-			data = json.loads(responsedata)
+			data = response.json()
 			imgurl = self.metadata_parse_response_artist(data)
 		else:
 			imgurl = None
@@ -245,13 +247,15 @@ def get_image_album(self,album):
 		artists, title = album
 		artiststring = urllib.parse.quote(", ".join(artists or []))
 		titlestring = urllib.parse.quote(title)
-		response = urllib.request.urlopen(
-			self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings)
+		response = requests.get(
+			self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings),
+			headers={
+				"User-Agent":self.useragent
+			}
 		)
 
-		responsedata = response.read()
 		if self.metadata["response_type"] == "json":
-			data = json.loads(responsedata)
+			data = response.json()
 			imgurl = self.metadata_parse_response_album(data)
 		else:
 			imgurl = None

maloja/thirdparty/lastfm.py

@@ -1,6 +1,7 @@
 from . import MetadataInterface, ProxyScrobbleInterface, utf
 import hashlib
-import urllib.parse, urllib.request
+import requests
+import xml.etree.ElementTree as ElementTree
 from doreah.logging import log
 
 class LastFM(MetadataInterface, ProxyScrobbleInterface):
@@ -54,27 +55,37 @@ def proxyscrobble_postdata(self,artists,title,timestamp):
 
 	def authorize(self):
 		try:
-			result = self.request(
-				self.proxyscrobble['scrobbleurl'],
-				self.query_compose({
+			response = requests.post(
+				url=self.proxyscrobble['scrobbleurl'],
+				params=self.query_compose({
 					"method":"auth.getMobileSession",
 					"username":self.settings["username"],
 					"password":self.settings["password"],
 					"api_key":self.settings["apikey"]
 				}),
-				responsetype="xml"
+				headers={
+					"User-Agent":self.useragent
+				}
 			)
-			self.settings["sk"] = result.find("session").findtext("key")
+
+			data = ElementTree.fromstring(response.text)
+			self.settings["sk"] = data.find("session").findtext("key")
 		except Exception as e:
-			pass
-			#log("Error while authenticating with LastFM: " + repr(e))
+			log("Error while authenticating with LastFM: " + repr(e))
 
 
-	# creates signature and returns full query string
+	# creates signature and returns full query
 	def query_compose(self,parameters):
 		m = hashlib.md5()
 		keys = sorted(str(k) for k in parameters)
 		m.update(utf("".join(str(k) + str(parameters[k]) for k in keys)))
 		m.update(utf(self.settings["secret"]))
 		sig = m.hexdigest()
-		return urllib.parse.urlencode(parameters) + "&api_sig=" + sig
+		return {**parameters,"api_sig":sig}
+
+	def handle_json_result_error(self,result):
+		if "track" in result and not result.get("track").get('album',{}):
+			return True
+
+		if "error" in result and result.get("error") == 6:
+			return True

maloja/thirdparty/maloja.py

@@ -1,5 +1,5 @@
 from . import ProxyScrobbleInterface, ImportInterface
-import urllib.request
+import requests
 from doreah.logging import log
 import json
 
@@ -32,8 +32,8 @@ def active_proxyscrobble(self):
 	def get_remote_scrobbles(self):
 		url = f"{self.settings['instance']}/apis/mlj_1/scrobbles"
 
-		response = urllib.request.urlopen(url)
-		data = json.loads(response.read().decode('utf-8'))
+		response = requests.get(url)
+		data = response.json()
 
 		for scrobble in data['list']:
 			yield scrobble

maloja/thirdparty/musicbrainz.py

@@ -1,17 +1,14 @@
 from . import MetadataInterface
-import urllib.parse, urllib.request
-import json
+import requests
 import time
 import threading
-from ..__pkginfo__ import USER_AGENT
 
 class MusicBrainz(MetadataInterface):
 	name = "MusicBrainz"
 	identifier = "musicbrainz"
 
 	# musicbrainz is rate-limited
 	lock = threading.Lock()
-	useragent = USER_AGENT
 
 
 	thumbnailsize_order = ['500','large','1200','250','small']
@@ -35,30 +32,32 @@ def get_image_album(self,album):
 			searchstr = f'release:"{title}"'
 			for artist in artists:
 				searchstr += f' artist:"{artist}"'
-			querystr = urllib.parse.urlencode({
-				"fmt":"json",
-				"query":searchstr
-			})
-			req = urllib.request.Request(**{
-				"url":"https://musicbrainz.org/ws/2/release?" + querystr,
-				"method":"GET",
+			res = requests.get(**{
+				"url":"https://musicbrainz.org/ws/2/release",
+				"params":{
+					"fmt":"json",
+					"query":searchstr
+				},
 				"headers":{
 					"User-Agent":self.useragent
 				}
 			})
-			response = urllib.request.urlopen(req)
-			responsedata = response.read()
-			data = json.loads(responsedata)
+			data = res.json()
 			entity = data["releases"][0]
 			coverartendpoint = "release"
 			while True:
 				mbid = entity["id"]
 				try:
-					response = urllib.request.urlopen(
-						f"https://coverartarchive.org/{coverartendpoint}/{mbid}?fmt=json"
+					response = requests.get(
+						f"https://coverartarchive.org/{coverartendpoint}/{mbid}",
+						params={
+							"fmt":"json"
+						},
+						headers={
+							"User-Agent":self.useragent
+						}
 					)
-					responsedata = response.read()
-					data = json.loads(responsedata)
+					data = response.json()
 					thumbnails = data['images'][0]['thumbnails']
 					for size in self.thumbnailsize_order:
 						if thumbnails.get(size) is not None:
@@ -88,30 +87,29 @@ def get_image_track(self,track):
 			searchstr = f'recording:"{title}"'
 			for artist in artists:
 				searchstr += f' artist:"{artist}"'
-			querystr = urllib.parse.urlencode({
-				"fmt":"json",
-				"query":searchstr
-			})
-			req = urllib.request.Request(**{
-				"url":"https://musicbrainz.org/ws/2/recording?" + querystr,
-				"method":"GET",
+			res = requests.get(**{
+				"url":"https://musicbrainz.org/ws/2/recording",
+				"params":{
+					"fmt":"json",
+					"query":searchstr
+				},
 				"headers":{
 					"User-Agent":self.useragent
 				}
 			})
-			response = urllib.request.urlopen(req)
-			responsedata = response.read()
-			data = json.loads(responsedata)
+			data = res.json()
 			entity = data["recordings"][0]["releases"][0]
 			coverartendpoint = "release"
 			while True:
 				mbid = entity["id"]
 				try:
-					response = urllib.request.urlopen(
-						f"https://coverartarchive.org/{coverartendpoint}/{mbid}?fmt=json"
+					response = requests.get(
+						f"https://coverartarchive.org/{coverartendpoint}/{mbid}",
+						params={
+							"fmt":"json"
+						}
 					)
-					responsedata = response.read()
-					data = json.loads(responsedata)
+					data = response.json()
 					thumbnails = data['images'][0]['thumbnails']
 					for size in self.thumbnailsize_order:
 						if thumbnails.get(size) is not None:

maloja/thirdparty/spotify.py

@@ -1,6 +1,5 @@
 from . import MetadataInterface, utf, b64
-import urllib.parse, urllib.request
-import json
+import requests
 from threading import Timer
 from doreah.logging import log
 
@@ -31,15 +30,14 @@ def authorize(self):
 			try:
 				keys = {
 					"url":"https://accounts.spotify.com/api/token",
-					"method":"POST",
 					"headers":{
-						"Authorization":"Basic " + b64(utf(self.settings["apiid"] + ":" + self.settings["secret"])).decode("utf-8")
+						"Authorization":"Basic " + b64(utf(self.settings["apiid"] + ":" + self.settings["secret"])).decode("utf-8"),
+						"User-Agent": self.useragent
 					},
-					"data":bytes(urllib.parse.urlencode({"grant_type":"client_credentials"}),encoding="utf-8")
+					"data":{"grant_type":"client_credentials"}
 				}
-				req = urllib.request.Request(**keys)
-				response = urllib.request.urlopen(req)
-				responsedata = json.loads(response.read())
+				res = requests.post(**keys)
+				responsedata = res.json()
 				if "error" in responsedata:
 					log("Error authenticating with Spotify: " + responsedata['error_description'])
 					expire = 3600
@@ -52,3 +50,8 @@ def authorize(self):
 				t.start()
 			except Exception as e:
 				log("Error while authenticating with Spotify: " + repr(e))
+
+	def handle_json_result_error(self,result):
+		result = result.get('tracks') or result.get('albums') or result.get('artists')
+		if not result['items']:
+			return True