From e47a2863e8dcc245dac02026220237c59c55d2cd Mon Sep 17 00:00:00 2001 From: RemixDev Date: Thu, 11 Feb 2021 17:30:53 +0100 Subject: [PATCH 01/20] Removed eventlet --- deemix/app/downloadjob.py | 21 +++++++++++---------- deemix/app/queuemanager.py | 3 +-- deemix/app/spotifyhelper.py | 5 +---- deemix/types/Track.py | 6 +++--- requirements.txt | 1 - setup.py | 2 +- 6 files changed, 17 insertions(+), 21 deletions(-) diff --git a/deemix/app/downloadjob.py b/deemix/app/downloadjob.py index f0e08b4..d436422 100644 --- a/deemix/app/downloadjob.py +++ b/deemix/app/downloadjob.py @@ -1,9 +1,10 @@ -import eventlet -from eventlet.green.subprocess import call as execute -requests = eventlet.import_patched('requests') +import requests get = requests.get request_exception = requests.exceptions +from concurrent.futures import ThreadPoolExecutor +from time import sleep + from os.path import sep as pathSep from pathlib import Path from shlex import quote @@ -74,12 +75,12 @@ def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): pictureSize = int(pictureUrl[:pictureUrl.find("x")]) if pictureSize > 1200: logger.warn("Couldn't download "+str(pictureSize)+"x"+str(pictureSize)+" image, falling back to 1200x1200") - eventlet.sleep(1) + sleep(1) return downloadImage(urlBase+pictureUrl.replace(str(pictureSize)+"x"+str(pictureSize), '1200x1200'), path, overwrite) logger.error("Image not found: "+url) except (request_exception.ConnectionError, request_exception.ChunkedEncodingError, u3SSLError) as e: logger.error("Couldn't download Image, retrying in 5 seconds...: "+url+"\n") - eventlet.sleep(5) + sleep(5) return downloadImage(url, path, overwrite) except OSError as e: if e.errno == errno.ENOSPC: @@ -114,9 +115,9 @@ class DownloadJob: if result: self.singleAfterDownload(result) elif isinstance(self.queueItem, QICollection): tracks = [None] * len(self.queueItem.collection) - pool = eventlet.GreenPool(size=self.settings['queueConcurrency']) - for pos, track in enumerate(self.queueItem.collection, start=0): - tracks[pos] = pool.spawn(self.downloadWrapper, track) + with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor: + for pos, track in enumerate(self.queueItem.collection, start=0): + tracks[pos] = executor.submit(self.downloadWrapper, track) pool.waitall() self.collectionAfterDownload(tracks) if self.interface: @@ -525,7 +526,7 @@ class DownloadJob: except (request_exception.ConnectionError, request_exception.ChunkedEncodingError) as e: if writepath.is_file(): writepath.unlink() logger.warn(f"[{track.mainArtist.name} - {track.title}] Error while downloading the track, trying again in 5s...") - eventlet.sleep(5) + sleep(5) return downloadMusic(track, trackAPI_gw) except OSError as e: if e.errno == errno.ENOSPC: @@ -680,7 +681,7 @@ class DownloadJob: logger.info(f'{itemName} retrying from byte {chunkLength}') return self.streamTrack(stream, track, chunkLength) except (request_exception.ConnectionError, requests.exceptions.ReadTimeout): - eventlet.sleep(2) + sleep(2) return self.streamTrack(stream, track, start) def updatePercentage(self): diff --git a/deemix/app/queuemanager.py b/deemix/app/queuemanager.py index 4547bf8..0305945 100644 --- a/deemix/app/queuemanager.py +++ b/deemix/app/queuemanager.py @@ -9,9 +9,8 @@ import logging from pathlib import Path import json from os import remove -import eventlet import uuid -urlopen = eventlet.import_patched('urllib.request').urlopen +from urllib.request import urlopen logging.basicConfig(level=logging.INFO) logger = logging.getLogger('deemix') diff --git a/deemix/app/spotifyhelper.py b/deemix/app/spotifyhelper.py index 0eaf907..d60e819 100644 --- a/deemix/app/spotifyhelper.py +++ b/deemix/app/spotifyhelper.py @@ -1,10 +1,7 @@ -import eventlet import json from pathlib import Path -eventlet.import_patched('requests.adapters') - -spotipy = eventlet.import_patched('spotipy') +import spotipy SpotifyClientCredentials = spotipy.oauth2.SpotifyClientCredentials from deemix.utils.localpaths import getConfigFolder from deemix.app.queueitem import QIConvertable diff --git a/deemix/types/Track.py b/deemix/types/Track.py index 523384f..a354772 100644 --- a/deemix/types/Track.py +++ b/deemix/types/Track.py @@ -1,5 +1,5 @@ -import eventlet -requests = eventlet.import_patched('requests') +from time import sleep +import requests import logging logging.basicConfig(level=logging.INFO) @@ -83,7 +83,7 @@ class Track: ) result_json = site.json() except: - eventlet.sleep(2) + sleep(2) return self.retriveFilesizes(dz) if len(result_json['error']): raise APIError(json.dumps(result_json['error'])) diff --git a/requirements.txt b/requirements.txt index 3242ce0..6f5ae55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,4 @@ pycryptodomex mutagen requests spotipy>=2.11.0 -eventlet deezer-py diff --git a/setup.py b/setup.py index f14b4e9..ec9d9b6 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( python_requires='>=3.6', packages=find_packages(exclude=("tests",)), include_package_data=True, - install_requires=["click", "pycryptodomex", "mutagen", "requests", "spotipy>=2.11.0", "eventlet", "deezer-py"], + install_requires=["click", "pycryptodomex", "mutagen", "requests", "spotipy>=2.11.0", "deezer-py"], entry_points={ "console_scripts": [ "deemix=deemix.__main__:download", -- 2.25.1 From bd8a1948d41d63c492ae6b1b8a6c99050f7aba7b Mon Sep 17 00:00:00 2001 From: RemixDev Date: Thu, 11 Feb 2021 19:05:06 +0100 Subject: [PATCH 02/20] Started queuemanager refactoring --- deemix/app/__init__.py | 5 ++--- deemix/app/downloadjob.py | 15 +++++++-------- deemix/app/queuemanager.py | 34 ++++++++++++++++++++++++---------- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/deemix/app/__init__.py b/deemix/app/__init__.py index 9b628bb..225936f 100644 --- a/deemix/app/__init__.py +++ b/deemix/app/__init__.py @@ -6,7 +6,6 @@ from deemix.app.spotifyhelper import SpotifyHelper class deemix: def __init__(self, configFolder=None, overwriteDownloadFolder=None): self.set = Settings(configFolder, overwriteDownloadFolder=overwriteDownloadFolder) - self.dz = Deezer() - self.dz.set_accept_language(self.set.settings.get('tagsLanguage')) + self.dz = Deezer(self.set.settings.get('tagsLanguage')) self.sp = SpotifyHelper(configFolder) - self.qm = QueueManager(self.sp) + self.qm = QueueManager(self.dz, self.sp) diff --git a/deemix/app/downloadjob.py b/deemix/app/downloadjob.py index d436422..d001b46 100644 --- a/deemix/app/downloadjob.py +++ b/deemix/app/downloadjob.py @@ -1,6 +1,5 @@ import requests -get = requests.get -request_exception = requests.exceptions +from requests import get from concurrent.futures import ThreadPoolExecutor from time import sleep @@ -68,7 +67,7 @@ def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): with open(path, 'wb') as f: f.write(image.content) return path - except request_exception.HTTPError: + except requests.exceptions.HTTPError: if 'cdns-images.dzcdn.net' in url: urlBase = url[:url.rfind("/")+1] pictureUrl = url[len(urlBase):] @@ -78,7 +77,7 @@ def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): sleep(1) return downloadImage(urlBase+pictureUrl.replace(str(pictureSize)+"x"+str(pictureSize), '1200x1200'), path, overwrite) logger.error("Image not found: "+url) - except (request_exception.ConnectionError, request_exception.ChunkedEncodingError, u3SSLError) as e: + except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError, u3SSLError) as e: logger.error("Couldn't download Image, retrying in 5 seconds...: "+url+"\n") sleep(5) return downloadImage(url, path, overwrite) @@ -492,7 +491,7 @@ class DownloadJob: except DownloadCancelled: if writepath.is_file(): writepath.unlink() raise DownloadCancelled - except (request_exception.HTTPError, DownloadEmpty): + except (requests.exceptions.HTTPError, DownloadEmpty): if writepath.is_file(): writepath.unlink() if track.fallbackId != "0": logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available, using fallback id") @@ -523,7 +522,7 @@ class DownloadJob: raise DownloadFailed("notAvailableNoAlternative") else: raise DownloadFailed("notAvailable") - except (request_exception.ConnectionError, request_exception.ChunkedEncodingError) as e: + except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e: if writepath.is_file(): writepath.unlink() logger.warn(f"[{track.mainArtist.name} - {track.title}] Error while downloading the track, trying again in 5s...") sleep(5) @@ -616,7 +615,7 @@ class DownloadJob: try: request.raise_for_status() return formatNumber - except request_exception.HTTPError: # if the format is not available, Deezer returns a 403 error + except requests.exceptions.HTTPError: # if the format is not available, Deezer returns a 403 error pass if not shouldFallback: raise PreferredBitrateNotFound @@ -680,7 +679,7 @@ class DownloadJob: except (SSLError, u3SSLError) as e: logger.info(f'{itemName} retrying from byte {chunkLength}') return self.streamTrack(stream, track, chunkLength) - except (request_exception.ConnectionError, requests.exceptions.ReadTimeout): + except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): sleep(2) return self.streamTrack(stream, track, start) diff --git a/deemix/app/queuemanager.py b/deemix/app/queuemanager.py index 0305945..cf2f9dc 100644 --- a/deemix/app/queuemanager.py +++ b/deemix/app/queuemanager.py @@ -1,8 +1,11 @@ from deemix.app.downloadjob import DownloadJob from deemix.utils import getIDFromLink, getTypeFromLink, getBitrateInt + +from deezer import Deezer from deezer.gw import APIError as gwAPIError, LyricsStatus from deezer.api import APIError from deezer.utils import map_user_playlist + from spotipy.exceptions import SpotifyException from deemix.app.queueitem import QueueItem, QISingle, QICollection, QIConvertable import logging @@ -16,14 +19,16 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger('deemix') class QueueManager: - def __init__(self, spotifyHelper=None): + def __init__(self, deezerHelper=None, spotifyHelper=None): self.queue = [] self.queueList = {} self.queueComplete = [] self.currentItem = "" + self.dz = deezerHelper or Deezer() self.sp = spotifyHelper - def generateTrackQueueItem(self, dz, id, settings, bitrate, trackAPI=None, albumAPI=None): + def generateTrackQueueItem(self, id, settings, bitrate, trackAPI=None, albumAPI=None, dz=None): + if not dz: dz = self.dz # Check if is an isrc: url if str(id).startswith("isrc"): try: @@ -72,7 +77,8 @@ class QueueManager: single=trackAPI_gw, ) - def generateAlbumQueueItem(self, dz, id, settings, bitrate, rootArtist=None): + def generateAlbumQueueItem(self, id, settings, bitrate, rootArtist=None, dz=None): + if not dz: dz = self.dz # Get essential album info try: albumAPI = dz.api.get_album(id) @@ -125,7 +131,8 @@ class QueueManager: collection=collection, ) - def generatePlaylistQueueItem(self, dz, id, settings, bitrate): + def generatePlaylistQueueItem(self, id, settings, bitrate, dz=None): + if not dz: dz = self.dz # Get essential playlist info try: playlistAPI = dz.api.get_playlist(id) @@ -178,7 +185,8 @@ class QueueManager: collection=collection, ) - def generateArtistQueueItem(self, dz, id, settings, bitrate, interface=None): + def generateArtistQueueItem(self, id, settings, bitrate, dz=None, interface=None): + if not dz: dz = self.dz # Get essential artist info try: artistAPI = dz.api.get_artist(id) @@ -201,7 +209,8 @@ class QueueManager: if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) return albumList - def generateArtistDiscographyQueueItem(self, dz, id, settings, bitrate, interface=None): + def generateArtistDiscographyQueueItem(self, id, settings, bitrate, dz=None, interface=None): + if not dz: dz = self.dz # Get essential artist info try: artistAPI = dz.api.get_artist(id) @@ -225,7 +234,8 @@ class QueueManager: if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) return albumList - def generateArtistTopQueueItem(self, dz, id, settings, bitrate, interface=None): + def generateArtistTopQueueItem(self, id, settings, bitrate, dz=None, interface=None): + if not dz: dz = self.dz # Get essential artist info try: artistAPI = dz.api.get_artist(id) @@ -293,7 +303,8 @@ class QueueManager: collection=collection, ) - def generateQueueItem(self, dz, url, settings, bitrate=None, interface=None): + def generateQueueItem(self, url, settings, bitrate=None, dz=None, interface=None): + if not dz: dz = self.dz bitrate = getBitrateInt(bitrate) or settings['maxBitrate'] if 'deezer.page.link' in url: url = urlopen(url).url if 'link.tospotify.com' in url: url = urlopen(url).url @@ -359,7 +370,9 @@ class QueueManager: logger.warn("URL not supported yet") return QueueError(url, "URL not supported yet", "unsupportedURL") - def addToQueue(self, dz, url, settings, bitrate=None, interface=None, ack=None): + def addToQueue(self, url, settings, bitrate=None, dz=None, interface=None, ack=None): + if not dz: dz = self.dz + if not dz.logged_in: if interface: interface.send("loginNeededToDownload") return False @@ -430,7 +443,8 @@ class QueueManager: self.nextItem(dz, interface) return True - def nextItem(self, dz, interface=None): + def nextItem(self, dz=None, interface=None): + if not dz: dz = self.dz # Check that nothing is already downloading and # that the queue is not empty if self.currentItem != "": return None -- 2.25.1 From 9c49bf5d23e8b2d48a2f07d926a460929ef12970 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Sun, 28 Feb 2021 23:24:04 +0100 Subject: [PATCH 03/20] Removed dz as first parameter --- .gitignore | 1 + deemix/app/cli.py | 4 ++-- deemix/app/queuemanager.py | 24 ++++++++++++------------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index d6fb3b0..3a64c4a 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ yarn-error.log* /build /*egg-info updatePyPi.sh +/deezer diff --git a/deemix/app/cli.py b/deemix/app/cli.py index 379e11f..b1d2bf8 100644 --- a/deemix/app/cli.py +++ b/deemix/app/cli.py @@ -14,9 +14,9 @@ class cli(deemix): for link in url: if ';' in link: for l in link.split(";"): - self.qm.addToQueue(self.dz, l, self.set.settings, bitrate) + self.qm.addToQueue(l, self.set.settings, bitrate) else: - self.qm.addToQueue(self.dz, link, self.set.settings, bitrate) + self.qm.addToQueue(link, self.set.settings, bitrate) def requestValidArl(self): while True: diff --git a/deemix/app/queuemanager.py b/deemix/app/queuemanager.py index cf2f9dc..ffbb0a4 100644 --- a/deemix/app/queuemanager.py +++ b/deemix/app/queuemanager.py @@ -97,7 +97,7 @@ class QueueManager: # If the album is a single download as a track if albumAPI['nb_tracks'] == 1: - return self.generateTrackQueueItem(dz, albumAPI['tracks']['data'][0]['id'], settings, bitrate, albumAPI=albumAPI) + return self.generateTrackQueueItem(albumAPI['tracks']['data'][0]['id'], settings, bitrate, albumAPI=albumAPI, dz=dz) tracksArray = dz.gw.get_album_tracks(id) @@ -204,7 +204,7 @@ class QueueManager: allReleases = artistDiscographyAPI.pop('all', []) albumList = [] for album in allReleases: - albumList.append(self.generateAlbumQueueItem(dz, album['id'], settings, bitrate, rootArtist=rootArtist)) + albumList.append(self.generateAlbumQueueItem(album['id'], settings, bitrate, rootArtist=rootArtist, dz=dz)) if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) return albumList @@ -229,7 +229,7 @@ class QueueManager: albumList = [] for type in artistDiscographyAPI: for album in artistDiscographyAPI[type]: - albumList.append(self.generateAlbumQueueItem(dz, album['id'], settings, bitrate, rootArtist=rootArtist)) + albumList.append(self.generateAlbumQueueItem(album['id'], settings, bitrate, rootArtist=rootArtist, dz=dz)) if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) return albumList @@ -316,17 +316,17 @@ class QueueManager: return QueueError(url, "URL not recognized", "invalidURL") if type == "track": - return self.generateTrackQueueItem(dz, id, settings, bitrate) + return self.generateTrackQueueItem(id, settings, bitrate, dz=dz) elif type == "album": - return self.generateAlbumQueueItem(dz, id, settings, bitrate) + return self.generateAlbumQueueItem(id, settings, bitrate, dz=dz) elif type == "playlist": - return self.generatePlaylistQueueItem(dz, id, settings, bitrate) + return self.generatePlaylistQueueItem(id, settings, bitrate, dz=dz) elif type == "artist": - return self.generateArtistQueueItem(dz, id, settings, bitrate, interface=interface) + return self.generateArtistQueueItem(id, settings, bitrate, interface=interface, dz=dz) elif type == "artistdiscography": - return self.generateArtistDiscographyQueueItem(dz, id, settings, bitrate, interface=interface) + return self.generateArtistDiscographyQueueItem(id, settings, bitrate, interface=interface, dz=dz) elif type == "artisttop": - return self.generateArtistTopQueueItem(dz, id, settings, bitrate, interface=interface) + return self.generateArtistTopQueueItem(id, settings, bitrate, interface=interface, dz=dz) elif type.startswith("spotify") and self.sp: if not self.sp.spotifyEnabled: logger.warn("Spotify Features is not setted up correctly.") @@ -341,7 +341,7 @@ class QueueManager: return QueueError(url, "Something went wrong: "+str(e)) if track_id != "0": - return self.generateTrackQueueItem(dz, track_id, settings, bitrate, trackAPI=trackAPI) + return self.generateTrackQueueItem(track_id, settings, bitrate, trackAPI=trackAPI, dz=dz) else: logger.warn("Track not found on deezer!") return QueueError(url, "Track not found on deezer!", "trackNotOnDeezer") @@ -355,7 +355,7 @@ class QueueManager: return QueueError(url, "Something went wrong: "+str(e)) if album_id != "0": - return self.generateAlbumQueueItem(dz, album_id, settings, bitrate) + return self.generateAlbumQueueItem(album_id, settings, bitrate, dz=dz) else: logger.warn("Album not found on deezer!") return QueueError(url, "Album not found on deezer!", "albumNotOnDeezer") @@ -381,7 +381,7 @@ class QueueManager: link = link.strip() if link == "": return False logger.info("Generating queue item for: "+link) - item = self.generateQueueItem(dz, link, settings, bitrate, interface=interface) + item = self.generateQueueItem(link, settings, bitrate, interface=interface, dz=dz) # Add ack to all items if type(item) is list: -- 2.25.1 From 318ad689ea2875ca510b43c12198049c195ed0a5 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Mon, 1 Mar 2021 19:11:07 +0100 Subject: [PATCH 04/20] Made nextitem work on a thread --- deemix/app/downloadjob.py | 3 +-- deemix/app/queuemanager.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/deemix/app/downloadjob.py b/deemix/app/downloadjob.py index d001b46..d7eb36e 100644 --- a/deemix/app/downloadjob.py +++ b/deemix/app/downloadjob.py @@ -117,7 +117,6 @@ class DownloadJob: with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor: for pos, track in enumerate(self.queueItem.collection, start=0): tracks[pos] = executor.submit(self.downloadWrapper, track) - pool.waitall() self.collectionAfterDownload(tracks) if self.interface: if self.queueItem.cancel: @@ -159,7 +158,7 @@ class DownloadJob: searched = "" for i in range(len(tracks)): - result = tracks[i].wait() + result = tracks[i].result() if not result: return None # Check if item is cancelled # Log errors to file diff --git a/deemix/app/queuemanager.py b/deemix/app/queuemanager.py index ffbb0a4..8ebcb2f 100644 --- a/deemix/app/queuemanager.py +++ b/deemix/app/queuemanager.py @@ -15,6 +15,8 @@ from os import remove import uuid from urllib.request import urlopen +import threading + logging.basicConfig(level=logging.INFO) logger = logging.getLogger('deemix') @@ -26,6 +28,7 @@ class QueueManager: self.currentItem = "" self.dz = deezerHelper or Deezer() self.sp = spotifyHelper + self.queueThread = None def generateTrackQueueItem(self, id, settings, bitrate, trackAPI=None, albumAPI=None, dz=None): if not dz: dz = self.dz @@ -440,15 +443,18 @@ class QueueManager: if interface: interface.send("addedToQueue", queueItem.getSlimmedItem()) else: return False - self.nextItem(dz, interface) + if not self.queueThread: + self.queueThread = threading.Thread(target=self.nextItem, args=(dz, interface)) + self.queueThread.start() return True def nextItem(self, dz=None, interface=None): if not dz: dz = self.dz # Check that nothing is already downloading and # that the queue is not empty - if self.currentItem != "": return None - if not len(self.queue): return None + if self.currentItem != "" or not len(self.queue): + self.queueThread = None + return None self.currentItem = self.queue.pop(0) -- 2.25.1 From b91d2a1af379e96299b03b32440c7574aea198e5 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Sun, 7 Mar 2021 12:03:50 +0100 Subject: [PATCH 05/20] Added start queue function --- deemix/app/queuemanager.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/deemix/app/queuemanager.py b/deemix/app/queuemanager.py index 8ebcb2f..03a8a89 100644 --- a/deemix/app/queuemanager.py +++ b/deemix/app/queuemanager.py @@ -443,9 +443,7 @@ class QueueManager: if interface: interface.send("addedToQueue", queueItem.getSlimmedItem()) else: return False - if not self.queueThread: - self.queueThread = threading.Thread(target=self.nextItem, args=(dz, interface)) - self.queueThread.start() + self.startQueue(interface, dz) return True def nextItem(self, dz=None, interface=None): @@ -529,6 +527,12 @@ class QueueManager: 'restored': True }) + def startQueue(self, interface=None, dz=None): + if not dz: dz = self.dz + if dz.logged_in and not self.queueThread: + self.queueThread = threading.Thread(target=self.nextItem, args=(dz, interface)) + self.queueThread.start() + def restoreQueue(self, queue, queueComplete, queueList, settings): self.queue = queue self.queueComplete = queueComplete -- 2.25.1 From 0f733ceaaa1cb2c15d9718057860a53d1cf15d3c Mon Sep 17 00:00:00 2001 From: RemixDev Date: Sat, 13 Mar 2021 11:54:01 +0100 Subject: [PATCH 06/20] Some rework done on types --- deemix/types/Album.py | 17 ++++++++++------- deemix/types/Artist.py | 4 ++-- deemix/types/Date.py | 2 +- deemix/types/Lyrics.py | 8 +++----- deemix/types/Picture.py | 29 ++++++++++++++--------------- deemix/types/Playlist.py | 27 ++++++++++++++------------- 6 files changed, 44 insertions(+), 43 deletions(-) diff --git a/deemix/types/Album.py b/deemix/types/Album.py index a6472cb..2ac9015 100644 --- a/deemix/types/Album.py +++ b/deemix/types/Album.py @@ -14,19 +14,19 @@ class Album: self.artist = {"Main": []} self.artists = [] self.mainArtist = None - self.dateString = None - self.barcode = "Unknown" self.date = None + self.dateString = None + self.trackTotal = "0" self.discTotal = "0" self.embeddedCoverPath = None self.embeddedCoverURL = None self.explicit = False self.genre = [] + self.barcode = "Unknown" self.label = "Unknown" self.recordType = "album" - self.rootArtist = None - self.trackTotal = "0" self.bitrate = 0 + self.rootArtist = None self.variousArtists = None def parseAlbum(self, albumAPI): @@ -42,9 +42,12 @@ class Album: pic_md5 = artistPicture ) if albumAPI.get('root_artist'): + artistPicture = albumAPI['root_artist']['picture_small'] + artistPicture = artistPicture[artistPicture.find('artist/') + 7:-24] self.rootArtist = Artist( id = albumAPI['root_artist']['id'], - name = albumAPI['root_artist']['name'] + name = albumAPI['root_artist']['name'], + pic_md5 = artistPicture ) for artist in albumAPI['contributors']: @@ -82,7 +85,7 @@ class Album: self.discTotal = albumAPI.get('nb_disk') self.copyright = albumAPI.get('copyright') - if not self.pic.md5: + if self.pic.md5 == "": # Getting album cover MD5 # ex: https://e-cdns-images.dzcdn.net/images/cover/2e018122cb56986277102d2041a592c8/56x56-000000-80-0-0.jpg self.pic.md5 = albumAPI['cover_small'][albumAPI['cover_small'].find('cover/') + 6:-24] @@ -106,7 +109,7 @@ class Album: explicitLyricsStatus = albumAPI_gw.get('EXPLICIT_ALBUM_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) self.explicit = explicitLyricsStatus in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT] - if not self.pic.md5: + if self.pic.md5 == "": self.pic.md5 = albumAPI_gw['ALB_PICTURE'] if 'PHYSICAL_RELEASE_DATE' in albumAPI_gw: day = albumAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] diff --git a/deemix/types/Artist.py b/deemix/types/Artist.py index 42cb573..cfc49c4 100644 --- a/deemix/types/Artist.py +++ b/deemix/types/Artist.py @@ -2,11 +2,11 @@ from deemix.types.Picture import Picture from deemix import VARIOUS_ARTISTS class Artist: - def __init__(self, id="0", name="", pic_md5="", role=""): + def __init__(self, id="0", name="", role="", pic_md5=""): self.id = str(id) self.name = name self.pic = Picture(md5=pic_md5, type="artist") - self.role = "" + self.role = role self.save = True def isVariousArtists(self): diff --git a/deemix/types/Date.py b/deemix/types/Date.py index b74e04f..061f2eb 100644 --- a/deemix/types/Date.py +++ b/deemix/types/Date.py @@ -1,5 +1,5 @@ class Date(object): - def __init__(self, year="XXXX", month="00", day="00"): + def __init__(self, day="00", month="00", year="XXXX"): self.year = year self.month = month self.day = day diff --git a/deemix/types/Lyrics.py b/deemix/types/Lyrics.py index a21beb1..8a02a4c 100644 --- a/deemix/types/Lyrics.py +++ b/deemix/types/Lyrics.py @@ -1,16 +1,14 @@ class Lyrics: def __init__(self, id="0"): self.id = id - self.sync = None - self.unsync = None - self.syncID3 = None + self.sync = "" + self.unsync = "" + self.syncID3 = [] def parseLyrics(self, lyricsAPI): self.unsync = lyricsAPI.get("LYRICS_TEXT") if "LYRICS_SYNC_JSON" in lyricsAPI: syncLyricsJson = lyricsAPI["LYRICS_SYNC_JSON"] - self.sync = "" - self.syncID3 = [] timestamp = "" milliseconds = 0 for line in range(len(syncLyricsJson)): diff --git a/deemix/types/Picture.py b/deemix/types/Picture.py index ca00f49..6cf3d99 100644 --- a/deemix/types/Picture.py +++ b/deemix/types/Picture.py @@ -1,27 +1,26 @@ class Picture: - def __init__(self, md5="", type=None, url=None): + def __init__(self, md5="", type="", url=None): self.md5 = md5 self.type = type - self.url = url + self.staticUrl = url def generatePictureURL(self, size, format): - if self.url: return self.url + if self.staticUrl: return self.staticUrl + + url = "https://e-cdns-images.dzcdn.net/images/{}/{}/{}x{}".format( + self.type, + self.md5, + size, size + ) + if format.startswith("jpg"): if '-' in format: quality = format[4:] else: quality = 80 format = 'jpg' - return "https://e-cdns-images.dzcdn.net/images/{}/{}/{}x{}-{}".format( - self.type, - self.md5, - size, size, - f'000000-{quality}-0-0.jpg' - ) + return url + f'-000000-{quality}-0-0.jpg' if format == 'png': - return "https://e-cdns-images.dzcdn.net/images/{}/{}/{}x{}-{}".format( - self.type, - self.md5, - size, size, - 'none-100-0-0.png' - ) + return url + '-none-100-0-0.png' + + return url+'.jpg' diff --git a/deemix/types/Playlist.py b/deemix/types/Playlist.py index 9625719..e358674 100644 --- a/deemix/types/Playlist.py +++ b/deemix/types/Playlist.py @@ -4,17 +4,6 @@ from deemix.types.Picture import Picture class Playlist: def __init__(self, playlistAPI): - if 'various_artist' in playlistAPI: - playlistAPI['various_artist']['role'] = "Main" - self.variousArtists = Artist( - id = playlistAPI['various_artist']['id'], - name = playlistAPI['various_artist']['name'], - pic_md5 = playlistAPI['various_artist']['picture_small'][ - playlistAPI['various_artist']['picture_small'].find('artist/') + 7:-24], - role = playlistAPI['various_artist']['role'] - ) - self.mainArtist = self.variousArtists - self.id = "pl_" + str(playlistAPI['id']) self.title = playlistAPI['title'] self.rootArtist = None @@ -30,11 +19,12 @@ class Playlist: year = playlistAPI["creation_date"][0:4] month = playlistAPI["creation_date"][5:7] day = playlistAPI["creation_date"][8:10] - self.date = Date(year, month, day) + self.date = Date(day, month, year) self.discTotal = "1" - self.playlistId = playlistAPI['id'] + self.playlistID = playlistAPI['id'] self.owner = playlistAPI['creator'] + if 'dzcdn.net' in playlistAPI['picture_small']: url = playlistAPI['picture_small'] picType = url[url.find('images/')+7:] @@ -46,3 +36,14 @@ class Playlist: ) else: self.pic = Picture(url = playlistAPI['picture_xl']) + + if 'various_artist' in playlistAPI: + pic_md5 = playlistAPI['various_artist']['picture_small'] + pic_md5 = pic_md5[pic_md5.indexOf('artist/') + 7:-24] + self.variousArtists = Artist( + id = playlistAPI['various_artist']['id'], + name = playlistAPI['various_artist']['name'], + role = "Main", + pic_md5 = pic_md5 + ) + self.mainArtist = self.variousArtists -- 2.25.1 From 5ee81ced449f508bd0a78a3bf9868db8e628cd99 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Fri, 19 Mar 2021 14:31:32 +0100 Subject: [PATCH 07/20] Total rework of the library (WIP) --- deemix/__init__.py | 59 +- deemix/__main__.py | 55 +- deemix/app/__init__.py | 11 - deemix/app/cli.py | 40 -- deemix/app/messageinterface.py | 4 - deemix/app/queueitem.py | 115 ---- deemix/app/queuemanager.py | 592 ------------------ deemix/app/settings.py | 220 ------- deemix/app/spotifyhelper.py | 346 ----------- deemix/{utils => }/decryption.py | 21 +- deemix/{app/downloadjob.py => downloader.py} | 622 ++++++++----------- deemix/itemgen.py | 246 ++++++++ deemix/plugins/spotify.py | 0 deemix/settings.py | 139 +++++ deemix/{utils => }/taggers.py | 0 deemix/types/Album.py | 2 +- deemix/types/Artist.py | 2 +- deemix/types/DownloadObjects.py | 126 ++++ deemix/types/Track.py | 87 ++- deemix/types/__init__.py | 8 +- deemix/utils/__init__.py | 62 +- setup.py | 5 +- updatePyPi.sh | 4 +- 23 files changed, 979 insertions(+), 1787 deletions(-) delete mode 100644 deemix/app/__init__.py delete mode 100644 deemix/app/cli.py delete mode 100644 deemix/app/messageinterface.py delete mode 100644 deemix/app/queueitem.py delete mode 100644 deemix/app/queuemanager.py delete mode 100644 deemix/app/settings.py delete mode 100644 deemix/app/spotifyhelper.py rename deemix/{utils => }/decryption.py (69%) rename deemix/{app/downloadjob.py => downloader.py} (63%) create mode 100644 deemix/itemgen.py create mode 100644 deemix/plugins/spotify.py create mode 100644 deemix/settings.py rename deemix/{utils => }/taggers.py (100%) create mode 100644 deemix/types/DownloadObjects.py diff --git a/deemix/__init__.py b/deemix/__init__.py index de69d03..ea1b8ae 100644 --- a/deemix/__init__.py +++ b/deemix/__init__.py @@ -1,6 +1,63 @@ #!/usr/bin/env python3 +import re +from urllib.request import urlopen + +from deemix.itemgen import generateTrackItem, generateAlbumItem, generatePlaylistItem, generateArtistItem, generateArtistDiscographyItem, generateArtistTopItem __version__ = "2.0.16" USER_AGENT_HEADER = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) " \ "Chrome/79.0.3945.130 Safari/537.36" -VARIOUS_ARTISTS = "5080" + +# Returns the Resolved URL, the Type and the ID +def parseLink(link): + if 'deezer.page.link' in link: link = urlopen(url).url # Resolve URL shortner + # Remove extra stuff + if '?' in link: link = link[:link.find('?')] + if '&' in link: link = link[:link.find('&')] + if link.endswith('/'): link = link[:-1] # Remove last slash if present + + type = None + id = None + + if not 'deezer' in link: return (link, type, id) # return if not a deezer link + + if '/track' in link: + type = 'track' + id = link[link.rfind("/") + 1:] + elif '/playlist' in link: + type = 'playlist' + id = re.search("\/playlist\/(\d+)", link)[0] + elif '/album' in link: + type = 'album' + id = link[link.rfind("/") + 1:] + elif re.search("\/artist\/(\d+)\/top_track", link): + type = 'artist_top' + id = re.search("\/artist\/(\d+)\/top_track", link)[0] + elif re.search("\/artist\/(\d+)\/discography", link): + type = 'artist_discography' + id = re.search("\/artist\/(\d+)\/discography", link)[0] + elif '/artist' in link: + type = 'artist' + id = re.search("\/artist\/(\d+)", link)[0] + + return (link, type, id) + +def generateDownloadItem(dz, link, bitrate): + (link, type, id) = parseLink(link) + + if type == None or id == None: return None + + if type == "track": + return generateTrackItem(dz, id, bitrate) + elif type == "album": + return generateAlbumItem(dz, id, bitrate) + elif type == "playlist": + return generatePlaylistItem(dz, id, bitrate) + elif type == "artist": + return generateArtistItem(dz, id, bitrate) + elif type == "artist_discography": + return generateArtistDiscographyItem(dz, id, bitrate) + elif type == "artist_top": + return generateArtistTopItem(dz, id, bitrate) + + return None diff --git a/deemix/__main__.py b/deemix/__main__.py index 35bb938..fde6781 100644 --- a/deemix/__main__.py +++ b/deemix/__main__.py @@ -1,26 +1,65 @@ #!/usr/bin/env python3 import click - -from deemix.app.cli import cli from pathlib import Path +from deezer import Deezer +from deezer import TrackFormats + +from deemix import generateDownloadItem +from deemix.settings import loadSettings +from deemix.utils import getBitrateNumberFromText +import deemix.utils.localpaths as localpaths +from deemix.downloader import Downloader + @click.command() @click.option('--portable', is_flag=True, help='Creates the config folder in the same directory where the script is launched') @click.option('-b', '--bitrate', default=None, help='Overwrites the default bitrate selected') @click.option('-p', '--path', type=str, help='Downloads in the given folder') @click.argument('url', nargs=-1, required=True) def download(url, bitrate, portable, path): - + # Check for local configFolder localpath = Path('.') - configFolder = localpath / 'config' if portable else None + configFolder = localpath / 'config' if portable else localpaths.getConfigFolder() + + settings = loadSettings(configFolder) + dz = Deezer(settings.get('tagsLanguage')) + + def requestValidArl(): + while True: + arl = input("Paste here your arl:") + if dz.login_via_arl(arl.strip()): break + return arl + + if (configFolder / '.arl').is_file(): + with open(configFolder / '.arl', 'r') as f: + arl = f.readline().rstrip("\n").strip() + if not dz.login_via_arl(arl): arl = requestValidArl() + else: arl = requestValidArl() + with open(configFolder / '.arl', 'w') as f: + f.write(arl) + + def downloadLinks(url, bitrate=None): + if not bitrate: bitrate = settings.get("maxBitrate", TrackFormats.MP3_320) + links = [] + for link in url: + if ';' in link: + for l in link.split(";"): + links.append(l) + else: + links.append(link) + + for link in links: + downloadItem = generateDownloadItem(dz, link, bitrate) + Downloader(dz, downloadItem, settings).start() + if path is not None: if path == '': path = '.' path = Path(path) - - app = cli(path, configFolder) - app.login() + settings['downloadLocation'] = str(path) url = list(url) + if bitrate: bitrate = getBitrateNumberFromText(bitrate) + # If first url is filepath readfile and use them as URLs try: isfile = Path(url[0]).is_file() except: @@ -30,7 +69,7 @@ def download(url, bitrate, portable, path): with open(filename) as f: url = f.readlines() - app.downloadLink(url, bitrate) + downloadLinks(url, bitrate) click.echo("All done!") if __name__ == '__main__': diff --git a/deemix/app/__init__.py b/deemix/app/__init__.py deleted file mode 100644 index 225936f..0000000 --- a/deemix/app/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from deezer import Deezer -from deemix.app.settings import Settings -from deemix.app.queuemanager import QueueManager -from deemix.app.spotifyhelper import SpotifyHelper - -class deemix: - def __init__(self, configFolder=None, overwriteDownloadFolder=None): - self.set = Settings(configFolder, overwriteDownloadFolder=overwriteDownloadFolder) - self.dz = Deezer(self.set.settings.get('tagsLanguage')) - self.sp = SpotifyHelper(configFolder) - self.qm = QueueManager(self.dz, self.sp) diff --git a/deemix/app/cli.py b/deemix/app/cli.py deleted file mode 100644 index b1d2bf8..0000000 --- a/deemix/app/cli.py +++ /dev/null @@ -1,40 +0,0 @@ -from pathlib import Path -from os import makedirs - -from deemix.app import deemix -from deemix.utils import checkFolder - -class cli(deemix): - def __init__(self, downloadpath, configFolder=None): - super().__init__(configFolder, overwriteDownloadFolder=downloadpath) - if downloadpath: - print("Using folder: "+self.set.settings['downloadLocation']) - - def downloadLink(self, url, bitrate=None): - for link in url: - if ';' in link: - for l in link.split(";"): - self.qm.addToQueue(l, self.set.settings, bitrate) - else: - self.qm.addToQueue(link, self.set.settings, bitrate) - - def requestValidArl(self): - while True: - arl = input("Paste here your arl:") - if self.dz.login_via_arl(arl): - break - return arl - - def login(self): - configFolder = Path(self.set.configFolder) - if not configFolder.is_dir(): - makedirs(configFolder, exist_ok=True) - if (configFolder / '.arl').is_file(): - with open(configFolder / '.arl', 'r') as f: - arl = f.readline().rstrip("\n") - if not self.dz.login_via_arl(arl): - arl = self.requestValidArl() - else: - arl = self.requestValidArl() - with open(configFolder / '.arl', 'w') as f: - f.write(arl) diff --git a/deemix/app/messageinterface.py b/deemix/app/messageinterface.py deleted file mode 100644 index ef910c2..0000000 --- a/deemix/app/messageinterface.py +++ /dev/null @@ -1,4 +0,0 @@ -class MessageInterface: - def send(self, message, value=None): - """Implement this class to process updates and messages from the core""" - pass diff --git a/deemix/app/queueitem.py b/deemix/app/queueitem.py deleted file mode 100644 index 49e223b..0000000 --- a/deemix/app/queueitem.py +++ /dev/null @@ -1,115 +0,0 @@ -class QueueItem: - def __init__(self, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, type=None, settings=None, queueItemDict=None): - if queueItemDict: - self.title = queueItemDict['title'] - self.artist = queueItemDict['artist'] - self.cover = queueItemDict['cover'] - self.explicit = queueItemDict.get('explicit', False) - self.size = queueItemDict['size'] - self.type = queueItemDict['type'] - self.id = queueItemDict['id'] - self.bitrate = queueItemDict['bitrate'] - self.extrasPath = queueItemDict.get('extrasPath', '') - self.files = queueItemDict['files'] - self.downloaded = queueItemDict['downloaded'] - self.failed = queueItemDict['failed'] - self.errors = queueItemDict['errors'] - self.progress = queueItemDict['progress'] - self.settings = queueItemDict.get('settings') - else: - self.title = title - self.artist = artist - self.cover = cover - self.explicit = explicit - self.size = size - self.type = type - self.id = id - self.bitrate = bitrate - self.extrasPath = None - self.files = [] - self.settings = settings - self.downloaded = 0 - self.failed = 0 - self.errors = [] - self.progress = 0 - self.uuid = f"{self.type}_{self.id}_{self.bitrate}" - self.cancel = False - self.ack = None - - def toDict(self): - return { - 'title': self.title, - 'artist': self.artist, - 'cover': self.cover, - 'explicit': self.explicit, - 'size': self.size, - 'extrasPath': self.extrasPath, - 'files': self.files, - 'downloaded': self.downloaded, - 'failed': self.failed, - 'errors': self.errors, - 'progress': self.progress, - 'type': self.type, - 'id': self.id, - 'bitrate': self.bitrate, - 'uuid': self.uuid, - 'ack': self.ack - } - - def getResettedItem(self): - item = self.toDict() - item['downloaded'] = 0 - item['failed'] = 0 - item['progress'] = 0 - item['errors'] = [] - return item - - def getSlimmedItem(self): - light = self.toDict() - propertiesToDelete = ['single', 'collection', '_EXTRA', 'settings'] - for property in propertiesToDelete: - if property in light: - del light[property] - return light - -class QISingle(QueueItem): - def __init__(self, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, type=None, settings=None, single=None, queueItemDict=None): - if queueItemDict: - super().__init__(queueItemDict=queueItemDict) - self.single = queueItemDict['single'] - else: - super().__init__(id, bitrate, title, artist, cover, explicit, 1, type, settings) - self.single = single - - def toDict(self): - queueItem = super().toDict() - queueItem['single'] = self.single - return queueItem - -class QICollection(QueueItem): - def __init__(self, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, type=None, settings=None, collection=None, queueItemDict=None): - if queueItemDict: - super().__init__(queueItemDict=queueItemDict) - self.collection = queueItemDict['collection'] - else: - super().__init__(id, bitrate, title, artist, cover, explicit, size, type, settings) - self.collection = collection - - def toDict(self): - queueItem = super().toDict() - queueItem['collection'] = self.collection - return queueItem - -class QIConvertable(QICollection): - def __init__(self, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, type=None, settings=None, extra=None, queueItemDict=None): - if queueItemDict: - super().__init__(queueItemDict=queueItemDict) - self.extra = queueItemDict['_EXTRA'] - else: - super().__init__(id, bitrate, title, artist, cover, explicit, size, type, settings, []) - self.extra = extra - - def toDict(self): - queueItem = super().toDict() - queueItem['_EXTRA'] = self.extra - return queueItem diff --git a/deemix/app/queuemanager.py b/deemix/app/queuemanager.py deleted file mode 100644 index 03a8a89..0000000 --- a/deemix/app/queuemanager.py +++ /dev/null @@ -1,592 +0,0 @@ -from deemix.app.downloadjob import DownloadJob -from deemix.utils import getIDFromLink, getTypeFromLink, getBitrateInt - -from deezer import Deezer -from deezer.gw import APIError as gwAPIError, LyricsStatus -from deezer.api import APIError -from deezer.utils import map_user_playlist - -from spotipy.exceptions import SpotifyException -from deemix.app.queueitem import QueueItem, QISingle, QICollection, QIConvertable -import logging -from pathlib import Path -import json -from os import remove -import uuid -from urllib.request import urlopen - -import threading - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('deemix') - -class QueueManager: - def __init__(self, deezerHelper=None, spotifyHelper=None): - self.queue = [] - self.queueList = {} - self.queueComplete = [] - self.currentItem = "" - self.dz = deezerHelper or Deezer() - self.sp = spotifyHelper - self.queueThread = None - - def generateTrackQueueItem(self, id, settings, bitrate, trackAPI=None, albumAPI=None, dz=None): - if not dz: dz = self.dz - # Check if is an isrc: url - if str(id).startswith("isrc"): - try: - trackAPI = dz.api.get_track(id) - except APIError as e: - e = str(e) - return QueueError("https://deezer.com/track/"+str(id), f"Wrong URL: {e}") - if 'id' in trackAPI and 'title' in trackAPI: - id = trackAPI['id'] - else: - return QueueError("https://deezer.com/track/"+str(id), "Track ISRC is not available on deezer", "ISRCnotOnDeezer") - - # Get essential track info - try: - trackAPI_gw = dz.gw.get_track_with_fallback(id) - except gwAPIError as e: - e = str(e) - message = "Wrong URL" - if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}" - return QueueError("https://deezer.com/track/"+str(id), message) - - if albumAPI: trackAPI_gw['_EXTRA_ALBUM'] = albumAPI - if trackAPI: trackAPI_gw['_EXTRA_TRACK'] = trackAPI - - if settings['createSingleFolder']: - trackAPI_gw['FILENAME_TEMPLATE'] = settings['albumTracknameTemplate'] - else: - trackAPI_gw['FILENAME_TEMPLATE'] = settings['tracknameTemplate'] - - trackAPI_gw['SINGLE_TRACK'] = True - - title = trackAPI_gw['SNG_TITLE'].strip() - if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: - title += f" {trackAPI_gw['VERSION']}".strip() - explicit = bool(int(trackAPI_gw.get('EXPLICIT_LYRICS', 0))) - - return QISingle( - id=id, - bitrate=bitrate, - title=title, - artist=trackAPI_gw['ART_NAME'], - cover=f"https://e-cdns-images.dzcdn.net/images/cover/{trackAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg", - explicit=explicit, - type='track', - settings=settings, - single=trackAPI_gw, - ) - - def generateAlbumQueueItem(self, id, settings, bitrate, rootArtist=None, dz=None): - if not dz: dz = self.dz - # Get essential album info - try: - albumAPI = dz.api.get_album(id) - except APIError as e: - e = str(e) - return QueueError("https://deezer.com/album/"+str(id), f"Wrong URL: {e}") - - if str(id).startswith('upc'): id = albumAPI['id'] - - # Get extra info about album - # This saves extra api calls when downloading - albumAPI_gw = dz.gw.get_album(id) - albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK'] - albumAPI['copyright'] = albumAPI_gw['COPYRIGHT'] - albumAPI['root_artist'] = rootArtist - - # If the album is a single download as a track - if albumAPI['nb_tracks'] == 1: - return self.generateTrackQueueItem(albumAPI['tracks']['data'][0]['id'], settings, bitrate, albumAPI=albumAPI, dz=dz) - - tracksArray = dz.gw.get_album_tracks(id) - - if albumAPI['cover_small'] != None: - cover = albumAPI['cover_small'][:-24] + '/75x75-000000-80-0-0.jpg' - else: - cover = f"https://e-cdns-images.dzcdn.net/images/cover/{albumAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg" - - totalSize = len(tracksArray) - albumAPI['nb_tracks'] = totalSize - collection = [] - for pos, trackAPI in enumerate(tracksArray, start=1): - trackAPI['_EXTRA_ALBUM'] = albumAPI - trackAPI['POSITION'] = pos - trackAPI['SIZE'] = totalSize - trackAPI['FILENAME_TEMPLATE'] = settings['albumTracknameTemplate'] - collection.append(trackAPI) - - explicit = albumAPI_gw.get('EXPLICIT_ALBUM_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT] - - return QICollection( - id=id, - bitrate=bitrate, - title=albumAPI['title'], - artist=albumAPI['artist']['name'], - cover=cover, - explicit=explicit, - size=totalSize, - type='album', - settings=settings, - collection=collection, - ) - - def generatePlaylistQueueItem(self, id, settings, bitrate, dz=None): - if not dz: dz = self.dz - # Get essential playlist info - try: - playlistAPI = dz.api.get_playlist(id) - except: - playlistAPI = None - # Fallback to gw api if the playlist is private - if not playlistAPI: - try: - userPlaylist = dz.gw.get_playlist_page(id) - playlistAPI = map_user_playlist(userPlaylist['DATA']) - except gwAPIError as e: - e = str(e) - message = "Wrong URL" - if "DATA_ERROR" in e: - message += f": {e['DATA_ERROR']}" - return QueueError("https://deezer.com/playlist/"+str(id), message) - - # Check if private playlist and owner - if not playlistAPI.get('public', False) and playlistAPI['creator']['id'] != str(dz.current_user['id']): - logger.warning("You can't download others private playlists.") - return QueueError("https://deezer.com/playlist/"+str(id), "You can't download others private playlists.", "notYourPrivatePlaylist") - - playlistTracksAPI = dz.gw.get_playlist_tracks(id) - playlistAPI['various_artist'] = dz.api.get_artist(5080) # Useful for save as compilation - - totalSize = len(playlistTracksAPI) - playlistAPI['nb_tracks'] = totalSize - collection = [] - for pos, trackAPI in enumerate(playlistTracksAPI, start=1): - if trackAPI.get('EXPLICIT_TRACK_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT]: - playlistAPI['explicit'] = True - trackAPI['_EXTRA_PLAYLIST'] = playlistAPI - trackAPI['POSITION'] = pos - trackAPI['SIZE'] = totalSize - trackAPI['FILENAME_TEMPLATE'] = settings['playlistTracknameTemplate'] - collection.append(trackAPI) - if not 'explicit' in playlistAPI: - playlistAPI['explicit'] = False - - return QICollection( - id=id, - bitrate=bitrate, - title=playlistAPI['title'], - artist=playlistAPI['creator']['name'], - cover=playlistAPI['picture_small'][:-24] + '/75x75-000000-80-0-0.jpg', - explicit=playlistAPI['explicit'], - size=totalSize, - type='playlist', - settings=settings, - collection=collection, - ) - - def generateArtistQueueItem(self, id, settings, bitrate, dz=None, interface=None): - if not dz: dz = self.dz - # Get essential artist info - try: - artistAPI = dz.api.get_artist(id) - except APIError as e: - e = str(e) - return QueueError("https://deezer.com/artist/"+str(id), f"Wrong URL: {e}") - - if interface: interface.send("startAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) - rootArtist = { - 'id': artistAPI['id'], - 'name': artistAPI['name'] - } - - artistDiscographyAPI = dz.gw.get_artist_discography_tabs(id, 100) - allReleases = artistDiscographyAPI.pop('all', []) - albumList = [] - for album in allReleases: - albumList.append(self.generateAlbumQueueItem(album['id'], settings, bitrate, rootArtist=rootArtist, dz=dz)) - - if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) - return albumList - - def generateArtistDiscographyQueueItem(self, id, settings, bitrate, dz=None, interface=None): - if not dz: dz = self.dz - # Get essential artist info - try: - artistAPI = dz.api.get_artist(id) - except APIError as e: - e = str(e) - return QueueError("https://deezer.com/artist/"+str(id)+"/discography", f"Wrong URL: {e}") - - if interface: interface.send("startAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) - rootArtist = { - 'id': artistAPI['id'], - 'name': artistAPI['name'] - } - - artistDiscographyAPI = dz.gw.get_artist_discography_tabs(id, 100) - artistDiscographyAPI.pop('all', None) # all contains albums and singles, so its all duplicates. This removes them - albumList = [] - for type in artistDiscographyAPI: - for album in artistDiscographyAPI[type]: - albumList.append(self.generateAlbumQueueItem(album['id'], settings, bitrate, rootArtist=rootArtist, dz=dz)) - - if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) - return albumList - - def generateArtistTopQueueItem(self, id, settings, bitrate, dz=None, interface=None): - if not dz: dz = self.dz - # Get essential artist info - try: - artistAPI = dz.api.get_artist(id) - except APIError as e: - e = str(e) - return QueueError("https://deezer.com/artist/"+str(id)+"/top_track", f"Wrong URL: {e}") - - # Emulate the creation of a playlist - # Can't use generatePlaylistQueueItem as this is not a real playlist - playlistAPI = { - 'id': str(artistAPI['id'])+"_top_track", - 'title': artistAPI['name']+" - Top Tracks", - 'description': "Top Tracks for "+artistAPI['name'], - 'duration': 0, - 'public': True, - 'is_loved_track': False, - 'collaborative': False, - 'nb_tracks': 0, - 'fans': artistAPI['nb_fan'], - 'link': "https://www.deezer.com/artist/"+str(artistAPI['id'])+"/top_track", - 'share': None, - 'picture': artistAPI['picture'], - 'picture_small': artistAPI['picture_small'], - 'picture_medium': artistAPI['picture_medium'], - 'picture_big': artistAPI['picture_big'], - 'picture_xl': artistAPI['picture_xl'], - 'checksum': None, - 'tracklist': "https://api.deezer.com/artist/"+str(artistAPI['id'])+"/top", - 'creation_date': "XXXX-00-00", - 'creator': { - 'id': "art_"+str(artistAPI['id']), - 'name': artistAPI['name'], - 'type': "user" - }, - 'type': "playlist" - } - - artistTopTracksAPI_gw = dz.gw.get_artist_toptracks(id) - playlistAPI['various_artist'] = dz.api.get_artist(5080) # Useful for save as compilation - - totalSize = len(artistTopTracksAPI_gw) - playlistAPI['nb_tracks'] = totalSize - collection = [] - for pos, trackAPI in enumerate(artistTopTracksAPI_gw, start=1): - if trackAPI.get('EXPLICIT_TRACK_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT]: - playlistAPI['explicit'] = True - trackAPI['_EXTRA_PLAYLIST'] = playlistAPI - trackAPI['POSITION'] = pos - trackAPI['SIZE'] = totalSize - trackAPI['FILENAME_TEMPLATE'] = settings['playlistTracknameTemplate'] - collection.append(trackAPI) - if not 'explicit' in playlistAPI: - playlistAPI['explicit'] = False - - return QICollection( - id=id, - bitrate=bitrate, - title=playlistAPI['title'], - artist=playlistAPI['creator']['name'], - cover=playlistAPI['picture_small'][:-24] + '/75x75-000000-80-0-0.jpg', - explicit=playlistAPI['explicit'], - size=totalSize, - type='playlist', - settings=settings, - collection=collection, - ) - - def generateQueueItem(self, url, settings, bitrate=None, dz=None, interface=None): - if not dz: dz = self.dz - bitrate = getBitrateInt(bitrate) or settings['maxBitrate'] - if 'deezer.page.link' in url: url = urlopen(url).url - if 'link.tospotify.com' in url: url = urlopen(url).url - - type = getTypeFromLink(url) - id = getIDFromLink(url, type) - if type == None or id == None: - logger.warn("URL not recognized") - return QueueError(url, "URL not recognized", "invalidURL") - - if type == "track": - return self.generateTrackQueueItem(id, settings, bitrate, dz=dz) - elif type == "album": - return self.generateAlbumQueueItem(id, settings, bitrate, dz=dz) - elif type == "playlist": - return self.generatePlaylistQueueItem(id, settings, bitrate, dz=dz) - elif type == "artist": - return self.generateArtistQueueItem(id, settings, bitrate, interface=interface, dz=dz) - elif type == "artistdiscography": - return self.generateArtistDiscographyQueueItem(id, settings, bitrate, interface=interface, dz=dz) - elif type == "artisttop": - return self.generateArtistTopQueueItem(id, settings, bitrate, interface=interface, dz=dz) - elif type.startswith("spotify") and self.sp: - if not self.sp.spotifyEnabled: - logger.warn("Spotify Features is not setted up correctly.") - return QueueError(url, "Spotify Features is not setted up correctly.", "spotifyDisabled") - - if type == "spotifytrack": - try: - (track_id, trackAPI, _) = self.sp.get_trackid_spotify(dz, id, settings['fallbackSearch']) - except SpotifyException as e: - return QueueError(url, "Wrong URL: "+e.msg[e.msg.find('\n')+2:]) - except Exception as e: - return QueueError(url, "Something went wrong: "+str(e)) - - if track_id != "0": - return self.generateTrackQueueItem(track_id, settings, bitrate, trackAPI=trackAPI, dz=dz) - else: - logger.warn("Track not found on deezer!") - return QueueError(url, "Track not found on deezer!", "trackNotOnDeezer") - - elif type == "spotifyalbum": - try: - album_id = self.sp.get_albumid_spotify(dz, id) - except SpotifyException as e: - return QueueError(url, "Wrong URL: "+e.msg[e.msg.find('\n')+2:]) - except Exception as e: - return QueueError(url, "Something went wrong: "+str(e)) - - if album_id != "0": - return self.generateAlbumQueueItem(album_id, settings, bitrate, dz=dz) - else: - logger.warn("Album not found on deezer!") - return QueueError(url, "Album not found on deezer!", "albumNotOnDeezer") - - elif type == "spotifyplaylist": - try: - return self.sp.generate_playlist_queueitem(dz, id, bitrate, settings) - except SpotifyException as e: - return QueueError(url, "Wrong URL: "+e.msg[e.msg.find('\n')+2:]) - except Exception as e: - return QueueError(url, "Something went wrong: "+str(e)) - logger.warn("URL not supported yet") - return QueueError(url, "URL not supported yet", "unsupportedURL") - - def addToQueue(self, url, settings, bitrate=None, dz=None, interface=None, ack=None): - if not dz: dz = self.dz - - if not dz.logged_in: - if interface: interface.send("loginNeededToDownload") - return False - - def parseLink(link): - link = link.strip() - if link == "": return False - logger.info("Generating queue item for: "+link) - item = self.generateQueueItem(link, settings, bitrate, interface=interface, dz=dz) - - # Add ack to all items - if type(item) is list: - for i in item: - if isinstance(i, QueueItem): - i.ack = ack - elif isinstance(item, QueueItem): - item.ack = ack - return item - - if type(url) is list: - queueItem = [] - request_uuid = str(uuid.uuid4()) - if interface: interface.send("startGeneratingItems", {'uuid': request_uuid, 'total': len(url)}) - for link in url: - item = parseLink(link) - if not item: continue - if type(item) is list: - queueItem += item - else: - queueItem.append(item) - if interface: interface.send("finishGeneratingItems", {'uuid': request_uuid, 'total': len(queueItem)}) - if not len(queueItem): - return False - else: - queueItem = parseLink(url) - if not queueItem: - return False - - def processQueueItem(item, silent=False): - if isinstance(item, QueueError): - logger.error(f"[{item.link}] {item.message}") - if interface: interface.send("queueError", item.toDict()) - return False - if item.uuid in list(self.queueList.keys()): - logger.warn(f"[{item.uuid}] Already in queue, will not be added again.") - if interface and not silent: interface.send("alreadyInQueue", {'uuid': item.uuid, 'title': item.title}) - return False - self.queue.append(item.uuid) - self.queueList[item.uuid] = item - logger.info(f"[{item.uuid}] Added to queue.") - return True - - if type(queueItem) is list: - slimmedItems = [] - for item in queueItem: - if processQueueItem(item, silent=True): - slimmedItems.append(item.getSlimmedItem()) - else: - continue - if not len(slimmedItems): - return False - if interface: interface.send("addedToQueue", slimmedItems) - else: - if processQueueItem(queueItem): - if interface: interface.send("addedToQueue", queueItem.getSlimmedItem()) - else: - return False - self.startQueue(interface, dz) - return True - - def nextItem(self, dz=None, interface=None): - if not dz: dz = self.dz - # Check that nothing is already downloading and - # that the queue is not empty - if self.currentItem != "" or not len(self.queue): - self.queueThread = None - return None - - self.currentItem = self.queue.pop(0) - - if isinstance(self.queueList[self.currentItem], QIConvertable) and self.queueList[self.currentItem].extra: - logger.info(f"[{self.currentItem}] Converting tracks to deezer.") - self.sp.convert_spotify_playlist(dz, self.queueList[self.currentItem], interface=interface) - logger.info(f"[{self.currentItem}] Tracks converted.") - - if interface: interface.send("startDownload", self.currentItem) - logger.info(f"[{self.currentItem}] Started downloading.") - - DownloadJob(dz, self.queueList[self.currentItem], interface).start() - - if self.queueList[self.currentItem].cancel: - del self.queueList[self.currentItem] - else: - self.queueComplete.append(self.currentItem) - logger.info(f"[{self.currentItem}] Finished downloading.") - self.currentItem = "" - self.nextItem(dz, interface) - - def getQueue(self): - return (self.queue, self.queueComplete, self.slimQueueList(), self.currentItem) - - def saveQueue(self, configFolder): - if len(self.queueList) > 0: - if self.currentItem != "": - self.queue.insert(0, self.currentItem) - with open(Path(configFolder) / 'queue.json', 'w') as f: - json.dump({ - 'queue': self.queue, - 'queueComplete': self.queueComplete, - 'queueList': self.exportQueueList() - }, f) - - def exportQueueList(self): - queueList = {} - for uuid in self.queueList: - if uuid in self.queue: - queueList[uuid] = self.queueList[uuid].getResettedItem() - else: - queueList[uuid] = self.queueList[uuid].toDict() - return queueList - - def slimQueueList(self): - queueList = {} - for uuid in self.queueList: - queueList[uuid] = self.queueList[uuid].getSlimmedItem() - return queueList - - def loadQueue(self, configFolder, settings, interface=None): - configFolder = Path(configFolder) - if (configFolder / 'queue.json').is_file() and not len(self.queue): - if interface: interface.send('restoringQueue') - with open(configFolder / 'queue.json', 'r') as f: - try: - qd = json.load(f) - except json.decoder.JSONDecodeError: - logger.warn("Saved queue is corrupted, resetting it") - qd = { - 'queue': [], - 'queueComplete': [], - 'queueList': {} - } - remove(configFolder / 'queue.json') - self.restoreQueue(qd['queue'], qd['queueComplete'], qd['queueList'], settings) - if interface: - interface.send('init_downloadQueue', { - 'queue': self.queue, - 'queueComplete': self.queueComplete, - 'queueList': self.slimQueueList(), - 'restored': True - }) - - def startQueue(self, interface=None, dz=None): - if not dz: dz = self.dz - if dz.logged_in and not self.queueThread: - self.queueThread = threading.Thread(target=self.nextItem, args=(dz, interface)) - self.queueThread.start() - - def restoreQueue(self, queue, queueComplete, queueList, settings): - self.queue = queue - self.queueComplete = queueComplete - self.queueList = {} - for uuid in queueList: - if 'single' in queueList[uuid]: - self.queueList[uuid] = QISingle(queueItemDict = queueList[uuid]) - if 'collection' in queueList[uuid]: - self.queueList[uuid] = QICollection(queueItemDict = queueList[uuid]) - if '_EXTRA' in queueList[uuid]: - self.queueList[uuid] = QIConvertable(queueItemDict = queueList[uuid]) - self.queueList[uuid].settings = settings - - def removeFromQueue(self, uuid, interface=None): - if uuid == self.currentItem: - if interface: interface.send("cancellingCurrentItem", uuid) - self.queueList[uuid].cancel = True - return - if uuid in self.queue: - self.queue.remove(uuid) - elif uuid in self.queueComplete: - self.queueComplete.remove(uuid) - else: - return - del self.queueList[uuid] - if interface: interface.send("removedFromQueue", uuid) - - - def cancelAllDownloads(self, interface=None): - self.queue = [] - self.queueComplete = [] - if self.currentItem != "": - if interface: interface.send("cancellingCurrentItem", self.currentItem) - self.queueList[self.currentItem].cancel = True - for uuid in list(self.queueList.keys()): - if uuid != self.currentItem: del self.queueList[uuid] - if interface: interface.send("removedAllDownloads", self.currentItem) - - - def removeFinishedDownloads(self, interface=None): - for uuid in self.queueComplete: - del self.queueList[uuid] - self.queueComplete = [] - if interface: interface.send("removedFinishedDownloads") - -class QueueError: - def __init__(self, link, message, errid=None): - self.link = link - self.message = message - self.errid = errid - - def toDict(self): - return { - 'link': self.link, - 'error': self.message, - 'errid': self.errid - } diff --git a/deemix/app/settings.py b/deemix/app/settings.py deleted file mode 100644 index 2390048..0000000 --- a/deemix/app/settings.py +++ /dev/null @@ -1,220 +0,0 @@ -import json -from pathlib import Path -from os import makedirs, listdir -from deemix import __version__ as deemixVersion -from deezer import TrackFormats -from deemix.utils import checkFolder -import logging -import datetime -import platform - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('deemix') - -import deemix.utils.localpaths as localpaths - -class OverwriteOption(): - """Should the lib overwrite files?""" - - OVERWRITE = 'y' - """Yes, overwrite the file""" - - DONT_OVERWRITE = 'n' - """No, don't overwrite the file""" - - DONT_CHECK_EXT = 'e' - """No, and don't check for extensions""" - - KEEP_BOTH = 'b' - """No, and keep both files""" - - ONLY_TAGS = 't' - """Overwrite only the tags""" - -class FeaturesOption(): - """What should I do with featured artists?""" - - NO_CHANGE = "0" - """Do nothing""" - - REMOVE_TITLE = "1" - """Remove from track title""" - - REMOVE_TITLE_ALBUM = "3" - """Remove from track title and album title""" - - MOVE_TITLE = "2" - """Move to track title""" - -DEFAULT_SETTINGS = { - "downloadLocation": str(localpaths.getMusicFolder()), - "tracknameTemplate": "%artist% - %title%", - "albumTracknameTemplate": "%tracknumber% - %title%", - "playlistTracknameTemplate": "%position% - %artist% - %title%", - "createPlaylistFolder": True, - "playlistNameTemplate": "%playlist%", - "createArtistFolder": False, - "artistNameTemplate": "%artist%", - "createAlbumFolder": True, - "albumNameTemplate": "%artist% - %album%", - "createCDFolder": True, - "createStructurePlaylist": False, - "createSingleFolder": False, - "padTracks": True, - "paddingSize": "0", - "illegalCharacterReplacer": "_", - "queueConcurrency": 3, - "maxBitrate": str(TrackFormats.MP3_320), - "fallbackBitrate": True, - "fallbackSearch": False, - "logErrors": True, - "logSearched": False, - "saveDownloadQueue": False, - "overwriteFile": OverwriteOption.DONT_OVERWRITE, - "createM3U8File": False, - "playlistFilenameTemplate": "playlist", - "syncedLyrics": False, - "embeddedArtworkSize": 800, - "embeddedArtworkPNG": False, - "localArtworkSize": 1400, - "localArtworkFormat": "jpg", - "saveArtwork": True, - "coverImageTemplate": "cover", - "saveArtworkArtist": False, - "artistImageTemplate": "folder", - "jpegImageQuality": 80, - "dateFormat": "Y-M-D", - "albumVariousArtists": True, - "removeAlbumVersion": False, - "removeDuplicateArtists": False, - "tagsLanguage": "", - "featuredToTitle": FeaturesOption.NO_CHANGE, - "titleCasing": "nothing", - "artistCasing": "nothing", - "executeCommand": "", - "tags": { - "title": True, - "artist": True, - "album": True, - "cover": True, - "trackNumber": True, - "trackTotal": False, - "discNumber": True, - "discTotal": False, - "albumArtist": True, - "genre": True, - "year": True, - "date": True, - "explicit": False, - "isrc": True, - "length": True, - "barcode": True, - "bpm": True, - "replayGain": False, - "label": True, - "lyrics": False, - "syncedLyrics": False, - "copyright": False, - "composer": False, - "involvedPeople": False, - "source": False, - "savePlaylistAsCompilation": False, - "useNullSeparator": False, - "saveID3v1": True, - "multiArtistSeparator": "default", - "singleAlbumArtist": False, - "coverDescriptionUTF8": False - } -} - -class Settings: - def __init__(self, configFolder=None, overwriteDownloadFolder=None): - self.settings = {} - self.configFolder = Path(configFolder or localpaths.getConfigFolder()) - - # Create config folder if it doesn't exsist - makedirs(self.configFolder, exist_ok=True) - - # Create config file if it doesn't exsist - if not (self.configFolder / 'config.json').is_file(): - with open(self.configFolder / 'config.json', 'w') as f: - json.dump(DEFAULT_SETTINGS, f, indent=2) - - # Read config file - with open(self.configFolder / 'config.json', 'r') as configFile: - self.settings = json.load(configFile) - - # Check for overwriteDownloadFolder - # This prevents the creation of the original download folder when - # using overwriteDownloadFolder - originalDownloadFolder = self.settings['downloadLocation'] - if overwriteDownloadFolder: - overwriteDownloadFolder = str(overwriteDownloadFolder) - self.settings['downloadLocation'] = overwriteDownloadFolder - - # Make sure the download path exsits, fallback to default - invalidDownloadFolder = False - if self.settings['downloadLocation'] == "" or not checkFolder(self.settings['downloadLocation']): - self.settings['downloadLocation'] = DEFAULT_SETTINGS['downloadLocation'] - originalDownloadFolder = self.settings['downloadLocation'] - invalidDownloadFolder = True - - # Check the settings and save them if something changed - if self.settingsCheck() > 0 or invalidDownloadFolder: - makedirs(self.settings['downloadLocation'], exist_ok=True) - self.settings['downloadLocation'] = originalDownloadFolder # Prevents the saving of the overwritten path - self.saveSettings() - self.settings['downloadLocation'] = overwriteDownloadFolder or originalDownloadFolder # Restores the correct path - - # LOGFILES - - # Create logfile name and path - logspath = self.configFolder / 'logs' - now = datetime.datetime.now() - logfile = now.strftime("%Y-%m-%d_%H%M%S")+".log" - makedirs(logspath, exist_ok=True) - - # Add handler for logging - fh = logging.FileHandler(logspath / logfile, 'w', 'utf-8') - fh.setLevel(logging.DEBUG) - fh.setFormatter(logging.Formatter('%(asctime)s - [%(levelname)s] %(message)s')) - logger.addHandler(fh) - logger.info(f"{platform.platform(True, True)} - Python {platform.python_version()}, deemix {deemixVersion}") - - # Only keep last 5 logfiles (to preserve disk space) - logslist = listdir(logspath) - logslist.sort() - if len(logslist)>5: - for i in range(len(logslist)-5): - (logspath / logslist[i]).unlink() - - # Saves the settings - def saveSettings(self, newSettings=None, dz=None): - if newSettings: - if dz and newSettings.get('tagsLanguage') != self.settings.get('tagsLanguage'): dz.set_accept_language(newSettings.get('tagsLanguage')) - if newSettings.get('downloadLocation') != self.settings.get('downloadLocation') and not checkFolder(newSettings.get('downloadLocation')): - newSettings['downloadLocation'] = DEFAULT_SETTINGS['downloadLocation'] - makedirs(newSettings['downloadLocation'], exist_ok=True) - self.settings = newSettings - with open(self.configFolder / 'config.json', 'w') as configFile: - json.dump(self.settings, configFile, indent=2) - - # Checks if the default settings have changed - def settingsCheck(self): - changes = 0 - for set in DEFAULT_SETTINGS: - if not set in self.settings or type(self.settings[set]) != type(DEFAULT_SETTINGS[set]): - self.settings[set] = DEFAULT_SETTINGS[set] - changes += 1 - for set in DEFAULT_SETTINGS['tags']: - if not set in self.settings['tags'] or type(self.settings['tags'][set]) != type(DEFAULT_SETTINGS['tags'][set]): - self.settings['tags'][set] = DEFAULT_SETTINGS['tags'][set] - changes += 1 - if self.settings['downloadLocation'] == "": - self.settings['downloadLocation'] = DEFAULT_SETTINGS['downloadLocation'] - changes += 1 - for template in ['tracknameTemplate', 'albumTracknameTemplate', 'playlistTracknameTemplate', 'playlistNameTemplate', 'artistNameTemplate', 'albumNameTemplate', 'playlistFilenameTemplate', 'coverImageTemplate', 'artistImageTemplate', 'paddingSize']: - if self.settings[template] == "": - self.settings[template] = DEFAULT_SETTINGS[template] - changes += 1 - return changes diff --git a/deemix/app/spotifyhelper.py b/deemix/app/spotifyhelper.py deleted file mode 100644 index d60e819..0000000 --- a/deemix/app/spotifyhelper.py +++ /dev/null @@ -1,346 +0,0 @@ -import json -from pathlib import Path - -import spotipy -SpotifyClientCredentials = spotipy.oauth2.SpotifyClientCredentials -from deemix.utils.localpaths import getConfigFolder -from deemix.app.queueitem import QIConvertable - -emptyPlaylist = { - 'collaborative': False, - 'description': "", - 'external_urls': {'spotify': None}, - 'followers': {'total': 0, 'href': None}, - 'id': None, - 'images': [], - 'name': "Something went wrong", - 'owner': { - 'display_name': "Error", - 'id': None - }, - 'public': True, - 'tracks' : [], - 'type': 'playlist', - 'uri': None -} - -class SpotifyHelper: - def __init__(self, configFolder=None): - self.credentials = {} - self.spotifyEnabled = False - self.sp = None - self.configFolder = configFolder - - # Make sure config folder exists - if not self.configFolder: - self.configFolder = getConfigFolder() - self.configFolder = Path(self.configFolder) - if not self.configFolder.is_dir(): - self.configFolder.mkdir() - - # Make sure authCredentials exsits - if not (self.configFolder / 'authCredentials.json').is_file(): - with open(self.configFolder / 'authCredentials.json', 'w') as f: - json.dump({'clientId': "", 'clientSecret': ""}, f, indent=2) - - # Load spotify id and secret and check if they are usable - with open(self.configFolder / 'authCredentials.json', 'r') as credentialsFile: - self.credentials = json.load(credentialsFile) - self.checkCredentials() - self.checkValidCache() - - def checkValidCache(self): - if (self.configFolder / 'spotifyCache.json').is_file(): - with open(self.configFolder / 'spotifyCache.json', 'r') as spotifyCache: - try: - cache = json.load(spotifyCache) - except Exception as e: - print(str(e)) - (self.configFolder / 'spotifyCache.json').unlink() - return - # Remove old versions of cache - if len(cache['tracks'].values()) and isinstance(list(cache['tracks'].values())[0], int) or \ - len(cache['albums'].values()) and isinstance(list(cache['albums'].values())[0], int): - (self.configFolder / 'spotifyCache.json').unlink() - - def checkCredentials(self): - if self.credentials['clientId'] == "" or self.credentials['clientSecret'] == "": - spotifyEnabled = False - else: - try: - client_credentials_manager = SpotifyClientCredentials(client_id=self.credentials['clientId'], - client_secret=self.credentials['clientSecret']) - self.sp = spotipy.Spotify(client_credentials_manager=client_credentials_manager) - self.sp.user_playlists('spotify') - self.spotifyEnabled = True - except Exception as e: - self.spotifyEnabled = False - return self.spotifyEnabled - - def getCredentials(self): - return self.credentials - - def setCredentials(self, spotifyCredentials): - # Remove extra spaces, just to be sure - spotifyCredentials['clientId'] = spotifyCredentials['clientId'].strip() - spotifyCredentials['clientSecret'] = spotifyCredentials['clientSecret'].strip() - - # Save them to disk - with open(self.configFolder / 'authCredentials.json', 'w') as f: - json.dump(spotifyCredentials, f, indent=2) - - # Check if they are usable - self.credentials = spotifyCredentials - self.checkCredentials() - - # Converts spotify API playlist structure to deezer's playlist structure - def _convert_playlist_structure(self, spotify_obj): - if len(spotify_obj['images']): - url = spotify_obj['images'][0]['url'] - else: - url = False - deezer_obj = { - 'checksum': spotify_obj['snapshot_id'], - 'collaborative': spotify_obj['collaborative'], - 'creation_date': "XXXX-00-00", - 'creator': { - 'id': spotify_obj['owner']['id'], - 'name': spotify_obj['owner']['display_name'], - 'tracklist': spotify_obj['owner']['href'], - 'type': "user" - }, - 'description': spotify_obj['description'], - 'duration': 0, - 'fans': spotify_obj['followers']['total'] if 'followers' in spotify_obj else 0, - 'id': spotify_obj['id'], - 'is_loved_track': False, - 'link': spotify_obj['external_urls']['spotify'], - 'nb_tracks': spotify_obj['tracks']['total'], - 'picture': url, - 'picture_small': url, - 'picture_medium': url, - 'picture_big': url, - 'picture_xl': url, - 'public': spotify_obj['public'], - 'share': spotify_obj['external_urls']['spotify'], - 'title': spotify_obj['name'], - 'tracklist': spotify_obj['tracks']['href'], - 'type': "playlist" - } - if not url: - deezer_obj['picture_small'] = "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/56x56-000000-80-0-0.jpg" - deezer_obj['picture_medium'] = "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/250x250-000000-80-0-0.jpg" - deezer_obj['picture_big'] = "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/500x500-000000-80-0-0.jpg" - deezer_obj['picture_xl'] = "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/1000x1000-000000-80-0-0.jpg" - return deezer_obj - - # Returns deezer song_id from spotify track_id or track dict - def get_trackid_spotify(self, dz, track_id, fallbackSearch, spotifyTrack=None): - if not self.spotifyEnabled: - raise spotifyFeaturesNotEnabled - singleTrack = False - if not spotifyTrack: - if (self.configFolder / 'spotifyCache.json').is_file(): - with open(self.configFolder / 'spotifyCache.json', 'r') as spotifyCache: - cache = json.load(spotifyCache) - else: - cache = {'tracks': {}, 'albums': {}} - if str(track_id) in cache['tracks']: - dz_track = None - if cache['tracks'][str(track_id)]['isrc']: - dz_track = dz.api.get_track_by_ISRC(cache['tracks'][str(track_id)]['isrc']) - dz_id = dz_track['id'] if 'id' in dz_track and 'title' in dz_track else "0" - cache['tracks'][str(track_id)]['id'] = dz_id - return (cache['tracks'][str(track_id)]['id'], dz_track, cache['tracks'][str(track_id)]['isrc']) - singleTrack = True - spotify_track = self.sp.track(track_id) - else: - spotify_track = spotifyTrack - dz_id = "0" - dz_track = None - isrc = None - if 'external_ids' in spotify_track and 'isrc' in spotify_track['external_ids']: - try: - dz_track = dz.api.get_track_by_ISRC(spotify_track['external_ids']['isrc']) - dz_id = dz_track['id'] if 'id' in dz_track and 'title' in dz_track else "0" - isrc = spotify_track['external_ids']['isrc'] - except: - dz_id = dz.api.get_track_id_from_metadata( - artist=spotify_track['artists'][0]['name'], - track=spotify_track['name'], - album=spotify_track['album']['name'] - ) if fallbackSearch else "0" - elif fallbackSearch: - dz_id = dz.api.get_track_id_from_metadata( - artist=spotify_track['artists'][0]['name'], - track=spotify_track['name'], - album=spotify_track['album']['name'] - ) - if singleTrack: - cache['tracks'][str(track_id)] = {'id': dz_id, 'isrc': isrc} - with open(self.configFolder / 'spotifyCache.json', 'w') as spotifyCache: - json.dump(cache, spotifyCache) - return (dz_id, dz_track, isrc) - - # Returns deezer album_id from spotify album_id - def get_albumid_spotify(self, dz, album_id): - if not self.spotifyEnabled: - raise spotifyFeaturesNotEnabled - if (self.configFolder / 'spotifyCache.json').is_file(): - with open(self.configFolder / 'spotifyCache.json', 'r') as spotifyCache: - cache = json.load(spotifyCache) - else: - cache = {'tracks': {}, 'albums': {}} - if str(album_id) in cache['albums']: - return cache['albums'][str(album_id)]['id'] - spotify_album = self.sp.album(album_id) - dz_album = "0" - upc = None - if 'external_ids' in spotify_album and 'upc' in spotify_album['external_ids']: - try: - dz_album = dz.api.get_album_by_UPC(spotify_album['external_ids']['upc']) - dz_album = dz_album['id'] if 'id' in dz_album else "0" - upc = spotify_album['external_ids']['upc'] - except: - try: - dz_album = dz.api.get_album_by_UPC(int(spotify_album['external_ids']['upc'])) - dz_album = dz_album['id'] if 'id' in dz_album else "0" - except: - dz_album = "0" - cache['albums'][str(album_id)] = {'id': dz_album, 'upc': upc} - with open(self.configFolder / 'spotifyCache.json', 'w') as spotifyCache: - json.dump(cache, spotifyCache) - return dz_album - - - def generate_playlist_queueitem(self, dz, playlist_id, bitrate, settings): - if not self.spotifyEnabled: - raise spotifyFeaturesNotEnabled - spotify_playlist = self.sp.playlist(playlist_id) - - if len(spotify_playlist['images']): - cover = spotify_playlist['images'][0]['url'] - else: - cover = "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/75x75-000000-80-0-0.jpg" - - playlistAPI = self._convert_playlist_structure(spotify_playlist) - playlistAPI['various_artist'] = dz.api.get_artist(5080) - - extra = {} - extra['unconverted'] = [] - - tracklistTmp = spotify_playlist['tracks']['items'] - while spotify_playlist['tracks']['next']: - spotify_playlist['tracks'] = self.sp.next(spotify_playlist['tracks']) - tracklistTmp += spotify_playlist['tracks']['items'] - for item in tracklistTmp: - if item['track']: - if item['track']['explicit']: - playlistAPI['explicit'] = True - extra['unconverted'].append(item['track']) - - totalSize = len(extra['unconverted']) - if not 'explicit' in playlistAPI: - playlistAPI['explicit'] = False - extra['playlistAPI'] = playlistAPI - return QIConvertable( - playlist_id, - bitrate, - spotify_playlist['name'], - spotify_playlist['owner']['display_name'], - cover, - playlistAPI['explicit'], - totalSize, - 'spotify_playlist', - settings, - extra, - ) - - def convert_spotify_playlist(self, dz, queueItem, interface=None): - convertPercentage = 0 - lastPercentage = 0 - if (self.configFolder / 'spotifyCache.json').is_file(): - with open(self.configFolder / 'spotifyCache.json', 'r') as spotifyCache: - cache = json.load(spotifyCache) - else: - cache = {'tracks': {}, 'albums': {}} - if interface: - interface.send("startConversion", queueItem.uuid) - collection = [] - for pos, track in enumerate(queueItem.extra['unconverted'], start=1): - if queueItem.cancel: - return - if str(track['id']) in cache['tracks']: - trackID = cache['tracks'][str(track['id'])]['id'] - trackAPI = None - if cache['tracks'][str(track['id'])]['isrc']: - trackAPI = dz.api.get_track_by_ISRC(cache['tracks'][str(track['id'])]['isrc']) - else: - (trackID, trackAPI, isrc) = self.get_trackid_spotify(dz, "0", queueItem.settings['fallbackSearch'], track) - cache['tracks'][str(track['id'])] = { - 'id': trackID, - 'isrc': isrc - } - if str(trackID) == "0": - deezerTrack = { - 'SNG_ID': "0", - 'SNG_TITLE': track['name'], - 'DURATION': 0, - 'MD5_ORIGIN': 0, - 'MEDIA_VERSION': 0, - 'FILESIZE': 0, - 'ALB_TITLE': track['album']['name'], - 'ALB_PICTURE': "", - 'ART_ID': 0, - 'ART_NAME': track['artists'][0]['name'] - } - else: - deezerTrack = dz.gw.get_track_with_fallback(trackID) - deezerTrack['_EXTRA_PLAYLIST'] = queueItem.extra['playlistAPI'] - if trackAPI: - deezerTrack['_EXTRA_TRACK'] = trackAPI - deezerTrack['POSITION'] = pos - deezerTrack['SIZE'] = queueItem.size - deezerTrack['FILENAME_TEMPLATE'] = queueItem.settings['playlistTracknameTemplate'] - collection.append(deezerTrack) - - convertPercentage = (pos / queueItem.size) * 100 - if round(convertPercentage) != lastPercentage and round(convertPercentage) % 5 == 0: - lastPercentage = round(convertPercentage) - if interface: - interface.send("updateQueue", {'uuid': queueItem.uuid, 'conversion': lastPercentage}) - - queueItem.extra = None - queueItem.collection = collection - - with open(self.configFolder / 'spotifyCache.json', 'w') as spotifyCache: - json.dump(cache, spotifyCache) - - def get_user_playlists(self, user): - if not self.spotifyEnabled: - raise spotifyFeaturesNotEnabled - result = [] - playlists = self.sp.user_playlists(user) - while playlists: - for playlist in playlists['items']: - result.append(self._convert_playlist_structure(playlist)) - if playlists['next']: - playlists = self.sp.next(playlists) - else: - playlists = None - return result - - def get_playlist_tracklist(self, id): - if not self.spotifyEnabled: - raise spotifyFeaturesNotEnabled - playlist = self.sp.playlist(id) - tracklist = playlist['tracks']['items'] - while playlist['tracks']['next']: - playlist['tracks'] = self.sp.next(playlist['tracks']) - tracklist += playlist['tracks']['items'] - playlist['tracks'] = tracklist - return playlist - - -class spotifyFeaturesNotEnabled(Exception): - pass diff --git a/deemix/utils/decryption.py b/deemix/decryption.py similarity index 69% rename from deemix/utils/decryption.py rename to deemix/decryption.py index 616bbac..0dec77f 100644 --- a/deemix/utils/decryption.py +++ b/deemix/decryption.py @@ -8,24 +8,35 @@ def _md5(data): return h.hexdigest() def generateBlowfishKey(trackId): - SECRET = 'g4el58wc' + '0zvf9na1' + SECRET = 'g4el58wc0zvf9na1' idMd5 = _md5(trackId) bfKey = "" for i in range(16): bfKey += chr(ord(idMd5[i]) ^ ord(idMd5[i + 16]) ^ ord(SECRET[i])) return bfKey -def generateStreamURL(sng_id, md5, media_version, format): +def generateStreamPath(sng_id, md5, media_version, format): urlPart = b'\xa4'.join( [str.encode(md5), str.encode(str(format)), str.encode(str(sng_id)), str.encode(str(media_version))]) md5val = _md5(urlPart) step2 = str.encode(md5val) + b'\xa4' + urlPart + b'\xa4' step2 = step2 + (b'.' * (16 - (len(step2) % 16))) urlPart = binascii.hexlify(AES.new(b'jo6aey6haid2Teih', AES.MODE_ECB).encrypt(step2)) - return "https://e-cdns-proxy-" + md5[0] + ".dzcdn.net/mobile/1/" + urlPart.decode("utf-8") + return urlPart.decode("utf-8") -def reverseStreamURL(url): - urlPart = url[42:] +def reverseStreamPath(urlPart): step2 = AES.new(b'jo6aey6haid2Teih', AES.MODE_ECB).decrypt(binascii.unhexlify(urlPart.encode("utf-8"))) (md5val, md5, format, sng_id, media_version, _) = step2.split(b'\xa4') return (sng_id.decode('utf-8'), md5.decode('utf-8'), media_version.decode('utf-8'), format.decode('utf-8')) + +def generateStreamURL(sng_id, md5, media_version, format): + urlPart = generateStreamPath(sng_id, md5, media_version, format) + return "https://e-cdns-proxy-" + md5[0] + ".dzcdn.net/mobile/1/" + urlPart + +def generateUnencryptedStreamURL(sng_id, md5, media_version, format): + urlPart = generateStreamPath(sng_id, md5, media_version, format) + return "https://e-cdns-proxy-" + md5[0] + ".dzcdn.net/api/1/" + urlPart + +def reverseStreamURL(url): + urlPart = url[url.find("/1/")+3:] + return generateStreamPath(urlPart) diff --git a/deemix/app/downloadjob.py b/deemix/downloader.py similarity index 63% rename from deemix/app/downloadjob.py rename to deemix/downloader.py index d7eb36e..ff8da30 100644 --- a/deemix/app/downloadjob.py +++ b/deemix/downloader.py @@ -12,18 +12,17 @@ import errno from ssl import SSLError from os import makedirs -from tempfile import gettempdir from urllib3.exceptions import SSLError as u3SSLError -from deemix.app.queueitem import QISingle, QICollection +from deemix.types.DownloadObjects import Single, Collection from deemix.types.Track import Track, AlbumDoesntExists from deemix.utils import changeCase from deemix.utils.pathtemplates import generateFilename, generateFilepath, settingsRegexAlbum, settingsRegexArtist, settingsRegexPlaylistFile from deezer import TrackFormats from deemix import USER_AGENT_HEADER -from deemix.utils.taggers import tagID3, tagFLAC -from deemix.utils.decryption import generateStreamURL, generateBlowfishKey -from deemix.app.settings import OverwriteOption, FeaturesOption +from deemix.taggers import tagID3, tagFLAC +from deemix.decryption import generateStreamURL, generateBlowfishKey +from deemix.settings import OverwriteOption from Cryptodome.Cipher import Blowfish from mutagen.flac import FLACNoHeaderError, error as FLACError @@ -32,6 +31,8 @@ import logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger('deemix') +from tempfile import gettempdir + TEMPDIR = Path(gettempdir()) / 'deemix-imgs' if not TEMPDIR.is_dir(): makedirs(TEMPDIR) @@ -75,17 +76,15 @@ def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): if pictureSize > 1200: logger.warn("Couldn't download "+str(pictureSize)+"x"+str(pictureSize)+" image, falling back to 1200x1200") sleep(1) - return downloadImage(urlBase+pictureUrl.replace(str(pictureSize)+"x"+str(pictureSize), '1200x1200'), path, overwrite) + return downloadImage(urlBase+pictureUrl.replace(str(pictureSize)+"x"+str(pictureSize), '1200x1200'), path, overwrite) logger.error("Image not found: "+url) except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError, u3SSLError) as e: logger.error("Couldn't download Image, retrying in 5 seconds...: "+url+"\n") sleep(5) return downloadImage(url, path, overwrite) except OSError as e: - if e.errno == errno.ENOSPC: - raise DownloadFailed("noSpaceLeft") - else: - logger.exception(f"Error while downloading an image, you should report this to the developers: {str(e)}") + if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") + else: logger.exception(f"Error while downloading an image, you should report this to the developers: {str(e)}") except Exception as e: logger.exception(f"Error while downloading an image, you should report this to the developers: {str(e)}") if path.is_file(): path.unlink() @@ -93,14 +92,73 @@ def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): else: return path +def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectUUID=None, interface=None): + if track.localTrack: return TrackFormats.LOCAL + + falledBack = False + + formats_non_360 = { + TrackFormats.FLAC: "FLAC", + TrackFormats.MP3_320: "MP3_320", + TrackFormats.MP3_128: "MP3_128", + } + formats_360 = { + TrackFormats.MP4_RA3: "MP4_RA3", + TrackFormats.MP4_RA2: "MP4_RA2", + TrackFormats.MP4_RA1: "MP4_RA1", + } + + is360format = int(preferredBitrate) in formats_360 + + if not shouldFallback: + formats = formats_360 + formats.update(formats_non_360) + elif is360format: + formats = formats_360 + else: + formats = formats_non_360 + + for formatNumber, formatName in formats.items(): + if formatNumber <= int(preferredBitrate): + if f"FILESIZE_{formatName}" in track.filesizes: + if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber + if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]: + request = requests.head( + generateStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), + headers={'User-Agent': USER_AGENT_HEADER}, + timeout=30 + ) + try: + request.raise_for_status() + return formatNumber + except requests.exceptions.HTTPError: # if the format is not available, Deezer returns a 403 error + pass + if not shouldFallback: + raise PreferredBitrateNotFound + else: + if not falledBack: + falledBack = True + logger.info(f"[{track.mainArtist.name} - {track.title}] Fallback to lower bitrate") + if interface and downloadObjectUUID: + interface.send('queueUpdate', { + 'uuid': downloadObjectUUID, + 'bitrateFallback': True, + 'data': { + 'id': track.id, + 'title': track.title, + 'artist': track.mainArtist.name + }, + }) + if is360format: raise TrackNot360 + return TrackFormats.DEFAULT -class DownloadJob: - def __init__(self, dz, queueItem, interface=None): +class Downloader: + def __init__(self, dz, downloadObject, settings, interface=None): self.dz = dz + self.downloadObject = downloadObject + self.settings = settings + self.bitrate = downloadObject.bitrate self.interface = interface - self.queueItem = queueItem - self.settings = queueItem.settings - self.bitrate = queueItem.bitrate self.downloadPercentage = 0 self.lastPercentage = 0 self.extrasPath = None @@ -108,191 +166,51 @@ class DownloadJob: self.playlistURLs = [] def start(self): - if not self.queueItem.cancel: - if isinstance(self.queueItem, QISingle): - result = self.downloadWrapper(self.queueItem.single) - if result: self.singleAfterDownload(result) - elif isinstance(self.queueItem, QICollection): - tracks = [None] * len(self.queueItem.collection) - with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor: - for pos, track in enumerate(self.queueItem.collection, start=0): - tracks[pos] = executor.submit(self.downloadWrapper, track) - self.collectionAfterDownload(tracks) + if isinstance(self.downloadObject, Single): + result = self.downloadWrapper(self.downloadObject.single['trackAPI_gw'], self.downloadObject.single['trackAPI'], self.downloadObject.single['albumAPI']) + if result: self.singleAfterDownload(result) + elif isinstance(self.downloadObject, Collection): + tracks = [None] * len(self.downloadObject.collection['tracks_gw']) + with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor: + for pos, track in enumerate(self.downloadObject.collection['tracks_gw'], start=0): + tracks[pos] = executor.submit(self.downloadWrapper, track, None, self.downloadObject.collection['albumAPI'], self.downloadObject.collection['playlistAPI']) + self.collectionAfterDownload(tracks) if self.interface: - if self.queueItem.cancel: - self.interface.send('currentItemCancelled', self.queueItem.uuid) - self.interface.send("removedFromQueue", self.queueItem.uuid) - else: - self.interface.send("finishDownload", self.queueItem.uuid) + self.interface.send("finishDownload", self.downloadObject.uuid) return self.extrasPath - def singleAfterDownload(self, result): - if not self.extrasPath: self.extrasPath = Path(self.settings['downloadLocation']) - - # Save Album Cover - if self.settings['saveArtwork'] and 'albumPath' in result: - for image in result['albumURLs']: - downloadImage(image['url'], result['albumPath'] / f"{result['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) - - # Save Artist Artwork - if self.settings['saveArtworkArtist'] and 'artistPath' in result: - for image in result['artistURLs']: - downloadImage(image['url'], result['artistPath'] / f"{result['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) - - # Create searched logfile - if self.settings['logSearched'] and 'searched' in result: - with open(self.extrasPath / 'searched.txt', 'wb+') as f: - orig = f.read().decode('utf-8') - if not result['searched'] in orig: - if orig != "": orig += "\r\n" - orig += result['searched'] + "\r\n" - f.write(orig.encode('utf-8')) - # Execute command after download - if self.settings['executeCommand'] != "": - execute(self.settings['executeCommand'].replace("%folder%", quote(str(self.extrasPath))).replace("%filename%", quote(result['filename'])), shell=True) - - def collectionAfterDownload(self, tracks): - if not self.extrasPath: self.extrasPath = Path(self.settings['downloadLocation']) - playlist = [None] * len(tracks) - errors = "" - searched = "" - - for i in range(len(tracks)): - result = tracks[i].result() - if not result: return None # Check if item is cancelled - - # Log errors to file - if result.get('error'): - if not result['error'].get('data'): result['error']['data'] = {'id': "0", 'title': 'Unknown', 'artist': 'Unknown'} - errors += f"{result['error']['data']['id']} | {result['error']['data']['artist']} - {result['error']['data']['title']} | {result['error']['message']}\r\n" - - # Log searched to file - if 'searched' in result: searched += result['searched'] + "\r\n" - - # Save Album Cover - if self.settings['saveArtwork'] and 'albumPath' in result: - for image in result['albumURLs']: - downloadImage(image['url'], result['albumPath'] / f"{result['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) - - # Save Artist Artwork - if self.settings['saveArtworkArtist'] and 'artistPath' in result: - for image in result['artistURLs']: - downloadImage(image['url'], result['artistPath'] / f"{result['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) - - # Save filename for playlist file - playlist[i] = result.get('filename', "") - - # Create errors logfile - if self.settings['logErrors'] and errors != "": - with open(self.extrasPath / 'errors.txt', 'wb') as f: - f.write(errors.encode('utf-8')) - - # Create searched logfile - if self.settings['logSearched'] and searched != "": - with open(self.extrasPath / 'searched.txt', 'wb') as f: - f.write(searched.encode('utf-8')) - - # Save Playlist Artwork - if self.settings['saveArtwork'] and self.playlistCoverName and not self.settings['tags']['savePlaylistAsCompilation']: - for image in self.playlistURLs: - downloadImage(image['url'], self.extrasPath / f"{self.playlistCoverName}.{image['ext']}", self.settings['overwriteFile']) - - # Create M3U8 File - if self.settings['createM3U8File']: - filename = settingsRegexPlaylistFile(self.settings['playlistFilenameTemplate'], self.queueItem, self.settings) or "playlist" - with open(self.extrasPath / f'{filename}.m3u8', 'wb') as f: - for line in playlist: - f.write((line + "\n").encode('utf-8')) - - # Execute command after download - if self.settings['executeCommand'] != "": - execute(self.settings['executeCommand'].replace("%folder%", quote(str(self.extrasPath))), shell=True) - - def download(self, trackAPI_gw, track=None): + def download(self, trackAPI_gw, trackAPI=None, albumAPI=None, playlistAPI=None, track=None): result = {} - if self.queueItem.cancel: raise DownloadCancelled if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer") # Create Track object + print(track) if not track: logger.info(f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}] Getting the tags") try: track = Track().parseData( dz=self.dz, trackAPI_gw=trackAPI_gw, - trackAPI=trackAPI_gw['_EXTRA_TRACK'] if '_EXTRA_TRACK' in trackAPI_gw else None, - albumAPI=trackAPI_gw['_EXTRA_ALBUM'] if '_EXTRA_ALBUM' in trackAPI_gw else None, - playlistAPI = trackAPI_gw['_EXTRA_PLAYLIST'] if '_EXTRA_PLAYLIST' in trackAPI_gw else None + trackAPI=trackAPI, + albumAPI=albumAPI, + playlistAPI=playlistAPI ) except AlbumDoesntExists: raise DownloadError('albumDoesntExists') - if self.queueItem.cancel: raise DownloadCancelled # Check if track not yet encoded - if track.MD5 == '': - if track.fallbackId != "0": - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not yet encoded, using fallback id") - newTrack = self.dz.gw.get_track_with_fallback(track.fallbackId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - return self.download(trackAPI_gw, track) - elif not track.searched and self.settings['fallbackSearch']: - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not yet encoded, searching for alternative") - searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) - if searchedId != "0": - newTrack = self.dz.gw.get_track_with_fallback(searchedId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - track.searched = True - if self.interface: - self.interface.send('queueUpdate', { - 'uuid': self.queueItem.uuid, - 'searchFallback': True, - 'data': { - 'id': track.id, - 'title': track.title, - 'artist': track.mainArtist.name - }, - }) - return self.download(trackAPI_gw, track) - else: - raise DownloadFailed("notEncodedNoAlternative") - else: - raise DownloadFailed("notEncoded") + if track.MD5 == '': raise DownloadFailed("notEncoded", track) # Choose the target bitrate try: - selectedFormat = self.getPreferredBitrate(track) + selectedFormat = getPreferredBitrate( + track, + self.bitrate, + self.settings['fallbackBitrate'], + self.downloadObject.uuid, self.interface + ) except PreferredBitrateNotFound: - if track.fallbackId != "0": - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not found at desired bitrate, using fallback id") - newTrack = self.dz.gw.get_track_with_fallback(track.fallbackId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - return self.download(trackAPI_gw, track) - elif not track.searched and self.settings['fallbackSearch']: - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not found at desired bitrate, searching for alternative") - searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) - if searchedId != "0": - newTrack = self.dz.gw.get_track_with_fallback(searchedId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - track.searched = True - if self.interface: - self.interface.send('queueUpdate', { - 'uuid': self.queueItem.uuid, - 'searchFallback': True, - 'data': { - 'id': track.id, - 'title': track.title, - 'artist': track.mainArtist.name - }, - }) - return self.download(trackAPI_gw, track) - else: - raise DownloadFailed("wrongBitrateNoAlternative") - else: - raise DownloadFailed("wrongBitrate") + raise DownloadFailed("wrongBitrate", track) except TrackNot360: raise DownloadFailed("no360RA") track.selectedFormat = selectedFormat @@ -302,92 +220,23 @@ class DownloadJob: embeddedImageFormat = f'jpg-{self.settings["jpegImageQuality"]}' if self.settings['embeddedArtworkPNG']: imageFormat = 'png' - if self.settings['tags']['savePlaylistAsCompilation'] and track.playlist: - track.trackNumber = track.position - track.discNumber = "1" - track.album.makePlaylistCompilation(track.playlist) - track.album.embeddedCoverURL = track.playlist.pic.generatePictureURL(self.settings['embeddedArtworkSize'], embeddedImageFormat) - - ext = track.album.embeddedCoverURL[-4:] - if ext[0] != ".": ext = ".jpg" # Check for Spotify images - - track.album.embeddedCoverPath = TEMPDIR / f"pl{trackAPI_gw['_EXTRA_PLAYLIST']['id']}_{self.settings['embeddedArtworkSize']}{ext}" - else: - if track.album.date: track.date = track.album.date - track.album.embeddedCoverURL = track.album.pic.generatePictureURL(self.settings['embeddedArtworkSize'], embeddedImageFormat) - - ext = track.album.embeddedCoverURL[-4:] - track.album.embeddedCoverPath = TEMPDIR / f"alb{track.album.id}_{self.settings['embeddedArtworkSize']}{ext}" - - track.dateString = track.date.format(self.settings['dateFormat']) - track.album.dateString = track.album.date.format(self.settings['dateFormat']) - if track.playlist: track.playlist.dateString = track.playlist.date.format(self.settings['dateFormat']) - - # Check various artist option - if self.settings['albumVariousArtists'] and track.album.variousArtists: - artist = track.album.variousArtists - isMainArtist = artist.role == "Main" - - if artist.name not in track.album.artists: - track.album.artists.insert(0, artist.name) - - if isMainArtist or artist.name not in track.album.artist['Main'] and not isMainArtist: - if not artist.role in track.album.artist: - track.album.artist[artist.role] = [] - track.album.artist[artist.role].insert(0, artist.name) - track.album.mainArtist.save = not track.album.mainArtist.isVariousArtists() or self.settings['albumVariousArtists'] and track.album.mainArtist.isVariousArtists() - - # Check removeDuplicateArtists - if self.settings['removeDuplicateArtists']: track.removeDuplicateArtists() - - # Check if user wants the feat in the title - if str(self.settings['featuredToTitle']) == FeaturesOption.REMOVE_TITLE: - track.title = track.getCleanTitle() - elif str(self.settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: - track.title = track.getFeatTitle() - elif str(self.settings['featuredToTitle']) == FeaturesOption.REMOVE_TITLE_ALBUM: - track.title = track.getCleanTitle() - track.album.title = track.album.getCleanTitle() - - # Remove (Album Version) from tracks that have that - if self.settings['removeAlbumVersion']: - if "Album Version" in track.title: - track.title = re.sub(r' ?\(Album Version\)', "", track.title).strip() - - # Change Title and Artists casing if needed - if self.settings['titleCasing'] != "nothing": - track.title = changeCase(track.title, self.settings['titleCasing']) - if self.settings['artistCasing'] != "nothing": - track.mainArtist.name = changeCase(track.mainArtist.name, self.settings['artistCasing']) - for i, artist in enumerate(track.artists): - track.artists[i] = changeCase(artist, self.settings['artistCasing']) - for type in track.artist: - for i, artist in enumerate(track.artist[type]): - track.artist[type][i] = changeCase(artist, self.settings['artistCasing']) - track.generateMainFeatStrings() - - # Generate artist tag - if self.settings['tags']['multiArtistSeparator'] == "default": - if str(self.settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: - track.artistsString = ", ".join(track.artist['Main']) - else: - track.artistsString = ", ".join(track.artists) - elif self.settings['tags']['multiArtistSeparator'] == "andFeat": - track.artistsString = track.mainArtistsString - if track.featArtistsString and str(self.settings['featuredToTitle']) != FeaturesOption.MOVE_TITLE: - track.artistsString += " " + track.featArtistsString - else: - separator = self.settings['tags']['multiArtistSeparator'] - if str(self.settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: - track.artistsString = separator.join(track.artist['Main']) - else: - track.artistsString = separator.join(track.artists) + track.applySettings(self.settings, TEMPDIR, embeddedImageFormat) # Generate filename and filepath from metadata - filename = generateFilename(track, self.settings, trackAPI_gw['FILENAME_TEMPLATE']) + filename = generateFilename(track, self.settings, "%artist% - %title%") (filepath, artistPath, coverPath, extrasPath) = generateFilepath(track, self.settings) - - if self.queueItem.cancel: raise DownloadCancelled + # Remove subfolders from filename and add it to filepath + if pathSep in filename: + tempPath = filename[:filename.rfind(pathSep)] + filepath = filepath / tempPath + filename = filename[filename.rfind(pathSep) + len(pathSep):] + # Make sure the filepath exists + makedirs(filepath, exist_ok=True) + writepath = filepath / f"{filename}{extensions[track.selectedFormat]}" + # Save extrasPath + if extrasPath: + if not self.extrasPath: self.extrasPath = extrasPath + result['filename'] = str(writepath)[len(str(extrasPath))+ len(pathSep):] # Download and cache coverart logger.info(f"[{track.mainArtist.name} - {track.title}] Getting the album cover") @@ -423,7 +272,7 @@ class DownloadJob: result['artistPath'] = artistPath result['artistFilename'] = f"{settingsRegexArtist(self.settings['artistImageTemplate'], track.album.mainArtist, self.settings, rootArtist=track.album.rootArtist)}" - # Save playlist cover + # Save playlist art if track.playlist: if not len(self.playlistURLs): for format in self.settings['localArtworkFormat'].split(","): @@ -438,22 +287,13 @@ class DownloadJob: track.playlist.dateString = track.playlist.date.format(self.settings['dateFormat']) self.playlistCoverName = f"{settingsRegexAlbum(self.settings['coverImageTemplate'], track.playlist, self.settings, track.playlist)}" - # Remove subfolders from filename and add it to filepath - if pathSep in filename: - tempPath = filename[:filename.rfind(pathSep)] - filepath = filepath / tempPath - filename = filename[filename.rfind(pathSep) + len(pathSep):] - - # Make sure the filepath exists - makedirs(filepath, exist_ok=True) - writepath = filepath / f"{filename}{extensions[track.selectedFormat]}" - # Save lyrics in lrc file if self.settings['syncedLyrics'] and track.lyrics.sync: if not (filepath / f"{filename}.lrc").is_file() or self.settings['overwriteFile'] in [OverwriteOption.OVERWRITE, OverwriteOption.ONLY_TAGS]: with open(filepath / f"{filename}.lrc", 'wb') as f: f.write(track.lyrics.sync.encode('utf-8')) + # Check for overwrite settings trackAlreadyDownloaded = writepath.is_file() # Don't overwrite and don't mind extension @@ -463,7 +303,6 @@ class DownloadJob: for ext in exts: trackAlreadyDownloaded = Path(baseFilename+ext).is_file() if trackAlreadyDownloaded: break - # Don't overwrite and keep both files if trackAlreadyDownloaded and self.settings['overwriteFile'] == OverwriteOption.KEEP_BOTH: baseFilename = str(filepath / filename) @@ -475,10 +314,6 @@ class DownloadJob: trackAlreadyDownloaded = False writepath = Path(currentFilename) - if extrasPath: - if not self.extrasPath: self.extrasPath = extrasPath - result['filename'] = str(writepath)[len(str(extrasPath))+ len(pathSep):] - if not trackAlreadyDownloaded or self.settings['overwriteFile'] == OverwriteOption.OVERWRITE: logger.info(f"[{track.mainArtist.name} - {track.title}] Downloading the track") track.downloadUrl = generateStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat) @@ -508,7 +343,7 @@ class DownloadJob: track.searched = True if self.interface: self.interface.send('queueUpdate', { - 'uuid': self.queueItem.uuid, + 'uuid': self.downloadObject.uuid, 'searchFallback': True, 'data': { 'id': track.id, @@ -544,7 +379,7 @@ class DownloadJob: except Exception as e: raise e - if not trackDownloaded: return self.download(trackAPI_gw, track) + if not trackDownloaded: return self.download(trackAPI_gw, track=track) else: logger.info(f"[{track.mainArtist.name} - {track.title}] Skipping track as it's already downloaded") self.completeTrackPercentage() @@ -563,80 +398,18 @@ class DownloadJob: self.removeTrackPercentage() track.filesizes['FILESIZE_FLAC'] = "0" track.filesizes['FILESIZE_FLAC_TESTED'] = True - return self.download(trackAPI_gw, track) + return self.download(trackAPI_gw, track=track) if track.searched: result['searched'] = f"{track.mainArtist.name} - {track.title}" logger.info(f"[{track.mainArtist.name} - {track.title}] Track download completed\n{str(writepath)}") - self.queueItem.downloaded += 1 - self.queueItem.files.append(str(writepath)) - self.queueItem.extrasPath = str(self.extrasPath) + self.downloadObject.downloaded += 1 + self.downloadObject.files.append(str(writepath)) + self.downloadObject.extrasPath = str(self.extrasPath) if self.interface: - self.interface.send("updateQueue", {'uuid': self.queueItem.uuid, 'downloaded': True, 'downloadPath': str(writepath), 'extrasPath': str(self.extrasPath)}) + self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'downloaded': True, 'downloadPath': str(writepath), 'extrasPath': str(self.extrasPath)}) return result - def getPreferredBitrate(self, track): - if track.localTrack: return TrackFormats.LOCAL - - shouldFallback = self.settings['fallbackBitrate'] - falledBack = False - - formats_non_360 = { - TrackFormats.FLAC: "FLAC", - TrackFormats.MP3_320: "MP3_320", - TrackFormats.MP3_128: "MP3_128", - } - formats_360 = { - TrackFormats.MP4_RA3: "MP4_RA3", - TrackFormats.MP4_RA2: "MP4_RA2", - TrackFormats.MP4_RA1: "MP4_RA1", - } - - is360format = int(self.bitrate) in formats_360 - - if not shouldFallback: - formats = formats_360 - formats.update(formats_non_360) - elif is360format: - formats = formats_360 - else: - formats = formats_non_360 - - for formatNumber, formatName in formats.items(): - if formatNumber <= int(self.bitrate): - if f"FILESIZE_{formatName}" in track.filesizes: - if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber - if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]: - request = requests.head( - generateStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), - headers={'User-Agent': USER_AGENT_HEADER}, - timeout=30 - ) - try: - request.raise_for_status() - return formatNumber - except requests.exceptions.HTTPError: # if the format is not available, Deezer returns a 403 error - pass - if not shouldFallback: - raise PreferredBitrateNotFound - else: - if not falledBack: - falledBack = True - logger.info(f"[{track.mainArtist.name} - {track.title}] Fallback to lower bitrate") - if self.interface: - self.interface.send('queueUpdate', { - 'uuid': self.queueItem.uuid, - 'bitrateFallback': True, - 'data': { - 'id': track.id, - 'title': track.title, - 'artist': track.mainArtist.name - }, - }) - if is360format: raise TrackNot360 - return TrackFormats.DEFAULT - def streamTrack(self, stream, track, start=0): - if self.queueItem.cancel: raise DownloadCancelled headers=dict(self.dz.http_headers) if range != 0: headers['Range'] = f'bytes={start}-' @@ -659,7 +432,6 @@ class DownloadJob: logger.info(f'{itemName} downloading {complete} bytes') for chunk in request.iter_content(2048 * 3): - if self.queueItem.cancel: raise DownloadCancelled if len(chunk) >= 2048: chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:] @@ -667,11 +439,11 @@ class DownloadJob: stream.write(chunk) chunkLength += len(chunk) - if isinstance(self.queueItem, QISingle): + if isinstance(self.downloadObject, Single): percentage = (chunkLength / (complete + start)) * 100 self.downloadPercentage = percentage else: - chunkProgres = (len(chunk) / (complete + start)) / self.queueItem.size * 100 + chunkProgres = (len(chunk) / (complete + start)) / self.downloadObject.size * 100 self.downloadPercentage += chunkProgres self.updatePercentage() @@ -685,58 +457,87 @@ class DownloadJob: def updatePercentage(self): if round(self.downloadPercentage) != self.lastPercentage and round(self.downloadPercentage) % 2 == 0: self.lastPercentage = round(self.downloadPercentage) - self.queueItem.progress = self.lastPercentage - if self.interface: self.interface.send("updateQueue", {'uuid': self.queueItem.uuid, 'progress': self.lastPercentage}) + self.downloadObject.progress = self.lastPercentage + if self.interface: self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'progress': self.lastPercentage}) def completeTrackPercentage(self): - if isinstance(self.queueItem, QISingle): + if isinstance(self.downloadObject, Single): self.downloadPercentage = 100 else: - self.downloadPercentage += (1 / self.queueItem.size) * 100 + self.downloadPercentage += (1 / self.downloadObject.size) * 100 self.updatePercentage() def removeTrackPercentage(self): - if isinstance(self.queueItem, QISingle): + if isinstance(self.downloadObject, Single): self.downloadPercentage = 0 else: - self.downloadPercentage -= (1 / self.queueItem.size) * 100 + self.downloadPercentage -= (1 / self.downloadObject.size) * 100 self.updatePercentage() - def downloadWrapper(self, trackAPI_gw): - track = { + def downloadWrapper(self, trackAPI_gw, trackAPI=None, albumAPI=None, playlistAPI=None, track=None): + # Temp metadata to generate logs + tempTrack = { 'id': trackAPI_gw['SNG_ID'], 'title': trackAPI_gw['SNG_TITLE'].strip(), 'artist': trackAPI_gw['ART_NAME'] } if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: - track['title'] += f" {trackAPI_gw['VERSION']}".strip() + tempTrack['title'] += f" {trackAPI_gw['VERSION']}".strip() try: - result = self.download(trackAPI_gw) - except DownloadCancelled: - return None + result = self.download(trackAPI_gw, trackAPI, albumAPI, playlistAPI, track) except DownloadFailed as error: - logger.error(f"[{track['artist']} - {track['title']}] {error.message}") + if error.track: + track = error.track + if track.fallbackId != "0": + logger.warn(f"[{track.mainArtist.name} - {track.title}] {error.message} Using fallback id") + newTrack = self.dz.gw.get_track_with_fallback(track.fallbackId) + track.parseEssentialData(newTrack) + track.retriveFilesizes(self.dz) + return self.downloadWrapper(trackAPI_gw, trackAPI, albumAPI, playlistAPI, track) + elif not track.searched and self.settings['fallbackSearch']: + logger.warn(f"[{track.mainArtist.name} - {track.title}] {error.message} Searching for alternative") + searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) + if searchedId != "0": + newTrack = self.dz.gw.get_track_with_fallback(searchedId) + track.parseEssentialData(newTrack) + track.retriveFilesizes(self.dz) + track.searched = True + if self.interface: + self.interface.send('queueUpdate', { + 'uuid': self.queueItem.uuid, + 'searchFallback': True, + 'data': { + 'id': track.id, + 'title': track.title, + 'artist': track.mainArtist.name + }, + }) + return self.downloadWrapper(trackAPI_gw, trackAPI, albumAPI, playlistAPI, track) + else: + error.errid += "NoAlternative" + error.message = errorMessages[error.errid] + logger.error(f"[{tempTrack['artist']} - {tempTrack['title']}] {error.message}") result = {'error': { 'message': error.message, 'errid': error.errid, - 'data': track + 'data': tempTrack }} except Exception as e: - logger.exception(f"[{track['artist']} - {track['title']}] {str(e)}") + logger.exception(f"[{tempTrack['artist']} - {tempTrack['title']}] {str(e)}") result = {'error': { 'message': str(e), - 'data': track + 'data': tempTrack }} if 'error' in result: self.completeTrackPercentage() - self.queueItem.failed += 1 - self.queueItem.errors.append(result['error']) + self.downloadObject.failed += 1 + self.downloadObject.errors.append(result['error']) if self.interface: error = result['error'] self.interface.send("updateQueue", { - 'uuid': self.queueItem.uuid, + 'uuid': self.downloadObject.uuid, 'failed': True, 'data': error['data'], 'error': error['message'], @@ -744,14 +545,97 @@ class DownloadJob: }) return result + def singleAfterDownload(self, result): + if not self.extrasPath: self.extrasPath = Path(self.settings['downloadLocation']) + + # Save Album Cover + if self.settings['saveArtwork'] and 'albumPath' in result: + for image in result['albumURLs']: + downloadImage(image['url'], result['albumPath'] / f"{result['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) + + # Save Artist Artwork + if self.settings['saveArtworkArtist'] and 'artistPath' in result: + for image in result['artistURLs']: + downloadImage(image['url'], result['artistPath'] / f"{result['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) + + # Create searched logfile + if self.settings['logSearched'] and 'searched' in result: + with open(self.extrasPath / 'searched.txt', 'wb+') as f: + orig = f.read().decode('utf-8') + if not result['searched'] in orig: + if orig != "": orig += "\r\n" + orig += result['searched'] + "\r\n" + f.write(orig.encode('utf-8')) + # Execute command after download + if self.settings['executeCommand'] != "": + execute(self.settings['executeCommand'].replace("%folder%", quote(str(self.extrasPath))).replace("%filename%", quote(result['filename'])), shell=True) + + def collectionAfterDownload(self, tracks): + if not self.extrasPath: self.extrasPath = Path(self.settings['downloadLocation']) + playlist = [None] * len(tracks) + errors = "" + searched = "" + + for i in range(len(tracks)): + result = tracks[i].result() + if not result: return None # Check if item is cancelled + + # Log errors to file + if result.get('error'): + if not result['error'].get('data'): result['error']['data'] = {'id': "0", 'title': 'Unknown', 'artist': 'Unknown'} + errors += f"{result['error']['data']['id']} | {result['error']['data']['artist']} - {result['error']['data']['title']} | {result['error']['message']}\r\n" + + # Log searched to file + if 'searched' in result: searched += result['searched'] + "\r\n" + + # Save Album Cover + if self.settings['saveArtwork'] and 'albumPath' in result: + for image in result['albumURLs']: + downloadImage(image['url'], result['albumPath'] / f"{result['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) + + # Save Artist Artwork + if self.settings['saveArtworkArtist'] and 'artistPath' in result: + for image in result['artistURLs']: + downloadImage(image['url'], result['artistPath'] / f"{result['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) + + # Save filename for playlist file + playlist[i] = result.get('filename', "") + + # Create errors logfile + if self.settings['logErrors'] and errors != "": + with open(self.extrasPath / 'errors.txt', 'wb') as f: + f.write(errors.encode('utf-8')) + + # Create searched logfile + if self.settings['logSearched'] and searched != "": + with open(self.extrasPath / 'searched.txt', 'wb') as f: + f.write(searched.encode('utf-8')) + + # Save Playlist Artwork + if self.settings['saveArtwork'] and self.playlistCoverName and not self.settings['tags']['savePlaylistAsCompilation']: + for image in self.playlistURLs: + downloadImage(image['url'], self.extrasPath / f"{self.playlistCoverName}.{image['ext']}", self.settings['overwriteFile']) + + # Create M3U8 File + if self.settings['createM3U8File']: + filename = settingsRegexPlaylistFile(self.settings['playlistFilenameTemplate'], self.downloadObject, self.settings) or "playlist" + with open(self.extrasPath / f'{filename}.m3u8', 'wb') as f: + for line in playlist: + f.write((line + "\n").encode('utf-8')) + + # Execute command after download + if self.settings['executeCommand'] != "": + execute(self.settings['executeCommand'].replace("%folder%", quote(str(self.extrasPath))), shell=True) + class DownloadError(Exception): """Base class for exceptions in this module.""" pass class DownloadFailed(DownloadError): - def __init__(self, errid): + def __init__(self, errid, track=None): self.errid = errid self.message = errorMessages[self.errid] + self.track = track class DownloadCancelled(DownloadError): pass diff --git a/deemix/itemgen.py b/deemix/itemgen.py new file mode 100644 index 0000000..9629044 --- /dev/null +++ b/deemix/itemgen.py @@ -0,0 +1,246 @@ +from deemix.types.DownloadObjects import Single, Collection + +class GenerationError(Exception): + def __init__(self, link, message, errid=None): + self.link = link + self.message = message + self.errid = errid + + def toDict(self): + return { + 'link': self.link, + 'error': self.message, + 'errid': self.errid + } + +def generateTrackItem(dz, id, bitrate, trackAPI=None, albumAPI=None): + # Check if is an isrc: url + if str(id).startswith("isrc"): + try: + trackAPI = dz.api.get_track(id) + except APIError as e: + e = str(e) + raise GenerationError("https://deezer.com/track/"+str(id), f"Wrong URL: {e}") + if 'id' in trackAPI and 'title' in trackAPI: + id = trackAPI['id'] + else: + raise GenerationError("https://deezer.com/track/"+str(id), "Track ISRC is not available on deezer", "ISRCnotOnDeezer") + + # Get essential track info + try: + trackAPI_gw = dz.gw.get_track_with_fallback(id) + except gwAPIError as e: + e = str(e) + message = "Wrong URL" + if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}" + raise GenerationError("https://deezer.com/track/"+str(id), message) + + title = trackAPI_gw['SNG_TITLE'].strip() + if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: + title += f" {trackAPI_gw['VERSION']}".strip() + explicit = bool(int(trackAPI_gw.get('EXPLICIT_LYRICS', 0))) + + return Single( + 'track', + id, + bitrate, + title, + trackAPI_gw['ART_NAME'], + f"https://e-cdns-images.dzcdn.net/images/cover/{trackAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg", + explicit, + trackAPI_gw, + trackAPI, + albumAPI + ) + +def generateAlbumItem(dz, id, bitrate, rootArtist=None): + # Get essential album info + try: + albumAPI = dz.api.get_album(id) + except APIError as e: + e = str(e) + raise GenerationError("https://deezer.com/album/"+str(id), f"Wrong URL: {e}") + + if str(id).startswith('upc'): id = albumAPI['id'] + + # Get extra info about album + # This saves extra api calls when downloading + albumAPI_gw = dz.gw.get_album(id) + albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK'] + albumAPI['copyright'] = albumAPI_gw['COPYRIGHT'] + albumAPI['root_artist'] = rootArtist + + # If the album is a single download as a track + if albumAPI['nb_tracks'] == 1: + return generateTrackItem(dz, albumAPI['tracks']['data'][0]['id'], bitrate, albumAPI=albumAPI) + + tracksArray = dz.gw.get_album_tracks(id) + + if albumAPI['cover_small'] != None: + cover = albumAPI['cover_small'][:-24] + '/75x75-000000-80-0-0.jpg' + else: + cover = f"https://e-cdns-images.dzcdn.net/images/cover/{albumAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg" + + totalSize = len(tracksArray) + albumAPI['nb_tracks'] = totalSize + collection = [] + for pos, trackAPI in enumerate(tracksArray, start=1): + trackAPI['POSITION'] = pos + trackAPI['SIZE'] = totalSize + collection.append(trackAPI) + + explicit = albumAPI_gw.get('EXPLICIT_ALBUM_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT] + + return Collection( + 'album', + id, + bitrate, + albumAPI['title'], + albumAPI['artist']['name'], + cover, + explicit, + totalSize, + tracks_gw=collection, + albumAPI=albumAPI + ) + +def generatePlaylistItem(dz, id, bitrate, playlistAPI=None, playlistTracksAPI=None): + if not playlistAPI: + # Get essential playlist info + try: + playlistAPI = dz.api.get_playlist(id) + except: + playlistAPI = None + # Fallback to gw api if the playlist is private + if not playlistAPI: + try: + userPlaylist = dz.gw.get_playlist_page(id) + playlistAPI = map_user_playlist(userPlaylist['DATA']) + except gwAPIError as e: + e = str(e) + message = "Wrong URL" + if "DATA_ERROR" in e: + message += f": {e['DATA_ERROR']}" + raise GenerationError("https://deezer.com/playlist/"+str(id), message) + + # Check if private playlist and owner + if not playlistAPI.get('public', False) and playlistAPI['creator']['id'] != str(dz.current_user['id']): + logger.warning("You can't download others private playlists.") + raise GenerationError("https://deezer.com/playlist/"+str(id), "You can't download others private playlists.", "notYourPrivatePlaylist") + + if not playlistTracksAPI: + playlistTracksAPI = dz.gw.get_playlist_tracks(id) + playlistAPI['various_artist'] = dz.api.get_artist(5080) # Useful for save as compilation + + totalSize = len(playlistTracksAPI) + playlistAPI['nb_tracks'] = totalSize + collection = [] + for pos, trackAPI in enumerate(playlistTracksAPI, start=1): + if trackAPI.get('EXPLICIT_TRACK_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT]: + playlistAPI['explicit'] = True + trackAPI['POSITION'] = pos + trackAPI['SIZE'] = totalSize + collection.append(trackAPI) + + if not 'explicit' in playlistAPI: playlistAPI['explicit'] = False + + return Collection( + 'playlist', + id, + bitrate, + playlistAPI['title'], + playlistAPI['creator']['name'], + playlistAPI['picture_small'][:-24] + '/75x75-000000-80-0-0.jpg', + playlistAPI['explicit'], + totalSize, + tracks_gw=collection, + playlistAPI=playlistAPI + ) + +def generateArtistItem(dz, id, bitrate, interface=None): + # Get essential artist info + try: + artistAPI = dz.api.get_artist(id) + except APIError as e: + e = str(e) + raise GenerationError("https://deezer.com/artist/"+str(id), f"Wrong URL: {e}") + + if interface: interface.send("startAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) + rootArtist = { + 'id': artistAPI['id'], + 'name': artistAPI['name'] + } + + artistDiscographyAPI = dz.gw.get_artist_discography_tabs(id, 100) + allReleases = artistDiscographyAPI.pop('all', []) + albumList = [] + for album in allReleases: + albumList.append(generateAlbumItem(dz, album['id'], bitrate, rootArtist=rootArtist)) + + if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) + return albumList + +def generateArtistDiscographyItem(dz, id, bitrate, interface=None): + # Get essential artist info + try: + artistAPI = dz.api.get_artist(id) + except APIError as e: + e = str(e) + raise GenerationError("https://deezer.com/artist/"+str(id)+"/discography", f"Wrong URL: {e}") + + if interface: interface.send("startAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) + rootArtist = { + 'id': artistAPI['id'], + 'name': artistAPI['name'] + } + + artistDiscographyAPI = dz.gw.get_artist_discography_tabs(id, 100) + artistDiscographyAPI.pop('all', None) # all contains albums and singles, so its all duplicates. This removes them + albumList = [] + for type in artistDiscographyAPI: + for album in artistDiscographyAPI[type]: + albumList.append(generateAlbumItem(dz, album['id'], bitrate, rootArtist=rootArtist)) + + if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) + return albumList + +def generateArtistTopItem(dz, id, bitrate, interface=None): + # Get essential artist info + try: + artistAPI = dz.api.get_artist(id) + except APIError as e: + e = str(e) + raise GenerationError("https://deezer.com/artist/"+str(id)+"/top_track", f"Wrong URL: {e}") + + # Emulate the creation of a playlist + # Can't use generatePlaylistItem directly as this is not a real playlist + playlistAPI = { + 'id': str(artistAPI['id'])+"_top_track", + 'title': artistAPI['name']+" - Top Tracks", + 'description': "Top Tracks for "+artistAPI['name'], + 'duration': 0, + 'public': True, + 'is_loved_track': False, + 'collaborative': False, + 'nb_tracks': 0, + 'fans': artistAPI['nb_fan'], + 'link': "https://www.deezer.com/artist/"+str(artistAPI['id'])+"/top_track", + 'share': None, + 'picture': artistAPI['picture'], + 'picture_small': artistAPI['picture_small'], + 'picture_medium': artistAPI['picture_medium'], + 'picture_big': artistAPI['picture_big'], + 'picture_xl': artistAPI['picture_xl'], + 'checksum': None, + 'tracklist': "https://api.deezer.com/artist/"+str(artistAPI['id'])+"/top", + 'creation_date': "XXXX-00-00", + 'creator': { + 'id': "art_"+str(artistAPI['id']), + 'name': artistAPI['name'], + 'type': "user" + }, + 'type': "playlist" + } + + artistTopTracksAPI_gw = dz.gw.get_artist_toptracks(id) + return generatePlaylistItem(dz, playlistAPI['id'], bitrate, playlistAPI=playlistAPI, playlistTracksAPI=artistTopTracksAPI_gw) diff --git a/deemix/plugins/spotify.py b/deemix/plugins/spotify.py new file mode 100644 index 0000000..e69de29 diff --git a/deemix/settings.py b/deemix/settings.py new file mode 100644 index 0000000..9d3993f --- /dev/null +++ b/deemix/settings.py @@ -0,0 +1,139 @@ +import json +from pathlib import Path +from os import makedirs +from deezer import TrackFormats +import deemix.utils.localpaths as localpaths + +"""Should the lib overwrite files?""" +class OverwriteOption(): + OVERWRITE = 'y' # Yes, overwrite the file + DONT_OVERWRITE = 'n' # No, don't overwrite the file + DONT_CHECK_EXT = 'e' # No, and don't check for extensions + KEEP_BOTH = 'b' # No, and keep both files + ONLY_TAGS = 't' # Overwrite only the tags + +"""What should I do with featured artists?""" +class FeaturesOption(): + NO_CHANGE = "0" # Do nothing + REMOVE_TITLE = "1" # Remove from track title + REMOVE_TITLE_ALBUM = "3" # Remove from track title and album title + MOVE_TITLE = "2" # Move to track title + +DEFAULTS = { + "downloadLocation": "", + "tracknameTemplate": "%artist% - %title%", + "albumTracknameTemplate": "%tracknumber% - %title%", + "playlistTracknameTemplate": "%position% - %artist% - %title%", + "createPlaylistFolder": True, + "playlistNameTemplate": "%playlist%", + "createArtistFolder": False, + "artistNameTemplate": "%artist%", + "createAlbumFolder": True, + "albumNameTemplate": "%artist% - %album%", + "createCDFolder": True, + "createStructurePlaylist": False, + "createSingleFolder": False, + "padTracks": True, + "paddingSize": "0", + "illegalCharacterReplacer": "_", + "queueConcurrency": 3, + "maxBitrate": str(TrackFormats.MP3_320), + "fallbackBitrate": True, + "fallbackSearch": False, + "logErrors": True, + "logSearched": False, + "saveDownloadQueue": False, + "overwriteFile": OverwriteOption.DONT_OVERWRITE, + "createM3U8File": False, + "playlistFilenameTemplate": "playlist", + "syncedLyrics": False, + "embeddedArtworkSize": 800, + "embeddedArtworkPNG": False, + "localArtworkSize": 1400, + "localArtworkFormat": "jpg", + "saveArtwork": True, + "coverImageTemplate": "cover", + "saveArtworkArtist": False, + "artistImageTemplate": "folder", + "jpegImageQuality": 80, + "dateFormat": "Y-M-D", + "albumVariousArtists": True, + "removeAlbumVersion": False, + "removeDuplicateArtists": False, + "tagsLanguage": "", + "featuredToTitle": FeaturesOption.NO_CHANGE, + "titleCasing": "nothing", + "artistCasing": "nothing", + "executeCommand": "", + "tags": { + "title": True, + "artist": True, + "album": True, + "cover": True, + "trackNumber": True, + "trackTotal": False, + "discNumber": True, + "discTotal": False, + "albumArtist": True, + "genre": True, + "year": True, + "date": True, + "explicit": False, + "isrc": True, + "length": True, + "barcode": True, + "bpm": True, + "replayGain": False, + "label": True, + "lyrics": False, + "syncedLyrics": False, + "copyright": False, + "composer": False, + "involvedPeople": False, + "source": False, + "savePlaylistAsCompilation": False, + "useNullSeparator": False, + "saveID3v1": True, + "multiArtistSeparator": "default", + "singleAlbumArtist": False, + "coverDescriptionUTF8": False + } +} + +def saveSettings(settings, configFolder=None): + configFolder = Path(configFolder or localpaths.getConfigFolder()) + makedirs(configFolder, exist_ok=True) # Create config folder if it doesn't exsist + + with open(configFolder / 'config.json', 'w') as configFile: + json.dump(settings, configFile, indent=2) + +def loadSettings(configFolder=None): + configFolder = Path(configFolder or localpaths.getConfigFolder()) + makedirs(configFolder, exist_ok=True) # Create config folder if it doesn't exsist + if not (configFolder / 'config.json').is_file(): saveSettings(DEFAULTS, configFolder) # Create config file if it doesn't exsist + + # Read config file + with open(configFolder / 'config.json', 'r') as configFile: + settings = json.load(configFile) + + if checkSettings(settings) > 0: saveSettings(settings) # Check the settings and save them if something changed + return settings + +def checkSettings(settings): + changes = 0 + for set in DEFAULTS: + if not set in settings or type(settings[set]) != type(DEFAULTS[set]): + settings[set] = DEFAULTS[set] + changes += 1 + for set in DEFAULTS['tags']: + if not set in settings['tags'] or type(settings['tags'][set]) != type(DEFAULTS['tags'][set]): + settings['tags'][set] = DEFAULTS['tags'][set] + changes += 1 + if settings['downloadLocation'] == "": + settings['downloadLocation'] = DEFAULTS['downloadLocation'] + changes += 1 + for template in ['tracknameTemplate', 'albumTracknameTemplate', 'playlistTracknameTemplate', 'playlistNameTemplate', 'artistNameTemplate', 'albumNameTemplate', 'playlistFilenameTemplate', 'coverImageTemplate', 'artistImageTemplate', 'paddingSize']: + if settings[template] == "": + settings[template] = DEFAULTS[template] + changes += 1 + return changes diff --git a/deemix/utils/taggers.py b/deemix/taggers.py similarity index 100% rename from deemix/utils/taggers.py rename to deemix/taggers.py diff --git a/deemix/types/Album.py b/deemix/types/Album.py index 2ac9015..5c7d7f9 100644 --- a/deemix/types/Album.py +++ b/deemix/types/Album.py @@ -4,7 +4,7 @@ from deemix.utils import removeDuplicateArtists, removeFeatures from deemix.types.Artist import Artist from deemix.types.Date import Date from deemix.types.Picture import Picture -from deemix import VARIOUS_ARTISTS +from deemix.types import VARIOUS_ARTISTS class Album: def __init__(self, id="0", title="", pic_md5=""): diff --git a/deemix/types/Artist.py b/deemix/types/Artist.py index cfc49c4..2e0bb1b 100644 --- a/deemix/types/Artist.py +++ b/deemix/types/Artist.py @@ -1,5 +1,5 @@ from deemix.types.Picture import Picture -from deemix import VARIOUS_ARTISTS +from deemix.types import VARIOUS_ARTISTS class Artist: def __init__(self, id="0", name="", role="", pic_md5=""): diff --git a/deemix/types/DownloadObjects.py b/deemix/types/DownloadObjects.py new file mode 100644 index 0000000..e7b43b5 --- /dev/null +++ b/deemix/types/DownloadObjects.py @@ -0,0 +1,126 @@ +class IDownloadObject: + def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, dictItem=None): + if dictItem: + self.type = dictItem['type'] + self.id = dictItem['id'] + self.bitrate = dictItem['bitrate'] + self.title = dictItem['title'] + self.artist = dictItem['artist'] + self.cover = dictItem['cover'] + self.explicit = dictItem.get('explicit', False) + self.size = dictItem['size'] + self.downloaded = dictItem['downloaded'] + self.failed = dictItem['failed'] + self.progress = dictItem['progress'] + self.errors = dictItem['errors'] + self.files = dictItem['files'] + else: + self.type = type + self.id = id + self.bitrate = bitrate + self.title = title + self.artist = artist + self.cover = cover + self.explicit = explicit + self.size = size + self.downloaded = 0 + self.failed = 0 + self.progress = 0 + self.errors = [] + self.files = [] + self.uuid = f"{self.type}_{self.id}_{self.bitrate}" + self.ack = None + self.__type__ = None + + def toDict(self): + return { + 'type': self.type, + 'id': self.id, + 'bitrate': self.bitrate, + 'uuid': self.uuid, + 'title': self.title, + 'artist': self.artist, + 'cover': self.cover, + 'explicit': self.explicit, + 'size': self.size, + 'downloaded': self.downloaded, + 'failed': self.failed, + 'progress': self.progress, + 'errors': self.errors, + 'files': self.files, + 'ack': self.ack, + '__type__': self.__type__ + } + + def getResettedDict(self): + item = self.toDict() + item['downloaded'] = 0 + item['failed'] = 0 + item['progress'] = 0 + item['errors'] = [] + item['files'] = [] + return item + + def getSlimmedDict(self): + light = self.toDict() + propertiesToDelete = ['single', 'collection', 'convertable'] + for property in propertiesToDelete: + if property in light: + del light[property] + return light + +class Single(IDownloadObject): + def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, trackAPI_gw=None, trackAPI=None, albumAPI=None, dictItem=None): + if dictItem: + super().__init__(dictItem=dictItem) + self.single = dictItem['single'] + else: + super().__init__(type, id, bitrate, title, artist, cover, explicit, 1) + self.single = { + 'trackAPI_gw': trackAPI_gw, + 'trackAPI': trackAPI, + 'albumAPI': albumAPI + } + self.__type__ = "Single" + + def toDict(self): + item = super().toDict() + item['single'] = self.single + return item + +class Collection(IDownloadObject): + def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, tracks_gw=None, albumAPI=None, playlistAPI=None, dictItem=None): + if dictItem: + super().__init__(dictItem=dictItem) + self.collection = dictItem['collection'] + else: + super().__init__(type, id, bitrate, title, artist, cover, explicit, size) + self.collection = { + 'tracks_gw': tracks_gw, + 'albumAPI': albumAPI, + 'playlistAPI': playlistAPI + } + self.__type__ = "Collection" + + def toDict(self): + item = super().toDict() + item['collection'] = self.collection + return item + +class Convertable(Collection): + def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, plugin=None, conversion_data=None, dictItem=None): + if dictItem: + super().__init__(dictItem=dictItem) + self.plugin = dictItem['plugin'] + self.conversion_data = dictItem['conversion_data'] + else: + super().__init__(type, id, bitrate, title, artist, cover, explicit, size) + self.plugin = plugin + self.conversion_data = conversion_data + self.__type__ = "Convertable" + + def toDict(self): + item = super().toDict() + item['plugin'] = self.plugin + item['conversion_data'] = self.conversion_data + return item diff --git a/deemix/types/Track.py b/deemix/types/Track.py index a354772..b75db67 100644 --- a/deemix/types/Track.py +++ b/deemix/types/Track.py @@ -14,7 +14,7 @@ from deemix.types.Date import Date from deemix.types.Picture import Picture from deemix.types.Playlist import Playlist from deemix.types.Lyrics import Lyrics -from deemix import VARIOUS_ARTISTS +from deemix.types import VARIOUS_ARTISTS class Track: def __init__(self, id="0", name=""): @@ -259,6 +259,91 @@ class Track: if 'Featured' in self.artist: self.featArtistsString = "feat. "+andCommaConcat(self.artist['Featured']) + def applySettings(self, settings, TEMPDIR, embeddedImageFormat): + from deemix.settings import FeaturesOption + + # Check if should save the playlist as a compilation + if self.playlist and settings['tags']['savePlaylistAsCompilation']: + self.trackNumber = self.position + self.discNumber = "1" + self.album.makePlaylistCompilation(self.playlist) + self.album.embeddedCoverURL = self.playlist.pic.generatePictureURL(settings['embeddedArtworkSize'], embeddedImageFormat) + + ext = self.album.embeddedCoverURL[-4:] + if ext[0] != ".": ext = ".jpg" # Check for Spotify images + + self.album.embeddedCoverPath = TEMPDIR / f"pl{trackAPI_gw['_EXTRA_PLAYLIST']['id']}_{settings['embeddedArtworkSize']}{ext}" + else: + if self.album.date: self.date = self.album.date + self.album.embeddedCoverURL = self.album.pic.generatePictureURL(settings['embeddedArtworkSize'], embeddedImageFormat) + + ext = self.album.embeddedCoverURL[-4:] + self.album.embeddedCoverPath = TEMPDIR / f"alb{self.album.id}_{settings['embeddedArtworkSize']}{ext}" + + self.dateString = self.date.format(settings['dateFormat']) + self.album.dateString = self.album.date.format(settings['dateFormat']) + if self.playlist: self.playlist.dateString = self.playlist.date.format(settings['dateFormat']) + + # Check various artist option + if settings['albumVariousArtists'] and self.album.variousArtists: + artist = self.album.variousArtists + isMainArtist = artist.role == "Main" + + if artist.name not in self.album.artists: + self.album.artists.insert(0, artist.name) + + if isMainArtist or artist.name not in self.album.artist['Main'] and not isMainArtist: + if not artist.role in self.album.artist: + self.album.artist[artist.role] = [] + self.album.artist[artist.role].insert(0, artist.name) + self.album.mainArtist.save = not self.album.mainArtist.isVariousArtists() or settings['albumVariousArtists'] and self.album.mainArtist.isVariousArtists() + + # Check removeDuplicateArtists + if settings['removeDuplicateArtists']: self.removeDuplicateArtists() + + # Check if user wants the feat in the title + if str(settings['featuredToTitle']) == FeaturesOption.REMOVE_TITLE: + self.title = self.getCleanTitle() + elif str(settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: + self.title = self.getFeatTitle() + elif str(settings['featuredToTitle']) == FeaturesOption.REMOVE_TITLE_ALBUM: + self.title = self.getCleanTitle() + self.album.title = self.album.getCleanTitle() + + # Remove (Album Version) from tracks that have that + if settings['removeAlbumVersion']: + if "Album Version" in self.title: + self.title = re.sub(r' ?\(Album Version\)', "", self.title).strip() + + # Change Title and Artists casing if needed + if settings['titleCasing'] != "nothing": + self.title = changeCase(self.title, settings['titleCasing']) + if settings['artistCasing'] != "nothing": + self.mainArtist.name = changeCase(self.mainArtist.name, settings['artistCasing']) + for i, artist in enumerate(self.artists): + self.artists[i] = changeCase(artist, settings['artistCasing']) + for type in self.artist: + for i, artist in enumerate(self.artist[type]): + self.artist[type][i] = changeCase(artist, settings['artistCasing']) + self.generateMainFeatStrings() + + # Generate artist tag + if settings['tags']['multiArtistSeparator'] == "default": + if str(settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: + self.artistsString = ", ".join(self.artist['Main']) + else: + self.artistsString = ", ".join(self.artists) + elif settings['tags']['multiArtistSeparator'] == "andFeat": + self.artistsString = self.mainArtistsString + if self.featArtistsString and str(settings['featuredToTitle']) != FeaturesOption.MOVE_TITLE: + self.artistsString += " " + self.featArtistsString + else: + separator = settings['tags']['multiArtistSeparator'] + if str(settings['featuredToTitle']) == FeaturesOption.MOVE_TITLE: + self.artistsString = separator.join(self.artist['Main']) + else: + self.artistsString = separator.join(self.artists) + class TrackError(Exception): """Base class for exceptions in this module.""" pass diff --git a/deemix/types/__init__.py b/deemix/types/__init__.py index 9db5426..3c0325c 100644 --- a/deemix/types/__init__.py +++ b/deemix/types/__init__.py @@ -1,7 +1 @@ -from deemix.types.Date import Date -from deemix.types.Picture import Picture -from deemix.types.Lyrics import Lyrics -from deemix.types.Album import Album -from deemix.types.Artist import Artist -from deemix.types.Playlist import Playlist -from deemix.types.Track import Track +VARIOUS_ARTISTS = "5080" diff --git a/deemix/utils/__init__.py b/deemix/utils/__init__.py index 5936119..8f67ace 100644 --- a/deemix/utils/__init__.py +++ b/deemix/utils/__init__.py @@ -1,4 +1,3 @@ -import re import string from deezer import TrackFormats import os @@ -6,7 +5,7 @@ import os def generateReplayGainString(trackGain): return "{0:.2f} dB".format((float(trackGain) + 18.4) * -1) -def getBitrateInt(txt): +def getBitrateNumberFromText(txt): txt = str(txt).lower() if txt in ['flac', 'lossless', '9']: return TrackFormats.FLAC @@ -23,7 +22,6 @@ def getBitrateInt(txt): else: return None - def changeCase(str, type): if type == "lower": return str.lower() @@ -36,7 +34,6 @@ def changeCase(str, type): else: return str - def removeFeatures(title): clean = title if "(feat." in clean.lower(): @@ -48,7 +45,6 @@ def removeFeatures(title): clean = ' '.join(clean.split()) return clean - def andCommaConcat(lst): tot = len(lst) result = "" @@ -61,62 +57,6 @@ def andCommaConcat(lst): result += ", " return result - -def getIDFromLink(link, type): - if '?' in link: - link = link[:link.find('?')] - if link.endswith("/"): - link = link[:-1] - - if link.startswith("http") and 'open.spotify.com/' in link: - if '&' in link: link = link[:link.find('&')] - if type == "spotifyplaylist": - return link[link.find("/playlist/") + 10:] - if type == "spotifytrack": - return link[link.find("/track/") + 7:] - if type == "spotifyalbum": - return link[link.find("/album/") + 7:] - elif link.startswith("spotify:"): - if type == "spotifyplaylist": - return link[link.find("playlist:") + 9:] - if type == "spotifytrack": - return link[link.find("track:") + 6:] - if type == "spotifyalbum": - return link[link.find("album:") + 6:] - elif type == "artisttop": - return re.search(r"\/artist\/(\d+)\/top_track", link)[1] - elif type == "artistdiscography": - return re.search(r"\/artist\/(\d+)\/discography", link)[1] - else: - return link[link.rfind("/") + 1:] - - -def getTypeFromLink(link): - type = '' - if 'spotify' in link: - type = 'spotify' - if 'playlist' in link: - type += 'playlist' - elif 'track' in link: - type += 'track' - elif 'album' in link: - type += 'album' - elif 'deezer' in link: - if '/track' in link: - type = 'track' - elif '/playlist' in link: - type = 'playlist' - elif '/album' in link: - type = 'album' - elif re.search("\/artist\/(\d+)\/top_track", link): - type = 'artisttop' - elif re.search("\/artist\/(\d+)\/discography", link): - type = 'artistdiscography' - elif '/artist' in link: - type = 'artist' - return type - - def uniqueArray(arr): for iPrinc, namePrinc in enumerate(arr): for iRest, nRest in enumerate(arr): diff --git a/setup.py b/setup.py index ec9d9b6..f0a818d 100644 --- a/setup.py +++ b/setup.py @@ -16,12 +16,11 @@ setup( license="GPL3", classifiers=[ "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", - "Development Status :: 4 - Beta", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", "Operating System :: OS Independent", ], - python_requires='>=3.6', + python_requires='>=3.7', packages=find_packages(exclude=("tests",)), include_package_data=True, install_requires=["click", "pycryptodomex", "mutagen", "requests", "spotipy>=2.11.0", "deezer-py"], diff --git a/updatePyPi.sh b/updatePyPi.sh index 9c400e0..767fd01 100755 --- a/updatePyPi.sh +++ b/updatePyPi.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash rm -rd build rm -rd dist -python -m bump -python -m bump deemix/__init__.py +#python -m bump +#python -m bump deemix/__init__.py python3 setup.py sdist bdist_wheel python3 -m twine upload dist/* -- 2.25.1 From dc6adc7887297cb4d454ea249a37f9d80f8341c8 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Fri, 19 Mar 2021 15:44:21 +0100 Subject: [PATCH 08/20] More work on the library (WIP) --- deemix/__init__.py | 8 +- deemix/decryption.py | 136 ++++++++++++++++++++++++++++++++ deemix/downloader.py | 96 +++------------------- deemix/itemgen.py | 6 +- deemix/types/DownloadObjects.py | 22 ++++++ deemix/types/Playlist.py | 2 +- deemix/types/Track.py | 11 +-- deemix/utils/pathtemplates.py | 6 +- 8 files changed, 187 insertions(+), 100 deletions(-) diff --git a/deemix/__init__.py b/deemix/__init__.py index ea1b8ae..8f10dc1 100644 --- a/deemix/__init__.py +++ b/deemix/__init__.py @@ -26,19 +26,19 @@ def parseLink(link): id = link[link.rfind("/") + 1:] elif '/playlist' in link: type = 'playlist' - id = re.search("\/playlist\/(\d+)", link)[0] + id = re.search("\/playlist\/(\d+)", link).group(1) elif '/album' in link: type = 'album' id = link[link.rfind("/") + 1:] elif re.search("\/artist\/(\d+)\/top_track", link): type = 'artist_top' - id = re.search("\/artist\/(\d+)\/top_track", link)[0] + id = re.search("\/artist\/(\d+)\/top_track", link).group(1) elif re.search("\/artist\/(\d+)\/discography", link): type = 'artist_discography' - id = re.search("\/artist\/(\d+)\/discography", link)[0] + id = re.search("\/artist\/(\d+)\/discography", link).group(1) elif '/artist' in link: type = 'artist' - id = re.search("\/artist\/(\d+)", link)[0] + id = re.search("\/artist\/(\d+)", link).group(1) return (link, type, id) diff --git a/deemix/decryption.py b/deemix/decryption.py index 0dec77f..867cb52 100644 --- a/deemix/decryption.py +++ b/deemix/decryption.py @@ -2,6 +2,18 @@ import binascii from Cryptodome.Cipher import Blowfish, AES from Cryptodome.Hash import MD5 +from deemix import USER_AGENT_HEADER +from deemix.types.DownloadObjects import Single, Collection + +from requests import get + +from requests.exceptions import ConnectionError, ReadTimeout +from ssl import SSLError +from urllib3.exceptions import SSLError as u3SSLError + +import logging +logger = logging.getLogger('deemix') + def _md5(data): h = MD5.new() h.update(str.encode(data) if isinstance(data, str) else data) @@ -40,3 +52,127 @@ def generateUnencryptedStreamURL(sng_id, md5, media_version, format): def reverseStreamURL(url): urlPart = url[url.find("/1/")+3:] return generateStreamPath(urlPart) + +def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None): + headers= {'User-Agent': USER_AGENT_HEADER} + chunkLength = start + percentage = 0 + + itemName = f"[{track.mainArtist.name} - {track.title}]" + + try: + with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: + request.raise_for_status() + + complete = int(request.headers["Content-Length"]) + if complete == 0: raise DownloadEmpty + if start != 0: + responseRange = request.headers["Content-Range"] + logger.info(f'{itemName} downloading range {responseRange}') + else: + logger.info(f'{itemName} downloading {complete} bytes') + + for chunk in request.iter_content(2048 * 3): + outputStream.write(chunk) + chunkLength += len(chunk) + + if downloadObject: + if isinstance(downloadObject, Single): + percentage = (chunkLength / (complete + start)) * 100 + downloadObject.progressNext = percentage + else: + chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 + downloadObject.progressNext += chunkProgres + downloadObject.updateProgress(interface) + + except (SSLError, u3SSLError) as e: + logger.info(f'{itemName} retrying from byte {chunkLength}') + return streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface) + except (ConnectionError, ReadTimeout): + sleep(2) + return streamUnencryptedTrack(outputStream, track, start, downloadObject, interface) + +def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None): + headers= {'User-Agent': USER_AGENT_HEADER} + chunkLength = start + percentage = 0 + + itemName = f"[{track.mainArtist.name} - {track.title}]" + + try: + with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: + request.raise_for_status() + + complete = int(request.headers["Content-Length"]) + if complete == 0: raise DownloadEmpty + if start != 0: + responseRange = request.headers["Content-Range"] + logger.info(f'{itemName} downloading range {responseRange}') + else: + logger.info(f'{itemName} downloading {complete} bytes') + + for chunk in request.iter_content(2048 * 3): + outputStream.write(chunk) + chunkLength += len(chunk) + + if downloadObject: + if isinstance(downloadObject, Single): + percentage = (chunkLength / (complete + start)) * 100 + downloadObject.progressNext = percentage + else: + chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 + downloadObject.progressNext += chunkProgres + downloadObject.updateProgress(interface) + + except (SSLError, u3SSLError) as e: + logger.info(f'{itemName} retrying from byte {chunkLength}') + return streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface) + except (ConnectionError, ReadTimeout): + sleep(2) + return streamUnencryptedTrack(outputStream, track, start, downloadObject, interface) + +def streamTrack(outputStream, track, start=0, downloadObject=None, interface=None): + headers= {'User-Agent': USER_AGENT_HEADER} + chunkLength = start + percentage = 0 + + itemName = f"[{track.mainArtist.name} - {track.title}]" + + try: + with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: + request.raise_for_status() + blowfish_key = str.encode(generateBlowfishKey(str(track.id))) + + complete = int(request.headers["Content-Length"]) + if complete == 0: raise DownloadEmpty + if start != 0: + responseRange = request.headers["Content-Range"] + logger.info(f'{itemName} downloading range {responseRange}') + else: + logger.info(f'{itemName} downloading {complete} bytes') + + for chunk in request.iter_content(2048 * 3): + if len(chunk) >= 2048: + chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:] + + outputStream.write(chunk) + chunkLength += len(chunk) + + if downloadObject: + if isinstance(downloadObject, Single): + percentage = (chunkLength / (complete + start)) * 100 + downloadObject.progressNext = percentage + else: + chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 + downloadObject.progressNext += chunkProgres + downloadObject.updateProgress(interface) + + except (SSLError, u3SSLError) as e: + logger.info(f'{itemName} retrying from byte {chunkLength}') + return streamTrack(outputStream, track, chunkLength, downloadObject, interface) + except (ConnectionError, ReadTimeout): + sleep(2) + return streamTrack(outputStream, track, start, downloadObject, interface) + +class DownloadEmpty(Exception): + pass diff --git a/deemix/downloader.py b/deemix/downloader.py index ff8da30..f2a985f 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -11,24 +11,21 @@ import re import errno from ssl import SSLError -from os import makedirs from urllib3.exceptions import SSLError as u3SSLError +from os import makedirs from deemix.types.DownloadObjects import Single, Collection from deemix.types.Track import Track, AlbumDoesntExists -from deemix.utils import changeCase from deemix.utils.pathtemplates import generateFilename, generateFilepath, settingsRegexAlbum, settingsRegexArtist, settingsRegexPlaylistFile from deezer import TrackFormats from deemix import USER_AGENT_HEADER from deemix.taggers import tagID3, tagFLAC -from deemix.decryption import generateStreamURL, generateBlowfishKey +from deemix.decryption import generateUnencryptedStreamURL, streamUnencryptedTrack from deemix.settings import OverwriteOption -from Cryptodome.Cipher import Blowfish from mutagen.flac import FLACNoHeaderError, error as FLACError -import logging -logging.basicConfig(level=logging.INFO) +import logging logger = logging.getLogger('deemix') from tempfile import gettempdir @@ -124,7 +121,7 @@ def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectU if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]: request = requests.head( - generateStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), + generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), headers={'User-Agent': USER_AGENT_HEADER}, timeout=30 ) @@ -159,8 +156,6 @@ class Downloader: self.settings = settings self.bitrate = downloadObject.bitrate self.interface = interface - self.downloadPercentage = 0 - self.lastPercentage = 0 self.extrasPath = None self.playlistCoverName = None self.playlistURLs = [] @@ -184,7 +179,6 @@ class Downloader: if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer") # Create Track object - print(track) if not track: logger.info(f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}] Getting the tags") try: @@ -252,7 +246,7 @@ class Downloader: url = track.album.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) if self.settings['tags']['savePlaylistAsCompilation'] \ and track.playlist \ - and track.playlist.pic.url \ + and track.playlist.pic.staticUrl \ and not format.startswith("jpg"): continue result['albumURLs'].append({'url': url, 'ext': format}) @@ -280,7 +274,7 @@ class Downloader: extendedFormat = format if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" url = track.playlist.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) - if track.playlist.pic.url and not format.startswith("jpg"): continue + if track.playlist.pic.staticUrl and not format.startswith("jpg"): continue self.playlistURLs.append({'url': url, 'ext': format}) if not self.playlistCoverName: track.playlist.bitrate = selectedFormat @@ -316,12 +310,12 @@ class Downloader: if not trackAlreadyDownloaded or self.settings['overwriteFile'] == OverwriteOption.OVERWRITE: logger.info(f"[{track.mainArtist.name} - {track.title}] Downloading the track") - track.downloadUrl = generateStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat) + track.downloadUrl = generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat) def downloadMusic(track, trackAPI_gw): try: with open(writepath, 'wb') as stream: - self.streamTrack(stream, track) + streamUnencryptedTrack(stream, track, downloadObject=self.downloadObject, interface=self.interface) except DownloadCancelled: if writepath.is_file(): writepath.unlink() raise DownloadCancelled @@ -382,7 +376,7 @@ class Downloader: if not trackDownloaded: return self.download(trackAPI_gw, track=track) else: logger.info(f"[{track.mainArtist.name} - {track.title}] Skipping track as it's already downloaded") - self.completeTrackPercentage() + self.downloadObject.completeTrackProgress(self.interface) # Adding tags if (not trackAlreadyDownloaded or self.settings['overwriteFile'] in [OverwriteOption.ONLY_TAGS, OverwriteOption.OVERWRITE]) and not track.localTrack: @@ -395,7 +389,7 @@ class Downloader: except (FLACNoHeaderError, FLACError): if writepath.is_file(): writepath.unlink() logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available in FLAC, falling back if necessary") - self.removeTrackPercentage() + self.downloadObject.removeTrackProgress(self.interface) track.filesizes['FILESIZE_FLAC'] = "0" track.filesizes['FILESIZE_FLAC_TESTED'] = True return self.download(trackAPI_gw, track=track) @@ -409,71 +403,6 @@ class Downloader: self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'downloaded': True, 'downloadPath': str(writepath), 'extrasPath': str(self.extrasPath)}) return result - def streamTrack(self, stream, track, start=0): - - headers=dict(self.dz.http_headers) - if range != 0: headers['Range'] = f'bytes={start}-' - chunkLength = start - percentage = 0 - - itemName = f"[{track.mainArtist.name} - {track.title}]" - - try: - with self.dz.session.get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: - request.raise_for_status() - blowfish_key = str.encode(generateBlowfishKey(str(track.id))) - - complete = int(request.headers["Content-Length"]) - if complete == 0: raise DownloadEmpty - if start != 0: - responseRange = request.headers["Content-Range"] - logger.info(f'{itemName} downloading range {responseRange}') - else: - logger.info(f'{itemName} downloading {complete} bytes') - - for chunk in request.iter_content(2048 * 3): - - if len(chunk) >= 2048: - chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:] - - stream.write(chunk) - chunkLength += len(chunk) - - if isinstance(self.downloadObject, Single): - percentage = (chunkLength / (complete + start)) * 100 - self.downloadPercentage = percentage - else: - chunkProgres = (len(chunk) / (complete + start)) / self.downloadObject.size * 100 - self.downloadPercentage += chunkProgres - self.updatePercentage() - - except (SSLError, u3SSLError) as e: - logger.info(f'{itemName} retrying from byte {chunkLength}') - return self.streamTrack(stream, track, chunkLength) - except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout): - sleep(2) - return self.streamTrack(stream, track, start) - - def updatePercentage(self): - if round(self.downloadPercentage) != self.lastPercentage and round(self.downloadPercentage) % 2 == 0: - self.lastPercentage = round(self.downloadPercentage) - self.downloadObject.progress = self.lastPercentage - if self.interface: self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'progress': self.lastPercentage}) - - def completeTrackPercentage(self): - if isinstance(self.downloadObject, Single): - self.downloadPercentage = 100 - else: - self.downloadPercentage += (1 / self.downloadObject.size) * 100 - self.updatePercentage() - - def removeTrackPercentage(self): - if isinstance(self.downloadObject, Single): - self.downloadPercentage = 0 - else: - self.downloadPercentage -= (1 / self.downloadObject.size) * 100 - self.updatePercentage() - def downloadWrapper(self, trackAPI_gw, trackAPI=None, albumAPI=None, playlistAPI=None, track=None): # Temp metadata to generate logs tempTrack = { @@ -531,7 +460,7 @@ class Downloader: }} if 'error' in result: - self.completeTrackPercentage() + self.downloadObject.completeTrackProgress(self.interface) self.downloadObject.failed += 1 self.downloadObject.errors.append(result['error']) if self.interface: @@ -640,9 +569,6 @@ class DownloadFailed(DownloadError): class DownloadCancelled(DownloadError): pass -class DownloadEmpty(DownloadError): - pass - class PreferredBitrateNotFound(DownloadError): pass diff --git a/deemix/itemgen.py b/deemix/itemgen.py index 9629044..15d6eb0 100644 --- a/deemix/itemgen.py +++ b/deemix/itemgen.py @@ -1,4 +1,6 @@ from deemix.types.DownloadObjects import Single, Collection +from deezer.api import APIError +from deezer.gw import GWAPIError, LyricsStatus class GenerationError(Exception): def __init__(self, link, message, errid=None): @@ -29,7 +31,7 @@ def generateTrackItem(dz, id, bitrate, trackAPI=None, albumAPI=None): # Get essential track info try: trackAPI_gw = dz.gw.get_track_with_fallback(id) - except gwAPIError as e: + except GWAPIError as e: e = str(e) message = "Wrong URL" if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}" @@ -116,7 +118,7 @@ def generatePlaylistItem(dz, id, bitrate, playlistAPI=None, playlistTracksAPI=No try: userPlaylist = dz.gw.get_playlist_page(id) playlistAPI = map_user_playlist(userPlaylist['DATA']) - except gwAPIError as e: + except GWAPIError as e: e = str(e) message = "Wrong URL" if "DATA_ERROR" in e: diff --git a/deemix/types/DownloadObjects.py b/deemix/types/DownloadObjects.py index e7b43b5..0fd2664 100644 --- a/deemix/types/DownloadObjects.py +++ b/deemix/types/DownloadObjects.py @@ -28,6 +28,7 @@ class IDownloadObject: self.progress = 0 self.errors = [] self.files = [] + self.progressNext = 0 self.uuid = f"{self.type}_{self.id}_{self.bitrate}" self.ack = None self.__type__ = None @@ -69,6 +70,11 @@ class IDownloadObject: del light[property] return light + def updateProgress(self, interface=None): + if round(self.progressNext) != self.progress and round(self.progressNext) % 2 == 0: + self.progress = round(self.progressNext) + if interface: interface.send("updateQueue", {'uuid': self.uuid, 'progress': self.progress}) + class Single(IDownloadObject): def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, trackAPI_gw=None, trackAPI=None, albumAPI=None, dictItem=None): if dictItem: @@ -88,6 +94,14 @@ class Single(IDownloadObject): item['single'] = self.single return item + def completeTrackProgress(self, interface=None): + self.progressNext = 100 + self.updateProgress(interface) + + def removeTrackProgress(self, interface=None): + self.progressNext = 0 + self.updateProgress(interface) + class Collection(IDownloadObject): def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, tracks_gw=None, albumAPI=None, playlistAPI=None, dictItem=None): if dictItem: @@ -107,6 +121,14 @@ class Collection(IDownloadObject): item['collection'] = self.collection return item + def completeTrackProgress(self, interface=None): + self.progressNext += (1 / self.size) * 100 + self.updateProgress(interface) + + def removeTrackProgress(self, interface=None): + self.progressNext -= (1 / self.size) * 100 + self.updateProgress(interface) + class Convertable(Collection): def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, plugin=None, conversion_data=None, dictItem=None): if dictItem: diff --git a/deemix/types/Playlist.py b/deemix/types/Playlist.py index e358674..61412ee 100644 --- a/deemix/types/Playlist.py +++ b/deemix/types/Playlist.py @@ -39,7 +39,7 @@ class Playlist: if 'various_artist' in playlistAPI: pic_md5 = playlistAPI['various_artist']['picture_small'] - pic_md5 = pic_md5[pic_md5.indexOf('artist/') + 7:-24] + pic_md5 = pic_md5[pic_md5.find('artist/') + 7:-24] self.variousArtists = Artist( id = playlistAPI['various_artist']['id'], name = playlistAPI['various_artist']['name'], diff --git a/deemix/types/Track.py b/deemix/types/Track.py index b75db67..e130578 100644 --- a/deemix/types/Track.py +++ b/deemix/types/Track.py @@ -5,9 +5,10 @@ import logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger('deemix') -from deezer.gw import APIError as gwAPIError +from deezer.gw import GWAPIError from deezer.api import APIError -from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString +from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString, changeCase + from deemix.types.Album import Album from deemix.types.Artist import Artist from deemix.types.Date import Date @@ -114,7 +115,7 @@ class Track: # Get Lyrics data if not "LYRICS" in trackAPI_gw and self.lyrics.id != "0": try: trackAPI_gw["LYRICS"] = dz.gw.get_track_lyrics(self.id) - except gwAPIError: self.lyrics.id = "0" + except GWAPIError: self.lyrics.id = "0" if self.lyrics.id != "0": self.lyrics.parseLyrics(trackAPI_gw["LYRICS"]) # Parse Album data @@ -132,7 +133,7 @@ class Track: # Get album_gw Data if not albumAPI_gw: try: albumAPI_gw = dz.gw.get_album(self.album.id) - except gwAPIError: albumAPI_gw = None + except GWAPIError: albumAPI_gw = None if albumAPI: self.album.parseAlbum(albumAPI) @@ -261,7 +262,7 @@ class Track: def applySettings(self, settings, TEMPDIR, embeddedImageFormat): from deemix.settings import FeaturesOption - + # Check if should save the playlist as a compilation if self.playlist and settings['tags']['savePlaylistAsCompilation']: self.trackNumber = self.position diff --git a/deemix/utils/pathtemplates.py b/deemix/utils/pathtemplates.py index 3d04dce..3d5dafe 100644 --- a/deemix/utils/pathtemplates.py +++ b/deemix/utils/pathtemplates.py @@ -148,7 +148,7 @@ def settingsRegex(filename, track, settings): filename = filename.replace("%album_id%", str(track.album.id)) filename = filename.replace("%artist_id%", str(track.mainArtist.id)) if track.playlist: - filename = filename.replace("%playlist_id%", str(track.playlist.playlistId)) + filename = filename.replace("%playlist_id%", str(track.playlist.playlistID)) filename = filename.replace("%position%", pad(track.position, track.playlist.trackTotal, settings)) else: filename = filename.replace("%playlist_id%", '') @@ -159,7 +159,7 @@ def settingsRegex(filename, track, settings): def settingsRegexAlbum(foldername, album, settings, playlist=None): if playlist and settings['tags']['savePlaylistAsCompilation']: - foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistId)) + foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistID)) foldername = foldername.replace("%genre%", "Compile") else: foldername = foldername.replace("%album_id%", str(album.id)) @@ -205,7 +205,7 @@ def settingsRegexArtist(foldername, artist, settings, rootArtist=None): def settingsRegexPlaylist(foldername, playlist, settings): foldername = foldername.replace("%playlist%", fixName(playlist.title, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistId, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistID, settings['illegalCharacterReplacer'])) foldername = foldername.replace("%owner%", fixName(playlist.owner['name'], settings['illegalCharacterReplacer'])) foldername = foldername.replace("%owner_id%", str(playlist.owner['id'])) foldername = foldername.replace("%year%", str(playlist.date.year)) -- 2.25.1 From eda8fd3d138439ca14b273c7c3440ac85a1f4041 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Wed, 24 Mar 2021 17:41:03 +0100 Subject: [PATCH 09/20] Even more rework on the library --- deemix/__init__.py | 6 +-- deemix/__main__.py | 6 +-- deemix/downloader.py | 8 ++-- deemix/itemgen.py | 78 ++++++++++++++++-------------- deemix/types/DownloadObjects.py | 85 ++++++++++----------------------- deemix/types/Picture.py | 3 +- deemix/types/Track.py | 38 +++++++-------- 7 files changed, 95 insertions(+), 129 deletions(-) diff --git a/deemix/__init__.py b/deemix/__init__.py index 8f10dc1..f5caa61 100644 --- a/deemix/__init__.py +++ b/deemix/__init__.py @@ -23,13 +23,13 @@ def parseLink(link): if '/track' in link: type = 'track' - id = link[link.rfind("/") + 1:] + id = re.search("\/track\/(.+)", link).group(1) elif '/playlist' in link: type = 'playlist' id = re.search("\/playlist\/(\d+)", link).group(1) elif '/album' in link: type = 'album' - id = link[link.rfind("/") + 1:] + id = re.search("\/album\/(.+)", link).group(1) elif re.search("\/artist\/(\d+)\/top_track", link): type = 'artist_top' id = re.search("\/artist\/(\d+)\/top_track", link).group(1) @@ -42,7 +42,7 @@ def parseLink(link): return (link, type, id) -def generateDownloadItem(dz, link, bitrate): +def generateDownloadObject(dz, link, bitrate): (link, type, id) = parseLink(link) if type == None or id == None: return None diff --git a/deemix/__main__.py b/deemix/__main__.py index fde6781..5602e36 100644 --- a/deemix/__main__.py +++ b/deemix/__main__.py @@ -5,7 +5,7 @@ from pathlib import Path from deezer import Deezer from deezer import TrackFormats -from deemix import generateDownloadItem +from deemix import generateDownloadObject from deemix.settings import loadSettings from deemix.utils import getBitrateNumberFromText import deemix.utils.localpaths as localpaths @@ -49,8 +49,8 @@ def download(url, bitrate, portable, path): links.append(link) for link in links: - downloadItem = generateDownloadItem(dz, link, bitrate) - Downloader(dz, downloadItem, settings).start() + downloadObject = generateDownloadObject(dz, link, bitrate) + Downloader(dz, downloadObject, settings).start() if path is not None: if path == '': path = '.' diff --git a/deemix/downloader.py b/deemix/downloader.py index f2a985f..8f6273c 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -321,9 +321,9 @@ class Downloader: raise DownloadCancelled except (requests.exceptions.HTTPError, DownloadEmpty): if writepath.is_file(): writepath.unlink() - if track.fallbackId != "0": + if track.fallbackID != "0": logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available, using fallback id") - newTrack = self.dz.gw.get_track_with_fallback(track.fallbackId) + newTrack = self.dz.gw.get_track_with_fallback(track.fallbackID) track.parseEssentialData(newTrack) track.retriveFilesizes(self.dz) return False @@ -418,9 +418,9 @@ class Downloader: except DownloadFailed as error: if error.track: track = error.track - if track.fallbackId != "0": + if track.fallbackID != "0": logger.warn(f"[{track.mainArtist.name} - {track.title}] {error.message} Using fallback id") - newTrack = self.dz.gw.get_track_with_fallback(track.fallbackId) + newTrack = self.dz.gw.get_track_with_fallback(track.fallbackID) track.parseEssentialData(newTrack) track.retriveFilesizes(self.dz) return self.downloadWrapper(trackAPI_gw, trackAPI, albumAPI, playlistAPI, track) diff --git a/deemix/itemgen.py b/deemix/itemgen.py index 15d6eb0..6087d2c 100644 --- a/deemix/itemgen.py +++ b/deemix/itemgen.py @@ -42,18 +42,20 @@ def generateTrackItem(dz, id, bitrate, trackAPI=None, albumAPI=None): title += f" {trackAPI_gw['VERSION']}".strip() explicit = bool(int(trackAPI_gw.get('EXPLICIT_LYRICS', 0))) - return Single( - 'track', - id, - bitrate, - title, - trackAPI_gw['ART_NAME'], - f"https://e-cdns-images.dzcdn.net/images/cover/{trackAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg", - explicit, - trackAPI_gw, - trackAPI, - albumAPI - ) + return Single({ + 'type': 'track', + 'id': id, + 'bitrate': bitrate, + 'title': title, + 'artist': trackAPI_gw['ART_NAME'], + 'cover': f"https://e-cdns-images.dzcdn.net/images/cover/{trackAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg", + 'explicit': explicit, + 'single': { + 'trackAPI_gw': trackAPI_gw, + 'trackAPI': trackAPI, + 'albumAPI': albumAPI + } + }) def generateAlbumItem(dz, id, bitrate, rootArtist=None): # Get essential album info @@ -93,18 +95,20 @@ def generateAlbumItem(dz, id, bitrate, rootArtist=None): explicit = albumAPI_gw.get('EXPLICIT_ALBUM_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT] - return Collection( - 'album', - id, - bitrate, - albumAPI['title'], - albumAPI['artist']['name'], - cover, - explicit, - totalSize, - tracks_gw=collection, - albumAPI=albumAPI - ) + return Collection({ + 'type': 'album', + 'id': id, + 'bitrate': bitrate, + 'title': albumAPI['title'], + 'artist': albumAPI['artist']['name'], + 'cover': cover, + 'explicit': explicit, + 'size': totalSize, + 'collection': { + 'tracks_gw': collection, + 'albumAPI': albumAPI + } + }) def generatePlaylistItem(dz, id, bitrate, playlistAPI=None, playlistTracksAPI=None): if not playlistAPI: @@ -146,18 +150,20 @@ def generatePlaylistItem(dz, id, bitrate, playlistAPI=None, playlistTracksAPI=No if not 'explicit' in playlistAPI: playlistAPI['explicit'] = False - return Collection( - 'playlist', - id, - bitrate, - playlistAPI['title'], - playlistAPI['creator']['name'], - playlistAPI['picture_small'][:-24] + '/75x75-000000-80-0-0.jpg', - playlistAPI['explicit'], - totalSize, - tracks_gw=collection, - playlistAPI=playlistAPI - ) + return Collection({ + 'type': 'playlist', + 'id': id, + 'bitrate': bitrate, + 'title': playlistAPI['title'], + 'artist': playlistAPI['creator']['name'], + 'cover': playlistAPI['picture_small'][:-24] + '/75x75-000000-80-0-0.jpg', + 'explicit': playlistAPI['explicit'], + 'size': totalSize, + 'collection': { + 'tracks_gw': collection, + 'playlistAPI': playlistAPI + } + }) def generateArtistItem(dz, id, bitrate, interface=None): # Get essential artist info diff --git a/deemix/types/DownloadObjects.py b/deemix/types/DownloadObjects.py index 0fd2664..c0e6736 100644 --- a/deemix/types/DownloadObjects.py +++ b/deemix/types/DownloadObjects.py @@ -1,33 +1,18 @@ class IDownloadObject: - def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, dictItem=None): - if dictItem: - self.type = dictItem['type'] - self.id = dictItem['id'] - self.bitrate = dictItem['bitrate'] - self.title = dictItem['title'] - self.artist = dictItem['artist'] - self.cover = dictItem['cover'] - self.explicit = dictItem.get('explicit', False) - self.size = dictItem['size'] - self.downloaded = dictItem['downloaded'] - self.failed = dictItem['failed'] - self.progress = dictItem['progress'] - self.errors = dictItem['errors'] - self.files = dictItem['files'] - else: - self.type = type - self.id = id - self.bitrate = bitrate - self.title = title - self.artist = artist - self.cover = cover - self.explicit = explicit - self.size = size - self.downloaded = 0 - self.failed = 0 - self.progress = 0 - self.errors = [] - self.files = [] + def __init__(self, obj): + self.type = obj['type'] + self.id = obj['id'] + self.bitrate = obj['bitrate'] + self.title = obj['title'] + self.artist = obj['artist'] + self.cover = obj['cover'] + self.explicit = obj.get('explicit', False) + self.size = obj['size'] + self.downloaded = obj.get('downloaded', 0) + self.failed = obj.get('failed', 0) + self.progress = obj.get('progress', 0) + self.errors = obj.get('errors', []) + self.files = obj.get('files', []) self.progressNext = 0 self.uuid = f"{self.type}_{self.id}_{self.bitrate}" self.ack = None @@ -76,17 +61,10 @@ class IDownloadObject: if interface: interface.send("updateQueue", {'uuid': self.uuid, 'progress': self.progress}) class Single(IDownloadObject): - def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, trackAPI_gw=None, trackAPI=None, albumAPI=None, dictItem=None): - if dictItem: - super().__init__(dictItem=dictItem) - self.single = dictItem['single'] - else: - super().__init__(type, id, bitrate, title, artist, cover, explicit, 1) - self.single = { - 'trackAPI_gw': trackAPI_gw, - 'trackAPI': trackAPI, - 'albumAPI': albumAPI - } + def __init__(self, obj): + super().__init__(obj) + self.size = 1 + self.single = obj['single'] self.__type__ = "Single" def toDict(self): @@ -103,17 +81,9 @@ class Single(IDownloadObject): self.updateProgress(interface) class Collection(IDownloadObject): - def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, tracks_gw=None, albumAPI=None, playlistAPI=None, dictItem=None): - if dictItem: - super().__init__(dictItem=dictItem) - self.collection = dictItem['collection'] - else: - super().__init__(type, id, bitrate, title, artist, cover, explicit, size) - self.collection = { - 'tracks_gw': tracks_gw, - 'albumAPI': albumAPI, - 'playlistAPI': playlistAPI - } + def __init__(self, obj): + super().__init__(obj) + self.collection = obj['collection'] self.__type__ = "Collection" def toDict(self): @@ -130,15 +100,10 @@ class Collection(IDownloadObject): self.updateProgress(interface) class Convertable(Collection): - def __init__(self, type=None, id=None, bitrate=None, title=None, artist=None, cover=None, explicit=False, size=None, plugin=None, conversion_data=None, dictItem=None): - if dictItem: - super().__init__(dictItem=dictItem) - self.plugin = dictItem['plugin'] - self.conversion_data = dictItem['conversion_data'] - else: - super().__init__(type, id, bitrate, title, artist, cover, explicit, size) - self.plugin = plugin - self.conversion_data = conversion_data + def __init__(self, obj): + super().__init__(obj) + self.plugin = obj['plugin'] + self.conversion_data = obj['conversion_data'] self.__type__ = "Convertable" def toDict(self): diff --git a/deemix/types/Picture.py b/deemix/types/Picture.py index 6cf3d99..859e333 100644 --- a/deemix/types/Picture.py +++ b/deemix/types/Picture.py @@ -14,10 +14,9 @@ class Picture: ) if format.startswith("jpg"): + quality = 80 if '-' in format: quality = format[4:] - else: - quality = 80 format = 'jpg' return url + f'-000000-{quality}-0-0.jpg' if format == 'png': diff --git a/deemix/types/Track.py b/deemix/types/Track.py index e130578..43c814c 100644 --- a/deemix/types/Track.py +++ b/deemix/types/Track.py @@ -1,12 +1,9 @@ -from time import sleep import requests - -import logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('deemix') +from time import sleep from deezer.gw import GWAPIError from deezer.api import APIError + from deemix.utils import removeFeatures, andCommaConcat, removeDuplicateArtists, generateReplayGainString, changeCase from deemix.types.Album import Album @@ -24,7 +21,7 @@ class Track: self.MD5 = "" self.mediaVersion = "" self.duration = 0 - self.fallbackId = "0" + self.fallbackID = "0" self.filesizes = {} self.localTrack = False self.mainArtist = None @@ -46,7 +43,7 @@ class Track: self.searched = False self.selectedFormat = 0 self.singleDownload = False - self.dateString = None + self.dateString = "" self.artistsString = "" self.mainArtistsString = "" self.featArtistsString = "" @@ -61,14 +58,14 @@ class Track: else: raise MD5NotFound self.mediaVersion = trackAPI_gw['MEDIA_VERSION'] - self.fallbackId = "0" + self.fallbackID = "0" if 'FALLBACK' in trackAPI_gw: - self.fallbackId = trackAPI_gw['FALLBACK']['SNG_ID'] + self.fallbackID = trackAPI_gw['FALLBACK']['SNG_ID'] self.localTrack = int(self.id) < 0 def retriveFilesizes(self, dz): + guest_sid = dz.session.cookies.get('sid') try: - guest_sid = dz.session.cookies.get('sid') site = requests.post( "https://api.deezer.com/1.0/gateway.php", params={ @@ -97,8 +94,7 @@ class Track: self.filesizes = filesizes def parseData(self, dz, id=None, trackAPI_gw=None, trackAPI=None, albumAPI_gw=None, albumAPI=None, playlistAPI=None): - if id: - if not trackAPI_gw: trackAPI_gw = dz.gw.get_track_with_fallback(id) + if id and not trackAPI_gw: trackAPI_gw = dz.gw.get_track_with_fallback(id) elif not trackAPI_gw: raise NoDataToParse if not trackAPI: try: trackAPI = dz.api.get_track(trackAPI_gw['SNG_ID']) @@ -110,15 +106,15 @@ class Track: self.parseLocalTrackData(trackAPI_gw) else: self.retriveFilesizes(dz) - self.parseTrackGW(trackAPI_gw) + # Get Lyrics data if not "LYRICS" in trackAPI_gw and self.lyrics.id != "0": try: trackAPI_gw["LYRICS"] = dz.gw.get_track_lyrics(self.id) except GWAPIError: self.lyrics.id = "0" if self.lyrics.id != "0": self.lyrics.parseLyrics(trackAPI_gw["LYRICS"]) - # Parse Album data + # Parse Album Data self.album = Album( id = trackAPI_gw['ALB_ID'], title = trackAPI_gw['ALB_TITLE'], @@ -161,7 +157,7 @@ class Track: if not len(self.artist['Main']): self.artist['Main'] = [self.mainArtist['name']] - self.singleDownload = trackAPI_gw.get('SINGLE_TRACK', False) + self.singleDownload = trackAPI_gw.get('SINGLE_TRACK', False) # TODO: To change self.position = trackAPI_gw.get('POSITION') # Add playlist data if track is in a playlist @@ -184,16 +180,16 @@ class Track: self.artist = { 'Main': [trackAPI_gw['ART_NAME']] } + self.date = Date() self.album.artist = self.artist self.album.artists = self.artists self.album.date = self.date self.album.mainArtist = self.mainArtist - self.date = Date() def parseTrackGW(self, trackAPI_gw): self.title = trackAPI_gw['SNG_TITLE'].strip() - if trackAPI_gw.get('VERSION') and not trackAPI_gw['VERSION'] in trackAPI_gw['SNG_TITLE']: - self.title += " " + trackAPI_gw['VERSION'].strip() + if trackAPI_gw.get('VERSION') and not trackAPI_gw['VERSION'].strip() in this.title: + self.title += f" {trackAPI_gw['VERSION'].strip()}" self.discNumber = trackAPI_gw.get('DISK_NUMBER') self.explicit = bool(int(trackAPI_gw.get('EXPLICIT_LYRICS', "0"))) @@ -215,7 +211,7 @@ class Track: day = trackAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] month = trackAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] year = trackAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] - self.date = Date(year, month, day) + self.date = Date(day, month, year) def parseTrack(self, trackAPI): self.bpm = trackAPI['bpm'] @@ -250,8 +246,8 @@ class Track: return removeFeatures(self.title) def getFeatTitle(self): - if self.featArtistsString and not "(feat." in self.title.lower(): - return self.title + " ({})".format(self.featArtistsString) + if self.featArtistsString and not "feat." in self.title.lower(): + return f"{self.title} ({self.featArtistsString})" return self.title def generateMainFeatStrings(self): -- 2.25.1 From 69c165e2bcc738fd3bcf94e9345daadfa08eb22f Mon Sep 17 00:00:00 2001 From: RemixDev Date: Sat, 10 Apr 2021 11:53:52 +0200 Subject: [PATCH 10/20] Code cleanup with pylint --- .pylintrc | 2 + deemix/__init__.py | 70 +++++----- deemix/__main__.py | 2 +- deemix/decryption.py | 106 +++++---------- deemix/downloader.py | 233 ++++++++++++++++---------------- deemix/itemgen.py | 106 ++++++++------- deemix/settings.py | 16 +-- deemix/types/Album.py | 15 +- deemix/types/Artist.py | 4 +- deemix/types/Date.py | 2 +- deemix/types/DownloadObjects.py | 7 +- deemix/types/Lyrics.py | 6 +- deemix/types/Picture.py | 20 +-- deemix/types/Playlist.py | 4 +- deemix/types/Track.py | 36 ++--- deemix/utils/__init__.py | 34 +++-- deemix/utils/localpaths.py | 3 +- deemix/utils/pathtemplates.py | 7 +- 18 files changed, 323 insertions(+), 350 deletions(-) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..cf11d0f --- /dev/null +++ b/.pylintrc @@ -0,0 +1,2 @@ +[MESSAGES CONTROL] +disable=C0301,C0103,R0902,R0903,C0321,R0911,R0912,R0913,R0914,R0915,R0916 diff --git a/deemix/__init__.py b/deemix/__init__.py index f5caa61..374cd5a 100644 --- a/deemix/__init__.py +++ b/deemix/__init__.py @@ -10,54 +10,54 @@ USER_AGENT_HEADER = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, # Returns the Resolved URL, the Type and the ID def parseLink(link): - if 'deezer.page.link' in link: link = urlopen(url).url # Resolve URL shortner + if 'deezer.page.link' in link: link = urlopen(link).url # Resolve URL shortner # Remove extra stuff if '?' in link: link = link[:link.find('?')] if '&' in link: link = link[:link.find('&')] if link.endswith('/'): link = link[:-1] # Remove last slash if present - type = None - id = None + link_type = None + link_id = None - if not 'deezer' in link: return (link, type, id) # return if not a deezer link + if not 'deezer' in link: return (link, link_type, link_id) # return if not a deezer link if '/track' in link: - type = 'track' - id = re.search("\/track\/(.+)", link).group(1) + link_type = 'track' + link_id = re.search(r"/track/(.+)", link).group(1) elif '/playlist' in link: - type = 'playlist' - id = re.search("\/playlist\/(\d+)", link).group(1) + link_type = 'playlist' + link_id = re.search(r"/playlist/(\d+)", link).group(1) elif '/album' in link: - type = 'album' - id = re.search("\/album\/(.+)", link).group(1) - elif re.search("\/artist\/(\d+)\/top_track", link): - type = 'artist_top' - id = re.search("\/artist\/(\d+)\/top_track", link).group(1) - elif re.search("\/artist\/(\d+)\/discography", link): - type = 'artist_discography' - id = re.search("\/artist\/(\d+)\/discography", link).group(1) + link_type = 'album' + link_id = re.search(r"/album/(.+)", link).group(1) + elif re.search(r"/artist/(\d+)/top_track", link): + link_type = 'artist_top' + link_id = re.search(r"/artist/(\d+)/top_track", link).group(1) + elif re.search(r"/artist/(\d+)/discography", link): + link_type = 'artist_discography' + link_id = re.search(r"/artist/(\d+)/discography", link).group(1) elif '/artist' in link: - type = 'artist' - id = re.search("\/artist\/(\d+)", link).group(1) + link_type = 'artist' + link_id = re.search(r"/artist/(\d+)", link).group(1) - return (link, type, id) + return (link, link_type, link_id) def generateDownloadObject(dz, link, bitrate): - (link, type, id) = parseLink(link) - - if type == None or id == None: return None - - if type == "track": - return generateTrackItem(dz, id, bitrate) - elif type == "album": - return generateAlbumItem(dz, id, bitrate) - elif type == "playlist": - return generatePlaylistItem(dz, id, bitrate) - elif type == "artist": - return generateArtistItem(dz, id, bitrate) - elif type == "artist_discography": - return generateArtistDiscographyItem(dz, id, bitrate) - elif type == "artist_top": - return generateArtistTopItem(dz, id, bitrate) + (link, link_type, link_id) = parseLink(link) + + if link_type is None or link_id is None: + return None + if link_type == "track": + return generateTrackItem(dz, link_id, bitrate) + if link_type == "album": + return generateAlbumItem(dz, link_id, bitrate) + if link_type == "playlist": + return generatePlaylistItem(dz, link_id, bitrate) + if link_type == "artist": + return generateArtistItem(dz, link_id, bitrate) + if link_type == "artist_discography": + return generateArtistDiscographyItem(dz, link_id, bitrate) + if link_type == "artist_top": + return generateArtistTopItem(dz, link_id, bitrate) return None diff --git a/deemix/__main__.py b/deemix/__main__.py index 5602e36..ca7109e 100644 --- a/deemix/__main__.py +++ b/deemix/__main__.py @@ -73,4 +73,4 @@ def download(url, bitrate, portable, path): click.echo("All done!") if __name__ == '__main__': - download() + download() # pylint: disable=E1120 diff --git a/deemix/decryption.py b/deemix/decryption.py index 867cb52..1e71acd 100644 --- a/deemix/decryption.py +++ b/deemix/decryption.py @@ -1,22 +1,24 @@ import binascii +from ssl import SSLError +from time import sleep + +import logging + from Cryptodome.Cipher import Blowfish, AES from Cryptodome.Hash import MD5 -from deemix import USER_AGENT_HEADER -from deemix.types.DownloadObjects import Single, Collection - from requests import get - -from requests.exceptions import ConnectionError, ReadTimeout -from ssl import SSLError +from requests.exceptions import ConnectionError as RequestsConnectionError, ReadTimeout from urllib3.exceptions import SSLError as u3SSLError -import logging +from deemix import USER_AGENT_HEADER +from deemix.types.DownloadObjects import Single + logger = logging.getLogger('deemix') def _md5(data): h = MD5.new() - h.update(str.encode(data) if isinstance(data, str) else data) + h.update(data.encode() if isinstance(data, str) else data) return h.hexdigest() def generateBlowfishKey(trackId): @@ -27,75 +29,35 @@ def generateBlowfishKey(trackId): bfKey += chr(ord(idMd5[i]) ^ ord(idMd5[i + 16]) ^ ord(SECRET[i])) return bfKey -def generateStreamPath(sng_id, md5, media_version, format): +def generateStreamPath(sng_id, md5, media_version, media_format): urlPart = b'\xa4'.join( - [str.encode(md5), str.encode(str(format)), str.encode(str(sng_id)), str.encode(str(media_version))]) + [md5.encode(), str(media_format).encode(), str(sng_id).encode(), str(media_version).encode()]) md5val = _md5(urlPart) - step2 = str.encode(md5val) + b'\xa4' + urlPart + b'\xa4' + step2 = md5val.encode() + b'\xa4' + urlPart + b'\xa4' step2 = step2 + (b'.' * (16 - (len(step2) % 16))) urlPart = binascii.hexlify(AES.new(b'jo6aey6haid2Teih', AES.MODE_ECB).encrypt(step2)) return urlPart.decode("utf-8") def reverseStreamPath(urlPart): step2 = AES.new(b'jo6aey6haid2Teih', AES.MODE_ECB).decrypt(binascii.unhexlify(urlPart.encode("utf-8"))) - (md5val, md5, format, sng_id, media_version, _) = step2.split(b'\xa4') - return (sng_id.decode('utf-8'), md5.decode('utf-8'), media_version.decode('utf-8'), format.decode('utf-8')) + (_, md5, media_format, sng_id, media_version, _) = step2.split(b'\xa4') + return (sng_id.decode('utf-8'), md5.decode('utf-8'), media_version.decode('utf-8'), media_format.decode('utf-8')) -def generateStreamURL(sng_id, md5, media_version, format): - urlPart = generateStreamPath(sng_id, md5, media_version, format) +def generateStreamURL(sng_id, md5, media_version, media_format): + urlPart = generateStreamPath(sng_id, md5, media_version, media_format) return "https://e-cdns-proxy-" + md5[0] + ".dzcdn.net/mobile/1/" + urlPart -def generateUnencryptedStreamURL(sng_id, md5, media_version, format): - urlPart = generateStreamPath(sng_id, md5, media_version, format) +def generateUnencryptedStreamURL(sng_id, md5, media_version, media_format): + urlPart = generateStreamPath(sng_id, md5, media_version, media_format) return "https://e-cdns-proxy-" + md5[0] + ".dzcdn.net/api/1/" + urlPart def reverseStreamURL(url): urlPart = url[url.find("/1/")+3:] - return generateStreamPath(urlPart) - -def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None): - headers= {'User-Agent': USER_AGENT_HEADER} - chunkLength = start - percentage = 0 - - itemName = f"[{track.mainArtist.name} - {track.title}]" - - try: - with get(track.downloadUrl, headers=headers, stream=True, timeout=10) as request: - request.raise_for_status() - - complete = int(request.headers["Content-Length"]) - if complete == 0: raise DownloadEmpty - if start != 0: - responseRange = request.headers["Content-Range"] - logger.info(f'{itemName} downloading range {responseRange}') - else: - logger.info(f'{itemName} downloading {complete} bytes') - - for chunk in request.iter_content(2048 * 3): - outputStream.write(chunk) - chunkLength += len(chunk) - - if downloadObject: - if isinstance(downloadObject, Single): - percentage = (chunkLength / (complete + start)) * 100 - downloadObject.progressNext = percentage - else: - chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 - downloadObject.progressNext += chunkProgres - downloadObject.updateProgress(interface) - - except (SSLError, u3SSLError) as e: - logger.info(f'{itemName} retrying from byte {chunkLength}') - return streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface) - except (ConnectionError, ReadTimeout): - sleep(2) - return streamUnencryptedTrack(outputStream, track, start, downloadObject, interface) + return reverseStreamPath(urlPart) def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None): headers= {'User-Agent': USER_AGENT_HEADER} chunkLength = start - percentage = 0 itemName = f"[{track.mainArtist.name} - {track.title}]" @@ -107,9 +69,9 @@ def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, in if complete == 0: raise DownloadEmpty if start != 0: responseRange = request.headers["Content-Range"] - logger.info(f'{itemName} downloading range {responseRange}') + logger.info('%s downloading range %s', itemName, responseRange) else: - logger.info(f'{itemName} downloading {complete} bytes') + logger.info('%s downloading %s bytes', itemName, complete) for chunk in request.iter_content(2048 * 3): outputStream.write(chunk) @@ -124,12 +86,12 @@ def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, in downloadObject.progressNext += chunkProgres downloadObject.updateProgress(interface) - except (SSLError, u3SSLError) as e: - logger.info(f'{itemName} retrying from byte {chunkLength}') - return streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface) - except (ConnectionError, ReadTimeout): + except (SSLError, u3SSLError): + logger.info('%s retrying from byte %s', itemName, chunkLength) + streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface) + except (RequestsConnectionError, ReadTimeout): sleep(2) - return streamUnencryptedTrack(outputStream, track, start, downloadObject, interface) + streamUnencryptedTrack(outputStream, track, start, downloadObject, interface) def streamTrack(outputStream, track, start=0, downloadObject=None, interface=None): headers= {'User-Agent': USER_AGENT_HEADER} @@ -147,9 +109,9 @@ def streamTrack(outputStream, track, start=0, downloadObject=None, interface=Non if complete == 0: raise DownloadEmpty if start != 0: responseRange = request.headers["Content-Range"] - logger.info(f'{itemName} downloading range {responseRange}') + logger.info('%s downloading range %s', itemName, responseRange) else: - logger.info(f'{itemName} downloading {complete} bytes') + logger.info('%s downloading %s bytes', itemName, complete) for chunk in request.iter_content(2048 * 3): if len(chunk) >= 2048: @@ -167,12 +129,12 @@ def streamTrack(outputStream, track, start=0, downloadObject=None, interface=Non downloadObject.progressNext += chunkProgres downloadObject.updateProgress(interface) - except (SSLError, u3SSLError) as e: - logger.info(f'{itemName} retrying from byte {chunkLength}') - return streamTrack(outputStream, track, chunkLength, downloadObject, interface) - except (ConnectionError, ReadTimeout): + except (SSLError, u3SSLError): + logger.info('%s retrying from byte %s', itemName, chunkLength) + streamTrack(outputStream, track, chunkLength, downloadObject, interface) + except (RequestsConnectionError, ReadTimeout): sleep(2) - return streamTrack(outputStream, track, start, downloadObject, interface) + streamTrack(outputStream, track, start, downloadObject, interface) class DownloadEmpty(Exception): pass diff --git a/deemix/downloader.py b/deemix/downloader.py index 8f6273c..fe72009 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -1,35 +1,33 @@ -import requests -from requests import get - from concurrent.futures import ThreadPoolExecutor from time import sleep from os.path import sep as pathSep +from os import makedirs, system as execute from pathlib import Path from shlex import quote -import re import errno -from ssl import SSLError +import logging +from tempfile import gettempdir + +import requests +from requests import get + from urllib3.exceptions import SSLError as u3SSLError -from os import makedirs +from mutagen.flac import FLACNoHeaderError, error as FLACError + +from deezer import TrackFormats +from deemix import USER_AGENT_HEADER from deemix.types.DownloadObjects import Single, Collection from deemix.types.Track import Track, AlbumDoesntExists from deemix.utils.pathtemplates import generateFilename, generateFilepath, settingsRegexAlbum, settingsRegexArtist, settingsRegexPlaylistFile -from deezer import TrackFormats -from deemix import USER_AGENT_HEADER from deemix.taggers import tagID3, tagFLAC from deemix.decryption import generateUnencryptedStreamURL, streamUnencryptedTrack from deemix.settings import OverwriteOption -from mutagen.flac import FLACNoHeaderError, error as FLACError - -import logging logger = logging.getLogger('deemix') -from tempfile import gettempdir - TEMPDIR = Path(gettempdir()) / 'deemix-imgs' if not TEMPDIR.is_dir(): makedirs(TEMPDIR) @@ -71,23 +69,22 @@ def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): pictureUrl = url[len(urlBase):] pictureSize = int(pictureUrl[:pictureUrl.find("x")]) if pictureSize > 1200: - logger.warn("Couldn't download "+str(pictureSize)+"x"+str(pictureSize)+" image, falling back to 1200x1200") + logger.warning("Couldn't download %sx%s image, falling back to 1200x1200", pictureSize, pictureSize) sleep(1) return downloadImage(urlBase+pictureUrl.replace(str(pictureSize)+"x"+str(pictureSize), '1200x1200'), path, overwrite) - logger.error("Image not found: "+url) + logger.error("Image not found: %s", url) except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError, u3SSLError) as e: - logger.error("Couldn't download Image, retrying in 5 seconds...: "+url+"\n") + logger.error("Couldn't download Image, retrying in 5 seconds...: %s", url) sleep(5) return downloadImage(url, path, overwrite) except OSError as e: - if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") - else: logger.exception(f"Error while downloading an image, you should report this to the developers: {str(e)}") + if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") from e + logger.exception("Error while downloading an image, you should report this to the developers: %s", e) except Exception as e: - logger.exception(f"Error while downloading an image, you should report this to the developers: {str(e)}") + logger.exception("Error while downloading an image, you should report this to the developers: %s", e) if path.is_file(): path.unlink() return None - else: - return path + return path def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectUUID=None, interface=None): if track.localTrack: return TrackFormats.LOCAL @@ -116,36 +113,36 @@ def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectU formats = formats_non_360 for formatNumber, formatName in formats.items(): - if formatNumber <= int(preferredBitrate): - if f"FILESIZE_{formatName}" in track.filesizes: - if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber - if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]: - request = requests.head( - generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), - headers={'User-Agent': USER_AGENT_HEADER}, - timeout=30 - ) - try: - request.raise_for_status() - return formatNumber - except requests.exceptions.HTTPError: # if the format is not available, Deezer returns a 403 error - pass - if not shouldFallback: - raise PreferredBitrateNotFound - else: - if not falledBack: - falledBack = True - logger.info(f"[{track.mainArtist.name} - {track.title}] Fallback to lower bitrate") - if interface and downloadObjectUUID: - interface.send('queueUpdate', { - 'uuid': downloadObjectUUID, - 'bitrateFallback': True, - 'data': { - 'id': track.id, - 'title': track.title, - 'artist': track.mainArtist.name - }, - }) + if formatNumber >= int(preferredBitrate): continue + if f"FILESIZE_{formatName}" in track.filesizes: + if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber + if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]: + request = requests.head( + generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), + headers={'User-Agent': USER_AGENT_HEADER}, + timeout=30 + ) + try: + request.raise_for_status() + return formatNumber + except requests.exceptions.HTTPError: # if the format is not available, Deezer returns a 403 error + pass + + if not shouldFallback: + raise PreferredBitrateNotFound + if not falledBack: + falledBack = True + logger.info("%s Fallback to lower bitrate", f"[{track.mainArtist.name} - {track.title}]") + if interface and downloadObjectUUID: + interface.send('queueUpdate', { + 'uuid': downloadObjectUUID, + 'bitrateFallback': True, + 'data': { + 'id': track.id, + 'title': track.title, + 'artist': track.mainArtist.name + }, + }) if is360format: raise TrackNot360 return TrackFormats.DEFAULT @@ -178,9 +175,11 @@ class Downloader: result = {} if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer") + itemName = f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}]" + # Create Track object if not track: - logger.info(f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}] Getting the tags") + logger.info("%s Getting the tags", itemName) try: track = Track().parseData( dz=self.dz, @@ -189,8 +188,10 @@ class Downloader: albumAPI=albumAPI, playlistAPI=playlistAPI ) - except AlbumDoesntExists: - raise DownloadError('albumDoesntExists') + except AlbumDoesntExists as e: + raise DownloadError('albumDoesntExists') from e + + itemName = f"[{track.mainArtist.name} - {track.title}]" # Check if track not yet encoded if track.MD5 == '': raise DownloadFailed("notEncoded", track) @@ -203,16 +204,16 @@ class Downloader: self.settings['fallbackBitrate'], self.downloadObject.uuid, self.interface ) - except PreferredBitrateNotFound: - raise DownloadFailed("wrongBitrate", track) - except TrackNot360: - raise DownloadFailed("no360RA") + except PreferredBitrateNotFound as e: + raise DownloadFailed("wrongBitrate", track) from e + except TrackNot360 as e: + raise DownloadFailed("no360RA") from e track.selectedFormat = selectedFormat track.album.bitrate = selectedFormat # Generate covers URLs embeddedImageFormat = f'jpg-{self.settings["jpegImageQuality"]}' - if self.settings['embeddedArtworkPNG']: imageFormat = 'png' + if self.settings['embeddedArtworkPNG']: embeddedImageFormat = 'png' track.applySettings(self.settings, TEMPDIR, embeddedImageFormat) @@ -233,49 +234,49 @@ class Downloader: result['filename'] = str(writepath)[len(str(extrasPath))+ len(pathSep):] # Download and cache coverart - logger.info(f"[{track.mainArtist.name} - {track.title}] Getting the album cover") + logger.info("%s Getting the album cover", itemName) track.album.embeddedCoverPath = downloadImage(track.album.embeddedCoverURL, track.album.embeddedCoverPath) # Save local album art if coverPath: result['albumURLs'] = [] - for format in self.settings['localArtworkFormat'].split(","): - if format in ["png","jpg"]: - extendedFormat = format + for pic_format in self.settings['localArtworkFormat'].split(","): + if pic_format in ["png","jpg"]: + extendedFormat = pic_format if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" url = track.album.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) if self.settings['tags']['savePlaylistAsCompilation'] \ and track.playlist \ and track.playlist.pic.staticUrl \ - and not format.startswith("jpg"): - continue - result['albumURLs'].append({'url': url, 'ext': format}) + and not pic_format.startswith("jpg"): + continue + result['albumURLs'].append({'url': url, 'ext': pic_format}) result['albumPath'] = coverPath result['albumFilename'] = f"{settingsRegexAlbum(self.settings['coverImageTemplate'], track.album, self.settings, track.playlist)}" # Save artist art if artistPath: result['artistURLs'] = [] - for format in self.settings['localArtworkFormat'].split(","): - if format in ["png","jpg"]: - extendedFormat = format + for pic_format in self.settings['localArtworkFormat'].split(","): + if pic_format in ["png","jpg"]: + extendedFormat = pic_format if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" url = track.album.mainArtist.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) - if track.album.mainArtist.pic.md5 == "" and not format.startswith("jpg"): continue - result['artistURLs'].append({'url': url, 'ext': format}) + if track.album.mainArtist.pic.md5 == "" and not pic_format.startswith("jpg"): continue + result['artistURLs'].append({'url': url, 'ext': pic_format}) result['artistPath'] = artistPath result['artistFilename'] = f"{settingsRegexArtist(self.settings['artistImageTemplate'], track.album.mainArtist, self.settings, rootArtist=track.album.rootArtist)}" # Save playlist art if track.playlist: - if not len(self.playlistURLs): - for format in self.settings['localArtworkFormat'].split(","): - if format in ["png","jpg"]: - extendedFormat = format + if self.playlistURLs == []: + for pic_format in self.settings['localArtworkFormat'].split(","): + if pic_format in ["png","jpg"]: + extendedFormat = pic_format if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" url = track.playlist.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) - if track.playlist.pic.staticUrl and not format.startswith("jpg"): continue - self.playlistURLs.append({'url': url, 'ext': format}) + if track.playlist.pic.staticUrl and not pic_format.startswith("jpg"): continue + self.playlistURLs.append({'url': url, 'ext': pic_format}) if not self.playlistCoverName: track.playlist.bitrate = selectedFormat track.playlist.dateString = track.playlist.date.format(self.settings['dateFormat']) @@ -309,26 +310,26 @@ class Downloader: writepath = Path(currentFilename) if not trackAlreadyDownloaded or self.settings['overwriteFile'] == OverwriteOption.OVERWRITE: - logger.info(f"[{track.mainArtist.name} - {track.title}] Downloading the track") + logger.info("%s Downloading the track", itemName) track.downloadUrl = generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat) def downloadMusic(track, trackAPI_gw): try: with open(writepath, 'wb') as stream: streamUnencryptedTrack(stream, track, downloadObject=self.downloadObject, interface=self.interface) - except DownloadCancelled: + except DownloadCancelled as e: if writepath.is_file(): writepath.unlink() - raise DownloadCancelled - except (requests.exceptions.HTTPError, DownloadEmpty): + raise e + except (requests.exceptions.HTTPError, DownloadEmpty) as e: if writepath.is_file(): writepath.unlink() if track.fallbackID != "0": - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available, using fallback id") + logger.warning("%s Track not available, using fallback id", itemName) newTrack = self.dz.gw.get_track_with_fallback(track.fallbackID) track.parseEssentialData(newTrack) track.retriveFilesizes(self.dz) return False - elif not track.searched and self.settings['fallbackSearch']: - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available, searching for alternative") + if not track.searched and self.settings['fallbackSearch']: + logger.warning("%s Track not available, searching for alternative", itemName) searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) if searchedId != "0": newTrack = self.dz.gw.get_track_with_fallback(searchedId) @@ -346,25 +347,21 @@ class Downloader: }, }) return False - else: - raise DownloadFailed("notAvailableNoAlternative") - else: - raise DownloadFailed("notAvailable") + raise DownloadFailed("notAvailableNoAlternative") from e + raise DownloadFailed("notAvailable") from e except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e: if writepath.is_file(): writepath.unlink() - logger.warn(f"[{track.mainArtist.name} - {track.title}] Error while downloading the track, trying again in 5s...") + logger.warning("%s Error while downloading the track, trying again in 5s...", itemName) sleep(5) return downloadMusic(track, trackAPI_gw) except OSError as e: - if e.errno == errno.ENOSPC: - raise DownloadFailed("noSpaceLeft") - else: - if writepath.is_file(): writepath.unlink() - logger.exception(f"[{track.mainArtist.name} - {track.title}] Error while downloading the track, you should report this to the developers: {str(e)}") - raise e + if writepath.is_file(): writepath.unlink() + if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") from e + logger.exception("%s Error while downloading the track, you should report this to the developers: %s", itemName, e) + raise e except Exception as e: if writepath.is_file(): writepath.unlink() - logger.exception(f"[{track.mainArtist.name} - {track.title}] Error while downloading the track, you should report this to the developers: {str(e)}") + logger.exception("%s Error while downloading the track, you should report this to the developers: %s", itemName, e) raise e return True @@ -375,12 +372,12 @@ class Downloader: if not trackDownloaded: return self.download(trackAPI_gw, track=track) else: - logger.info(f"[{track.mainArtist.name} - {track.title}] Skipping track as it's already downloaded") + logger.info("%s Skipping track as it's already downloaded", itemName) self.downloadObject.completeTrackProgress(self.interface) # Adding tags if (not trackAlreadyDownloaded or self.settings['overwriteFile'] in [OverwriteOption.ONLY_TAGS, OverwriteOption.OVERWRITE]) and not track.localTrack: - logger.info(f"[{track.mainArtist.name} - {track.title}] Applying tags to the track") + logger.info("%s Applying tags to the track", itemName) if track.selectedFormat in [TrackFormats.MP3_320, TrackFormats.MP3_128, TrackFormats.DEFAULT]: tagID3(writepath, track, self.settings['tags']) elif track.selectedFormat == TrackFormats.FLAC: @@ -388,14 +385,14 @@ class Downloader: tagFLAC(writepath, track, self.settings['tags']) except (FLACNoHeaderError, FLACError): if writepath.is_file(): writepath.unlink() - logger.warn(f"[{track.mainArtist.name} - {track.title}] Track not available in FLAC, falling back if necessary") + logger.warning("%s Track not available in FLAC, falling back if necessary", itemName) self.downloadObject.removeTrackProgress(self.interface) track.filesizes['FILESIZE_FLAC'] = "0" track.filesizes['FILESIZE_FLAC_TESTED'] = True return self.download(trackAPI_gw, track=track) if track.searched: result['searched'] = f"{track.mainArtist.name} - {track.title}" - logger.info(f"[{track.mainArtist.name} - {track.title}] Track download completed\n{str(writepath)}") + logger.info("%s Track download completed\n%s", itemName, writepath) self.downloadObject.downloaded += 1 self.downloadObject.files.append(str(writepath)) self.downloadObject.extrasPath = str(self.extrasPath) @@ -413,19 +410,21 @@ class Downloader: if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: tempTrack['title'] += f" {trackAPI_gw['VERSION']}".strip() + itemName = f"[{track.mainArtist.name} - {track.title}]" + try: result = self.download(trackAPI_gw, trackAPI, albumAPI, playlistAPI, track) except DownloadFailed as error: if error.track: track = error.track if track.fallbackID != "0": - logger.warn(f"[{track.mainArtist.name} - {track.title}] {error.message} Using fallback id") + logger.warning("%s %s Using fallback id", itemName, error.message) newTrack = self.dz.gw.get_track_with_fallback(track.fallbackID) track.parseEssentialData(newTrack) track.retriveFilesizes(self.dz) return self.downloadWrapper(trackAPI_gw, trackAPI, albumAPI, playlistAPI, track) - elif not track.searched and self.settings['fallbackSearch']: - logger.warn(f"[{track.mainArtist.name} - {track.title}] {error.message} Searching for alternative") + if not track.searched and self.settings['fallbackSearch']: + logger.warning("%s %s Searching for alternative", itemName, error.message) searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) if searchedId != "0": newTrack = self.dz.gw.get_track_with_fallback(searchedId) @@ -434,7 +433,7 @@ class Downloader: track.searched = True if self.interface: self.interface.send('queueUpdate', { - 'uuid': self.queueItem.uuid, + 'uuid': self.downloadObject.uuid, 'searchFallback': True, 'data': { 'id': track.id, @@ -443,17 +442,16 @@ class Downloader: }, }) return self.downloadWrapper(trackAPI_gw, trackAPI, albumAPI, playlistAPI, track) - else: - error.errid += "NoAlternative" - error.message = errorMessages[error.errid] - logger.error(f"[{tempTrack['artist']} - {tempTrack['title']}] {error.message}") + error.errid += "NoAlternative" + error.message = errorMessages[error.errid] + logger.error("%s %s", itemName, error.message) result = {'error': { - 'message': error.message, - 'errid': error.errid, - 'data': tempTrack - }} + 'message': error.message, + 'errid': error.errid, + 'data': tempTrack + }} except Exception as e: - logger.exception(f"[{tempTrack['artist']} - {tempTrack['title']}] {str(e)}") + logger.exception("%s %s", itemName, e) result = {'error': { 'message': str(e), 'data': tempTrack @@ -505,9 +503,9 @@ class Downloader: errors = "" searched = "" - for i in range(len(tracks)): + for i in enumerate(tracks): result = tracks[i].result() - if not result: return None # Check if item is cancelled + if not result: return # Check if item is cancelled # Log errors to file if result.get('error'): @@ -558,10 +556,10 @@ class Downloader: class DownloadError(Exception): """Base class for exceptions in this module.""" - pass class DownloadFailed(DownloadError): def __init__(self, errid, track=None): + super().__init__() self.errid = errid self.message = errorMessages[self.errid] self.track = track @@ -569,6 +567,9 @@ class DownloadFailed(DownloadError): class DownloadCancelled(DownloadError): pass +class DownloadEmpty(DownloadError): + pass + class PreferredBitrateNotFound(DownloadError): pass diff --git a/deemix/itemgen.py b/deemix/itemgen.py index 6087d2c..3cf9e5e 100644 --- a/deemix/itemgen.py +++ b/deemix/itemgen.py @@ -1,9 +1,15 @@ +import logging + from deemix.types.DownloadObjects import Single, Collection +from deezer.utils import map_user_playlist from deezer.api import APIError from deezer.gw import GWAPIError, LyricsStatus +logger = logging.getLogger('deemix') + class GenerationError(Exception): def __init__(self, link, message, errid=None): + super().__init__() self.link = link self.message = message self.errid = errid @@ -15,27 +21,26 @@ class GenerationError(Exception): 'errid': self.errid } -def generateTrackItem(dz, id, bitrate, trackAPI=None, albumAPI=None): +def generateTrackItem(dz, link_id, bitrate, trackAPI=None, albumAPI=None): # Check if is an isrc: url - if str(id).startswith("isrc"): + if str(link_id).startswith("isrc"): try: - trackAPI = dz.api.get_track(id) + trackAPI = dz.api.get_track(link_id) except APIError as e: - e = str(e) - raise GenerationError("https://deezer.com/track/"+str(id), f"Wrong URL: {e}") + raise GenerationError("https://deezer.com/track/"+str(link_id), f"Wrong URL: {e}") from e if 'id' in trackAPI and 'title' in trackAPI: - id = trackAPI['id'] + link_id = trackAPI['id'] else: - raise GenerationError("https://deezer.com/track/"+str(id), "Track ISRC is not available on deezer", "ISRCnotOnDeezer") + raise GenerationError("https://deezer.com/track/"+str(link_id), "Track ISRC is not available on deezer", "ISRCnotOnDeezer") # Get essential track info try: - trackAPI_gw = dz.gw.get_track_with_fallback(id) + trackAPI_gw = dz.gw.get_track_with_fallback(link_id) except GWAPIError as e: - e = str(e) message = "Wrong URL" - if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}" - raise GenerationError("https://deezer.com/track/"+str(id), message) + # TODO: FIX + # if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}" + raise GenerationError("https://deezer.com/track/"+str(link_id), message) from e title = trackAPI_gw['SNG_TITLE'].strip() if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: @@ -44,7 +49,7 @@ def generateTrackItem(dz, id, bitrate, trackAPI=None, albumAPI=None): return Single({ 'type': 'track', - 'id': id, + 'id': link_id, 'bitrate': bitrate, 'title': title, 'artist': trackAPI_gw['ART_NAME'], @@ -57,19 +62,18 @@ def generateTrackItem(dz, id, bitrate, trackAPI=None, albumAPI=None): } }) -def generateAlbumItem(dz, id, bitrate, rootArtist=None): +def generateAlbumItem(dz, link_id, bitrate, rootArtist=None): # Get essential album info try: - albumAPI = dz.api.get_album(id) + albumAPI = dz.api.get_album(link_id) except APIError as e: - e = str(e) - raise GenerationError("https://deezer.com/album/"+str(id), f"Wrong URL: {e}") + raise GenerationError("https://deezer.com/album/"+str(link_id), f"Wrong URL: {e}") from e - if str(id).startswith('upc'): id = albumAPI['id'] + if str(link_id).startswith('upc'): link_id = albumAPI['id'] # Get extra info about album # This saves extra api calls when downloading - albumAPI_gw = dz.gw.get_album(id) + albumAPI_gw = dz.gw.get_album(link_id) albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK'] albumAPI['copyright'] = albumAPI_gw['COPYRIGHT'] albumAPI['root_artist'] = rootArtist @@ -78,9 +82,9 @@ def generateAlbumItem(dz, id, bitrate, rootArtist=None): if albumAPI['nb_tracks'] == 1: return generateTrackItem(dz, albumAPI['tracks']['data'][0]['id'], bitrate, albumAPI=albumAPI) - tracksArray = dz.gw.get_album_tracks(id) + tracksArray = dz.gw.get_album_tracks(link_id) - if albumAPI['cover_small'] != None: + if albumAPI['cover_small'] is not None: cover = albumAPI['cover_small'][:-24] + '/75x75-000000-80-0-0.jpg' else: cover = f"https://e-cdns-images.dzcdn.net/images/cover/{albumAPI_gw['ALB_PICTURE']}/75x75-000000-80-0-0.jpg" @@ -97,7 +101,7 @@ def generateAlbumItem(dz, id, bitrate, rootArtist=None): return Collection({ 'type': 'album', - 'id': id, + 'id': link_id, 'bitrate': bitrate, 'title': albumAPI['title'], 'artist': albumAPI['artist']['name'], @@ -110,32 +114,31 @@ def generateAlbumItem(dz, id, bitrate, rootArtist=None): } }) -def generatePlaylistItem(dz, id, bitrate, playlistAPI=None, playlistTracksAPI=None): +def generatePlaylistItem(dz, link_id, bitrate, playlistAPI=None, playlistTracksAPI=None): if not playlistAPI: # Get essential playlist info try: - playlistAPI = dz.api.get_playlist(id) - except: + playlistAPI = dz.api.get_playlist(link_id) + except APIError: playlistAPI = None # Fallback to gw api if the playlist is private if not playlistAPI: try: - userPlaylist = dz.gw.get_playlist_page(id) + userPlaylist = dz.gw.get_playlist_page(link_id) playlistAPI = map_user_playlist(userPlaylist['DATA']) except GWAPIError as e: - e = str(e) message = "Wrong URL" - if "DATA_ERROR" in e: - message += f": {e['DATA_ERROR']}" - raise GenerationError("https://deezer.com/playlist/"+str(id), message) + # TODO: FIX + # if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}" + raise GenerationError("https://deezer.com/playlist/"+str(link_id), message) from e # Check if private playlist and owner if not playlistAPI.get('public', False) and playlistAPI['creator']['id'] != str(dz.current_user['id']): logger.warning("You can't download others private playlists.") - raise GenerationError("https://deezer.com/playlist/"+str(id), "You can't download others private playlists.", "notYourPrivatePlaylist") + raise GenerationError("https://deezer.com/playlist/"+str(link_id), "You can't download others private playlists.", "notYourPrivatePlaylist") if not playlistTracksAPI: - playlistTracksAPI = dz.gw.get_playlist_tracks(id) + playlistTracksAPI = dz.gw.get_playlist_tracks(link_id) playlistAPI['various_artist'] = dz.api.get_artist(5080) # Useful for save as compilation totalSize = len(playlistTracksAPI) @@ -148,11 +151,11 @@ def generatePlaylistItem(dz, id, bitrate, playlistAPI=None, playlistTracksAPI=No trackAPI['SIZE'] = totalSize collection.append(trackAPI) - if not 'explicit' in playlistAPI: playlistAPI['explicit'] = False + if 'explicit' not in playlistAPI: playlistAPI['explicit'] = False return Collection({ 'type': 'playlist', - 'id': id, + 'id': link_id, 'bitrate': bitrate, 'title': playlistAPI['title'], 'artist': playlistAPI['creator']['name'], @@ -165,60 +168,59 @@ def generatePlaylistItem(dz, id, bitrate, playlistAPI=None, playlistTracksAPI=No } }) -def generateArtistItem(dz, id, bitrate, interface=None): +def generateArtistItem(dz, link_id, bitrate, interface=None): # Get essential artist info try: - artistAPI = dz.api.get_artist(id) + artistAPI = dz.api.get_artist(link_id) except APIError as e: - e = str(e) - raise GenerationError("https://deezer.com/artist/"+str(id), f"Wrong URL: {e}") + raise GenerationError("https://deezer.com/artist/"+str(link_id), f"Wrong URL: {e}") from e - if interface: interface.send("startAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) rootArtist = { 'id': artistAPI['id'], 'name': artistAPI['name'] } + if interface: interface.send("startAddingArtist", rootArtist) - artistDiscographyAPI = dz.gw.get_artist_discography_tabs(id, 100) + artistDiscographyAPI = dz.gw.get_artist_discography_tabs(link_id, 100) allReleases = artistDiscographyAPI.pop('all', []) albumList = [] for album in allReleases: albumList.append(generateAlbumItem(dz, album['id'], bitrate, rootArtist=rootArtist)) - if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) + if interface: interface.send("finishAddingArtist", rootArtist) return albumList -def generateArtistDiscographyItem(dz, id, bitrate, interface=None): +def generateArtistDiscographyItem(dz, link_id, bitrate, interface=None): # Get essential artist info try: - artistAPI = dz.api.get_artist(id) + artistAPI = dz.api.get_artist(link_id) except APIError as e: e = str(e) - raise GenerationError("https://deezer.com/artist/"+str(id)+"/discography", f"Wrong URL: {e}") + raise GenerationError("https://deezer.com/artist/"+str(link_id)+"/discography", f"Wrong URL: {e}") - if interface: interface.send("startAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) rootArtist = { 'id': artistAPI['id'], 'name': artistAPI['name'] } + if interface: interface.send("startAddingArtist", rootArtist) - artistDiscographyAPI = dz.gw.get_artist_discography_tabs(id, 100) + artistDiscographyAPI = dz.gw.get_artist_discography_tabs(link_id, 100) artistDiscographyAPI.pop('all', None) # all contains albums and singles, so its all duplicates. This removes them albumList = [] - for type in artistDiscographyAPI: - for album in artistDiscographyAPI[type]: + for releaseType in artistDiscographyAPI: + for album in artistDiscographyAPI[releaseType]: albumList.append(generateAlbumItem(dz, album['id'], bitrate, rootArtist=rootArtist)) - if interface: interface.send("finishAddingArtist", {'name': artistAPI['name'], 'id': artistAPI['id']}) + if interface: interface.send("finishAddingArtist", rootArtist) return albumList -def generateArtistTopItem(dz, id, bitrate, interface=None): +def generateArtistTopItem(dz, link_id, bitrate, interface=None): # Get essential artist info try: - artistAPI = dz.api.get_artist(id) + artistAPI = dz.api.get_artist(link_id) except APIError as e: e = str(e) - raise GenerationError("https://deezer.com/artist/"+str(id)+"/top_track", f"Wrong URL: {e}") + raise GenerationError("https://deezer.com/artist/"+str(link_id)+"/top_track", f"Wrong URL: {e}") # Emulate the creation of a playlist # Can't use generatePlaylistItem directly as this is not a real playlist @@ -250,5 +252,5 @@ def generateArtistTopItem(dz, id, bitrate, interface=None): 'type': "playlist" } - artistTopTracksAPI_gw = dz.gw.get_artist_toptracks(id) + artistTopTracksAPI_gw = dz.gw.get_artist_toptracks(link_id) return generatePlaylistItem(dz, playlistAPI['id'], bitrate, playlistAPI=playlistAPI, playlistTracksAPI=artistTopTracksAPI_gw) diff --git a/deemix/settings.py b/deemix/settings.py index 9d3993f..fd46656 100644 --- a/deemix/settings.py +++ b/deemix/settings.py @@ -4,16 +4,16 @@ from os import makedirs from deezer import TrackFormats import deemix.utils.localpaths as localpaths -"""Should the lib overwrite files?""" class OverwriteOption(): + """Should the lib overwrite files?""" OVERWRITE = 'y' # Yes, overwrite the file DONT_OVERWRITE = 'n' # No, don't overwrite the file DONT_CHECK_EXT = 'e' # No, and don't check for extensions KEEP_BOTH = 'b' # No, and keep both files ONLY_TAGS = 't' # Overwrite only the tags -"""What should I do with featured artists?""" class FeaturesOption(): + """What should I do with featured artists?""" NO_CHANGE = "0" # Do nothing REMOVE_TITLE = "1" # Remove from track title REMOVE_TITLE_ALBUM = "3" # Remove from track title and album title @@ -121,13 +121,13 @@ def loadSettings(configFolder=None): def checkSettings(settings): changes = 0 - for set in DEFAULTS: - if not set in settings or type(settings[set]) != type(DEFAULTS[set]): - settings[set] = DEFAULTS[set] + for i_set in DEFAULTS: + if not i_set in settings or not isinstance(settings[i_set], DEFAULTS[i_set]): + settings[i_set] = DEFAULTS[i_set] changes += 1 - for set in DEFAULTS['tags']: - if not set in settings['tags'] or type(settings['tags'][set]) != type(DEFAULTS['tags'][set]): - settings['tags'][set] = DEFAULTS['tags'][set] + for i_set in DEFAULTS['tags']: + if not i_set in settings['tags'] or not isinstance(settings['tags'][i_set], DEFAULTS['tags'][i_set]): + settings['tags'][i_set] = DEFAULTS['tags'][i_set] changes += 1 if settings['downloadLocation'] == "": settings['downloadLocation'] = DEFAULTS['downloadLocation'] diff --git a/deemix/types/Album.py b/deemix/types/Album.py index 5c7d7f9..9161aa5 100644 --- a/deemix/types/Album.py +++ b/deemix/types/Album.py @@ -7,8 +7,8 @@ from deemix.types.Picture import Picture from deemix.types import VARIOUS_ARTISTS class Album: - def __init__(self, id="0", title="", pic_md5=""): - self.id = id + def __init__(self, alb_id="0", title="", pic_md5=""): + self.id = alb_id self.title = title self.pic = Picture(md5=pic_md5, type="cover") self.artist = {"Main": []} @@ -24,11 +24,15 @@ class Album: self.genre = [] self.barcode = "Unknown" self.label = "Unknown" + self.copyright = None self.recordType = "album" self.bitrate = 0 self.rootArtist = None self.variousArtists = None + self.playlistId = None + self.owner = None + def parseAlbum(self, albumAPI): self.title = albumAPI['title'] @@ -80,7 +84,7 @@ class Album: day = albumAPI["release_date"][8:10] month = albumAPI["release_date"][5:7] year = albumAPI["release_date"][0:4] - self.date = Date(year, month, day) + self.date = Date(day, month, year) self.discTotal = albumAPI.get('nb_disk') self.copyright = albumAPI.get('copyright') @@ -115,7 +119,7 @@ class Album: day = albumAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] month = albumAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] year = albumAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] - self.date = Date(year, month, day) + self.date = Date(day, month, year) def makePlaylistCompilation(self, playlist): self.variousArtists = playlist.variousArtists @@ -136,8 +140,9 @@ class Album: self.pic = playlist.pic def removeDuplicateArtists(self): + """Removes duplicate artists for both artist array and artists dict""" (self.artist, self.artists) = removeDuplicateArtists(self.artist, self.artists) - # Removes featuring from the album name def getCleanTitle(self): + """Removes featuring from the album name""" return removeFeatures(self.title) diff --git a/deemix/types/Artist.py b/deemix/types/Artist.py index 2e0bb1b..576dc0a 100644 --- a/deemix/types/Artist.py +++ b/deemix/types/Artist.py @@ -2,8 +2,8 @@ from deemix.types.Picture import Picture from deemix.types import VARIOUS_ARTISTS class Artist: - def __init__(self, id="0", name="", role="", pic_md5=""): - self.id = str(id) + def __init__(self, art_id="0", name="", role="", pic_md5=""): + self.id = str(art_id) self.name = name self.pic = Picture(md5=pic_md5, type="artist") self.role = role diff --git a/deemix/types/Date.py b/deemix/types/Date.py index 061f2eb..196612c 100644 --- a/deemix/types/Date.py +++ b/deemix/types/Date.py @@ -1,4 +1,4 @@ -class Date(object): +class Date: def __init__(self, day="00", month="00", year="XXXX"): self.year = year self.month = month diff --git a/deemix/types/DownloadObjects.py b/deemix/types/DownloadObjects.py index c0e6736..d725561 100644 --- a/deemix/types/DownloadObjects.py +++ b/deemix/types/DownloadObjects.py @@ -1,4 +1,5 @@ class IDownloadObject: + """DownloadObject interface""" def __init__(self, obj): self.type = obj['type'] self.id = obj['id'] @@ -50,9 +51,9 @@ class IDownloadObject: def getSlimmedDict(self): light = self.toDict() propertiesToDelete = ['single', 'collection', 'convertable'] - for property in propertiesToDelete: - if property in light: - del light[property] + for prop in propertiesToDelete: + if prop in light: + del light[prop] return light def updateProgress(self, interface=None): diff --git a/deemix/types/Lyrics.py b/deemix/types/Lyrics.py index 8a02a4c..938e4da 100644 --- a/deemix/types/Lyrics.py +++ b/deemix/types/Lyrics.py @@ -1,6 +1,6 @@ class Lyrics: - def __init__(self, id="0"): - self.id = id + def __init__(self, lyr_id="0"): + self.id = lyr_id self.sync = "" self.unsync = "" self.syncID3 = [] @@ -11,7 +11,7 @@ class Lyrics: syncLyricsJson = lyricsAPI["LYRICS_SYNC_JSON"] timestamp = "" milliseconds = 0 - for line in range(len(syncLyricsJson)): + for line in enumerate(syncLyricsJson): if syncLyricsJson[line]["line"] != "": timestamp = syncLyricsJson[line]["lrc_timestamp"] milliseconds = int(syncLyricsJson[line]["milliseconds"]) diff --git a/deemix/types/Picture.py b/deemix/types/Picture.py index 859e333..1488dd1 100644 --- a/deemix/types/Picture.py +++ b/deemix/types/Picture.py @@ -1,25 +1,25 @@ class Picture: - def __init__(self, md5="", type="", url=None): + def __init__(self, md5="", pic_type="", url=None): self.md5 = md5 - self.type = type + self.type = pic_type self.staticUrl = url - def generatePictureURL(self, size, format): + def generatePictureURL(self, size, pic_format): if self.staticUrl: return self.staticUrl - url = "https://e-cdns-images.dzcdn.net/images/{}/{}/{}x{}".format( + url = "https://e-cdns-images.dzcdn.net/images/{}/{}/{size}x{size}".format( self.type, self.md5, - size, size + size=size ) - if format.startswith("jpg"): + if pic_format.startswith("jpg"): quality = 80 - if '-' in format: - quality = format[4:] - format = 'jpg' + if '-' in pic_format: + quality = pic_format[4:] + pic_format = 'jpg' return url + f'-000000-{quality}-0-0.jpg' - if format == 'png': + if pic_format == 'png': return url + '-none-100-0-0.png' return url+'.jpg' diff --git a/deemix/types/Playlist.py b/deemix/types/Playlist.py index 61412ee..9d85455 100644 --- a/deemix/types/Playlist.py +++ b/deemix/types/Playlist.py @@ -32,7 +32,7 @@ class Playlist: md5 = url[url.find(picType+'/') + len(picType)+1:-24] self.pic = Picture( md5 = md5, - type = picType + pic_type = picType ) else: self.pic = Picture(url = playlistAPI['picture_xl']) @@ -41,7 +41,7 @@ class Playlist: pic_md5 = playlistAPI['various_artist']['picture_small'] pic_md5 = pic_md5[pic_md5.find('artist/') + 7:-24] self.variousArtists = Artist( - id = playlistAPI['various_artist']['id'], + art_id = playlistAPI['various_artist']['id'], name = playlistAPI['various_artist']['name'], role = "Main", pic_md5 = pic_md5 diff --git a/deemix/types/Track.py b/deemix/types/Track.py index 43c814c..d7fb13f 100644 --- a/deemix/types/Track.py +++ b/deemix/types/Track.py @@ -1,5 +1,6 @@ -import requests from time import sleep +import re +import requests from deezer.gw import GWAPIError from deezer.api import APIError @@ -14,9 +15,11 @@ from deemix.types.Playlist import Playlist from deemix.types.Lyrics import Lyrics from deemix.types import VARIOUS_ARTISTS +from deemix.settings import FeaturesOption + class Track: - def __init__(self, id="0", name=""): - self.id = id + def __init__(self, sng_id="0", name=""): + self.id = sng_id self.title = name self.MD5 = "" self.mediaVersion = "" @@ -82,9 +85,9 @@ class Track: result_json = site.json() except: sleep(2) - return self.retriveFilesizes(dz) + self.retriveFilesizes(dz) if len(result_json['error']): - raise APIError(json.dumps(result_json['error'])) + raise APIError(result_json.dumps(result_json['error'])) response = result_json.get("results") filesizes = {} for key, value in response.items(): @@ -116,7 +119,7 @@ class Track: # Parse Album Data self.album = Album( - id = trackAPI_gw['ALB_ID'], + alb_id = trackAPI_gw['ALB_ID'], title = trackAPI_gw['ALB_TITLE'], pic_md5 = trackAPI_gw.get('ALB_PICTURE') ) @@ -157,7 +160,7 @@ class Track: if not len(self.artist['Main']): self.artist['Main'] = [self.mainArtist['name']] - self.singleDownload = trackAPI_gw.get('SINGLE_TRACK', False) # TODO: To change + self.singleDownload = trackAPI_gw.get('SINGLE_TRACK', False) # TODO: Change self.position = trackAPI_gw.get('POSITION') # Add playlist data if track is in a playlist @@ -173,7 +176,7 @@ class Track: self.album = Album(title=trackAPI_gw['ALB_TITLE']) self.album.pic = Picture( md5 = trackAPI_gw.get('ALB_PICTURE', ""), - type = "cover" + pic_type = "cover" ) self.mainArtist = Artist(name=trackAPI_gw['ART_NAME']) self.artists = [trackAPI_gw['ART_NAME']] @@ -188,7 +191,7 @@ class Track: def parseTrackGW(self, trackAPI_gw): self.title = trackAPI_gw['SNG_TITLE'].strip() - if trackAPI_gw.get('VERSION') and not trackAPI_gw['VERSION'].strip() in this.title: + if trackAPI_gw.get('VERSION') and not trackAPI_gw['VERSION'].strip() in self.title: self.title += f" {trackAPI_gw['VERSION'].strip()}" self.discNumber = trackAPI_gw.get('DISK_NUMBER') @@ -202,7 +205,7 @@ class Track: self.lyrics = Lyrics(trackAPI_gw.get('LYRICS_ID', "0")) self.mainArtist = Artist( - id = trackAPI_gw['ART_ID'], + art_id = trackAPI_gw['ART_ID'], name = trackAPI_gw['ART_NAME'], pic_md5 = trackAPI_gw.get('ART_PICTURE') ) @@ -257,7 +260,6 @@ class Track: self.featArtistsString = "feat. "+andCommaConcat(self.artist['Featured']) def applySettings(self, settings, TEMPDIR, embeddedImageFormat): - from deemix.settings import FeaturesOption # Check if should save the playlist as a compilation if self.playlist and settings['tags']['savePlaylistAsCompilation']: @@ -269,7 +271,8 @@ class Track: ext = self.album.embeddedCoverURL[-4:] if ext[0] != ".": ext = ".jpg" # Check for Spotify images - self.album.embeddedCoverPath = TEMPDIR / f"pl{trackAPI_gw['_EXTRA_PLAYLIST']['id']}_{settings['embeddedArtworkSize']}{ext}" + # TODO: FIX + # self.album.embeddedCoverPath = TEMPDIR / f"pl{trackAPI_gw['_EXTRA_PLAYLIST']['id']}_{settings['embeddedArtworkSize']}{ext}" else: if self.album.date: self.date = self.album.date self.album.embeddedCoverURL = self.album.pic.generatePictureURL(settings['embeddedArtworkSize'], embeddedImageFormat) @@ -290,7 +293,7 @@ class Track: self.album.artists.insert(0, artist.name) if isMainArtist or artist.name not in self.album.artist['Main'] and not isMainArtist: - if not artist.role in self.album.artist: + if artist.role not in self.album.artist: self.album.artist[artist.role] = [] self.album.artist[artist.role].insert(0, artist.name) self.album.mainArtist.save = not self.album.mainArtist.isVariousArtists() or settings['albumVariousArtists'] and self.album.mainArtist.isVariousArtists() @@ -319,9 +322,9 @@ class Track: self.mainArtist.name = changeCase(self.mainArtist.name, settings['artistCasing']) for i, artist in enumerate(self.artists): self.artists[i] = changeCase(artist, settings['artistCasing']) - for type in self.artist: - for i, artist in enumerate(self.artist[type]): - self.artist[type][i] = changeCase(artist, settings['artistCasing']) + for art_type in self.artist: + for i, artist in enumerate(self.artist[art_type]): + self.artist[art_type][i] = changeCase(artist, settings['artistCasing']) self.generateMainFeatStrings() # Generate artist tag @@ -343,7 +346,6 @@ class Track: class TrackError(Exception): """Base class for exceptions in this module.""" - pass class AlbumDoesntExists(TrackError): pass diff --git a/deemix/utils/__init__.py b/deemix/utils/__init__.py index 8f67ace..48fa9eb 100644 --- a/deemix/utils/__init__.py +++ b/deemix/utils/__init__.py @@ -9,30 +9,28 @@ def getBitrateNumberFromText(txt): txt = str(txt).lower() if txt in ['flac', 'lossless', '9']: return TrackFormats.FLAC - elif txt in ['mp3', '320', '3']: + if txt in ['mp3', '320', '3']: return TrackFormats.MP3_320 - elif txt in ['128', '1']: + if txt in ['128', '1']: return TrackFormats.MP3_128 - elif txt in ['360', '360_hq', '15']: + if txt in ['360', '360_hq', '15']: return TrackFormats.MP4_RA3 - elif txt in ['360_mq', '14']: + if txt in ['360_mq', '14']: return TrackFormats.MP4_RA2 - elif txt in ['360_lq', '13']: + if txt in ['360_lq', '13']: return TrackFormats.MP4_RA1 - else: - return None + return None -def changeCase(str, type): - if type == "lower": - return str.lower() - elif type == "upper": - return str.upper() - elif type == "start": - return string.capwords(str) - elif type == "sentence": - return str.capitalize() - else: - return str +def changeCase(txt, case_type): + if case_type == "lower": + return txt.lower() + if case_type == "upper": + return txt.upper() + if case_type == "start": + return string.capwords(txt) + if case_type == "sentence": + return txt.capitalize() + return str def removeFeatures(title): clean = title diff --git a/deemix/utils/localpaths.py b/deemix/utils/localpaths.py index e9a39b0..4250776 100644 --- a/deemix/utils/localpaths.py +++ b/deemix/utils/localpaths.py @@ -1,6 +1,8 @@ from pathlib import Path import sys import os +if os.name == 'nt': + import winreg # pylint: disable=E0401 homedata = Path.home() userdata = "" @@ -23,7 +25,6 @@ if os.getenv("DEEMIX_MUSIC_DIR"): elif os.getenv("XDG_MUSIC_DIR"): musicdata = Path(os.getenv("XDG_MUSIC_DIR")) / "deemix Music" elif os.name == 'nt': - import winreg sub_key = r'SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders' music_guid = '{4BD8D571-6D19-48D3-BE97-422220080E43}' with winreg.OpenKey(winreg.HKEY_CURRENT_USER, sub_key) as key: diff --git a/deemix/utils/pathtemplates.py b/deemix/utils/pathtemplates.py index 3d5dafe..9e48b8a 100644 --- a/deemix/utils/pathtemplates.py +++ b/deemix/utils/pathtemplates.py @@ -52,17 +52,16 @@ def antiDot(string): return string -def pad(num, max, settings): +def pad(num, max_val, settings): if int(settings['paddingSize']) == 0: - paddingSize = len(str(max)) + paddingSize = len(str(max_val)) else: paddingSize = len(str(10 ** (int(settings['paddingSize']) - 1))) if paddingSize == 1: paddingSize = 2 if settings['padTracks']: return str(num).zfill(paddingSize) - else: - return str(num) + return str(num) def generateFilename(track, settings, template): filename = template or "%artist% - %title%" -- 2.25.1 From 224a62aad2881b63e05c4a75921f24c8c2efde5e Mon Sep 17 00:00:00 2001 From: RemixDev Date: Mon, 7 Jun 2021 20:25:51 +0200 Subject: [PATCH 11/20] Code parity with deemix-js --- deemix/__init__.py | 32 ++- deemix/__main__.py | 4 +- deemix/decryption.py | 104 ++++--- deemix/downloader.py | 473 +++++++++++++++---------------- deemix/itemgen.py | 140 +++++---- deemix/settings.py | 12 +- deemix/{taggers.py => tagger.py} | 10 +- deemix/types/Album.py | 65 +++-- deemix/types/Artist.py | 2 +- deemix/types/Date.py | 4 +- deemix/types/DownloadObjects.py | 39 ++- deemix/types/Lyrics.py | 2 +- deemix/types/Picture.py | 14 +- deemix/types/Playlist.py | 17 +- deemix/types/Track.py | 52 ++-- deemix/utils/__init__.py | 14 +- deemix/utils/crypto.py | 26 ++ deemix/utils/deezer.py | 32 +++ deemix/utils/localpaths.py | 89 ++++-- deemix/utils/pathtemplates.py | 135 +++++---- setup.py | 2 +- 21 files changed, 714 insertions(+), 554 deletions(-) rename deemix/{taggers.py => tagger.py} (98%) create mode 100644 deemix/utils/crypto.py create mode 100644 deemix/utils/deezer.py diff --git a/deemix/__init__.py b/deemix/__init__.py index 374cd5a..d3aa8d6 100644 --- a/deemix/__init__.py +++ b/deemix/__init__.py @@ -2,11 +2,16 @@ import re from urllib.request import urlopen -from deemix.itemgen import generateTrackItem, generateAlbumItem, generatePlaylistItem, generateArtistItem, generateArtistDiscographyItem, generateArtistTopItem +from deemix.itemgen import generateTrackItem, \ + generateAlbumItem, \ + generatePlaylistItem, \ + generateArtistItem, \ + generateArtistDiscographyItem, \ + generateArtistTopItem, \ + LinkNotRecognized, \ + LinkNotSupported -__version__ = "2.0.16" -USER_AGENT_HEADER = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) " \ - "Chrome/79.0.3945.130 Safari/537.36" +__version__ = "3.0.0" # Returns the Resolved URL, the Type and the ID def parseLink(link): @@ -42,11 +47,20 @@ def parseLink(link): return (link, link_type, link_id) -def generateDownloadObject(dz, link, bitrate): +def generateDownloadObject(dz, link, bitrate, plugins=None, listener=None): (link, link_type, link_id) = parseLink(link) if link_type is None or link_id is None: - return None + if plugins is None: plugins = {} + plugin_names = plugins.keys() + current_plugin = None + item = None + for plugin in plugin_names: + current_plugin = plugins[plugin] + item = current_plugin.generateDownloadObject(dz, link, bitrate, listener) + if item: return item + raise LinkNotRecognized(link) + if link_type == "track": return generateTrackItem(dz, link_id, bitrate) if link_type == "album": @@ -54,10 +68,10 @@ def generateDownloadObject(dz, link, bitrate): if link_type == "playlist": return generatePlaylistItem(dz, link_id, bitrate) if link_type == "artist": - return generateArtistItem(dz, link_id, bitrate) + return generateArtistItem(dz, link_id, bitrate, listener) if link_type == "artist_discography": - return generateArtistDiscographyItem(dz, link_id, bitrate) + return generateArtistDiscographyItem(dz, link_id, bitrate, listener) if link_type == "artist_top": return generateArtistTopItem(dz, link_id, bitrate) - return None + raise LinkNotSupported(link) diff --git a/deemix/__main__.py b/deemix/__main__.py index ca7109e..ce87e19 100644 --- a/deemix/__main__.py +++ b/deemix/__main__.py @@ -6,7 +6,7 @@ from deezer import Deezer from deezer import TrackFormats from deemix import generateDownloadObject -from deemix.settings import loadSettings +from deemix.settings import load as loadSettings from deemix.utils import getBitrateNumberFromText import deemix.utils.localpaths as localpaths from deemix.downloader import Downloader @@ -62,7 +62,7 @@ def download(url, bitrate, portable, path): # If first url is filepath readfile and use them as URLs try: isfile = Path(url[0]).is_file() - except: + except Exception: isfile = False if isfile: filename = url[0] diff --git a/deemix/decryption.py b/deemix/decryption.py index 1e71acd..51bef59 100644 --- a/deemix/decryption.py +++ b/deemix/decryption.py @@ -1,53 +1,37 @@ -import binascii from ssl import SSLError from time import sleep - import logging -from Cryptodome.Cipher import Blowfish, AES -from Cryptodome.Hash import MD5 - from requests import get -from requests.exceptions import ConnectionError as RequestsConnectionError, ReadTimeout +from requests.exceptions import ConnectionError as RequestsConnectionError, ReadTimeout, ChunkedEncodingError from urllib3.exceptions import SSLError as u3SSLError -from deemix import USER_AGENT_HEADER +from deemix.utils.crypto import _md5, _ecbCrypt, _ecbDecrypt, generateBlowfishKey, decryptChunk + +from deemix.utils import USER_AGENT_HEADER from deemix.types.DownloadObjects import Single logger = logging.getLogger('deemix') -def _md5(data): - h = MD5.new() - h.update(data.encode() if isinstance(data, str) else data) - return h.hexdigest() - -def generateBlowfishKey(trackId): - SECRET = 'g4el58wc0zvf9na1' - idMd5 = _md5(trackId) - bfKey = "" - for i in range(16): - bfKey += chr(ord(idMd5[i]) ^ ord(idMd5[i + 16]) ^ ord(SECRET[i])) - return bfKey - def generateStreamPath(sng_id, md5, media_version, media_format): urlPart = b'\xa4'.join( [md5.encode(), str(media_format).encode(), str(sng_id).encode(), str(media_version).encode()]) md5val = _md5(urlPart) step2 = md5val.encode() + b'\xa4' + urlPart + b'\xa4' step2 = step2 + (b'.' * (16 - (len(step2) % 16))) - urlPart = binascii.hexlify(AES.new(b'jo6aey6haid2Teih', AES.MODE_ECB).encrypt(step2)) + urlPart = _ecbCrypt('jo6aey6haid2Teih', step2) return urlPart.decode("utf-8") def reverseStreamPath(urlPart): - step2 = AES.new(b'jo6aey6haid2Teih', AES.MODE_ECB).decrypt(binascii.unhexlify(urlPart.encode("utf-8"))) + step2 = _ecbDecrypt('jo6aey6haid2Teih', urlPart) (_, md5, media_format, sng_id, media_version, _) = step2.split(b'\xa4') return (sng_id.decode('utf-8'), md5.decode('utf-8'), media_version.decode('utf-8'), media_format.decode('utf-8')) -def generateStreamURL(sng_id, md5, media_version, media_format): +def generateCryptedStreamURL(sng_id, md5, media_version, media_format): urlPart = generateStreamPath(sng_id, md5, media_version, media_format) return "https://e-cdns-proxy-" + md5[0] + ".dzcdn.net/mobile/1/" + urlPart -def generateUnencryptedStreamURL(sng_id, md5, media_version, media_format): +def generateStreamURL(sng_id, md5, media_version, media_format): urlPart = generateStreamPath(sng_id, md5, media_version, media_format) return "https://e-cdns-proxy-" + md5[0] + ".dzcdn.net/api/1/" + urlPart @@ -55,7 +39,8 @@ def reverseStreamURL(url): urlPart = url[url.find("/1/")+3:] return reverseStreamPath(urlPart) -def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, interface=None): +def streamTrack(outputStream, track, start=0, downloadObject=None, listener=None): + if downloadObject.isCanceled: raise DownloadCanceled headers= {'User-Agent': USER_AGENT_HEADER} chunkLength = start @@ -69,9 +54,23 @@ def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, in if complete == 0: raise DownloadEmpty if start != 0: responseRange = request.headers["Content-Range"] - logger.info('%s downloading range %s', itemName, responseRange) + if listener: + listener.send('downloadInfo', { + 'uuid': downloadObject.uuid, + 'itemName': itemName, + 'state': "downloading", + 'alreadyStarted': True, + 'value': responseRange + }) else: - logger.info('%s downloading %s bytes', itemName, complete) + if listener: + listener.send('downloadInfo', { + 'uuid': downloadObject.uuid, + 'itemName': itemName, + 'state': "downloading", + 'alreadyStarted': False, + 'value': complete + }) for chunk in request.iter_content(2048 * 3): outputStream.write(chunk) @@ -79,24 +78,24 @@ def streamUnencryptedTrack(outputStream, track, start=0, downloadObject=None, in if downloadObject: if isinstance(downloadObject, Single): - percentage = (chunkLength / (complete + start)) * 100 - downloadObject.progressNext = percentage + chunkProgres = (chunkLength / (complete + start)) * 100 + downloadObject.progressNext = chunkProgres else: chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 downloadObject.progressNext += chunkProgres - downloadObject.updateProgress(interface) + downloadObject.updateProgress(listener) except (SSLError, u3SSLError): logger.info('%s retrying from byte %s', itemName, chunkLength) - streamUnencryptedTrack(outputStream, track, chunkLength, downloadObject, interface) - except (RequestsConnectionError, ReadTimeout): + streamTrack(outputStream, track, chunkLength, downloadObject, listener) + except (RequestsConnectionError, ReadTimeout, ChunkedEncodingError): sleep(2) - streamUnencryptedTrack(outputStream, track, start, downloadObject, interface) + streamTrack(outputStream, track, start, downloadObject, listener) -def streamTrack(outputStream, track, start=0, downloadObject=None, interface=None): +def streamCryptedTrack(outputStream, track, start=0, downloadObject=None, listener=None): + if downloadObject.isCanceled: raise DownloadCanceled headers= {'User-Agent': USER_AGENT_HEADER} chunkLength = start - percentage = 0 itemName = f"[{track.mainArtist.name} - {track.title}]" @@ -109,32 +108,49 @@ def streamTrack(outputStream, track, start=0, downloadObject=None, interface=Non if complete == 0: raise DownloadEmpty if start != 0: responseRange = request.headers["Content-Range"] - logger.info('%s downloading range %s', itemName, responseRange) + if listener: + listener.send('downloadInfo', { + 'uuid': downloadObject.uuid, + 'itemName': itemName, + 'state': "downloading", + 'alreadyStarted': True, + 'value': responseRange + }) else: - logger.info('%s downloading %s bytes', itemName, complete) + if listener: + listener.send('downloadInfo', { + 'uuid': downloadObject.uuid, + 'itemName': itemName, + 'state': "downloading", + 'alreadyStarted': False, + 'value': complete + }) for chunk in request.iter_content(2048 * 3): if len(chunk) >= 2048: - chunk = Blowfish.new(blowfish_key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(chunk[0:2048]) + chunk[2048:] + chunk = decryptChunk(blowfish_key, chunk[0:2048]) + chunk[2048:] outputStream.write(chunk) chunkLength += len(chunk) if downloadObject: if isinstance(downloadObject, Single): - percentage = (chunkLength / (complete + start)) * 100 - downloadObject.progressNext = percentage + chunkProgres = (chunkLength / (complete + start)) * 100 + downloadObject.progressNext = chunkProgres else: chunkProgres = (len(chunk) / (complete + start)) / downloadObject.size * 100 downloadObject.progressNext += chunkProgres - downloadObject.updateProgress(interface) + downloadObject.updateProgress(listener) except (SSLError, u3SSLError): logger.info('%s retrying from byte %s', itemName, chunkLength) - streamTrack(outputStream, track, chunkLength, downloadObject, interface) - except (RequestsConnectionError, ReadTimeout): + streamCryptedTrack(outputStream, track, chunkLength, downloadObject, listener) + except (RequestsConnectionError, ReadTimeout, ChunkedEncodingError): sleep(2) - streamTrack(outputStream, track, start, downloadObject, interface) + streamCryptedTrack(outputStream, track, start, downloadObject, listener) + +class DownloadCanceled(Exception): + pass class DownloadEmpty(Exception): pass diff --git a/deemix/downloader.py b/deemix/downloader.py index fe72009..36286b1 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -18,19 +18,17 @@ from urllib3.exceptions import SSLError as u3SSLError from mutagen.flac import FLACNoHeaderError, error as FLACError from deezer import TrackFormats -from deemix import USER_AGENT_HEADER from deemix.types.DownloadObjects import Single, Collection -from deemix.types.Track import Track, AlbumDoesntExists -from deemix.utils.pathtemplates import generateFilename, generateFilepath, settingsRegexAlbum, settingsRegexArtist, settingsRegexPlaylistFile -from deemix.taggers import tagID3, tagFLAC -from deemix.decryption import generateUnencryptedStreamURL, streamUnencryptedTrack +from deemix.types.Track import Track, AlbumDoesntExists, MD5NotFound +from deemix.types.Picture import StaticPicture +from deemix.utils import USER_AGENT_HEADER +from deemix.utils.pathtemplates import generatePath, generateAlbumName, generateArtistName, generateDownloadObjectName +from deemix.tagger import tagID3, tagFLAC +from deemix.decryption import generateStreamURL, streamTrack, DownloadCanceled from deemix.settings import OverwriteOption logger = logging.getLogger('deemix') -TEMPDIR = Path(gettempdir()) / 'deemix-imgs' -if not TEMPDIR.is_dir(): makedirs(TEMPDIR) - extensions = { TrackFormats.FLAC: '.flac', TrackFormats.LOCAL: '.mp3', @@ -42,52 +40,39 @@ extensions = { TrackFormats.MP4_RA1: '.mp4' } -errorMessages = { - 'notOnDeezer': "Track not available on Deezer!", - 'notEncoded': "Track not yet encoded!", - 'notEncodedNoAlternative': "Track not yet encoded and no alternative found!", - 'wrongBitrate': "Track not found at desired bitrate.", - 'wrongBitrateNoAlternative': "Track not found at desired bitrate and no alternative found!", - 'no360RA': "Track is not available in Reality Audio 360.", - 'notAvailable': "Track not available on deezer's servers!", - 'notAvailableNoAlternative': "Track not available on deezer's servers and no alternative found!", - 'noSpaceLeft': "No space left on target drive, clean up some space for the tracks", - 'albumDoesntExists': "Track's album does not exsist, failed to gather info" -} +TEMPDIR = Path(gettempdir()) / 'deemix-imgs' +if not TEMPDIR.is_dir(): makedirs(TEMPDIR) def downloadImage(url, path, overwrite=OverwriteOption.DONT_OVERWRITE): - if not path.is_file() or overwrite in [OverwriteOption.OVERWRITE, OverwriteOption.ONLY_TAGS, OverwriteOption.KEEP_BOTH]: - try: - image = get(url, headers={'User-Agent': USER_AGENT_HEADER}, timeout=30) - image.raise_for_status() - with open(path, 'wb') as f: - f.write(image.content) - return path - except requests.exceptions.HTTPError: - if 'cdns-images.dzcdn.net' in url: - urlBase = url[:url.rfind("/")+1] - pictureUrl = url[len(urlBase):] - pictureSize = int(pictureUrl[:pictureUrl.find("x")]) - if pictureSize > 1200: - logger.warning("Couldn't download %sx%s image, falling back to 1200x1200", pictureSize, pictureSize) - sleep(1) - return downloadImage(urlBase+pictureUrl.replace(str(pictureSize)+"x"+str(pictureSize), '1200x1200'), path, overwrite) - logger.error("Image not found: %s", url) - except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError, u3SSLError) as e: - logger.error("Couldn't download Image, retrying in 5 seconds...: %s", url) - sleep(5) - return downloadImage(url, path, overwrite) - except OSError as e: - if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") from e - logger.exception("Error while downloading an image, you should report this to the developers: %s", e) - except Exception as e: - logger.exception("Error while downloading an image, you should report this to the developers: %s", e) + if path.is_file() and overwrite not in [OverwriteOption.OVERWRITE, OverwriteOption.ONLY_TAGS, OverwriteOption.KEEP_BOTH]: return path + + try: + image = get(url, headers={'User-Agent': USER_AGENT_HEADER}, timeout=30) + image.raise_for_status() + with open(path, 'wb') as f: + f.write(image.content) + return path + except requests.exceptions.HTTPError: + if path.is_file(): path.unlink() + if 'cdns-images.dzcdn.net' in url: + urlBase = url[:url.rfind("/")+1] + pictureUrl = url[len(urlBase):] + pictureSize = int(pictureUrl[:pictureUrl.find("x")]) + if pictureSize > 1200: + return downloadImage(urlBase+pictureUrl.replace(f"{pictureSize}x{pictureSize}", '1200x1200'), path, overwrite) + except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError, u3SSLError) as e: + if path.is_file(): path.unlink() + sleep(5) + return downloadImage(url, path, overwrite) + except OSError as e: if path.is_file(): path.unlink() - return None - return path + if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") from e + logger.exception("Error while downloading an image, you should report this to the developers: %s", e) + return None -def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectUUID=None, interface=None): - if track.localTrack: return TrackFormats.LOCAL +def getPreferredBitrate(track, bitrate, shouldFallback, uuid=None, listener=None): + bitrate = int(bitrate) + if track.local: return TrackFormats.LOCAL falledBack = False @@ -102,7 +87,7 @@ def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectU TrackFormats.MP4_RA1: "MP4_RA1", } - is360format = int(preferredBitrate) in formats_360 + is360format = bitrate in formats_360.keys() if not shouldFallback: formats = formats_360 @@ -112,30 +97,36 @@ def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectU else: formats = formats_non_360 + def testBitrate(track, formatNumber, formatName): + request = requests.head( + generateStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), + headers={'User-Agent': USER_AGENT_HEADER}, + timeout=30 + ) + try: + request.raise_for_status() + track.filesizes[f"FILESIZE_{formatName}"] = request.headers["Content-Length"] + track.filesizes[f"FILESIZE_{formatName}_TESTED"] = True + return formatNumber + except requests.exceptions.HTTPError: # if the format is not available, Deezer returns a 403 error + return None + for formatNumber, formatName in formats.items(): - if formatNumber >= int(preferredBitrate): continue + if formatNumber >= int(bitrate): continue if f"FILESIZE_{formatName}" in track.filesizes: if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]: - request = requests.head( - generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, formatNumber), - headers={'User-Agent': USER_AGENT_HEADER}, - timeout=30 - ) - try: - request.raise_for_status() - return formatNumber - except requests.exceptions.HTTPError: # if the format is not available, Deezer returns a 403 error - pass + testedBitrate = testBitrate(track, formatNumber, formatName) + if testedBitrate: return testedBitrate if not shouldFallback: raise PreferredBitrateNotFound if not falledBack: falledBack = True logger.info("%s Fallback to lower bitrate", f"[{track.mainArtist.name} - {track.title}]") - if interface and downloadObjectUUID: - interface.send('queueUpdate', { - 'uuid': downloadObjectUUID, + if listener and uuid: + listener.send('queueUpdate', { + 'uuid': uuid, 'bitrateFallback': True, 'data': { 'id': track.id, @@ -147,32 +138,52 @@ def getPreferredBitrate(track, preferredBitrate, shouldFallback, downloadObjectU return TrackFormats.DEFAULT class Downloader: - def __init__(self, dz, downloadObject, settings, interface=None): + def __init__(self, dz, downloadObject, settings, listener=None): self.dz = dz self.downloadObject = downloadObject self.settings = settings self.bitrate = downloadObject.bitrate - self.interface = interface + self.listener = listener + self.extrasPath = None self.playlistCoverName = None self.playlistURLs = [] def start(self): + if self.downloadObject.isCanceled: + if self.listener: + self.listener.send('currentItemCancelled', self.downloadObject.uuid) + self.listener.send("removedFromQueue", self.downloadObject.uuid) + return + if isinstance(self.downloadObject, Single): - result = self.downloadWrapper(self.downloadObject.single['trackAPI_gw'], self.downloadObject.single['trackAPI'], self.downloadObject.single['albumAPI']) - if result: self.singleAfterDownload(result) + track = self.downloadWrapper({ + 'trackAPI_gw': self.downloadObject.single['trackAPI_gw'], + 'trackAPI': self.downloadObject.single['trackAPI'], + 'albumAPI': self.downloadObject.single['albumAPI'] + }) + if track: self.afterDownloadSingle(track) elif isinstance(self.downloadObject, Collection): tracks = [None] * len(self.downloadObject.collection['tracks_gw']) with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor: for pos, track in enumerate(self.downloadObject.collection['tracks_gw'], start=0): - tracks[pos] = executor.submit(self.downloadWrapper, track, None, self.downloadObject.collection['albumAPI'], self.downloadObject.collection['playlistAPI']) - self.collectionAfterDownload(tracks) - if self.interface: - self.interface.send("finishDownload", self.downloadObject.uuid) - return self.extrasPath - - def download(self, trackAPI_gw, trackAPI=None, albumAPI=None, playlistAPI=None, track=None): - result = {} + tracks[pos] = executor.submit(self.downloadWrapper, { + 'trackAPI_gw': track, + 'albumAPI': self.downloadObject.collection['albumAPI'], + 'playlistAPI': self.downloadObject.collection['playlistAPI'] + }) + self.afterDownloadCollection(tracks) + + if self.listener: + self.listener.send("finishDownload", self.downloadObject.uuid) + + def download(self, extraData, track=None): + returnData = {} + trackAPI_gw = extraData['trackAPI_gw'] + trackAPI = extraData['trackAPI'] + albumAPI = extraData['albumAPI'] + playlistAPI = extraData['playlistAPI'] + if self.downloadObject.isCanceled: raise DownloadCanceled if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer") itemName = f"[{trackAPI_gw['ART_NAME']} - {trackAPI_gw['SNG_TITLE']}]" @@ -190,6 +201,8 @@ class Downloader: ) except AlbumDoesntExists as e: raise DownloadError('albumDoesntExists') from e + except MD5NotFound as e: + raise DownloadError('notLoggedIn') from e itemName = f"[{track.mainArtist.name} - {track.title}]" @@ -202,36 +215,37 @@ class Downloader: track, self.bitrate, self.settings['fallbackBitrate'], - self.downloadObject.uuid, self.interface + self.downloadObject.uuid, self.listener ) except PreferredBitrateNotFound as e: raise DownloadFailed("wrongBitrate", track) from e except TrackNot360 as e: raise DownloadFailed("no360RA") from e - track.selectedFormat = selectedFormat + track.bitrate = selectedFormat track.album.bitrate = selectedFormat - # Generate covers URLs - embeddedImageFormat = f'jpg-{self.settings["jpegImageQuality"]}' - if self.settings['embeddedArtworkPNG']: embeddedImageFormat = 'png' - - track.applySettings(self.settings, TEMPDIR, embeddedImageFormat) + # Apply settings + track.applySettings(self.settings) # Generate filename and filepath from metadata - filename = generateFilename(track, self.settings, "%artist% - %title%") - (filepath, artistPath, coverPath, extrasPath) = generateFilepath(track, self.settings) - # Remove subfolders from filename and add it to filepath - if pathSep in filename: - tempPath = filename[:filename.rfind(pathSep)] - filepath = filepath / tempPath - filename = filename[filename.rfind(pathSep) + len(pathSep):] + (filename, filepath, artistPath, coverPath, extrasPath) = generatePath(track, self.downloadObject, self.settings) + # Make sure the filepath exists makedirs(filepath, exist_ok=True) - writepath = filepath / f"{filename}{extensions[track.selectedFormat]}" + extension = extensions[track.bitrate] + writepath = filepath / f"{filename}{extension}" + # Save extrasPath - if extrasPath: - if not self.extrasPath: self.extrasPath = extrasPath - result['filename'] = str(writepath)[len(str(extrasPath))+ len(pathSep):] + if extrasPath and not self.extrasPath: self.extrasPath = extrasPath + + # Generate covers URLs + embeddedImageFormat = f'jpg-{self.settings["jpegImageQuality"]}' + if self.settings['embeddedArtworkPNG']: embeddedImageFormat = 'png' + + track.album.embeddedCoverURL = track.album.pic.getURL(self.settings['embeddedArtworkSize'], embeddedImageFormat) + ext = track.album.embeddedCoverURL[-4:] + if ext[0] != ".": ext = ".jpg" # Check for Spotify images + track.album.embeddedCoverPath = TEMPDIR / ((f"pl{track.playlist.id}" if track.album.isPlaylist else f"alb{track.album.id}") + f"_{self.settings['embeddedArtworkSize']}{ext}") # Download and cache coverart logger.info("%s Getting the album cover", itemName) @@ -239,48 +253,46 @@ class Downloader: # Save local album art if coverPath: - result['albumURLs'] = [] + returnData['albumURLs'] = [] for pic_format in self.settings['localArtworkFormat'].split(","): if pic_format in ["png","jpg"]: extendedFormat = pic_format if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" - url = track.album.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) - if self.settings['tags']['savePlaylistAsCompilation'] \ - and track.playlist \ - and track.playlist.pic.staticUrl \ - and not pic_format.startswith("jpg"): + url = track.album.pic.getURL(self.settings['localArtworkSize'], extendedFormat) + # Skip non deezer pictures at the wrong format + if isinstance(track.album.pic, StaticPicture) and pic_format != "jpg": continue - result['albumURLs'].append({'url': url, 'ext': pic_format}) - result['albumPath'] = coverPath - result['albumFilename'] = f"{settingsRegexAlbum(self.settings['coverImageTemplate'], track.album, self.settings, track.playlist)}" + returnData['albumURLs'].append({'url': url, 'ext': pic_format}) + returnData['albumPath'] = coverPath + returnData['albumFilename'] = generateAlbumName(self.settings['coverImageTemplate'], track.album, self.settings, track.playlist) # Save artist art if artistPath: - result['artistURLs'] = [] + returnData['artistURLs'] = [] for pic_format in self.settings['localArtworkFormat'].split(","): - if pic_format in ["png","jpg"]: - extendedFormat = pic_format - if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" - url = track.album.mainArtist.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) - if track.album.mainArtist.pic.md5 == "" and not pic_format.startswith("jpg"): continue - result['artistURLs'].append({'url': url, 'ext': pic_format}) - result['artistPath'] = artistPath - result['artistFilename'] = f"{settingsRegexArtist(self.settings['artistImageTemplate'], track.album.mainArtist, self.settings, rootArtist=track.album.rootArtist)}" + # Deezer doesn't support png artist images + if pic_format == "jpg": + extendedFormat = f"{pic_format}-{self.settings['jpegImageQuality']}" + url = track.album.mainArtist.pic.getURL(self.settings['localArtworkSize'], extendedFormat) + if track.album.mainArtist.pic.md5 == "": continue + returnData['artistURLs'].append({'url': url, 'ext': pic_format}) + returnData['artistPath'] = artistPath + returnData['artistFilename'] = generateArtistName(self.settings['artistImageTemplate'], track.album.mainArtist, self.settings, rootArtist=track.album.rootArtist) # Save playlist art if track.playlist: - if self.playlistURLs == []: + if len(self.playlistURLs) == 0: for pic_format in self.settings['localArtworkFormat'].split(","): if pic_format in ["png","jpg"]: extendedFormat = pic_format if extendedFormat == "jpg": extendedFormat += f"-{self.settings['jpegImageQuality']}" - url = track.playlist.pic.generatePictureURL(self.settings['localArtworkSize'], extendedFormat) - if track.playlist.pic.staticUrl and not pic_format.startswith("jpg"): continue + url = track.playlist.pic.getURL(self.settings['localArtworkSize'], extendedFormat) + if isinstance(track.playlist.pic, StaticPicture) and pic_format != "jpg": continue self.playlistURLs.append({'url': url, 'ext': pic_format}) if not self.playlistCoverName: track.playlist.bitrate = selectedFormat track.playlist.dateString = track.playlist.date.format(self.settings['dateFormat']) - self.playlistCoverName = f"{settingsRegexAlbum(self.settings['coverImageTemplate'], track.playlist, self.settings, track.playlist)}" + self.playlistCoverName = generateAlbumName(self.settings['coverImageTemplate'], track.playlist, self.settings, track.playlist) # Save lyrics in lrc file if self.settings['syncedLyrics'] and track.lyrics.sync: @@ -301,106 +313,67 @@ class Downloader: # Don't overwrite and keep both files if trackAlreadyDownloaded and self.settings['overwriteFile'] == OverwriteOption.KEEP_BOTH: baseFilename = str(filepath / filename) - i = 1 - currentFilename = baseFilename+' ('+str(i)+')'+ extensions[track.selectedFormat] + c = 1 + currentFilename = baseFilename+' ('+str(c)+')'+ extension while Path(currentFilename).is_file(): - i += 1 - currentFilename = baseFilename+' ('+str(i)+')'+ extensions[track.selectedFormat] + c += 1 + currentFilename = baseFilename+' ('+str(c)+')'+ extension trackAlreadyDownloaded = False writepath = Path(currentFilename) if not trackAlreadyDownloaded or self.settings['overwriteFile'] == OverwriteOption.OVERWRITE: logger.info("%s Downloading the track", itemName) - track.downloadUrl = generateUnencryptedStreamURL(track.id, track.MD5, track.mediaVersion, track.selectedFormat) - - def downloadMusic(track, trackAPI_gw): - try: - with open(writepath, 'wb') as stream: - streamUnencryptedTrack(stream, track, downloadObject=self.downloadObject, interface=self.interface) - except DownloadCancelled as e: - if writepath.is_file(): writepath.unlink() - raise e - except (requests.exceptions.HTTPError, DownloadEmpty) as e: - if writepath.is_file(): writepath.unlink() - if track.fallbackID != "0": - logger.warning("%s Track not available, using fallback id", itemName) - newTrack = self.dz.gw.get_track_with_fallback(track.fallbackID) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - return False - if not track.searched and self.settings['fallbackSearch']: - logger.warning("%s Track not available, searching for alternative", itemName) - searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) - if searchedId != "0": - newTrack = self.dz.gw.get_track_with_fallback(searchedId) - track.parseEssentialData(newTrack) - track.retriveFilesizes(self.dz) - track.searched = True - if self.interface: - self.interface.send('queueUpdate', { - 'uuid': self.downloadObject.uuid, - 'searchFallback': True, - 'data': { - 'id': track.id, - 'title': track.title, - 'artist': track.mainArtist.name - }, - }) - return False - raise DownloadFailed("notAvailableNoAlternative") from e - raise DownloadFailed("notAvailable") from e - except (requests.exceptions.ConnectionError, requests.exceptions.ChunkedEncodingError) as e: - if writepath.is_file(): writepath.unlink() - logger.warning("%s Error while downloading the track, trying again in 5s...", itemName) - sleep(5) - return downloadMusic(track, trackAPI_gw) - except OSError as e: - if writepath.is_file(): writepath.unlink() - if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") from e - logger.exception("%s Error while downloading the track, you should report this to the developers: %s", itemName, e) - raise e - except Exception as e: - if writepath.is_file(): writepath.unlink() - logger.exception("%s Error while downloading the track, you should report this to the developers: %s", itemName, e) - raise e - return True + track.downloadUrl = generateStreamURL(track.id, track.MD5, track.mediaVersion, track.bitrate) try: - trackDownloaded = downloadMusic(track, trackAPI_gw) - except Exception as e: + with open(writepath, 'wb') as stream: + streamTrack(stream, track, downloadObject=self.downloadObject, listener=self.listener) + except OSError as e: + if writepath.is_file(): writepath.unlink() + if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") from e raise e - if not trackDownloaded: return self.download(trackAPI_gw, track=track) else: logger.info("%s Skipping track as it's already downloaded", itemName) - self.downloadObject.completeTrackProgress(self.interface) + self.downloadObject.completeTrackProgress(self.listener) # Adding tags - if (not trackAlreadyDownloaded or self.settings['overwriteFile'] in [OverwriteOption.ONLY_TAGS, OverwriteOption.OVERWRITE]) and not track.localTrack: + if (not trackAlreadyDownloaded or self.settings['overwriteFile'] in [OverwriteOption.ONLY_TAGS, OverwriteOption.OVERWRITE]) and not track.local: logger.info("%s Applying tags to the track", itemName) - if track.selectedFormat in [TrackFormats.MP3_320, TrackFormats.MP3_128, TrackFormats.DEFAULT]: + if extension == '.mp3': tagID3(writepath, track, self.settings['tags']) - elif track.selectedFormat == TrackFormats.FLAC: + elif extension == '.flac': try: tagFLAC(writepath, track, self.settings['tags']) except (FLACNoHeaderError, FLACError): - if writepath.is_file(): writepath.unlink() + writepath.unlink() logger.warning("%s Track not available in FLAC, falling back if necessary", itemName) - self.downloadObject.removeTrackProgress(self.interface) + self.downloadObject.removeTrackProgress(self.listener) track.filesizes['FILESIZE_FLAC'] = "0" track.filesizes['FILESIZE_FLAC_TESTED'] = True return self.download(trackAPI_gw, track=track) - if track.searched: result['searched'] = f"{track.mainArtist.name} - {track.title}" - logger.info("%s Track download completed\n%s", itemName, writepath) + if track.searched: returnData['searched'] = True self.downloadObject.downloaded += 1 self.downloadObject.files.append(str(writepath)) self.downloadObject.extrasPath = str(self.extrasPath) - if self.interface: - self.interface.send("updateQueue", {'uuid': self.downloadObject.uuid, 'downloaded': True, 'downloadPath': str(writepath), 'extrasPath': str(self.extrasPath)}) - return result + logger.info("%s Track download completed\n%s", itemName, writepath) + if self.listener: self.listener.send("updateQueue", { + 'uuid': self.downloadObject.uuid, + 'downloaded': True, + 'downloadPath': str(writepath), + 'extrasPath': str(self.extrasPath) + }) + returnData['filename'] = str(writepath)[len(str(extrasPath))+ len(pathSep):] + returnData['data'] = { + 'id': track.id, + 'title': track.title, + 'artist': track.mainArtist.name + } + return returnData - def downloadWrapper(self, trackAPI_gw, trackAPI=None, albumAPI=None, playlistAPI=None, track=None): + def downloadWrapper(self, extraData, track=None): + trackAPI_gw = extraData['trackAPI_gw'] # Temp metadata to generate logs tempTrack = { 'id': trackAPI_gw['SNG_ID'], @@ -413,7 +386,7 @@ class Downloader: itemName = f"[{track.mainArtist.name} - {track.title}]" try: - result = self.download(trackAPI_gw, trackAPI, albumAPI, playlistAPI, track) + result = self.download(extraData, track) except DownloadFailed as error: if error.track: track = error.track @@ -422,7 +395,7 @@ class Downloader: newTrack = self.dz.gw.get_track_with_fallback(track.fallbackID) track.parseEssentialData(newTrack) track.retriveFilesizes(self.dz) - return self.downloadWrapper(trackAPI_gw, trackAPI, albumAPI, playlistAPI, track) + return self.downloadWrapper(extraData, track) if not track.searched and self.settings['fallbackSearch']: logger.warning("%s %s Searching for alternative", itemName, error.message) searchedId = self.dz.api.get_track_id_from_metadata(track.mainArtist.name, track.title, track.album.title) @@ -431,19 +404,18 @@ class Downloader: track.parseEssentialData(newTrack) track.retriveFilesizes(self.dz) track.searched = True - if self.interface: - self.interface.send('queueUpdate', { - 'uuid': self.downloadObject.uuid, - 'searchFallback': True, - 'data': { - 'id': track.id, - 'title': track.title, - 'artist': track.mainArtist.name - }, - }) - return self.downloadWrapper(trackAPI_gw, trackAPI, albumAPI, playlistAPI, track) - error.errid += "NoAlternative" - error.message = errorMessages[error.errid] + if self.listener: self.listener.send('queueUpdate', { + 'uuid': self.downloadObject.uuid, + 'searchFallback': True, + 'data': { + 'id': track.id, + 'title': track.title, + 'artist': track.mainArtist.name + }, + }) + return self.downloadWrapper(extraData, track) + error.errid += "NoAlternative" + error.message = errorMessages[error.errid] logger.error("%s %s", itemName, error.message) result = {'error': { 'message': error.message, @@ -453,17 +425,17 @@ class Downloader: except Exception as e: logger.exception("%s %s", itemName, e) result = {'error': { - 'message': str(e), - 'data': tempTrack - }} + 'message': str(e), + 'data': tempTrack + }} if 'error' in result: - self.downloadObject.completeTrackProgress(self.interface) + self.downloadObject.completeTrackProgress(self.listener) self.downloadObject.failed += 1 self.downloadObject.errors.append(result['error']) - if self.interface: + if self.listener: error = result['error'] - self.interface.send("updateQueue", { + self.listener.send("updateQueue", { 'uuid': self.downloadObject.uuid, 'failed': True, 'data': error['data'], @@ -472,61 +444,63 @@ class Downloader: }) return result - def singleAfterDownload(self, result): + def afterDownloadSingle(self, track): if not self.extrasPath: self.extrasPath = Path(self.settings['downloadLocation']) # Save Album Cover - if self.settings['saveArtwork'] and 'albumPath' in result: - for image in result['albumURLs']: - downloadImage(image['url'], result['albumPath'] / f"{result['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) + if self.settings['saveArtwork'] and 'albumPath' in track: + for image in track['albumURLs']: + downloadImage(image['url'], track['albumPath'] / f"{track['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) # Save Artist Artwork - if self.settings['saveArtworkArtist'] and 'artistPath' in result: - for image in result['artistURLs']: - downloadImage(image['url'], result['artistPath'] / f"{result['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) + if self.settings['saveArtworkArtist'] and 'artistPath' in track: + for image in track['artistURLs']: + downloadImage(image['url'], track['artistPath'] / f"{track['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) # Create searched logfile - if self.settings['logSearched'] and 'searched' in result: + if self.settings['logSearched'] and 'searched' in track: + filename = f"{track.data.artist} - {track.data.title}" with open(self.extrasPath / 'searched.txt', 'wb+') as f: - orig = f.read().decode('utf-8') - if not result['searched'] in orig: - if orig != "": orig += "\r\n" - orig += result['searched'] + "\r\n" - f.write(orig.encode('utf-8')) + searchedFile = f.read().decode('utf-8') + if not filename in searchedFile: + if searchedFile != "": searchedFile += "\r\n" + searchedFile += filename + "\r\n" + f.write(searchedFile.encode('utf-8')) + # Execute command after download if self.settings['executeCommand'] != "": - execute(self.settings['executeCommand'].replace("%folder%", quote(str(self.extrasPath))).replace("%filename%", quote(result['filename'])), shell=True) + execute(self.settings['executeCommand'].replace("%folder%", quote(str(self.extrasPath))).replace("%filename%", quote(track['filename'])), shell=True) - def collectionAfterDownload(self, tracks): + def afterDownloadCollection(self, tracks): if not self.extrasPath: self.extrasPath = Path(self.settings['downloadLocation']) playlist = [None] * len(tracks) errors = "" searched = "" - for i in enumerate(tracks): - result = tracks[i].result() - if not result: return # Check if item is cancelled + for i, track in enumerate(tracks): + track = track.result() + if not track: return # Check if item is cancelled # Log errors to file - if result.get('error'): - if not result['error'].get('data'): result['error']['data'] = {'id': "0", 'title': 'Unknown', 'artist': 'Unknown'} - errors += f"{result['error']['data']['id']} | {result['error']['data']['artist']} - {result['error']['data']['title']} | {result['error']['message']}\r\n" + if track.get('error'): + if not track['error'].get('data'): track['error']['data'] = {'id': "0", 'title': 'Unknown', 'artist': 'Unknown'} + errors += f"{track['error']['data']['id']} | {track['error']['data']['artist']} - {track['error']['data']['title']} | {track['error']['message']}\r\n" # Log searched to file - if 'searched' in result: searched += result['searched'] + "\r\n" + if 'searched' in track: searched += track['searched'] + "\r\n" # Save Album Cover - if self.settings['saveArtwork'] and 'albumPath' in result: - for image in result['albumURLs']: - downloadImage(image['url'], result['albumPath'] / f"{result['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) + if self.settings['saveArtwork'] and 'albumPath' in track: + for image in track['albumURLs']: + downloadImage(image['url'], track['albumPath'] / f"{track['albumFilename']}.{image['ext']}", self.settings['overwriteFile']) # Save Artist Artwork - if self.settings['saveArtworkArtist'] and 'artistPath' in result: - for image in result['artistURLs']: - downloadImage(image['url'], result['artistPath'] / f"{result['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) + if self.settings['saveArtworkArtist'] and 'artistPath' in track: + for image in track['artistURLs']: + downloadImage(image['url'], track['artistPath'] / f"{track['artistFilename']}.{image['ext']}", self.settings['overwriteFile']) # Save filename for playlist file - playlist[i] = result.get('filename', "") + playlist[i] = track.get('filename', "") # Create errors logfile if self.settings['logErrors'] and errors != "": @@ -545,7 +519,7 @@ class Downloader: # Create M3U8 File if self.settings['createM3U8File']: - filename = settingsRegexPlaylistFile(self.settings['playlistFilenameTemplate'], self.downloadObject, self.settings) or "playlist" + filename = generateDownloadObjectName(self.settings['playlistFilenameTemplate'], self.downloadObject, self.settings) or "playlist" with open(self.extrasPath / f'{filename}.m3u8', 'wb') as f: for line in playlist: f.write((line + "\n").encode('utf-8')) @@ -557,6 +531,19 @@ class Downloader: class DownloadError(Exception): """Base class for exceptions in this module.""" +errorMessages = { + 'notOnDeezer': "Track not available on Deezer!", + 'notEncoded': "Track not yet encoded!", + 'notEncodedNoAlternative': "Track not yet encoded and no alternative found!", + 'wrongBitrate': "Track not found at desired bitrate.", + 'wrongBitrateNoAlternative': "Track not found at desired bitrate and no alternative found!", + 'no360RA': "Track is not available in Reality Audio 360.", + 'notAvailable': "Track not available on deezer's servers!", + 'notAvailableNoAlternative': "Track not available on deezer's servers and no alternative found!", + 'noSpaceLeft': "No space left on target drive, clean up some space for the tracks", + 'albumDoesntExists': "Track's album does not exsist, failed to gather info" +} + class DownloadFailed(DownloadError): def __init__(self, errid, track=None): super().__init__() @@ -564,12 +551,6 @@ class DownloadFailed(DownloadError): self.message = errorMessages[self.errid] self.track = track -class DownloadCancelled(DownloadError): - pass - -class DownloadEmpty(DownloadError): - pass - class PreferredBitrateNotFound(DownloadError): pass diff --git a/deemix/itemgen.py b/deemix/itemgen.py index 3cf9e5e..36821db 100644 --- a/deemix/itemgen.py +++ b/deemix/itemgen.py @@ -1,46 +1,31 @@ import logging from deemix.types.DownloadObjects import Single, Collection -from deezer.utils import map_user_playlist -from deezer.api import APIError from deezer.gw import GWAPIError, LyricsStatus +from deezer.api import APIError +from deezer.utils import map_user_playlist logger = logging.getLogger('deemix') -class GenerationError(Exception): - def __init__(self, link, message, errid=None): - super().__init__() - self.link = link - self.message = message - self.errid = errid - - def toDict(self): - return { - 'link': self.link, - 'error': self.message, - 'errid': self.errid - } - def generateTrackItem(dz, link_id, bitrate, trackAPI=None, albumAPI=None): # Check if is an isrc: url if str(link_id).startswith("isrc"): try: trackAPI = dz.api.get_track(link_id) except APIError as e: - raise GenerationError("https://deezer.com/track/"+str(link_id), f"Wrong URL: {e}") from e + raise GenerationError(f"https://deezer.com/track/{link_id}", str(e)) from e + if 'id' in trackAPI and 'title' in trackAPI: link_id = trackAPI['id'] else: - raise GenerationError("https://deezer.com/track/"+str(link_id), "Track ISRC is not available on deezer", "ISRCnotOnDeezer") + raise ISRCnotOnDeezer(f"https://deezer.com/track/{link_id}") + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/track/{link_id}") # Get essential track info try: trackAPI_gw = dz.gw.get_track_with_fallback(link_id) except GWAPIError as e: - message = "Wrong URL" - # TODO: FIX - # if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}" - raise GenerationError("https://deezer.com/track/"+str(link_id), message) from e + raise GenerationError(f"https://deezer.com/track/{link_id}", str(e)) from e title = trackAPI_gw['SNG_TITLE'].strip() if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: @@ -67,20 +52,24 @@ def generateAlbumItem(dz, link_id, bitrate, rootArtist=None): try: albumAPI = dz.api.get_album(link_id) except APIError as e: - raise GenerationError("https://deezer.com/album/"+str(link_id), f"Wrong URL: {e}") from e + raise GenerationError(f"https://deezer.com/album/{link_id}", str(e)) from e if str(link_id).startswith('upc'): link_id = albumAPI['id'] + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/album/{link_id}") # Get extra info about album # This saves extra api calls when downloading albumAPI_gw = dz.gw.get_album(link_id) albumAPI['nb_disk'] = albumAPI_gw['NUMBER_DISK'] albumAPI['copyright'] = albumAPI_gw['COPYRIGHT'] + albumAPI['release_date'] = albumAPI_gw['PHYSICAL_RELEASE_DATE'] albumAPI['root_artist'] = rootArtist # If the album is a single download as a track if albumAPI['nb_tracks'] == 1: - return generateTrackItem(dz, albumAPI['tracks']['data'][0]['id'], bitrate, albumAPI=albumAPI) + if len(albumAPI['tracks']['data']): + return generateTrackItem(dz, albumAPI['tracks']['data'][0]['id'], bitrate, albumAPI=albumAPI) + raise GenerationError(f"https://deezer.com/album/{link_id}", "Single has no tracks.") tracksArray = dz.gw.get_album_tracks(link_id) @@ -116,6 +105,7 @@ def generateAlbumItem(dz, link_id, bitrate, rootArtist=None): def generatePlaylistItem(dz, link_id, bitrate, playlistAPI=None, playlistTracksAPI=None): if not playlistAPI: + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/playlist/{link_id}") # Get essential playlist info try: playlistAPI = dz.api.get_playlist(link_id) @@ -127,15 +117,12 @@ def generatePlaylistItem(dz, link_id, bitrate, playlistAPI=None, playlistTracksA userPlaylist = dz.gw.get_playlist_page(link_id) playlistAPI = map_user_playlist(userPlaylist['DATA']) except GWAPIError as e: - message = "Wrong URL" - # TODO: FIX - # if "DATA_ERROR" in e: message += f": {e['DATA_ERROR']}" - raise GenerationError("https://deezer.com/playlist/"+str(link_id), message) from e + raise GenerationError(f"https://deezer.com/playlist/{link_id}", str(e)) from e # Check if private playlist and owner if not playlistAPI.get('public', False) and playlistAPI['creator']['id'] != str(dz.current_user['id']): logger.warning("You can't download others private playlists.") - raise GenerationError("https://deezer.com/playlist/"+str(link_id), "You can't download others private playlists.", "notYourPrivatePlaylist") + raise NotYourPrivatePlaylist(f"https://deezer.com/playlist/{link_id}") if not playlistTracksAPI: playlistTracksAPI = dz.gw.get_playlist_tracks(link_id) @@ -168,73 +155,82 @@ def generatePlaylistItem(dz, link_id, bitrate, playlistAPI=None, playlistTracksA } }) -def generateArtistItem(dz, link_id, bitrate, interface=None): +def generateArtistItem(dz, link_id, bitrate, listener=None): + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/artist/{link_id}") # Get essential artist info try: artistAPI = dz.api.get_artist(link_id) except APIError as e: - raise GenerationError("https://deezer.com/artist/"+str(link_id), f"Wrong URL: {e}") from e + raise GenerationError(f"https://deezer.com/artist/{link_id}", str(e)) from e rootArtist = { 'id': artistAPI['id'], - 'name': artistAPI['name'] + 'name': artistAPI['name'], + 'picture_small': artistAPI['picture_small'] } - if interface: interface.send("startAddingArtist", rootArtist) + if listener: listener.send("startAddingArtist", rootArtist) artistDiscographyAPI = dz.gw.get_artist_discography_tabs(link_id, 100) allReleases = artistDiscographyAPI.pop('all', []) albumList = [] for album in allReleases: - albumList.append(generateAlbumItem(dz, album['id'], bitrate, rootArtist=rootArtist)) + try: + albumList.append(generateAlbumItem(dz, album['id'], bitrate, rootArtist=rootArtist)) + except GenerationError as e: + logger.warning("Album %s has no data: %s", str(album['id']), str(e)) - if interface: interface.send("finishAddingArtist", rootArtist) + if listener: listener.send("finishAddingArtist", rootArtist) return albumList -def generateArtistDiscographyItem(dz, link_id, bitrate, interface=None): +def generateArtistDiscographyItem(dz, link_id, bitrate, listener=None): + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/artist/{link_id}/discography") # Get essential artist info try: artistAPI = dz.api.get_artist(link_id) except APIError as e: - e = str(e) - raise GenerationError("https://deezer.com/artist/"+str(link_id)+"/discography", f"Wrong URL: {e}") + raise GenerationError(f"https://deezer.com/artist/{link_id}/discography", str(e)) from e rootArtist = { 'id': artistAPI['id'], - 'name': artistAPI['name'] + 'name': artistAPI['name'], + 'picture_small': artistAPI['picture_small'] } - if interface: interface.send("startAddingArtist", rootArtist) + if listener: listener.send("startAddingArtist", rootArtist) artistDiscographyAPI = dz.gw.get_artist_discography_tabs(link_id, 100) artistDiscographyAPI.pop('all', None) # all contains albums and singles, so its all duplicates. This removes them albumList = [] for releaseType in artistDiscographyAPI: for album in artistDiscographyAPI[releaseType]: - albumList.append(generateAlbumItem(dz, album['id'], bitrate, rootArtist=rootArtist)) + try: + albumList.append(generateAlbumItem(dz, album['id'], bitrate, rootArtist=rootArtist)) + except GenerationError as e: + logger.warning("Album %s has no data: %s", str(album['id']), str(e)) - if interface: interface.send("finishAddingArtist", rootArtist) + if listener: listener.send("finishAddingArtist", rootArtist) return albumList -def generateArtistTopItem(dz, link_id, bitrate, interface=None): +def generateArtistTopItem(dz, link_id, bitrate): + if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/artist/{link_id}/top_track") # Get essential artist info try: artistAPI = dz.api.get_artist(link_id) except APIError as e: - e = str(e) - raise GenerationError("https://deezer.com/artist/"+str(link_id)+"/top_track", f"Wrong URL: {e}") + raise GenerationError(f"https://deezer.com/artist/{link_id}/top_track", str(e)) from e # Emulate the creation of a playlist # Can't use generatePlaylistItem directly as this is not a real playlist playlistAPI = { - 'id': str(artistAPI['id'])+"_top_track", - 'title': artistAPI['name']+" - Top Tracks", - 'description': "Top Tracks for "+artistAPI['name'], + 'id':f"{artistAPI['id']}_top_track", + 'title': f"{artistAPI['name']} - Top Tracks", + 'description': f"Top Tracks for {artistAPI['name']}", 'duration': 0, 'public': True, 'is_loved_track': False, 'collaborative': False, 'nb_tracks': 0, 'fans': artistAPI['nb_fan'], - 'link': "https://www.deezer.com/artist/"+str(artistAPI['id'])+"/top_track", + 'link': f"https://www.deezer.com/artist/{artistAPI['id']}/top_track", 'share': None, 'picture': artistAPI['picture'], 'picture_small': artistAPI['picture_small'], @@ -242,10 +238,10 @@ def generateArtistTopItem(dz, link_id, bitrate, interface=None): 'picture_big': artistAPI['picture_big'], 'picture_xl': artistAPI['picture_xl'], 'checksum': None, - 'tracklist': "https://api.deezer.com/artist/"+str(artistAPI['id'])+"/top", + 'tracklist': f"https://api.deezer.com/artist/{artistAPI['id']}/top", 'creation_date': "XXXX-00-00", 'creator': { - 'id': "art_"+str(artistAPI['id']), + 'id': f"art_{artistAPI['id']}", 'name': artistAPI['name'], 'type': "user" }, @@ -254,3 +250,45 @@ def generateArtistTopItem(dz, link_id, bitrate, interface=None): artistTopTracksAPI_gw = dz.gw.get_artist_toptracks(link_id) return generatePlaylistItem(dz, playlistAPI['id'], bitrate, playlistAPI=playlistAPI, playlistTracksAPI=artistTopTracksAPI_gw) + +class GenerationError(Exception): + def __init__(self, link, message, errid=None): + super().__init__() + self.link = link + self.message = message + self.errid = errid + + def toDict(self): + return { + 'link': self.link, + 'error': self.message, + 'errid': self.errid + } + +class ISRCnotOnDeezer(GenerationError): + def __init__(self, link): + super().__init__(link, "Track ISRC is not available on deezer", "ISRCnotOnDeezer") + +class NotYourPrivatePlaylist(GenerationError): + def __init__(self, link): + super().__init__(link, "You can't download others private playlists.", "notYourPrivatePlaylist") + +class TrackNotOnDeezer(GenerationError): + def __init__(self, link): + super().__init__(link, "Track not found on deezer!", "trackNotOnDeezer") + +class AlbumNotOnDeezer(GenerationError): + def __init__(self, link): + super().__init__(link, "Album not found on deezer!", "albumNotOnDeezer") + +class InvalidID(GenerationError): + def __init__(self, link): + super().__init__(link, "Link ID is invalid!", "invalidID") + +class LinkNotSupported(GenerationError): + def __init__(self, link): + super().__init__(link, "Link is not supported.", "unsupportedURL") + +class LinkNotRecognized(GenerationError): + def __init__(self, link): + super().__init__(link, "Link is not recognized.", "invalidURL") diff --git a/deemix/settings.py b/deemix/settings.py index fd46656..e1298ef 100644 --- a/deemix/settings.py +++ b/deemix/settings.py @@ -20,7 +20,7 @@ class FeaturesOption(): MOVE_TITLE = "2" # Move to track title DEFAULTS = { - "downloadLocation": "", + "downloadLocation": localpaths.getMusicFolder(), "tracknameTemplate": "%artist% - %title%", "albumTracknameTemplate": "%tracknumber% - %title%", "playlistTracknameTemplate": "%position% - %artist% - %title%", @@ -100,26 +100,26 @@ DEFAULTS = { } } -def saveSettings(settings, configFolder=None): +def save(settings, configFolder=None): configFolder = Path(configFolder or localpaths.getConfigFolder()) makedirs(configFolder, exist_ok=True) # Create config folder if it doesn't exsist with open(configFolder / 'config.json', 'w') as configFile: json.dump(settings, configFile, indent=2) -def loadSettings(configFolder=None): +def load(configFolder=None): configFolder = Path(configFolder or localpaths.getConfigFolder()) makedirs(configFolder, exist_ok=True) # Create config folder if it doesn't exsist - if not (configFolder / 'config.json').is_file(): saveSettings(DEFAULTS, configFolder) # Create config file if it doesn't exsist + if not (configFolder / 'config.json').is_file(): save(DEFAULTS, configFolder) # Create config file if it doesn't exsist # Read config file with open(configFolder / 'config.json', 'r') as configFile: settings = json.load(configFile) - if checkSettings(settings) > 0: saveSettings(settings) # Check the settings and save them if something changed + if check(settings) > 0: save(settings, configFolder) # Check the settings and save them if something changed return settings -def checkSettings(settings): +def check(settings): changes = 0 for i_set in DEFAULTS: if not i_set in settings or not isinstance(settings[i_set], DEFAULTS[i_set]): diff --git a/deemix/taggers.py b/deemix/tagger.py similarity index 98% rename from deemix/taggers.py rename to deemix/tagger.py index b363752..e3cdbf6 100644 --- a/deemix/taggers.py +++ b/deemix/tagger.py @@ -4,10 +4,10 @@ from mutagen.id3 import ID3, ID3NoHeaderError, \ TPUB, TSRC, USLT, SYLT, APIC, IPLS, TCOM, TCOP, TCMP, Encoding, PictureType # Adds tags to a MP3 file -def tagID3(stream, track, save): +def tagID3(path, track, save): # Delete exsisting tags try: - tag = ID3(stream) + tag = ID3(path) tag.delete() except ID3NoHeaderError: tag = ID3() @@ -111,15 +111,15 @@ def tagID3(stream, track, save): with open(track.album.embeddedCoverPath, 'rb') as f: tag.add(APIC(descEncoding, mimeType, PictureType.COVER_FRONT, desc='cover', data=f.read())) - tag.save( stream, + tag.save( path, v1=2 if save['saveID3v1'] else 0, v2_version=3, v23_sep=None if save['useNullSeparator'] else '/' ) # Adds tags to a FLAC file -def tagFLAC(stream, track, save): +def tagFLAC(path, track, save): # Delete exsisting tags - tag = FLAC(stream) + tag = FLAC(path) tag.delete() tag.clear_pictures() diff --git a/deemix/types/Album.py b/deemix/types/Album.py index 9161aa5..d1e0fda 100644 --- a/deemix/types/Album.py +++ b/deemix/types/Album.py @@ -10,21 +10,21 @@ class Album: def __init__(self, alb_id="0", title="", pic_md5=""): self.id = alb_id self.title = title - self.pic = Picture(md5=pic_md5, type="cover") + self.pic = Picture(pic_md5, "cover") self.artist = {"Main": []} self.artists = [] self.mainArtist = None - self.date = None - self.dateString = None + self.date = Date() + self.dateString = "" self.trackTotal = "0" self.discTotal = "0" - self.embeddedCoverPath = None - self.embeddedCoverURL = None + self.embeddedCoverPath = "" + self.embeddedCoverURL = "" self.explicit = False self.genre = [] self.barcode = "Unknown" self.label = "Unknown" - self.copyright = None + self.copyright = "" self.recordType = "album" self.bitrate = 0 self.rootArtist = None @@ -32,26 +32,29 @@ class Album: self.playlistId = None self.owner = None + self.isPlaylist = False def parseAlbum(self, albumAPI): self.title = albumAPI['title'] # Getting artist image ID # ex: https://e-cdns-images.dzcdn.net/images/artist/f2bc007e9133c946ac3c3907ddc5d2ea/56x56-000000-80-0-0.jpg - artistPicture = albumAPI['artist']['picture_small'] - artistPicture = artistPicture[artistPicture.find('artist/') + 7:-24] + art_pic = albumAPI['artist']['picture_small'] + art_pic = art_pic[art_pic.find('artist/') + 7:-24] self.mainArtist = Artist( - id = albumAPI['artist']['id'], - name = albumAPI['artist']['name'], - pic_md5 = artistPicture + albumAPI['artist']['id'], + albumAPI['artist']['name'], + "Main", + art_pic ) if albumAPI.get('root_artist'): - artistPicture = albumAPI['root_artist']['picture_small'] - artistPicture = artistPicture[artistPicture.find('artist/') + 7:-24] + art_pic = albumAPI['root_artist']['picture_small'] + art_pic = art_pic[art_pic.find('artist/') + 7:-24] self.rootArtist = Artist( - id = albumAPI['root_artist']['id'], - name = albumAPI['root_artist']['name'], - pic_md5 = artistPicture + albumAPI['root_artist']['id'], + albumAPI['root_artist']['name'], + "Root", + art_pic ) for artist in albumAPI['contributors']: @@ -60,7 +63,7 @@ class Album: if isVariousArtists: self.variousArtists = Artist( - id = artist['id'], + art_id = artist['id'], name = artist['name'], role = artist['role'] ) @@ -81,10 +84,10 @@ class Album: self.label = albumAPI.get('label', self.label) self.explicit = bool(albumAPI.get('explicit_lyrics', False)) if 'release_date' in albumAPI: - day = albumAPI["release_date"][8:10] - month = albumAPI["release_date"][5:7] - year = albumAPI["release_date"][0:4] - self.date = Date(day, month, year) + self.date.day = albumAPI["release_date"][8:10] + self.date.month = albumAPI["release_date"][5:7] + self.date.year = albumAPI["release_date"][0:4] + self.date.fixDayMonth() self.discTotal = albumAPI.get('nb_disk') self.copyright = albumAPI.get('copyright') @@ -92,7 +95,8 @@ class Album: if self.pic.md5 == "": # Getting album cover MD5 # ex: https://e-cdns-images.dzcdn.net/images/cover/2e018122cb56986277102d2041a592c8/56x56-000000-80-0-0.jpg - self.pic.md5 = albumAPI['cover_small'][albumAPI['cover_small'].find('cover/') + 6:-24] + alb_pic = albumAPI['cover_small'] + self.pic.md5 = alb_pic[alb_pic.find('cover/') + 6:-24] if albumAPI.get('genres') and len(albumAPI['genres'].get('data', [])) > 0: for genre in albumAPI['genres']['data']: @@ -101,8 +105,9 @@ class Album: def parseAlbumGW(self, albumAPI_gw): self.title = albumAPI_gw['ALB_TITLE'] self.mainArtist = Artist( - id = albumAPI_gw['ART_ID'], - name = albumAPI_gw['ART_NAME'] + art_id = albumAPI_gw['ART_ID'], + name = albumAPI_gw['ART_NAME'], + role = "Main" ) self.artists = [albumAPI_gw['ART_NAME']] @@ -113,13 +118,16 @@ class Album: explicitLyricsStatus = albumAPI_gw.get('EXPLICIT_ALBUM_CONTENT', {}).get('EXPLICIT_LYRICS_STATUS', LyricsStatus.UNKNOWN) self.explicit = explicitLyricsStatus in [LyricsStatus.EXPLICIT, LyricsStatus.PARTIALLY_EXPLICIT] + self.addExtraAlbumGWData(albumAPI_gw) + + def addExtraAlbumGWData(self, albumAPI_gw): if self.pic.md5 == "": self.pic.md5 = albumAPI_gw['ALB_PICTURE'] if 'PHYSICAL_RELEASE_DATE' in albumAPI_gw: - day = albumAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] - month = albumAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] - year = albumAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] - self.date = Date(day, month, year) + self.date.day = albumAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] + self.date.month = albumAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] + self.date.year = albumAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] + self.date.fixDayMonth() def makePlaylistCompilation(self, playlist): self.variousArtists = playlist.variousArtists @@ -138,6 +146,7 @@ class Album: self.playlistId = playlist.playlistId self.owner = playlist.owner self.pic = playlist.pic + self.isPlaylist = True def removeDuplicateArtists(self): """Removes duplicate artists for both artist array and artists dict""" diff --git a/deemix/types/Artist.py b/deemix/types/Artist.py index 576dc0a..9b18262 100644 --- a/deemix/types/Artist.py +++ b/deemix/types/Artist.py @@ -5,7 +5,7 @@ class Artist: def __init__(self, art_id="0", name="", role="", pic_md5=""): self.id = str(art_id) self.name = name - self.pic = Picture(md5=pic_md5, type="artist") + self.pic = Picture(md5=pic_md5, pic_type="artist") self.role = role self.save = True diff --git a/deemix/types/Date.py b/deemix/types/Date.py index 196612c..e0c73d3 100644 --- a/deemix/types/Date.py +++ b/deemix/types/Date.py @@ -1,8 +1,8 @@ class Date: def __init__(self, day="00", month="00", year="XXXX"): - self.year = year - self.month = month self.day = day + self.month = month + self.year = year self.fixDayMonth() # Fix incorrect day month when detectable diff --git a/deemix/types/DownloadObjects.py b/deemix/types/DownloadObjects.py index d725561..b0c684b 100644 --- a/deemix/types/DownloadObjects.py +++ b/deemix/types/DownloadObjects.py @@ -1,5 +1,5 @@ class IDownloadObject: - """DownloadObject interface""" + """DownloadObject Interface""" def __init__(self, obj): self.type = obj['type'] self.id = obj['id'] @@ -16,7 +16,6 @@ class IDownloadObject: self.files = obj.get('files', []) self.progressNext = 0 self.uuid = f"{self.type}_{self.id}_{self.bitrate}" - self.ack = None self.__type__ = None def toDict(self): @@ -35,7 +34,6 @@ class IDownloadObject: 'progress': self.progress, 'errors': self.errors, 'files': self.files, - 'ack': self.ack, '__type__': self.__type__ } @@ -50,16 +48,29 @@ class IDownloadObject: def getSlimmedDict(self): light = self.toDict() - propertiesToDelete = ['single', 'collection', 'convertable'] + propertiesToDelete = ['single', 'collection', 'plugin', 'conversion_data'] for prop in propertiesToDelete: if prop in light: del light[prop] return light - def updateProgress(self, interface=None): + def getEssentialDict(self): + return { + 'type': self.type, + 'id': self.id, + 'bitrate': self.bitrate, + 'uuid': self.uuid, + 'title': self.title, + 'artist': self.artist, + 'cover': self.cover, + 'explicit': self.explicit, + 'size': self.size + } + + def updateProgress(self, listener=None): if round(self.progressNext) != self.progress and round(self.progressNext) % 2 == 0: self.progress = round(self.progressNext) - if interface: interface.send("updateQueue", {'uuid': self.uuid, 'progress': self.progress}) + if listener: listener.send("updateQueue", {'uuid': self.uuid, 'progress': self.progress}) class Single(IDownloadObject): def __init__(self, obj): @@ -73,13 +84,13 @@ class Single(IDownloadObject): item['single'] = self.single return item - def completeTrackProgress(self, interface=None): + def completeTrackProgress(self, listener=None): self.progressNext = 100 - self.updateProgress(interface) + self.updateProgress(listener) - def removeTrackProgress(self, interface=None): + def removeTrackProgress(self, listener=None): self.progressNext = 0 - self.updateProgress(interface) + self.updateProgress(listener) class Collection(IDownloadObject): def __init__(self, obj): @@ -92,13 +103,13 @@ class Collection(IDownloadObject): item['collection'] = self.collection return item - def completeTrackProgress(self, interface=None): + def completeTrackProgress(self, listener=None): self.progressNext += (1 / self.size) * 100 - self.updateProgress(interface) + self.updateProgress(listener) - def removeTrackProgress(self, interface=None): + def removeTrackProgress(self, listener=None): self.progressNext -= (1 / self.size) * 100 - self.updateProgress(interface) + self.updateProgress(listener) class Convertable(Collection): def __init__(self, obj): diff --git a/deemix/types/Lyrics.py b/deemix/types/Lyrics.py index 938e4da..dfb32ef 100644 --- a/deemix/types/Lyrics.py +++ b/deemix/types/Lyrics.py @@ -19,6 +19,6 @@ class Lyrics: else: notEmptyLine = line + 1 while syncLyricsJson[notEmptyLine]["line"] == "": - notEmptyLine = notEmptyLine + 1 + notEmptyLine += 1 timestamp = syncLyricsJson[notEmptyLine]["lrc_timestamp"] self.sync += timestamp + syncLyricsJson[line]["line"] + "\r\n" diff --git a/deemix/types/Picture.py b/deemix/types/Picture.py index 1488dd1..b3fed1b 100644 --- a/deemix/types/Picture.py +++ b/deemix/types/Picture.py @@ -1,12 +1,9 @@ class Picture: - def __init__(self, md5="", pic_type="", url=None): + def __init__(self, md5="", pic_type=""): self.md5 = md5 self.type = pic_type - self.staticUrl = url - - def generatePictureURL(self, size, pic_format): - if self.staticUrl: return self.staticUrl + def getURL(self, size, pic_format): url = "https://e-cdns-images.dzcdn.net/images/{}/{}/{size}x{size}".format( self.type, self.md5, @@ -23,3 +20,10 @@ class Picture: return url + '-none-100-0-0.png' return url+'.jpg' + +class StaticPicture: + def __init__(self, url): + self.staticURL = url + + def getURL(self): + return self.staticURL diff --git a/deemix/types/Playlist.py b/deemix/types/Playlist.py index 9d85455..f936f44 100644 --- a/deemix/types/Playlist.py +++ b/deemix/types/Playlist.py @@ -1,6 +1,6 @@ from deemix.types.Artist import Artist from deemix.types.Date import Date -from deemix.types.Picture import Picture +from deemix.types.Picture import Picture, StaticPicture class Playlist: def __init__(self, playlistAPI): @@ -30,20 +30,17 @@ class Playlist: picType = url[url.find('images/')+7:] picType = picType[:picType.find('/')] md5 = url[url.find(picType+'/') + len(picType)+1:-24] - self.pic = Picture( - md5 = md5, - pic_type = picType - ) + self.pic = Picture(md5, picType) else: - self.pic = Picture(url = playlistAPI['picture_xl']) + self.pic = StaticPicture(playlistAPI['picture_xl']) if 'various_artist' in playlistAPI: pic_md5 = playlistAPI['various_artist']['picture_small'] pic_md5 = pic_md5[pic_md5.find('artist/') + 7:-24] self.variousArtists = Artist( - art_id = playlistAPI['various_artist']['id'], - name = playlistAPI['various_artist']['name'], - role = "Main", - pic_md5 = pic_md5 + playlistAPI['various_artist']['id'], + playlistAPI['various_artist']['name'], + "Main", + pic_md5 ) self.mainArtist = self.variousArtists diff --git a/deemix/types/Track.py b/deemix/types/Track.py index d7fb13f..5014601 100644 --- a/deemix/types/Track.py +++ b/deemix/types/Track.py @@ -26,14 +26,14 @@ class Track: self.duration = 0 self.fallbackID = "0" self.filesizes = {} - self.localTrack = False + self.local = False self.mainArtist = None self.artist = {"Main": []} self.artists = [] self.album = None self.trackNumber = "0" self.discNumber = "0" - self.date = None + self.date = Date() self.lyrics = None self.bpm = 0 self.contributors = {} @@ -64,7 +64,7 @@ class Track: self.fallbackID = "0" if 'FALLBACK' in trackAPI_gw: self.fallbackID = trackAPI_gw['FALLBACK']['SNG_ID'] - self.localTrack = int(self.id) < 0 + self.local = int(self.id) < 0 def retriveFilesizes(self, dz): guest_sid = dz.session.cookies.get('sid') @@ -87,8 +87,8 @@ class Track: sleep(2) self.retriveFilesizes(dz) if len(result_json['error']): - raise APIError(result_json.dumps(result_json['error'])) - response = result_json.get("results") + raise TrackError(result_json.dumps(result_json['error'])) + response = result_json.get("results", {}) filesizes = {} for key, value in response.items(): if key.startswith("FILESIZE_"): @@ -96,8 +96,8 @@ class Track: filesizes[key+"_TESTED"] = False self.filesizes = filesizes - def parseData(self, dz, id=None, trackAPI_gw=None, trackAPI=None, albumAPI_gw=None, albumAPI=None, playlistAPI=None): - if id and not trackAPI_gw: trackAPI_gw = dz.gw.get_track_with_fallback(id) + def parseData(self, dz, track_id=None, trackAPI_gw=None, trackAPI=None, albumAPI_gw=None, albumAPI=None, playlistAPI=None): + if track_id and not trackAPI_gw: trackAPI_gw = dz.gw.get_track_with_fallback(track_id) elif not trackAPI_gw: raise NoDataToParse if not trackAPI: try: trackAPI = dz.api.get_track(trackAPI_gw['SNG_ID']) @@ -105,7 +105,7 @@ class Track: self.parseEssentialData(trackAPI_gw, trackAPI) - if self.localTrack: + if self.local: self.parseLocalTrackData(trackAPI_gw) else: self.retriveFilesizes(dz) @@ -147,6 +147,7 @@ class Track: raise AlbumDoesntExists # Fill missing data + if albumAPI_gw: self.album.addExtraAlbumGWData(albumAPI_gw) if self.album.date and not self.date: self.date = self.album.date if not self.album.discTotal: self.album.discTotal = albumAPI_gw.get('NUMBER_DISK', "1") if not self.copyright: self.copyright = albumAPI_gw['COPYRIGHT'] @@ -157,10 +158,9 @@ class Track: self.title = ' '.join(self.title.split()) # Make sure there is at least one artist - if not len(self.artist['Main']): + if len(self.artist['Main']) == 0: self.artist['Main'] = [self.mainArtist['name']] - self.singleDownload = trackAPI_gw.get('SINGLE_TRACK', False) # TODO: Change self.position = trackAPI_gw.get('POSITION') # Add playlist data if track is in a playlist @@ -178,12 +178,11 @@ class Track: md5 = trackAPI_gw.get('ALB_PICTURE', ""), pic_type = "cover" ) - self.mainArtist = Artist(name=trackAPI_gw['ART_NAME']) + self.mainArtist = Artist(name=trackAPI_gw['ART_NAME'], role="Main") self.artists = [trackAPI_gw['ART_NAME']] self.artist = { 'Main': [trackAPI_gw['ART_NAME']] } - self.date = Date() self.album.artist = self.artist self.album.artists = self.artists self.album.date = self.date @@ -207,14 +206,15 @@ class Track: self.mainArtist = Artist( art_id = trackAPI_gw['ART_ID'], name = trackAPI_gw['ART_NAME'], + role = "Main", pic_md5 = trackAPI_gw.get('ART_PICTURE') ) if 'PHYSICAL_RELEASE_DATE' in trackAPI_gw: - day = trackAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] - month = trackAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] - year = trackAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] - self.date = Date(day, month, year) + self.date.day = trackAPI_gw["PHYSICAL_RELEASE_DATE"][8:10] + self.date.month = trackAPI_gw["PHYSICAL_RELEASE_DATE"][5:7] + self.date.year = trackAPI_gw["PHYSICAL_RELEASE_DATE"][0:4] + self.date.fixDayMonth() def parseTrack(self, trackAPI): self.bpm = trackAPI['bpm'] @@ -249,7 +249,7 @@ class Track: return removeFeatures(self.title) def getFeatTitle(self): - if self.featArtistsString and not "feat." in self.title.lower(): + if self.featArtistsString and "feat." not in self.title.lower(): return f"{self.title} ({self.featArtistsString})" return self.title @@ -259,26 +259,15 @@ class Track: if 'Featured' in self.artist: self.featArtistsString = "feat. "+andCommaConcat(self.artist['Featured']) - def applySettings(self, settings, TEMPDIR, embeddedImageFormat): + def applySettings(self, settings): # Check if should save the playlist as a compilation if self.playlist and settings['tags']['savePlaylistAsCompilation']: self.trackNumber = self.position self.discNumber = "1" self.album.makePlaylistCompilation(self.playlist) - self.album.embeddedCoverURL = self.playlist.pic.generatePictureURL(settings['embeddedArtworkSize'], embeddedImageFormat) - - ext = self.album.embeddedCoverURL[-4:] - if ext[0] != ".": ext = ".jpg" # Check for Spotify images - - # TODO: FIX - # self.album.embeddedCoverPath = TEMPDIR / f"pl{trackAPI_gw['_EXTRA_PLAYLIST']['id']}_{settings['embeddedArtworkSize']}{ext}" else: if self.album.date: self.date = self.album.date - self.album.embeddedCoverURL = self.album.pic.generatePictureURL(settings['embeddedArtworkSize'], embeddedImageFormat) - - ext = self.album.embeddedCoverURL[-4:] - self.album.embeddedCoverPath = TEMPDIR / f"alb{self.album.id}_{settings['embeddedArtworkSize']}{ext}" self.dateString = self.date.format(settings['dateFormat']) self.album.dateString = self.album.date.format(settings['dateFormat']) @@ -311,9 +300,8 @@ class Track: self.album.title = self.album.getCleanTitle() # Remove (Album Version) from tracks that have that - if settings['removeAlbumVersion']: - if "Album Version" in self.title: - self.title = re.sub(r' ?\(Album Version\)', "", self.title).strip() + if settings['removeAlbumVersion'] and "Album Version" in self.title: + self.title = re.sub(r' ?\(Album Version\)', "", self.title).strip() # Change Title and Artists casing if needed if settings['titleCasing'] != "nothing": diff --git a/deemix/utils/__init__.py b/deemix/utils/__init__.py index 48fa9eb..3b79e97 100644 --- a/deemix/utils/__init__.py +++ b/deemix/utils/__init__.py @@ -2,6 +2,12 @@ import string from deezer import TrackFormats import os +USER_AGENT_HEADER = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) " \ + "Chrome/79.0.3945.130 Safari/537.36" + +def canWrite(folder): + return os.access(folder, os.W_OK) + def generateReplayGainString(trackGain): return "{0:.2f} dB".format((float(trackGain) + 18.4) * -1) @@ -67,11 +73,3 @@ def removeDuplicateArtists(artist, artists): for role in artist.keys(): artist[role] = uniqueArray(artist[role]) return (artist, artists) - -def checkFolder(folder): - try: - os.makedirs(folder, exist_ok=True) - except Exception as e: - print(str(e)) - return False - return os.access(folder, os.W_OK) diff --git a/deemix/utils/crypto.py b/deemix/utils/crypto.py new file mode 100644 index 0000000..6edb49b --- /dev/null +++ b/deemix/utils/crypto.py @@ -0,0 +1,26 @@ +import binascii + +from Cryptodome.Cipher import Blowfish, AES +from Cryptodome.Hash import MD5 + +def _md5(data): + h = MD5.new() + h.update(data.encode() if isinstance(data, str) else data) + return h.hexdigest() + +def _ecbCrypt(key, data): + return binascii.hexlify(AES.new(key.encode(), AES.MODE_ECB).encrypt(data)) + +def _ecbDecrypt(key, data): + return AES.new(key.encode(), AES.MODE_ECB).decrypt(binascii.unhexlify(data.encode("utf-8"))) + +def generateBlowfishKey(trackId): + SECRET = 'g4el58wc0zvf9na1' + idMd5 = _md5(trackId) + bfKey = "" + for i in range(16): + bfKey += chr(ord(idMd5[i]) ^ ord(idMd5[i + 16]) ^ ord(SECRET[i])) + return bfKey + +def decryptChunk(key, data): + return Blowfish.new(key, Blowfish.MODE_CBC, b"\x00\x01\x02\x03\x04\x05\x06\x07").decrypt(data) diff --git a/deemix/utils/deezer.py b/deemix/utils/deezer.py new file mode 100644 index 0000000..7841a5e --- /dev/null +++ b/deemix/utils/deezer.py @@ -0,0 +1,32 @@ +import requests +from deemix.utils.crypto import _md5 +from deemix.utils import USER_AGENT_HEADER +CLIENT_ID = "172365" +CLIENT_SECRET = "fb0bec7ccc063dab0417eb7b0d847f34" + +def getAccessToken(email, password): + password = _md5(password) + request_hash = _md5(''.join([CLIENT_ID, email, password, CLIENT_SECRET])) + response = requests.get( + 'https://api.deezer.com/auth/token', + params={ + 'app_id': CLIENT_ID, + 'login': email, + 'password': password, + 'hash': request_hash + }, + headers={"User-Agent": USER_AGENT_HEADER} + ).json() + return response.get('access_token') + +def getArtFromAccessToken(accessToken): + session = requests.Session() + session.get( + "https://api.deezer.com/platform/generic/track/3135556", + headers={"Authorization": f"Bearer {accessToken}", "User-Agent": USER_AGENT_HEADER} + ) + response = session.get( + 'https://www.deezer.com/ajax/gw-light.php?method=user.getArl&input=3&api_version=1.0&api_token=null', + headers={"User-Agent": USER_AGENT_HEADER} + ).json() + return response.get('results') diff --git a/deemix/utils/localpaths.py b/deemix/utils/localpaths.py index 4250776..2e7670c 100644 --- a/deemix/utils/localpaths.py +++ b/deemix/utils/localpaths.py @@ -1,45 +1,72 @@ from pathlib import Path import sys import os -if os.name == 'nt': - import winreg # pylint: disable=E0401 +import re +from deemix.utils import canWrite homedata = Path.home() userdata = "" musicdata = "" - -if os.getenv("DEEMIX_DATA_DIR"): - userdata = Path(os.getenv("DEEMIX_DATA_DIR")) -elif os.getenv("XDG_CONFIG_HOME"): - userdata = Path(os.getenv("XDG_CONFIG_HOME")) / 'deemix' -elif os.getenv("APPDATA"): - userdata = Path(os.getenv("APPDATA")) / "deemix" -elif sys.platform.startswith('darwin'): - userdata = homedata / 'Library' / 'Application Support' / 'deemix' -else: - userdata = homedata / '.config' / 'deemix' - -if os.getenv("DEEMIX_MUSIC_DIR"): - musicdata = Path(os.getenv("DEEMIX_MUSIC_DIR")) -elif os.getenv("XDG_MUSIC_DIR"): - musicdata = Path(os.getenv("XDG_MUSIC_DIR")) / "deemix Music" -elif os.name == 'nt': - sub_key = r'SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders' - music_guid = '{4BD8D571-6D19-48D3-BE97-422220080E43}' - with winreg.OpenKey(winreg.HKEY_CURRENT_USER, sub_key) as key: - location = None - try: location = winreg.QueryValueEx(key, music_guid)[0] - except: pass - try: location = winreg.QueryValueEx(key, 'My Music')[0] - except: pass - if not location: location = homedata / "Music" - musicdata = Path(location) / "deemix Music" -else: - musicdata = homedata / "Music" / "deemix Music" +def checkPath(path): + if path == "": return "" + if not path.is_dir(): return "" + if not canWrite(path): return "" + return path def getConfigFolder(): + global userdata + if userdata != "": return userdata + if os.getenv("XDG_CONFIG_HOME") and userdata == "": + userdata = Path(os.getenv("XDG_CONFIG_HOME")) + userdata = checkPath(userdata) + if os.getenv("APPDATA") and userdata == "": + userdata = Path(os.getenv("APPDATA")) + userdata = checkPath(userdata) + if sys.platform.startswith('darwin') and userdata == "": + userdata = homedata / 'Library' / 'Application Support' + userdata = checkPath(userdata) + if userdata == "": + userdata = homedata / '.config' + userdata = checkPath(userdata) + + if userdata == "": userdata = Path(os.getcwd()) / 'config' + else: userdata = userdata / 'deemix' + + if os.getenv("DEEMIX_DATA_DIR"): + userdata = Path(os.getenv("DEEMIX_DATA_DIR")) return userdata def getMusicFolder(): + global musicdata + if musicdata != "": return musicdata + if os.getenv("XDG_MUSIC_DIR") and musicdata == "": + musicdata = Path(os.getenv("XDG_MUSIC_DIR")) + musicdata = checkPath(musicdata) + if (homedata / '.config' / 'user-dirs.dirs').is_file() and musicdata == "": + with open(homedata / '.config' / 'user-dirs.dirs', 'r') as f: + userDirs = f.read() + musicdata = re.search(r"XDG_MUSIC_DIR=\"(.*)\"", userDirs).group(1) + musicdata = Path(os.path.expandvars(musicdata)) + musicdata = checkPath(musicdata) + if os.name == 'nt' and musicdata == "": + musicKeys = ['My Music', '{4BD8D571-6D19-48D3-BE97-422220080E43}'] + regData = os.popen(r'reg.exe query "HKCU\SOFTWARE\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"').read().split('\r\n') + for i, line in enumerate(regData): + if line == "": continue + if i == 1: continue + line = line.split(' ') + if line[1] in musicKeys: + musicdata = Path(line[3]) + break + musicdata = checkPath(musicdata) + if musicdata == "": + musicdata = homedata / 'Music' + musicdata = checkPath(musicdata) + + if musicdata == "": musicdata = Path(os.getcwd()) / 'music' + else: musicdata = musicdata / 'deemix Music' + + if os.getenv("DEEMIX_MUSIC_DIR"): + musicdata = Path(os.getenv("DEEMIX_MUSIC_DIR")) return musicdata diff --git a/deemix/utils/pathtemplates.py b/deemix/utils/pathtemplates.py index 9e48b8a..32fc69a 100644 --- a/deemix/utils/pathtemplates.py +++ b/deemix/utils/pathtemplates.py @@ -21,14 +21,13 @@ def fixName(txt, char='_'): txt = normalize("NFC", txt) return txt -def fixEndOfData(bString): - try: - bString.decode() - return True - except: - return False - def fixLongName(name): + def fixEndOfData(bString): + try: + bString.decode() + return True + except Exception: + return False if pathSep in name: sepName = name.split(pathSep) name = "" @@ -63,18 +62,29 @@ def pad(num, max_val, settings): return str(num).zfill(paddingSize) return str(num) -def generateFilename(track, settings, template): - filename = template or "%artist% - %title%" - return settingsRegex(filename, track, settings) +def generatePath(track, downloadObject, settings): + filenameTemplate = "%artist% - %title%" + singleTrack = False + if downloadObject.type == "track": + if settings['createSingleFolder']: + filenameTemplate = settings['albumTracknameTemplate'] + else: + filenameTemplate = settings['tracknameTemplate'] + singleTrack = True + elif downloadObject.type == "album": + filenameTemplate = settings['albumTracknameTemplate'] + else: + filenameTemplate = settings['plyalistTracknameTemplate'] + + filename = generateTrackName(filenameTemplate, track, settings) -def generateFilepath(track, settings): - filepath = Path(settings['downloadLocation']) + filepath = Path(settings['downloadLocation'] or '.') artistPath = None coverPath = None extrasPath = None if settings['createPlaylistFolder'] and track.playlist and not settings['tags']['savePlaylistAsCompilation']: - filepath = filepath / settingsRegexPlaylist(settings['playlistNameTemplate'], track.playlist, settings) + filepath = filepath / generatePlaylistName(settings['playlistNameTemplate'], track.playlist, settings) if track.playlist and not settings['tags']['savePlaylistAsCompilation']: extrasPath = filepath @@ -84,61 +94,66 @@ def generateFilepath(track, settings): (settings['createArtistFolder'] and track.playlist and settings['tags']['savePlaylistAsCompilation']) or (settings['createArtistFolder'] and track.playlist and settings['createStructurePlaylist']) ): - filepath = filepath / settingsRegexArtist(settings['artistNameTemplate'], track.album.mainArtist, settings, rootArtist=track.album.rootArtist) + filepath = filepath / generateArtistName(settings['artistNameTemplate'], track.album.mainArtist, settings, rootArtist=track.album.rootArtist) artistPath = filepath if (settings['createAlbumFolder'] and - (not track.singleDownload or (track.singleDownload and settings['createSingleFolder'])) and + (not singleTrack or (singleTrack and settings['createSingleFolder'])) and (not track.playlist or (track.playlist and settings['tags']['savePlaylistAsCompilation']) or (track.playlist and settings['createStructurePlaylist']) ) ): - filepath = filepath / settingsRegexAlbum(settings['albumNameTemplate'], track.album, settings, track.playlist) + filepath = filepath / generateAlbumName(settings['albumNameTemplate'], track.album, settings, track.playlist) coverPath = filepath - if not (track.playlist and not settings['tags']['savePlaylistAsCompilation']): - extrasPath = filepath + if not extrasPath: extrasPath = filepath if ( - int(track.album.discTotal) > 1 and ( + int(track.album.discTotal) > 1 and ( (settings['createAlbumFolder'] and settings['createCDFolder']) and - (not track.singleDownload or (track.singleDownload and settings['createSingleFolder'])) and + (not singleTrack or (singleTrack and settings['createSingleFolder'])) and (not track.playlist or (track.playlist and settings['tags']['savePlaylistAsCompilation']) or (track.playlist and settings['createStructurePlaylist']) - ) + ) )): - filepath = filepath / f'CD{str(track.discNumber)}' + filepath = filepath / f'CD{track.discNumber}' + + # Remove subfolders from filename and add it to filepath + if pathSep in filename: + tempPath = filename[:filename.rfind(pathSep)] + filepath = filepath / tempPath + filename = filename[filename.rfind(pathSep) + len(pathSep):] - return (filepath, artistPath, coverPath, extrasPath) + return (filename, filepath, artistPath, coverPath, extrasPath) -def settingsRegex(filename, track, settings): - filename = filename.replace("%title%", fixName(track.title, settings['illegalCharacterReplacer'])) - filename = filename.replace("%artist%", fixName(track.mainArtist.name, settings['illegalCharacterReplacer'])) - filename = filename.replace("%artists%", fixName(", ".join(track.artists), settings['illegalCharacterReplacer'])) - filename = filename.replace("%allartists%", fixName(track.artistsString, settings['illegalCharacterReplacer'])) - filename = filename.replace("%mainartists%", fixName(track.mainArtistsString, settings['illegalCharacterReplacer'])) +def generateTrackName(filename, track, settings): + c = settings['illegalCharacterReplacer'] + filename = filename.replace("%title%", fixName(track.title, c)) + filename = filename.replace("%artist%", fixName(track.mainArtist.name, c)) + filename = filename.replace("%artists%", fixName(", ".join(track.artists), c)) + filename = filename.replace("%allartists%", fixName(track.artistsString, c)) + filename = filename.replace("%mainartists%", fixName(track.mainArtistsString, c)) if track.featArtistsString: - filename = filename.replace("%featartists%", fixName('('+track.featArtistsString+')', settings['illegalCharacterReplacer'])) + filename = filename.replace("%featartists%", fixName('('+track.featArtistsString+')', c)) else: filename = filename.replace("%featartists%", '') - filename = filename.replace("%album%", fixName(track.album.title, settings['illegalCharacterReplacer'])) - filename = filename.replace("%albumartist%", fixName(track.album.mainArtist.name, settings['illegalCharacterReplacer'])) + filename = filename.replace("%album%", fixName(track.album.title, c)) + filename = filename.replace("%albumartist%", fixName(track.album.mainArtist.name, c)) filename = filename.replace("%tracknumber%", pad(track.trackNumber, track.album.trackTotal, settings)) filename = filename.replace("%tracktotal%", str(track.album.trackTotal)) filename = filename.replace("%discnumber%", str(track.discNumber)) filename = filename.replace("%disctotal%", str(track.album.discTotal)) if len(track.album.genre) > 0: - filename = filename.replace("%genre%", - fixName(track.album.genre[0], settings['illegalCharacterReplacer'])) + filename = filename.replace("%genre%", fixName(track.album.genre[0], c)) else: filename = filename.replace("%genre%", "Unknown") filename = filename.replace("%year%", str(track.date.year)) filename = filename.replace("%date%", track.dateString) filename = filename.replace("%bpm%", str(track.bpm)) - filename = filename.replace("%label%", fixName(track.album.label, settings['illegalCharacterReplacer'])) + filename = filename.replace("%label%", fixName(track.album.label, c)) filename = filename.replace("%isrc%", track.ISRC) filename = filename.replace("%upc%", track.album.barcode) filename = filename.replace("%explicit%", "(Explicit)" if track.explicit else "") @@ -151,36 +166,37 @@ def settingsRegex(filename, track, settings): filename = filename.replace("%position%", pad(track.position, track.playlist.trackTotal, settings)) else: filename = filename.replace("%playlist_id%", '') - filename = filename.replace("%position%", pad(track.trackNumber, track.album.trackTotal, settings)) + filename = filename.replace("%position%", pad(track.position, track.album.trackTotal, settings)) filename = filename.replace('\\', pathSep).replace('/', pathSep) return antiDot(fixLongName(filename)) -def settingsRegexAlbum(foldername, album, settings, playlist=None): +def generateAlbumName(foldername, album, settings, playlist=None): + c = settings['illegalCharacterReplacer'] if playlist and settings['tags']['savePlaylistAsCompilation']: foldername = foldername.replace("%album_id%", "pl_" + str(playlist.playlistID)) foldername = foldername.replace("%genre%", "Compile") else: foldername = foldername.replace("%album_id%", str(album.id)) if len(album.genre) > 0: - foldername = foldername.replace("%genre%", fixName(album.genre[0], settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%genre%", fixName(album.genre[0], c)) else: foldername = foldername.replace("%genre%", "Unknown") - foldername = foldername.replace("%album%", fixName(album.title, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%artist%", fixName(album.mainArtist.name, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%album%", fixName(album.title, c)) + foldername = foldername.replace("%artist%", fixName(album.mainArtist.name, c)) foldername = foldername.replace("%artist_id%", str(album.mainArtist.id)) if album.rootArtist: - foldername = foldername.replace("%root_artist%", fixName(album.rootArtist.name, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%root_artist%", fixName(album.rootArtist.name, c)) foldername = foldername.replace("%root_artist_id%", str(album.rootArtist.id)) else: - foldername = foldername.replace("%root_artist%", fixName(album.mainArtist.name, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%root_artist%", fixName(album.mainArtist.name, c)) foldername = foldername.replace("%root_artist_id%", str(album.mainArtist.id)) foldername = foldername.replace("%tracktotal%", str(album.trackTotal)) foldername = foldername.replace("%disctotal%", str(album.discTotal)) - foldername = foldername.replace("%type%", fixName(album.recordType.capitalize(), settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%type%", fixName(album.recordType.capitalize(), c)) foldername = foldername.replace("%upc%", album.barcode) foldername = foldername.replace("%explicit%", "(Explicit)" if album.explicit else "") - foldername = foldername.replace("%label%", fixName(album.label, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%label%", fixName(album.label, c)) foldername = foldername.replace("%year%", str(album.date.year)) foldername = foldername.replace("%date%", album.dateString) foldername = foldername.replace("%bitrate%", bitrateLabels[int(album.bitrate)]) @@ -189,23 +205,25 @@ def settingsRegexAlbum(foldername, album, settings, playlist=None): return antiDot(fixLongName(foldername)) -def settingsRegexArtist(foldername, artist, settings, rootArtist=None): - foldername = foldername.replace("%artist%", fixName(artist.name, settings['illegalCharacterReplacer'])) +def generateArtistName(foldername, artist, settings, rootArtist=None): + c = settings['illegalCharacterReplacer'] + foldername = foldername.replace("%artist%", fixName(artist.name, c)) foldername = foldername.replace("%artist_id%", str(artist.id)) if rootArtist: - foldername = foldername.replace("%root_artist%", fixName(rootArtist.name, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%root_artist%", fixName(rootArtist.name, c)) foldername = foldername.replace("%root_artist_id%", str(rootArtist.id)) else: - foldername = foldername.replace("%root_artist%", fixName(artist.name, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%root_artist%", fixName(artist.name, c)) foldername = foldername.replace("%root_artist_id%", str(artist.id)) foldername = foldername.replace('\\', pathSep).replace('/', pathSep) return antiDot(fixLongName(foldername)) -def settingsRegexPlaylist(foldername, playlist, settings): - foldername = foldername.replace("%playlist%", fixName(playlist.title, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistID, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%owner%", fixName(playlist.owner['name'], settings['illegalCharacterReplacer'])) +def generatePlaylistName(foldername, playlist, settings): + c = settings['illegalCharacterReplacer'] + foldername = foldername.replace("%playlist%", fixName(playlist.title, c)) + foldername = foldername.replace("%playlist_id%", fixName(playlist.playlistID, c)) + foldername = foldername.replace("%owner%", fixName(playlist.owner['name'], c)) foldername = foldername.replace("%owner_id%", str(playlist.owner['id'])) foldername = foldername.replace("%year%", str(playlist.date.year)) foldername = foldername.replace("%date%", str(playlist.dateString)) @@ -213,12 +231,13 @@ def settingsRegexPlaylist(foldername, playlist, settings): foldername = foldername.replace('\\', pathSep).replace('/', pathSep) return antiDot(fixLongName(foldername)) -def settingsRegexPlaylistFile(foldername, queueItem, settings): - foldername = foldername.replace("%title%", fixName(queueItem.title, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%artist%", fixName(queueItem.artist, settings['illegalCharacterReplacer'])) +def generateDownloadObjectName(foldername, queueItem, settings): + c = settings['illegalCharacterReplacer'] + foldername = foldername.replace("%title%", fixName(queueItem.title, c)) + foldername = foldername.replace("%artist%", fixName(queueItem.artist, c)) foldername = foldername.replace("%size%", str(queueItem.size)) - foldername = foldername.replace("%type%", fixName(queueItem.type, settings['illegalCharacterReplacer'])) - foldername = foldername.replace("%id%", fixName(queueItem.id, settings['illegalCharacterReplacer'])) + foldername = foldername.replace("%type%", fixName(queueItem.type, c)) + foldername = foldername.replace("%id%", fixName(queueItem.id, c)) foldername = foldername.replace("%bitrate%", bitrateLabels[int(queueItem.bitrate)]) - foldername = foldername.replace('\\', pathSep).replace('/', pathSep).replace(pathSep, settings['illegalCharacterReplacer']) + foldername = foldername.replace('\\', pathSep).replace('/', pathSep).replace(pathSep, c) return antiDot(fixLongName(foldername)) diff --git a/setup.py b/setup.py index f0a818d..e853813 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ README = (HERE / "README.md").read_text() setup( name="deemix", - version="2.0.16", + version="3.0.0", description="A barebone deezer downloader library", long_description=README, long_description_content_type="text/markdown", -- 2.25.1 From 261b7adb364f1f5e1abd0ec53642fa49132a8fd7 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Tue, 8 Jun 2021 11:11:08 +0200 Subject: [PATCH 12/20] Fixed queue item not cancelling correctly --- deemix/downloader.py | 45 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/deemix/downloader.py b/deemix/downloader.py index 36286b1..85e99b9 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -150,32 +150,31 @@ class Downloader: self.playlistURLs = [] def start(self): - if self.downloadObject.isCanceled: + if not self.downloadObject.isCanceled: + if isinstance(self.downloadObject, Single): + track = self.downloadWrapper({ + 'trackAPI_gw': self.downloadObject.single['trackAPI_gw'], + 'trackAPI': self.downloadObject.single['trackAPI'], + 'albumAPI': self.downloadObject.single['albumAPI'] + }) + if track: self.afterDownloadSingle(track) + elif isinstance(self.downloadObject, Collection): + tracks = [None] * len(self.downloadObject.collection['tracks_gw']) + with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor: + for pos, track in enumerate(self.downloadObject.collection['tracks_gw'], start=0): + tracks[pos] = executor.submit(self.downloadWrapper, { + 'trackAPI_gw': track, + 'albumAPI': self.downloadObject.collection['albumAPI'], + 'playlistAPI': self.downloadObject.collection['playlistAPI'] + }) + self.afterDownloadCollection(tracks) + + if self.listener: if self.listener: self.listener.send('currentItemCancelled', self.downloadObject.uuid) self.listener.send("removedFromQueue", self.downloadObject.uuid) - return - - if isinstance(self.downloadObject, Single): - track = self.downloadWrapper({ - 'trackAPI_gw': self.downloadObject.single['trackAPI_gw'], - 'trackAPI': self.downloadObject.single['trackAPI'], - 'albumAPI': self.downloadObject.single['albumAPI'] - }) - if track: self.afterDownloadSingle(track) - elif isinstance(self.downloadObject, Collection): - tracks = [None] * len(self.downloadObject.collection['tracks_gw']) - with ThreadPoolExecutor(self.settings['queueConcurrency']) as executor: - for pos, track in enumerate(self.downloadObject.collection['tracks_gw'], start=0): - tracks[pos] = executor.submit(self.downloadWrapper, { - 'trackAPI_gw': track, - 'albumAPI': self.downloadObject.collection['albumAPI'], - 'playlistAPI': self.downloadObject.collection['playlistAPI'] - }) - self.afterDownloadCollection(tracks) - - if self.listener: - self.listener.send("finishDownload", self.downloadObject.uuid) + else: + self.listener.send("finishDownload", self.downloadObject.uuid) def download(self, extraData, track=None): returnData = {} -- 2.25.1 From c47e394039ea6a756bdac0bd9651319e97a6e02b Mon Sep 17 00:00:00 2001 From: RemixDev Date: Tue, 8 Jun 2021 11:16:52 +0200 Subject: [PATCH 13/20] Better handling of albums upcs --- deemix/itemgen.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/deemix/itemgen.py b/deemix/itemgen.py index 36821db..fd5f6b8 100644 --- a/deemix/itemgen.py +++ b/deemix/itemgen.py @@ -49,12 +49,25 @@ def generateTrackItem(dz, link_id, bitrate, trackAPI=None, albumAPI=None): def generateAlbumItem(dz, link_id, bitrate, rootArtist=None): # Get essential album info - try: - albumAPI = dz.api.get_album(link_id) - except APIError as e: - raise GenerationError(f"https://deezer.com/album/{link_id}", str(e)) from e + if str(link_id).startswith('upc'): + upcs = [link_id[4:],] + upcs.append(int(upcs[0])) + lastError = None + for upc in upcs: + try: + albumAPI = dz.api.get_album(f"upc:{upc}") + except APIError as e: + lastError = e + albumAPI = None + if not albumAPI: + raise GenerationError(f"https://deezer.com/album/{link_id}", str(lastError)) from lastError + link_id = albumAPI['id'] + else: + try: + albumAPI = dz.api.get_album(link_id) + except APIError as e: + raise GenerationError(f"https://deezer.com/album/{link_id}", str(e)) from e - if str(link_id).startswith('upc'): link_id = albumAPI['id'] if not link_id.isdecimal(): raise InvalidID(f"https://deezer.com/album/{link_id}") # Get extra info about album -- 2.25.1 From 8b7417f8c442ba8ac11e7eb33b3a5e8ecdaa5c4c Mon Sep 17 00:00:00 2001 From: RemixDev Date: Tue, 8 Jun 2021 19:11:46 +0200 Subject: [PATCH 14/20] Implemented spotify plugin back --- deemix/downloader.py | 4 + deemix/plugins/__init__.py | 12 ++ deemix/plugins/spotify.py | 346 +++++++++++++++++++++++++++++++++++++ 3 files changed, 362 insertions(+) create mode 100644 deemix/plugins/__init__.py diff --git a/deemix/downloader.py b/deemix/downloader.py index 85e99b9..58bd12d 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -373,6 +373,10 @@ class Downloader: def downloadWrapper(self, extraData, track=None): trackAPI_gw = extraData['trackAPI_gw'] + if ('_EXTRA_TRACK' in trackAPI_gw): + extraData['trackAPI'] = trackAPI_gw['_EXTRA_TRACK'].copy() + del extraData['trackAPI_gw']['_EXTRA_TRACK'] + del trackAPI_gw['_EXTRA_TRACK'] # Temp metadata to generate logs tempTrack = { 'id': trackAPI_gw['SNG_ID'], diff --git a/deemix/plugins/__init__.py b/deemix/plugins/__init__.py new file mode 100644 index 0000000..59bfeea --- /dev/null +++ b/deemix/plugins/__init__.py @@ -0,0 +1,12 @@ +class Plugin: + def __init__(self): + pass + + def setup(self): + pass + + def parseLink(self, link): + pass + + def generateDownloadObject(self, dz, link, bitrate, listener): + pass diff --git a/deemix/plugins/spotify.py b/deemix/plugins/spotify.py index e69de29..b36eb17 100644 --- a/deemix/plugins/spotify.py +++ b/deemix/plugins/spotify.py @@ -0,0 +1,346 @@ +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import re +from urllib.request import urlopen +from deemix.plugins import Plugin +from deemix.utils.localpaths import getConfigFolder +from deemix.itemgen import generateTrackItem, generateAlbumItem, GenerationError, TrackNotOnDeezer, AlbumNotOnDeezer +from deemix.types.DownloadObjects import Convertable + +import spotipy +SpotifyClientCredentials = spotipy.oauth2.SpotifyClientCredentials + +class Spotify(Plugin): + def __init__(self, configFolder=None): + super().__init__() + self.credentials = {'clientId': "", 'clientSecret': ""} + self.settings = { + 'fallbackSearch': False + } + self.enabled = False + self.sp = None + self.configFolder = Path(configFolder or getConfigFolder()) + self.configFolder /= 'spotify' + + def setup(self): + if not self.configFolder.is_dir(): self.configFolder.mkdir() + + self.loadSettings() + return self + + @classmethod + def parseLink(cls, link): + if 'link.tospotify.com' in link: link = urlopen(link).url + # Remove extra stuff + if '?' in link: link = link[:link.find('?')] + if '&' in link: link = link[:link.find('&')] + if link.endswith('/'): link = link[:-1] # Remove last slash if present + + link_type = None + link_id = None + + if not 'spotify' in link: return (link, link_type, link_id) # return if not a spotify link + + if re.search(r"[/:]track[/:](.+)", link): + link_type = 'track' + link_id = re.search(r"[/:]track[/:](.+)", link).group(1) + elif re.search(r"[/:]album[/:](.+)", link): + link_type = 'album' + link_id = re.search(r"[/:]album[/:](.+)", link).group(1) + elif re.search(r"[/:]playlist[/:](.+)", link): + link_type = 'playlist' + link_id = re.search(r"[/:]playlist[/:](.+)", link).group(1) + + return (link, link_type, link_id) + + def generateDownloadObject(self, dz, link, bitrate): + (link, link_type, link_id) = self.parseLink(link) + + if link_type is None or link_id is None: return None + + if link_type == "track": + return self.generateTrackItem(dz, link_id, bitrate) + if link_type == "album": + return self.generateAlbumItem(dz, link_id, bitrate) + if link_type == "playlist": + return self.generatePlaylistItem(dz, link_id, bitrate) + return None + + def generateTrackItem(self, dz, link_id, bitrate): + cache = self.loadCache() + + if link_id in cache['tracks']: + cachedTrack = cache['tracks'][link_id] + else: + cachedTrack = self.getTrack(link_id) + cache['tracks'][link_id] = cachedTrack + self.saveCache(cache) + + if 'isrc' in cachedTrack: + try: return generateTrackItem(dz, f"isrc:{cachedTrack['isrc']}", bitrate) + except GenerationError: pass + if self.settings['fallbackSearch']: + if 'id' not in cachedTrack or cachedTrack['id'] == "0": + trackID = dz.api.get_track_id_from_metadata( + cachedTrack['data']['artist'], + cachedTrack['data']['title'], + cachedTrack['data']['album'], + ) + if trackID != "0": + cachedTrack['id'] = trackID + cache['tracks'][link_id] = cachedTrack + self.saveCache(cache) + if cachedTrack['id'] != "0": return generateTrackItem(dz, cachedTrack['id'], bitrate) + raise TrackNotOnDeezer(f"https://open.spotify.com/track/{link_id}") + + def generateAlbumItem(self, dz, link_id, bitrate): + cache = self.loadCache() + + if link_id in cache['albums']: + cachedAlbum = cache['albums'][link_id] + else: + cachedAlbum = self.getAlbum(link_id) + cache['albums'][link_id] = cachedAlbum + self.saveCache(cache) + + try: return generateAlbumItem(dz, f"upc:{cachedAlbum['upc']}", bitrate) + except GenerationError as e: raise AlbumNotOnDeezer(f"https://open.spotify.com/album/{link_id}") from e + + def generatePlaylistItem(self, dz, link_id, bitrate): + if not self.enabled: raise Exception("Spotify plugin not enabled") + spotifyPlaylist = self.sp.playlist(link_id) + + playlistAPI = self._convertPlaylistStructure(spotifyPlaylist) + playlistAPI.various_artist = dz.api.get_artist(5080) # Useful for save as compilation + + tracklistTemp = spotifyPlaylist.track.items + while spotifyPlaylist['tracks']['next']: + spotifyPlaylist['tracks'] = self.sp.next(spotifyPlaylist['tracks']) + tracklistTemp += spotifyPlaylist['tracks']['items'] + + tracklist = [] + for item in tracklistTemp: + if item['track']: + if item['track']['explicit']: + playlistAPI['explicit'] = True + tracklist.append(item['track']) + if 'explicit' not in playlistAPI: playlistAPI['explicit'] = False + + return Convertable({ + 'type': 'spotify_playlist', + 'id': link_id, + 'bitrate': bitrate, + 'title': spotifyPlaylist['name'], + 'artist': spotifyPlaylist['owner']['display_name'], + 'cover': playlistAPI['picture_thumbnail'], + 'explicit': playlistAPI['explicit'], + 'size': len(tracklist), + 'collection': { + 'tracks_gw': [], + 'playlistAPI': playlistAPI + }, + 'plugin': 'spotify', + 'conversion_data': tracklist + }) + + def getTrack(self, track_id, spotifyTrack=None): + if not self.enabled: raise Exception("Spotify plugin not enabled") + cachedTrack = { + 'isrc': None, + 'data': None + } + + if not spotifyTrack: + spotifyTrack = self.sp.track(track_id) + if 'isrc' in spotifyTrack.get('external_ids', {}): + cachedTrack['isrc'] = spotifyTrack['external_ids']['isrc'] + cachedTrack['data'] = { + 'title': spotifyTrack['name'], + 'artist': spotifyTrack['artists'][0]['name'], + 'album': spotifyTrack['album']['name'] + } + return cachedTrack + + def getAlbum(self, album_id, spotifyAlbum=None): + if not self.enabled: raise Exception("Spotify plugin not enabled") + cachedAlbum = { + 'upc': None, + 'data': None + } + + if not spotifyAlbum: + spotifyAlbum = self.sp.album(album_id) + if 'upc' in spotifyAlbum.get('external_ids', {}): + cachedAlbum['upc'] = spotifyAlbum['external_ids']['upc'] + cachedAlbum['data'] = { + 'title': spotifyAlbum['name'], + 'artist': spotifyAlbum['artists'][0]['name'] + } + return cachedAlbum + + def convertTrack(self, dz, downloadObject, track, pos, conversion, conversionNext, cache, listener): + if downloadObject.isCanceled: return + + if track['id'] in cache['tracks']: + cachedTrack = cache['tracks'][track['id']] + else: + cachedTrack = self.getTrack(track['id'], track) + cache['tracks'][track['id']] = cachedTrack + self.saveCache(cache) + + if 'isrc' in cachedTrack: + try: + trackAPI = dz.api.get_track_by_ISRC(cachedTrack['isrc']) + if 'id' not in trackAPI or 'title' not in trackAPI: trackAPI = None + except GenerationError: pass + if self.settings['fallbackSearch'] and not trackAPI: + if 'id' not in cachedTrack or cachedTrack['id'] == "0": + trackID = dz.api.get_track_id_from_metadata( + cachedTrack['data']['artist'], + cachedTrack['data']['title'], + cachedTrack['data']['album'], + ) + if trackID != "0": + cachedTrack['id'] = trackID + cache['tracks'][track['id']] = cachedTrack + self.saveCache(cache) + if cachedTrack['id'] != "0": trackAPI = dz.api.get_track(cachedTrack['id']) + + if not trackAPI: + deezerTrack = { + 'SNG_ID': "0", + 'SNG_TITLE': track['name'], + 'DURATION': 0, + 'MD5_ORIGIN': 0, + 'MEDIA_VERSION': 0, + 'FILESIZE': 0, + 'ALB_TITLE': track['album']['name'], + 'ALB_PICTURE': "", + 'ART_ID': 0, + 'ART_NAME': track['artists'][0]['name'] + } + else: + deezerTrack = dz.gw.get_track_with_fallback(trackAPI['id']) + deezerTrack['_EXTRA_TRACK'] = trackAPI + deezerTrack['POSITION'] = pos+1 + + conversionNext += (1 / downloadObject.size) * 100 + if round(conversionNext) != conversion and round(conversionNext) % 2 == 0: + conversion = round(conversionNext) + if listener: listener.send("updateQueue", {'uuid': downloadObject.uuid, 'conversion': conversion}) + + def convert(self, dz, downloadObject, settings, listener=None): + cache = self.loadCache() + + conversion = 0 + conversionNext = 0 + + collection = [None] * len(downloadObject.conversion_data) + with ThreadPoolExecutor(settings['queueConcurrency']) as executor: + for pos, track in enumerate(downloadObject.conversion_data, start=0): + collection[pos] = executor.submit(self.convertTrack, + dz, downloadObject, + track, pos, + conversion, conversionNext, + cache, listener + ) + + @classmethod + def _convertPlaylistStructure(cls, spotifyPlaylist): + cover = None + if len(spotifyPlaylist['images']): cover = spotifyPlaylist['images'][0]['url'] + + deezerPlaylist = { + 'checksum': spotifyPlaylist['snapshot_id'], + 'collaborative': spotifyPlaylist['collaborative'], + 'creation_date': "XXXX-00-00", + 'creator': { + 'id': spotifyPlaylist['owner']['id'], + 'name': spotifyPlaylist['owner']['display_name'], + 'tracklist': spotifyPlaylist['owner']['href'], + 'type': "user" + }, + 'description': spotifyPlaylist['description'], + 'duration': 0, + 'fans': spotifyPlaylist['followers']['total'] if 'followers' in spotifyPlaylist else 0, + 'id': spotifyPlaylist['id'], + 'is_loved_track': False, + 'link': spotifyPlaylist['external_urls']['spotify'], + 'nb_tracks': spotifyPlaylist['tracks']['total'], + 'picture': cover, + 'picture_small': cover or "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/56x56-000000-80-0-0.jpg", + 'picture_medium': cover or "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/250x250-000000-80-0-0.jpg", + 'picture_big': cover or "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/500x500-000000-80-0-0.jpg", + 'picture_xl': cover or "https://e-cdns-images.dzcdn.net/images/cover/d41d8cd98f00b204e9800998ecf8427e/1000x1000-000000-80-0-0.jpg", + 'public': spotifyPlaylist['public'], + 'share': spotifyPlaylist['external_urls']['spotify'], + 'title': spotifyPlaylist['name'], + 'tracklist': spotifyPlaylist['tracks']['href'], + 'type': "playlist" + } + return deezerPlaylist + + def loadSettings(self): + if not (self.configFolder / 'settings.json').is_file(): + with open(self.configFolder / 'settings.json', 'w') as f: + json.dump({**self.credentials, **self.settings}, f, indent=2) + + with open(self.configFolder / 'settings.json', 'r') as settingsFile: + settings = json.load(settingsFile) + self.setSettings(settings) + self.checkCredentials() + + def saveSettings(self, newSettings=None): + if newSettings: self.setSettings(newSettings) + self.checkCredentials() + with open(self.configFolder / 'settings.json', 'w') as f: + json.dump({**self.credentials, **self.settings}, f, indent=2) + + def getSettings(self): + return {**self.credentials, **self.settings} + + def setSettings(self, newSettings): + self.credentials = { 'clientId': newSettings['clientId'], 'clientSecret': newSettings['clientSecret'] } + settings = {**newSettings} + del settings['clientId'] + del settings['clientSecret'] + self.settings = settings + + def loadCache(self): + if (self.configFolder / 'cache.json').is_file(): + with open(self.configFolder / 'cache.json', 'r') as f: + cache = json.load(f) + else: + cache = {'tracks': {}, 'albums': {}} + return cache + + def saveCache(self, newCache): + with open(self.configFolder / 'cache.json', 'w') as spotifyCache: + json.dump(newCache, spotifyCache) + + def checkCredentials(self): + if self.credentials['clientId'] == "" or self.credentials['clientSecret'] == "": + self.enabled = False + return + + try: + client_credentials_manager = SpotifyClientCredentials(client_id=self.credentials['clientId'], + client_secret=self.credentials['clientSecret']) + self.sp = spotipy.Spotify(client_credentials_manager=client_credentials_manager) + self.sp.user_playlists('spotify') + self.enabled = True + except Exception: + self.enabled = False + + def getCredentials(self): + return self.credentials + + def setCredentials(self, clientId, clientSecret): + # Remove extra spaces, just to be sure + clientId = clientId.strip() + clientSecret = clientSecret.strip() + + # Save them to disk + self.credentials = { 'clientId': clientId, 'clientSecret': clientSecret} + self.saveSettings() -- 2.25.1 From 97cd903289b5cf23ec6fb7017d2093f8ed6d98ba Mon Sep 17 00:00:00 2001 From: RemixDev Date: Tue, 8 Jun 2021 19:12:15 +0200 Subject: [PATCH 15/20] Fixed some issues to make the lib work --- deemix/downloader.py | 18 ++++++++++-------- deemix/settings.py | 6 +++--- deemix/types/DownloadObjects.py | 3 ++- deemix/types/Lyrics.py | 2 +- deemix/utils/pathtemplates.py | 2 +- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/deemix/downloader.py b/deemix/downloader.py index 58bd12d..22327ab 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -154,8 +154,8 @@ class Downloader: if isinstance(self.downloadObject, Single): track = self.downloadWrapper({ 'trackAPI_gw': self.downloadObject.single['trackAPI_gw'], - 'trackAPI': self.downloadObject.single['trackAPI'], - 'albumAPI': self.downloadObject.single['albumAPI'] + 'trackAPI': self.downloadObject.single.get('trackAPI'), + 'albumAPI': self.downloadObject.single.get('albumAPI') }) if track: self.afterDownloadSingle(track) elif isinstance(self.downloadObject, Collection): @@ -164,8 +164,8 @@ class Downloader: for pos, track in enumerate(self.downloadObject.collection['tracks_gw'], start=0): tracks[pos] = executor.submit(self.downloadWrapper, { 'trackAPI_gw': track, - 'albumAPI': self.downloadObject.collection['albumAPI'], - 'playlistAPI': self.downloadObject.collection['playlistAPI'] + 'albumAPI': self.downloadObject.collection.get('albumAPI'), + 'playlistAPI': self.downloadObject.collection.get('playlistAPI') }) self.afterDownloadCollection(tracks) @@ -179,9 +179,9 @@ class Downloader: def download(self, extraData, track=None): returnData = {} trackAPI_gw = extraData['trackAPI_gw'] - trackAPI = extraData['trackAPI'] - albumAPI = extraData['albumAPI'] - playlistAPI = extraData['playlistAPI'] + trackAPI = extraData.get('trackAPI') + albumAPI = extraData.get('albumAPI') + playlistAPI = extraData.get('playlistAPI') if self.downloadObject.isCanceled: raise DownloadCanceled if trackAPI_gw['SNG_ID'] == "0": raise DownloadFailed("notOnDeezer") @@ -327,6 +327,8 @@ class Downloader: try: with open(writepath, 'wb') as stream: streamTrack(stream, track, downloadObject=self.downloadObject, listener=self.listener) + except requests.exceptions.HTTPError as e: + raise DownloadFailed('notAvailable', track) from e except OSError as e: if writepath.is_file(): writepath.unlink() if e.errno == errno.ENOSPC: raise DownloadFailed("noSpaceLeft") from e @@ -386,7 +388,7 @@ class Downloader: if trackAPI_gw.get('VERSION') and trackAPI_gw['VERSION'] not in trackAPI_gw['SNG_TITLE']: tempTrack['title'] += f" {trackAPI_gw['VERSION']}".strip() - itemName = f"[{track.mainArtist.name} - {track.title}]" + itemName = f"[{tempTrack['artist']} - {tempTrack['title']}]" try: result = self.download(extraData, track) diff --git a/deemix/settings.py b/deemix/settings.py index e1298ef..15f15f0 100644 --- a/deemix/settings.py +++ b/deemix/settings.py @@ -20,7 +20,7 @@ class FeaturesOption(): MOVE_TITLE = "2" # Move to track title DEFAULTS = { - "downloadLocation": localpaths.getMusicFolder(), + "downloadLocation": str(localpaths.getMusicFolder()), "tracknameTemplate": "%artist% - %title%", "albumTracknameTemplate": "%tracknumber% - %title%", "playlistTracknameTemplate": "%position% - %artist% - %title%", @@ -122,11 +122,11 @@ def load(configFolder=None): def check(settings): changes = 0 for i_set in DEFAULTS: - if not i_set in settings or not isinstance(settings[i_set], DEFAULTS[i_set]): + if not i_set in settings or not isinstance(settings[i_set], type(DEFAULTS[i_set])): settings[i_set] = DEFAULTS[i_set] changes += 1 for i_set in DEFAULTS['tags']: - if not i_set in settings['tags'] or not isinstance(settings['tags'][i_set], DEFAULTS['tags'][i_set]): + if not i_set in settings['tags'] or not isinstance(settings['tags'][i_set], type(DEFAULTS['tags'][i_set])): settings['tags'][i_set] = DEFAULTS['tags'][i_set] changes += 1 if settings['downloadLocation'] == "": diff --git a/deemix/types/DownloadObjects.py b/deemix/types/DownloadObjects.py index b0c684b..a2f54ac 100644 --- a/deemix/types/DownloadObjects.py +++ b/deemix/types/DownloadObjects.py @@ -8,7 +8,7 @@ class IDownloadObject: self.artist = obj['artist'] self.cover = obj['cover'] self.explicit = obj.get('explicit', False) - self.size = obj['size'] + self.size = obj.get('size', 0) self.downloaded = obj.get('downloaded', 0) self.failed = obj.get('failed', 0) self.progress = obj.get('progress', 0) @@ -16,6 +16,7 @@ class IDownloadObject: self.files = obj.get('files', []) self.progressNext = 0 self.uuid = f"{self.type}_{self.id}_{self.bitrate}" + self.isCanceled = False self.__type__ = None def toDict(self): diff --git a/deemix/types/Lyrics.py b/deemix/types/Lyrics.py index dfb32ef..f16e960 100644 --- a/deemix/types/Lyrics.py +++ b/deemix/types/Lyrics.py @@ -11,7 +11,7 @@ class Lyrics: syncLyricsJson = lyricsAPI["LYRICS_SYNC_JSON"] timestamp = "" milliseconds = 0 - for line in enumerate(syncLyricsJson): + for line, _ in enumerate(syncLyricsJson): if syncLyricsJson[line]["line"] != "": timestamp = syncLyricsJson[line]["lrc_timestamp"] milliseconds = int(syncLyricsJson[line]["milliseconds"]) diff --git a/deemix/utils/pathtemplates.py b/deemix/utils/pathtemplates.py index 32fc69a..b654428 100644 --- a/deemix/utils/pathtemplates.py +++ b/deemix/utils/pathtemplates.py @@ -74,7 +74,7 @@ def generatePath(track, downloadObject, settings): elif downloadObject.type == "album": filenameTemplate = settings['albumTracknameTemplate'] else: - filenameTemplate = settings['plyalistTracknameTemplate'] + filenameTemplate = settings['playlistTracknameTemplate'] filename = generateTrackName(filenameTemplate, track, settings) -- 2.25.1 From 5dc923d057659852e3d9f3b289295501f5651ef9 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Tue, 8 Jun 2021 19:39:56 +0200 Subject: [PATCH 16/20] Fixed bitrate fallback not working --- deemix/downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deemix/downloader.py b/deemix/downloader.py index 22327ab..22a1915 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -112,7 +112,7 @@ def getPreferredBitrate(track, bitrate, shouldFallback, uuid=None, listener=None return None for formatNumber, formatName in formats.items(): - if formatNumber >= int(bitrate): continue + if formatNumber > bitrate: continue if f"FILESIZE_{formatName}" in track.filesizes: if int(track.filesizes[f"FILESIZE_{formatName}"]) != 0: return formatNumber if not track.filesizes[f"FILESIZE_{formatName}_TESTED"]: -- 2.25.1 From f8b835229cad0b58500918ac67a62b9a29155842 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Sun, 13 Jun 2021 14:06:05 +0200 Subject: [PATCH 17/20] Use overwriteFile setting when downloading embedded covers --- deemix/downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deemix/downloader.py b/deemix/downloader.py index 22a1915..e2daa84 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -248,7 +248,7 @@ class Downloader: # Download and cache coverart logger.info("%s Getting the album cover", itemName) - track.album.embeddedCoverPath = downloadImage(track.album.embeddedCoverURL, track.album.embeddedCoverPath) + track.album.embeddedCoverPath = downloadImage(track.album.embeddedCoverURL, track.album.embeddedCoverPath, self.settings.overwriteFile) # Save local album art if coverPath: -- 2.25.1 From c42eff7f95c19d48dcea6d389f93fadfb41b928a Mon Sep 17 00:00:00 2001 From: RemixDev Date: Sun, 13 Jun 2021 14:06:17 +0200 Subject: [PATCH 18/20] Fixed bitrate fallback check --- deemix/downloader.py | 3 ++- deemix/types/Track.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/deemix/downloader.py b/deemix/downloader.py index e2daa84..ea9659c 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -105,8 +105,9 @@ def getPreferredBitrate(track, bitrate, shouldFallback, uuid=None, listener=None ) try: request.raise_for_status() - track.filesizes[f"FILESIZE_{formatName}"] = request.headers["Content-Length"] + track.filesizes[f"FILESIZE_{formatName}"] = int(request.headers["Content-Length"]) track.filesizes[f"FILESIZE_{formatName}_TESTED"] = True + if track.filesizes[f"FILESIZE_{formatName}"] == 0: return None return formatNumber except requests.exceptions.HTTPError: # if the format is not available, Deezer returns a 403 error return None diff --git a/deemix/types/Track.py b/deemix/types/Track.py index 5014601..85e359e 100644 --- a/deemix/types/Track.py +++ b/deemix/types/Track.py @@ -92,7 +92,7 @@ class Track: filesizes = {} for key, value in response.items(): if key.startswith("FILESIZE_"): - filesizes[key] = value + filesizes[key] = int(value) filesizes[key+"_TESTED"] = False self.filesizes = filesizes -- 2.25.1 From 9827cfddb01cb839690f317d351f0c867b791e60 Mon Sep 17 00:00:00 2001 From: RemixDev Date: Wed, 16 Jun 2021 15:27:22 +0200 Subject: [PATCH 19/20] Revert embedded cover change --- deemix/downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deemix/downloader.py b/deemix/downloader.py index ea9659c..a9c7247 100644 --- a/deemix/downloader.py +++ b/deemix/downloader.py @@ -249,7 +249,7 @@ class Downloader: # Download and cache coverart logger.info("%s Getting the album cover", itemName) - track.album.embeddedCoverPath = downloadImage(track.album.embeddedCoverURL, track.album.embeddedCoverPath, self.settings.overwriteFile) + track.album.embeddedCoverPath = downloadImage(track.album.embeddedCoverURL, track.album.embeddedCoverPath) # Save local album art if coverPath: -- 2.25.1 From be131d914f35a1ec2e561feb5bde053f0fdfac1c Mon Sep 17 00:00:00 2001 From: RemixDev Date: Sun, 27 Jun 2021 19:53:41 +0200 Subject: [PATCH 20/20] Removed saveDownloadQueue and tagsLanguage from lib settings --- deemix/__main__.py | 2 +- deemix/settings.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/deemix/__main__.py b/deemix/__main__.py index ce87e19..7885794 100644 --- a/deemix/__main__.py +++ b/deemix/__main__.py @@ -22,7 +22,7 @@ def download(url, bitrate, portable, path): configFolder = localpath / 'config' if portable else localpaths.getConfigFolder() settings = loadSettings(configFolder) - dz = Deezer(settings.get('tagsLanguage')) + dz = Deezer(settings.get('tagsLanguage', "")) def requestValidArl(): while True: diff --git a/deemix/settings.py b/deemix/settings.py index 15f15f0..d1c6d5e 100644 --- a/deemix/settings.py +++ b/deemix/settings.py @@ -42,7 +42,6 @@ DEFAULTS = { "fallbackSearch": False, "logErrors": True, "logSearched": False, - "saveDownloadQueue": False, "overwriteFile": OverwriteOption.DONT_OVERWRITE, "createM3U8File": False, "playlistFilenameTemplate": "playlist", @@ -60,7 +59,6 @@ DEFAULTS = { "albumVariousArtists": True, "removeAlbumVersion": False, "removeDuplicateArtists": False, - "tagsLanguage": "", "featuredToTitle": FeaturesOption.NO_CHANGE, "titleCasing": "nothing", "artistCasing": "nothing", -- 2.25.1