diff options
| author | Robert Honz <exislow+github@gmail.com> | 2024-11-29 17:41:25 +0100 |
|---|---|---|
| committer | Robert Honz <exislow+github@gmail.com> | 2024-11-29 17:41:25 +0100 |
| commit | 33a30537e9a205dd4b43e891b00c0b5f15a9b71b (patch) | |
| tree | 9ccb8c4c273ce96fd6ebce8ea2077d281d812712 /tidal_dl_ng | |
| parent | f2b3cc2505f4c64f09efacd775de3390ae204be0 (diff) | |
| parent | 4deaf94fb53ad304db73f9384f817b29afcac4a8 (diff) | |
| download | tidal-dl-ng-33a30537e9a205dd4b43e891b00c0b5f15a9b71b.tar.gz | |
✨ Merge with master.
Diffstat (limited to 'tidal_dl_ng')
| -rw-r--r-- | tidal_dl_ng/constants.py | 10 | ||||
| -rw-r--r-- | tidal_dl_ng/dialog.py | 4 | ||||
| -rw-r--r-- | tidal_dl_ng/download.py | 432 | ||||
| -rw-r--r-- | tidal_dl_ng/gui.py | 2 | ||||
| -rw-r--r-- | tidal_dl_ng/helper/decryption.py | 7 | ||||
| -rw-r--r-- | tidal_dl_ng/helper/path.py | 36 | ||||
| -rw-r--r-- | tidal_dl_ng/helper/tidal.py | 8 | ||||
| -rw-r--r-- | tidal_dl_ng/helper/wrapper.py | 3 | ||||
| -rw-r--r-- | tidal_dl_ng/metadata.py | 2 | ||||
| -rw-r--r-- | tidal_dl_ng/model/cfg.py | 18 | ||||
| -rw-r--r-- | tidal_dl_ng/model/downloader.py | 13 | ||||
| -rw-r--r-- | tidal_dl_ng/ui/dialog_settings.py | 52 | ||||
| -rw-r--r-- | tidal_dl_ng/ui/dialog_settings.ui | 51 |
13 files changed, 385 insertions, 253 deletions
diff --git a/tidal_dl_ng/constants.py b/tidal_dl_ng/constants.py index d3ea2b8..10fb93d 100644 --- a/tidal_dl_ng/constants.py +++ b/tidal_dl_ng/constants.py @@ -6,6 +6,9 @@ EXTENSION_LYRICS: str = ".lrc" UNIQUIFY_THRESHOLD: int = 99 FILENAME_SANITIZE_PLACEHOLDER: str = "_" COVER_NAME: str = "cover.jpg" +BLOCK_SIZE: int = 4096 +BLOCKS: int = 1024 +CHUNK_SIZE: int = BLOCK_SIZE * BLOCKS class QualityVideo(StrEnum): @@ -24,13 +27,6 @@ class MediaType(StrEnum): ARTIST: str = "artist" -class SkipExisting(StrEnum): - Disabled: str = "False" - Filename: str = "exact" - ExtensionIgnore: str = "extension_ignore" - Append: str = "append" - - class CoverDimensions(StrEnum): Px80: str = "80" Px160: str = "160" diff --git a/tidal_dl_ng/dialog.py b/tidal_dl_ng/dialog.py index ff31633..928f6c1 100644 --- a/tidal_dl_ng/dialog.py +++ b/tidal_dl_ng/dialog.py @@ -10,7 +10,7 @@ from tidalapi import Quality as QualityAudio from tidal_dl_ng import __version__ from tidal_dl_ng.config import Settings -from tidal_dl_ng.constants import CoverDimensions, QualityVideo, SkipExisting +from tidal_dl_ng.constants import CoverDimensions, QualityVideo from tidal_dl_ng.model.cfg import HelpSettings from tidal_dl_ng.model.cfg import Settings as ModelSettings from tidal_dl_ng.model.meta import ReleaseLatest @@ -159,7 +159,6 @@ class DialogPreferences(QtWidgets.QDialog): def _init_comboboxes(self): self.parameters_combo = [ - ("skip_existing", SkipExisting), ("quality_audio", QualityAudio), ("quality_video", QualityVideo), ("metadata_cover_dimension", CoverDimensions), @@ -175,6 +174,7 @@ class DialogPreferences(QtWidgets.QDialog): "extract_flac", "metadata_cover_embed", "cover_album_file", + "skip_existing", ] def gui_populate(self): diff --git a/tidal_dl_ng/download.py b/tidal_dl_ng/download.py index b6d5f13..c755411 100644 --- a/tidal_dl_ng/download.py +++ b/tidal_dl_ng/download.py @@ -5,22 +5,31 @@ import shutil import tempfile import time from collections.abc import Callable +from concurrent import futures from uuid import uuid4 import ffmpeg import m3u8 import requests -from constants import COVER_NAME +from requests.adapters import HTTPAdapter, Retry from requests.exceptions import HTTPError from rich.progress import Progress, TaskID from tidalapi import Album, Mix, Playlist, Session, Track, UserPlaylist, Video +from tidalapi.exceptions import TooManyRequests from tidalapi.media import AudioExtensions, Codec, Quality, StreamManifest, VideoExtensions from tidal_dl_ng.config import Settings -from tidal_dl_ng.constants import EXTENSION_LYRICS, REQUESTS_TIMEOUT_SEC, MediaType, QualityVideo, SkipExisting +from tidal_dl_ng.constants import ( + CHUNK_SIZE, + COVER_NAME, + EXTENSION_LYRICS, + REQUESTS_TIMEOUT_SEC, + MediaType, + QualityVideo, +) from tidal_dl_ng.helper.decryption import decrypt_file, decrypt_security_token from tidal_dl_ng.helper.exceptions import MediaMissing -from tidal_dl_ng.helper.path import check_file_exists, format_path_media, path_file_sanitize +from tidal_dl_ng.helper.path import check_file_exists, format_path_media, path_file_sanitize, url_to_filename from tidal_dl_ng.helper.tidal import ( instantiate_media, items_results_all, @@ -30,6 +39,7 @@ from tidal_dl_ng.helper.tidal import ( name_builder_title, ) from tidal_dl_ng.metadata import Metadata +from tidal_dl_ng.model.downloader import DownloadSegmentResult from tidal_dl_ng.model.gui_data import ProgressBars @@ -51,7 +61,7 @@ class RequestsClient: class Download: settings: Settings session: Session - skip_existing: SkipExisting = SkipExisting.Disabled + skip_existing: bool = False fn_logger: Callable progress_gui: ProgressBars progress: Progress @@ -61,7 +71,7 @@ class Download: session: Session, path_base: str, fn_logger: Callable, - skip_existing: SkipExisting = SkipExisting.Disabled, + skip_existing: bool = False, progress_gui: ProgressBars = None, progress: Progress = None, ): @@ -88,27 +98,34 @@ class Download: def _download( self, media: Track | Video, - path_file: str, - ) -> str: + stream_manifest: StreamManifest, + path_file: pathlib.Path, + ) -> (bool, pathlib.Path): media_name: str = name_builder_item(media) urls: [str] + path_base: pathlib.Path = path_file.parent + result_segments: bool = True + dl_segment_results: [DownloadSegmentResult] = [] + result_merge: bool = False # Get urls for media. - if isinstance(media, Track): - urls = media.get_stream().get_stream_manifest().urls - stream_manifest: StreamManifest = media.get_stream().get_stream_manifest() - elif isinstance(media, Video): - m3u8_variant: m3u8.M3U8 = m3u8.load(media.get_url()) - # Find the desired video resolution or the next best one. - m3u8_playlist, codecs = self._extract_video_stream(m3u8_variant, int(self.settings.data.quality_video)) - # Populate urls. - urls = m3u8_playlist.files + try: + if isinstance(media, Track): + urls = stream_manifest.get_urls() + elif isinstance(media, Video): + m3u8_variant: m3u8.M3U8 = m3u8.load(media.get_url()) + # Find the desired video resolution or the next best one. + m3u8_playlist, codecs = self._extract_video_stream(m3u8_variant, int(self.settings.data.quality_video)) + # Populate urls. + urls = m3u8_playlist.files + except Exception: + return False, path_file # Set the correct progress output channel. if self.progress_gui is None: - progress_stdout: bool = True + progress_to_stdout: bool = True else: - progress_stdout: bool = False + progress_to_stdout: bool = False # Send signal to GUI with media name self.progress_gui.item_name.emit(media_name[:30]) @@ -119,8 +136,11 @@ class Download: progress_total: int = urls_count block_size: int | None = None elif urls_count == 1: - # Will be computed later. - progress_total: float = None + # Get file size and compute progress steps + r = requests.head(urls[0], timeout=REQUESTS_TIMEOUT_SEC) + total_size_in_bytes: int = int(r.headers.get("content-length", 0)) + block_size: int | None = 1048576 + progress_total: float = total_size_in_bytes / block_size else: raise ValueError @@ -128,57 +148,115 @@ class Download: p_task: TaskID = self.progress.add_task( f"[blue]Item '{media_name[:30]}'", total=progress_total, - visible=progress_stdout, + visible=progress_to_stdout, ) - # Write content to file until progress is finished. + # Download segments until progress is finished. + # TODO: Compute download speed (https://github.com/Textualize/rich/blob/master/examples/downloader.py) while not self.progress.tasks[p_task].finished: - with open(path_file, "wb") as f: - for url in urls: - try: - # Create the request object with stream=True, so the content won't be loaded into memory at once. - r = requests.get(url, stream=True, timeout=REQUESTS_TIMEOUT_SEC) + with futures.ThreadPoolExecutor( + max_workers=self.settings.data.downloads_simultaneous_per_track_max + ) as executor: + # Dispatch all download tasks to worker threads + l_futures: [any] = [ + executor.submit(self._download_segment, url, path_base, block_size, p_task, progress_to_stdout) + for url in urls + ] + # Report results as they become available + for future in futures.as_completed(l_futures): + # Retrieve result + result_dl_segment: DownloadSegmentResult = future.result() - r.raise_for_status() + dl_segment_results.append(result_dl_segment) - # Compute progress iterations based on the file size and update task details. - if not progress_total: - # Get file size and compute progress steps - total_size_in_bytes: int = int(r.headers.get("content-length", 0)) - block_size: int | None = 1048576 - progress_total: float = total_size_in_bytes / block_size + # check for a link that was skipped + if not result_dl_segment.result and (result_dl_segment.url is not urls[-1]): + # Sometimes it happens, if a track is very short (< 8 seconds or so), that the last URL in `urls` is + # invalid (HTTP Error 500) and not necessary. File won't be corrupt. + # If this is NOT the case, but any other URL has resulted in an error, + # mark the whole thing as corrupt. + result_segments = False + self.fn_logger.error(f"Something went wrong while downloading {media_name}. File is corrupt!") - self.progress.update(p_task, total=progress_total) + tmp_path_file_decrypted: pathlib.Path = path_file - # Write the content to disk. If `chunk_size` is set to `None` the whole file will be written at once. - for data in r.iter_content(chunk_size=block_size): - f.write(data) - # Advance progress bar. - self.progress.advance(p_task) - except HTTPError as e: - if url is urls[-1]: - # It happens, if a track is very short (< 8 seconds or so), that the last URL in `urls` is - # invalid (HTTP Error 500) and not necessary. File won't be corrupt. - # Thus, advance progress bar to avoid infinity loops. - self.progress.advance(p_task) - else: - # Finish downloading early and report error. - # TODO: The track should somehow be marked as corrupt. - self.progress.update(p_task, completed=progress_total) - self.fn_logger.error(e) - finally: - # To send the progress to the GUI, we need to emit the percentage. - if not progress_stdout: - self.progress_gui.item.emit(self.progress.tasks[p_task].percentage) + # Only if no error happened while downloading. + if result_segments: + # Bring list into right order, so segments can be easily merged. + dl_segment_results.sort(key=lambda x: x.id_segment) + result_merge: bool = self._segments_merge(path_file, dl_segment_results) - if isinstance(media, Track) and stream_manifest.is_encrypted: - key, nonce = decrypt_security_token(stream_manifest.encryption_key) - tmp_path_file_decrypted = path_file + "_decrypted" - decrypt_file(path_file, tmp_path_file_decrypted, key, nonce) - else: - tmp_path_file_decrypted = path_file + if not result_merge: + self.fn_logger.error(f"Something went wrong while writing to {media_name}. File is corrupt!") + elif result_merge and isinstance(media, Track) and stream_manifest.is_encrypted: + key, nonce = decrypt_security_token(stream_manifest.encryption_key) + tmp_path_file_decrypted = path_file.with_suffix(".decrypted") + decrypt_file(path_file, tmp_path_file_decrypted, key, nonce) + + return result_merge, tmp_path_file_decrypted + + def _segments_merge(self, path_file, dl_segment_results) -> bool: + result: bool + + # Copy the content of all segments into one file. + try: + with path_file.open("wb") as f_target: + for dl_segment_result in dl_segment_results: + with dl_segment_result.path_segment.open("rb") as f_segment: + # Read and write junks, which gives better HDD write performance + while segment := f_segment.read(CHUNK_SIZE): + f_target.write(segment) + + # Delete segment from HDD + dl_segment_result.path_segment.unlink() + + result = True + except Exception: + result = False + + return result + + def _download_segment( + self, url: str, path_base: pathlib.Path, block_size: int | None, p_task: TaskID, progress_to_stdout: bool + ) -> DownloadSegmentResult: + result: bool = False + path_segment: pathlib.Path = path_base / url_to_filename(url) + # Calculate the segment ID based on the file name within the URL. + filename_stem: str = str(path_segment.stem).split("_")[-1] + # CAUTION: This is a workaround, so BTS (LOW quality) track will work. They usually have only ONE link. + id_segment: int = int(filename_stem) if filename_stem.isdecimal() else 0 + error: HTTPError | None = None - return tmp_path_file_decrypted + # Retry download on failed segments, with an exponential delay between retries + s = requests.Session() + retries = Retry(total=5, backoff_factor=1) # , status_forcelist=[ 502, 503, 504 ]) + + s.mount("https://", HTTPAdapter(max_retries=retries)) + + try: + # Create the request object with stream=True, so the content won't be loaded into memory at once. + r = s.get(url, stream=True, timeout=REQUESTS_TIMEOUT_SEC) + + r.raise_for_status() + + # Write the content to disk. If `chunk_size` is set to `None` the whole file will be written at once. + with path_segment.open("wb") as f: + for data in r.iter_content(chunk_size=block_size): + f.write(data) + # Advance progress bar. + self.progress.advance(p_task) + + result = True + except Exception: + self.progress.advance(p_task) + + # To send the progress to the GUI, we need to emit the percentage. + if not progress_to_stdout: + self.progress_gui.item.emit(self.progress.tasks[p_task].percentage) + + return DownloadSegmentResult( + result=result, url=url, path_segment=path_segment, id_segment=id_segment, error=error + ) def item( self, @@ -219,103 +297,139 @@ class Download: return False, "" - # Get extension. - file_extension: str - - if isinstance(media, Track): - # If a quality is explicitly set, change it. - if quality_audio: - quality_audio_old: Quality = self.adjust_quality_audio(quality_audio) - - file_extension = media.get_stream().get_stream_manifest().file_extension - # Use M4A extension for MP4 audio tracks, because it looks better and is completely interchangeable. - file_extension = AudioExtensions.M4A if file_extension == AudioExtensions.MP4 else file_extension - - if self.settings.data.extract_flac: - do_flac_extract = False - - if ( - media.get_stream().get_stream_manifest().codecs.upper() == Codec.FLAC - and file_extension != AudioExtensions.FLAC - ): - file_extension = AudioExtensions.FLAC - do_flac_extract = True - elif isinstance(media, Video): - if quality_video: - quality_video_old: QualityVideo = self.adjust_quality_video(quality_video) - - file_extension = AudioExtensions.MP4 if self.settings.data.video_convert_mp4 else VideoExtensions.TS - # Create file name and path + file_extension_dummy: str = AudioExtensions.FLAC file_name_relative = format_path_media(file_template, media) - path_media_dst = os.path.abspath( - os.path.normpath(os.path.join(os.path.expanduser(self.path_base), file_name_relative)) + path_media_dst: pathlib.Path = ( + (pathlib.Path(self.path_base).expanduser() / (file_name_relative + file_extension_dummy)) + .resolve() + .absolute() ) # Sanitize final path_file to fit into OS boundaries. - uniquify: bool = self.skip_existing == SkipExisting.Append - path_media_dst = path_file_sanitize(path_media_dst + file_extension, adapt=True, uniquify=uniquify) + path_media_dst = pathlib.Path(path_file_sanitize(str(path_media_dst), adapt=True)) # Compute if and how downloads need to be skipped. - if self.skip_existing in (SkipExisting.ExtensionIgnore, SkipExisting.Filename): - extension_ignore: bool = self.skip_existing == SkipExisting.ExtensionIgnore - file_exists: bool = check_file_exists(path_media_dst, extension_ignore=extension_ignore) + if self.skip_existing: + skip_file: bool = check_file_exists(path_media_dst, extension_ignore=True) else: - file_exists: bool = False + skip_file: bool = False + + if not skip_file: + # Get extension. + file_extension: str + do_flac_extract = False + # If a quality is explicitly set, change it and remember the previously set quality. + quality_audio_old: Quality = self.adjust_quality_audio(quality_audio) if quality_audio else quality_audio + quality_video_old: QualityVideo = ( + self.adjust_quality_video(quality_video) if quality_video else quality_video + ) + + try: + media_stream = media.get_stream() + stream_manifest: StreamManifest = media_stream.get_stream_manifest() + except TooManyRequests: + self.fn_logger.exception( + f"Too many requests against TIDAL backend. Skipping '{name_builder_item(media)}'. " + f"Consider to activate delay between downloads." + ) + + return False, "" + except Exception: + self.fn_logger.exception(f"Something went wrong. Skipping '{name_builder_item(media)}'.") + + return False, "" + + if isinstance(media, Track): + # TODO: HOTFIX! Go back to + # file_extension = media_stream.get_stream_manifest().file_extension + # After tidalapi has fixed #304 + if stream_manifest.file_extension is VideoExtensions.TS: + file_extension = stream_manifest.file_extension + elif stream_manifest.dash_info and AudioExtensions.FLAC in stream_manifest.dash_info.first_url: + file_extension = AudioExtensions.FLAC + else: + file_extension = AudioExtensions.M4A + + if self.settings.data.extract_flac and ( + stream_manifest.codecs.upper() == Codec.FLAC and file_extension != AudioExtensions.FLAC + ): + file_extension = AudioExtensions.FLAC + do_flac_extract = True + elif isinstance(media, Video): + file_extension = AudioExtensions.MP4 if self.settings.data.video_convert_mp4 else VideoExtensions.TS + + # Compute file name and create destination directory + path_media_dst = path_media_dst.with_suffix(file_extension) + os.makedirs(path_media_dst.parent, exist_ok=True) - if not file_exists: # Create a temp directory and file. with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_path_dir: - tmp_path_file = os.path.join(tmp_path_dir, str(uuid4())) + tmp_path_file: pathlib.Path = pathlib.Path(tmp_path_dir) / str(uuid4()) + + # Create empty file + tmp_path_file.touch() + # Download media. - tmp_path_file = self._download(media=media, path_file=tmp_path_file) + result_download, tmp_path_file = self._download( + media=media, stream_manifest=stream_manifest, path_file=tmp_path_file + ) - # Convert video from TS to MP4 - if isinstance(media, Video) and self.settings.data.video_convert_mp4: - # Convert `*.ts` file to `*.mp4` using ffmpeg - tmp_path_file = self._video_convert(tmp_path_file) + if result_download: + # Convert video from TS to MP4 + if isinstance(media, Video) and self.settings.data.video_convert_mp4: + # Convert `*.ts` file to `*.mp4` using ffmpeg + tmp_path_file = self._video_convert(tmp_path_file) - # Extract FLAC from MP4 container using ffmpeg - if isinstance(media, Track) and self.settings.data.extract_flac and do_flac_extract: - tmp_path_file = self._extract_flac(tmp_path_file) + # Extract FLAC from MP4 container using ffmpeg + if isinstance(media, Track) and self.settings.data.extract_flac and do_flac_extract: + tmp_path_file = self._extract_flac(tmp_path_file) - tmp_path_lyrics: pathlib.Path | None = None - tmp_path_cover: pathlib.Path | None = None + tmp_path_lyrics: pathlib.Path | None = None + tmp_path_cover: pathlib.Path | None = None - # Write metadata to file. - if not isinstance(media, Video): - result_metadata, tmp_path_lyrics, tmp_path_cover = self.metadata_write( - media, tmp_path_file, is_parent_album - ) + # Write metadata to file. + if not isinstance(media, Video): + result_metadata, tmp_path_lyrics, tmp_path_cover = self.metadata_write( + media, tmp_path_file, is_parent_album + ) + + # Move lyrics file + if self.settings.data.lyrics_file and not isinstance(media, Video) and tmp_path_lyrics: + self._move_lyrics(tmp_path_lyrics, path_media_dst) + + # Move cover file + # TODO: Cover is downloaded with every track of the album. Needs refactoring, so cover is only + # downloaded for an album once. + if self.settings.data.cover_album_file and tmp_path_cover: + self._move_cover(tmp_path_cover, path_media_dst) + + self.fn_logger.info(f"Downloaded item '{name_builder_item(media)}'.") # Move final file to the configured destination directory. - os.makedirs(os.path.dirname(path_media_dst), exist_ok=True) shutil.move(tmp_path_file, path_media_dst) - # Move lyrics file - if self.settings.data.lyrics_file and not isinstance(media, Video): - self._move_lyrics(tmp_path_lyrics, path_media_dst) + if quality_audio: + # Set quality back to the global user value + self.adjust_quality_audio(quality_audio_old) - # Move cover file - if self.settings.data.cover_album_file: - self._move_cover(tmp_path_cover, path_media_dst) + if quality_video: + # Set quality back to the global user value + self.adjust_quality_video(quality_video_old) else: self.fn_logger.debug(f"Download skipped, since file exists: '{path_media_dst}'") - status_download: bool = not file_exists - - if quality_audio: - # Set quality back to the global user value - self.adjust_quality_audio(quality_audio_old) - - if quality_video: - # Set quality back to the global user value - self.adjust_quality_video(quality_video_old) + status_download: bool = not skip_file # Whether a file was downloaded or skipped and the download delay is enabled, wait until the next download. # Only use this, if you have a list of several Track items. - if download_delay: - time_sleep: float = round(random.SystemRandom().uniform(2, 5), 1) + if download_delay and not skip_file: + time_sleep: float = round( + random.SystemRandom().uniform( + self.settings.data.download_delay_sec_min, self.settings.data.download_delay_sec_max + ), + 1, + ) self.fn_logger.debug(f"Next download will start in {time_sleep} seconds.") time.sleep(time_sleep) @@ -340,7 +454,7 @@ class Download: result: bool # Check if the file was downloaded - if path_file_source and os.path.isfile(path_file_source): + if path_file_source and path_file_source.is_file(): # Move it. shutil.move(path_file_source, path_file_destination) @@ -350,16 +464,16 @@ class Download: return result - def _move_lyrics(self, path_lyrics: pathlib.Path, file_media_dst: str) -> bool: + def _move_lyrics(self, path_lyrics: pathlib.Path, file_media_dst: pathlib.Path) -> bool: # Build tmp lyrics filename - path_file_lyrics: str = os.path.splitext(file_media_dst)[0] + EXTENSION_LYRICS + path_file_lyrics: pathlib.Path = file_media_dst.with_suffix(EXTENSION_LYRICS) result: bool = self._move_file(path_lyrics, path_file_lyrics) return result - def _move_cover(self, path_cover: pathlib.Path, file_media_dst: str) -> bool: + def _move_cover(self, path_cover: pathlib.Path, file_media_dst: pathlib.Path) -> bool: # Build tmp lyrics filename - path_file_cover: pathlib.Path = pathlib.Path(file_media_dst).parent.absolute() / COVER_NAME + path_file_cover: pathlib.Path = file_media_dst.parent / COVER_NAME result: bool = self._move_file(path_cover, path_file_cover) return result @@ -403,7 +517,7 @@ class Download: return result def metadata_write( - self, track: Track, path_media: str, is_parent_album: str + self, track: Track, path_media: pathlib.Path, is_parent_album: bool ) -> (bool, pathlib.Path | None, pathlib.Path | None): result: bool = False path_lyrics: pathlib.Path | None = None @@ -433,14 +547,14 @@ class Download: print(f"Could not retrieve lyrics for `{name_builder_item(track)}`.") if lyrics and self.settings.data.lyrics_file: - path_lyrics = self.lyrics_to_file(pathlib.Path(path_media).parent, lyrics) + path_lyrics = self.lyrics_to_file(path_media.parent, lyrics) if self.settings.data.metadata_cover_embed or (self.settings.data.cover_album_file and is_parent_album): url_cover = track.album.image(int(self.settings.data.metadata_cover_dimension)) cover_data = self.cover_data(url=url_cover) if cover_data and self.settings.data.cover_album_file and is_parent_album: - path_cover = self.cover_to_file(pathlib.Path(path_media).parent, cover_data) + path_cover = self.cover_to_file(path_media.parent, cover_data) # `None` values are not allowed. m: Metadata = Metadata( @@ -487,29 +601,22 @@ class Download: file_name_relative = format_path_media(file_template, media) # Get the name of the list and check, if videos should be included. - videos_include: bool = True - - if isinstance(media, Mix): - list_media_name = media.title[:30] - elif video_download: - list_media_name = name_builder_title(media)[:30] - else: - videos_include = False - list_media_name = name_builder_title(media)[:30] + list_media_name: str = name_builder_title(media) + list_media_name_short: str = list_media_name[:30] # Get all items of the list. - items = items_results_all(media, videos_include=videos_include) + items = items_results_all(media, videos_include=video_download) # Determine where to redirect the progress information. if self.progress_gui is None: progress_stdout: bool = True else: progress_stdout: bool = False - self.progress_gui.list_name.emit(list_media_name[:30]) + self.progress_gui.list_name.emit(list_media_name_short[:30]) # Create the list progress task. p_task1: TaskID = self.progress.add_task( - f"[green]List '{list_media_name}'", total=len(items), visible=progress_stdout + f"[green]List '{list_media_name_short}'", total=len(items), visible=progress_stdout ) is_album: bool = isinstance(media, Album) @@ -518,7 +625,7 @@ class Download: while not self.progress.finished: for item_media in items: # Download the item. - status_download, result_path_file = self.item( + status, result_path_file = self.item( media=item_media, file_template=file_name_relative, quality_audio=quality_audio, @@ -533,22 +640,29 @@ class Download: if not progress_stdout: self.progress_gui.list_item.emit(self.progress.tasks[p_task1].percentage) - def _video_convert(self, path_file: str) -> str: - path_file_out = path_file + AudioExtensions.MP4 + self.fn_logger.info(f"Finished list '{list_media_name}'.") + + def _video_convert(self, path_file: pathlib.Path) -> pathlib.Path: + path_file_out: pathlib.Path = path_file.with_suffix(AudioExtensions.MP4) result, _ = ( ffmpeg.input(path_file) - .output(path_file_out, map=0, c="copy", loglevel="quiet") + .output(str(path_file_out), map=0, c="copy", loglevel="quiet") .run(cmd=self.settings.data.path_binary_ffmpeg) ) return path_file_out - def _extract_flac(self, path_media_src: str) -> str: - path_media_out = path_media_src + AudioExtensions.FLAC + def _extract_flac(self, path_media_src: pathlib.Path) -> pathlib.Path: + path_media_out = path_media_src.with_suffix(AudioExtensions.FLAC) result, _ = ( ffmpeg.input(path_media_src) .output( - path_media_out, map=0, movflags="use_metadata_tags", acodec="copy", map_metadata="0:g", loglevel="quiet" + str(path_media_out), + map=0, + movflags="use_metadata_tags", + acodec="copy", + map_metadata="0:g", + loglevel="quiet", ) .run(cmd=self.settings.data.path_binary_ffmpeg) ) diff --git a/tidal_dl_ng/gui.py b/tidal_dl_ng/gui.py index f7d84ca..4d3c38a 100644 --- a/tidal_dl_ng/gui.py +++ b/tidal_dl_ng/gui.py @@ -516,6 +516,8 @@ class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow): self._init_dl() def search(self, query: str, types_media: SearchTypes) -> [ResultItem]: + query = query.strip() + # If a direct link was searched for, skip search and create the object from the link directly. if "http" in query: media_type = get_tidal_media_type(query) diff --git a/tidal_dl_ng/helper/decryption.py b/tidal_dl_ng/helper/decryption.py index ebb946a..17ea19b 100644 --- a/tidal_dl_ng/helper/decryption.py +++ b/tidal_dl_ng/helper/decryption.py @@ -1,4 +1,5 @@ import base64 +import pathlib from Crypto.Cipher import AES from Crypto.Util import Counter @@ -35,7 +36,7 @@ def decrypt_security_token(security_token: str) -> (str, str): return key, nonce -def decrypt_file(path_file_encrypted: str, path_file_destination: str, key: str, nonce: str) -> None: +def decrypt_file(path_file_encrypted: pathlib.Path, path_file_destination: pathlib.Path, key: str, nonce: str) -> None: """ Decrypts an encrypted MQA file given the file, key and nonce. TODO: Is it really only necessary for MQA of for all other formats, too? @@ -46,9 +47,9 @@ def decrypt_file(path_file_encrypted: str, path_file_destination: str, key: str, decryptor = AES.new(key, AES.MODE_CTR, counter=counter) # Open and decrypt - with open(path_file_encrypted, "rb") as f_src: + with path_file_encrypted.open("rb") as f_src: audio_decrypted = decryptor.decrypt(f_src.read()) # Replace with decrypted file - with open(path_file_destination, "wb") as f_dst: + with path_file_destination.open("wb") as f_dst: f_dst.write(audio_decrypted) diff --git a/tidal_dl_ng/helper/path.py b/tidal_dl_ng/helper/path.py index cc8d5b6..fb7bb87 100644 --- a/tidal_dl_ng/helper/path.py +++ b/tidal_dl_ng/helper/path.py @@ -1,8 +1,10 @@ import math import os +import pathlib +import posixpath import re import sys -from pathlib import Path, PosixPath +from urllib.parse import unquote, urlsplit from pathvalidate import sanitize_filename, sanitize_filepath from pathvalidate.error import ValidationError @@ -77,8 +79,7 @@ def format_str_media(name: str, media: Track | Album | Playlist | UserPlaylist | elif hasattr(media, "artist"): result = media.artist.name case "album_artist": - if isinstance(media, Album): - result = name_builder_album_artist(media) + result = name_builder_album_artist(media) case "track_title": if isinstance(media, Track | Video): result = name_builder_title(media) @@ -195,7 +196,7 @@ def get_format_template( def path_file_sanitize(path_file: str, adapt: bool = False, uniquify: bool = False) -> (bool, str): # Split into path and filename pathname, filename = os.path.split(path_file) - file_extension: str = Path(path_file).suffix + file_extension: str = pathlib.Path(path_file).suffix # Sanitize path try: @@ -205,7 +206,7 @@ def path_file_sanitize(path_file: str, adapt: bool = False, uniquify: bool = Fal except ValidationError: # If adaption of path is allowed in case of an error set path to HOME. if adapt: - pathname_sanitized: str = Path.home() + pathname_sanitized: str = str(pathlib.Path.home()) else: raise @@ -272,10 +273,10 @@ def file_unique_suffix(path_file: str, seperator: str = "_") -> str: return unique_suffix -def check_file_exists(path_file: str, extension_ignore: bool = False) -> bool: +def check_file_exists(path_file: pathlib.Path, extension_ignore: bool = False) -> bool: if extension_ignore: - path_file_stem: str = Path(path_file).stem - path_parent: PosixPath = Path(path_file).parent + path_file_stem: str = pathlib.Path(path_file).stem + path_parent: pathlib.Path = pathlib.Path(path_file).parent path_files: [str] = [] for extension in AudioExtensions: @@ -295,3 +296,22 @@ def resource_path(relative_path): base_path = os.path.abspath(".") return os.path.join(base_path, relative_path) + + +def url_to_filename(url: str) -> str: + """Return basename corresponding to url. + >>> print(url_to_filename('http://example.com/path/to/file%C3%80?opt=1')) + fileÀ + >>> print(url_to_filename('http://example.com/slash%2fname')) # '/' in name + Taken from https://gist.github.com/zed/c2168b9c52b032b5fb7d + Traceback (most recent call last): + ... + ValueError + """ + urlpath: str = urlsplit(url).path + basename: str = posixpath.basename(unquote(urlpath)) + + if os.path.basename(basename) != basename or unquote(posixpath.basename(urlpath)) != basename: + raise ValueError # reject '%2f' or 'dir%5Cbasename.ext' on Windows + + return basename diff --git a/tidal_dl_ng/helper/tidal.py b/tidal_dl_ng/helper/tidal.py index d679d5e..e568cc6 100644 --- a/tidal_dl_ng/helper/tidal.py +++ b/tidal_dl_ng/helper/tidal.py @@ -25,8 +25,10 @@ def name_builder_album_artist(media: Track | Album) -> str: return ", ".join(artists_tmp) -def name_builder_title(media: Track | Video) -> str: - result: str = media.full_name if hasattr(media, "full_name") else media.name +def name_builder_title(media: Track | Video | Mix | Playlist | Album | Video) -> str: + result: str = ( + media.title if isinstance(media, Mix) else media.full_name if hasattr(media, "full_name") else media.name + ) return result @@ -186,7 +188,7 @@ def instantiate_media( def quality_audio_highest(media: Track | Album) -> Quality: quality: Quality - if MediaMetadataTags.hires_lossless in media.media_metadata_tags: + if MediaMetadataTags.hi_res_lossless in media.media_metadata_tags: quality = Quality.hi_res_lossless elif MediaMetadataTags.lossless in media.media_metadata_tags: quality = Quality.high_lossless diff --git a/tidal_dl_ng/helper/wrapper.py b/tidal_dl_ng/helper/wrapper.py index d8ffb09..cd8e6ba 100644 --- a/tidal_dl_ng/helper/wrapper.py +++ b/tidal_dl_ng/helper/wrapper.py @@ -21,3 +21,6 @@ class LoggerWrapped: def critical(self, value): self.fn_print(value) + + def exception(self, value): + self.fn_print(value) diff --git a/tidal_dl_ng/metadata.py b/tidal_dl_ng/metadata.py index d9ecb8c..e1fb44b 100644 --- a/tidal_dl_ng/metadata.py +++ b/tidal_dl_ng/metadata.py @@ -111,6 +111,8 @@ class Metadata: self.m.tags["LYRICS"] = self.lyrics def set_mp3(self): + # ID3 Frame (tags) overview: https://exiftool.org/TagNames/ID3.html / https://id3.org/id3v2.3.0 + # Mapping overview: https://docs.mp3tag.de/mapping/ self.m.tags.add(TIT2(encoding=3, text=self.title)) self.m.tags.add(TALB(encoding=3, text=self.album)) self.m.tags.add(TOPE(encoding=3, text=self.albumartist)) diff --git a/tidal_dl_ng/model/cfg.py b/tidal_dl_ng/model/cfg.py index fae3590..06614f4 100644 --- a/tidal_dl_ng/model/cfg.py +++ b/tidal_dl_ng/model/cfg.py @@ -3,13 +3,13 @@ from dataclasses import dataclass from dataclasses_json import dataclass_json from tidalapi import Quality -from tidal_dl_ng.constants import CoverDimensions, QualityVideo, SkipExisting +from tidal_dl_ng.constants import CoverDimensions, QualityVideo @dataclass_json @dataclass class Settings: - skip_existing: SkipExisting = SkipExisting.Disabled + skip_existing: bool = True lyrics_embed: bool = False lyrics_file: bool = False # TODO: Implement API KEY selection. @@ -25,7 +25,7 @@ class Settings: quality_video: QualityVideo = QualityVideo.P480 format_album: str = ( "Albums/{album_artist} - {album_title}{album_explicit}/{track_volume_num_optional}" - "{album_track_num}. {artist_name} - {track_title}" + "{album_track_num}. {artist_name} - {track_title}{album_explicit}" ) format_playlist: str = "Playlists/{playlist_name}/{artist_name} - {track_title}" format_mix: str = "Mix/{mix_name}/{artist_name} - {track_title}" @@ -37,16 +37,15 @@ class Settings: metadata_cover_embed: bool = True cover_album_file: bool = True extract_flac: bool = True + downloads_simultaneous_per_track_max: int = 20 + download_delay_sec_min: float = 3.0 + download_delay_sec_max: float = 5.0 @dataclass_json @dataclass class HelpSettings: - skip_existing: str = ( - "Do not download, if file already exists. Possible option false = do not skip, " - "'exact' = if filename already exists, 'extension_ignore' = skip even if a file with a " - "different file extension exists." - ) + skip_existing: str = "Skip download if file already exists." album_cover_save: str = "Safe cover to album folder." lyrics_embed: str = "Embed lyrics in audio file, if lyrics are available." lyrics_file: str = "Save lyrics to separate *.lrc file, if lyrics are available." @@ -82,6 +81,9 @@ class HelpSettings: metadata_cover_embed: str = "Embed album cover into file." cover_album_file: str = "Save cover to 'cover.jpg', if an album is downloaded." extract_flac: str = "Extract FLAC audio tracks from MP4 containers and save them as `*.flac` (uses FFmpeg)." + downloads_simultaneous_per_track_max: str = "Maximum number of simultaneous chunk downloads per track." + download_delay_sec_min: float = "Lower boundary for the calculation of the download delay in seconds." + download_delay_sec_max: float = "Upper boundary for the calculation of the download delay in seconds." @dataclass_json diff --git a/tidal_dl_ng/model/downloader.py b/tidal_dl_ng/model/downloader.py new file mode 100644 index 0000000..3f6151f --- /dev/null +++ b/tidal_dl_ng/model/downloader.py @@ -0,0 +1,13 @@ +import pathlib +from dataclasses import dataclass + +from requests import HTTPError + + +@dataclass +class DownloadSegmentResult: + result: bool + url: str + path_segment: pathlib.Path + id_segment: int + error: HTTPError | None = None diff --git a/tidal_dl_ng/ui/dialog_settings.py b/tidal_dl_ng/ui/dialog_settings.py index ad45f1d..a624e34 100644 --- a/tidal_dl_ng/ui/dialog_settings.py +++ b/tidal_dl_ng/ui/dialog_settings.py @@ -153,6 +153,24 @@ class Ui_DialogSettings: self.lv_flags.addLayout(self.lh_flags_4) + self.horizontalLayout = QHBoxLayout() + self.horizontalLayout.setObjectName("horizontalLayout") + self.lv_flag_skip_existing = QVBoxLayout() + self.lv_flag_skip_existing.setObjectName("lv_flag_skip_existing") + self.cb_skip_existing = QCheckBox(self.gb_flags) + self.cb_skip_existing.setObjectName("cb_skip_existing") + + self.lv_flag_skip_existing.addWidget(self.cb_skip_existing) + + self.horizontalLayout.addLayout(self.lv_flag_skip_existing) + + self.verticalLayout_3 = QVBoxLayout() + self.verticalLayout_3.setObjectName("verticalLayout_3") + + self.horizontalLayout.addLayout(self.verticalLayout_3) + + self.lv_flags.addLayout(self.horizontalLayout) + self.lv_main.addWidget(self.gb_flags) self.gb_choices = QGroupBox(DialogSettings) @@ -164,38 +182,13 @@ class Ui_DialogSettings: self.gb_choices.setSizePolicy(sizePolicy3) self.lv_choices = QVBoxLayout(self.gb_choices) self.lv_choices.setObjectName("lv_choices") - self.lh_choices_skip_existing = QHBoxLayout() - self.lh_choices_skip_existing.setObjectName("lh_choices_skip_existing") - self.l_icon_skip_existing = QLabel(self.gb_choices) - self.l_icon_skip_existing.setObjectName("l_icon_skip_existing") - sizePolicy4 = QSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed) - sizePolicy4.setHorizontalStretch(0) - sizePolicy4.setVerticalStretch(0) - sizePolicy4.setHeightForWidth(self.l_icon_skip_existing.sizePolicy().hasHeightForWidth()) - self.l_icon_skip_existing.setSizePolicy(sizePolicy4) - - self.lh_choices_skip_existing.addWidget(self.l_icon_skip_existing) - - self.l_skip_existing = QLabel(self.gb_choices) - self.l_skip_existing.setObjectName("l_skip_existing") - sizePolicy4.setHeightForWidth(self.l_skip_existing.sizePolicy().hasHeightForWidth()) - self.l_skip_existing.setSizePolicy(sizePolicy4) - - self.lh_choices_skip_existing.addWidget(self.l_skip_existing) - - self.c_skip_existing = QComboBox(self.gb_choices) - self.c_skip_existing.setObjectName("c_skip_existing") - - self.lh_choices_skip_existing.addWidget(self.c_skip_existing) - - self.lh_choices_skip_existing.setStretch(2, 50) - - self.lv_choices.addLayout(self.lh_choices_skip_existing) - self.lh_choices_quality_audio = QHBoxLayout() self.lh_choices_quality_audio.setObjectName("lh_choices_quality_audio") self.l_icon_quality_audio = QLabel(self.gb_choices) self.l_icon_quality_audio.setObjectName("l_icon_quality_audio") + sizePolicy4 = QSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed) + sizePolicy4.setHorizontalStretch(0) + sizePolicy4.setVerticalStretch(0) sizePolicy4.setHeightForWidth(self.l_icon_quality_audio.sizePolicy().hasHeightForWidth()) self.l_icon_quality_audio.setSizePolicy(sizePolicy4) @@ -532,9 +525,8 @@ class Ui_DialogSettings: self.cb_extract_flac.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None)) self.cb_metadata_cover_embed.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None)) self.cb_cover_album_file.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None)) + self.cb_skip_existing.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None)) self.gb_choices.setTitle(QCoreApplication.translate("DialogSettings", "Choices", None)) - self.l_icon_skip_existing.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None)) - self.l_skip_existing.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None)) self.l_icon_quality_audio.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None)) self.l_quality_audio.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None)) self.l_icon_quality_video.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None)) diff --git a/tidal_dl_ng/ui/dialog_settings.ui b/tidal_dl_ng/ui/dialog_settings.ui index 9bc894d..4fa07cf 100644 --- a/tidal_dl_ng/ui/dialog_settings.ui +++ b/tidal_dl_ng/ui/dialog_settings.ui @@ -210,6 +210,24 @@ </item> </layout> </item> + <item> + <layout class="QHBoxLayout" name="horizontalLayout"> + <item> + <layout class="QVBoxLayout" name="lv_flag_skip_existing"> + <item> + <widget class="QCheckBox" name="cb_skip_existing"> + <property name="text"> + <string>CheckBox</string> + </property> + </widget> + </item> + </layout> + </item> + <item> + <layout class="QVBoxLayout" name="verticalLayout_3"/> + </item> + </layout> + </item> </layout> </widget> </item> @@ -226,39 +244,6 @@ </property> <layout class="QVBoxLayout" name="lv_choices"> <item> - <layout class="QHBoxLayout" name="lh_choices_skip_existing" stretch="0,0,50"> - <item> - <widget class="QLabel" name="l_icon_skip_existing"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Fixed" vsizetype="Fixed"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="text"> - <string>TextLabel</string> - </property> - </widget> - </item> - <item> - <widget class="QLabel" name="l_skip_existing"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Fixed" vsizetype="Fixed"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="text"> - <string>TextLabel</string> - </property> - </widget> - </item> - <item> - <widget class="QComboBox" name="c_skip_existing"/> - </item> - </layout> - </item> - <item> <layout class="QHBoxLayout" name="lh_choices_quality_audio" stretch="0,0,50"> <item> <widget class="QLabel" name="l_icon_quality_audio"> |
