summaryrefslogtreecommitdiff
path: root/tidal_dl_ng
diff options
context:
space:
mode:
authorRobert Honz <exislow+github@gmail.com>2024-11-29 17:41:25 +0100
committerRobert Honz <exislow+github@gmail.com>2024-11-29 17:41:25 +0100
commit33a30537e9a205dd4b43e891b00c0b5f15a9b71b (patch)
tree9ccb8c4c273ce96fd6ebce8ea2077d281d812712 /tidal_dl_ng
parentf2b3cc2505f4c64f09efacd775de3390ae204be0 (diff)
parent4deaf94fb53ad304db73f9384f817b29afcac4a8 (diff)
downloadtidal-dl-ng-33a30537e9a205dd4b43e891b00c0b5f15a9b71b.tar.gz
✨ Merge with master.
Diffstat (limited to 'tidal_dl_ng')
-rw-r--r--tidal_dl_ng/constants.py10
-rw-r--r--tidal_dl_ng/dialog.py4
-rw-r--r--tidal_dl_ng/download.py432
-rw-r--r--tidal_dl_ng/gui.py2
-rw-r--r--tidal_dl_ng/helper/decryption.py7
-rw-r--r--tidal_dl_ng/helper/path.py36
-rw-r--r--tidal_dl_ng/helper/tidal.py8
-rw-r--r--tidal_dl_ng/helper/wrapper.py3
-rw-r--r--tidal_dl_ng/metadata.py2
-rw-r--r--tidal_dl_ng/model/cfg.py18
-rw-r--r--tidal_dl_ng/model/downloader.py13
-rw-r--r--tidal_dl_ng/ui/dialog_settings.py52
-rw-r--r--tidal_dl_ng/ui/dialog_settings.ui51
13 files changed, 385 insertions, 253 deletions
diff --git a/tidal_dl_ng/constants.py b/tidal_dl_ng/constants.py
index d3ea2b8..10fb93d 100644
--- a/tidal_dl_ng/constants.py
+++ b/tidal_dl_ng/constants.py
@@ -6,6 +6,9 @@ EXTENSION_LYRICS: str = ".lrc"
UNIQUIFY_THRESHOLD: int = 99
FILENAME_SANITIZE_PLACEHOLDER: str = "_"
COVER_NAME: str = "cover.jpg"
+BLOCK_SIZE: int = 4096
+BLOCKS: int = 1024
+CHUNK_SIZE: int = BLOCK_SIZE * BLOCKS
class QualityVideo(StrEnum):
@@ -24,13 +27,6 @@ class MediaType(StrEnum):
ARTIST: str = "artist"
-class SkipExisting(StrEnum):
- Disabled: str = "False"
- Filename: str = "exact"
- ExtensionIgnore: str = "extension_ignore"
- Append: str = "append"
-
-
class CoverDimensions(StrEnum):
Px80: str = "80"
Px160: str = "160"
diff --git a/tidal_dl_ng/dialog.py b/tidal_dl_ng/dialog.py
index ff31633..928f6c1 100644
--- a/tidal_dl_ng/dialog.py
+++ b/tidal_dl_ng/dialog.py
@@ -10,7 +10,7 @@ from tidalapi import Quality as QualityAudio
from tidal_dl_ng import __version__
from tidal_dl_ng.config import Settings
-from tidal_dl_ng.constants import CoverDimensions, QualityVideo, SkipExisting
+from tidal_dl_ng.constants import CoverDimensions, QualityVideo
from tidal_dl_ng.model.cfg import HelpSettings
from tidal_dl_ng.model.cfg import Settings as ModelSettings
from tidal_dl_ng.model.meta import ReleaseLatest
@@ -159,7 +159,6 @@ class DialogPreferences(QtWidgets.QDialog):
def _init_comboboxes(self):
self.parameters_combo = [
- ("skip_existing", SkipExisting),
("quality_audio", QualityAudio),
("quality_video", QualityVideo),
("metadata_cover_dimension", CoverDimensions),
@@ -175,6 +174,7 @@ class DialogPreferences(QtWidgets.QDialog):
"extract_flac",
"metadata_cover_embed",
"cover_album_file",
+ "skip_existing",
]
def gui_populate(self):
diff --git a/tidal_dl_ng/download.py b/tidal_dl_ng/download.py
index b6d5f13..c755411 100644
--- a/tidal_dl_ng/download.py
+++ b/tidal_dl_ng/download.py
@@ -5,22 +5,31 @@ import shutil
import tempfile
import time
from collections.abc import Callable
+from concurrent import futures
from uuid import uuid4
import ffmpeg
import m3u8
import requests
-from constants import COVER_NAME
+from requests.adapters import HTTPAdapter, Retry
from requests.exceptions import HTTPError
from rich.progress import Progress, TaskID
from tidalapi import Album, Mix, Playlist, Session, Track, UserPlaylist, Video
+from tidalapi.exceptions import TooManyRequests
from tidalapi.media import AudioExtensions, Codec, Quality, StreamManifest, VideoExtensions
from tidal_dl_ng.config import Settings
-from tidal_dl_ng.constants import EXTENSION_LYRICS, REQUESTS_TIMEOUT_SEC, MediaType, QualityVideo, SkipExisting
+from tidal_dl_ng.constants import (
+ CHUNK_SIZE,
+ COVER_NAME,
+ EXTENSION_LYRICS,
+ REQUESTS_TIMEOUT_SEC,
+ MediaType,
+ QualityVideo,
+)
from tidal_dl_ng.helper.decryption import decrypt_file, decrypt_security_token
from tidal_dl_ng.helper.exceptions import MediaMissing
-from tidal_dl_ng.helper.path import check_file_exists, format_path_media, path_file_sanitize
+from tidal_dl_ng.helper.path import check_file_exists, format_path_media, path_file_sanitize, url_to_filename
from tidal_dl_ng.helper.tidal import (
instantiate_media,
items_results_all,
@@ -30,6 +39,7 @@ from tidal_dl_ng.helper.tidal import (
name_builder_title,
)
from tidal_dl_ng.metadata import Metadata
+from tidal_dl_ng.model.downloader import DownloadSegmentResult
from tidal_dl_ng.model.gui_data import ProgressBars
@@ -51,7 +61,7 @@ class RequestsClient:
class Download:
settings: Settings
session: Session
- skip_existing: SkipExisting = SkipExisting.Disabled
+ skip_existing: bool = False
fn_logger: Callable
progress_gui: ProgressBars
progress: Progress
@@ -61,7 +71,7 @@ class Download:
session: Session,
path_base: str,
fn_logger: Callable,
- skip_existing: SkipExisting = SkipExisting.Disabled,
+ skip_existing: bool = False,
progress_gui: ProgressBars = None,
progress: Progress = None,
):
@@ -88,27 +98,34 @@ class Download:
def _download(
self,
media: Track | Video,
- path_file: str,
- ) -> str:
+ stream_manifest: StreamManifest,
+ path_file: pathlib.Path,
+ ) -> (bool, pathlib.Path):
media_name: str = name_builder_item(media)
urls: [str]
+ path_base: pathlib.Path = path_file.parent
+ result_segments: bool = True
+ dl_segment_results: [DownloadSegmentResult] = []
+ result_merge: bool = False
# Get urls for media.
- if isinstance(media, Track):
- urls = media.get_stream().get_stream_manifest().urls
- stream_manifest: StreamManifest = media.get_stream().get_stream_manifest()
- elif isinstance(media, Video):
- m3u8_variant: m3u8.M3U8 = m3u8.load(media.get_url())
- # Find the desired video resolution or the next best one.
- m3u8_playlist, codecs = self._extract_video_stream(m3u8_variant, int(self.settings.data.quality_video))
- # Populate urls.
- urls = m3u8_playlist.files
+ try:
+ if isinstance(media, Track):
+ urls = stream_manifest.get_urls()
+ elif isinstance(media, Video):
+ m3u8_variant: m3u8.M3U8 = m3u8.load(media.get_url())
+ # Find the desired video resolution or the next best one.
+ m3u8_playlist, codecs = self._extract_video_stream(m3u8_variant, int(self.settings.data.quality_video))
+ # Populate urls.
+ urls = m3u8_playlist.files
+ except Exception:
+ return False, path_file
# Set the correct progress output channel.
if self.progress_gui is None:
- progress_stdout: bool = True
+ progress_to_stdout: bool = True
else:
- progress_stdout: bool = False
+ progress_to_stdout: bool = False
# Send signal to GUI with media name
self.progress_gui.item_name.emit(media_name[:30])
@@ -119,8 +136,11 @@ class Download:
progress_total: int = urls_count
block_size: int | None = None
elif urls_count == 1:
- # Will be computed later.
- progress_total: float = None
+ # Get file size and compute progress steps
+ r = requests.head(urls[0], timeout=REQUESTS_TIMEOUT_SEC)
+ total_size_in_bytes: int = int(r.headers.get("content-length", 0))
+ block_size: int | None = 1048576
+ progress_total: float = total_size_in_bytes / block_size
else:
raise ValueError
@@ -128,57 +148,115 @@ class Download:
p_task: TaskID = self.progress.add_task(
f"[blue]Item '{media_name[:30]}'",
total=progress_total,
- visible=progress_stdout,
+ visible=progress_to_stdout,
)
- # Write content to file until progress is finished.
+ # Download segments until progress is finished.
+ # TODO: Compute download speed (https://github.com/Textualize/rich/blob/master/examples/downloader.py)
while not self.progress.tasks[p_task].finished:
- with open(path_file, "wb") as f:
- for url in urls:
- try:
- # Create the request object with stream=True, so the content won't be loaded into memory at once.
- r = requests.get(url, stream=True, timeout=REQUESTS_TIMEOUT_SEC)
+ with futures.ThreadPoolExecutor(
+ max_workers=self.settings.data.downloads_simultaneous_per_track_max
+ ) as executor:
+ # Dispatch all download tasks to worker threads
+ l_futures: [any] = [
+ executor.submit(self._download_segment, url, path_base, block_size, p_task, progress_to_stdout)
+ for url in urls
+ ]
+ # Report results as they become available
+ for future in futures.as_completed(l_futures):
+ # Retrieve result
+ result_dl_segment: DownloadSegmentResult = future.result()
- r.raise_for_status()
+ dl_segment_results.append(result_dl_segment)
- # Compute progress iterations based on the file size and update task details.
- if not progress_total:
- # Get file size and compute progress steps
- total_size_in_bytes: int = int(r.headers.get("content-length", 0))
- block_size: int | None = 1048576
- progress_total: float = total_size_in_bytes / block_size
+ # check for a link that was skipped
+ if not result_dl_segment.result and (result_dl_segment.url is not urls[-1]):
+ # Sometimes it happens, if a track is very short (< 8 seconds or so), that the last URL in `urls` is
+ # invalid (HTTP Error 500) and not necessary. File won't be corrupt.
+ # If this is NOT the case, but any other URL has resulted in an error,
+ # mark the whole thing as corrupt.
+ result_segments = False
+ self.fn_logger.error(f"Something went wrong while downloading {media_name}. File is corrupt!")
- self.progress.update(p_task, total=progress_total)
+ tmp_path_file_decrypted: pathlib.Path = path_file
- # Write the content to disk. If `chunk_size` is set to `None` the whole file will be written at once.
- for data in r.iter_content(chunk_size=block_size):
- f.write(data)
- # Advance progress bar.
- self.progress.advance(p_task)
- except HTTPError as e:
- if url is urls[-1]:
- # It happens, if a track is very short (< 8 seconds or so), that the last URL in `urls` is
- # invalid (HTTP Error 500) and not necessary. File won't be corrupt.
- # Thus, advance progress bar to avoid infinity loops.
- self.progress.advance(p_task)
- else:
- # Finish downloading early and report error.
- # TODO: The track should somehow be marked as corrupt.
- self.progress.update(p_task, completed=progress_total)
- self.fn_logger.error(e)
- finally:
- # To send the progress to the GUI, we need to emit the percentage.
- if not progress_stdout:
- self.progress_gui.item.emit(self.progress.tasks[p_task].percentage)
+ # Only if no error happened while downloading.
+ if result_segments:
+ # Bring list into right order, so segments can be easily merged.
+ dl_segment_results.sort(key=lambda x: x.id_segment)
+ result_merge: bool = self._segments_merge(path_file, dl_segment_results)
- if isinstance(media, Track) and stream_manifest.is_encrypted:
- key, nonce = decrypt_security_token(stream_manifest.encryption_key)
- tmp_path_file_decrypted = path_file + "_decrypted"
- decrypt_file(path_file, tmp_path_file_decrypted, key, nonce)
- else:
- tmp_path_file_decrypted = path_file
+ if not result_merge:
+ self.fn_logger.error(f"Something went wrong while writing to {media_name}. File is corrupt!")
+ elif result_merge and isinstance(media, Track) and stream_manifest.is_encrypted:
+ key, nonce = decrypt_security_token(stream_manifest.encryption_key)
+ tmp_path_file_decrypted = path_file.with_suffix(".decrypted")
+ decrypt_file(path_file, tmp_path_file_decrypted, key, nonce)
+
+ return result_merge, tmp_path_file_decrypted
+
+ def _segments_merge(self, path_file, dl_segment_results) -> bool:
+ result: bool
+
+ # Copy the content of all segments into one file.
+ try:
+ with path_file.open("wb") as f_target:
+ for dl_segment_result in dl_segment_results:
+ with dl_segment_result.path_segment.open("rb") as f_segment:
+ # Read and write junks, which gives better HDD write performance
+ while segment := f_segment.read(CHUNK_SIZE):
+ f_target.write(segment)
+
+ # Delete segment from HDD
+ dl_segment_result.path_segment.unlink()
+
+ result = True
+ except Exception:
+ result = False
+
+ return result
+
+ def _download_segment(
+ self, url: str, path_base: pathlib.Path, block_size: int | None, p_task: TaskID, progress_to_stdout: bool
+ ) -> DownloadSegmentResult:
+ result: bool = False
+ path_segment: pathlib.Path = path_base / url_to_filename(url)
+ # Calculate the segment ID based on the file name within the URL.
+ filename_stem: str = str(path_segment.stem).split("_")[-1]
+ # CAUTION: This is a workaround, so BTS (LOW quality) track will work. They usually have only ONE link.
+ id_segment: int = int(filename_stem) if filename_stem.isdecimal() else 0
+ error: HTTPError | None = None
- return tmp_path_file_decrypted
+ # Retry download on failed segments, with an exponential delay between retries
+ s = requests.Session()
+ retries = Retry(total=5, backoff_factor=1) # , status_forcelist=[ 502, 503, 504 ])
+
+ s.mount("https://", HTTPAdapter(max_retries=retries))
+
+ try:
+ # Create the request object with stream=True, so the content won't be loaded into memory at once.
+ r = s.get(url, stream=True, timeout=REQUESTS_TIMEOUT_SEC)
+
+ r.raise_for_status()
+
+ # Write the content to disk. If `chunk_size` is set to `None` the whole file will be written at once.
+ with path_segment.open("wb") as f:
+ for data in r.iter_content(chunk_size=block_size):
+ f.write(data)
+ # Advance progress bar.
+ self.progress.advance(p_task)
+
+ result = True
+ except Exception:
+ self.progress.advance(p_task)
+
+ # To send the progress to the GUI, we need to emit the percentage.
+ if not progress_to_stdout:
+ self.progress_gui.item.emit(self.progress.tasks[p_task].percentage)
+
+ return DownloadSegmentResult(
+ result=result, url=url, path_segment=path_segment, id_segment=id_segment, error=error
+ )
def item(
self,
@@ -219,103 +297,139 @@ class Download:
return False, ""
- # Get extension.
- file_extension: str
-
- if isinstance(media, Track):
- # If a quality is explicitly set, change it.
- if quality_audio:
- quality_audio_old: Quality = self.adjust_quality_audio(quality_audio)
-
- file_extension = media.get_stream().get_stream_manifest().file_extension
- # Use M4A extension for MP4 audio tracks, because it looks better and is completely interchangeable.
- file_extension = AudioExtensions.M4A if file_extension == AudioExtensions.MP4 else file_extension
-
- if self.settings.data.extract_flac:
- do_flac_extract = False
-
- if (
- media.get_stream().get_stream_manifest().codecs.upper() == Codec.FLAC
- and file_extension != AudioExtensions.FLAC
- ):
- file_extension = AudioExtensions.FLAC
- do_flac_extract = True
- elif isinstance(media, Video):
- if quality_video:
- quality_video_old: QualityVideo = self.adjust_quality_video(quality_video)
-
- file_extension = AudioExtensions.MP4 if self.settings.data.video_convert_mp4 else VideoExtensions.TS
-
# Create file name and path
+ file_extension_dummy: str = AudioExtensions.FLAC
file_name_relative = format_path_media(file_template, media)
- path_media_dst = os.path.abspath(
- os.path.normpath(os.path.join(os.path.expanduser(self.path_base), file_name_relative))
+ path_media_dst: pathlib.Path = (
+ (pathlib.Path(self.path_base).expanduser() / (file_name_relative + file_extension_dummy))
+ .resolve()
+ .absolute()
)
# Sanitize final path_file to fit into OS boundaries.
- uniquify: bool = self.skip_existing == SkipExisting.Append
- path_media_dst = path_file_sanitize(path_media_dst + file_extension, adapt=True, uniquify=uniquify)
+ path_media_dst = pathlib.Path(path_file_sanitize(str(path_media_dst), adapt=True))
# Compute if and how downloads need to be skipped.
- if self.skip_existing in (SkipExisting.ExtensionIgnore, SkipExisting.Filename):
- extension_ignore: bool = self.skip_existing == SkipExisting.ExtensionIgnore
- file_exists: bool = check_file_exists(path_media_dst, extension_ignore=extension_ignore)
+ if self.skip_existing:
+ skip_file: bool = check_file_exists(path_media_dst, extension_ignore=True)
else:
- file_exists: bool = False
+ skip_file: bool = False
+
+ if not skip_file:
+ # Get extension.
+ file_extension: str
+ do_flac_extract = False
+ # If a quality is explicitly set, change it and remember the previously set quality.
+ quality_audio_old: Quality = self.adjust_quality_audio(quality_audio) if quality_audio else quality_audio
+ quality_video_old: QualityVideo = (
+ self.adjust_quality_video(quality_video) if quality_video else quality_video
+ )
+
+ try:
+ media_stream = media.get_stream()
+ stream_manifest: StreamManifest = media_stream.get_stream_manifest()
+ except TooManyRequests:
+ self.fn_logger.exception(
+ f"Too many requests against TIDAL backend. Skipping '{name_builder_item(media)}'. "
+ f"Consider to activate delay between downloads."
+ )
+
+ return False, ""
+ except Exception:
+ self.fn_logger.exception(f"Something went wrong. Skipping '{name_builder_item(media)}'.")
+
+ return False, ""
+
+ if isinstance(media, Track):
+ # TODO: HOTFIX! Go back to
+ # file_extension = media_stream.get_stream_manifest().file_extension
+ # After tidalapi has fixed #304
+ if stream_manifest.file_extension is VideoExtensions.TS:
+ file_extension = stream_manifest.file_extension
+ elif stream_manifest.dash_info and AudioExtensions.FLAC in stream_manifest.dash_info.first_url:
+ file_extension = AudioExtensions.FLAC
+ else:
+ file_extension = AudioExtensions.M4A
+
+ if self.settings.data.extract_flac and (
+ stream_manifest.codecs.upper() == Codec.FLAC and file_extension != AudioExtensions.FLAC
+ ):
+ file_extension = AudioExtensions.FLAC
+ do_flac_extract = True
+ elif isinstance(media, Video):
+ file_extension = AudioExtensions.MP4 if self.settings.data.video_convert_mp4 else VideoExtensions.TS
+
+ # Compute file name and create destination directory
+ path_media_dst = path_media_dst.with_suffix(file_extension)
+ os.makedirs(path_media_dst.parent, exist_ok=True)
- if not file_exists:
# Create a temp directory and file.
with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as tmp_path_dir:
- tmp_path_file = os.path.join(tmp_path_dir, str(uuid4()))
+ tmp_path_file: pathlib.Path = pathlib.Path(tmp_path_dir) / str(uuid4())
+
+ # Create empty file
+ tmp_path_file.touch()
+
# Download media.
- tmp_path_file = self._download(media=media, path_file=tmp_path_file)
+ result_download, tmp_path_file = self._download(
+ media=media, stream_manifest=stream_manifest, path_file=tmp_path_file
+ )
- # Convert video from TS to MP4
- if isinstance(media, Video) and self.settings.data.video_convert_mp4:
- # Convert `*.ts` file to `*.mp4` using ffmpeg
- tmp_path_file = self._video_convert(tmp_path_file)
+ if result_download:
+ # Convert video from TS to MP4
+ if isinstance(media, Video) and self.settings.data.video_convert_mp4:
+ # Convert `*.ts` file to `*.mp4` using ffmpeg
+ tmp_path_file = self._video_convert(tmp_path_file)
- # Extract FLAC from MP4 container using ffmpeg
- if isinstance(media, Track) and self.settings.data.extract_flac and do_flac_extract:
- tmp_path_file = self._extract_flac(tmp_path_file)
+ # Extract FLAC from MP4 container using ffmpeg
+ if isinstance(media, Track) and self.settings.data.extract_flac and do_flac_extract:
+ tmp_path_file = self._extract_flac(tmp_path_file)
- tmp_path_lyrics: pathlib.Path | None = None
- tmp_path_cover: pathlib.Path | None = None
+ tmp_path_lyrics: pathlib.Path | None = None
+ tmp_path_cover: pathlib.Path | None = None
- # Write metadata to file.
- if not isinstance(media, Video):
- result_metadata, tmp_path_lyrics, tmp_path_cover = self.metadata_write(
- media, tmp_path_file, is_parent_album
- )
+ # Write metadata to file.
+ if not isinstance(media, Video):
+ result_metadata, tmp_path_lyrics, tmp_path_cover = self.metadata_write(
+ media, tmp_path_file, is_parent_album
+ )
+
+ # Move lyrics file
+ if self.settings.data.lyrics_file and not isinstance(media, Video) and tmp_path_lyrics:
+ self._move_lyrics(tmp_path_lyrics, path_media_dst)
+
+ # Move cover file
+ # TODO: Cover is downloaded with every track of the album. Needs refactoring, so cover is only
+ # downloaded for an album once.
+ if self.settings.data.cover_album_file and tmp_path_cover:
+ self._move_cover(tmp_path_cover, path_media_dst)
+
+ self.fn_logger.info(f"Downloaded item '{name_builder_item(media)}'.")
# Move final file to the configured destination directory.
- os.makedirs(os.path.dirname(path_media_dst), exist_ok=True)
shutil.move(tmp_path_file, path_media_dst)
- # Move lyrics file
- if self.settings.data.lyrics_file and not isinstance(media, Video):
- self._move_lyrics(tmp_path_lyrics, path_media_dst)
+ if quality_audio:
+ # Set quality back to the global user value
+ self.adjust_quality_audio(quality_audio_old)
- # Move cover file
- if self.settings.data.cover_album_file:
- self._move_cover(tmp_path_cover, path_media_dst)
+ if quality_video:
+ # Set quality back to the global user value
+ self.adjust_quality_video(quality_video_old)
else:
self.fn_logger.debug(f"Download skipped, since file exists: '{path_media_dst}'")
- status_download: bool = not file_exists
-
- if quality_audio:
- # Set quality back to the global user value
- self.adjust_quality_audio(quality_audio_old)
-
- if quality_video:
- # Set quality back to the global user value
- self.adjust_quality_video(quality_video_old)
+ status_download: bool = not skip_file
# Whether a file was downloaded or skipped and the download delay is enabled, wait until the next download.
# Only use this, if you have a list of several Track items.
- if download_delay:
- time_sleep: float = round(random.SystemRandom().uniform(2, 5), 1)
+ if download_delay and not skip_file:
+ time_sleep: float = round(
+ random.SystemRandom().uniform(
+ self.settings.data.download_delay_sec_min, self.settings.data.download_delay_sec_max
+ ),
+ 1,
+ )
self.fn_logger.debug(f"Next download will start in {time_sleep} seconds.")
time.sleep(time_sleep)
@@ -340,7 +454,7 @@ class Download:
result: bool
# Check if the file was downloaded
- if path_file_source and os.path.isfile(path_file_source):
+ if path_file_source and path_file_source.is_file():
# Move it.
shutil.move(path_file_source, path_file_destination)
@@ -350,16 +464,16 @@ class Download:
return result
- def _move_lyrics(self, path_lyrics: pathlib.Path, file_media_dst: str) -> bool:
+ def _move_lyrics(self, path_lyrics: pathlib.Path, file_media_dst: pathlib.Path) -> bool:
# Build tmp lyrics filename
- path_file_lyrics: str = os.path.splitext(file_media_dst)[0] + EXTENSION_LYRICS
+ path_file_lyrics: pathlib.Path = file_media_dst.with_suffix(EXTENSION_LYRICS)
result: bool = self._move_file(path_lyrics, path_file_lyrics)
return result
- def _move_cover(self, path_cover: pathlib.Path, file_media_dst: str) -> bool:
+ def _move_cover(self, path_cover: pathlib.Path, file_media_dst: pathlib.Path) -> bool:
# Build tmp lyrics filename
- path_file_cover: pathlib.Path = pathlib.Path(file_media_dst).parent.absolute() / COVER_NAME
+ path_file_cover: pathlib.Path = file_media_dst.parent / COVER_NAME
result: bool = self._move_file(path_cover, path_file_cover)
return result
@@ -403,7 +517,7 @@ class Download:
return result
def metadata_write(
- self, track: Track, path_media: str, is_parent_album: str
+ self, track: Track, path_media: pathlib.Path, is_parent_album: bool
) -> (bool, pathlib.Path | None, pathlib.Path | None):
result: bool = False
path_lyrics: pathlib.Path | None = None
@@ -433,14 +547,14 @@ class Download:
print(f"Could not retrieve lyrics for `{name_builder_item(track)}`.")
if lyrics and self.settings.data.lyrics_file:
- path_lyrics = self.lyrics_to_file(pathlib.Path(path_media).parent, lyrics)
+ path_lyrics = self.lyrics_to_file(path_media.parent, lyrics)
if self.settings.data.metadata_cover_embed or (self.settings.data.cover_album_file and is_parent_album):
url_cover = track.album.image(int(self.settings.data.metadata_cover_dimension))
cover_data = self.cover_data(url=url_cover)
if cover_data and self.settings.data.cover_album_file and is_parent_album:
- path_cover = self.cover_to_file(pathlib.Path(path_media).parent, cover_data)
+ path_cover = self.cover_to_file(path_media.parent, cover_data)
# `None` values are not allowed.
m: Metadata = Metadata(
@@ -487,29 +601,22 @@ class Download:
file_name_relative = format_path_media(file_template, media)
# Get the name of the list and check, if videos should be included.
- videos_include: bool = True
-
- if isinstance(media, Mix):
- list_media_name = media.title[:30]
- elif video_download:
- list_media_name = name_builder_title(media)[:30]
- else:
- videos_include = False
- list_media_name = name_builder_title(media)[:30]
+ list_media_name: str = name_builder_title(media)
+ list_media_name_short: str = list_media_name[:30]
# Get all items of the list.
- items = items_results_all(media, videos_include=videos_include)
+ items = items_results_all(media, videos_include=video_download)
# Determine where to redirect the progress information.
if self.progress_gui is None:
progress_stdout: bool = True
else:
progress_stdout: bool = False
- self.progress_gui.list_name.emit(list_media_name[:30])
+ self.progress_gui.list_name.emit(list_media_name_short[:30])
# Create the list progress task.
p_task1: TaskID = self.progress.add_task(
- f"[green]List '{list_media_name}'", total=len(items), visible=progress_stdout
+ f"[green]List '{list_media_name_short}'", total=len(items), visible=progress_stdout
)
is_album: bool = isinstance(media, Album)
@@ -518,7 +625,7 @@ class Download:
while not self.progress.finished:
for item_media in items:
# Download the item.
- status_download, result_path_file = self.item(
+ status, result_path_file = self.item(
media=item_media,
file_template=file_name_relative,
quality_audio=quality_audio,
@@ -533,22 +640,29 @@ class Download:
if not progress_stdout:
self.progress_gui.list_item.emit(self.progress.tasks[p_task1].percentage)
- def _video_convert(self, path_file: str) -> str:
- path_file_out = path_file + AudioExtensions.MP4
+ self.fn_logger.info(f"Finished list '{list_media_name}'.")
+
+ def _video_convert(self, path_file: pathlib.Path) -> pathlib.Path:
+ path_file_out: pathlib.Path = path_file.with_suffix(AudioExtensions.MP4)
result, _ = (
ffmpeg.input(path_file)
- .output(path_file_out, map=0, c="copy", loglevel="quiet")
+ .output(str(path_file_out), map=0, c="copy", loglevel="quiet")
.run(cmd=self.settings.data.path_binary_ffmpeg)
)
return path_file_out
- def _extract_flac(self, path_media_src: str) -> str:
- path_media_out = path_media_src + AudioExtensions.FLAC
+ def _extract_flac(self, path_media_src: pathlib.Path) -> pathlib.Path:
+ path_media_out = path_media_src.with_suffix(AudioExtensions.FLAC)
result, _ = (
ffmpeg.input(path_media_src)
.output(
- path_media_out, map=0, movflags="use_metadata_tags", acodec="copy", map_metadata="0:g", loglevel="quiet"
+ str(path_media_out),
+ map=0,
+ movflags="use_metadata_tags",
+ acodec="copy",
+ map_metadata="0:g",
+ loglevel="quiet",
)
.run(cmd=self.settings.data.path_binary_ffmpeg)
)
diff --git a/tidal_dl_ng/gui.py b/tidal_dl_ng/gui.py
index f7d84ca..4d3c38a 100644
--- a/tidal_dl_ng/gui.py
+++ b/tidal_dl_ng/gui.py
@@ -516,6 +516,8 @@ class MainWindow(QtWidgets.QMainWindow, Ui_MainWindow):
self._init_dl()
def search(self, query: str, types_media: SearchTypes) -> [ResultItem]:
+ query = query.strip()
+
# If a direct link was searched for, skip search and create the object from the link directly.
if "http" in query:
media_type = get_tidal_media_type(query)
diff --git a/tidal_dl_ng/helper/decryption.py b/tidal_dl_ng/helper/decryption.py
index ebb946a..17ea19b 100644
--- a/tidal_dl_ng/helper/decryption.py
+++ b/tidal_dl_ng/helper/decryption.py
@@ -1,4 +1,5 @@
import base64
+import pathlib
from Crypto.Cipher import AES
from Crypto.Util import Counter
@@ -35,7 +36,7 @@ def decrypt_security_token(security_token: str) -> (str, str):
return key, nonce
-def decrypt_file(path_file_encrypted: str, path_file_destination: str, key: str, nonce: str) -> None:
+def decrypt_file(path_file_encrypted: pathlib.Path, path_file_destination: pathlib.Path, key: str, nonce: str) -> None:
"""
Decrypts an encrypted MQA file given the file, key and nonce.
TODO: Is it really only necessary for MQA of for all other formats, too?
@@ -46,9 +47,9 @@ def decrypt_file(path_file_encrypted: str, path_file_destination: str, key: str,
decryptor = AES.new(key, AES.MODE_CTR, counter=counter)
# Open and decrypt
- with open(path_file_encrypted, "rb") as f_src:
+ with path_file_encrypted.open("rb") as f_src:
audio_decrypted = decryptor.decrypt(f_src.read())
# Replace with decrypted file
- with open(path_file_destination, "wb") as f_dst:
+ with path_file_destination.open("wb") as f_dst:
f_dst.write(audio_decrypted)
diff --git a/tidal_dl_ng/helper/path.py b/tidal_dl_ng/helper/path.py
index cc8d5b6..fb7bb87 100644
--- a/tidal_dl_ng/helper/path.py
+++ b/tidal_dl_ng/helper/path.py
@@ -1,8 +1,10 @@
import math
import os
+import pathlib
+import posixpath
import re
import sys
-from pathlib import Path, PosixPath
+from urllib.parse import unquote, urlsplit
from pathvalidate import sanitize_filename, sanitize_filepath
from pathvalidate.error import ValidationError
@@ -77,8 +79,7 @@ def format_str_media(name: str, media: Track | Album | Playlist | UserPlaylist |
elif hasattr(media, "artist"):
result = media.artist.name
case "album_artist":
- if isinstance(media, Album):
- result = name_builder_album_artist(media)
+ result = name_builder_album_artist(media)
case "track_title":
if isinstance(media, Track | Video):
result = name_builder_title(media)
@@ -195,7 +196,7 @@ def get_format_template(
def path_file_sanitize(path_file: str, adapt: bool = False, uniquify: bool = False) -> (bool, str):
# Split into path and filename
pathname, filename = os.path.split(path_file)
- file_extension: str = Path(path_file).suffix
+ file_extension: str = pathlib.Path(path_file).suffix
# Sanitize path
try:
@@ -205,7 +206,7 @@ def path_file_sanitize(path_file: str, adapt: bool = False, uniquify: bool = Fal
except ValidationError:
# If adaption of path is allowed in case of an error set path to HOME.
if adapt:
- pathname_sanitized: str = Path.home()
+ pathname_sanitized: str = str(pathlib.Path.home())
else:
raise
@@ -272,10 +273,10 @@ def file_unique_suffix(path_file: str, seperator: str = "_") -> str:
return unique_suffix
-def check_file_exists(path_file: str, extension_ignore: bool = False) -> bool:
+def check_file_exists(path_file: pathlib.Path, extension_ignore: bool = False) -> bool:
if extension_ignore:
- path_file_stem: str = Path(path_file).stem
- path_parent: PosixPath = Path(path_file).parent
+ path_file_stem: str = pathlib.Path(path_file).stem
+ path_parent: pathlib.Path = pathlib.Path(path_file).parent
path_files: [str] = []
for extension in AudioExtensions:
@@ -295,3 +296,22 @@ def resource_path(relative_path):
base_path = os.path.abspath(".")
return os.path.join(base_path, relative_path)
+
+
+def url_to_filename(url: str) -> str:
+ """Return basename corresponding to url.
+ >>> print(url_to_filename('http://example.com/path/to/file%C3%80?opt=1'))
+ fileÀ
+ >>> print(url_to_filename('http://example.com/slash%2fname')) # '/' in name
+ Taken from https://gist.github.com/zed/c2168b9c52b032b5fb7d
+ Traceback (most recent call last):
+ ...
+ ValueError
+ """
+ urlpath: str = urlsplit(url).path
+ basename: str = posixpath.basename(unquote(urlpath))
+
+ if os.path.basename(basename) != basename or unquote(posixpath.basename(urlpath)) != basename:
+ raise ValueError # reject '%2f' or 'dir%5Cbasename.ext' on Windows
+
+ return basename
diff --git a/tidal_dl_ng/helper/tidal.py b/tidal_dl_ng/helper/tidal.py
index d679d5e..e568cc6 100644
--- a/tidal_dl_ng/helper/tidal.py
+++ b/tidal_dl_ng/helper/tidal.py
@@ -25,8 +25,10 @@ def name_builder_album_artist(media: Track | Album) -> str:
return ", ".join(artists_tmp)
-def name_builder_title(media: Track | Video) -> str:
- result: str = media.full_name if hasattr(media, "full_name") else media.name
+def name_builder_title(media: Track | Video | Mix | Playlist | Album | Video) -> str:
+ result: str = (
+ media.title if isinstance(media, Mix) else media.full_name if hasattr(media, "full_name") else media.name
+ )
return result
@@ -186,7 +188,7 @@ def instantiate_media(
def quality_audio_highest(media: Track | Album) -> Quality:
quality: Quality
- if MediaMetadataTags.hires_lossless in media.media_metadata_tags:
+ if MediaMetadataTags.hi_res_lossless in media.media_metadata_tags:
quality = Quality.hi_res_lossless
elif MediaMetadataTags.lossless in media.media_metadata_tags:
quality = Quality.high_lossless
diff --git a/tidal_dl_ng/helper/wrapper.py b/tidal_dl_ng/helper/wrapper.py
index d8ffb09..cd8e6ba 100644
--- a/tidal_dl_ng/helper/wrapper.py
+++ b/tidal_dl_ng/helper/wrapper.py
@@ -21,3 +21,6 @@ class LoggerWrapped:
def critical(self, value):
self.fn_print(value)
+
+ def exception(self, value):
+ self.fn_print(value)
diff --git a/tidal_dl_ng/metadata.py b/tidal_dl_ng/metadata.py
index d9ecb8c..e1fb44b 100644
--- a/tidal_dl_ng/metadata.py
+++ b/tidal_dl_ng/metadata.py
@@ -111,6 +111,8 @@ class Metadata:
self.m.tags["LYRICS"] = self.lyrics
def set_mp3(self):
+ # ID3 Frame (tags) overview: https://exiftool.org/TagNames/ID3.html / https://id3.org/id3v2.3.0
+ # Mapping overview: https://docs.mp3tag.de/mapping/
self.m.tags.add(TIT2(encoding=3, text=self.title))
self.m.tags.add(TALB(encoding=3, text=self.album))
self.m.tags.add(TOPE(encoding=3, text=self.albumartist))
diff --git a/tidal_dl_ng/model/cfg.py b/tidal_dl_ng/model/cfg.py
index fae3590..06614f4 100644
--- a/tidal_dl_ng/model/cfg.py
+++ b/tidal_dl_ng/model/cfg.py
@@ -3,13 +3,13 @@ from dataclasses import dataclass
from dataclasses_json import dataclass_json
from tidalapi import Quality
-from tidal_dl_ng.constants import CoverDimensions, QualityVideo, SkipExisting
+from tidal_dl_ng.constants import CoverDimensions, QualityVideo
@dataclass_json
@dataclass
class Settings:
- skip_existing: SkipExisting = SkipExisting.Disabled
+ skip_existing: bool = True
lyrics_embed: bool = False
lyrics_file: bool = False
# TODO: Implement API KEY selection.
@@ -25,7 +25,7 @@ class Settings:
quality_video: QualityVideo = QualityVideo.P480
format_album: str = (
"Albums/{album_artist} - {album_title}{album_explicit}/{track_volume_num_optional}"
- "{album_track_num}. {artist_name} - {track_title}"
+ "{album_track_num}. {artist_name} - {track_title}{album_explicit}"
)
format_playlist: str = "Playlists/{playlist_name}/{artist_name} - {track_title}"
format_mix: str = "Mix/{mix_name}/{artist_name} - {track_title}"
@@ -37,16 +37,15 @@ class Settings:
metadata_cover_embed: bool = True
cover_album_file: bool = True
extract_flac: bool = True
+ downloads_simultaneous_per_track_max: int = 20
+ download_delay_sec_min: float = 3.0
+ download_delay_sec_max: float = 5.0
@dataclass_json
@dataclass
class HelpSettings:
- skip_existing: str = (
- "Do not download, if file already exists. Possible option false = do not skip, "
- "'exact' = if filename already exists, 'extension_ignore' = skip even if a file with a "
- "different file extension exists."
- )
+ skip_existing: str = "Skip download if file already exists."
album_cover_save: str = "Safe cover to album folder."
lyrics_embed: str = "Embed lyrics in audio file, if lyrics are available."
lyrics_file: str = "Save lyrics to separate *.lrc file, if lyrics are available."
@@ -82,6 +81,9 @@ class HelpSettings:
metadata_cover_embed: str = "Embed album cover into file."
cover_album_file: str = "Save cover to 'cover.jpg', if an album is downloaded."
extract_flac: str = "Extract FLAC audio tracks from MP4 containers and save them as `*.flac` (uses FFmpeg)."
+ downloads_simultaneous_per_track_max: str = "Maximum number of simultaneous chunk downloads per track."
+ download_delay_sec_min: float = "Lower boundary for the calculation of the download delay in seconds."
+ download_delay_sec_max: float = "Upper boundary for the calculation of the download delay in seconds."
@dataclass_json
diff --git a/tidal_dl_ng/model/downloader.py b/tidal_dl_ng/model/downloader.py
new file mode 100644
index 0000000..3f6151f
--- /dev/null
+++ b/tidal_dl_ng/model/downloader.py
@@ -0,0 +1,13 @@
+import pathlib
+from dataclasses import dataclass
+
+from requests import HTTPError
+
+
+@dataclass
+class DownloadSegmentResult:
+ result: bool
+ url: str
+ path_segment: pathlib.Path
+ id_segment: int
+ error: HTTPError | None = None
diff --git a/tidal_dl_ng/ui/dialog_settings.py b/tidal_dl_ng/ui/dialog_settings.py
index ad45f1d..a624e34 100644
--- a/tidal_dl_ng/ui/dialog_settings.py
+++ b/tidal_dl_ng/ui/dialog_settings.py
@@ -153,6 +153,24 @@ class Ui_DialogSettings:
self.lv_flags.addLayout(self.lh_flags_4)
+ self.horizontalLayout = QHBoxLayout()
+ self.horizontalLayout.setObjectName("horizontalLayout")
+ self.lv_flag_skip_existing = QVBoxLayout()
+ self.lv_flag_skip_existing.setObjectName("lv_flag_skip_existing")
+ self.cb_skip_existing = QCheckBox(self.gb_flags)
+ self.cb_skip_existing.setObjectName("cb_skip_existing")
+
+ self.lv_flag_skip_existing.addWidget(self.cb_skip_existing)
+
+ self.horizontalLayout.addLayout(self.lv_flag_skip_existing)
+
+ self.verticalLayout_3 = QVBoxLayout()
+ self.verticalLayout_3.setObjectName("verticalLayout_3")
+
+ self.horizontalLayout.addLayout(self.verticalLayout_3)
+
+ self.lv_flags.addLayout(self.horizontalLayout)
+
self.lv_main.addWidget(self.gb_flags)
self.gb_choices = QGroupBox(DialogSettings)
@@ -164,38 +182,13 @@ class Ui_DialogSettings:
self.gb_choices.setSizePolicy(sizePolicy3)
self.lv_choices = QVBoxLayout(self.gb_choices)
self.lv_choices.setObjectName("lv_choices")
- self.lh_choices_skip_existing = QHBoxLayout()
- self.lh_choices_skip_existing.setObjectName("lh_choices_skip_existing")
- self.l_icon_skip_existing = QLabel(self.gb_choices)
- self.l_icon_skip_existing.setObjectName("l_icon_skip_existing")
- sizePolicy4 = QSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed)
- sizePolicy4.setHorizontalStretch(0)
- sizePolicy4.setVerticalStretch(0)
- sizePolicy4.setHeightForWidth(self.l_icon_skip_existing.sizePolicy().hasHeightForWidth())
- self.l_icon_skip_existing.setSizePolicy(sizePolicy4)
-
- self.lh_choices_skip_existing.addWidget(self.l_icon_skip_existing)
-
- self.l_skip_existing = QLabel(self.gb_choices)
- self.l_skip_existing.setObjectName("l_skip_existing")
- sizePolicy4.setHeightForWidth(self.l_skip_existing.sizePolicy().hasHeightForWidth())
- self.l_skip_existing.setSizePolicy(sizePolicy4)
-
- self.lh_choices_skip_existing.addWidget(self.l_skip_existing)
-
- self.c_skip_existing = QComboBox(self.gb_choices)
- self.c_skip_existing.setObjectName("c_skip_existing")
-
- self.lh_choices_skip_existing.addWidget(self.c_skip_existing)
-
- self.lh_choices_skip_existing.setStretch(2, 50)
-
- self.lv_choices.addLayout(self.lh_choices_skip_existing)
-
self.lh_choices_quality_audio = QHBoxLayout()
self.lh_choices_quality_audio.setObjectName("lh_choices_quality_audio")
self.l_icon_quality_audio = QLabel(self.gb_choices)
self.l_icon_quality_audio.setObjectName("l_icon_quality_audio")
+ sizePolicy4 = QSizePolicy(QSizePolicy.Policy.Fixed, QSizePolicy.Policy.Fixed)
+ sizePolicy4.setHorizontalStretch(0)
+ sizePolicy4.setVerticalStretch(0)
sizePolicy4.setHeightForWidth(self.l_icon_quality_audio.sizePolicy().hasHeightForWidth())
self.l_icon_quality_audio.setSizePolicy(sizePolicy4)
@@ -532,9 +525,8 @@ class Ui_DialogSettings:
self.cb_extract_flac.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None))
self.cb_metadata_cover_embed.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None))
self.cb_cover_album_file.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None))
+ self.cb_skip_existing.setText(QCoreApplication.translate("DialogSettings", "CheckBox", None))
self.gb_choices.setTitle(QCoreApplication.translate("DialogSettings", "Choices", None))
- self.l_icon_skip_existing.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None))
- self.l_skip_existing.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None))
self.l_icon_quality_audio.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None))
self.l_quality_audio.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None))
self.l_icon_quality_video.setText(QCoreApplication.translate("DialogSettings", "TextLabel", None))
diff --git a/tidal_dl_ng/ui/dialog_settings.ui b/tidal_dl_ng/ui/dialog_settings.ui
index 9bc894d..4fa07cf 100644
--- a/tidal_dl_ng/ui/dialog_settings.ui
+++ b/tidal_dl_ng/ui/dialog_settings.ui
@@ -210,6 +210,24 @@
</item>
</layout>
</item>
+ <item>
+ <layout class="QHBoxLayout" name="horizontalLayout">
+ <item>
+ <layout class="QVBoxLayout" name="lv_flag_skip_existing">
+ <item>
+ <widget class="QCheckBox" name="cb_skip_existing">
+ <property name="text">
+ <string>CheckBox</string>
+ </property>
+ </widget>
+ </item>
+ </layout>
+ </item>
+ <item>
+ <layout class="QVBoxLayout" name="verticalLayout_3"/>
+ </item>
+ </layout>
+ </item>
</layout>
</widget>
</item>
@@ -226,39 +244,6 @@
</property>
<layout class="QVBoxLayout" name="lv_choices">
<item>
- <layout class="QHBoxLayout" name="lh_choices_skip_existing" stretch="0,0,50">
- <item>
- <widget class="QLabel" name="l_icon_skip_existing">
- <property name="sizePolicy">
- <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
- <horstretch>0</horstretch>
- <verstretch>0</verstretch>
- </sizepolicy>
- </property>
- <property name="text">
- <string>TextLabel</string>
- </property>
- </widget>
- </item>
- <item>
- <widget class="QLabel" name="l_skip_existing">
- <property name="sizePolicy">
- <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
- <horstretch>0</horstretch>
- <verstretch>0</verstretch>
- </sizepolicy>
- </property>
- <property name="text">
- <string>TextLabel</string>
- </property>
- </widget>
- </item>
- <item>
- <widget class="QComboBox" name="c_skip_existing"/>
- </item>
- </layout>
- </item>
- <item>
<layout class="QHBoxLayout" name="lh_choices_quality_audio" stretch="0,0,50">
<item>
<widget class="QLabel" name="l_icon_quality_audio">