diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 74f752780..824865215 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -1,6 +1,7 @@ import os import re import uuid +from collections import deque as queue from pathlib import Path from django import db from django.conf import settings @@ -527,23 +528,31 @@ def get_index(self, type): days = timezone.timedelta(seconds=self.download_cap).days response = indexer(self.get_index_url(type=type), days=days) if not isinstance(response, dict): - return [] - entries = response.get('entries', []) + return list() + entries = response.get('entries', list()) return entries def index_media(self): ''' - Index the media source returning a list of media metadata as dicts. + Index the media source returning a queue of media metadata as dicts. ''' - entries = list() + entries = queue(list(), getattr(settings, 'MAX_ENTRIES_PROCESSING', 0) or None) if self.index_videos: - entries += self.get_index('videos') + entries.extend(reversed(self.get_index('videos'))) + # Playlists do something different that I have yet to figure out if not self.is_playlist: if self.index_streams: - entries += self.get_index('streams') + streams = self.get_index('streams') + if entries.maxlen is None or 0 == len(entries): + entries.extend(reversed(streams)) + else: + # share the queue between streams and videos + allowed_streams = max( + entries.maxlen // 2, + entries.maxlen - len(entries), + ) + entries.extend(reversed(streams[-1 * allowed_streams :])) - if settings.MAX_ENTRIES_PROCESSING: - entries = entries[:settings.MAX_ENTRIES_PROCESSING] return entries diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index 0467a4fd1..6d0727c01 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -255,7 +255,7 @@ def cleanup_old_media(): schedule_media_servers_update() -def cleanup_removed_media(source, videos): +def cleanup_removed_media(source, video_keys): if not source.delete_removed_media: return log.info(f'Cleaning up media no longer in source: {source}') @@ -265,8 +265,7 @@ def cleanup_removed_media(source, videos): source=source, ) for media in qs_gen(mqs): - matching_source_item = [video['id'] for video in videos if video['id'] == media.key] - if not matching_source_item: + if media.key not in video_keys: log.info(f'{media.name} is no longer in source, removing') with atomic(): media.delete() @@ -316,12 +315,17 @@ def index_source_task(source_id): end=task.verbose_name.find('Index'), ) tvn_format = '{:,}' + f'/{num_videos:,}' - for vn, video in enumerate(videos, start=1): + vn = 0 + video_keys = set() + while len(videos) > 0: + vn += 1 + video = videos.popleft() # Create or update each video as a Media object key = video.get(source.key_field, None) if not key: # Video has no unique key (ID), it can't be indexed continue + video_keys.add(key) update_task_status(task, tvn_format.format(vn)) # media, new_media = Media.objects.get_or_create(key=key, source=source) try: @@ -376,7 +380,7 @@ def index_source_task(source_id): # Reset task.verbose_name to the saved value update_task_status(task, None) # Cleanup of media no longer available from the source - cleanup_removed_media(source, videos) + cleanup_removed_media(source, video_keys) videos = video = None diff --git a/tubesync/sync/youtube.py b/tubesync/sync/youtube.py index 9814db248..c795261a1 100644 --- a/tubesync/sync/youtube.py +++ b/tubesync/sync/youtube.py @@ -326,13 +326,10 @@ def download_media( final_path = Path(output_file).resolve(strict=False) expected_file = shell_quote(str(final_path)) cmds = pp_opts.exec_cmd.get('after_move', list()) - # It is important that we use a tuple for strings. - # Otherwise, list adds each character instead. - # That last comma is really necessary! - cmds += ( + cmds.append( f'test -f {expected_file} || ' 'mv -T -u -- %(filepath,_filename|)q ' - f'{expected_file}', + f'{expected_file}' ) # assignment is the quickest way to cover both 'get' cases pp_opts.exec_cmd['after_move'] = cmds @@ -387,7 +384,7 @@ def download_media( youtube_ea_dict = ytopts['extractor_args'].get('youtube', dict()) formats_list = youtube_ea_dict.get('formats', list()) if 'missing_pot' not in formats_list: - formats_list += ('missing_pot',) + formats_list.append('missing_pot') youtube_ea_dict.update({ 'formats': formats_list, })