From 8f75f37a301fa450ffc0a92e58b8b75b54ac6a69 Mon Sep 17 00:00:00 2001 From: Nicolaj L J Date: Sun, 24 Sep 2023 14:10:38 +0200 Subject: [PATCH 1/8] can use the no sub dir command --- XMADownloader.App/Models/CommandLineOptions.cs | 4 ++-- XMADownloader.App/Program.cs | 2 +- .../Models/XmaDownloaderSettings.cs | 4 ++-- .../XmaCrawledUrlProcessor.cs | 10 +++++----- XMADownloader.Implementation/XmaPageCrawler.cs | 14 +++++++------- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/XMADownloader.App/Models/CommandLineOptions.cs b/XMADownloader.App/Models/CommandLineOptions.cs index 00d0554..623729a 100644 --- a/XMADownloader.App/Models/CommandLineOptions.cs +++ b/XMADownloader.App/Models/CommandLineOptions.cs @@ -44,8 +44,8 @@ class CommandLineOptions [Option("remote-browser-address", Required = false, HelpText = "Advanced users only. Address of the browser with remote debugging enabled. Refer to documentation for more details.")] public string RemoteBrowserAddress { get; set; } - /*[Option("use-sub-directories", Required = false, HelpText = "Create a new directory inside of the download directory for every post instead of placing all files into a single directory.")] - public bool UseSubDirectories { get; set; }*/ + [Option("use-sub-directories", Required = false, HelpText = "Create a new directory inside of the download directory for every post instead of placing all files into a single directory.")] + public bool UseSubDirectories { get; set; } [Option("sub-directory-pattern", Required = false, HelpText = "Pattern which will be used to create a name for the sub directories if --use-sub-directories is used. Supported parameters: %ModId%, %PublishedAt%, %PostTitle%.", Default = "[%ModId%] %PublishedAt% %PostTitle%")] public string SubDirectoryPattern { get; set; } diff --git a/XMADownloader.App/Program.cs b/XMADownloader.App/Program.cs index 8a59a5c..f168394 100644 --- a/XMADownloader.App/Program.cs +++ b/XMADownloader.App/Program.cs @@ -156,7 +156,7 @@ private static async Task InitializeSettings(CommandLineO DownloadDirectory = commandLineOptions.DownloadDirectory, FileExistsAction = commandLineOptions.FileExistsAction, IsCheckRemoteFileSize = !commandLineOptions.IsDisableRemoteFileSizeCheck, - //IsUseSubDirectories = commandLineOptions.UseSubDirectories, + IsUseSubDirectories = commandLineOptions.UseSubDirectories, SubDirectoryPattern = commandLineOptions.SubDirectoryPattern, MaxSubdirectoryNameLength = commandLineOptions.MaxSubdirectoryNameLength, MaxFilenameLength = commandLineOptions.MaxFilenameLength, diff --git a/XMADownloader.Common/Models/XmaDownloaderSettings.cs b/XMADownloader.Common/Models/XmaDownloaderSettings.cs index b09b0c0..7743945 100644 --- a/XMADownloader.Common/Models/XmaDownloaderSettings.cs +++ b/XMADownloader.Common/Models/XmaDownloaderSettings.cs @@ -18,7 +18,7 @@ public record XmaDownloaderSettings : UniversalDownloaderPlatformSettings, IPupp /// /// Create a new directory for every post and store files of said post in that directory /// - //public bool IsUseSubDirectories { get; init; } + public bool IsUseSubDirectories { get; init; } /// /// Pattern used to generate directory name if UseSubDirectories is enabled @@ -50,7 +50,7 @@ public XmaDownloaderSettings() { SaveDescriptions = true; SaveHtml = true; - //IsUseSubDirectories = false; + IsUseSubDirectories = false; SubDirectoryPattern = "[%ModId%] %PublishedAt% %PostTitle%"; FallbackToContentTypeFilenames = false; MaxFilenameLength = 100; diff --git a/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs b/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs index 1f63bed..4cd72a8 100644 --- a/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs +++ b/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs @@ -56,10 +56,10 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) if (!crawledUrl.IsProcessedByPlugin) { - /*if (!_XMADownloaderSettings.IsUseSubDirectories) + if (!_xmaDownloaderSettings.IsUseSubDirectories) filename = $"{crawledUrl.ModId}_"; else - filename = "";*/ + filename = ""; if (crawledUrl.Filename == null) throw new DownloadException($"[{crawledUrl.ModId}] No filename for {crawledUrl.Url}!"); @@ -111,9 +111,9 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) string downloadDirectory = crawledUrl.UserId.ToString(); - if (/*_XMADownloaderSettings.IsUseSubDirectories*/true) - downloadDirectory = Path.Combine(downloadDirectory, PostSubdirectoryHelper.CreateNameFromPattern(crawledUrl, _xmaDownloaderSettings.SubDirectoryPattern, _xmaDownloaderSettings.MaxSubdirectoryNameLength)); - + //if (_xmaDownloaderSettings.IsUseSubDirectories) + // downloadDirectory = Path.Combine(downloadDirectory, PostSubdirectoryHelper.CreateNameFromPattern(crawledUrl, _xmaDownloaderSettings.SubDirectoryPattern, _xmaDownloaderSettings.MaxSubdirectoryNameLength)); + //_logger.Debug(crawledUrl.DownloadPath); crawledUrl.DownloadPath = !crawledUrl.IsProcessedByPlugin ? Path.Combine(downloadDirectory, filename) : downloadDirectory + Path.DirectorySeparatorChar; return true; diff --git a/XMADownloader.Implementation/XmaPageCrawler.cs b/XMADownloader.Implementation/XmaPageCrawler.cs index 34e66aa..33a1374 100644 --- a/XMADownloader.Implementation/XmaPageCrawler.cs +++ b/XMADownloader.Implementation/XmaPageCrawler.cs @@ -296,13 +296,13 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) }; string additionalFilesSaveDirectory = Path.Combine(_xmaDownloaderSettings.DownloadDirectory, entry.UserId.ToString()); - if (/*_xmaDownloaderSettings.IsUseSubDirectories*/true && - (_xmaDownloaderSettings.SaveDescriptions || _xmaDownloaderSettings.SaveDescriptions) - ) - { - additionalFilesSaveDirectory = Path.Combine(additionalFilesSaveDirectory, - PostSubdirectoryHelper.CreateNameFromPattern(entry, _xmaDownloaderSettings.SubDirectoryPattern, _xmaDownloaderSettings.MaxSubdirectoryNameLength)); - } + //if (_xmaDownloaderSettings.IsUseSubDirectories && + // (_xmaDownloaderSettings.SaveDescriptions || _xmaDownloaderSettings.SaveDescriptions) + // ) + //{ + // additionalFilesSaveDirectory = Path.Combine(additionalFilesSaveDirectory, + // PostSubdirectoryHelper.CreateNameFromPattern(entry, _xmaDownloaderSettings.SubDirectoryPattern, _xmaDownloaderSettings.MaxSubdirectoryNameLength)); + //} if (!Directory.Exists(additionalFilesSaveDirectory)) Directory.CreateDirectory(additionalFilesSaveDirectory); From eb90db0acbf89fde07beacd31c2a319a5eb447dd Mon Sep 17 00:00:00 2001 From: Nicolaj L J Date: Sat, 28 Oct 2023 19:58:47 +0200 Subject: [PATCH 2/8] dont download everything --- .../Models/CommandLineOptions.cs | 3 ++ XMADownloader.App/Program.cs | 40 ++++++++++--------- .../Models/XmaDownloaderSettings.cs | 2 + .../Models/XmaCrawledUrl.cs | 5 +++ .../XmaPageCrawler.cs | 33 ++++++++++----- 5 files changed, 53 insertions(+), 30 deletions(-) diff --git a/XMADownloader.App/Models/CommandLineOptions.cs b/XMADownloader.App/Models/CommandLineOptions.cs index 623729a..4c269db 100644 --- a/XMADownloader.App/Models/CommandLineOptions.cs +++ b/XMADownloader.App/Models/CommandLineOptions.cs @@ -9,6 +9,9 @@ class CommandLineOptions [Option("url", Required = true, HelpText = "Url of the user page to download")] public string Url { get; set; } + [Option("nsfw", Required = false, HelpText = "Url of the user page to download",Default = false)] + public bool Nsfw { get; set; } + [Option("descriptions", Required = false, HelpText = "Save mod descriptions into a separate html files", Default = false)] public bool SaveDescriptions { get; set; } diff --git a/XMADownloader.App/Program.cs b/XMADownloader.App/Program.cs index f168394..dd65d43 100644 --- a/XMADownloader.App/Program.cs +++ b/XMADownloader.App/Program.cs @@ -30,24 +30,24 @@ static async Task Main(string[] args) NLogManager.ReconfigureNLog(); - try - { - UpdateChecker updateChecker = new UpdateChecker(); - (bool isUpdateAvailable, string updateMessage) = await updateChecker.IsNewVersionAvailable(); - if (isUpdateAvailable) - { - _logger.Warn("New version is available at https://github.com/AlexCSDev/XMADownloader/releases"); - if (updateMessage != null && !updateMessage.StartsWith("!")) - _logger.Warn($"Note from developer: {updateMessage}"); - } - - if (updateMessage != null && updateMessage.StartsWith("!")) - _logger.Warn($"Note from developer: {updateMessage.Substring(1)}"); - } - catch (Exception ex) - { - _logger.Error($"Error encountered while checking for updates: {ex}", ex); - } + //try + //{ + // UpdateChecker updateChecker = new UpdateChecker(); + // (bool isUpdateAvailable, string updateMessage) = await updateChecker.IsNewVersionAvailable(); + // if (isUpdateAvailable) + // { + // _logger.Warn("New version is available at https://github.com/AlexCSDev/XMADownloader/releases"); + // if (updateMessage != null && !updateMessage.StartsWith("!")) + // _logger.Warn($"Note from developer: {updateMessage}"); + // } + + // if (updateMessage != null && updateMessage.StartsWith("!")) + // _logger.Warn($"Note from developer: {updateMessage.Substring(1)}"); + //} + //catch (Exception ex) + //{ + // _logger.Error($"Error encountered while checking for updates: {ex}", ex); + //} AppDomain.CurrentDomain.ProcessExit += CurrentDomain_ProcessExit; Console.CancelKeyPress += ConsoleOnCancelKeyPress; @@ -163,7 +163,9 @@ private static async Task InitializeSettings(CommandLineO FallbackToContentTypeFilenames = commandLineOptions.FilenamesFallbackToContentType, ProxyServerAddress = commandLineOptions.ProxyServerAddress, RemoteBrowserAddress = commandLineOptions.RemoteBrowserAddress != null ? new Uri(commandLineOptions.RemoteBrowserAddress) : null, - ExportCrawlResults = commandLineOptions.ExportCrawlJson + ExportCrawlResults = commandLineOptions.ExportCrawlJson, + + Nsfw = commandLineOptions.Nsfw }; return settings; diff --git a/XMADownloader.Common/Models/XmaDownloaderSettings.cs b/XMADownloader.Common/Models/XmaDownloaderSettings.cs index 7743945..032fad9 100644 --- a/XMADownloader.Common/Models/XmaDownloaderSettings.cs +++ b/XMADownloader.Common/Models/XmaDownloaderSettings.cs @@ -45,6 +45,7 @@ public record XmaDownloaderSettings : UniversalDownloaderPlatformSettings, IPupp public Uri RemoteBrowserAddress { get; init; } public bool IsHeadlessBrowser { get; init; } public bool ExportCrawlResults { get; set; } + public bool Nsfw { get; set; } public XmaDownloaderSettings() { @@ -56,6 +57,7 @@ public XmaDownloaderSettings() MaxFilenameLength = 100; MaxSubdirectoryNameLength = 100; IsHeadlessBrowser = true; + Nsfw = false; } } } diff --git a/XMADownloader.Implementation/Models/XmaCrawledUrl.cs b/XMADownloader.Implementation/Models/XmaCrawledUrl.cs index f63e9cf..db379a9 100644 --- a/XMADownloader.Implementation/Models/XmaCrawledUrl.cs +++ b/XMADownloader.Implementation/Models/XmaCrawledUrl.cs @@ -53,5 +53,10 @@ public object Clone() UserId = UserId }; } + + public override string ToString() + { + return Url; + } } } diff --git a/XMADownloader.Implementation/XmaPageCrawler.cs b/XMADownloader.Implementation/XmaPageCrawler.cs index 33a1374..d6f2b44 100644 --- a/XMADownloader.Implementation/XmaPageCrawler.cs +++ b/XMADownloader.Implementation/XmaPageCrawler.cs @@ -36,6 +36,8 @@ internal sealed class XmaPageCrawler : IPageCrawler private readonly IPluginManager _pluginManager; private readonly Logger _logger = LogManager.GetCurrentClassLogger(); private readonly Random _random; + private readonly bool _downloadUrlsInDescription; + private readonly bool _downloadUrlsInFilesTab; private XmaDownloaderSettings _xmaDownloaderSettings; @@ -52,6 +54,8 @@ public XmaPageCrawler(IWebDownloader webDownloader, IPluginManager pluginManager { _webDownloader = (XmaWebDownloader)webDownloader ?? throw new ArgumentNullException(nameof(webDownloader)); _pluginManager = pluginManager ?? throw new ArgumentNullException(nameof(pluginManager)); + _downloadUrlsInDescription = false;//TODO + _downloadUrlsInFilesTab = false; _random = new Random(); } @@ -210,6 +214,10 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) if (descriptionNode == null) throw new Exception($"[{id}] Description node not found!"); + HtmlNode imageNode = doc.DocumentNode.SelectSingleNode("//img[contains(@class,\"mod-carousel-image\")]"); + if (imageNode == null) + throw new Exception("Image was not found"); + DateTime? publishDate = null; DateTime? lastUpdateDate = null; HtmlNodeCollection modDateNodes = doc.DocumentNode.SelectNodes("//div[contains(@class,\"mod-meta-block\")]"); @@ -247,7 +255,8 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) throw new Exception($"[{id}] Last update date not found!"); string currentUrl = await _webDownloader.GetActualUrl(HttpUtility.HtmlDecode(primaryUrlNode.Attributes["href"].Value)); - + string modImage = await _webDownloader.GetActualUrl(HttpUtility.HtmlDecode(imageNode.Attributes["data-src"].Value)); + _parsedUrls.Add(modImage); if (!_parsedUrls.Contains(currentUrl)) { _parsedUrls.Add(currentUrl); @@ -255,7 +264,7 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) _logger.Debug($"[{id}] New primary url: {currentUrl}"); } - if(additionalUrlNodes != null) + if(additionalUrlNodes != null && _downloadUrlsInFilesTab) { foreach (HtmlNode node in additionalUrlNodes) { @@ -274,17 +283,21 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) } //External urls via plugins (including direct via default plugin) - List pluginUrls = await _pluginManager.ExtractSupportedUrls(HttpUtility.HtmlDecode(descriptionNode.InnerHtml)); - foreach (string url in pluginUrls) + if (_downloadUrlsInDescription) { - currentUrl = await _webDownloader.GetActualUrl(url); - if (!_parsedUrls.Contains(currentUrl)) + List pluginUrls = await _pluginManager.ExtractSupportedUrls(HttpUtility.HtmlDecode(descriptionNode.InnerHtml)); + foreach (string url in pluginUrls) { - _parsedUrls.Add(currentUrl); - parsedUrlsForThisMod.Add(currentUrl); - _logger.Debug($"[{id}] New external entry: {currentUrl}"); + currentUrl = await _webDownloader.GetActualUrl(url); + if (!_parsedUrls.Contains(currentUrl)) + { + _parsedUrls.Add(currentUrl); + parsedUrlsForThisMod.Add(currentUrl); + _logger.Debug($"[{id}] New external entry: {currentUrl}"); + } } } + XmaCrawledUrl entry = new XmaCrawledUrl { @@ -345,8 +358,6 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) OnPostCrawlEnd(new PostCrawlEventArgs(id, true)); - - return (crawledUrls, crawledMods); } From 4082ecb0990590b0ebf536de9c2826a3cb82fa2b Mon Sep 17 00:00:00 2001 From: Nicolaj L J Date: Sun, 29 Oct 2023 12:39:36 +0100 Subject: [PATCH 3/8] I should rename this branch.. --- XMADownloader.App/Models/CommandLineOptions.cs | 14 +++++++++++--- .../Models/XmaDownloaderSettings.cs | 2 +- .../Models/XmaCrawledUrl.cs | 2 +- .../XmaCrawledUrlProcessor.cs | 17 +++++++++++++++-- .../XmaDefaultPlugin.cs | 6 +++++- XMADownloader.Implementation/XmaPageCrawler.cs | 8 ++++++-- 6 files changed, 39 insertions(+), 10 deletions(-) diff --git a/XMADownloader.App/Models/CommandLineOptions.cs b/XMADownloader.App/Models/CommandLineOptions.cs index 4c269db..2272418 100644 --- a/XMADownloader.App/Models/CommandLineOptions.cs +++ b/XMADownloader.App/Models/CommandLineOptions.cs @@ -9,9 +9,6 @@ class CommandLineOptions [Option("url", Required = true, HelpText = "Url of the user page to download")] public string Url { get; set; } - [Option("nsfw", Required = false, HelpText = "Url of the user page to download",Default = false)] - public bool Nsfw { get; set; } - [Option("descriptions", Required = false, HelpText = "Save mod descriptions into a separate html files", Default = false)] public bool SaveDescriptions { get; set; } @@ -64,5 +61,16 @@ class CommandLineOptions [Option("proxy-server-address", Required = false, HelpText = "The address of proxy server to use in the following format: [://][:]. Supported protocols: http(s), socks4, socks4a, socks5.")] public string ProxyServerAddress { get; set; } + + [Option("nsfw", Required = false, HelpText = "Url of the user page to download", Default = false)] + public bool Nsfw { get; set; } + + /// + /// 1 = Gear mods + /// 2 = Body replacement mods + /// 3 = + /// + [Option("types", Required = false, HelpText = "Url of the user page to download", Default = false)] + public int[] ModTypes { get; set; } } } diff --git a/XMADownloader.Common/Models/XmaDownloaderSettings.cs b/XMADownloader.Common/Models/XmaDownloaderSettings.cs index 032fad9..2219f80 100644 --- a/XMADownloader.Common/Models/XmaDownloaderSettings.cs +++ b/XMADownloader.Common/Models/XmaDownloaderSettings.cs @@ -51,7 +51,7 @@ public XmaDownloaderSettings() { SaveDescriptions = true; SaveHtml = true; - IsUseSubDirectories = false; + IsUseSubDirectories = true; SubDirectoryPattern = "[%ModId%] %PublishedAt% %PostTitle%"; FallbackToContentTypeFilenames = false; MaxFilenameLength = 100; diff --git a/XMADownloader.Implementation/Models/XmaCrawledUrl.cs b/XMADownloader.Implementation/Models/XmaCrawledUrl.cs index db379a9..db874db 100644 --- a/XMADownloader.Implementation/Models/XmaCrawledUrl.cs +++ b/XMADownloader.Implementation/Models/XmaCrawledUrl.cs @@ -56,7 +56,7 @@ public object Clone() public override string ToString() { - return Url; + return Name; } } } diff --git a/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs b/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs index 4cd72a8..8cd38c4 100644 --- a/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs +++ b/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs @@ -51,6 +51,7 @@ public async Task BeforeStart(IUniversalDownloaderPlatformSettings settings) public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) { XmaCrawledUrl crawledUrl = (XmaCrawledUrl)udpCrawledUrl; + _logger.Info("HERR "+crawledUrl.Name); string filename = ""; @@ -64,16 +65,26 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) if (crawledUrl.Filename == null) throw new DownloadException($"[{crawledUrl.ModId}] No filename for {crawledUrl.Url}!"); - filename = crawledUrl.Filename; + + string extension = Path.GetExtension(crawledUrl.Filename); + _logger.Info("HERR " + extension); + + //If the downloaded file is an image rename it to the mods name + if (extension == ".jpg" || extension == ".png" || extension == ".jpeg") + { + filename = crawledUrl.Name + extension; + } + else //Else we set it to the orginal filename + filename = crawledUrl.Filename; _logger.Debug($"Sanitizing filename: {filename}"); filename = PathSanitizer.SanitizePath(filename); _logger.Debug($"Sanitized filename: {filename}"); + if (filename.Length > _xmaDownloaderSettings.MaxFilenameLength) { _logger.Debug($"Filename is too long, will be truncated: {filename}"); - string extension = Path.GetExtension(filename); if (extension.Length > 4) { _logger.Warn($"File extension for file {filename} is longer 4 characters and won't be appended to truncated filename!"); @@ -83,6 +94,7 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) _logger.Debug($"Truncated filename: {filename}"); } + string key = $"{crawledUrl.ModId}_{filename.ToLowerInvariant()}"; _fileCountDict.AddOrUpdate(key, 0, (key, oldValue) => oldValue + 1); @@ -116,6 +128,7 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) //_logger.Debug(crawledUrl.DownloadPath); crawledUrl.DownloadPath = !crawledUrl.IsProcessedByPlugin ? Path.Combine(downloadDirectory, filename) : downloadDirectory + Path.DirectorySeparatorChar; + _logger.Info("END " + filename); return true; } } diff --git a/XMADownloader.Implementation/XmaDefaultPlugin.cs b/XMADownloader.Implementation/XmaDefaultPlugin.cs index 12863b6..446e860 100644 --- a/XMADownloader.Implementation/XmaDefaultPlugin.cs +++ b/XMADownloader.Implementation/XmaDefaultPlugin.cs @@ -167,7 +167,11 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) } crawledUrl.Filename = filename; - crawledUrl.FileSize = fileSize; + + if (_settings.IsCheckRemoteFileSize) + crawledUrl.FileSize = fileSize; + else + crawledUrl.FileSize = 0; } catch (Exception ex) { diff --git a/XMADownloader.Implementation/XmaPageCrawler.cs b/XMADownloader.Implementation/XmaPageCrawler.cs index d6f2b44..cebb8f8 100644 --- a/XMADownloader.Implementation/XmaPageCrawler.cs +++ b/XMADownloader.Implementation/XmaPageCrawler.cs @@ -255,12 +255,16 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) throw new Exception($"[{id}] Last update date not found!"); string currentUrl = await _webDownloader.GetActualUrl(HttpUtility.HtmlDecode(primaryUrlNode.Attributes["href"].Value)); - string modImage = await _webDownloader.GetActualUrl(HttpUtility.HtmlDecode(imageNode.Attributes["data-src"].Value)); - _parsedUrls.Add(modImage); + string modImage = await _webDownloader.GetActualUrl(HttpUtility.HtmlDecode(imageNode.Attributes["src"].Value)); + if (!_parsedUrls.Contains(currentUrl)) { _parsedUrls.Add(currentUrl); parsedUrlsForThisMod.Add(currentUrl); + + _parsedUrls.Add(modImage); + parsedUrlsForThisMod.Add(modImage); + _logger.Debug($"[{id}] New primary url: {currentUrl}"); } From f13f5e6c325ef68add02fbfe9154246d937cc364 Mon Sep 17 00:00:00 2001 From: Nicolaj L J Date: Sun, 29 Oct 2023 14:14:44 +0100 Subject: [PATCH 4/8] Hey it works --- .../Models/CommandLineOptions.cs | 32 +++++++++++-- XMADownloader.App/Program.cs | 6 ++- .../Models/XmaDownloaderSettings.cs | 14 +++++- .../XmaPageCrawler.cs | 48 ++++++++++++++----- 4 files changed, 80 insertions(+), 20 deletions(-) diff --git a/XMADownloader.App/Models/CommandLineOptions.cs b/XMADownloader.App/Models/CommandLineOptions.cs index 2272418..2760a7a 100644 --- a/XMADownloader.App/Models/CommandLineOptions.cs +++ b/XMADownloader.App/Models/CommandLineOptions.cs @@ -1,6 +1,7 @@ using CommandLine; using XMADownloader.App.Enums; using UniversalDownloaderPlatform.Common.Enums; +using System.Collections.Generic; namespace XMADownloader.App.Models { @@ -62,15 +63,36 @@ class CommandLineOptions [Option("proxy-server-address", Required = false, HelpText = "The address of proxy server to use in the following format: [://][:]. Supported protocols: http(s), socks4, socks4a, socks5.")] public string ProxyServerAddress { get; set; } - [Option("nsfw", Required = false, HelpText = "Url of the user page to download", Default = false)] - public bool Nsfw { get; set; } + [Option("download-urls-in-description", Required = false, HelpText = "Scrapes the description text for urls and downloads the files found in that url", Default = false)] + public bool DownloadUrlsInDescription { get; set; } + + [Option("download-urls-in-filestab", Required = false, HelpText = "Download all of the files in the filetab", Default = false)] + public bool DownloadUrlsInFilesTab { get; set; } + + [Option("download-mod-image", Required = false, HelpText = "Download the cover image for the mod", Default = true)] + public bool DownloadModImage { get; set; } + + [Option("content-type", Required = false, HelpText = "1 = Both, 2 = SFW only, 3 = NSFW only", Default = 1)] + public int ContentType { get; set; } /// /// 1 = Gear mods /// 2 = Body replacement mods - /// 3 = + /// 3 = Face mods + /// 4 = Hair mods + /// 5 = Shaders + /// 6 = Other mods + /// 7 = Minion mods + /// 8 = Mount mods + /// 10 = Skin mods + /// 11 = Concept matrix pose + /// 12 = Racial scaling mods + /// 13 = Anamnesis pose + /// 14 = VFX + /// 15 = Animation + /// 16 = Sound /// - [Option("types", Required = false, HelpText = "Url of the user page to download", Default = false)] - public int[] ModTypes { get; set; } + [Option("types", Required = false, HelpText = "Choose the modtypes you want to search for\r\nExample: --types 1 6 16\r\n1 = Gear mods\r\n2 = Body replacement mods\r\n3 = Face mods\r\n4 = Hair mods\r\n5 = Shaders\r\n6 = Other mods\r\n7 = Minion mods\r\n8 = Mount mods\r\n10 = Skin mods\r\n11 = Concept matrix pose\r\n12 = Racial scaling mods\r\n13 = Anamnesis pose\r\n14 = VFX\r\n15 = Animation\r\n16 = Sound")] + public IEnumerable ModTypes { get; set; } } } diff --git a/XMADownloader.App/Program.cs b/XMADownloader.App/Program.cs index dd65d43..307fbba 100644 --- a/XMADownloader.App/Program.cs +++ b/XMADownloader.App/Program.cs @@ -165,7 +165,11 @@ private static async Task InitializeSettings(CommandLineO RemoteBrowserAddress = commandLineOptions.RemoteBrowserAddress != null ? new Uri(commandLineOptions.RemoteBrowserAddress) : null, ExportCrawlResults = commandLineOptions.ExportCrawlJson, - Nsfw = commandLineOptions.Nsfw + ContentType = commandLineOptions.ContentType, + DownloadModImage = commandLineOptions.DownloadModImage, + DownloadUrlsInDescription = commandLineOptions.DownloadUrlsInDescription, + DownloadUrlsInFilesTab = commandLineOptions.DownloadUrlsInFilesTab, + ModTypes = commandLineOptions.ModTypes }; return settings; diff --git a/XMADownloader.Common/Models/XmaDownloaderSettings.cs b/XMADownloader.Common/Models/XmaDownloaderSettings.cs index 2219f80..cf31f5a 100644 --- a/XMADownloader.Common/Models/XmaDownloaderSettings.cs +++ b/XMADownloader.Common/Models/XmaDownloaderSettings.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Net.Mime; using System.Text; using UniversalDownloaderPlatform.Common.Enums; using UniversalDownloaderPlatform.Common.Helpers; @@ -45,7 +46,11 @@ public record XmaDownloaderSettings : UniversalDownloaderPlatformSettings, IPupp public Uri RemoteBrowserAddress { get; init; } public bool IsHeadlessBrowser { get; init; } public bool ExportCrawlResults { get; set; } - public bool Nsfw { get; set; } + public bool DownloadUrlsInDescription { get; set; } + public bool DownloadUrlsInFilesTab { get; set; } + public bool DownloadModImage { get; set; } + public int ContentType { get; set; } + public IEnumerable ModTypes { get; set; } public XmaDownloaderSettings() { @@ -57,7 +62,12 @@ public XmaDownloaderSettings() MaxFilenameLength = 100; MaxSubdirectoryNameLength = 100; IsHeadlessBrowser = true; - Nsfw = false; + DownloadUrlsInDescription = false; + DownloadUrlsInFilesTab = false; + DownloadModImage = true; + ContentType = 1; + ModTypes = new int[0]; + } } } diff --git a/XMADownloader.Implementation/XmaPageCrawler.cs b/XMADownloader.Implementation/XmaPageCrawler.cs index cebb8f8..8fc48cf 100644 --- a/XMADownloader.Implementation/XmaPageCrawler.cs +++ b/XMADownloader.Implementation/XmaPageCrawler.cs @@ -24,20 +24,20 @@ using XMADownloader.Implementation.Models.Export; using UniversalDownloaderPlatform.DefaultImplementations.Interfaces; using XMADownloader.Common.Models; +using Castle.Core.Internal; namespace XMADownloader.Implementation { internal sealed class XmaPageCrawler : IPageCrawler { - private const string CrawlStartUrl = "https://xivmodarchive.com/search?sortby=time_posted&sortorder=desc&types=1%2C3%2C7%2C9%2C12%2C15%2C2%2C4%2C8%2C10%2C14%2C11%2C5%2C13%2C6"; + //private const string CrawlStartUrl = "https://xivmodarchive.com/search?sortby=time_posted&sortorder=desc&types=1%2C3%2C7%2C9%2C12%2C15%2C2%2C4%2C8%2C10%2C14%2C11%2C5%2C13%2C6"; + private const string CrawlStartUrl = "https://xivmodarchive.com/search?sortby=time_posted&sortorder=desc&types="; private static Regex _modPageUrlMatchRegex = new Regex("https:\\/\\/(?>www\\.)?xivmodarchive\\.com\\/(modid|private)\\/([a-z\\-0-9]+)(\\/.+)?"); private readonly XmaWebDownloader _webDownloader; private readonly IPluginManager _pluginManager; private readonly Logger _logger = LogManager.GetCurrentClassLogger(); private readonly Random _random; - private readonly bool _downloadUrlsInDescription; - private readonly bool _downloadUrlsInFilesTab; private XmaDownloaderSettings _xmaDownloaderSettings; @@ -54,9 +54,7 @@ public XmaPageCrawler(IWebDownloader webDownloader, IPluginManager pluginManager { _webDownloader = (XmaWebDownloader)webDownloader ?? throw new ArgumentNullException(nameof(webDownloader)); _pluginManager = pluginManager ?? throw new ArgumentNullException(nameof(pluginManager)); - _downloadUrlsInDescription = false;//TODO - _downloadUrlsInFilesTab = false; - + _random = new Random(); } @@ -81,7 +79,28 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) List crawledUrls = new List(); Random rnd = new Random(Guid.NewGuid().GetHashCode()); - string basePageUrl = CrawlStartUrl + $"&author=id-{xmaCrawlTargetInfo.Id}&page="; + string basePageUrl = CrawlStartUrl; + if (_xmaDownloaderSettings.ModTypes.IsNullOrEmpty()) + basePageUrl += "1%2C3%2C7%2C9%2C12%2C15%2C2%2C4%2C8%2C10%2C14%2C11%2C5%2C13%2C6"; + else + { + int[] modtypes = (int[])_xmaDownloaderSettings.ModTypes; + basePageUrl += modtypes.First(); + + if (modtypes.Length > 1) + foreach(var modtype in modtypes) + { + basePageUrl += "%2C" + modtype; + } + } + + if (_xmaDownloaderSettings.ContentType == 2) + basePageUrl += "&nsfw=false"; + else if (_xmaDownloaderSettings.ContentType == 3) + basePageUrl += "&nsfw=true"; + + basePageUrl += $"&author=id-{xmaCrawlTargetInfo.Id}&page="; + _logger.Info("HTML: " + basePageUrl); int page = 0; while (true) @@ -117,6 +136,7 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) private async Task<(List, List)> ParseSearchPage(string html) { + List crawledUrls = new List(); List crawledMods = new List(); @@ -255,20 +275,24 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) throw new Exception($"[{id}] Last update date not found!"); string currentUrl = await _webDownloader.GetActualUrl(HttpUtility.HtmlDecode(primaryUrlNode.Attributes["href"].Value)); - string modImage = await _webDownloader.GetActualUrl(HttpUtility.HtmlDecode(imageNode.Attributes["src"].Value)); + if (!_parsedUrls.Contains(currentUrl)) { _parsedUrls.Add(currentUrl); parsedUrlsForThisMod.Add(currentUrl); + _logger.Debug($"[{id}] New primary url: {currentUrl}"); + } + + if (_xmaDownloaderSettings.DownloadModImage) + { + string modImage = await _webDownloader.GetActualUrl(HttpUtility.HtmlDecode(imageNode.Attributes["src"].Value)); _parsedUrls.Add(modImage); parsedUrlsForThisMod.Add(modImage); - - _logger.Debug($"[{id}] New primary url: {currentUrl}"); } - if(additionalUrlNodes != null && _downloadUrlsInFilesTab) + if(additionalUrlNodes != null && _xmaDownloaderSettings.DownloadUrlsInFilesTab) { foreach (HtmlNode node in additionalUrlNodes) { @@ -287,7 +311,7 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) } //External urls via plugins (including direct via default plugin) - if (_downloadUrlsInDescription) + if (_xmaDownloaderSettings.DownloadUrlsInDescription) { List pluginUrls = await _pluginManager.ExtractSupportedUrls(HttpUtility.HtmlDecode(descriptionNode.InnerHtml)); foreach (string url in pluginUrls) From e2f409a2e72085dc1f4a4d35d2e837889488676a Mon Sep 17 00:00:00 2001 From: Nicolaj L J Date: Tue, 31 Oct 2023 13:13:25 +0100 Subject: [PATCH 5/8] Update CommandLineOptions.cs --- XMADownloader.App/Models/CommandLineOptions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/XMADownloader.App/Models/CommandLineOptions.cs b/XMADownloader.App/Models/CommandLineOptions.cs index 2760a7a..e96a413 100644 --- a/XMADownloader.App/Models/CommandLineOptions.cs +++ b/XMADownloader.App/Models/CommandLineOptions.cs @@ -45,7 +45,7 @@ class CommandLineOptions [Option("remote-browser-address", Required = false, HelpText = "Advanced users only. Address of the browser with remote debugging enabled. Refer to documentation for more details.")] public string RemoteBrowserAddress { get; set; } - [Option("use-sub-directories", Required = false, HelpText = "Create a new directory inside of the download directory for every post instead of placing all files into a single directory.")] + [Option("use-sub-directories", Required = false, HelpText = "Create a new directory inside of the download directory for every post instead of placing all files into a single directory.", Default = true)] public bool UseSubDirectories { get; set; } [Option("sub-directory-pattern", Required = false, HelpText = "Pattern which will be used to create a name for the sub directories if --use-sub-directories is used. Supported parameters: %ModId%, %PublishedAt%, %PostTitle%.", Default = "[%ModId%] %PublishedAt% %PostTitle%")] From 8ec397e8b3c07497d73351033f6fc83a45d15a7d Mon Sep 17 00:00:00 2001 From: Nicolaj L J Date: Wed, 1 Nov 2023 13:33:48 +0100 Subject: [PATCH 6/8] Clean up --- XMADownloader.Implementation/XmaCrawledUrlProcessor.cs | 3 --- XMADownloader.Implementation/XmaPageCrawler.cs | 1 - 2 files changed, 4 deletions(-) diff --git a/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs b/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs index 8cd38c4..69f21c5 100644 --- a/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs +++ b/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs @@ -51,7 +51,6 @@ public async Task BeforeStart(IUniversalDownloaderPlatformSettings settings) public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) { XmaCrawledUrl crawledUrl = (XmaCrawledUrl)udpCrawledUrl; - _logger.Info("HERR "+crawledUrl.Name); string filename = ""; @@ -67,7 +66,6 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) string extension = Path.GetExtension(crawledUrl.Filename); - _logger.Info("HERR " + extension); //If the downloaded file is an image rename it to the mods name if (extension == ".jpg" || extension == ".png" || extension == ".jpeg") @@ -128,7 +126,6 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) //_logger.Debug(crawledUrl.DownloadPath); crawledUrl.DownloadPath = !crawledUrl.IsProcessedByPlugin ? Path.Combine(downloadDirectory, filename) : downloadDirectory + Path.DirectorySeparatorChar; - _logger.Info("END " + filename); return true; } } diff --git a/XMADownloader.Implementation/XmaPageCrawler.cs b/XMADownloader.Implementation/XmaPageCrawler.cs index 8fc48cf..f559148 100644 --- a/XMADownloader.Implementation/XmaPageCrawler.cs +++ b/XMADownloader.Implementation/XmaPageCrawler.cs @@ -100,7 +100,6 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) basePageUrl += "&nsfw=true"; basePageUrl += $"&author=id-{xmaCrawlTargetInfo.Id}&page="; - _logger.Info("HTML: " + basePageUrl); int page = 0; while (true) From cf789045c97c0432f5c713fa8d63162119697efb Mon Sep 17 00:00:00 2001 From: Nicolaj L J Date: Sun, 12 Nov 2023 12:41:56 +0100 Subject: [PATCH 7/8] Can search for text --- .../Models/CommandLineOptions.cs | 3 ++ XMADownloader.App/Program.cs | 40 +++++++++---------- .../Models/XmaDownloaderSettings.cs | 2 + .../XmaPageCrawler.cs | 8 ++++ 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/XMADownloader.App/Models/CommandLineOptions.cs b/XMADownloader.App/Models/CommandLineOptions.cs index e96a413..dfdbb8a 100644 --- a/XMADownloader.App/Models/CommandLineOptions.cs +++ b/XMADownloader.App/Models/CommandLineOptions.cs @@ -75,6 +75,9 @@ class CommandLineOptions [Option("content-type", Required = false, HelpText = "1 = Both, 2 = SFW only, 3 = NSFW only", Default = 1)] public int ContentType { get; set; } + [Option("search-text", Required = false, HelpText = "Search for posts that include the text somewhere (title, description, tags etc)", Default = "")] + public string SearchText { get; set; } + /// /// 1 = Gear mods /// 2 = Body replacement mods diff --git a/XMADownloader.App/Program.cs b/XMADownloader.App/Program.cs index 307fbba..4eb653a 100644 --- a/XMADownloader.App/Program.cs +++ b/XMADownloader.App/Program.cs @@ -30,24 +30,24 @@ static async Task Main(string[] args) NLogManager.ReconfigureNLog(); - //try - //{ - // UpdateChecker updateChecker = new UpdateChecker(); - // (bool isUpdateAvailable, string updateMessage) = await updateChecker.IsNewVersionAvailable(); - // if (isUpdateAvailable) - // { - // _logger.Warn("New version is available at https://github.com/AlexCSDev/XMADownloader/releases"); - // if (updateMessage != null && !updateMessage.StartsWith("!")) - // _logger.Warn($"Note from developer: {updateMessage}"); - // } - - // if (updateMessage != null && updateMessage.StartsWith("!")) - // _logger.Warn($"Note from developer: {updateMessage.Substring(1)}"); - //} - //catch (Exception ex) - //{ - // _logger.Error($"Error encountered while checking for updates: {ex}", ex); - //} + try + { + UpdateChecker updateChecker = new UpdateChecker(); + (bool isUpdateAvailable, string updateMessage) = await updateChecker.IsNewVersionAvailable(); + if (isUpdateAvailable) + { + _logger.Warn("New version is available at https://github.com/AlexCSDev/XMADownloader/releases"); + if (updateMessage != null && !updateMessage.StartsWith("!")) + _logger.Warn($"Note from developer: {updateMessage}"); + } + + if (updateMessage != null && updateMessage.StartsWith("!")) + _logger.Warn($"Note from developer: {updateMessage.Substring(1)}"); + } + catch (Exception ex) + { + _logger.Error($"Error encountered while checking for updates: {ex}", ex); + } AppDomain.CurrentDomain.ProcessExit += CurrentDomain_ProcessExit; Console.CancelKeyPress += ConsoleOnCancelKeyPress; @@ -164,11 +164,11 @@ private static async Task InitializeSettings(CommandLineO ProxyServerAddress = commandLineOptions.ProxyServerAddress, RemoteBrowserAddress = commandLineOptions.RemoteBrowserAddress != null ? new Uri(commandLineOptions.RemoteBrowserAddress) : null, ExportCrawlResults = commandLineOptions.ExportCrawlJson, - - ContentType = commandLineOptions.ContentType, DownloadModImage = commandLineOptions.DownloadModImage, DownloadUrlsInDescription = commandLineOptions.DownloadUrlsInDescription, DownloadUrlsInFilesTab = commandLineOptions.DownloadUrlsInFilesTab, + SearchText = commandLineOptions.SearchText, + ContentType = commandLineOptions.ContentType, ModTypes = commandLineOptions.ModTypes }; diff --git a/XMADownloader.Common/Models/XmaDownloaderSettings.cs b/XMADownloader.Common/Models/XmaDownloaderSettings.cs index cf31f5a..bd0b8f9 100644 --- a/XMADownloader.Common/Models/XmaDownloaderSettings.cs +++ b/XMADownloader.Common/Models/XmaDownloaderSettings.cs @@ -50,6 +50,7 @@ public record XmaDownloaderSettings : UniversalDownloaderPlatformSettings, IPupp public bool DownloadUrlsInFilesTab { get; set; } public bool DownloadModImage { get; set; } public int ContentType { get; set; } + public string SearchText { get; set; } public IEnumerable ModTypes { get; set; } public XmaDownloaderSettings() @@ -66,6 +67,7 @@ public XmaDownloaderSettings() DownloadUrlsInFilesTab = false; DownloadModImage = true; ContentType = 1; + SearchText = ""; ModTypes = new int[0]; } diff --git a/XMADownloader.Implementation/XmaPageCrawler.cs b/XMADownloader.Implementation/XmaPageCrawler.cs index f559148..a9f16ae 100644 --- a/XMADownloader.Implementation/XmaPageCrawler.cs +++ b/XMADownloader.Implementation/XmaPageCrawler.cs @@ -94,6 +94,14 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) } } + if (!_xmaDownloaderSettings.SearchText.IsNullOrEmpty()) + { + string replaceSpace = _xmaDownloaderSettings.SearchText.Replace(" ", "%20"); + basePageUrl += "&basic_text=" + replaceSpace; + } + + + if (_xmaDownloaderSettings.ContentType == 2) basePageUrl += "&nsfw=false"; else if (_xmaDownloaderSettings.ContentType == 3) From 6bd2c2c0f04ea9c1e724042dd549207d27163200 Mon Sep 17 00:00:00 2001 From: Hapse Date: Fri, 29 Dec 2023 13:18:31 +0100 Subject: [PATCH 8/8] Added pages and sort-by --- .../Models/CommandLineOptions.cs | 10 +++++-- XMADownloader.App/Program.cs | 4 ++- .../net3.1-linux-x64-release.pubxml | 12 ++++---- .../Models/XmaDownloaderSettings.cs | 4 +++ .../XmaPageCrawler.cs | 28 +++++++++++++++++-- 5 files changed, 45 insertions(+), 13 deletions(-) diff --git a/XMADownloader.App/Models/CommandLineOptions.cs b/XMADownloader.App/Models/CommandLineOptions.cs index dfdbb8a..bfe2851 100644 --- a/XMADownloader.App/Models/CommandLineOptions.cs +++ b/XMADownloader.App/Models/CommandLineOptions.cs @@ -75,7 +75,7 @@ class CommandLineOptions [Option("content-type", Required = false, HelpText = "1 = Both, 2 = SFW only, 3 = NSFW only", Default = 1)] public int ContentType { get; set; } - [Option("search-text", Required = false, HelpText = "Search for posts that include the text somewhere (title, description, tags etc)", Default = "")] + [Option("search-text", Required = false, HelpText = "Search for posts that include the text somewhere (title, description, tags etc)")] public string SearchText { get; set; } /// @@ -95,7 +95,13 @@ class CommandLineOptions /// 15 = Animation /// 16 = Sound /// - [Option("types", Required = false, HelpText = "Choose the modtypes you want to search for\r\nExample: --types 1 6 16\r\n1 = Gear mods\r\n2 = Body replacement mods\r\n3 = Face mods\r\n4 = Hair mods\r\n5 = Shaders\r\n6 = Other mods\r\n7 = Minion mods\r\n8 = Mount mods\r\n10 = Skin mods\r\n11 = Concept matrix pose\r\n12 = Racial scaling mods\r\n13 = Anamnesis pose\r\n14 = VFX\r\n15 = Animation\r\n16 = Sound")] + [Option("types", Required = false, HelpText = "Choose the modtypes you want to search for\r\nExample: --types 11 13 for poses\r\n1 = Gear mods\r\n2 = Body replacement mods\r\n3 = Face mods\r\n4 = Hair mods\r\n5 = Shaders\r\n6 = Other mods\r\n7 = Minion mods\r\n8 = Mount mods\r\n10 = Skin mods\r\n11 = Concept matrix pose\r\n12 = Racial scaling mods\r\n13 = Anamnesis pose\r\n14 = VFX\r\n15 = Animation\r\n16 = Sound")] public IEnumerable ModTypes { get; set; } + + [Option("pages", Required = false, HelpText = "If you only want some certain pages. Example: --pages 1 4 5 will give you page 1, 4 and 5")] + public IEnumerable Pages { get; set; } + + [Option("sort-by", Required = false, HelpText = "Use the sort by feature on the website, always uses descending\r\nrank = Relevance\r\ntime_edited = Last Version Update\r\ntime_posted = Release Date\r\nname = Name\r\nview = Views\r\nviews_today = Views Today\r\ndownloads = Downloads\r\nfollowers = Followers", Default = "time_posted")] + public string SortBy { get; set; } } } diff --git a/XMADownloader.App/Program.cs b/XMADownloader.App/Program.cs index 4eb653a..639a8c4 100644 --- a/XMADownloader.App/Program.cs +++ b/XMADownloader.App/Program.cs @@ -169,7 +169,9 @@ private static async Task InitializeSettings(CommandLineO DownloadUrlsInFilesTab = commandLineOptions.DownloadUrlsInFilesTab, SearchText = commandLineOptions.SearchText, ContentType = commandLineOptions.ContentType, - ModTypes = commandLineOptions.ModTypes + ModTypes = commandLineOptions.ModTypes, + Pages = commandLineOptions.Pages, + SortBy = commandLineOptions.SortBy }; return settings; diff --git a/XMADownloader.App/Properties/PublishProfiles/net3.1-linux-x64-release.pubxml b/XMADownloader.App/Properties/PublishProfiles/net3.1-linux-x64-release.pubxml index 2d8602c..5e21dcc 100644 --- a/XMADownloader.App/Properties/PublishProfiles/net3.1-linux-x64-release.pubxml +++ b/XMADownloader.App/Properties/PublishProfiles/net3.1-linux-x64-release.pubxml @@ -7,13 +7,11 @@ https://go.microsoft.com/fwlink/?LinkID=208121. FileSystem Release Any CPU - netcoreapp3.1 - bin\publish\net3.1-linux-x64-release - linux-x64 - true + net6.0 + C:\Users\hapse\Documents\GitHub\XMADownloader\XMADownloader.App\bin\Release\net6.0 + win-x64 + false <_IsPortable>false - False - False - False + false \ No newline at end of file diff --git a/XMADownloader.Common/Models/XmaDownloaderSettings.cs b/XMADownloader.Common/Models/XmaDownloaderSettings.cs index bd0b8f9..f10cce5 100644 --- a/XMADownloader.Common/Models/XmaDownloaderSettings.cs +++ b/XMADownloader.Common/Models/XmaDownloaderSettings.cs @@ -52,6 +52,8 @@ public record XmaDownloaderSettings : UniversalDownloaderPlatformSettings, IPupp public int ContentType { get; set; } public string SearchText { get; set; } public IEnumerable ModTypes { get; set; } + public IEnumerable Pages { get; set; } + public string SortBy { get; set; } public XmaDownloaderSettings() { @@ -69,6 +71,8 @@ public XmaDownloaderSettings() ContentType = 1; SearchText = ""; ModTypes = new int[0]; + Pages = new int[0]; + SortBy = "time_posted"; } } diff --git a/XMADownloader.Implementation/XmaPageCrawler.cs b/XMADownloader.Implementation/XmaPageCrawler.cs index a9f16ae..9d3cabb 100644 --- a/XMADownloader.Implementation/XmaPageCrawler.cs +++ b/XMADownloader.Implementation/XmaPageCrawler.cs @@ -31,7 +31,7 @@ namespace XMADownloader.Implementation internal sealed class XmaPageCrawler : IPageCrawler { //private const string CrawlStartUrl = "https://xivmodarchive.com/search?sortby=time_posted&sortorder=desc&types=1%2C3%2C7%2C9%2C12%2C15%2C2%2C4%2C8%2C10%2C14%2C11%2C5%2C13%2C6"; - private const string CrawlStartUrl = "https://xivmodarchive.com/search?sortby=time_posted&sortorder=desc&types="; + private const string CrawlStartUrl = "https://xivmodarchive.com/search?"; private static Regex _modPageUrlMatchRegex = new Regex("https:\\/\\/(?>www\\.)?xivmodarchive\\.com\\/(modid|private)\\/([a-z\\-0-9]+)(\\/.+)?"); private readonly XmaWebDownloader _webDownloader; @@ -79,7 +79,15 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) List crawledUrls = new List(); Random rnd = new Random(Guid.NewGuid().GetHashCode()); + string basePageUrl = CrawlStartUrl; + + //add sort by order + basePageUrl += "sortby=" + _xmaDownloaderSettings.SortBy + "&sortorder=desc"; + + + //add types + basePageUrl += "&types="; if (_xmaDownloaderSettings.ModTypes.IsNullOrEmpty()) basePageUrl += "1%2C3%2C7%2C9%2C12%2C15%2C2%2C4%2C8%2C10%2C14%2C11%2C5%2C13%2C6"; else @@ -109,10 +117,25 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) basePageUrl += $"&author=id-{xmaCrawlTargetInfo.Id}&page="; + List pages = _xmaDownloaderSettings.Pages.ToList(); int page = 0; while (true) { - page++; + //if only certain pages are requested + if(!_xmaDownloaderSettings.Pages.IsNullOrEmpty()) + { + if (pages.IsNullOrEmpty())//once its done all pages, break + break; + + page = pages.First(); + pages.Remove(page); + + } + else + { + page++; + } + _logger.Debug($"Page #{page}"); string searchPageHtml = await _webDownloader.DownloadString(basePageUrl + page); @@ -143,7 +166,6 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) private async Task<(List, List)> ParseSearchPage(string html) { - List crawledUrls = new List(); List crawledMods = new List();