diff --git a/XMADownloader.App/Models/CommandLineOptions.cs b/XMADownloader.App/Models/CommandLineOptions.cs index 00d0554..bfe2851 100644 --- a/XMADownloader.App/Models/CommandLineOptions.cs +++ b/XMADownloader.App/Models/CommandLineOptions.cs @@ -1,6 +1,7 @@ using CommandLine; using XMADownloader.App.Enums; using UniversalDownloaderPlatform.Common.Enums; +using System.Collections.Generic; namespace XMADownloader.App.Models { @@ -44,8 +45,8 @@ class CommandLineOptions [Option("remote-browser-address", Required = false, HelpText = "Advanced users only. Address of the browser with remote debugging enabled. Refer to documentation for more details.")] public string RemoteBrowserAddress { get; set; } - /*[Option("use-sub-directories", Required = false, HelpText = "Create a new directory inside of the download directory for every post instead of placing all files into a single directory.")] - public bool UseSubDirectories { get; set; }*/ + [Option("use-sub-directories", Required = false, HelpText = "Create a new directory inside of the download directory for every post instead of placing all files into a single directory.", Default = true)] + public bool UseSubDirectories { get; set; } [Option("sub-directory-pattern", Required = false, HelpText = "Pattern which will be used to create a name for the sub directories if --use-sub-directories is used. Supported parameters: %ModId%, %PublishedAt%, %PostTitle%.", Default = "[%ModId%] %PublishedAt% %PostTitle%")] public string SubDirectoryPattern { get; set; } @@ -61,5 +62,46 @@ class CommandLineOptions [Option("proxy-server-address", Required = false, HelpText = "The address of proxy server to use in the following format: [://][:]. Supported protocols: http(s), socks4, socks4a, socks5.")] public string ProxyServerAddress { get; set; } + + [Option("download-urls-in-description", Required = false, HelpText = "Scrapes the description text for urls and downloads the files found in that url", Default = false)] + public bool DownloadUrlsInDescription { get; set; } + + [Option("download-urls-in-filestab", Required = false, HelpText = "Download all of the files in the filetab", Default = false)] + public bool DownloadUrlsInFilesTab { get; set; } + + [Option("download-mod-image", Required = false, HelpText = "Download the cover image for the mod", Default = true)] + public bool DownloadModImage { get; set; } + + [Option("content-type", Required = false, HelpText = "1 = Both, 2 = SFW only, 3 = NSFW only", Default = 1)] + public int ContentType { get; set; } + + [Option("search-text", Required = false, HelpText = "Search for posts that include the text somewhere (title, description, tags etc)")] + public string SearchText { get; set; } + + /// + /// 1 = Gear mods + /// 2 = Body replacement mods + /// 3 = Face mods + /// 4 = Hair mods + /// 5 = Shaders + /// 6 = Other mods + /// 7 = Minion mods + /// 8 = Mount mods + /// 10 = Skin mods + /// 11 = Concept matrix pose + /// 12 = Racial scaling mods + /// 13 = Anamnesis pose + /// 14 = VFX + /// 15 = Animation + /// 16 = Sound + /// + [Option("types", Required = false, HelpText = "Choose the modtypes you want to search for\r\nExample: --types 11 13 for poses\r\n1 = Gear mods\r\n2 = Body replacement mods\r\n3 = Face mods\r\n4 = Hair mods\r\n5 = Shaders\r\n6 = Other mods\r\n7 = Minion mods\r\n8 = Mount mods\r\n10 = Skin mods\r\n11 = Concept matrix pose\r\n12 = Racial scaling mods\r\n13 = Anamnesis pose\r\n14 = VFX\r\n15 = Animation\r\n16 = Sound")] + public IEnumerable ModTypes { get; set; } + + [Option("pages", Required = false, HelpText = "If you only want some certain pages. Example: --pages 1 4 5 will give you page 1, 4 and 5")] + public IEnumerable Pages { get; set; } + + [Option("sort-by", Required = false, HelpText = "Use the sort by feature on the website, always uses descending\r\nrank = Relevance\r\ntime_edited = Last Version Update\r\ntime_posted = Release Date\r\nname = Name\r\nview = Views\r\nviews_today = Views Today\r\ndownloads = Downloads\r\nfollowers = Followers", Default = "time_posted")] + public string SortBy { get; set; } } } diff --git a/XMADownloader.App/Program.cs b/XMADownloader.App/Program.cs index 8a59a5c..639a8c4 100644 --- a/XMADownloader.App/Program.cs +++ b/XMADownloader.App/Program.cs @@ -156,14 +156,22 @@ private static async Task InitializeSettings(CommandLineO DownloadDirectory = commandLineOptions.DownloadDirectory, FileExistsAction = commandLineOptions.FileExistsAction, IsCheckRemoteFileSize = !commandLineOptions.IsDisableRemoteFileSizeCheck, - //IsUseSubDirectories = commandLineOptions.UseSubDirectories, + IsUseSubDirectories = commandLineOptions.UseSubDirectories, SubDirectoryPattern = commandLineOptions.SubDirectoryPattern, MaxSubdirectoryNameLength = commandLineOptions.MaxSubdirectoryNameLength, MaxFilenameLength = commandLineOptions.MaxFilenameLength, FallbackToContentTypeFilenames = commandLineOptions.FilenamesFallbackToContentType, ProxyServerAddress = commandLineOptions.ProxyServerAddress, RemoteBrowserAddress = commandLineOptions.RemoteBrowserAddress != null ? new Uri(commandLineOptions.RemoteBrowserAddress) : null, - ExportCrawlResults = commandLineOptions.ExportCrawlJson + ExportCrawlResults = commandLineOptions.ExportCrawlJson, + DownloadModImage = commandLineOptions.DownloadModImage, + DownloadUrlsInDescription = commandLineOptions.DownloadUrlsInDescription, + DownloadUrlsInFilesTab = commandLineOptions.DownloadUrlsInFilesTab, + SearchText = commandLineOptions.SearchText, + ContentType = commandLineOptions.ContentType, + ModTypes = commandLineOptions.ModTypes, + Pages = commandLineOptions.Pages, + SortBy = commandLineOptions.SortBy }; return settings; diff --git a/XMADownloader.App/Properties/PublishProfiles/net3.1-linux-x64-release.pubxml b/XMADownloader.App/Properties/PublishProfiles/net3.1-linux-x64-release.pubxml index 2d8602c..5e21dcc 100644 --- a/XMADownloader.App/Properties/PublishProfiles/net3.1-linux-x64-release.pubxml +++ b/XMADownloader.App/Properties/PublishProfiles/net3.1-linux-x64-release.pubxml @@ -7,13 +7,11 @@ https://go.microsoft.com/fwlink/?LinkID=208121. FileSystem Release Any CPU - netcoreapp3.1 - bin\publish\net3.1-linux-x64-release - linux-x64 - true + net6.0 + C:\Users\hapse\Documents\GitHub\XMADownloader\XMADownloader.App\bin\Release\net6.0 + win-x64 + false <_IsPortable>false - False - False - False + false \ No newline at end of file diff --git a/XMADownloader.Common/Models/XmaDownloaderSettings.cs b/XMADownloader.Common/Models/XmaDownloaderSettings.cs index b09b0c0..f10cce5 100644 --- a/XMADownloader.Common/Models/XmaDownloaderSettings.cs +++ b/XMADownloader.Common/Models/XmaDownloaderSettings.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Net.Mime; using System.Text; using UniversalDownloaderPlatform.Common.Enums; using UniversalDownloaderPlatform.Common.Helpers; @@ -18,7 +19,7 @@ public record XmaDownloaderSettings : UniversalDownloaderPlatformSettings, IPupp /// /// Create a new directory for every post and store files of said post in that directory /// - //public bool IsUseSubDirectories { get; init; } + public bool IsUseSubDirectories { get; init; } /// /// Pattern used to generate directory name if UseSubDirectories is enabled @@ -45,17 +46,34 @@ public record XmaDownloaderSettings : UniversalDownloaderPlatformSettings, IPupp public Uri RemoteBrowserAddress { get; init; } public bool IsHeadlessBrowser { get; init; } public bool ExportCrawlResults { get; set; } + public bool DownloadUrlsInDescription { get; set; } + public bool DownloadUrlsInFilesTab { get; set; } + public bool DownloadModImage { get; set; } + public int ContentType { get; set; } + public string SearchText { get; set; } + public IEnumerable ModTypes { get; set; } + public IEnumerable Pages { get; set; } + public string SortBy { get; set; } public XmaDownloaderSettings() { SaveDescriptions = true; SaveHtml = true; - //IsUseSubDirectories = false; + IsUseSubDirectories = true; SubDirectoryPattern = "[%ModId%] %PublishedAt% %PostTitle%"; FallbackToContentTypeFilenames = false; MaxFilenameLength = 100; MaxSubdirectoryNameLength = 100; IsHeadlessBrowser = true; + DownloadUrlsInDescription = false; + DownloadUrlsInFilesTab = false; + DownloadModImage = true; + ContentType = 1; + SearchText = ""; + ModTypes = new int[0]; + Pages = new int[0]; + SortBy = "time_posted"; + } } } diff --git a/XMADownloader.Implementation/Models/XmaCrawledUrl.cs b/XMADownloader.Implementation/Models/XmaCrawledUrl.cs index f63e9cf..db874db 100644 --- a/XMADownloader.Implementation/Models/XmaCrawledUrl.cs +++ b/XMADownloader.Implementation/Models/XmaCrawledUrl.cs @@ -53,5 +53,10 @@ public object Clone() UserId = UserId }; } + + public override string ToString() + { + return Name; + } } } diff --git a/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs b/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs index 1f63bed..69f21c5 100644 --- a/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs +++ b/XMADownloader.Implementation/XmaCrawledUrlProcessor.cs @@ -56,24 +56,33 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) if (!crawledUrl.IsProcessedByPlugin) { - /*if (!_XMADownloaderSettings.IsUseSubDirectories) + if (!_xmaDownloaderSettings.IsUseSubDirectories) filename = $"{crawledUrl.ModId}_"; else - filename = "";*/ + filename = ""; if (crawledUrl.Filename == null) throw new DownloadException($"[{crawledUrl.ModId}] No filename for {crawledUrl.Url}!"); - filename = crawledUrl.Filename; + + string extension = Path.GetExtension(crawledUrl.Filename); + + //If the downloaded file is an image rename it to the mods name + if (extension == ".jpg" || extension == ".png" || extension == ".jpeg") + { + filename = crawledUrl.Name + extension; + } + else //Else we set it to the orginal filename + filename = crawledUrl.Filename; _logger.Debug($"Sanitizing filename: {filename}"); filename = PathSanitizer.SanitizePath(filename); _logger.Debug($"Sanitized filename: {filename}"); + if (filename.Length > _xmaDownloaderSettings.MaxFilenameLength) { _logger.Debug($"Filename is too long, will be truncated: {filename}"); - string extension = Path.GetExtension(filename); if (extension.Length > 4) { _logger.Warn($"File extension for file {filename} is longer 4 characters and won't be appended to truncated filename!"); @@ -83,6 +92,7 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) _logger.Debug($"Truncated filename: {filename}"); } + string key = $"{crawledUrl.ModId}_{filename.ToLowerInvariant()}"; _fileCountDict.AddOrUpdate(key, 0, (key, oldValue) => oldValue + 1); @@ -111,9 +121,9 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) string downloadDirectory = crawledUrl.UserId.ToString(); - if (/*_XMADownloaderSettings.IsUseSubDirectories*/true) - downloadDirectory = Path.Combine(downloadDirectory, PostSubdirectoryHelper.CreateNameFromPattern(crawledUrl, _xmaDownloaderSettings.SubDirectoryPattern, _xmaDownloaderSettings.MaxSubdirectoryNameLength)); - + //if (_xmaDownloaderSettings.IsUseSubDirectories) + // downloadDirectory = Path.Combine(downloadDirectory, PostSubdirectoryHelper.CreateNameFromPattern(crawledUrl, _xmaDownloaderSettings.SubDirectoryPattern, _xmaDownloaderSettings.MaxSubdirectoryNameLength)); + //_logger.Debug(crawledUrl.DownloadPath); crawledUrl.DownloadPath = !crawledUrl.IsProcessedByPlugin ? Path.Combine(downloadDirectory, filename) : downloadDirectory + Path.DirectorySeparatorChar; return true; diff --git a/XMADownloader.Implementation/XmaDefaultPlugin.cs b/XMADownloader.Implementation/XmaDefaultPlugin.cs index 12863b6..446e860 100644 --- a/XMADownloader.Implementation/XmaDefaultPlugin.cs +++ b/XMADownloader.Implementation/XmaDefaultPlugin.cs @@ -167,7 +167,11 @@ public async Task ProcessCrawledUrl(ICrawledUrl udpCrawledUrl) } crawledUrl.Filename = filename; - crawledUrl.FileSize = fileSize; + + if (_settings.IsCheckRemoteFileSize) + crawledUrl.FileSize = fileSize; + else + crawledUrl.FileSize = 0; } catch (Exception ex) { diff --git a/XMADownloader.Implementation/XmaPageCrawler.cs b/XMADownloader.Implementation/XmaPageCrawler.cs index 34e66aa..9d3cabb 100644 --- a/XMADownloader.Implementation/XmaPageCrawler.cs +++ b/XMADownloader.Implementation/XmaPageCrawler.cs @@ -24,12 +24,14 @@ using XMADownloader.Implementation.Models.Export; using UniversalDownloaderPlatform.DefaultImplementations.Interfaces; using XMADownloader.Common.Models; +using Castle.Core.Internal; namespace XMADownloader.Implementation { internal sealed class XmaPageCrawler : IPageCrawler { - private const string CrawlStartUrl = "https://xivmodarchive.com/search?sortby=time_posted&sortorder=desc&types=1%2C3%2C7%2C9%2C12%2C15%2C2%2C4%2C8%2C10%2C14%2C11%2C5%2C13%2C6"; + //private const string CrawlStartUrl = "https://xivmodarchive.com/search?sortby=time_posted&sortorder=desc&types=1%2C3%2C7%2C9%2C12%2C15%2C2%2C4%2C8%2C10%2C14%2C11%2C5%2C13%2C6"; + private const string CrawlStartUrl = "https://xivmodarchive.com/search?"; private static Regex _modPageUrlMatchRegex = new Regex("https:\\/\\/(?>www\\.)?xivmodarchive\\.com\\/(modid|private)\\/([a-z\\-0-9]+)(\\/.+)?"); private readonly XmaWebDownloader _webDownloader; @@ -52,7 +54,7 @@ public XmaPageCrawler(IWebDownloader webDownloader, IPluginManager pluginManager { _webDownloader = (XmaWebDownloader)webDownloader ?? throw new ArgumentNullException(nameof(webDownloader)); _pluginManager = pluginManager ?? throw new ArgumentNullException(nameof(pluginManager)); - + _random = new Random(); } @@ -77,12 +79,63 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) List crawledUrls = new List(); Random rnd = new Random(Guid.NewGuid().GetHashCode()); - string basePageUrl = CrawlStartUrl + $"&author=id-{xmaCrawlTargetInfo.Id}&page="; + + string basePageUrl = CrawlStartUrl; + + //add sort by order + basePageUrl += "sortby=" + _xmaDownloaderSettings.SortBy + "&sortorder=desc"; + + + //add types + basePageUrl += "&types="; + if (_xmaDownloaderSettings.ModTypes.IsNullOrEmpty()) + basePageUrl += "1%2C3%2C7%2C9%2C12%2C15%2C2%2C4%2C8%2C10%2C14%2C11%2C5%2C13%2C6"; + else + { + int[] modtypes = (int[])_xmaDownloaderSettings.ModTypes; + basePageUrl += modtypes.First(); + + if (modtypes.Length > 1) + foreach(var modtype in modtypes) + { + basePageUrl += "%2C" + modtype; + } + } + if (!_xmaDownloaderSettings.SearchText.IsNullOrEmpty()) + { + string replaceSpace = _xmaDownloaderSettings.SearchText.Replace(" ", "%20"); + basePageUrl += "&basic_text=" + replaceSpace; + } + + + + if (_xmaDownloaderSettings.ContentType == 2) + basePageUrl += "&nsfw=false"; + else if (_xmaDownloaderSettings.ContentType == 3) + basePageUrl += "&nsfw=true"; + + basePageUrl += $"&author=id-{xmaCrawlTargetInfo.Id}&page="; + + List pages = _xmaDownloaderSettings.Pages.ToList(); int page = 0; while (true) { - page++; + //if only certain pages are requested + if(!_xmaDownloaderSettings.Pages.IsNullOrEmpty()) + { + if (pages.IsNullOrEmpty())//once its done all pages, break + break; + + page = pages.First(); + pages.Remove(page); + + } + else + { + page++; + } + _logger.Debug($"Page #{page}"); string searchPageHtml = await _webDownloader.DownloadString(basePageUrl + page); @@ -210,6 +263,10 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) if (descriptionNode == null) throw new Exception($"[{id}] Description node not found!"); + HtmlNode imageNode = doc.DocumentNode.SelectSingleNode("//img[contains(@class,\"mod-carousel-image\")]"); + if (imageNode == null) + throw new Exception("Image was not found"); + DateTime? publishDate = null; DateTime? lastUpdateDate = null; HtmlNodeCollection modDateNodes = doc.DocumentNode.SelectNodes("//div[contains(@class,\"mod-meta-block\")]"); @@ -247,15 +304,24 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) throw new Exception($"[{id}] Last update date not found!"); string currentUrl = await _webDownloader.GetActualUrl(HttpUtility.HtmlDecode(primaryUrlNode.Attributes["href"].Value)); - + + if (!_parsedUrls.Contains(currentUrl)) { _parsedUrls.Add(currentUrl); parsedUrlsForThisMod.Add(currentUrl); + _logger.Debug($"[{id}] New primary url: {currentUrl}"); } - if(additionalUrlNodes != null) + if (_xmaDownloaderSettings.DownloadModImage) + { + string modImage = await _webDownloader.GetActualUrl(HttpUtility.HtmlDecode(imageNode.Attributes["src"].Value)); + _parsedUrls.Add(modImage); + parsedUrlsForThisMod.Add(modImage); + } + + if(additionalUrlNodes != null && _xmaDownloaderSettings.DownloadUrlsInFilesTab) { foreach (HtmlNode node in additionalUrlNodes) { @@ -274,17 +340,21 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) } //External urls via plugins (including direct via default plugin) - List pluginUrls = await _pluginManager.ExtractSupportedUrls(HttpUtility.HtmlDecode(descriptionNode.InnerHtml)); - foreach (string url in pluginUrls) + if (_xmaDownloaderSettings.DownloadUrlsInDescription) { - currentUrl = await _webDownloader.GetActualUrl(url); - if (!_parsedUrls.Contains(currentUrl)) + List pluginUrls = await _pluginManager.ExtractSupportedUrls(HttpUtility.HtmlDecode(descriptionNode.InnerHtml)); + foreach (string url in pluginUrls) { - _parsedUrls.Add(currentUrl); - parsedUrlsForThisMod.Add(currentUrl); - _logger.Debug($"[{id}] New external entry: {currentUrl}"); + currentUrl = await _webDownloader.GetActualUrl(url); + if (!_parsedUrls.Contains(currentUrl)) + { + _parsedUrls.Add(currentUrl); + parsedUrlsForThisMod.Add(currentUrl); + _logger.Debug($"[{id}] New external entry: {currentUrl}"); + } } } + XmaCrawledUrl entry = new XmaCrawledUrl { @@ -296,13 +366,13 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) }; string additionalFilesSaveDirectory = Path.Combine(_xmaDownloaderSettings.DownloadDirectory, entry.UserId.ToString()); - if (/*_xmaDownloaderSettings.IsUseSubDirectories*/true && - (_xmaDownloaderSettings.SaveDescriptions || _xmaDownloaderSettings.SaveDescriptions) - ) - { - additionalFilesSaveDirectory = Path.Combine(additionalFilesSaveDirectory, - PostSubdirectoryHelper.CreateNameFromPattern(entry, _xmaDownloaderSettings.SubDirectoryPattern, _xmaDownloaderSettings.MaxSubdirectoryNameLength)); - } + //if (_xmaDownloaderSettings.IsUseSubDirectories && + // (_xmaDownloaderSettings.SaveDescriptions || _xmaDownloaderSettings.SaveDescriptions) + // ) + //{ + // additionalFilesSaveDirectory = Path.Combine(additionalFilesSaveDirectory, + // PostSubdirectoryHelper.CreateNameFromPattern(entry, _xmaDownloaderSettings.SubDirectoryPattern, _xmaDownloaderSettings.MaxSubdirectoryNameLength)); + //} if (!Directory.Exists(additionalFilesSaveDirectory)) Directory.CreateDirectory(additionalFilesSaveDirectory); @@ -345,8 +415,6 @@ public async Task> Crawl(ICrawlTargetInfo crawlTargetInfo) OnPostCrawlEnd(new PostCrawlEventArgs(id, true)); - - return (crawledUrls, crawledMods); }