No sub dir

XMADownloader.App/Models/CommandLineOptions.cs

@@ -1,6 +1,7 @@
 using CommandLine;
 using XMADownloader.App.Enums;
 using UniversalDownloaderPlatform.Common.Enums;
+using System.Collections.Generic;
 
 namespace XMADownloader.App.Models
 {
@@ -44,8 +45,8 @@ class CommandLineOptions
         [Option("remote-browser-address", Required = false, HelpText = "Advanced users only. Address of the browser with remote debugging enabled. Refer to documentation for more details.")]
         public string RemoteBrowserAddress { get; set; }
 
-        /*[Option("use-sub-directories", Required = false, HelpText = "Create a new directory inside of the download directory for every post instead of placing all files into a single directory.")]
-        public bool UseSubDirectories { get; set; }*/
+        [Option("use-sub-directories", Required = false, HelpText = "Create a new directory inside of the download directory for every post instead of placing all files into a single directory.", Default = true)]
+        public bool UseSubDirectories { get; set; }
 
         [Option("sub-directory-pattern", Required = false, HelpText = "Pattern which will be used to create a name for the sub directories if --use-sub-directories is used. Supported parameters: %ModId%, %PublishedAt%, %PostTitle%.", Default = "[%ModId%] %PublishedAt% %PostTitle%")]
         public string SubDirectoryPattern { get; set; }
@@ -61,5 +62,46 @@ class CommandLineOptions
 
         [Option("proxy-server-address", Required = false, HelpText = "The address of proxy server to use in the following format: [<proxy-scheme>://]<proxy-host>[:<proxy-port>]. Supported protocols: http(s), socks4, socks4a, socks5.")]
         public string ProxyServerAddress { get; set; }
+
+        [Option("download-urls-in-description", Required = false, HelpText = "Scrapes the description text for urls and downloads the files found in that url", Default = false)]
+        public bool DownloadUrlsInDescription { get; set; }
+
+        [Option("download-urls-in-filestab", Required = false, HelpText = "Download all of the files in the filetab", Default = false)]
+        public bool DownloadUrlsInFilesTab { get; set; }
+
+        [Option("download-mod-image", Required = false, HelpText = "Download the cover image for the mod", Default = true)]
+        public bool DownloadModImage { get; set; }
+
+        [Option("content-type", Required = false, HelpText = "1 = Both, 2 = SFW only, 3 = NSFW only", Default = 1)]
+        public int ContentType { get; set; }
+
+        [Option("search-text", Required = false, HelpText = "Search for posts that include the text somewhere (title, description, tags etc)")]
+        public string SearchText { get; set; }
+
+        /// <summary>
+        /// 1 = Gear mods
+        /// 2 = Body replacement mods
+        /// 3 = Face mods
+        /// 4 = Hair mods
+        /// 5 = Shaders
+        /// 6 = Other mods
+        /// 7 = Minion mods
+        /// 8 = Mount mods
+        /// 10 = Skin mods
+        /// 11 = Concept matrix pose
+        /// 12 = Racial scaling mods
+        /// 13 = Anamnesis pose
+        /// 14 = VFX
+        /// 15 = Animation
+        /// 16 = Sound
+        /// </summary>
+        [Option("types", Required = false, HelpText = "Choose the modtypes you want to search for\r\nExample: --types 11 13 for poses\r\n1 = Gear mods\r\n2 = Body replacement mods\r\n3 = Face mods\r\n4 = Hair mods\r\n5 = Shaders\r\n6 = Other mods\r\n7 = Minion mods\r\n8 = Mount mods\r\n10 = Skin mods\r\n11 = Concept matrix pose\r\n12 = Racial scaling mods\r\n13 = Anamnesis pose\r\n14 = VFX\r\n15 = Animation\r\n16 = Sound")]
+        public IEnumerable<int> ModTypes { get; set; }
+
+        [Option("pages", Required = false, HelpText = "If you only want some certain pages. Example: --pages 1 4 5 will give you page 1, 4 and 5")]
+        public IEnumerable<int> Pages { get; set; }
+
+        [Option("sort-by", Required = false, HelpText = "Use the sort by feature on the website, always uses descending\r\nrank = Relevance\r\ntime_edited = Last Version Update\r\ntime_posted = Release Date\r\nname = Name\r\nview = Views\r\nviews_today = Views Today\r\ndownloads = Downloads\r\nfollowers = Followers", Default = "time_posted")]
+        public string SortBy { get; set; }
     }
 }

XMADownloader.App/Program.cs

@@ -156,14 +156,22 @@ private static async Task<XmaDownloaderSettings> InitializeSettings(CommandLineO
                 DownloadDirectory = commandLineOptions.DownloadDirectory,
                 FileExistsAction = commandLineOptions.FileExistsAction,
                 IsCheckRemoteFileSize = !commandLineOptions.IsDisableRemoteFileSizeCheck,
-                //IsUseSubDirectories = commandLineOptions.UseSubDirectories,
+                IsUseSubDirectories = commandLineOptions.UseSubDirectories,
                 SubDirectoryPattern = commandLineOptions.SubDirectoryPattern,
                 MaxSubdirectoryNameLength = commandLineOptions.MaxSubdirectoryNameLength,
                 MaxFilenameLength = commandLineOptions.MaxFilenameLength,
                 FallbackToContentTypeFilenames = commandLineOptions.FilenamesFallbackToContentType,
                 ProxyServerAddress = commandLineOptions.ProxyServerAddress,
                 RemoteBrowserAddress = commandLineOptions.RemoteBrowserAddress != null ? new Uri(commandLineOptions.RemoteBrowserAddress) : null,
-                ExportCrawlResults = commandLineOptions.ExportCrawlJson
+                ExportCrawlResults = commandLineOptions.ExportCrawlJson,
+                DownloadModImage = commandLineOptions.DownloadModImage,
+                DownloadUrlsInDescription = commandLineOptions.DownloadUrlsInDescription,
+                DownloadUrlsInFilesTab = commandLineOptions.DownloadUrlsInFilesTab,
+                SearchText = commandLineOptions.SearchText,
+                ContentType = commandLineOptions.ContentType,
+                ModTypes = commandLineOptions.ModTypes,
+                Pages = commandLineOptions.Pages,
+                SortBy = commandLineOptions.SortBy
             };
 
             return settings;

XMADownloader.App/Properties/PublishProfiles/net3.1-linux-x64-release.pubxml

@@ -7,13 +7,11 @@ https://go.microsoft.com/fwlink/?LinkID=208121.
     <PublishProtocol>FileSystem</PublishProtocol>
     <Configuration>Release</Configuration>
     <Platform>Any CPU</Platform>
-    <TargetFramework>netcoreapp3.1</TargetFramework>
-    <PublishDir>bin\publish\net3.1-linux-x64-release</PublishDir>
-    <RuntimeIdentifier>linux-x64</RuntimeIdentifier>
-    <SelfContained>true</SelfContained>
+    <TargetFramework>net6.0</TargetFramework>
+    <PublishDir>C:\Users\hapse\Documents\GitHub\XMADownloader\XMADownloader.App\bin\Release\net6.0</PublishDir>
+    <RuntimeIdentifier>win-x64</RuntimeIdentifier>
+    <SelfContained>false</SelfContained>
     <_IsPortable>false</_IsPortable>
-    <PublishSingleFile>False</PublishSingleFile>
-    <PublishReadyToRun>False</PublishReadyToRun>
-    <PublishTrimmed>False</PublishTrimmed>
+    <PublishReadyToRun>false</PublishReadyToRun>
   </PropertyGroup>
 </Project>

XMADownloader.Common/Models/XmaDownloaderSettings.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Net.Mime;
 using System.Text;
 using UniversalDownloaderPlatform.Common.Enums;
 using UniversalDownloaderPlatform.Common.Helpers;
@@ -18,7 +19,7 @@ public record XmaDownloaderSettings : UniversalDownloaderPlatformSettings, IPupp
         /// <summary>
         /// Create a new directory for every post and store files of said post in that directory
         /// </summary>
-        //public bool IsUseSubDirectories { get; init; }
+        public bool IsUseSubDirectories { get; init; }
 
         /// <summary>
         /// Pattern used to generate directory name if UseSubDirectories is enabled
@@ -45,17 +46,34 @@ public record XmaDownloaderSettings : UniversalDownloaderPlatformSettings, IPupp
         public Uri RemoteBrowserAddress { get; init; }
         public bool IsHeadlessBrowser { get; init; }
         public bool ExportCrawlResults { get; set; }
+        public bool DownloadUrlsInDescription { get; set; }
+        public bool DownloadUrlsInFilesTab { get; set; }
+        public bool DownloadModImage { get; set; }
+        public int ContentType { get; set; }
+        public string SearchText { get; set; }
+        public IEnumerable<int> ModTypes { get; set; }
+        public IEnumerable<int> Pages { get; set; }
+        public string SortBy { get; set; }
 
         public XmaDownloaderSettings()
         {
             SaveDescriptions = true;
             SaveHtml = true;
-            //IsUseSubDirectories = false;
+            IsUseSubDirectories = true;
             SubDirectoryPattern = "[%ModId%] %PublishedAt% %PostTitle%";
             FallbackToContentTypeFilenames = false;
             MaxFilenameLength = 100;
             MaxSubdirectoryNameLength = 100;
             IsHeadlessBrowser = true;
+            DownloadUrlsInDescription = false;
+            DownloadUrlsInFilesTab = false;
+            DownloadModImage = true;
+            ContentType = 1;
+            SearchText = "";
+            ModTypes = new int[0];
+            Pages = new int[0];
+            SortBy = "time_posted";
+
         }
     }
 }

XMADownloader.Implementation/Models/XmaCrawledUrl.cs

@@ -53,5 +53,10 @@ public object Clone()
                 UserId = UserId
             };
         }
+
+        public override string ToString()
+        {
+            return Name;
+        }
     }
 }

XMADownloader.Implementation/XmaCrawledUrlProcessor.cs

@@ -56,24 +56,33 @@ public async Task<bool> ProcessCrawledUrl(ICrawledUrl udpCrawledUrl)
 
             if (!crawledUrl.IsProcessedByPlugin)
             {
-                /*if (!_XMADownloaderSettings.IsUseSubDirectories)
+                if (!_xmaDownloaderSettings.IsUseSubDirectories)
                     filename = $"{crawledUrl.ModId}_";
                 else
-                    filename = "";*/
+                    filename = "";
 
                 if (crawledUrl.Filename == null)
                     throw new DownloadException($"[{crawledUrl.ModId}] No filename for {crawledUrl.Url}!");
 
-                filename = crawledUrl.Filename;
+
+                string extension = Path.GetExtension(crawledUrl.Filename);
+
+                //If the downloaded file is an image rename it to the mods name
+                if (extension == ".jpg" || extension == ".png" || extension == ".jpeg")
+                {
+                    filename = crawledUrl.Name + extension;
+                }
+                else //Else we set it to the orginal filename
+                    filename = crawledUrl.Filename;
 
                 _logger.Debug($"Sanitizing filename: {filename}");
                 filename = PathSanitizer.SanitizePath(filename);
                 _logger.Debug($"Sanitized filename: {filename}");
 
+
                 if (filename.Length > _xmaDownloaderSettings.MaxFilenameLength)
                 {
                     _logger.Debug($"Filename is too long, will be truncated: {filename}");
-                    string extension = Path.GetExtension(filename);
                     if (extension.Length > 4)
                     {
                         _logger.Warn($"File extension for file {filename} is longer 4 characters and won't be appended to truncated filename!");
@@ -83,6 +92,7 @@ public async Task<bool> ProcessCrawledUrl(ICrawledUrl udpCrawledUrl)
                     _logger.Debug($"Truncated filename: {filename}");
                 }
 
+
                 string key = $"{crawledUrl.ModId}_{filename.ToLowerInvariant()}";
 
                 _fileCountDict.AddOrUpdate(key, 0, (key, oldValue) => oldValue + 1);
@@ -111,9 +121,9 @@ public async Task<bool> ProcessCrawledUrl(ICrawledUrl udpCrawledUrl)
 
             string downloadDirectory = crawledUrl.UserId.ToString();
 
-            if (/*_XMADownloaderSettings.IsUseSubDirectories*/true)
-                downloadDirectory = Path.Combine(downloadDirectory, PostSubdirectoryHelper.CreateNameFromPattern(crawledUrl, _xmaDownloaderSettings.SubDirectoryPattern, _xmaDownloaderSettings.MaxSubdirectoryNameLength));
-
+            //if (_xmaDownloaderSettings.IsUseSubDirectories)
+            //    downloadDirectory = Path.Combine(downloadDirectory, PostSubdirectoryHelper.CreateNameFromPattern(crawledUrl, _xmaDownloaderSettings.SubDirectoryPattern, _xmaDownloaderSettings.MaxSubdirectoryNameLength));
+            //_logger.Debug(crawledUrl.DownloadPath);
             crawledUrl.DownloadPath = !crawledUrl.IsProcessedByPlugin ? Path.Combine(downloadDirectory, filename) : downloadDirectory + Path.DirectorySeparatorChar;
 
             return true;

XMADownloader.Implementation/XmaDefaultPlugin.cs

@@ -167,7 +167,11 @@ public async Task<bool> ProcessCrawledUrl(ICrawledUrl udpCrawledUrl)
                 }
 
                 crawledUrl.Filename = filename;
-                crawledUrl.FileSize = fileSize;
+
+                if (_settings.IsCheckRemoteFileSize)
+                    crawledUrl.FileSize = fileSize;
+                else
+                    crawledUrl.FileSize = 0;
             }
             catch (Exception ex)
             {