Scan speed improvement

File-Integrity-Scanner/pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>org.pwss</groupId>
     <artifactId>File-Integrity-Scanner</artifactId>
-    <version>1.8.5</version>
+    <version>1.9</version>
     <packaging>jar</packaging>
     <description>A File Integrity Scanner</description>
     <licenses>
@@ -140,7 +140,7 @@
         <dependency>
             <groupId>io.github.pwssorg</groupId>
             <artifactId>algorithm-hash-extraction</artifactId>
-            <version>1.2.8</version>
+            <version>1.2.9</version>
         </dependency>
 
 

File-Integrity-Scanner/src/main/java/org/pwss/file_integrity_scanner/dsr/service/file_integrity_scanner/scan/FileHashComputer.java

@@ -2,6 +2,7 @@
 
 import lib.pwss.hash.file_hash_handler.BigFileHashHandler;
 import lib.pwss.hash.file_hash_handler.FileHashHandler;
+import lib.pwss.hash.file_hash_handler.parallel.ParallelFileHashHandler;
 import lib.pwss.hash.FileHash;
 import lib.pwss.hash.compare.util.HashCompareUtil;
 import lib.pwss.hash.model.HashForFilesOutput;
@@ -29,12 +30,21 @@ final class FileHashComputer {
 
     private final org.slf4j.Logger log;
 
-    // Instance of FileHashHandler for computing hashes of smaller files
+    /**
+     * Instance of FileHashHandler for computing hashes of smaller files
+     */
     private final FileHash fileHashHandler;
 
-    // Instance of BigFileHashHandler for computing hashes of larger files
+    /**
+     * Instance of BigFileHashHandler for computing hashes of larger files
+     */
     private final BigFileHashHandler bigFileHashHandler;
 
+    /**
+     * Handles parallel hash computation for files during scans.
+     */
+    private ParallelFileHashHandler parallelFileHashHandler;
+
     FileHashComputer() {
         this.log = org.slf4j.LoggerFactory.getLogger(FileHashComputer.class);
         this.fileHashHandler = new FileHashHandler();
@@ -54,13 +64,13 @@ Optional<HashForFilesOutput> computeHashes(File file) {
         try {
 
             if (file.length() > MEMORY_STRATEGY_LIMIT)
-                return Optional.of(bigFileHashHandler.GetAllHashes(file));
+                return Optional.of(parallelFileHashHandler.GetAllHashesInParallel(file));
             else
                 return Optional.of(fileHashHandler.GetAllHashes(file));
 
         } catch (OutOfMemoryError outOfMemoryError) {
-            log.debug("OutOfMemoryError occurred, switching to BigFileHashHandler for file: {}", file.getPath());
-            return Optional.of(bigFileHashHandler.GetAllHashes(file));
+            log.debug("OutOfMemoryError occurred, switching to ParallelFileHashHandler for file: {}", file.getPath());
+            return Optional.of(parallelFileHashHandler.GetAllHashesInParallel(file));
         }
 
         catch (NullPointerException nullPointerException) {
@@ -101,7 +111,24 @@ boolean compareHashes(Checksum first, Checksum second) {
      *                            user.
      */
     final void setUserDefinedMaxLimitInHashComputer(long userDefinedMaxLimit) {
-        bigFileHashHandler.setUserDefinedMaxLimit(userDefinedMaxLimit);
+        this.bigFileHashHandler.setUserDefinedMaxLimit(userDefinedMaxLimit);
+    }
+
+    /**
+     * Shuts down resources used for parallel hash computation.
+     */
+    final void shutdownParallelHashProcessor() {
+
+        this.parallelFileHashHandler.shutdownThreadPool();
+    }
+
+    /**
+     * Initializes resources required for parallel hash computation
+     * before starting a scan operation.
+     */
+    final void initializeParallelHashing() {
+
+        this.parallelFileHashHandler = new ParallelFileHashHandler(bigFileHashHandler);
     }
 
 }

File-Integrity-Scanner/src/main/java/org/pwss/file_integrity_scanner/dsr/service/file_integrity_scanner/scan/ScanServiceImpl.java

@@ -268,7 +268,7 @@ public void scanAllDirectories(StartAllRequest request)
                     Scan scan = new Scan(time, ScanStatus.IN_PROGRESS.toString(), dir, note, isBaseLineScan);
 
                     repository.save(scan);
-
+                    fileHashComputer.initializeParallelHashing();
                     fileTraverser = new FileTraverserImpl();
                     Future<List<File>> futureFiles;
 
@@ -353,7 +353,7 @@ public void scanSingleDirectory(StartScanByIdRequest request)
 
             this.isScanRunning = true;
             log.debug("Scan is running - {}", isScanRunning);
-
+            fileHashComputer.initializeParallelHashing();
             fileTraverser = new FileTraverserImpl();
 
             final Time time = new Time(OffsetDateTime.now(), OffsetDateTime.now());
@@ -594,7 +594,8 @@ private boolean finalizeScanTask(Scan scanInstance, List<File> files) {
 
                     // Shutdown the file traverser thread pool
                     fileTraverser.shutdownThreadPool();
-
+                    // Shutdown the parallel hash calculation thread pool
+                    fileHashComputer.shutdownParallelHashProcessor();
                     // Set state boolean to false so this method can be ran again
                     this.isScanRunning = false;
                 }

File-Integrity-Scanner/src/main/resources/logback.xml

@@ -17,6 +17,7 @@
     <!-- Configure specific packages from dependencies -->
     <logger name="org.pwss.io_file" level="INFO"/>
     <logger name="lib.pwss.hash.file_hash_handler" level="ERROR"/>
+    <logger name="lib.pwss.hash.file_hash_handler.parallel" level="ERROR"/>
     <logger name="org.pwss.util.PWSSDirectoryNavUtil" level="INFO"/>
     <logger name="org.pwss.quarantineManager_aes" level="INFO"/>
     <logger name="org.hibernate" level="INFO"/>