Merge pull request #669 from apache/ffm_phase2

FFM transition: Phase 2: HLL

Author: leerho

pom.xml

@@ -93,7 +93,7 @@ under the License.
     <testng.check-cpp-historical-files>check_cpp_historical_files</testng.check-cpp-historical-files>
 
     <!-- System-wide properties -->
-    <maven.version>3.6.3</maven.version>
+    <maven.version>3.9.10</maven.version>
     <java.version>24</java.version>
     <jvm-arguments></jvm-arguments>
     <maven.compiler.source>${java.version}</maven.compiler.source>
@@ -107,16 +107,16 @@ under the License.
 
     <!--  org.apache.maven plugins -->
     <maven-assembly-plugin.version>3.7.1</maven-assembly-plugin.version>
-    <maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
-    <maven-deploy-plugin.version>3.1.3</maven-deploy-plugin.version>
-    <maven-enforcer-plugin.version>3.5.0</maven-enforcer-plugin.version>
-    <maven-gpg-plugin.version>3.2.7</maven-gpg-plugin.version>
+    <maven-compiler-plugin.version>3.14.0</maven-compiler-plugin.version>
+    <maven-deploy-plugin.version>3.1.4</maven-deploy-plugin.version>
+    <maven-enforcer-plugin.version>3.6.0</maven-enforcer-plugin.version>
+    <maven-gpg-plugin.version>3.2.8</maven-gpg-plugin.version>
     <maven-jar-plugin.version>3.4.2</maven-jar-plugin.version>
     <maven-javadoc-plugin.version>3.11.2</maven-javadoc-plugin.version>
     <maven-release-plugin.version>3.1.1</maven-release-plugin.version>
     <maven-source-plugin.version>3.3.1</maven-source-plugin.version>
     <!-- for surefire, failsafe and surefire-report: -->
-    <maven-surefire-failsafe-plugins.version>3.5.2</maven-surefire-failsafe-plugins.version>
+    <maven-surefire-failsafe-plugins.version>3.5.3</maven-surefire-failsafe-plugins.version>
     <maven-toolchains-plugin.version>3.2.0</maven-toolchains-plugin.version>
     <!-- com.github plugins -->
     <git-commit-id-plugin.version>4.9.10</git-commit-id-plugin.version>

src/main/java/org/apache/datasketches/common/MemorySegmentStatus.java

@@ -20,6 +20,7 @@
 package org.apache.datasketches.common;
 
 import java.lang.foreign.MemorySegment;
+import java.util.Optional;
 
 /**
  * Methods for inquiring the status of a backing MemorySegment.
@@ -37,22 +38,41 @@ public interface MemorySegmentStatus {
    * Returns true if this object's internal data is backed by an off-heap (direct or native)) MemorySegment.
    * @return true if this object's internal data is backed by an off-heap (direct or native)) MemorySegment.
    */
-  boolean isDirect();
+  boolean isOffHeap();
 
   /**
-   * Returns true if the backing MemorySegment of this object refers to the same MemorySegment of <i>that</i>.
-   * They can either have the same off-heap memory location and size, or refer to the same on-heap array object.
+   * Returns true if an internally referenced MemorySegment has the same backing resource as <i>that</i>,
+   * or equivalently, if their two memory regions overlap.  This applies to both on-heap and off-heap MemorySegments.
    *
-   * <p>If both segment are off-heap, they both must have the same starting address and the same size.</p>
+   * <p>This returns false if either segment is <i>null</i> or not alive.</p>
    *
-   * <p>For on-heap segments, both segments must be based on or derived from the same array object and neither segment
-   * can be read-only.</p>
-   *
-   * <p>Returns false if either argument is null;</p>
+   * <p><b>Note:</b> If both segments are on-heap and not read-only, it can be determined if they were derived from
+   * the same backing memory (array).  However, this is not always possible off-heap. Because of this asymmetry, this definition
+   * of "isSameResource" is confined to the existence of an overlap.</p>
    *
    * @param that The given MemorySegment.
-   * @return true if the backing MemorySegment of this object hierarchy refers to the same MemorySegment of <i>that</i>.
+   * @return true if an internally referenced MemorySegment has the same backing resource as <i>that</i>.
    */
   boolean isSameResource(final MemorySegment that);
 
+  /**
+   * Returns true if the two given MemorySegments have to the same backing resource, or equivalently,
+   * if the two memory regions overlap.  This applies to both on-heap and off-heap MemorySegments.
+   *
+   * <p>This returns false if either segment is <i>null</i> or not alive.</p>
+   *
+   * <p><b>Note:</b> If both segments are on-heap and not read-only, it can be determined if they were derived from
+   * the same backing memory (array).  However, this is not always possible off-heap. Because of this asymmetry, this definition
+   * of "isSameResource" is confined to the existence of an overlap.</p>
+   *
+   * @param seg1 The first given MemorySegment
+   * @param seg2 The second given MemorySegment
+   * @return true if the two given MemorySegments have to the same backing resource.
+   */
+  static boolean isSameResource(final MemorySegment seg1, final MemorySegment seg2) {
+    if ((seg1 == null) || (seg2 == null) || !seg1.scope().isAlive() || !seg2.scope().isAlive()) { return false; }
+    final Optional<MemorySegment> opt = seg1.asOverlappingSlice(seg2);
+    return opt.isPresent();
+  }
+
 }

src/main/java/org/apache/datasketches/common/Util.java

@@ -101,10 +101,10 @@ private Util() {}
    * @return an int extracted from a Little-Endian byte array.
    */
   public static int bytesToInt(final byte[] arr) {
-    return arr[3] << 24
-          | (arr[2] & 0xff) << 16
-          | (arr[1] & 0xff) <<  8
-          | arr[0] & 0xff;
+    return (arr[3] << 24)
+          | ((arr[2] & 0xff) << 16)
+          | ((arr[1] & 0xff) <<  8)
+          | (arr[0] & 0xff);
   }
 
   /**
@@ -113,14 +113,14 @@ public static int bytesToInt(final byte[] arr) {
    * @return a long extracted from a Little-Endian byte array.
    */
   public static long bytesToLong(final byte[] arr) {
-    return (long)arr[7] << 56
-          | ((long)arr[6] & 0xff) << 48
-          | ((long)arr[5] & 0xff) << 40
-          | ((long)arr[4] & 0xff) << 32
-          | ((long)arr[3] & 0xff) << 24
-          | ((long)arr[2] & 0xff) << 16
-          | ((long)arr[1] & 0xff) <<  8
-          | (long)arr[0] & 0xff;
+    return ((long)arr[7] << 56)
+          | (((long)arr[6] & 0xff) << 48)
+          | (((long)arr[5] & 0xff) << 40)
+          | (((long)arr[4] & 0xff) << 32)
+          | (((long)arr[3] & 0xff) << 24)
+          | (((long)arr[2] & 0xff) << 16)
+          | (((long)arr[1] & 0xff) <<  8)
+          | ((long)arr[0] & 0xff);
   }
 
   /**
@@ -159,7 +159,7 @@ public static byte[] longToBytes(final long v, final byte[] arr) {
 
   static long[] convertToLongArray(final byte[] byteArr, final boolean littleEndian) {
     final int len = byteArr.length;
-    final long[] longArr = new long[len / 8 + (len % 8 != 0 ? 1 : 0)];
+    final long[] longArr = new long[(len / 8) + ((len % 8) != 0 ? 1 : 0)];
     int off = 0;
     int longArrIdx = 0;
     while (off < len) {
@@ -191,7 +191,7 @@ public static String longToHexBytes(final long v) {
     final long mask = 0XFFL;
     final StringBuilder sb = new StringBuilder();
     for (int i = 8; i-- > 0; ) {
-      final String s = Long.toHexString(v >>> i * 8 & mask);
+      final String s = Long.toHexString((v >>> (i * 8)) & mask);
       sb.append(zeroPad(s, 2)).append(" ");
     }
     return sb.toString();
@@ -211,7 +211,7 @@ public static String bytesToString(
     final int mask = signed ? 0XFFFFFFFF : 0XFF;
     final int arrLen = arr.length;
     if (littleEndian) {
-      for (int i = 0; i < arrLen - 1; i++) {
+      for (int i = 0; i < (arrLen - 1); i++) {
         sb.append(arr[i] & mask).append(sep);
       }
       sb.append(arr[arrLen - 1] & mask);
@@ -231,8 +231,8 @@ public static String bytesToString(
    */
   public static String nanoSecToString(final long nS) {
     final long rem_nS = (long)(nS % 1000.0);
-    final long rem_uS = (long)(nS / 1000.0 % 1000.0);
-    final long rem_mS = (long)(nS / 1000000.0 % 1000.0);
+    final long rem_uS = (long)((nS / 1000.0) % 1000.0);
+    final long rem_mS = (long)((nS / 1000000.0) % 1000.0);
     final long sec    = (long)(nS / 1000000000.0);
     final String nSstr = zeroPad(Long.toString(rem_nS), 3);
     final String uSstr = zeroPad(Long.toString(rem_uS), 3);
@@ -247,8 +247,8 @@ public static String nanoSecToString(final long nS) {
    */
   public static String milliSecToString(final long mS) {
     final long rem_mS = (long)(mS % 1000.0);
-    final long rem_sec = (long)(mS / 1000.0 % 60.0);
-    final long rem_min = (long)(mS / 60000.0 % 60.0);
+    final long rem_sec = (long)((mS / 1000.0) % 60.0);
+    final long rem_min = (long)((mS / 60000.0) % 60.0);
     final long hr  =     (long)(mS / 3600000.0);
     final String mSstr = zeroPad(Long.toString(rem_mS), 3);
     final String secStr = zeroPad(Long.toString(rem_sec), 2);
@@ -296,7 +296,7 @@ public static String characterPad(final String s, final int fieldLength, final c
    * @param argName This name will be part of the error message if the check fails.
    */
   public static void checkIfMultipleOf8AndGT0(final long v, final String argName) {
-    if ((v & 0X7L) == 0L && v > 0L) {
+    if (((v & 0X7L) == 0L) && (v > 0L)) {
       return;
     }
     throw new SketchesArgumentException("The value of the parameter \"" + argName
@@ -309,7 +309,7 @@ public static void checkIfMultipleOf8AndGT0(final long v, final String argName)
    * @return true if v is a multiple of 8 and greater than zero
    */
   public static boolean isMultipleOf8AndGT0(final long v) {
-    return (v & 0X7L) == 0L && v > 0L;
+    return ((v & 0X7L) == 0L) && (v > 0L);
   }
 
   //Powers of 2 or powers of base related
@@ -356,7 +356,7 @@ public static void checkIfPowerOf2(final long n, String argName) {
   public static int ceilingPowerOf2(final int n) {
     if (n <= 1) { return 1; }
     final int topIntPwrOf2 = 1 << 30;
-    return n >= topIntPwrOf2 ? topIntPwrOf2 : Integer.highestOneBit(n - 1 << 1);
+    return n >= topIntPwrOf2 ? topIntPwrOf2 : Integer.highestOneBit((n - 1) << 1);
   }
 
   /**
@@ -377,7 +377,7 @@ public static int ceilingPowerOf2(final int n) {
   public static long ceilingPowerOf2(final long n) {
     if (n <= 1L) { return 1L; }
     final long topIntPwrOf2 = 1L << 62;
-    return n >= topIntPwrOf2 ? topIntPwrOf2 : Long.highestOneBit(n - 1L << 1);
+    return n >= topIntPwrOf2 ? topIntPwrOf2 : Long.highestOneBit((n - 1L) << 1);
   }
 
   /**
@@ -430,8 +430,8 @@ public static long floorPowerOf2(final long n) {
    * @return  the inverse integer power of 2: 1/(2^e) = 2^(-e)
    */
   public static double invPow2(final int e) {
-    assert (e | 1024 - e - 1) >= 0 : "e cannot be negative or greater than 1023: " + e;
-    return Double.longBitsToDouble(1023L - e << 52);
+    assert (e | (1024 - e - 1)) >= 0 : "e cannot be negative or greater than 1023: " + e;
+    return Double.longBitsToDouble((1023L - e) << 52);
   }
 
   /**
@@ -695,7 +695,7 @@ public static void checkBounds(final long reqOff, final long reqLen, final long
    * @param argName Used in the thrown exception.
    */
   public static void checkProbability(final double p, final String argName) {
-    if (p >= 0.0 && p <= 1.0) {
+    if ((p >= 0.0) && (p <= 1.0)) {
       return;
     }
     throw new SketchesArgumentException("The value of the parameter \"" + argName
@@ -711,7 +711,7 @@ public static void checkProbability(final double p, final String argName) {
    * @return true if n1 &gt; n2.
    */
   public static boolean isLessThanUnsigned(final long n1, final long n2) {
-    return n1 < n2 ^ n1 < 0 != n2 < 0;
+    return (n1 < n2) ^ ((n1 < 0) != (n2 < 0));
   }
 
   /**
@@ -741,7 +741,7 @@ public static boolean isOdd(final long n) {
    * bit is at position zero.
    * @return a one if the bit at bitPos is a one, otherwise zero.
    */
-  public static final int bitAt(final long number, final int bitPos) {
+  public static int bitAt(final long number, final int bitPos) {
     return (number & (1L << bitPos)) > 0 ? 1 : 0;
   }
 
@@ -751,7 +751,7 @@ public static final int bitAt(final long number, final int bitPos) {
    * @return the number of decimal digits of the number n
    */
   public static int numDigits(long n) {
-    if (n % 10 == 0) { n++; }
+    if ((n % 10) == 0) { n++; }
     return (int) ceil(log(n) / log(10));
   }
 
@@ -898,39 +898,6 @@ public static void fill(final MemorySegment seg, final long offsetBytes, final l
     slice.fill(value);
   }
 
-  /**
-   * Returns true if the two given MemorySegments refer to the same backing resource,
-   * which is either an off-heap memory location and size, or the same on-heap array object.
-   *
-   * <p>If both segment are off-heap, they both must have the same starting address and the same size.</p>
-   *
-   * <p>For on-heap segments, both segments must be based on or derived from the same array object and neither segment
-   * can be read-only.</p>
-   *
-   * <p>Returns false if either argument is null;</p>
-   *
-   * @param seg1 The first given MemorySegment
-   * @param seg2 The second given MemorySegment
-   * @return true if both MemorySegments are determined to be the same backing memory.
-   */
-  public static boolean isSameResource(final MemorySegment seg1, final MemorySegment seg2) {
-    if ((seg1 == null) || (seg2 == null)) { return false; }
-    if (!seg1.scope().isAlive() || !seg2.scope().isAlive()) {
-      throw new IllegalArgumentException("Both arguments must be alive.");
-    }
-    final boolean seg1Native = seg1.isNative();
-    final boolean seg2Native = seg2.isNative();
-    if (seg1Native ^ seg2Native) { return false; }
-    if (seg1Native && seg2Native) { //both off heap
-      return (seg1.address() == seg2.address()) && (seg1.byteSize() == seg2.byteSize());
-    }
-    //both on heap
-    if (seg1.isReadOnly() || seg2.isReadOnly()) {
-      throw new IllegalArgumentException("Cannot determine 'isSameBackingMemory(..)' on heap if either MemorySegment is Read-only.");
-    }
-    return (seg1.heapBase().orElse(null) == seg2.heapBase().orElse(null));
-  }
-
   /**
    * Request a new heap MemorySegment with the given capacityBytes and either 8-byte aligned or one byte aligned.
    *

src/main/java/org/apache/datasketches/hll/AbstractCoupons.java

@@ -93,7 +93,7 @@ abstract class AbstractCoupons extends HllSketchImpl {
 
   @Override
   int getUpdatableSerializationBytes() {
-    return getMemDataStart() + (4 << getLgCouponArrInts());
+    return getSegDataStart() + (4 << getLgCouponArrInts());
   }
 
   @Override

src/main/java/org/apache/datasketches/hll/AbstractHllArray.java

@@ -145,7 +145,7 @@ int getCompactSerializationBytes() {
   }
 
   @Override
-  int getMemDataStart() {
+  int getSegDataStart() {
     return HLL_BYTE_ARR_START;
   }
 

src/main/java/org/apache/datasketches/hll/AuxHashMap.java

@@ -19,12 +19,15 @@
 
 package org.apache.datasketches.hll;
 
+import java.lang.foreign.MemorySegment;
+
 import org.apache.datasketches.common.SketchesStateException;
+import org.apache.datasketches.common.MemorySegmentStatus;
 
 /**
  * @author Lee Rhodes
  */
-interface AuxHashMap {
+interface AuxHashMap extends MemorySegmentStatus {
 
   AuxHashMap copy();
 
@@ -40,10 +43,15 @@ interface AuxHashMap {
 
   int getUpdatableSizeBytes();
 
-  boolean isMemory();
+  @Override
+  boolean hasMemorySegment();
 
+  @Override
   boolean isOffHeap();
 
+  @Override
+  boolean isSameResource(MemorySegment seg);
+
   /**
    * Adds the slotNo and value to the aux array.
    * @param slotNo the index from the HLL array