Merge pull request #673 from apache/ffm_phase5

FFM Phase 5: sketches: req, tdigest

Author: leerho

src/main/java/org/apache/datasketches/common/SpecialValueLayouts.java

@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datasketches.common;
+
+import java.lang.foreign.ValueLayout;
+import java.nio.ByteOrder;
+
+/**
+ * Value Layouts for Non-native Endianness
+ */
+public final class SpecialValueLayouts {
+
+  private SpecialValueLayouts() { }
+
+  /**
+   * The static final for NON <i>ByteOrder.nativeOrder()</i>.
+   */
+  public static final ByteOrder NON_NATIVE_BYTE_ORDER =
+      (ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN) ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN;
+
+  //Non-Native Endian Layouts
+
+  /**
+   * The static final for NON <i>ByteOrder.nativeOrder() char</i>.
+   */
+  public static final ValueLayout.OfChar JAVA_CHAR_UNALIGNED_NON_NATIVE =
+      ValueLayout.JAVA_CHAR_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER);
+
+  /**
+   * The static final for NON <i>ByteOrder.nativeOrder() double</i>.
+   */
+  public static final ValueLayout.OfDouble JAVA_DOUBLE_UNALIGNED_NON_NATIVE =
+      ValueLayout.JAVA_DOUBLE_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER);
+
+  /**
+   * The static final for NON <i>ByteOrder.nativeOrder() float</i>.
+   */
+  public static final ValueLayout.OfFloat JAVA_FLOAT_UNALIGNED_NON_NATIVE =
+      ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER);
+
+  /**
+   * The static final for NON <i>ByteOrder.nativeOrder() int</i>.
+   */
+  public static final ValueLayout.OfInt JAVA_INT_UNALIGNED_NON_NATIVE =
+      ValueLayout.JAVA_INT_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER);
+
+  /**
+   * The static final for NON <i>ByteOrder.nativeOrder() long</i>.
+   */
+  public static final ValueLayout.OfLong JAVA_LONG_UNALIGNED_NON_NATIVE =
+      ValueLayout.JAVA_LONG_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER);
+
+  /**
+   * The static final for NON <i>ByteOrder.nativeOrder() short</i>.
+   */
+  public static final ValueLayout.OfShort JAVA_SHORT_UNALIGNED_NON_NATIVE =
+      ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(NON_NATIVE_BYTE_ORDER);
+
+  //Big-Endian Layouts
+
+  /**
+   * The static final for <i>ByteOrder.BIG_ENDIAN char</i>.
+   */
+  public static final ValueLayout.OfChar JAVA_CHAR_UNALIGNED_BIG_ENDIAN =
+      ValueLayout.JAVA_CHAR_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN);
+
+  /**
+   * The static final for <i>ByteOrder.BIG_ENDIAN double</i>.
+   */
+  public static final ValueLayout.OfDouble JAVA_DOUBLE_UNALIGNED_BIG_ENDIAN =
+      ValueLayout.JAVA_DOUBLE_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN);
+
+  /**
+   * The static final for <i>ByteOrder.BIG_ENDIAN float</i>.
+   */
+  public static final ValueLayout.OfFloat JAVA_FLOAT_UNALIGNED_BIG_ENDIAN =
+      ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN);
+
+  /**
+   * The static final for <i>ByteOrder.BIG_ENDIAN int</i>.
+   */
+  public static final ValueLayout.OfInt JAVA_INT_UNALIGNED_BIG_ENDIAN =
+      ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN);
+
+  /**
+   * The static final for <i>ByteOrder.BIG_ENDIAN long</i>.
+   */
+  public static final ValueLayout.OfLong JAVA_LONG_UNALIGNED_BIG_ENDIAN =
+      ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN);
+
+  /**
+   * The static final for <i>ByteOrder.BIG_ENDIAN short</i>.
+   */
+  public static final ValueLayout.OfShort JAVA_SHORT_UNALIGNED_BIG_ENDIAN =
+      ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.BIG_ENDIAN);
+
+}

src/main/java/org/apache/datasketches/frequencies/ItemsSketch.java

@@ -278,8 +278,9 @@ public static <T> ItemsSketch<T> getInstance(final MemorySegment srcSeg,
 
     //Get countArray
     final long[] countArray = new long[activeItems];
-    final int reqBytes = preBytes + (activeItems * Long.BYTES); //count Arr only
+    final long reqBytes = (preBytes + ((long)activeItems * Long.BYTES)); //count Arr only
     checkBounds(0, reqBytes, srcSeg.byteSize()); //check MemorySegment capacity
+
     MemorySegment.copy(srcSeg, JAVA_LONG_UNALIGNED, preBytes, countArray, 0, activeItems);
 
     //Get itemArray
@@ -530,7 +531,7 @@ public byte[] toByteArray(final ArrayOfItemsSerDe2<T> serDe) {
 
       final int preBytes = preLongs << 3;
       MemorySegment.copy(hashMap.getActiveValues(), 0, seg, JAVA_LONG_UNALIGNED, preBytes, activeItems);
-      MemorySegment.copy(bytes, 0, seg, JAVA_BYTE, preBytes + (this.getNumActiveItems() << 3), bytes.length);
+      MemorySegment.copy(bytes, 0, seg, JAVA_BYTE, preBytes + (activeItems << 3), bytes.length);
     }
     return outArr;
   }

src/main/java/org/apache/datasketches/req/FloatBuffer.java

@@ -22,11 +22,11 @@
 import static org.apache.datasketches.common.Util.LS;
 import static org.apache.datasketches.quantilescommon.QuantileSearchCriteria.INCLUSIVE;
 
+import java.lang.foreign.MemorySegment;
 import java.util.Arrays;
 
+import org.apache.datasketches.common.positional.PositionalSegment;
 import org.apache.datasketches.common.SketchesArgumentException;
-import org.apache.datasketches.memory.WritableBuffer;
-import org.apache.datasketches.memory.WritableMemory;
 import org.apache.datasketches.quantilescommon.InequalitySearch;
 import org.apache.datasketches.quantilescommon.QuantileSearchCriteria;
 
@@ -176,7 +176,7 @@ FloatBuffer ensureCapacity(final int newCapacity) {
    * @return this
    */
   private FloatBuffer ensureSpace(final int space) {
-    if (count_ + space > capacity_) {
+    if ((count_ + space) > capacity_) {
       final int newCap = count_ + space + delta_;
       ensureCapacity(newCap);
     }
@@ -219,7 +219,7 @@ int getCountWithCriterion(final float item, final QuantileSearchCriteria searchC
     }
     final InequalitySearch crit = (searchCrit == INCLUSIVE) ? InequalitySearch.LE : InequalitySearch.LT;
     final int index = InequalitySearch.find(arr_, low, high, item, crit);
-    return index == -1 ? 0 : index - low + 1;
+    return index == -1 ? 0 : (index - low) + 1;
   }
 
   /**
@@ -234,8 +234,8 @@ int getCountWithCriterion(final float item, final QuantileSearchCriteria searchC
    * @return the selected odds from the range
    */
   FloatBuffer getEvensOrOdds(final int startOffset, final int endOffset, final boolean odds) {
-    final int start = spaceAtBottom_ ? capacity_ - count_ + startOffset : startOffset;
-    final int end = spaceAtBottom_ ? capacity_ - count_ + endOffset : endOffset;
+    final int start = spaceAtBottom_ ? (capacity_ - count_) + startOffset : startOffset;
+    final int end = spaceAtBottom_ ? (capacity_ - count_) + endOffset : endOffset;
     sort();
     final int range = endOffset - startOffset;
     if ((range & 1) == 1) {
@@ -265,7 +265,7 @@ float getItemFromIndex(final int index) {
    * @return an item given its offset
    */
   float getItem(final int offset) {
-    final int index = spaceAtBottom_ ? capacity_ - count_ + offset : offset;
+    final int index = spaceAtBottom_ ? (capacity_ - count_) + offset : offset;
     return arr_[index];
   }
 
@@ -317,11 +317,11 @@ boolean isEmpty() {
    * @return true iff this is exactly equal to that FloatBuffer.
    */
   boolean isEqualTo(final FloatBuffer that) {
-    if (capacity_ != that.capacity_
-        || count_ != that.count_
-        || delta_ != that.delta_
-        || sorted_ != that.sorted_
-        || spaceAtBottom_ != that.spaceAtBottom_) { return false; }
+    if ((capacity_ != that.capacity_)
+        || (count_ != that.count_)
+        || (delta_ != that.delta_)
+        || (sorted_ != that.sorted_)
+        || (spaceAtBottom_ != that.spaceAtBottom_)) { return false; }
     for (int i = 0; i < capacity_; i++) {
       if (arr_[i] != that.arr_[i]) { return false; }
     }
@@ -354,7 +354,7 @@ FloatBuffer mergeSortIn(final FloatBuffer bufIn) {
       int i = capacity_ - count_;
       int j = bufIn.capacity_ - bufIn.count_;
       for (int k = tgtStart; k < capacity_; k++) {
-        if (i < capacity_ && j < bufIn.capacity_) { //both valid
+        if ((i < capacity_) && (j < bufIn.capacity_)) { //both valid
           arr_[k] = arr_[i] <= arrIn[j] ? arr_[i++] : arrIn[j++];
         } else if (i < capacity_) { //i is valid
           arr_[k] = arr_[i++];
@@ -368,7 +368,7 @@ FloatBuffer mergeSortIn(final FloatBuffer bufIn) {
       int i = count_ - 1;
       int j = bufInLen - 1;
       for (int k = totLen; k-- > 0; ) {
-        if (i >= 0 && j >= 0) { //both valid
+        if ((i >= 0) && (j >= 0)) { //both valid
           arr_[k] = arr_[i] >= arrIn[j] ? arr_[i--] : arrIn[j--];
         } else if (i >= 0) { //i is valid
           arr_[k] = arr_[i--];
@@ -401,18 +401,18 @@ FloatBuffer sort() {
   byte[] floatsToBytes() {
     final int bytes = Float.BYTES * count_;
     final byte[] arr = new byte[bytes];
-    final WritableBuffer wbuf = WritableMemory.writableWrap(arr).asWritableBuffer();
+    final PositionalSegment posSeg = PositionalSegment.wrap(MemorySegment.ofArray(arr));
     if (spaceAtBottom_) {
-      wbuf.putFloatArray(arr_, capacity_ - count_, count_);
+      posSeg.setFloatArray(arr_, capacity_ - count_, count_);
     } else {
-      wbuf.putFloatArray(arr_, 0, count_);
+      posSeg.setFloatArray(arr_, 0, count_);
     }
-    assert wbuf.getPosition() == bytes;
+    assert posSeg.getPosition() == bytes;
     return arr;
   }
 
   /**
-   * Returns a printable formatted string of the items of this buffer separated by a single space.
+   * Returns a printable formatted string of the items of this FloatBuffer separated by a single space.
    * @param fmt The format for each printed item.
    * @param width the number of items to print per line
    * @return a printable, formatted string of the items of this buffer.
@@ -427,7 +427,7 @@ String toHorizList(final String fmt, final int width) {
     for (int i = start; i < end; i++) {
       final float v = arr_[i];
       final String str = String.format(fmt, v);
-      if (i > start && ++cnt % width == 0) { sb.append(LS).append(spaces); }
+      if ((i > start) && ((++cnt % width) == 0)) { sb.append(LS).append(spaces); }
       sb.append(str);
     }
     return sb.toString();

src/main/java/org/apache/datasketches/req/ReqCompactor.java

@@ -25,10 +25,10 @@
 import static org.apache.datasketches.req.ReqSketch.MIN_K;
 import static org.apache.datasketches.req.ReqSketch.NOM_CAP_MULT;
 
+import java.lang.foreign.MemorySegment;
 import java.util.Random;
 
-import org.apache.datasketches.memory.WritableBuffer;
-import org.apache.datasketches.memory.WritableMemory;
+import org.apache.datasketches.common.positional.PositionalSegment;
 import org.apache.datasketches.req.ReqSketch.CompactorReturn;
 
 /**
@@ -122,7 +122,7 @@ FloatBuffer compact(final CompactorReturn cReturn, final Random rand) {
     final long compactionRange = computeCompactionRange(secsToCompact);
     final int compactionStart = (int) (compactionRange & 0xFFFF_FFFFL); //low 32
     final int compactionEnd = (int) (compactionRange >>> 32); //high 32
-    assert compactionEnd - compactionStart >= 2;
+    assert (compactionEnd - compactionStart) >= 2;
 
     if ((state & 1L) == 1L) { coin = !coin; } //if numCompactions odd, flip coin;
     else { coin = rand.nextBoolean(); }       //random coin flip
@@ -137,7 +137,7 @@ FloatBuffer compact(final CompactorReturn cReturn, final Random rand) {
     buf.trimCount(buf.getCount() - (compactionEnd - compactionStart));
     state += 1;
     ensureEnoughSections();
-    cReturn.deltaRetItems = buf.getCount() - startRetItems + promote.getCount();
+    cReturn.deltaRetItems = (buf.getCount() - startRetItems) + promote.getCount();
     cReturn.deltaNomSize = getNomCapacity() - startNomCap;
     if (reqDebug != null) { reqDebug.emitCompactionDone(lgWeight); }
     return promote;
@@ -175,7 +175,7 @@ final int getNomCapacity() { //called from constructor
    */
   int getSerializationBytes() {
     final int count = buf.getCount();
-    return 8 + 4 + 1 + 1 + 2 + 4 + count * Float.BYTES; // 20 + array
+    return 8 + 4 + 1 + 1 + 2 + 4 + (count * Float.BYTES); // 20 + array
   }
 
   int getNumSections() {
@@ -227,9 +227,9 @@ ReqCompactor merge(final ReqCompactor other) {
   private boolean ensureEnoughSections() {
     final float szf;
     final int ne;
-    if (state >= 1L << numSections - 1
-        && sectionSize > MIN_K
-        && (ne = nearestEven(szf = (float)(sectionSizeFlt / SQRT2))) >= MIN_K)
+    if ((state >= (1L << (numSections - 1)))
+        && (sectionSize > MIN_K)
+        && ((ne = nearestEven(szf = (float)(sectionSizeFlt / SQRT2))) >= MIN_K))
     {
       sectionSizeFlt = szf;
       sectionSize = ne;
@@ -248,9 +248,9 @@ private boolean ensureEnoughSections() {
    */
   private long computeCompactionRange(final int secsToCompact) {
     final int bufLen = buf.getCount();
-    int nonCompact = getNomCapacity() / 2 + (numSections - secsToCompact) * sectionSize;
+    int nonCompact = (getNomCapacity() / 2) + ((numSections - secsToCompact) * sectionSize);
     //make compacted region even:
-    nonCompact = (bufLen - nonCompact & 1) == 1 ? nonCompact + 1 : nonCompact;
+    nonCompact = ((bufLen - nonCompact) & 1) == 1 ? nonCompact + 1 : nonCompact;
     final long low =  hra ? 0                   : nonCompact;
     final long high = hra ? bufLen - nonCompact : bufLen;
     return (high << 32) + low;
@@ -293,16 +293,16 @@ static final int nearestEven(final float fltVal) {
   byte[] toByteArray() {
     final int bytes = getSerializationBytes();
     final byte[] arr = new byte[bytes];
-    final WritableBuffer wbuf = WritableMemory.writableWrap(arr).asWritableBuffer();
-    wbuf.putLong(state);
-    wbuf.putFloat(sectionSizeFlt);
-    wbuf.putByte(lgWeight);
-    wbuf.putByte(numSections);
-    wbuf.incrementPosition(2); //pad 2
+    final PositionalSegment posSeg = PositionalSegment.wrap(MemorySegment.ofArray(arr));
+    posSeg.setLong(state);
+    posSeg.setFloat(sectionSizeFlt);
+    posSeg.setByte(lgWeight);
+    posSeg.setByte(numSections);
+    posSeg.incrementPosition(2); //pad 2
     //buf.sort(); //sort if necessary
-    wbuf.putInt(buf.getCount()); //count
-    wbuf.putByteArray(buf.floatsToBytes(), 0, Float.BYTES * buf.getCount());
-    assert wbuf.getPosition() == bytes;
+    posSeg.setInt(buf.getCount()); //count
+    posSeg.setByteArray(buf.floatsToBytes(), 0, Float.BYTES * buf.getCount());
+    assert posSeg.getPosition() == bytes;
     return arr;
   }
 
@@ -319,10 +319,9 @@ String toListPrefix() {
     final int secSz = getSectionSize();
     final int numSec = getNumSections();
     final long num = getState();
-    final String prefix = String.format(
+    return String.format(
       "  C:%d Len:%d NomSz:%d SecSz:%d NumSec:%d State:%d",
            h, len, nom, secSz, numSec, num);
-    return prefix;
   }
 
 }