Skip to content

Commit 2f31a88

Browse files
leerholeerho
authored
Merge pull request #675 from apache/ffm_phase7
FFM phase7: Classic Quantiles + cleanup
2 parents f62d552 + ed48b79 commit 2f31a88

125 files changed

Lines changed: 2802 additions & 3777 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎pom.xml‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -96,10 +96,10 @@ under the License.
9696
<!-- System-wide properties -->
9797
<maven.version>3.9.10</maven.version>
9898
<java.version>24</java.version>
99-
<jvm-arguments></jvm-arguments>
99+
<jvm-arguments>-Xmx10g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8</jvm-arguments>
100100
<maven.compiler.source>${java.version}</maven.compiler.source>
101101
<maven.compiler.target>${java.version}</maven.compiler.target>
102-
<argLine>-Xmx4g -Duser.language=en -Duser.country=US -Dfile.encoding=UTF-8 ${jvm-arguments}</argLine>
102+
<argLine>${jvm-arguments}</argLine>
103103
<charset.encoding>UTF-8</charset.encoding>
104104
<project.build.sourceEncoding>${charset.encoding}</project.build.sourceEncoding>
105105
<project.build.resourceEncoding>${charset.encoding}</project.build.resourceEncoding>
@@ -135,12 +135,6 @@ under the License.
135135
</properties>
136136

137137
<dependencies>
138-
<dependency>
139-
<groupId>org.apache.datasketches</groupId>
140-
<artifactId>datasketches-memory</artifactId>
141-
<version>${datasketches-memory.version}</version>
142-
</dependency>
143-
144138
<!-- Test Scope -->
145139
<dependency>
146140
<groupId>org.testng</groupId>
@@ -291,7 +285,7 @@ under the License.
291285
<useManifestOnlyJar>false</useManifestOnlyJar>
292286
<redirectTestOutputToFile>true</redirectTestOutputToFile>
293287
<reportsDirectory>${project.build.directory}/test-output/${maven.build.timestamp}</reportsDirectory>
294-
<excludedGroups>${testng.generate-java-files},${testng.check-cpp-files}</excludedGroups>
288+
<excludedGroups>${testng.generate-java-files},${testng.check-cpp-files},${testng.check-go-files},${testng.check-cpp-historical-files}</excludedGroups>
295289
</configuration>
296290
</plugin>
297291

@@ -663,8 +657,9 @@ under the License.
663657
<groupId>org.apache.maven.plugins</groupId>
664658
<artifactId>maven-surefire-plugin</artifactId>
665659
<configuration>
666-
<groups>${testng.check-cpp-files}</groups>
667-
<excludedGroups>${testng.generate-java-files},${testng.check-cpp-files},${testng.check-cpp-historical-files}</excludedGroups>
660+
<groups>${testng.check-go-files}</groups>
661+
<excludedGroups>${testng.generate-java-files},${testng.check-cpp-files},${testng.check-cpp-historical-files}
662+
</excludedGroups>
668663
</configuration>
669664
</plugin>
670665
</plugins>

‎src/main/java/org/apache/datasketches/common/ArrayOfBooleansSerDe.java‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@
1919

2020
package org.apache.datasketches.common;
2121

22-
import java.util.Objects;
22+
import static java.lang.foreign.ValueLayout.JAVA_BYTE;
2323

24-
import org.apache.datasketches.memory.Memory;
25-
import org.apache.datasketches.memory.WritableMemory;
24+
import java.lang.foreign.MemorySegment;
25+
import java.util.Objects;
2626

2727
/**
2828
* Methods of serializing and deserializing arrays of Boolean as a bit array.
@@ -54,42 +54,42 @@ public byte[] serializeToByteArray(final Boolean[] items) {
5454
Objects.requireNonNull(items, "Items must not be null");
5555
final int bytesNeeded = computeBytesNeeded(items.length);
5656
final byte[] bytes = new byte[bytesNeeded];
57-
final WritableMemory mem = WritableMemory.writableWrap(bytes);
57+
final MemorySegment seg = MemorySegment.ofArray(bytes);
5858

5959
byte val = 0;
6060
for (int i = 0; i < items.length; ++i) {
6161
if (items[i]) {
6262
val |= 0x1 << (i & 0x7);
6363
}
6464
if ((i & 0x7) == 0x7) {
65-
mem.putByte(i >>> 3, val);
65+
seg.set(JAVA_BYTE, i >>> 3, val);
6666
val = 0;
6767
}
6868
}
6969
// write out any remaining values (if val=0, still good to be explicit)
7070
if ((items.length & 0x7) > 0) {
71-
mem.putByte(bytesNeeded - 1, val);
71+
seg.set(JAVA_BYTE, bytesNeeded - 1, val);
7272
}
7373
return bytes;
7474
}
7575

7676
@Override
77-
public Boolean[] deserializeFromMemory(final Memory mem, final int numItems) {
78-
return deserializeFromMemory(mem, 0, numItems);
77+
public Boolean[] deserializeFromMemorySegment(final MemorySegment seg, final int numItems) {
78+
return deserializeFromMemorySegment(seg, 0, numItems);
7979
}
8080

8181
@Override
82-
public Boolean[] deserializeFromMemory(final Memory mem, final long offsetBytes, final int numItems) {
83-
Objects.requireNonNull(mem, "Memory must not be null");
82+
public Boolean[] deserializeFromMemorySegment(final MemorySegment seg, final long offsetBytes, final int numItems) {
83+
Objects.requireNonNull(seg, "MemorySegment must not be null");
8484
if (numItems <= 0) { return new Boolean[0]; }
8585
final int numBytes = computeBytesNeeded(numItems);
86-
Util.checkBounds(offsetBytes, numBytes, mem.getCapacity());
86+
Util.checkBounds(offsetBytes, numBytes, seg.byteSize());
8787
final Boolean[] array = new Boolean[numItems];
8888

8989
byte srcVal = 0;
9090
for (int i = 0, b = 0; i < numItems; ++i) {
9191
if ((i & 0x7) == 0x0) { // should trigger on first iteration
92-
srcVal = mem.getByte(offsetBytes + b++);
92+
srcVal = seg.get(JAVA_BYTE, offsetBytes + b++);
9393
}
9494
array[i] = ((srcVal >>> (i & 0x7)) & 0x1) == 1;
9595
}
@@ -109,8 +109,8 @@ public int sizeOf(final Boolean[] items) {
109109
}
110110

111111
@Override
112-
public int sizeOf(final Memory mem, final long offsetBytes, final int numItems) {
113-
Objects.requireNonNull(mem, "Memory must not be null");
112+
public int sizeOf(final MemorySegment seg, final long offsetBytes, final int numItems) {
113+
Objects.requireNonNull(seg, "MemorySegment must not be null");
114114
return computeBytesNeeded(numItems);
115115
}
116116

‎src/main/java/org/apache/datasketches/common/ArrayOfBooleansSerDe2.java‎

Lines changed: 0 additions & 125 deletions
This file was deleted.

‎src/main/java/org/apache/datasketches/common/ArrayOfDoublesSerDe.java‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,12 @@
1919

2020
package org.apache.datasketches.common;
2121

22+
import static java.lang.foreign.ValueLayout.JAVA_DOUBLE_UNALIGNED;
2223
import static org.apache.datasketches.common.ByteArrayUtil.putDoubleLE;
2324

25+
import java.lang.foreign.MemorySegment;
2426
import java.util.Objects;
2527

26-
import org.apache.datasketches.memory.Memory;
27-
import org.apache.datasketches.memory.WritableMemory;
28-
2928
/**
3029
* Methods of serializing and deserializing arrays of Double.
3130
*
@@ -46,30 +45,30 @@ public byte[] serializeToByteArray(final Double[] items) {
4645
Objects.requireNonNull(items, "Items must not be null");
4746
if (items.length == 0) { return new byte[0]; }
4847
final byte[] bytes = new byte[Double.BYTES * items.length];
49-
final WritableMemory mem = WritableMemory.writableWrap(bytes);
48+
final MemorySegment seg = MemorySegment.ofArray(bytes);
5049
long offset = 0;
5150
for (int i = 0; i < items.length; i++) {
52-
mem.putDouble(offset, items[i]);
51+
seg.set(JAVA_DOUBLE_UNALIGNED, offset, items[i]);
5352
offset += Double.BYTES;
5453
}
5554
return bytes;
5655
}
5756

5857
@Override
59-
public Double[] deserializeFromMemory(final Memory mem, final int numItems) {
60-
return deserializeFromMemory(mem, 0, numItems);
58+
public Double[] deserializeFromMemorySegment(final MemorySegment seg, final int numItems) {
59+
return deserializeFromMemorySegment(seg, 0, numItems);
6160
}
6261

6362
@Override
64-
public Double[] deserializeFromMemory(final Memory mem, final long offsetBytes, final int numItems) {
65-
Objects.requireNonNull(mem, "Memory must not be null");
63+
public Double[] deserializeFromMemorySegment(final MemorySegment seg, final long offsetBytes, final int numItems) {
64+
Objects.requireNonNull(seg, "MemorySegment must not be null");
6665
if (numItems <= 0) { return new Double[0]; }
6766
long offset = offsetBytes;
68-
Util.checkBounds(offset, Double.BYTES * (long)numItems, mem.getCapacity());
67+
Util.checkBounds(offset, Double.BYTES * (long)numItems, seg.byteSize());
6968
final Double[] array = new Double[numItems];
7069

7170
for (int i = 0; i < numItems; i++) {
72-
array[i] = mem.getDouble(offset);
71+
array[i] = seg.get(JAVA_DOUBLE_UNALIGNED, offset);
7372
offset += Double.BYTES;
7473
}
7574
return array;
@@ -88,8 +87,8 @@ public int sizeOf(final Double[] items) {
8887
}
8988

9089
@Override
91-
public int sizeOf(final Memory mem, final long offsetBytes, final int numItems) {
92-
Objects.requireNonNull(mem, "Memory must not be null");
90+
public int sizeOf(final MemorySegment seg, final long offsetBytes, final int numItems) {
91+
Objects.requireNonNull(seg, "MemorySegment must not be null");
9392
return numItems * Double.BYTES;
9493
}
9594

0 commit comments

Comments
 (0)