Skip to content

Commit e1662cb

Browse files
leerholeerho
committed
Made some minor changes.
Some of the //TODOs were left over from Kevin's C code, so I left the comment but removed the TODO. Java can't do "INLINE CODE". This includes a new, very detailed test of the HLL growth transitions and sizes.
1 parent 588d585 commit e1662cb

7 files changed

Lines changed: 119 additions & 10 deletions

File tree

‎src/main/java/org/apache/datasketches/cpc/CpcCompression.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@ static long lowLevelCompressPairs(
288288
final long golombLo = yDelta & golombLoMask; //long for bitBuf
289289
final long golombHi = yDelta >>> numBaseBits; //cannot exceed 2^26
290290

291-
//TODO Inline WriteUnary
291+
// Inline WriteUnary
292292
ptrArr[NEXT_WORD_IDX] = nextWordIndex;
293293
ptrArr[BIT_BUF] = bitBuf;
294294
ptrArr[BUF_BITS] = bufBits;
@@ -372,7 +372,7 @@ static void lowLevelUncompressPairs(
372372
bitBuf >>>= codeWordLength;
373373
bufBits -= codeWordLength;
374374

375-
//TODO Inline ReadUnary
375+
// Inline ReadUnary
376376
ptrArr[NEXT_WORD_IDX] = nextWordIndex;
377377
ptrArr[BIT_BUF] = bitBuf;
378378
ptrArr[BUF_BITS] = bufBits;

‎src/main/java/org/apache/datasketches/hll/DirectCouponHashSet.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import static org.apache.datasketches.hll.PreambleUtil.insertLgArr;
3636

3737
import org.apache.datasketches.common.SketchesArgumentException;
38+
import org.apache.datasketches.common.SketchesException;
3839
import org.apache.datasketches.common.SketchesStateException;
3940
import org.apache.datasketches.memory.Memory;
4041
import org.apache.datasketches.memory.WritableMemory;
@@ -105,13 +106,14 @@ private boolean checkGrowOrPromote() {
105106
if (lgCouponArrInts == (getLgConfigK() - 3)) {
106107
return true; // promote
107108
}
108-
//TODO if direct, ask for more memory
109109
insertLgArr(wmem, ++lgCouponArrInts);
110110
growHashSet(wmem, lgCouponArrInts);
111111
}
112112
return false;
113113
}
114114

115+
//This could fail if the user has undersized the given WritableMemory
116+
// and not used the public methods for sizing the Memory. See exception.
115117
private static final void growHashSet(final WritableMemory wmem, final int tgtLgCouponArrSize) {
116118
final int tgtArrSize = 1 << tgtLgCouponArrSize;
117119
final int[] tgtCouponIntArr = new int[tgtArrSize];
@@ -128,7 +130,11 @@ private static final void growHashSet(final WritableMemory wmem, final int tgtLg
128130
}
129131
}
130132
wmem.clear(HASH_SET_INT_ARR_START, tgtArrSize << 2);
131-
wmem.putIntArray(HASH_SET_INT_ARR_START, tgtCouponIntArr, 0, tgtArrSize);
133+
try { wmem.putIntArray(HASH_SET_INT_ARR_START, tgtCouponIntArr, 0, tgtArrSize); }
134+
catch (final IndexOutOfBoundsException e) {
135+
throw new SketchesException(
136+
"The WritableMemory is undersized. Use the public methods for properly sizing Memory.", e);
137+
}
132138
}
133139

134140
//Searches the Coupon hash table (embedded in Memory) for an empty slot

‎src/main/java/org/apache/datasketches/hll/HeapAuxHashMap.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,6 @@ else if (slotNo == (arrVal & configKmask)) { //Compares only on slotNo
205205
private void checkGrow() {
206206
if ((RESIZE_DENOM * auxCount) > (RESIZE_NUMER * auxIntArr.length)) {
207207
growAuxSpace();
208-
//TODO if direct, ask for more memory
209208
}
210209
}
211210

‎src/test/java/org/apache/datasketches/cpc/CpcCompressionTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ public void checkWriteReadUnary() {
5656

5757
for (int i = 0; i < 100; i++) {
5858

59-
//TODO Inline WriteUnary
59+
// Inline WriteUnary
6060
ptrArr[NEXT_WORD_IDX] = nextWordIndex;
6161
ptrArr[BIT_BUF] = bitBuf;
6262
ptrArr[BUF_BITS] = bufBits;
@@ -92,7 +92,7 @@ public void checkWriteReadUnary() {
9292

9393
for (int i = 0; i < 100; i++) {
9494

95-
//TODO Inline ReadUnary
95+
// Inline ReadUnary
9696
ptrArr[NEXT_WORD_IDX] = nextWordIndex;
9797
ptrArr[BIT_BUF] = bitBuf;
9898
ptrArr[BUF_BITS] = bufBits;

‎src/test/java/org/apache/datasketches/hll/CouponListTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ private static void checkIterator(boolean direct, int lgK, int n) {
4242
TgtHllType tgtHllType = TgtHllType.HLL_4;
4343
int bytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
4444
WritableMemory wmem = WritableMemory.allocate(bytes);
45-
HllSketch sk = (direct) ? new HllSketch(lgK, tgtHllType, wmem) : new HllSketch(lgK);
45+
HllSketch sk = (direct) ? new HllSketch(lgK, tgtHllType, wmem) : new HllSketch(lgK, tgtHllType);
4646
for (int i = 0; i < n; i++) { sk.update(i); }
4747
String store = direct ? "Memory" : "Heap";
4848
println("CurMode: " + sk.getCurMode().toString() + "; Store: " + store);

‎src/test/java/org/apache/datasketches/hll/SizeAndModeTransitions.java‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datasketches.hll;
21+
22+
import static org.testng.Assert.assertEquals;
23+
24+
import org.apache.datasketches.memory.WritableMemory;
25+
import org.testng.annotations.Test;
26+
27+
public class SizeAndModeTransitions {
28+
static String hfmt = "%6s %7s %4s %10s %5s %10s %10s %10s %10s %10s %14s\n";
29+
static String dfmt = "%6s %7s %4d %,10d %5s %,10d %,10d %,10d %,10d %,10d %,14.3f\n";
30+
static String[] hdr = {"Type", "Store", "LgK", "N", "Mode", "ActCBytes", "CmpBytes", "ActUBytes", "UpdBytes", "MaxBytes", "Estimate"};
31+
32+
@Test
33+
public void checkHLL8Heap() {
34+
TgtHllType tgtHllType = TgtHllType.HLL_8;
35+
boolean direct = false;
36+
int lgK = 10;
37+
int N = 97;
38+
printf(hfmt, (Object[]) hdr);
39+
int maxBytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
40+
WritableMemory wmem = null;;
41+
HllSketch sk;
42+
if (direct) {
43+
wmem = WritableMemory.allocate(maxBytes);
44+
sk = new HllSketch(lgK, tgtHllType, wmem);
45+
} else {
46+
sk = new HllSketch(lgK, tgtHllType);
47+
}
48+
String type = tgtHllType.toString();
49+
String store = direct ? "Memory" : "Heap";
50+
for (int i = 1; i <= N; i++) {
51+
sk.update(i);
52+
if (i == 7) { checkAtN(sk, tgtHllType, store, lgK, i, "LIST", 36, 40, 1064); }
53+
if (i == 8) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 44, 140, 1064); }
54+
if (i == 24) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 108, 140, 1064); }
55+
if (i == 25) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 112, 268, 1064); }
56+
if (i == 48) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 204, 268, 1064); }
57+
if (i == 49) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 208, 524, 1064); }
58+
if (i == 96) { checkAtN(sk, tgtHllType, store, lgK, i, "SET", 396, 524, 1064); }
59+
if (i == 97) { checkAtN(sk, tgtHllType, store, lgK, i, "HLL", 1064, 1064, 1064); }
60+
}
61+
println("");
62+
}
63+
64+
private static void checkAtN(HllSketch sk, TgtHllType tgtHllType, String store, int lgK, int n, String trueMode,
65+
int cmpTrueBytes, int updTrueBytes, int maxTrueBytes) {
66+
int maxBytes = HllSketch.getMaxUpdatableSerializationBytes(lgK, tgtHllType);
67+
String type = tgtHllType.toString();
68+
int cmpBytes = sk.getCompactSerializationBytes();
69+
int updBytes = sk.getUpdatableSerializationBytes();
70+
byte[] actCBytes = sk.toCompactByteArray();
71+
byte[] actUBytes = sk.toUpdatableByteArray();
72+
String mode = sk.getCurMode().toString();
73+
double est = sk.getEstimate();
74+
printf(dfmt, type, store, lgK, n, mode, actCBytes.length, cmpBytes, actUBytes.length, updBytes, maxBytes, est);
75+
assertEquals(mode, trueMode);
76+
assertEquals(cmpBytes, actCBytes.length);
77+
assertEquals(cmpBytes, cmpTrueBytes);
78+
assertEquals(updBytes, actUBytes.length);
79+
assertEquals(updBytes, updTrueBytes);
80+
assertEquals(maxBytes, maxTrueBytes);
81+
}
82+
83+
@Test
84+
public void printlnTest() {
85+
println("PRINTING: "+this.getClass().getName());
86+
}
87+
88+
/**
89+
* @param s value to print
90+
*/
91+
static void println(String s) {
92+
System.out.println(s); //disable here
93+
}
94+
95+
/**
96+
* @param fmt format
97+
* @param args arguments
98+
*/
99+
static void printf(String fmt, Object...args) {
100+
System.out.printf(fmt, args); //disable here
101+
}
102+
103+
}

‎src/test/java/org/apache/datasketches/quantiles/DirectQuantilesMemoryRequestTest.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,8 @@ public void checkUpdatableStorageBytes() {
125125
usk1.update(1.0);
126126
byte[] uarr = usk1.toByteArray();
127127
println("Actual Storage Bytes " + uarr.length);
128-
assertEquals(initBytes, uarr.length); //64
128+
assertEquals(initBytes, uarr.length);
129+
assertEquals(initBytes, 64);
129130
}
130131

131132

@@ -151,7 +152,7 @@ public void checkGrowFromWrappedEmptySketch() {
151152
assertEquals(usk2.getN(), 1);
152153
WritableMemory mem2 = usk2.getMemory();
153154
assertFalse(wmem.isSameResource(mem2));
154-
assertFalse(mem2.isDirect()); //should now be on-heap //TODO
155+
assertFalse(mem2.isDirect()); //should now be on-heap
155156

156157
final int expectedSize = COMBINED_BUFFER + ((2 * k) << 3);
157158
assertEquals(mem2.getCapacity(), expectedSize);

0 commit comments

Comments
 (0)