Skip to content

Commit 904e357

Browse files
committed
Updates for v2 arch
1 parent 5bc06a7 commit 904e357

30 files changed

Lines changed: 2286 additions & 90 deletions

src/main/java/org/hdf5javalib/examples/hdf5examples/HDF5Debug.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,11 @@ public static void main(String[] args) {
3737
private void run() {
3838
try {
3939
// List all .h5 files in HDF5Examples resources directory
40-
Path dirPath = Paths.get(Objects.requireNonNull(HDF5Debug.class.getClassLoader().getResource("HDF5Examples/h5ex_g_traverse.h5")).toURI());
40+
// ATL03_20250302235544_11742607_006_01
41+
// Path dirPath = Paths.get(Objects.requireNonNull(HDF5Debug.class.getClassLoader().getResource("HDF5Examples/h5ex_g_compact2.h5")).toURI());
42+
Path dirPath = Paths.get("c:/users/karln/Downloads/ATL03_20250302235544_11742607_006_01.h5");
4143
displayFile(dirPath);
42-
} catch (URISyntaxException e) {
44+
} catch (Exception e) {
4345
throw new IllegalStateException(e);
4446
}
4547
}

src/main/java/org/hdf5javalib/hdffile/dataobjects/messages/AttributeMessage.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ public static HdfMessage parseHeaderMessage(int flags, byte[] data, HdfDataFile
146146
byte[] dataBytes = new byte[dtDataSize];
147147
buffer.get(dataBytes);
148148
HdfData scalarValue = dt.getHdfDatatype().getInstance(HdfData.class, dataBytes);
149-
return new AttributeMessage(version, name, dt, ds, HdfDataHolder.ofScalar(scalarValue), flags, (short) data.length);
149+
return new AttributeMessage(version, name, dt, ds, HdfDataHolder.ofScalar(scalarValue), flags, data.length);
150150
}
151151

152152
// Case 2: Array data (dimensionality is 1 or more)
@@ -167,7 +167,7 @@ public static HdfMessage parseHeaderMessage(int flags, byte[] data, HdfDataFile
167167
// Step 2: Populate the array recursively from the flat buffer.
168168
populateArray(multiDimArray, dimensions, 0, buffer, dt.getHdfDatatype(), dtDataSize);
169169

170-
return new AttributeMessage(version, name, dt, ds, HdfDataHolder.ofArray(multiDimArray, dimensions), flags, (short) data.length);
170+
return new AttributeMessage(version, name, dt, ds, HdfDataHolder.ofArray(multiDimArray, dimensions), flags, data.length);
171171

172172
}
173173

src/main/java/org/hdf5javalib/hdffile/dataobjects/messages/BTreeKValuesMessage.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ public static BTreeKValuesMessage parseHeaderMessage(int flags, byte[] data, Hdf
9090
int indexedStorageInternalNodeK = Short.toUnsignedInt(buffer.getShort());
9191
int groupInternalNodeK = Short.toUnsignedInt(buffer.getShort());
9292
int groupLeafNodeK = Short.toUnsignedInt(buffer.getShort());
93-
return new BTreeKValuesMessage(version, indexedStorageInternalNodeK, groupInternalNodeK, groupLeafNodeK, flags, (short) data.length);
93+
return new BTreeKValuesMessage(version, indexedStorageInternalNodeK, groupInternalNodeK, groupLeafNodeK, flags, data.length);
9494
}
9595

9696
/**

src/main/java/org/hdf5javalib/hdffile/dataobjects/messages/DataspaceMessage.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ public static HdfMessage parseHeaderMessage(int flags, byte[] data, HdfDataFile
123123
}
124124
}
125125

126-
return new DataspaceMessage(version, dimensionality, flagSet, dimensions, maxDimensions, hasMaxDimensions, flags, (short) data.length);
126+
return new DataspaceMessage(version, dimensionality, flagSet, dimensions, maxDimensions, hasMaxDimensions, flags, data.length);
127127
}
128128

129129
/**

src/main/java/org/hdf5javalib/hdffile/dataobjects/messages/DatatypeMessage.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public class DatatypeMessage extends HdfMessage {
7272
* @param flags message flags
7373
* @param sizeMessageData the size of the message data in bytes
7474
*/
75-
public DatatypeMessage(Datatype datatype, int flags, short sizeMessageData) {
75+
public DatatypeMessage(Datatype datatype, int flags, int sizeMessageData) {
7676
super(MessageType.DATATYPE_MESSAGE, sizeMessageData, flags);
7777
this.datatype = datatype;
7878
}
@@ -88,7 +88,7 @@ public DatatypeMessage(Datatype datatype, int flags, short sizeMessageData) {
8888
public static HdfMessage parseHeaderMessage(int flags, byte[] data, HdfDataFile hdfDataFile) {
8989
ByteBuffer buffer = ByteBuffer.wrap(data).order(ByteOrder.LITTLE_ENDIAN);
9090
Datatype datatype = getHdfDatatype(buffer, hdfDataFile);
91-
return new DatatypeMessage(datatype, flags, (short) data.length);
91+
return new DatatypeMessage(datatype, flags, data.length);
9292
}
9393

9494
/**

src/main/java/org/hdf5javalib/hdffile/dataobjects/messages/FillMessage.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public static HdfMessage parseHeaderMessage(int flags, byte[] data, HdfDataFile
8787
buffer.get(fillValue);
8888

8989
// Return a constructed instance of FillValueMessage
90-
return new FillMessage(size, fillValue, flags, (short) data.length);
90+
return new FillMessage(size, fillValue, flags, data.length);
9191
}
9292

9393
/**

src/main/java/org/hdf5javalib/hdffile/dataobjects/messages/FillValueMessage.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ public static HdfMessage parseHeaderMessage(int flags, byte[] data, HdfDataFile
139139
}
140140

141141
// Return a constructed instance of FillValueMessage
142-
return new FillValueMessage(version, spaceAllocationTime, fillValueWriteTime, fillValueDefined, size, fillValue, flags, (short) data.length);
142+
return new FillValueMessage(version, spaceAllocationTime, fillValueWriteTime, fillValueDefined, size, fillValue, flags, data.length);
143143
}
144144

145145
/**

src/main/java/org/hdf5javalib/hdffile/dataobjects/messages/HdfMessage.java

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import java.nio.ByteOrder;
99
import java.nio.channels.SeekableByteChannel;
1010
import java.util.ArrayList;
11+
import java.util.Arrays;
1112
import java.util.List;
1213
import java.util.function.Function;
1314

@@ -146,6 +147,18 @@ public static class OBJECT_HEADER_PREFIX {
146147
return new OBJECT_HEADER_PREFIX(type, size, flags, order);
147148
};
148149

150+
// public static Function<ByteBuffer, OBJECT_HEADER_PREFIX> V2_OBJECT_HEADER_READ_PREFIX_V2 = buffer-> {
151+
// // Header Message Type (2 bytes, little-endian)
152+
// MessageType type = MessageType.fromValue(buffer.get());
153+
// int size = Short.toUnsignedInt(buffer.getShort());
154+
// int flags = Byte.toUnsignedInt(buffer.get());
155+
// int order = 0;
156+
// if ( (flags & (1 << 2)) != 0 ) {
157+
// order = Short.toUnsignedInt(buffer.getShort());
158+
// }
159+
// return new OBJECT_HEADER_PREFIX(type, size, flags, order);
160+
// };
161+
149162
/**
150163
* Reads and parses a list of HdfMessages from the provided file channel.
151164
*
@@ -166,16 +179,37 @@ public static List<HdfMessage> readMessagesFromByteBuffer(
166179
buffer.flip();
167180
List<HdfMessage> messages = new ArrayList<>();
168181

169-
while (buffer.hasRemaining()) {
182+
byte[] OCHKSignature = new byte[4];
183+
buffer.get(OCHKSignature);
184+
if (Arrays.compare(OCHKSignature, "OCHK".getBytes()) != 0 ) {
185+
buffer.rewind();
186+
}
187+
188+
boolean readContinue = true;
189+
while (buffer.hasRemaining() && readContinue) {
170190
OBJECT_HEADER_PREFIX prefix = prefixFunction.apply(buffer);
171191
// Header Message Data
172192
byte[] messageData = new byte[prefix.size];
173193
buffer.get(messageData);
174194

195+
175196
HdfMessage hdfMessage = parseHeaderMessage(prefix.type, prefix.flags, messageData, hdfDataFile);
176197
log.trace("Read: hdfMessage.sizeMessageData() + HDF_MESSAGE_PREAMBLE_SIZE = {} {}", hdfMessage.messageType, hdfMessage.getSizeMessageData() + HDF_MESSAGE_PREAMBLE_SIZE);
177198
// Add the message to the list
178199
messages.add(hdfMessage);
200+
//TODO: no good.
201+
if ( buffer.hasRemaining() ) {
202+
if ( buffer.remaining() <= 4 ) {
203+
readContinue = false;
204+
} else {
205+
buffer.mark();
206+
MessageType type = MessageType.fromValue(buffer.get());
207+
buffer.reset();
208+
if ( buffer.remaining() < 10 ) {
209+
readContinue = false;
210+
}
211+
}
212+
}
179213
}
180214
return messages;
181215
}
@@ -230,7 +264,7 @@ public static List<HdfMessage> parseContinuationMessage(
230264
Function<ByteBuffer, OBJECT_HEADER_PREFIX> prefixFunction
231265
) throws IOException, InvocationTargetException, InstantiationException, IllegalAccessException {
232266
long continuationOffset = objectHeaderContinuationMessage.getContinuationOffset().getInstance(Long.class);
233-
short continuationSize = objectHeaderContinuationMessage.getContinuationSize().getInstance(Long.class).shortValue();
267+
long continuationSize = objectHeaderContinuationMessage.getContinuationSize().getInstance(Long.class);
234268

235269
// Move to the continuation block offset
236270
fileChannel.position(continuationOffset);
Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
package org.hdf5javalib.hdffile.infrastructure.fractalheap.gemini;
2+
3+
import java.io.IOException;
4+
import java.nio.channels.SeekableByteChannel;
5+
import java.util.Arrays;
6+
import java.util.HashMap;
7+
import java.util.Map;
8+
9+
//##############################################################################
10+
//### FRACTAL HEAP STRUCTURES
11+
//##############################################################################
12+
public class FractalHeap {
13+
public final FractalHeapHeader header;
14+
15+
// The rootBlock object is removed. This class is now a "handle".
16+
// private final Object rootBlock; // REMOVED
17+
18+
// Context needed for object retrieval
19+
private final SeekableByteChannel channel;
20+
private final int sizeOfOffsets;
21+
private final int sizeOfLengths;
22+
private final int maxDirectRows;
23+
24+
// Cache to store blocks that have already been read from disk.
25+
// Key: Block Address, Value: Block Object (Direct or Indirect)
26+
private final Map<Long, Object> blockCache;
27+
28+
/**
29+
* Helper class to return both address and size of a located direct block.
30+
*/
31+
private static class DirectBlockInfo {
32+
final long address;
33+
final long size;
34+
35+
DirectBlockInfo(long address, long size) {
36+
this.address = address;
37+
this.size = size;
38+
}
39+
}
40+
41+
// MODIFIED CONSTRUCTOR: No longer accepts a rootBlock object.
42+
public FractalHeap(FractalHeapHeader header, SeekableByteChannel channel, int sizeOfOffsets, int sizeOfLengths) {
43+
this.header = header;
44+
this.channel = channel;
45+
this.sizeOfOffsets = sizeOfOffsets;
46+
this.sizeOfLengths = sizeOfLengths;
47+
this.blockCache = new HashMap<>();
48+
49+
// Pre-calculate max rows for direct blocks in any indirect block
50+
int log2MaxDirect = Hdf5Utils.ceilLog2(header.maximumDirectBlockSize);
51+
int log2Start = Hdf5Utils.ceilLog2(header.startingBlockSize);
52+
this.maxDirectRows = (log2MaxDirect - log2Start) + 2;
53+
}
54+
55+
public byte[] getObject(byte[] rawId) throws IOException {
56+
if (header.addressOfRootBlock == Hdf5Utils.UNDEFINED_ADDRESS) {
57+
throw new IOException("Cannot get object; fractal heap has no root block.");
58+
}
59+
60+
ParsedHeapId id = new ParsedHeapId(rawId, this.header);
61+
62+
63+
switch (id.type) {
64+
case 0: // Managed Object
65+
return findManagedObject(id);
66+
case 1: // Huge Object
67+
throw new UnsupportedOperationException("Huge object retrieval not implemented.");
68+
case 2: // Tiny Object
69+
int headerBits = 8; // type(2), version(2), reserved(4)
70+
long minOfMaxs = Math.min(header.maximumDirectBlockSize, header.maximumSizeOfManagedObjects);
71+
if (minOfMaxs < (1 << 8)) {
72+
headerBits += 8;
73+
} else if (minOfMaxs < (1 << 16)) {
74+
headerBits += 16;
75+
} else if (minOfMaxs < (1 << 24)) {
76+
headerBits += 24;
77+
} else {
78+
headerBits += 32;
79+
}
80+
int dataOffsetInIdBytes = (headerBits + 7) / 8;
81+
return Arrays.copyOfRange(rawId, dataOffsetInIdBytes, dataOffsetInIdBytes + id.length);
82+
default:
83+
throw new IOException("Unknown heap object type: " + id.type);
84+
}
85+
}
86+
87+
private byte[] findManagedObject(ParsedHeapId id) throws IOException {
88+
// 1. Find the address and size of the direct block containing our object's offset.
89+
DirectBlockInfo blockInfo = findDirectBlockForOffset(id.offset);
90+
if (blockInfo.address == Hdf5Utils.UNDEFINED_ADDRESS) {
91+
throw new IOException("Could not locate direct block for heap offset: " + id.offset);
92+
}
93+
94+
// 2. Read that specific direct block USING THE CACHED HELPER.
95+
FractalHeapDirectBlock directBlock = getDirectBlock(blockInfo.address, blockInfo.size);
96+
97+
// 3. The heap ID offset is from the start of the heap. The block's offset is also from the start of the heap.
98+
// We need the offset *within* this block's data array.
99+
long offsetInBlock = id.offset - directBlock.blockOffset;
100+
101+
// 4. Sanity checks.
102+
if (offsetInBlock < 0 || offsetInBlock + id.length > directBlock.objectData.length) {
103+
throw new IOException("Heap ID points outside of the located direct block's bounds. HeapID: " + id +
104+
", Block Offset: " + directBlock.blockOffset + ", Calculated Offset in Block: " + offsetInBlock);
105+
}
106+
107+
// 5. Extract and return the object's data.
108+
return Arrays.copyOfRange(directBlock.objectData, (int) offsetInBlock, (int) (offsetInBlock + id.length));
109+
}
110+
111+
private DirectBlockInfo findDirectBlockForOffset(long targetOffset) throws IOException {
112+
if (header.currentRowsInRootIndirectBlock == 0) {
113+
// Case 1: The root is a direct block.
114+
if (targetOffset < header.startingBlockSize) {
115+
return new DirectBlockInfo(header.addressOfRootBlock, header.startingBlockSize);
116+
} else {
117+
throw new IOException("Target offset " + targetOffset + " is out of bounds for a direct root block of size " + header.startingBlockSize);
118+
}
119+
} else {
120+
// Case 2: The root is an indirect block.
121+
return findAddressInIndirectBlock(header.addressOfRootBlock, header.currentRowsInRootIndirectBlock, targetOffset);
122+
}
123+
}
124+
125+
private DirectBlockInfo findAddressInIndirectBlock(long indirectBlockAddr, int rows, long targetOffset) throws IOException {
126+
// Read the indirect block USING THE CACHED HELPER.
127+
FractalHeapIndirectBlock indirectBlock = getIndirectBlock(indirectBlockAddr, rows);
128+
129+
for (int i = 0; i < indirectBlock.childDirectBlockAddresses.length; i++) {
130+
long childStartOffset = indirectBlock.blockOffsets[i];
131+
132+
int row = i / header.tableWidth;
133+
long childBlockSize = header.startingBlockSize * (1L << row);
134+
long childEndOffset = childStartOffset + childBlockSize;
135+
136+
if (targetOffset >= childStartOffset && targetOffset < childEndOffset) {
137+
long childAddress = indirectBlock.childDirectBlockAddresses[i];
138+
if (childAddress == Hdf5Utils.UNDEFINED_ADDRESS) {
139+
throw new IOException("Found entry for offset " + targetOffset + " but the block address is undefined.");
140+
}
141+
return new DirectBlockInfo(childAddress, childBlockSize);
142+
}
143+
}
144+
145+
if (indirectBlock.childIndirectBlockAddresses.length > 0) {
146+
throw new UnsupportedOperationException("Searching for objects in nested indirect blocks is not yet implemented.");
147+
}
148+
149+
throw new IOException("Failed to find a direct block containing offset " + targetOffset);
150+
}
151+
152+
// --- Caching Helper Methods ---
153+
154+
private FractalHeapDirectBlock getDirectBlock(long address, long size) throws IOException {
155+
return (FractalHeapDirectBlock) blockCache.computeIfAbsent(address, addr -> {
156+
try {
157+
return FractalHeapDirectBlock.read(channel, (Long) addr, size, header, sizeOfOffsets);
158+
} catch (IOException e) {
159+
// Lambda expressions can't throw checked exceptions directly, so we wrap them.
160+
throw new RuntimeException(e);
161+
}
162+
});
163+
}
164+
165+
private FractalHeapIndirectBlock getIndirectBlock(long address, int rows) throws IOException {
166+
try {
167+
return (FractalHeapIndirectBlock) blockCache.computeIfAbsent(address, addr -> {
168+
try {
169+
return FractalHeapIndirectBlock.read(channel, (Long) addr, rows, header, sizeOfOffsets, sizeOfLengths);
170+
} catch (IOException e) {
171+
throw new RuntimeException(e);
172+
}
173+
});
174+
} catch(RuntimeException e) {
175+
// Unwrap the original IOException
176+
if (e.getCause() instanceof IOException) {
177+
throw (IOException) e.getCause();
178+
}
179+
throw e;
180+
}
181+
}
182+
}
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package org.hdf5javalib.hdffile.infrastructure.fractalheap.gemini;
2+
3+
import java.io.IOException;
4+
import java.nio.ByteBuffer;
5+
import java.nio.ByteOrder;
6+
import java.nio.channels.SeekableByteChannel;
7+
8+
class FractalHeapDirectBlock {
9+
public final String signature;
10+
public final short version;
11+
public final long heapHeaderAddress;
12+
public final long blockOffset;
13+
public final byte[] objectData;
14+
public final Integer checksum;
15+
16+
private FractalHeapDirectBlock(ByteBuffer bb, FractalHeapHeader header, int sizeOfOffsets) throws IOException {
17+
bb.order(ByteOrder.LITTLE_ENDIAN);
18+
this.signature = Hdf5Utils.readSignature(bb, "FHDB");
19+
this.version = bb.get();
20+
this.heapHeaderAddress = Hdf5Utils.readOffset(bb, sizeOfOffsets);
21+
int blockOffsetSize = Hdf5Utils.ceilLog2(1L << header.maximumHeapSize) / 8;
22+
this.blockOffset = Hdf5Utils.readVariableSizeUnsigned(bb, blockOffsetSize);
23+
if ((header.flags & 2) != 0) {
24+
// dataSize -= 4;
25+
// bb.position(bb.position() + dataSize);
26+
this.checksum = bb.getInt();
27+
// bb.position(bb.position() - dataSize - 4);
28+
} else {
29+
this.checksum = null;
30+
}
31+
this.objectData = bb.array();
32+
}
33+
34+
public static FractalHeapDirectBlock read(SeekableByteChannel channel, long blockAddress, long expectedBlockSize, FractalHeapHeader header, int sizeOfOffsets) throws IOException {
35+
channel.position(blockAddress);
36+
ByteBuffer blockBuffer = ByteBuffer.allocate((int) expectedBlockSize);
37+
Hdf5Utils.readBytes(channel, blockBuffer);
38+
return new FractalHeapDirectBlock(blockBuffer, header, sizeOfOffsets);
39+
}
40+
}

0 commit comments

Comments
 (0)