Skip to content

Commit 9e6a85c

Browse files
karlnicholaskarlnicholas
committed
Updates for v2 arch
1 parent a7dd2a5 commit 9e6a85c

5 files changed

Lines changed: 133 additions & 104 deletions

File tree

‎src/main/java/org/hdf5javalib/dataclass/reference/HdfObjectReference.java‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.hdf5javalib.hdfjava.HdfTreeNode;
1111
import org.hdf5javalib.hdfjava.HdfDataObject;
1212
import org.hdf5javalib.utils.HdfDataHolder;
13+
import org.hdf5javalib.utils.HdfDisplayUtils;
1314

1415
import java.io.IOException;
1516
import java.lang.reflect.InvocationTargetException;
@@ -98,14 +99,16 @@ public HdfObjectReference(byte[] bytes, ReferenceDatatype dt, boolean external)
9899
if ( dataspaceSelectionInstance != null ) {
99100
this.hdfDataHolder = dataspaceSelectionInstance.getData(hdfDataObject, dt.getDataFile());
100101
} else {
101-
List<String> parents = new ArrayList<>();
102-
HdfDataObject currentNode = hdfDataObject;
103-
while(currentNode.getParent() != null) {
104-
parents.add(currentNode.getObjectName());
105-
currentNode = currentNode.getParent().getDataObject();
106-
}
107-
Collections.reverse(parents);
108-
String objectPathString = '/' + currentNode.getObjectName() + String.join("/", parents);
102+
// List<String> parents = new ArrayList<>();
103+
// HdfDataObject currentNode = hdfDataObject;
104+
// while(currentNode.getParent() != null) {
105+
// parents.add(currentNode.getObjectName());
106+
// currentNode = currentNode.getParent().getDataObject();
107+
// }
108+
// Collections.reverse(parents);
109+
// String objectPathString = '/' + currentNode.getObjectName() + String.join("/", parents);
110+
String objectPathString = HdfDisplayUtils.getDataObjectFullName(hdfDataObject);
111+
109112
this.hdfDataHolder = HdfDataHolder.ofScalar(
110113
new HdfString(objectPathString, new StringDatatype(
111114
StringDatatype.createClassAndVersion(),

‎src/main/java/org/hdf5javalib/hdffile/infrastructure/fractalheap/gemini/FractalHeapHeader.java‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -40,10 +40,10 @@ public class FractalHeapHeader {
4040
public byte[] ioFilterPipeline;
4141
public int checksum;
4242

43-
// --- Derived/Helper fields ---
43+
// --- Passed-in fields ---
4444
public final int sizeOfOffsets;
4545
public final int sizeOfLengths;
46-
public int maxRows;
46+
// The maxRows field has been removed as it is not used by the read-only logic.
4747

4848
private FractalHeapHeader(int sizeOfOffsets, int sizeOfLengths) {
4949
this.sizeOfOffsets = sizeOfOffsets;
@@ -109,12 +109,7 @@ public static FractalHeapHeader read(SeekableByteChannel channel, long position,
109109
}
110110
header.checksum = reader.readInt();
111111

112-
if (header.startingBlockSize > 0) {
113-
int startingBlockSizePowerOf2 = (int) Math.round(Math.log(header.startingBlockSize) / Math.log(2));
114-
header.maxRows = header.maximumHeapSize - startingBlockSizePowerOf2;
115-
} else {
116-
header.maxRows = 0;
117-
}
112+
// The unused maxRows calculation is now removed.
118113

119114
return header;
120115
}
@@ -156,7 +151,7 @@ public String toString() {
156151
.add("sizeOfOffsets=" + sizeOfOffsets).add("sizeOfLengths=" + sizeOfLengths)
157152
.add("tableWidth=" + tableWidth).add("startingBlockSize=" + startingBlockSize)
158153
.add("maximumDirectBlockSize=" + maximumDirectBlockSize).add("maximumHeapSize(log2)=" + maximumHeapSize)
159-
.add("maxRows=" + maxRows).add("numberOfManagedObjects=" + numberOfManagedObjects)
154+
.add("numberOfManagedObjects=" + numberOfManagedObjects)
160155
.add("startingRowsInRoot=" + startingRowsInRoot).add("currentRowsInRoot=" + currentRowsInRoot)
161156
.add("rootBlockAddress=" + rootBlockAddress)
162157
.toString();

‎src/main/java/org/hdf5javalib/hdffile/infrastructure/fractalheap/grok/FractalHeap.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ private static Block readDirectBlock(HdfReader reader, FractalHeapHeader header,
198198
db.data = data;
199199
db.filterMask = filterMask;
200200
db.checksum = checksum;
201+
db.headerSize = headerSize;
201202
return db;
202203
}
203204

@@ -291,11 +292,21 @@ public byte[] getObject(ParsedHeapId heapId) {
291292
for(Block db: ((IndirectBlock) rootBlock).children) {
292293
DirectBlock dBlock = (DirectBlock) db;
293294
if ( heapId.offset >= dBlock.blockOffset && heapId.offset < (dBlock.blockOffset + dBlock.blockSize) ) {
294-
int from = Math.toIntExact(heapId.offset - dBlock.blockOffset - 22);
295+
int from = Math.toIntExact(heapId.offset - dBlock.blockOffset - dBlock.headerSize);
295296
int to = from + heapId.length;
296297
return Arrays.copyOfRange(dBlock.data, from, to);
297298
}
298299
}
300+
} else {
301+
DirectBlock dBlock = (DirectBlock) rootBlock;
302+
if ( heapId.offset >= dBlock.blockOffset && heapId.offset < (dBlock.blockOffset + dBlock.blockSize) ) {
303+
int from = Math.toIntExact(heapId.offset - dBlock.blockOffset - dBlock.headerSize);
304+
int to = from + heapId.length;
305+
return Arrays.copyOfRange(dBlock.data, from, to);
306+
} else {
307+
throw new IllegalArgumentException("Unknown heap id " + heapId);
308+
}
309+
299310
}
300311
throw new IllegalArgumentException("Unknown heap id " + heapId);
301312
}
@@ -355,6 +366,7 @@ public static abstract class Block {
355366
}
356367

357368
public static class DirectBlock extends Block {
369+
public int headerSize;
358370
long blockSize;
359371
byte[] data;
360372
int filterMask;

‎src/main/java/org/hdf5javalib/hdfjava/HdfFileReader.java‎

Lines changed: 89 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,13 @@
55
import org.hdf5javalib.hdffile.dataobjects.HdfObjectHeaderPrefix;
66
import org.hdf5javalib.hdffile.dataobjects.HdfObjectHeaderPrefixV1;
77
import org.hdf5javalib.hdffile.dataobjects.HdfObjectHeaderPrefixV2;
8-
import org.hdf5javalib.hdffile.dataobjects.messages.AttributeInfoMessage;
9-
import org.hdf5javalib.hdffile.dataobjects.messages.HdfMessage;
10-
import org.hdf5javalib.hdffile.dataobjects.messages.ObjectHeaderContinuationMessage;
8+
import org.hdf5javalib.hdffile.dataobjects.messages.*;
119
import org.hdf5javalib.hdffile.infrastructure.*;
1210
import org.hdf5javalib.hdffile.infrastructure.fractalheap.grok.ParsedHeapId;
1311
import org.hdf5javalib.hdffile.infrastructure.fractalheap.grok.FractalHeap;
1412
import org.hdf5javalib.hdffile.infrastructure.v2btree.gemini.*;
1513
import org.hdf5javalib.hdffile.metadata.HdfSuperblock;
14+
import org.hdf5javalib.utils.HdfDisplayUtils;
1615
import org.hdf5javalib.utils.HdfReadUtils;
1716
import org.hdf5javalib.utils.HdfWriteUtils;
1817

@@ -141,67 +140,95 @@ private void readV1Arch(SeekableByteChannel fileChannel, HdfSymbolTableEntry roo
141140
}
142141

143142
private void readV2Arch(SeekableByteChannel fileChannel, HdfGroup rootGroup) throws Exception {
144-
//// HdfFixedPoint fractalHeapAddress = objectHeader.findMessageByType(LinkInfoMessage.class).orElseThrow().getFractalHeapAddress();
145-
//// long fractalHeapOffset = fractalHeapAddress.getInstance(Long.class);
146-
//// fileChannel.position(fractalHeapOffset);
147-
//
148-
// HdfFixedPoint v2BTreeNameIndexAddress = rootGroup.getObjectHeader().findMessageByType(LinkInfoMessage.class).orElseThrow().getV2BTreeNameIndexAddress();
149-
// long v2BTreeNameIndexOffset = v2BTreeNameIndexAddress.getInstance(Long.class);
150-
// fileChannel.position(v2BTreeNameIndexOffset);
151-
//
152-
// System.out.println("Starting B-tree v2 Read...");
153-
// BTreeV2Reader bTreeV2Reader = new BTreeV2Reader(fileChannel, 8, 8);
154-
// BTreeV2Header bTreeV2Header = bTreeV2Reader.getHeader();
155-
//
156-
// System.out.println("Header parsed successfully:");
157-
// System.out.println(" Type: " + bTreeV2Header.type);
158-
// System.out.println(" Depth: " + bTreeV2Header.depth);
159-
// System.out.println(" Total Records: " + bTreeV2Header.totalNumberOfRecordsInBTree);
160-
// System.out.println(" Root Node Address: " + bTreeV2Header.rootNodeAddress);
161-
//
162-
// List<BTreeV2Record> allRecords = bTreeV2Reader.getAllRecords();
163-
//
164-
// System.out.println("\nTraversal complete. Found " + allRecords.size() + " records:");
165-
//
166-
// HdfFixedPoint fractalHeapAddress = rootGroup.getObjectHeader().findMessageByType(LinkInfoMessage.class).orElseThrow().getFractalHeapAddress();
143+
// HdfFixedPoint fractalHeapAddress = objectHeader.findMessageByType(LinkInfoMessage.class).orElseThrow().getFractalHeapAddress();
167144
// long fractalHeapOffset = fractalHeapAddress.getInstance(Long.class);
168145
// fileChannel.position(fractalHeapOffset);
169-
//
170-
// // 6. --- EXECUTE THE READER ---
171-
// FractalHeapReader reader = new FractalHeapReader(fileChannel, 8, 8);
172-
// FractalHeap heap = reader.read();
173-
//
174-
// // 7. --- RETRIEVE THE OBJECT AND VERIFY ---
175-
// for (BTreeV2Record bTreeV2Record: allRecords) {
176-
//
177-
// byte[] heapId = ((Type5Record)bTreeV2Record).heapId;
178-
//
179-
//// byte[] retrievedData = heap.getObject(heapId);
180-
//// System.out.println("Row: " + Arrays.toString(Arrays.copyOfRange(retrievedData, 0, 10))
181-
//// + " : " + retrievedData[10]
182-
//// + " : " + new String(Arrays.copyOfRange(retrievedData, 11, 11+retrievedData[10]))
183-
//// + " : " + Arrays.toString(Arrays.copyOfRange(retrievedData, 10+1+retrievedData[10], retrievedData.length))
184-
//// );
185-
// ByteBuffer retrievedData = ByteBuffer.wrap(heap.getObject(heapId)).order(ByteOrder.LITTLE_ENDIAN);
186-
// byte[] rowHeader = new byte[10];
187-
// retrievedData.get(rowHeader);
188-
// int sLength = Byte.toUnsignedInt(retrievedData.get());
189-
// byte[] stringBuffer = new byte[sLength];
190-
// retrievedData.get(stringBuffer);
191-
// String groupName = new String(stringBuffer);
192-
// long objectHeaderOffset = retrievedData.getLong();
193-
// System.out.println("Row: " + Arrays.toString(rowHeader)
194-
// + " : " + sLength
195-
// + " : " + groupName
196-
// + " : " + objectHeaderOffset
146+
147+
HdfFixedPoint v2BTreeNameIndexAddress = rootGroup.getObjectHeader().findMessageByType(LinkInfoMessage.class).orElseThrow().getV2BTreeNameIndexAddress();
148+
long v2BTreeNameIndexOffset = v2BTreeNameIndexAddress.getInstance(Long.class);
149+
fileChannel.position(v2BTreeNameIndexOffset);
150+
151+
System.out.println("Starting B-tree v2 Read...");
152+
BTreeV2Reader bTreeV2Reader = new BTreeV2Reader(fileChannel, 8, 8);
153+
BTreeV2Header bTreeV2Header = bTreeV2Reader.getHeader();
154+
155+
System.out.println("Header parsed successfully:");
156+
System.out.println(" Type: " + bTreeV2Header.type);
157+
System.out.println(" Depth: " + bTreeV2Header.depth);
158+
System.out.println(" Total Records: " + bTreeV2Header.totalNumberOfRecordsInBTree);
159+
System.out.println(" Root Node Address: " + bTreeV2Header.rootNodeAddress);
160+
161+
List<BTreeV2Record> allRecords = bTreeV2Reader.getAllRecords();
162+
163+
System.out.println("\nTraversal complete. Found " + allRecords.size() + " records:");
164+
165+
HdfFixedPoint fractalHeapAddress = rootGroup.getObjectHeader().findMessageByType(LinkInfoMessage.class).orElseThrow().getFractalHeapAddress();
166+
long fractalHeapOffset = fractalHeapAddress.getInstance(Long.class);
167+
fileChannel.position(fractalHeapOffset);
168+
169+
// 6. --- EXECUTE THE READER ---
170+
FractalHeap fractalHeap = FractalHeap.read(fileChannel, fractalHeapOffset, 8, 8);
171+
172+
// 7. --- RETRIEVE THE OBJECT AND VERIFY ---
173+
for (BTreeV2Record bTreeV2Record: allRecords) {
174+
175+
byte[] heapId = ((Type5Record)bTreeV2Record).heapId;
176+
ParsedHeapId parsedHeapId = new ParsedHeapId(heapId, fractalHeap);
177+
178+
// byte[] retrievedData = heap.getObject(heapId);
179+
// System.out.println("Row: " + Arrays.toString(Arrays.copyOfRange(retrievedData, 0, 10))
180+
// + " : " + retrievedData[10]
181+
// + " : " + new String(Arrays.copyOfRange(retrievedData, 11, 11+retrievedData[10]))
182+
// + " : " + Arrays.toString(Arrays.copyOfRange(retrievedData, 10+1+retrievedData[10], retrievedData.length))
197183
// );
198-
//
199-
// // System.out.println("Heap ID: " + Arrays.toString(heapId));
200-
//// System.out.println("Retrieved Data: '" + new String(retrievedData, StandardCharsets.UTF_8) + "'");
184+
byte[] objectData = fractalHeap.getObject(parsedHeapId);
185+
ByteBuffer retrievedData = ByteBuffer.wrap(objectData).order(ByteOrder.LITTLE_ENDIAN);
186+
byte[] rowHeader = new byte[10];
187+
retrievedData.get(rowHeader);
188+
int sLength = Byte.toUnsignedInt(retrievedData.get());
189+
byte[] stringBuffer = new byte[sLength];
190+
retrievedData.get(stringBuffer);
191+
String groupName = new String(stringBuffer);
192+
long objectHeaderOffset = retrievedData.getLong();
193+
System.out.println("Row: " + Arrays.toString(rowHeader)
194+
+ " : " + sLength
195+
+ " : " + groupName
196+
+ " : " + objectHeaderOffset
197+
);
198+
199+
// System.out.println("Heap ID: " + Arrays.toString(heapId));
200+
// System.out.println("Retrieved Data: '" + new String(retrievedData, StandardCharsets.UTF_8) + "'");
201+
String hardLink = isHardLink(rootGroup, objectHeaderOffset);
202+
HdfObjectHeaderPrefix objectHeader = null;
203+
if ( hardLink == null ) {
204+
// objectHeader = readObjectHeader(fileChannel, objectHeaderOffset, this);
205+
objectHeader = readObjectHeaderPrefixFromSeekableByteChannel(fileChannel, objectHeaderOffset, this, groupName);
206+
}
201207
// HdfObjectHeaderPrefix objectHeaderG1 = readObjectHeaderPrefixFromSeekableByteChannel(fileChannel, objectHeaderOffset, this, groupName);
202-
//// System.out.println(groupName + ":" + objectHeaderG1.getHeaderMessages());
203-
// }
208+
// System.out.println(groupName + ":" + objectHeaderG1.getHeaderMessages());
209+
210+
if ( objectHeader != null && objectHeader.findMessageByType(DataLayoutMessage.class).isEmpty() ) {
211+
HdfGroup groupObject = new HdfGroup(groupName, objectHeader, rootGroup, hardLink);
212+
rootGroup.addChild(groupObject);
213+
System.out.println("ADDED GROUP: " + HdfDisplayUtils.getDataObjectFullName(groupObject));
214+
215+
if ( hardLink == null ) {
216+
LinkInfoMessage linkInfoMessage = objectHeader.findMessageByType(LinkInfoMessage.class).orElseThrow();
217+
if ( !linkInfoMessage.getFractalHeapAddress().isUndefined() && !linkInfoMessage.getV2BTreeNameIndexAddress().isUndefined() ) {
218+
readV2Arch(fileChannel, groupObject);
219+
}
220+
}
221+
222+
} else {
223+
HdfDataset datasetObject = new HdfDataset(groupName, objectHeader, rootGroup, hardLink);
224+
rootGroup.addChild(datasetObject);
225+
System.out.println("ADDED DATASET: " + HdfDisplayUtils.getDataObjectFullName(datasetObject));
226+
}
227+
228+
229+
}
204230

231+
/*
205232
HdfFixedPoint attributeInfoV2BTreeNameIndexAddress = rootGroup.getObjectHeader().findMessageByType(AttributeInfoMessage.class).orElseThrow().getAttributeNameV2BtreeAddress();
206233
long attributeInfoV2BTreeNameIndexOffset = attributeInfoV2BTreeNameIndexAddress.getInstance(Long.class);
207234
fileChannel.position(attributeInfoV2BTreeNameIndexOffset);
@@ -226,43 +253,18 @@ private void readV2Arch(SeekableByteChannel fileChannel, HdfGroup rootGroup) thr
226253
fileChannel.position(attributeInfoFractalHeapOffset);
227254
228255
FractalHeap fractalHeap = FractalHeap.read(fileChannel, attributeInfoFractalHeapOffset, 8, 8);
229-
//
230-
// // 6. --- EXECUTE THE READER ---
231-
// FractalHeapReader attributeInfoReader = new FractalHeapReader(fileChannel, 8, 8);
232-
// FractalHeap attributeInfoRHeap = attributeInfoReader.read();
233-
//
234256
// 7. --- RETRIEVE THE OBJECT AND VERIFY ---
235257
for (BTreeV2Record bTreeV2Record: attributeInfoAllRecords) {
236258
237259
byte[] heapId = ((Type8Record)bTreeV2Record).heapId;
238-
// System.out.println("Row: " + Arrays.toString(heapId));
239260
ParsedHeapId parsedHeapId = new ParsedHeapId(heapId, fractalHeap);
240261
241262
byte[] retrievedData = fractalHeap.getObject(parsedHeapId);
242263
System.out.println("Row: " + Arrays.toString(retrievedData));
243264
printRows(retrievedData);
244265
System.out.println();
245-
// System.out.println("Row: " + Arrays.toString(Arrays.copyOfRange(retrievedData, 0, 10))
246-
// + " : " + retrievedData[10]
247-
// + " : " + new String(Arrays.copyOfRange(retrievedData, 11, 11+retrievedData[10]))
248-
// + " : " + Arrays.toString(Arrays.copyOfRange(retrievedData, 10+1+retrievedData[10], retrievedData.length))
249-
// );
250-
// ByteBuffer retrievedData = ByteBuffer.wrap(attributeInfoRHeap.getObject(heapId)).order(ByteOrder.LITTLE_ENDIAN);
251-
// byte[] rowHeader = new byte[10];
252-
// retrievedData.get(rowHeader);
253-
// int sLength = Byte.toUnsignedInt(retrievedData.get());
254-
// byte[] stringBuffer = new byte[sLength];
255-
// retrievedData.get(stringBuffer);
256-
// String groupName = new String(stringBuffer);
257-
// long objectHeaderOffset = retrievedData.getLong();
258-
// System.out.println("Row: " + Arrays.toString(retrievedData.array()));
259-
260-
// System.out.println("Heap ID: " + Arrays.toString(heapId));
261-
// System.out.println("Retrieved Data: '" + new String(retrievedData, StandardCharsets.UTF_8) + "'");
262-
// HdfObjectHeaderPrefix objectHeaderG1 = readObjectHeaderPrefixFromSeekableByteChannel(fileChannel, objectHeaderOffset, this, groupName);
263-
// System.out.println(groupName + ":" + objectHeaderG1.getHeaderMessages());
264266
}
265-
267+
*/
266268

267269
}
268270
public static void printRows(byte[] input) {
@@ -409,7 +411,8 @@ public Iterator<HdfDataset> datasetIterator() {
409411
public List<HdfDataset> getDatasets() {
410412
// Convert Iterator to List using a for loop
411413
List<HdfDataset> resultList = new ArrayList<>();
412-
for (Iterator<HdfDataset> iterator = datasetIterator(); iterator.hasNext(); ) {
414+
Iterator<HdfDataset> iterator = datasetIterator();
415+
for (; iterator.hasNext(); ) {
413416
resultList.add(iterator.next());
414417
}
415418
return resultList;

‎src/main/java/org/hdf5javalib/utils/HdfDisplayUtils.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import org.hdf5javalib.datasource.TypedDataSource;
66
import org.hdf5javalib.hdffile.dataobjects.messages.AttributeMessage;
77
import org.hdf5javalib.hdfjava.HdfDataFile;
8+
import org.hdf5javalib.hdfjava.HdfDataObject;
89
import org.hdf5javalib.hdfjava.HdfDataset;
910
import org.hdf5javalib.hdfjava.HdfFileReader;
1011
import org.slf4j.Logger;
@@ -16,7 +17,10 @@
1617
import java.nio.file.Files;
1718
import java.nio.file.Path;
1819
import java.nio.file.StandardOpenOption;
20+
import java.util.ArrayList;
1921
import java.util.Arrays;
22+
import java.util.Collections;
23+
import java.util.List;
2024
import java.util.stream.Collectors;
2125

2226
/**
@@ -90,6 +94,18 @@ public static void displayAttributes(HdfDataset dataSet) throws InvocationTarget
9094
}
9195
}
9296

97+
public static String getDataObjectFullName(HdfDataObject hdfDataObject) {
98+
List<String> parents = new ArrayList<>();
99+
HdfDataObject currentNode = hdfDataObject;
100+
while(currentNode.getParent() != null) {
101+
parents.add(currentNode.getObjectName());
102+
currentNode = currentNode.getParent().getDataObject();
103+
}
104+
Collections.reverse(parents);
105+
String objectPathString = '/' + currentNode.getObjectName() + String.join("/", parents);
106+
return objectPathString;
107+
}
108+
93109
public static void displayData(SeekableByteChannel channel, HdfDataset ds, HdfFileReader reader) throws Exception {
94110
log.debug("Dataset path: {}", ds.getObjectPath());
95111
if (ds.hasData()) {

0 commit comments

Comments
 (0)