From 131161e1bd9907bb227b56a891685d5265864ac4 Mon Sep 17 00:00:00 2001 From: Emily Boudreaux Date: Sun, 16 Feb 2025 20:44:45 -0500 Subject: [PATCH] feat(utils/opatio): updated save to ascii function and fixed minor bug in loader the new general purpose index design introduced a bug in the loader related to the byte position of the byteStart field. This has been resolved. Further, I have updated the save to ascii format to provide an explicit warning that the ascii version is only for debugging purposes and not meant to be used directly. --- utils/opatio/src/opatio/opat/opat.py | 47 +++++++++++++++------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/utils/opatio/src/opatio/opat/opat.py b/utils/opatio/src/opatio/opat/opat.py index 744b8c9..f60239b 100644 --- a/utils/opatio/src/opatio/opat/opat.py +++ b/utils/opatio/src/opatio/opat/opat.py @@ -249,7 +249,7 @@ class OpatIO: logKappa = logKappa ) - self.tables.append(indicies, table)) + self.tables.append((indicies, table)) self.header.numTables += 1 @@ -309,7 +309,7 @@ class OpatIO: ) tableIndexBytes += tableIndex.sha256 - if len(tableIndexBytes) != 64: + if len(tableIndexBytes) != 16+self.header.numIndex*8+32: raise RuntimeError(f"Each table index entry must have 64 bytes. Due to an unknown error the table index entry for (X,Z)=({tableIndex.X},{tableIndex.Z}) header has {len(tableIndexBytes)} bytes") return tableIndexBytes @@ -347,7 +347,8 @@ class OpatIO: tableString.append(" ".join(tableIndexString)) tableString.append("-" * 80) # write logR across the top (reserving one col for where logT will be) - logRRow = f"{'':<10}" + tableString.append(f"{'':<10}{'logR':<10}") + logRRow = f"{'logT':<10}" logRRowTrue = "".join(f"{r:<10.4f}" for r in table.logR) tableString.append(logRRow + logRRowTrue) for i, logT in enumerate(table.logT): @@ -411,21 +412,27 @@ class OpatIO: currentStartByte += len(tableBytes) self.header.indexOffset = currentStartByte with open(filename, 'w') as f: - f.write(f"{self.header.magic}\n") - f.write(f"Version: {self.header.version}\n") - f.write(f"numTables: {self.header.numTables}\n") - f.write(f"headerSize (bytes): {self.header.headerSize}\n") - f.write(f"tableIndex Offset (bytes): {self.header.indexOffset}\n") - f.write(f"Creation Date: {self.header.creationDate}\n") - f.write(f"Source Info: {self.header.sourceInfo}\n") - f.write(f"Comment: {self.header.comment}\n") - f.write(f"numIndex: {self.header.numIndex}\n") - f.write("="*80 + "\n") + f.write("This is an ASCII representation of an OPAT file, it is not a valid OPAT file in and of itself.\n") + f.write("This file is meant to be human readable and is not meant to be read by a computer.\n") + f.write("The purpose of this file is to provide a human readable representation of the OPAT file which can be used for debugging purposes.\n") + f.write("The full binary specification of the OPAT file can be found in the OPAT file format documentation at:\n") + f.write(" https://github.com/4D-STAR/4DSSE/blob/main/specs/OPAT/OPAT.pdf\n") + f.write("="*35 + " HEADER " + "="*36 + "\n") + f.write(f">> {self.header.magic}\n") + f.write(f">> Version: {self.header.version}\n") + f.write(f">> numTables: {self.header.numTables}\n") + f.write(f">> headerSize (bytes): {self.header.headerSize}\n") + f.write(f">> tableIndex Offset (bytes): {self.header.indexOffset}\n") + f.write(f">> Creation Date: {self.header.creationDate}\n") + f.write(f">> Source Info: {self.header.sourceInfo}\n") + f.write(f">> Comment: {self.header.comment}\n") + f.write(f">> numIndex: {self.header.numIndex}\n") + f.write("="*37 + " DATA " + "="*37 + "\n") f.write("="*80 + "\n") for tableString in tableStrings: f.write(tableString) f.write("="*80 + "\n") - f.write("="*80 + "\n") + f.write("="*36 + " INDEX " + "="*37 + "\n") f.write(self.print_table_indexes(tableIndexs)) def save(self, filename: str) -> str: @@ -501,13 +508,11 @@ def loadOpat(filename: str) -> OpatIO: while tableIndexEntryBytes := f.read(tableIndexChunkSize): unpackedTableIndexEntry = struct.unpack(tableIndexFMTString, tableIndexEntryBytes) checksum = f.read(32) - # TODO: Update this to get the fully, general header index set (currently it still gets only X and Z without the update) - # TODO: Also update the spec to reflect the new header index set and the new table index format. + index = unpackedTableIndexEntry[:loadedHeader.numIndex] tableIndexEntry = TableIndex( - X = unpackedTableIndexEntry[0], - Z = unpackedTableIndexEntry[1], - byteStart = unpackedTableIndexEntry[2], - byteEnd = unpackedTableIndexEntry[3], + index = index, + byteStart = unpackedTableIndexEntry[loadedHeader.numIndex], + byteEnd = unpackedTableIndexEntry[loadedHeader.numIndex+1], sha256 = checksum ) tableIndices.append(tableIndexEntry) @@ -530,5 +535,5 @@ def loadOpat(filename: str) -> OpatIO: logT = np.array(unpackedData[N_R: N_R+N_T], dtype=np.float64) logKappa = np.array(unpackedData[N_R+N_T:], dtype=np.float64).reshape((N_R, N_T)) - opat.add_table(tableIndex.X, tableIndex.Z, logR, logT, logKappa) + opat.add_table(tableIndex.index, logR, logT, logKappa) return opat