Emily Boudreaux
7ab8641f51
There are loads of things (solutions, models, solver descriptions, mesh's, etc..) which need to be exported from SERiF and then either reimported into SERiF or into some other program. This module is responsible for all of that. Generally with the goal of feeling like the fmt / new format module in C++23 and on (e.g. by specializing formatting structs) Note that LLM's made signifigant contributions to this module (ChatGPT 6 astra max and Claude Opus 5.5 Medium). All code has been reviewd manually
425 lines
22 KiB
C++
425 lines
22 KiB
C++
#include "serif/utils/serialize/archive.hpp"
|
|
|
|
#include <algorithm>
|
|
#include <atomic>
|
|
#include <charconv>
|
|
#include <chrono>
|
|
#include <cstdio>
|
|
#include <iomanip>
|
|
#include <limits>
|
|
#include <locale>
|
|
#include <set>
|
|
#include <sstream>
|
|
#include <system_error>
|
|
|
|
#include "serif/config.hpp"
|
|
#include "format.hpp"
|
|
|
|
namespace serif::utils::serialize {
|
|
namespace {
|
|
struct Header {
|
|
Format format{Format::binary};
|
|
std::string version;
|
|
std::string timestamp;
|
|
std::uint64_t index_offset{detail::header_bytes};
|
|
std::uint64_t index_bytes{0};
|
|
std::uint64_t data_offset{0};
|
|
std::uint64_t file_bytes{0};
|
|
std::uint64_t entries{0};
|
|
std::string checksum;
|
|
};
|
|
|
|
struct Payload {
|
|
Entry entry;
|
|
std::string bytes;
|
|
};
|
|
|
|
/** @brief Keep the fixed header readable and independent of compiler structure padding. */
|
|
std::string print_header(const Header& header) {
|
|
std::ostringstream text;
|
|
text.imbue(std::locale::classic());
|
|
text << "SERIF_ARCHIVE 1\nformat " << (header.format == Format::binary ? "binary" : "text")
|
|
<< "\nserif_version " << std::quoted(header.version) << "\nwritten_utc " << std::quoted(header.timestamp)
|
|
<< "\nledger_offset " << header.index_offset << "\nledger_bytes " << header.index_bytes
|
|
<< "\ndata_offset " << header.data_offset << "\nfile_bytes " << header.file_bytes
|
|
<< "\nentries " << header.entries << "\nledger_checksum " << header.checksum << "\nEND_HEADER\n";
|
|
std::string result = text.str();
|
|
result.resize(detail::header_bytes, ' ');
|
|
result.back() = '\n';
|
|
return result;
|
|
}
|
|
|
|
/** @brief Parse a fixed sequence so duplicated or unknown control fields cannot be ambiguous. */
|
|
Result<Header> parse_header(std::string_view bytes) {
|
|
std::istringstream input{std::string{bytes}};
|
|
input.imbue(std::locale::classic());
|
|
Header header;
|
|
std::string name;
|
|
std::string format;
|
|
int version = 0;
|
|
/** @brief Consume the next required control-field name. */
|
|
const auto next = [&](std::string_view expected) { return static_cast<bool>(input >> name) && name == expected; };
|
|
const bool valid = next("SERIF_ARCHIVE") && (input >> version) && version == 1 && next("format") && (input >> format)
|
|
&& next("serif_version") && (input >> std::quoted(header.version)) && next("written_utc") && (input >> std::quoted(header.timestamp))
|
|
&& next("ledger_offset") && (input >> header.index_offset) && next("ledger_bytes") && (input >> header.index_bytes)
|
|
&& next("data_offset") && (input >> header.data_offset) && next("file_bytes") && (input >> header.file_bytes)
|
|
&& next("entries") && (input >> header.entries) && next("ledger_checksum") && (input >> header.checksum) && next("END_HEADER");
|
|
input >> std::ws;
|
|
if (!valid || !input.eof() || (format != "binary" && format != "text")) {
|
|
return std::unexpected(Error{"header", "Malformed header or unsupported archive format version."});
|
|
}
|
|
header.format = format == "binary" ? Format::binary : Format::text;
|
|
return header;
|
|
}
|
|
|
|
/** @brief Compare semantic major versions without interpreting arbitrary strings as zero. */
|
|
std::optional<std::uint64_t> major_version(std::string_view version) {
|
|
const std::size_t dot = version.find('.');
|
|
if (dot == std::string_view::npos || dot == 0) {
|
|
return {};
|
|
}
|
|
std::uint64_t major = 0;
|
|
const auto parsed = std::from_chars(version.data(), version.data() + dot, major);
|
|
if (parsed.ec != std::errc{} || parsed.ptr != version.data() + dot) {
|
|
return {};
|
|
}
|
|
return major;
|
|
}
|
|
|
|
/** @brief Flatten a component tree into independent payloads and reject ambiguous names. */
|
|
Result<void> collect(const Node& node, std::string path, Format format, std::vector<Payload>& result, std::size_t depth = 0) {
|
|
if (!detail::valid_path(path) || depth > 128 || node.version == 0 || node.type.empty() || node.type.size() > 1024
|
|
|| node.type.find_first_of("\r\n\0", 0, 3) != std::string::npos) {
|
|
return std::unexpected(Error{std::move(path), "Invalid component path, schema identifier, or nesting depth."});
|
|
}
|
|
if (!node.children.empty() && !std::holds_alternative<std::monostate>(node.value)) {
|
|
return std::unexpected(Error{std::move(path), "A component must contain either fields or a payload."});
|
|
}
|
|
Payload payload{
|
|
.entry = {.path = path, .type = node.type, .version = node.version, .kind = static_cast<std::uint32_t>(node.value.index()),
|
|
.count = detail::value_count(node.value)},
|
|
.bytes = detail::encode_value(node.value, format)};
|
|
payload.entry.bytes = payload.bytes.size();
|
|
payload.entry.checksum = fingerprint(payload.bytes);
|
|
result.push_back(std::move(payload));
|
|
for (const auto& [name, child] : node.children) {
|
|
if (name.find('/') != std::string::npos) {
|
|
return std::unexpected(Error{path, "A field name cannot contain a path separator."});
|
|
}
|
|
auto saved = collect(child, path + "/" + name, format, result, depth + 1);
|
|
if (!saved) {
|
|
return saved;
|
|
}
|
|
}
|
|
return {};
|
|
}
|
|
|
|
/** @brief Fixed-width offsets keep the ledger length unchanged when final locations are filled in. */
|
|
std::string print_ledger(const std::vector<Payload>& payloads) {
|
|
std::ostringstream output;
|
|
output.imbue(std::locale::classic());
|
|
output << "# path schema version kind count offset bytes checksum\n";
|
|
for (const Payload& payload : payloads) {
|
|
const Entry& entry = payload.entry;
|
|
output << std::quoted(entry.path) << ' ' << std::quoted(entry.type) << ' ' << entry.version << ' ' << entry.kind << ' '
|
|
<< entry.count << ' ' << std::setfill('0') << std::setw(20) << entry.offset << ' ' << std::setw(20) << entry.bytes
|
|
<< ' ' << entry.checksum << '\n';
|
|
}
|
|
return output.str();
|
|
}
|
|
|
|
/** @brief Give each text payload a visible section label outside its indexed bytes. */
|
|
std::string section_label(const Entry& entry, Format format) {
|
|
return format == Format::text ? "\n# " + entry.path + "\n" : "";
|
|
}
|
|
|
|
/** @brief Remove a failed temporary write without ever removing the previous destination. */
|
|
struct TemporaryFile {
|
|
std::filesystem::path path;
|
|
std::FILE* stream{nullptr};
|
|
bool owned{false};
|
|
/** @brief Clean up regardless of which validation or I/O operation failed. */
|
|
~TemporaryFile() {
|
|
if (stream != nullptr) {
|
|
std::fclose(stream);
|
|
}
|
|
if (owned) {
|
|
std::error_code error;
|
|
std::filesystem::remove(path, error);
|
|
}
|
|
}
|
|
/** @brief Check every byte of an output section. */
|
|
bool append(std::string_view bytes) const { return std::fwrite(bytes.data(), 1, bytes.size(), stream) == bytes.size(); }
|
|
};
|
|
|
|
/** @brief Read an exact bounded region, clearing EOF from any preceding failed read. */
|
|
Result<std::string> read_bytes(std::ifstream& file, std::uint64_t offset, std::uint64_t bytes, std::string_view path) {
|
|
if (bytes > static_cast<std::uint64_t>(std::numeric_limits<std::streamsize>::max())
|
|
|| offset > static_cast<std::uint64_t>(std::numeric_limits<std::streamoff>::max())) {
|
|
return std::unexpected(Error{std::string{path}, "File region exceeds this reader's addressable range."});
|
|
}
|
|
file.clear();
|
|
file.seekg(static_cast<std::streamoff>(offset));
|
|
std::string result(bytes, '\0');
|
|
file.read(result.data(), static_cast<std::streamsize>(bytes));
|
|
if (!file) {
|
|
return std::unexpected(Error{std::string{path}, "Unable to read the complete indexed file region."});
|
|
}
|
|
return result;
|
|
}
|
|
|
|
/** @brief Validate all structural bounds before any payload is allocated or interpreted. */
|
|
Result<std::vector<Entry>> parse_ledger(std::string_view bytes, const Header& header, const ReadOptions& options) {
|
|
std::istringstream input{std::string{bytes}};
|
|
input.imbue(std::locale::classic());
|
|
std::string line;
|
|
std::getline(input, line);
|
|
if (line != "# path schema version kind count offset bytes checksum") {
|
|
return std::unexpected(Error{"ledger", "Unrecognized ledger columns."});
|
|
}
|
|
std::vector<Entry> entries;
|
|
std::map<std::string, std::uint32_t> kinds;
|
|
std::uint64_t previous_end = header.data_offset;
|
|
while (std::getline(input, line)) {
|
|
Entry entry;
|
|
std::istringstream row{line};
|
|
row.imbue(std::locale::classic());
|
|
const bool parsed = static_cast<bool>(row >> std::quoted(entry.path) >> std::quoted(entry.type) >> entry.version >> entry.kind
|
|
>> entry.count >> entry.offset >> entry.bytes >> entry.checksum);
|
|
row >> std::ws;
|
|
if (!parsed || !row.eof() || !detail::valid_path(entry.path) || entry.type.empty() || entry.type.size() > 1024
|
|
|| entry.kind > 8 || entry.version == 0 || entries.size() >= options.maximum_entries
|
|
|| static_cast<std::uint64_t>(std::count(entry.path.begin(), entry.path.end(), '/')) > options.maximum_depth
|
|
|| (!entries.empty() && entries.back().path >= entry.path)) {
|
|
return std::unexpected(Error{entry.path, "Malformed, duplicated, unsorted, or oversized ledger entry."});
|
|
}
|
|
const std::size_t separator = entry.path.rfind('/');
|
|
if (separator != std::string::npos) {
|
|
const auto parent = kinds.find(entry.path.substr(0, separator));
|
|
if (parent == kinds.end() || parent->second != 0) {
|
|
return std::unexpected(Error{entry.path, "A field's parent must be an existing component directory."});
|
|
}
|
|
}
|
|
if (entry.kind == 0) {
|
|
if (entry.count != 0 || entry.offset != 0 || entry.bytes != 0 || entry.checksum != fingerprint("")) {
|
|
return std::unexpected(Error{entry.path, "Directory entries cannot reference payload bytes."});
|
|
}
|
|
} else {
|
|
if (entry.offset < previous_end || entry.offset > header.file_bytes || entry.bytes > header.file_bytes - entry.offset
|
|
|| entry.bytes > options.maximum_payload_bytes || entry.count > options.maximum_payload_bytes) {
|
|
return std::unexpected(Error{entry.path, "Payload overlaps another section, exceeds the file, or exceeds configured limits."});
|
|
}
|
|
previous_end = entry.offset + entry.bytes;
|
|
}
|
|
kinds.emplace(entry.path, entry.kind);
|
|
entries.push_back(std::move(entry));
|
|
}
|
|
if (entries.size() != header.entries || !kinds.contains("provenance")) {
|
|
return std::unexpected(Error{"ledger", "Entry count is inconsistent or required provenance is missing."});
|
|
}
|
|
return entries;
|
|
}
|
|
}
|
|
|
|
void add_component(Pack& contents, Field component) {
|
|
if (contents.error) {
|
|
return;
|
|
}
|
|
if (component.name == "provenance" || !detail::valid_path(component.name) || component.name.find('/') != std::string::npos) {
|
|
contents.error = Error{component.name, "Use a non-reserved, single-component archive name."};
|
|
return;
|
|
}
|
|
const auto found = contents.root.children.find(component.name);
|
|
if (found != contents.root.children.end() && found->second != component.node) {
|
|
contents.error = Error{component.name, "Two different components were packed under the same name."};
|
|
return;
|
|
}
|
|
contents.root.children.emplace(std::move(component.name), std::move(component.node));
|
|
}
|
|
|
|
Result<void> write(const std::filesystem::path& path, const Pack& contents, const WriteOptions& options) {
|
|
if (contents.error) {
|
|
return std::unexpected(*contents.error);
|
|
}
|
|
std::vector<Payload> payloads;
|
|
const Node metadata = detail::provenance(options);
|
|
auto saved = collect(metadata, "provenance", options.format, payloads);
|
|
if (!saved) {
|
|
return saved;
|
|
}
|
|
for (const auto& [name, node] : contents.root.children) {
|
|
if (name == "provenance") {
|
|
return std::unexpected(Error{name, "Provenance is written by the archive, not supplied as a model component."});
|
|
}
|
|
saved = collect(node, name, options.format, payloads);
|
|
if (!saved) {
|
|
return saved;
|
|
}
|
|
}
|
|
std::ranges::sort(payloads, {}, [](const Payload& payload) { return payload.entry.path; });
|
|
Header header{.format = options.format, .version = SERIF_VERSION,
|
|
.timestamp = std::get<std::string>(metadata.children.at("written_utc").value),
|
|
.index_bytes = print_ledger(payloads).size(), .entries = payloads.size()};
|
|
header.data_offset = header.index_offset + header.index_bytes;
|
|
std::uint64_t position = header.data_offset;
|
|
for (Payload& payload : payloads) {
|
|
if (payload.entry.kind != 0) {
|
|
position += section_label(payload.entry, options.format).size();
|
|
payload.entry.offset = position;
|
|
position += payload.bytes.size();
|
|
}
|
|
}
|
|
header.file_bytes = position;
|
|
const std::string ledger = print_ledger(payloads);
|
|
header.checksum = fingerprint(ledger);
|
|
|
|
static std::atomic_uint64_t sequence{0};
|
|
TemporaryFile temporary;
|
|
for (int attempt = 0; attempt < 10 && temporary.stream == nullptr; ++attempt) {
|
|
temporary.path = path;
|
|
temporary.path += ".tmp-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + "-" + std::to_string(sequence++);
|
|
temporary.stream = std::fopen(temporary.path.string().c_str(), "wbx");
|
|
}
|
|
if (temporary.stream == nullptr) {
|
|
return std::unexpected(Error{path.string(), "Cannot create an exclusive temporary file beside the destination."});
|
|
}
|
|
temporary.owned = true;
|
|
bool written = temporary.append(print_header(header)) && temporary.append(ledger);
|
|
for (const Payload& payload : payloads) {
|
|
if (written && payload.entry.kind != 0) {
|
|
written = temporary.append(section_label(payload.entry, options.format)) && temporary.append(payload.bytes);
|
|
}
|
|
}
|
|
const int close_status = std::fclose(temporary.stream);
|
|
temporary.stream = nullptr;
|
|
if (!written || close_status != 0) {
|
|
return std::unexpected(Error{path.string(), "The archive could not be written completely; the previous destination was retained."});
|
|
}
|
|
std::error_code error;
|
|
std::filesystem::rename(temporary.path, path, error);
|
|
if (error) {
|
|
return std::unexpected(Error{path.string(), "Cannot publish the completed archive: " + error.message()});
|
|
}
|
|
temporary.owned = false;
|
|
return {};
|
|
}
|
|
|
|
Result<Reader> Reader::open(const std::filesystem::path& path, ReadOptions options) {
|
|
Reader reader;
|
|
reader.m_options = options;
|
|
reader.m_file.open(path, std::ios::binary | std::ios::ate);
|
|
if (!reader.m_file) {
|
|
return std::unexpected(Error{path.string(), "Cannot open archive for reading."});
|
|
}
|
|
const std::streamoff file_size = reader.m_file.tellg();
|
|
if (file_size < static_cast<std::streamoff>(detail::header_bytes)) {
|
|
return std::unexpected(Error{"header", "Archive is truncated before the complete header."});
|
|
}
|
|
auto bytes = read_bytes(reader.m_file, 0, detail::header_bytes, "header");
|
|
if (!bytes) {
|
|
return std::unexpected(bytes.error());
|
|
}
|
|
auto header = parse_header(*bytes);
|
|
if (!header) {
|
|
return std::unexpected(header.error());
|
|
}
|
|
const bool same_major = major_version(header->version) && major_version(header->version) == major_version(SERIF_VERSION);
|
|
if ((options.version_tolerance == VersionTolerance::exact && header->version != SERIF_VERSION)
|
|
|| (options.version_tolerance == VersionTolerance::same_major && !same_major)) {
|
|
return std::unexpected(Error{"header/serif_version", "SERiF " + header->version + " is incompatible with the selected reader tolerance (reader " SERIF_VERSION ")."});
|
|
}
|
|
|
|
if (header->index_offset != detail::header_bytes || header->file_bytes != static_cast<std::uint64_t>(file_size)
|
|
|| header->index_bytes > options.maximum_index_bytes || header->index_bytes > header->file_bytes - detail::header_bytes
|
|
|| header->data_offset != header->index_offset + header->index_bytes || header->entries > options.maximum_entries) {
|
|
|
|
std::string cmp_string = std::format("header: [index_offset={}, index_bytes={}, data_offset={}, file_bytes={}, entries={}]", header->index_offset, header->index_bytes, header->data_offset, header->file_bytes, header->entries);
|
|
cmp_string = std::format("{}, file: [size={}]", cmp_string, file_size);
|
|
cmp_string = std::format("{}, limits: [max_index_bytes={}, max_entries={}]", cmp_string, options.maximum_index_bytes, options.maximum_entries);
|
|
cmp_string = std::format("{}, version_tolerance={}", cmp_string, static_cast<int>(options.version_tolerance));
|
|
cmp_string = std::format("{}, detail::header_bytes={}", cmp_string, detail::header_bytes);
|
|
|
|
return std::unexpected(Error{"header", std::format("Invalid file size, section offsets, or configured allocation limits. {}", cmp_string)});
|
|
}
|
|
auto ledger = read_bytes(reader.m_file, header->index_offset, header->index_bytes, "ledger");
|
|
if (!ledger) {
|
|
return std::unexpected(ledger.error());
|
|
}
|
|
if (fingerprint(*ledger) != header->checksum) {
|
|
return std::unexpected(Error{"ledger", "Ledger checksum does not match the header."});
|
|
}
|
|
auto entries = parse_ledger(*ledger, *header, options);
|
|
if (!entries) {
|
|
return std::unexpected(entries.error());
|
|
}
|
|
reader.m_entries = std::move(*entries);
|
|
reader.m_format = header->format;
|
|
reader.m_version = std::move(header->version);
|
|
return reader;
|
|
}
|
|
|
|
bool Reader::contains(std::string_view path) const {
|
|
const auto found = std::ranges::lower_bound(m_entries, path, {}, &Entry::path);
|
|
return found != m_entries.end() && found->path == path;
|
|
}
|
|
|
|
Result<Node> Reader::node(std::string_view path) const {
|
|
const auto first = std::ranges::lower_bound(m_entries, path, {}, &Entry::path);
|
|
if (first == m_entries.end() || first->path != path) {
|
|
return std::unexpected(Error{std::string{path}, "Requested component is not present in the archive ledger."});
|
|
}
|
|
Node root;
|
|
std::map<std::string, Node*> nodes;
|
|
const std::string prefix = std::string{path} + "/";
|
|
std::uint64_t total_bytes = 0;
|
|
for (auto entry = first; entry != m_entries.end(); ++entry) {
|
|
if (entry != first && !entry->path.starts_with(prefix)) {
|
|
// Sibling names can sort before a parent's descendants (a.b precedes a/b).
|
|
if (entry->path < prefix) {
|
|
continue;
|
|
}
|
|
break;
|
|
}
|
|
if (entry->bytes > m_options.maximum_payload_bytes - total_bytes) {
|
|
return std::unexpected(Error{entry->path, "Selected subtree exceeds the configured payload budget."});
|
|
}
|
|
total_bytes += entry->bytes;
|
|
auto bytes = read_bytes(m_file, entry->offset, entry->bytes, entry->path);
|
|
if (!bytes) {
|
|
return std::unexpected(bytes.error());
|
|
}
|
|
if (fingerprint(*bytes) != entry->checksum) {
|
|
return std::unexpected(Error{entry->path, "Payload checksum mismatch; the component is damaged."});
|
|
}
|
|
auto value = detail::decode_value(*bytes, *entry, m_format);
|
|
if (!value) {
|
|
return std::unexpected(value.error());
|
|
}
|
|
Node* destination = &root;
|
|
if (entry != first) {
|
|
const std::size_t separator = entry->path.rfind('/');
|
|
const auto parent = nodes.find(entry->path.substr(0, separator));
|
|
if (parent == nodes.end()) {
|
|
return std::unexpected(Error{entry->path, "Requested subtree has a missing parent."});
|
|
}
|
|
destination = &parent->second->children[entry->path.substr(separator + 1)];
|
|
}
|
|
*destination = Node{.type = entry->type, .version = entry->version, .value = std::move(*value)};
|
|
nodes.emplace(entry->path, destination);
|
|
}
|
|
return root;
|
|
}
|
|
|
|
Result<void> Reader::validate() const {
|
|
for (const Entry& entry : m_entries) {
|
|
if (entry.kind != 0) {
|
|
auto payload = node(entry.path);
|
|
if (!payload) {
|
|
return std::unexpected(payload.error());
|
|
}
|
|
}
|
|
}
|
|
return {};
|
|
}
|
|
}
|