MeanField_SERiF/tests/serialize/archive.cpp

#include <bit>
#include <chrono>
#include <filesystem>
#include <fstream>
#include <iomanip>
#include <iterator>
#include <limits>
#include <locale>
#include <random>
#include <sstream>

#include <catch2/catch_test_macros.hpp>
#include "serif/config.hpp"
#include "serif/utils/serialize/archive.hpp"
#include "serif/utils/serialize/solver.hpp"

namespace {
    namespace serialization = serif::utils::serialize;

    /** @brief Remove a test archive on success and on an assertion failure. */
    struct File {
        std::filesystem::path path = std::filesystem::temp_directory_path() /
            ("serif-archive-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()) + ".serif");
        /** @brief Keep all regression artifacts confined to their temporary lifetime. */
        ~File() { std::error_code error; std::filesystem::remove(path, error); }
    };

    /** @brief Read raw bytes so malformed archives can be tested independently of the writer. */
    std::string bytes(const File& file) {
        std::ifstream input(file.path, std::ios::binary);
        return {std::istreambuf_iterator<char>{input}, std::istreambuf_iterator<char>{}};
    }

    /** @brief Deliberately replace an archive with a malformed test fixture. */
    void replace(const File& file, std::string_view value) {
        std::ofstream output(file.path, std::ios::binary | std::ios::trunc);
        output.write(value.data(), value.size());
        REQUIRE(output.good());
    }

    /** @brief Change one public header field while preserving its fixed byte extent. */
    std::string header_field(std::string original, std::string_view key, std::string_view value) {
        const auto start = original.find(std::string{key} + ' ') + key.size() + 1;
        const auto end = original.find('\n', start);
        std::string header = original.substr(0, original.find("END_HEADER\n") + 11);
        header.replace(start, end - start, value);
        REQUIRE(header.size() < 512);
        header.resize(512, ' ');
        header.back() = '\n';
        return header + original.substr(512);
    }

    /** @brief Repair the ledger checksum after an intentional structurally invalid edit. */
    std::string ledger_checksum(std::string original) {
        const std::size_t start = original.find("ledger_bytes ") + 13;
        const std::size_t count = std::stoull(original.substr(start, original.find('\n', start) - start));
        return header_field(original, "ledger_checksum", serialization::fingerprint(std::string_view{original}.substr(512, count)));
    }
}

TEST_CASE("Both archive encodings preserve every binary64 bit pattern", "[serialize][archive]") {
    std::mt19937_64 random{271828};
    std::vector<double> values{0.0, -0.0, 1.0, -1.0, std::numeric_limits<double>::denorm_min(),
        std::numeric_limits<double>::min(), std::numeric_limits<double>::max(), std::numeric_limits<double>::infinity(),
        -std::numeric_limits<double>::infinity(), std::bit_cast<double>(0x7ff8123456789abcULL)};
    for (int index = 0; index < 2000; ++index) {
        values.push_back(std::bit_cast<double>(random()));
    }
    for (serialization::Format format : {serialization::Format::binary, serialization::Format::text}) {
        const File file;
        REQUIRE(serialization::write(file.path, serialization::pack(serialization::named("coefficients", values)), {.format = format}));
        auto reader = serialization::Reader::open(file.path);
        REQUIRE(reader);
        CHECK(reader->format() == format);
        const auto restored = reader->read<std::vector<double>>("coefficients");
        REQUIRE(restored);
        REQUIRE(restored->size() == values.size());
        for (std::size_t index = 0; index < values.size(); ++index) {
            CHECK(std::bit_cast<std::uint64_t>((*restored)[index]) == std::bit_cast<std::uint64_t>(values[index]));
        }
        CHECK(reader->validate());
    }
}

TEST_CASE("Archives carry provenance annotations and an immediately inspectable ledger", "[serialize][archive]") {
    const File file;
    const auto contents = serialization::pack(serialization::named("surface", serialization::object("harmonics",
        serialization::field("coefficients", std::vector<double>{1.0, 2.0}), serialization::field("description", "ξ, θ\n\"quoted\"\\"))));
    REQUIRE(serialization::write(file.path, contents, {.format = serialization::Format::text,
        .author = "Emily", .comment = "Restart after Newton step 4\nRetain the full surface.", .metadata = {{"study", "ESTER comparison"}}}));
    auto reader = serialization::Reader::open(file.path);
    REQUIRE(reader);
    REQUIRE(reader->contains("surface/coefficients"));
    CHECK_FALSE(reader->contains("unprovided_solver_options"));
    CHECK(reader->serif_version() == SERIF_VERSION);
    CHECK(reader->read<std::string>("provenance/author") == "Emily");
    CHECK(reader->read<std::string>("provenance/metadata/study") == "ESTER comparison");
    const auto time = reader->read<std::string>("provenance/written_utc");
    REQUIRE(time);
    CHECK(time->size() == 20);
    CHECK(time->back() == 'Z');
    CHECK(reader->contains("provenance/host/architecture"));
    CHECK(reader->contains("provenance/libraries/nameof"));
    CHECK(reader->contains("provenance/libraries/magic_enum"));
    CHECK(reader->read<std::string>("surface/description") == "ξ, θ\n\"quoted\"\\");
    for (const serialization::Entry& entry : reader->ledger()) {
        if (entry.path == "surface/coefficients") {
            std::ifstream input(file.path, std::ios::binary);
            input.seekg(entry.offset);
            double first = 0.0;
            input >> first;
            CHECK(first == 1.0);
            CHECK(entry.count == 2);
        }
    }
}

TEST_CASE("Pack handles absence unsupported components and conflicting definitions explicitly", "[serialize][archive]") {
    const File file;
    struct CustomPhysics {};
    auto contents = serialization::pack(serialization::named("physics", CustomPhysics{}), serialization::named("number", 42));
    REQUIRE(serialization::write(file.path, contents));
    auto reader = serialization::Reader::open(file.path);
    REQUIRE(reader);
    const auto unknown = reader->node("physics");
    REQUIRE(unknown);
    CHECK(unknown->type == "unsupported");
    CHECK_FALSE(reader->read<CustomPhysics>("physics"));
    CHECK_FALSE(reader->read<int>("missing"));
    const std::string original = bytes(file);
    CHECK_FALSE(serialization::write(file.path, serialization::pack(serialization::named("x", 1), serialization::named("x", 2))));
    CHECK(bytes(file) == original);
    CHECK(serialization::write(file.path, serialization::pack(serialization::named("x", 1), serialization::named("x", 1))));
    CHECK_FALSE(serialization::write(file.path, serialization::pack(serialization::named("provenance", 1))));
    CHECK_FALSE(serialization::write(file.path, serialization::pack(serialization::named("a/b", 1))));
    CHECK_FALSE(serialization::write(file.path, serialization::pack(serialization::named("zero", 0.0), serialization::named("zero", -0.0))));
    const double nan = std::bit_cast<double>(0x7ff8123456789abcULL);
    CHECK(serialization::write(file.path, serialization::pack(serialization::named("nan", nan), serialization::named("nan", nan))));
}

TEST_CASE("Readers implement configurable SERiF version tolerance independently of file versions", "[serialize][archive]") {
    const File file;
    REQUIRE(serialization::write(file.path, serialization::pack()));
    const std::string original = bytes(file);
    const std::string major = std::to_string(SERIF_VERSION_MAJOR);
    replace(file, header_field(original, "serif_version", '"' + major + ".999.17\""));
    CHECK(serialization::Reader::open(file.path));
    CHECK_FALSE(serialization::Reader::open(file.path, {.version_tolerance = serialization::VersionTolerance::exact}));
    replace(file, header_field(original, "serif_version", '"' + std::to_string(SERIF_VERSION_MAJOR + 1) + ".0.0\""));
    CHECK_FALSE(serialization::Reader::open(file.path));
    CHECK(serialization::Reader::open(file.path, {.version_tolerance = serialization::VersionTolerance::any}));
    std::string future = original;
    future[future.find("SERIF_ARCHIVE 1") + 14] = '2';
    replace(file, future);
    CHECK_FALSE(serialization::Reader::open(file.path, {.version_tolerance = serialization::VersionTolerance::any}));
}

TEST_CASE("Indexed reads isolate requested data and report damage when that section is read", "[serialize][archive]") {
    const File file;
    REQUIRE(serialization::write(file.path, serialization::pack(serialization::named("large", std::vector<double>(10000, 1.25)),
        serialization::named("a", serialization::object("group", serialization::field("value", 7))), serialization::named("a.b", 8))));
    const auto index = serialization::Reader::open(file.path);
    REQUIRE(index);
    std::string damaged = bytes(file);
    for (const auto& entry : index->ledger()) {
        if (entry.path == "large") {
            damaged[entry.offset + 3] ^= 1;
        }
    }
    replace(file, damaged);
    auto reader = serialization::Reader::open(file.path);
    REQUIRE(reader);
    CHECK(reader->metadata());
    const auto group = reader->node("a");
    REQUIRE(group);
    CHECK(serialization::read_field<int>(*group, "value") == 7);
    CHECK_FALSE(reader->read<std::vector<double>>("large"));
    CHECK_FALSE(reader->validate());
}

TEST_CASE("Malformed bounds truncation ledger and allocation requests are rejected", "[serialize][archive]") {
    const File file;
    REQUIRE(serialization::write(file.path, serialization::pack(serialization::named("value", 3.0))));
    const std::string original = bytes(file);
    for (std::size_t length : {std::size_t{0}, std::size_t{511}, std::size_t{512}, original.size() - 1}) {
        replace(file, original.substr(0, length));
        CHECK_FALSE(serialization::Reader::open(file.path));
    }
    for (std::string key : {"ledger_offset", "ledger_bytes", "data_offset", "file_bytes", "entries"}) {
        replace(file, header_field(original, key, "18446744073709551615"));
        CHECK_FALSE(serialization::Reader::open(file.path));
    }
    replace(file, original);
    CHECK_FALSE(serialization::Reader::open(file.path, {.maximum_index_bytes = 1}));
    CHECK_FALSE(serialization::Reader::open(file.path, {.maximum_payload_bytes = 1}));
    CHECK_FALSE(serialization::Reader::open(file.path, {.maximum_entries = 1}));
    CHECK_FALSE(serialization::Reader::open(file.path, {.maximum_depth = 0}));
    std::string damaged = original;
    damaged[550] ^= 1;
    replace(file, damaged);
    CHECK_FALSE(serialization::Reader::open(file.path));
}

TEST_CASE("Primitive schemas reject wrong extents ranges versions and representations", "[serialize][archive]") {
    CHECK_FALSE(serialization::decode<unsigned char>(serialization::encode(std::uint64_t{256})));
    CHECK_FALSE(serialization::decode<std::array<int, 2>>(serialization::encode(std::vector<int>{1})));
    CHECK_FALSE(serialization::decode<std::vector<int>>(serialization::encode(std::vector<double>{1.0})));
    CHECK_FALSE(serialization::decode<std::string>(serialization::encode(1.0)));
    auto future = serialization::encode(1.0);
    future.version = 2;
    CHECK_FALSE(serialization::decode<double>(future));
    const auto optional = serialization::decode<std::optional<int>>(serialization::encode(std::optional<int>{7}));
    REQUIRE(optional);
    CHECK(*optional == std::optional<int>{7});
    CHECK(serialization::decode<std::tuple<int, std::string>>(serialization::encode(std::tuple{7, std::string{"x"}})) == std::tuple{7, std::string{"x"}});
    for (serialization::Format format : {serialization::Format::text, serialization::Format::binary}) {
        const File file;
        REQUIRE(serialization::write(file.path, serialization::pack(serialization::named("empty", std::vector<double>{}),
            serialization::named("bounds", std::vector<std::int64_t>{std::numeric_limits<std::int64_t>::min(), std::numeric_limits<std::int64_t>::max()}),
            serialization::named("flag", true)), {.format = format}));
        auto reader = serialization::Reader::open(file.path);
        REQUIRE(reader);
        CHECK(reader->read<std::vector<double>>("empty") == std::vector<double>{});
        CHECK(reader->read<bool>("flag") == true);
        CHECK(reader->read<std::vector<std::int64_t>>("bounds") == std::vector<std::int64_t>{std::numeric_limits<std::int64_t>::min(), std::numeric_limits<std::int64_t>::max()});
    }
}

TEST_CASE("A valid checksum does not hide inconsistent ledger semantics", "[serialize][archive]") {
    const File file;
    REQUIRE(serialization::write(file.path, serialization::pack(serialization::named("aa", 1.0), serialization::named("bb", 2.0))));
    const std::string original = bytes(file);
    SECTION("Duplicate component paths") {
        std::string edited = original;
        edited.replace(edited.find("\"bb\" \"real\""), 4, "\"aa\"");
        replace(file, ledger_checksum(edited));
        CHECK_FALSE(serialization::Reader::open(file.path));
    }
    SECTION("Overlapping payloads") {
        std::string edited = original;
        const std::size_t first = edited.find("\"aa\" \"real\" 1 4 1 ") + 18;
        const std::size_t second = edited.find("\"bb\" \"real\" 1 4 1 ") + 18;
        edited.replace(second, 20, edited.substr(first, 20));
        replace(file, ledger_checksum(edited));
        CHECK_FALSE(serialization::Reader::open(file.path));
    }
    SECTION("Scalar with an array extent") {
        std::string edited = original;
        edited.replace(edited.find("\"aa\" \"real\" 1 4 1"), 17, "\"aa\" \"real\" 1 4 9");
        replace(file, ledger_checksum(edited));
        auto reader = serialization::Reader::open(file.path);
        REQUIRE(reader);
        CHECK_FALSE(reader->read<double>("aa"));
    }
    SECTION("Unknown payload encoding") {
        std::string edited = original;
        edited.replace(edited.find("\"aa\" \"real\" 1 4 1"), 17, "\"aa\" \"real\" 1 9 1");
        replace(file, ledger_checksum(edited));
        CHECK_FALSE(serialization::Reader::open(file.path));
    }
}

TEST_CASE("Failed archive publication retains existing destinations", "[serialize][archive]") {
    const File directory;
    REQUIRE(std::filesystem::create_directory(directory.path));
    CHECK_FALSE(serialization::write(directory.path, serialization::pack(serialization::named("value", 1))));
    CHECK(std::filesystem::is_directory(directory.path));
    CHECK(std::filesystem::is_empty(directory.path));
    CHECK_FALSE(serialization::write(directory.path / "missing" / "star.serif", serialization::pack()));
}

TEST_CASE("Archive bytes do not depend on the application's numeric locale", "[serialize][archive]") {
    struct Punctuation : std::numpunct<char> {
        /** @brief Simulate a locale with comma decimals and whitespace-separated digit groups. */
        char do_decimal_point() const override { return ','; }
        /** @brief Expose accidental locale-dependent checksum formatting as a ledger parse failure. */
        char do_thousands_sep() const override { return ' '; }
        /** @brief Group every third digit in formatted numbers. */
        std::string do_grouping() const override { return "\3"; }
    };
    struct RestoreLocale {
        std::locale original = std::locale();
        /** @brief Restore the application's locale even when an assertion aborts this case. */
        ~RestoreLocale() { std::locale::global(original); }
    } restore;
    const std::string expected = serialization::fingerprint("scientific data");
    std::locale::global(std::locale(std::locale::classic(), new Punctuation));
    CHECK(serialization::fingerprint("scientific data") == expected);
    const File file;
    REQUIRE(serialization::write(file.path, serialization::pack(serialization::named("value", 12345.6789)), {.format = serialization::Format::text}));
    std::locale::global(std::locale::classic());
    auto reader = serialization::Reader::open(file.path);
    REQUIRE(reader);
    CHECK(reader->read<double>("value") == 12345.6789);
}


TEST_CASE("Previous Newton diagnostic archives remain readable without solver-owned timers", "[unit][serialize][restart]") {
    using Progress = serif::solver::NewtonProgress;
    const Progress progress{
        .accepted_metric = {.residual_norm = 0.5, .merit = 0.125},
        .correction = {0.25},
        .next_step = 0.5,
        .diagnostics = {.attempted_iterations = 1, .accepted_iterations = 1, .line_search_trials = 2,
            .rejected_trials = 1, .initial_residual_norm = 2.0, .final_residual_norm = 0.5,
            .convergence_threshold = 1.0e-8, .last_accepted_step = 0.25}
    };
    auto previous = serialization::encode(progress);
    auto& diagnostics = previous.children.at("diagnostics");
    // These are the exact observational fields written by the preceding numerical-history schema.
    diagnostics.children.at("solve_seconds") = serialization::encode(17.0);
    diagnostics.children.at("linear_solve_seconds") = serialization::encode(9.0);
    for (const auto format : {serialization::Format::binary, serialization::Format::text}) {
        const File file;
        REQUIRE(serialization::write(file.path, serialization::pack(serialization::named("progress", previous)), {.format = format}));
        auto reader = serialization::Reader::open(file.path);
        REQUIRE(reader);
        const auto restored = reader->read<Progress>("progress");
        REQUIRE(restored);
        CHECK_FALSE(serif::solver::validate_newton_progress(*restored, 1));
        CHECK(restored->correction == progress.correction);
        CHECK(restored->next_step == progress.next_step);
        CHECK(restored->diagnostics.initial_residual_norm == progress.diagnostics.initial_residual_norm);
        CHECK(restored->diagnostics.attempted_iterations == 1);
        const auto current = serialization::encode(*restored).children.at("diagnostics");
        CHECK(serialization::decode<double>(current.children.at("solve_seconds")).value() == 0.0);
        CHECK(serialization::decode<double>(current.children.at("linear_solve_seconds")).value() == 0.0);
    }
}