archive.cpp serif/utils/serialize/archive.hpp algorithm atomic charconv chrono cstdio iomanip limits locale set sstream system_error serif/config.hpp format.hpp serif/utils/serialize/archive.hpp serif/utils/serialize/record.hpp src/lib/serif/utils/serialize/archive.cpp format.hpp algorithm array atomic bit charconv chrono concepts cstddef cstdint cstdio exception expected filesystem fstream functional iomanip limits locale map optional serif/config.hpp set sstream string string_view system_error tuple type_traits utility variant vector serif serif::utils serif::utils::serialize #include"serif/utils/serialize/archive.hpp" #include<algorithm> #include<atomic> #include<charconv> #include<chrono> #include<cstdio> #include<iomanip> #include<limits> #include<locale> #include<set> #include<sstream> #include<system_error> #include"serif/config.hpp" #include"format.hpp" namespaceserif::utils::serialize{ namespace{ structHeader{ Formatformat{Format::binary}; std::stringversion; std::stringtimestamp; std::uint64_tindex_offset{detail::header_bytes}; std::uint64_tindex_bytes{0}; std::uint64_tdata_offset{0}; std::uint64_tfile_bytes{0}; std::uint64_tentries{0}; std::stringchecksum; }; structPayload{ Entryentry; std::stringbytes; }; std::stringprint_header(constHeader&header){ std::ostringstreamtext; text.imbue(std::locale::classic()); text<<"SERIF_ARCHIVE1\nformat"<<(header.format==Format::binary?"binary":"text") <<"\nserif_version"<<std::quoted(header.version)<<"\nwritten_utc"<<std::quoted(header.timestamp) <<"\nledger_offset"<<header.index_offset<<"\nledger_bytes"<<header.index_bytes <<"\ndata_offset"<<header.data_offset<<"\nfile_bytes"<<header.file_bytes <<"\nentries"<<header.entries<<"\nledger_checksum"<<header.checksum<<"\nEND_HEADER\n"; std::stringresult=text.str(); result.resize(detail::header_bytes,''); result.back()='\n'; returnresult; } Result<Header>parse_header(std::string_viewbytes){ std::istringstreaminput{std::string{bytes}}; input.imbue(std::locale::classic()); Headerheader; std::stringname; std::stringformat; intversion=0; constautonext=[&](std::string_viewexpected){returnstatic_cast<bool>(input>>name)&&name==expected;}; constboolvalid=next("SERIF_ARCHIVE")&&(input>>version)&&version==1&&next("format")&&(input>>format) &&next("serif_version")&&(input>>std::quoted(header.version))&&next("written_utc")&&(input>>std::quoted(header.timestamp)) &&next("ledger_offset")&&(input>>header.index_offset)&&next("ledger_bytes")&&(input>>header.index_bytes) &&next("data_offset")&&(input>>header.data_offset)&&next("file_bytes")&&(input>>header.file_bytes) &&next("entries")&&(input>>header.entries)&&next("ledger_checksum")&&(input>>header.checksum)&&next("END_HEADER"); input>>std::ws; if(!valid||!input.eof()||(format!="binary"&&format!="text")){ returnstd::unexpected(Error{"header","Malformedheaderorunsupportedarchiveformatversion."}); } header.format=format=="binary"?Format::binary:Format::text; returnheader; } std::optional<std::uint64_t>major_version(std::string_viewversion){ conststd::size_tdot=version.find('.'); if(dot==std::string_view::npos||dot==0){ return{}; } std::uint64_tmajor=0; constautoparsed=std::from_chars(version.data(),version.data()+dot,major); if(parsed.ec!=std::errc{}||parsed.ptr!=version.data()+dot){ return{}; } returnmajor; } Result<void>collect(constNode&node,std::stringpath,Formatformat,std::vector<Payload>&result,std::size_tdepth=0){ if(!detail::valid_path(path)||depth>128||node.version==0||node.type.empty()||node.type.size()>1024 ||node.type.find_first_of("\r\n\0",0,3)!=std::string::npos){ returnstd::unexpected(Error{std::move(path),"Invalidcomponentpath,schemaidentifier,ornestingdepth."}); } if(!node.children.empty()&&!std::holds_alternative<std::monostate>(node.value)){ returnstd::unexpected(Error{std::move(path),"Acomponentmustcontaineitherfieldsorapayload."}); } Payloadpayload{ .entry={.path=path,.type=node.type,.version=node.version,.kind=static_cast<std::uint32_t>(node.value.index()), .count=detail::value_count(node.value)}, .bytes=detail::encode_value(node.value,format)}; payload.entry.bytes=payload.bytes.size(); payload.entry.checksum=fingerprint(payload.bytes); result.push_back(std::move(payload)); for(constauto&[name,child]:node.children){ if(name.find('/')!=std::string::npos){ returnstd::unexpected(Error{path,"Afieldnamecannotcontainapathseparator."}); } autosaved=collect(child,path+"/"+name,format,result,depth+1); if(!saved){ returnsaved; } } return{}; } std::stringprint_ledger(conststd::vector<Payload>&payloads){ std::ostringstreamoutput; output.imbue(std::locale::classic()); output<<"#pathschemaversionkindcountoffsetbyteschecksum\n"; for(constPayload&payload:payloads){ constEntry&entry=payload.entry; output<<std::quoted(entry.path)<<''<<std::quoted(entry.type)<<''<<entry.version<<''<<entry.kind<<'' <<entry.count<<''<<std::setfill('0')<<std::setw(20)<<entry.offset<<''<<std::setw(20)<<entry.bytes <<''<<entry.checksum<<'\n'; } returnoutput.str(); } std::stringsection_label(constEntry&entry,Formatformat){ returnformat==Format::text?"\n#"+entry.path+"\n":""; } structTemporaryFile{ std::filesystem::pathpath; std::FILE*stream{nullptr}; boolowned{false}; ~TemporaryFile(){ if(stream!=nullptr){ std::fclose(stream); } if(owned){ std::error_codeerror; std::filesystem::remove(path,error); } } boolappend(std::string_viewbytes)const{returnstd::fwrite(bytes.data(),1,bytes.size(),stream)==bytes.size();} }; Result<std::string>read_bytes(std::ifstream&file,std::uint64_toffset,std::uint64_tbytes,std::string_viewpath){ if(bytes>static_cast<std::uint64_t>(std::numeric_limits<std::streamsize>::max()) ||offset>static_cast<std::uint64_t>(std::numeric_limits<std::streamoff>::max())){ returnstd::unexpected(Error{std::string{path},"Fileregionexceedsthisreader'saddressablerange."}); } file.clear(); file.seekg(static_cast<std::streamoff>(offset)); std::stringresult(bytes,'\0'); file.read(result.data(),static_cast<std::streamsize>(bytes)); if(!file){ returnstd::unexpected(Error{std::string{path},"Unabletoreadthecompleteindexedfileregion."}); } returnresult; } Result<std::vector<Entry>>parse_ledger(std::string_viewbytes,constHeader&header,constReadOptions&options){ std::istringstreaminput{std::string{bytes}}; input.imbue(std::locale::classic()); std::stringline; std::getline(input,line); if(line!="#pathschemaversionkindcountoffsetbyteschecksum"){ returnstd::unexpected(Error{"ledger","Unrecognizedledgercolumns."}); } std::vector<Entry>entries; std::map<std::string,std::uint32_t>kinds; std::uint64_tprevious_end=header.data_offset; while(std::getline(input,line)){ Entryentry; std::istringstreamrow{line}; row.imbue(std::locale::classic()); constboolparsed=static_cast<bool>(row>>std::quoted(entry.path)>>std::quoted(entry.type)>>entry.version>>entry.kind >>entry.count>>entry.offset>>entry.bytes>>entry.checksum); row>>std::ws; if(!parsed||!row.eof()||!detail::valid_path(entry.path)||entry.type.empty()||entry.type.size()>1024 ||entry.kind>8||entry.version==0||entries.size()>=options.maximum_entries ||static_cast<std::uint64_t>(std::count(entry.path.begin(),entry.path.end(),'/'))>options.maximum_depth ||(!entries.empty()&&entries.back().path>=entry.path)){ returnstd::unexpected(Error{entry.path,"Malformed,duplicated,unsorted,oroversizedledgerentry."}); } conststd::size_tseparator=entry.path.rfind('/'); if(separator!=std::string::npos){ constautoparent=kinds.find(entry.path.substr(0,separator)); if(parent==kinds.end()||parent->second!=0){ returnstd::unexpected(Error{entry.path,"Afield'sparentmustbeanexistingcomponentdirectory."}); } } if(entry.kind==0){ if(entry.count!=0||entry.offset!=0||entry.bytes!=0||entry.checksum!=fingerprint("")){ returnstd::unexpected(Error{entry.path,"Directoryentriescannotreferencepayloadbytes."}); } }else{ if(entry.offset<previous_end||entry.offset>header.file_bytes||entry.bytes>header.file_bytes-entry.offset ||entry.bytes>options.maximum_payload_bytes||entry.count>options.maximum_payload_bytes){ returnstd::unexpected(Error{entry.path,"Payloadoverlapsanothersection,exceedsthefile,orexceedsconfiguredlimits."}); } previous_end=entry.offset+entry.bytes; } kinds.emplace(entry.path,entry.kind); entries.push_back(std::move(entry)); } if(entries.size()!=header.entries||!kinds.contains("provenance")){ returnstd::unexpected(Error{"ledger","Entrycountisinconsistentorrequiredprovenanceismissing."}); } returnentries; } } voidadd_component(Pack&contents,Fieldcomponent){ if(contents.error){ return; } if(component.name=="provenance"||!detail::valid_path(component.name)||component.name.find('/')!=std::string::npos){ contents.error=Error{component.name,"Useanon-reserved,single-componentarchivename."}; return; } constautofound=contents.root.children.find(component.name); if(found!=contents.root.children.end()&&found->second!=component.node){ contents.error=Error{component.name,"Twodifferentcomponentswerepackedunderthesamename."}; return; } contents.root.children.emplace(std::move(component.name),std::move(component.node)); } Result<void>write(conststd::filesystem::path&path,constPack&contents,constWriteOptions&options){ if(contents.error){ returnstd::unexpected(*contents.error); } std::vector<Payload>payloads; constNodemetadata=detail::provenance(options); autosaved=collect(metadata,"provenance",options.format,payloads); if(!saved){ returnsaved; } for(constauto&[name,node]:contents.root.children){ if(name=="provenance"){ returnstd::unexpected(Error{name,"Provenanceiswrittenbythearchive,notsuppliedasamodelcomponent."}); } saved=collect(node,name,options.format,payloads); if(!saved){ returnsaved; } } std::ranges::sort(payloads,{},[](constPayload&payload){returnpayload.entry.path;}); Headerheader{.format=options.format,.version=SERIF_VERSION, .timestamp=std::get<std::string>(metadata.children.at("written_utc").value), .index_bytes=print_ledger(payloads).size(),.entries=payloads.size()}; header.data_offset=header.index_offset+header.index_bytes; std::uint64_tposition=header.data_offset; for(Payload&payload:payloads){ if(payload.entry.kind!=0){ position+=section_label(payload.entry,options.format).size(); payload.entry.offset=position; position+=payload.bytes.size(); } } header.file_bytes=position; conststd::stringledger=print_ledger(payloads); header.checksum=fingerprint(ledger); staticstd::atomic_uint64_tsequence{0}; TemporaryFiletemporary; for(intattempt=0;attempt<10&&temporary.stream==nullptr;++attempt){ temporary.path=path; temporary.path+=".tmp-"+std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())+"-"+std::to_string(sequence++); temporary.stream=std::fopen(temporary.path.string().c_str(),"wbx"); } if(temporary.stream==nullptr){ returnstd::unexpected(Error{path.string(),"Cannotcreateanexclusivetemporaryfilebesidethedestination."}); } temporary.owned=true; boolwritten=temporary.append(print_header(header))&&temporary.append(ledger); for(constPayload&payload:payloads){ if(written&&payload.entry.kind!=0){ written=temporary.append(section_label(payload.entry,options.format))&&temporary.append(payload.bytes); } } constintclose_status=std::fclose(temporary.stream); temporary.stream=nullptr; if(!written||close_status!=0){ returnstd::unexpected(Error{path.string(),"Thearchivecouldnotbewrittencompletely;thepreviousdestinationwasretained."}); } std::error_codeerror; std::filesystem::rename(temporary.path,path,error); if(error){ returnstd::unexpected(Error{path.string(),"Cannotpublishthecompletedarchive:"+error.message()}); } temporary.owned=false; return{}; } Result<Reader>Reader::open(conststd::filesystem::path&path,ReadOptionsoptions){ Readerreader; reader.m_options=options; reader.m_file.open(path,std::ios::binary|std::ios::ate); if(!reader.m_file){ returnstd::unexpected(Error{path.string(),"Cannotopenarchiveforreading."}); } conststd::streamofffile_size=reader.m_file.tellg(); if(file_size<static_cast<std::streamoff>(detail::header_bytes)){ returnstd::unexpected(Error{"header","Archiveistruncatedbeforethecompleteheader."}); } autobytes=read_bytes(reader.m_file,0,detail::header_bytes,"header"); if(!bytes){ returnstd::unexpected(bytes.error()); } autoheader=parse_header(*bytes); if(!header){ returnstd::unexpected(header.error()); } constboolsame_major=major_version(header->version)&&major_version(header->version)==major_version(SERIF_VERSION); if((options.version_tolerance==VersionTolerance::exact&&header->version!=SERIF_VERSION) ||(options.version_tolerance==VersionTolerance::same_major&&!same_major)){ returnstd::unexpected(Error{"header/serif_version","SERiF"+header->version+"isincompatiblewiththeselectedreadertolerance(reader"SERIF_VERSION")."}); } if(header->index_offset!=detail::header_bytes||header->file_bytes!=static_cast<std::uint64_t>(file_size) ||header->index_bytes>options.maximum_index_bytes||header->index_bytes>header->file_bytes-detail::header_bytes ||header->data_offset!=header->index_offset+header->index_bytes||header->entries>options.maximum_entries){ std::stringcmp_string=std::format("header:[index_offset={},index_bytes={},data_offset={},file_bytes={},entries={}]",header->index_offset,header->index_bytes,header->data_offset,header->file_bytes,header->entries); cmp_string=std::format("{},file:[size={}]",cmp_string,file_size); cmp_string=std::format("{},limits:[max_index_bytes={},max_entries={}]",cmp_string,options.maximum_index_bytes,options.maximum_entries); cmp_string=std::format("{},version_tolerance={}",cmp_string,static_cast<int>(options.version_tolerance)); cmp_string=std::format("{},detail::header_bytes={}",cmp_string,detail::header_bytes); returnstd::unexpected(Error{"header",std::format("Invalidfilesize,sectionoffsets,orconfiguredallocationlimits.{}",cmp_string)}); } autoledger=read_bytes(reader.m_file,header->index_offset,header->index_bytes,"ledger"); if(!ledger){ returnstd::unexpected(ledger.error()); } if(fingerprint(*ledger)!=header->checksum){ returnstd::unexpected(Error{"ledger","Ledgerchecksumdoesnotmatchtheheader."}); } autoentries=parse_ledger(*ledger,*header,options); if(!entries){ returnstd::unexpected(entries.error()); } reader.m_entries=std::move(*entries); reader.m_format=header->format; reader.m_version=std::move(header->version); returnreader; } boolReader::contains(std::string_viewpath)const{ constautofound=std::ranges::lower_bound(m_entries,path,{},&Entry::path); returnfound!=m_entries.end()&&found->path==path; } Result<Node>Reader::node(std::string_viewpath)const{ constautofirst=std::ranges::lower_bound(m_entries,path,{},&Entry::path); if(first==m_entries.end()||first->path!=path){ returnstd::unexpected(Error{std::string{path},"Requestedcomponentisnotpresentinthearchiveledger."}); } Noderoot; std::map<std::string,Node*>nodes; conststd::stringprefix=std::string{path}+"/"; std::uint64_ttotal_bytes=0; for(autoentry=first;entry!=m_entries.end();++entry){ if(entry!=first&&!entry->path.starts_with(prefix)){ //Siblingnamescansortbeforeaparent'sdescendants(a.bprecedesa/b). if(entry->path<prefix){ continue; } break; } if(entry->bytes>m_options.maximum_payload_bytes-total_bytes){ returnstd::unexpected(Error{entry->path,"Selectedsubtreeexceedstheconfiguredpayloadbudget."}); } total_bytes+=entry->bytes; autobytes=read_bytes(m_file,entry->offset,entry->bytes,entry->path); if(!bytes){ returnstd::unexpected(bytes.error()); } if(fingerprint(*bytes)!=entry->checksum){ returnstd::unexpected(Error{entry->path,"Payloadchecksummismatch;thecomponentisdamaged."}); } autovalue=detail::decode_value(*bytes,*entry,m_format); if(!value){ returnstd::unexpected(value.error()); } Node*destination=&root; if(entry!=first){ conststd::size_tseparator=entry->path.rfind('/'); constautoparent=nodes.find(entry->path.substr(0,separator)); if(parent==nodes.end()){ returnstd::unexpected(Error{entry->path,"Requestedsubtreehasamissingparent."}); } destination=&parent->second->children[entry->path.substr(separator+1)]; } *destination=Node{.type=entry->type,.version=entry->version,.value=std::move(*value)}; nodes.emplace(entry->path,destination); } returnroot; } Result<void>Reader::validate()const{ for(constEntry&entry:m_entries){ if(entry.kind!=0){ autopayload=node(entry.path); if(!payload){ returnstd::unexpected(payload.error()); } } } return{}; } }