From d9085561668e6cf8c5fd11d7e3ed54555e01c561 Mon Sep 17 00:00:00 2001 From: gaozheng Date: Tue, 23 Jun 2026 11:00:47 +0800 Subject: [PATCH] =?UTF-8?q?feat(store):=20GPR=20=E4=B8=89=E7=BB=B4?= =?UTF-8?q?=E4=BD=93=E5=88=86=E5=9D=97=E5=8E=8B=E7=BC=A9=E8=90=BD=E7=9B=98?= =?UTF-8?q?=20ChunkedVolumeStore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增 geopro_store 库（B/C 方案共用基座）：int16 体逐块 qCompress 压缩写入 data.bin + nlohmann-json sidecar（meta.json 记几何/量化/逐块偏移索引）。 write/readMeta/readBrick 三接口 + 边缘块（< brick）支持；偏移/长度全程 64 位。 不引入 vtkHDFWriter，不加 vcpkg 依赖（压缩用 QtCore 自带 zlib）。 --- src/data/CMakeLists.txt | 3 + src/data/store/CMakeLists.txt | 16 ++ src/data/store/ChunkedVolumeStore.cpp | 203 ++++++++++++++++++ src/data/store/ChunkedVolumeStore.hpp | 73 +++++++ tests/CMakeLists.txt | 4 + .../data/store/test_chunked_volume_store.cpp | 43 ++++ 6 files changed, 342 insertions(+) create mode 100644 src/data/store/CMakeLists.txt create mode 100644 src/data/store/ChunkedVolumeStore.cpp create mode 100644 src/data/store/ChunkedVolumeStore.hpp create mode 100644 tests/data/store/test_chunked_volume_store.cpp diff --git a/src/data/CMakeLists.txt b/src/data/CMakeLists.txt index 73d39c4..6d928a8 100644 --- a/src/data/CMakeLists.txt +++ b/src/data/CMakeLists.txt @@ -21,3 +21,6 @@ target_include_directories(geopro_data PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(geopro_data PUBLIC geopro_core geopro_net Qt6::Core PRIVATE nlohmann_json::nlohmann_json) target_compile_features(geopro_data PUBLIC cxx_std_17) set_target_properties(geopro_data PROPERTIES AUTOMOC ON AUTOUIC OFF AUTORCC OFF) + +# GPR 三维体分块压缩落盘库（geopro_store，B/C 共用基座）。 +add_subdirectory(store) diff --git a/src/data/store/CMakeLists.txt b/src/data/store/CMakeLists.txt new file mode 100644 index 0000000..516cb9a --- /dev/null +++ b/src/data/store/CMakeLists.txt @@ -0,0 +1,16 @@ +# GPR 三维体分块压缩落盘（B/C 共用基座）。 +# 压缩用 Qt qCompress/qUncompress（QtCore 自带 zlib，免新依赖）；sidecar 用 nlohmann-json。 +find_package(nlohmann_json CONFIG REQUIRED) +find_package(Qt6 COMPONENTS Core REQUIRED) + +add_library(geopro_store STATIC + ChunkedVolumeStore.cpp) + +# include 根 = src/，使 #include "data/store/..." 与 "core/algo/..." 可解析 +# （geopro_tests 链 geopro_store 后透传）。 +target_include_directories(geopro_store PUBLIC ${CMAKE_SOURCE_DIR}/src) +target_link_libraries(geopro_store + PUBLIC geopro_core Qt6::Core + PRIVATE nlohmann_json::nlohmann_json) +target_compile_features(geopro_store PUBLIC cxx_std_17) +set_target_properties(geopro_store PROPERTIES AUTOMOC OFF AUTOUIC OFF AUTORCC OFF) diff --git a/src/data/store/ChunkedVolumeStore.cpp b/src/data/store/ChunkedVolumeStore.cpp new file mode 100644 index 0000000..d61d7f2 --- /dev/null +++ b/src/data/store/ChunkedVolumeStore.cpp @@ -0,0 +1,203 @@ +#include "data/store/ChunkedVolumeStore.hpp" + +#include +#include + +#include +#include +#include +#include + +#include "core/algo/GprVolumeBuilder.hpp" // geopro::core::BuiltI16 + +namespace geopro::data { + +namespace { +using nlohmann::json; +namespace fs = std::filesystem; + +constexpr const char* kMetaFile = "meta.json"; +constexpr const char* kDataFile = "data.bin"; + +int ceilDiv(int n, int brick) { return (n + brick - 1) / brick; } + +// 块尺寸（边缘块 < brick）：第 b 块沿该轴的体素数。 +int extent(int n, int b, int brick) { + const int got = n - b * brick; + return got < brick ? got : brick; +} + +// 从体中拷出一块的 int16（块内 i 最快、k 最慢，与体一致）。 +std::vector sliceBrick(const geopro::core::ScalarVolumeI16& vol, + int bx, int by, int bz, int brick, + int bw, int bh, int bd) { + std::vector out(static_cast(bw) * bh * bd); + const int i0 = bx * brick, j0 = by * brick, k0 = bz * brick; + std::size_t w = 0; + for (int kk = 0; kk < bd; ++kk) { + for (int jj = 0; jj < bh; ++jj) { + for (int ii = 0; ii < bw; ++ii) { + out[w++] = vol.at(i0 + ii, j0 + jj, k0 + kk); + } + } + } + return out; +} +} // namespace + +void ChunkedVolumeStore::write(const std::string& dir, + const geopro::core::BuiltI16& b, int brick) { + if (brick <= 0) throw std::invalid_argument("ChunkedVolumeStore: brick must be > 0"); + + const auto& vol = b.vol; + const int nx = vol.nx(), ny = vol.ny(), nz = vol.nz(); + + fs::create_directories(fs::path(dir)); + + const int bX = ceilDiv(nx, brick); + const int bY = ceilDiv(ny, brick); + const int bZ = ceilDiv(nz, brick); + + std::ofstream data((fs::path(dir) / kDataFile).string(), + std::ios::binary | std::ios::trunc); + if (!data) throw std::runtime_error("ChunkedVolumeStore: cannot open data.bin for write"); + + json bricks = json::array(); + std::int64_t offset = 0; + + // 固定遍历顺序：bz 最慢、bx 最快（与 brickIndex 一致）。 + for (int bz = 0; bz < bZ; ++bz) { + for (int by = 0; by < bY; ++by) { + for (int bx = 0; bx < bX; ++bx) { + const int bw = extent(nx, bx, brick); + const int bh = extent(ny, by, brick); + const int bd = extent(nz, bz, brick); + auto raw = sliceBrick(vol, bx, by, bz, brick, bw, bh, bd); + + const int rawBytes = + static_cast(raw.size() * sizeof(std::int16_t)); + const QByteArray compressed = qCompress( + reinterpret_cast(raw.data()), rawBytes); + const std::int64_t clen = compressed.size(); + + data.write(compressed.constData(), clen); + if (!data) throw std::runtime_error("ChunkedVolumeStore: data.bin write failed"); + + bricks.push_back(json{{"offset", offset}, + {"compressedLen", clen}, + {"bw", bw}, + {"bh", bh}, + {"bd", bd}}); + offset += clen; + } + } + } + data.close(); + + json meta; + meta["nx"] = nx; + meta["ny"] = ny; + meta["nz"] = nz; + meta["brick"] = brick; + meta["origin"] = {b.origin[0], b.origin[1], b.origin[2]}; + meta["spacing"] = {b.spacing[0], b.spacing[1], b.spacing[2]}; + meta["quant"] = {{"scale", b.quant.scale}, {"offset", b.quant.offset}}; + meta["vminPhys"] = b.vminPhys; + meta["vmaxPhys"] = b.vmaxPhys; + meta["bricks"] = std::move(bricks); + + std::ofstream metaOut((fs::path(dir) / kMetaFile).string(), + std::ios::trunc); + if (!metaOut) throw std::runtime_error("ChunkedVolumeStore: cannot open meta.json for write"); + metaOut << meta.dump(2); + if (!metaOut) throw std::runtime_error("ChunkedVolumeStore: meta.json write failed"); +} + +StoreMeta ChunkedVolumeStore::readMeta(const std::string& dir) { + std::ifstream in((fs::path(dir) / kMetaFile).string()); + if (!in) throw std::runtime_error("ChunkedVolumeStore: cannot open meta.json"); + json meta; + in >> meta; + + StoreMeta m; + m.nx = meta.at("nx").get(); + m.ny = meta.at("ny").get(); + m.nz = meta.at("nz").get(); + m.brick = meta.at("brick").get(); + const auto& o = meta.at("origin"); + m.origin = {o[0].get(), o[1].get(), o[2].get()}; + const auto& s = meta.at("spacing"); + m.spacing = {s[0].get(), s[1].get(), s[2].get()}; + m.quant.scale = meta.at("quant").at("scale").get(); + m.quant.offset = meta.at("quant").at("offset").get(); + m.vminPhys = meta.at("vminPhys").get(); + m.vmaxPhys = meta.at("vmaxPhys").get(); + return m; +} + +ChunkedVolumeStore::ChunkedVolumeStore(const std::string& dir) : dir_(dir) { + std::ifstream in((fs::path(dir) / kMetaFile).string()); + if (!in) throw std::runtime_error("ChunkedVolumeStore: cannot open meta.json"); + json meta; + in >> meta; + + meta_.nx = meta.at("nx").get(); + meta_.ny = meta.at("ny").get(); + meta_.nz = meta.at("nz").get(); + meta_.brick = meta.at("brick").get(); + const auto& o = meta.at("origin"); + meta_.origin = {o[0].get(), o[1].get(), o[2].get()}; + const auto& sp = meta.at("spacing"); + meta_.spacing = {sp[0].get(), sp[1].get(), sp[2].get()}; + meta_.quant.scale = meta.at("quant").at("scale").get(); + meta_.quant.offset = meta.at("quant").at("offset").get(); + meta_.vminPhys = meta.at("vminPhys").get(); + meta_.vmaxPhys = meta.at("vmaxPhys").get(); + + bricksX_ = ceilDiv(meta_.nx, meta_.brick); + bricksY_ = ceilDiv(meta_.ny, meta_.brick); + bricksZ_ = ceilDiv(meta_.nz, meta_.brick); + + const auto& arr = meta.at("bricks"); + bricks_.reserve(arr.size()); + for (const auto& e : arr) { + BrickEntry be; + be.offset = e.at("offset").get(); + be.compressedLen = e.at("compressedLen").get(); + be.bw = e.at("bw").get(); + be.bh = e.at("bh").get(); + be.bd = e.at("bd").get(); + bricks_.push_back(be); + } +} + +std::vector ChunkedVolumeStore::readBrick(int bx, int by, + int bz) const { + if (bx < 0 || by < 0 || bz < 0 || bx >= bricksX_ || by >= bricksY_ || + bz >= bricksZ_) { + throw std::out_of_range("ChunkedVolumeStore::readBrick: brick index out of range"); + } + const BrickEntry& be = bricks_.at(static_cast(brickIndex(bx, by, bz))); + + std::ifstream data((fs::path(dir_) / kDataFile).string(), std::ios::binary); + if (!data) throw std::runtime_error("ChunkedVolumeStore: cannot open data.bin"); + data.seekg(static_cast(be.offset), std::ios::beg); + + QByteArray compressed; + compressed.resize(static_cast(be.compressedLen)); + data.read(compressed.data(), be.compressedLen); + if (!data) throw std::runtime_error("ChunkedVolumeStore: data.bin read failed"); + + const QByteArray raw = qUncompress(compressed); + const std::size_t count = + static_cast(be.bw) * be.bh * be.bd; + if (static_cast(raw.size()) != count * sizeof(std::int16_t)) { + throw std::runtime_error("ChunkedVolumeStore: decompressed size mismatch"); + } + + std::vector out(count); + std::memcpy(out.data(), raw.constData(), raw.size()); + return out; +} + +} // namespace geopro::data diff --git a/src/data/store/ChunkedVolumeStore.hpp b/src/data/store/ChunkedVolumeStore.hpp new file mode 100644 index 0000000..af42357 --- /dev/null +++ b/src/data/store/ChunkedVolumeStore.hpp @@ -0,0 +1,73 @@ +#ifndef GEOPRO_DATA_STORE_CHUNKEDVOLUMESTORE_HPP +#define GEOPRO_DATA_STORE_CHUNKEDVOLUMESTORE_HPP + +#include +#include +#include +#include + +#include "model/ScalarVolumeI16.hpp" // geopro::core::Quant + +namespace geopro::core { +struct BuiltI16; // src/core/algo/GprVolumeBuilder.hpp +} + +namespace geopro::data { + +// 分块存储的 sidecar 元数据（meta.json 反序列化结果，不含逐块索引）。 +struct StoreMeta { + int nx = 0, ny = 0, nz = 0; + int brick = 64; + std::array origin{{0, 0, 0}}; + std::array spacing{{0, 0, 0}}; + geopro::core::Quant quant; // scale/offset + double vminPhys = 0, vmaxPhys = 0; +}; + +// GPR 三维体的分块压缩落盘（B/C 共用基座）。 +// 格式：dir/meta.json（几何 + 量化 + 逐块索引）+ dir/data.bin（逐块 qCompress 流）。 +// 块布局与体一致（块内 i 最快、k 最慢）；边缘块尺寸 < brick。 +// 偏移/长度全程 64 位（块偏移可能 > 2GB）。 +class ChunkedVolumeStore { + public: + // 落盘：dir/meta.json + dir/data.bin。逐块 int16 → qCompress 压缩流， + // 块索引/偏移/压缩长度记入 meta.json。dir 不存在则创建。 + static void write(const std::string& dir, const geopro::core::BuiltI16& b, + int brick = 64); + + // 只读 meta.json（不打开 data.bin）。 + static StoreMeta readMeta(const std::string& dir); + + // 读 meta + 打开 data.bin。 + explicit ChunkedVolumeStore(const std::string& dir); + + const StoreMeta& meta() const { return meta_; } + + int bricksX() const { return bricksX_; } + int bricksY() const { return bricksY_; } + int bricksZ() const { return bricksZ_; } + + // 读单块 → 还原 int16 vector。返回长度 = bw*bh*bd（内部块 = brick³，边缘块更小）。 + std::vector readBrick(int bx, int by, int bz) const; + + private: + // 单块在 data.bin 中的位置与未压缩尺寸。 + struct BrickEntry { + std::int64_t offset = 0; + std::int64_t compressedLen = 0; + int bw = 0, bh = 0, bd = 0; + }; + + int brickIndex(int bx, int by, int bz) const { + return (bz * bricksY_ + by) * bricksX_ + bx; + } + + std::string dir_; + StoreMeta meta_; + int bricksX_ = 0, bricksY_ = 0, bricksZ_ = 0; + std::vector bricks_; +}; + +} // namespace geopro::data + +#endif // GEOPRO_DATA_STORE_CHUNKEDVOLUMESTORE_HPP diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2a02e8f..17cf11c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -54,6 +54,10 @@ target_sources(geopro_tests PRIVATE data/test_async_repo_dispatch.cpp) target_sources(geopro_tests PRIVATE data/test_nav_request.cpp) target_link_libraries(geopro_tests PRIVATE geopro_data) +# store 层：ChunkedVolumeStore（GPR 三维体分块压缩落盘 round-trip + 边缘块 + 压缩生效）。 +target_sources(geopro_tests PRIVATE data/store/test_chunked_volume_store.cpp) +target_link_libraries(geopro_tests PRIVATE geopro_store) + # net 层：RSA 加密器。测试需直接用 OpenSSL 生成/解密密钥，故显式 find_package # 并链接 OpenSSL（geopro_net 的 PUBLIC 链接通常已传递，这里显式以防头文件找不到）。 find_package(OpenSSL REQUIRED) diff --git a/tests/data/store/test_chunked_volume_store.cpp b/tests/data/store/test_chunked_volume_store.cpp new file mode 100644 index 0000000..181be77 --- /dev/null +++ b/tests/data/store/test_chunked_volume_store.cpp @@ -0,0 +1,43 @@ +#include "data/store/ChunkedVolumeStore.hpp" +#include "core/algo/GprVolumeBuilder.hpp" +#include +#include +#include +using namespace geopro::data; +namespace { +geopro::core::BuiltI16 makeBuilt(int n) { + geopro::core::BuiltI16 b; + b.vol = geopro::core::ScalarVolumeI16(n,n,n); + for (auto& v : b.vol.data()) v = 7; // 常量→高压缩比 + b.quant = {1.0, 0.0}; b.origin={{0,0,0}}; b.spacing={{1,1,1}}; + b.vminPhys=0; b.vmaxPhys=7; + return b; +} +} +TEST(ChunkedVolumeStore, RoundTripBrickAndCompresses) { + auto dir = (std::filesystem::temp_directory_path() / "gpr_store_test").string(); + std::filesystem::remove_all(dir); + auto b = makeBuilt(128); + ChunkedVolumeStore::write(dir, b, 64); + auto m = ChunkedVolumeStore::readMeta(dir); + EXPECT_EQ(m.nx, 128); EXPECT_EQ(m.brick, 64); + ChunkedVolumeStore s(dir); + EXPECT_EQ(s.bricksX(), 2); + auto blk = s.readBrick(0,0,0); + EXPECT_EQ(blk.size(), 64u*64*64); + EXPECT_EQ(blk[0], 7); + auto dataSize = std::filesystem::file_size(std::filesystem::path(dir)/"data.bin"); + EXPECT_LT(dataSize, 128u*128*128*2); // 压缩生效 + std::filesystem::remove_all(dir); +} +TEST(ChunkedVolumeStore, EdgeBrickPartial) { + auto dir = (std::filesystem::temp_directory_path() / "gpr_store_edge").string(); + std::filesystem::remove_all(dir); + auto b = makeBuilt(100); // 100 不整除 64 → 边缘块 36 + ChunkedVolumeStore::write(dir, b, 64); + ChunkedVolumeStore s(dir); + EXPECT_EQ(s.bricksX(), 2); // ceil(100/64)=2 + auto edge = s.readBrick(1,0,0); // x 方向边缘块,bw=36 + EXPECT_EQ(edge.size(), 36u*64*64); + std::filesystem::remove_all(dir); +}