feat/vtk-3d-view #7
|
|
@ -21,3 +21,6 @@ target_include_directories(geopro_data PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
|||
target_link_libraries(geopro_data PUBLIC geopro_core geopro_net Qt6::Core PRIVATE nlohmann_json::nlohmann_json)
|
||||
target_compile_features(geopro_data PUBLIC cxx_std_17)
|
||||
set_target_properties(geopro_data PROPERTIES AUTOMOC ON AUTOUIC OFF AUTORCC OFF)
|
||||
|
||||
# GPR 三维体分块压缩落盘库（geopro_store，B/C 共用基座）。
|
||||
add_subdirectory(store)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
# GPR 三维体分块压缩落盘（B/C 共用基座）。
|
||||
# 压缩用 Qt qCompress/qUncompress（QtCore 自带 zlib，免新依赖）；sidecar 用 nlohmann-json。
|
||||
find_package(nlohmann_json CONFIG REQUIRED)
|
||||
find_package(Qt6 COMPONENTS Core REQUIRED)
|
||||
|
||||
add_library(geopro_store STATIC
|
||||
ChunkedVolumeStore.cpp)
|
||||
|
||||
# include 根 = src/，使 #include "data/store/..." 与 "core/algo/..." 可解析
|
||||
# （geopro_tests 链 geopro_store 后透传）。
|
||||
target_include_directories(geopro_store PUBLIC ${CMAKE_SOURCE_DIR}/src)
|
||||
target_link_libraries(geopro_store
|
||||
PUBLIC geopro_core Qt6::Core
|
||||
PRIVATE nlohmann_json::nlohmann_json)
|
||||
target_compile_features(geopro_store PUBLIC cxx_std_17)
|
||||
set_target_properties(geopro_store PROPERTIES AUTOMOC OFF AUTOUIC OFF AUTORCC OFF)
|
||||
|
|
@ -0,0 +1,203 @@
|
|||
#include "data/store/ChunkedVolumeStore.hpp"
|
||||
|
||||
#include <QByteArray>
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <stdexcept>
|
||||
|
||||
#include "core/algo/GprVolumeBuilder.hpp" // geopro::core::BuiltI16
|
||||
|
||||
namespace geopro::data {
|
||||
|
||||
namespace {
|
||||
using nlohmann::json;
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
constexpr const char* kMetaFile = "meta.json";
|
||||
constexpr const char* kDataFile = "data.bin";
|
||||
|
||||
int ceilDiv(int n, int brick) { return (n + brick - 1) / brick; }
|
||||
|
||||
// 块尺寸（边缘块 < brick）：第 b 块沿该轴的体素数。
|
||||
int extent(int n, int b, int brick) {
|
||||
const int got = n - b * brick;
|
||||
return got < brick ? got : brick;
|
||||
}
|
||||
|
||||
// 从体中拷出一块的 int16（块内 i 最快、k 最慢，与体一致）。
|
||||
std::vector<std::int16_t> sliceBrick(const geopro::core::ScalarVolumeI16& vol,
|
||||
int bx, int by, int bz, int brick,
|
||||
int bw, int bh, int bd) {
|
||||
std::vector<std::int16_t> out(static_cast<std::size_t>(bw) * bh * bd);
|
||||
const int i0 = bx * brick, j0 = by * brick, k0 = bz * brick;
|
||||
std::size_t w = 0;
|
||||
for (int kk = 0; kk < bd; ++kk) {
|
||||
for (int jj = 0; jj < bh; ++jj) {
|
||||
for (int ii = 0; ii < bw; ++ii) {
|
||||
out[w++] = vol.at(i0 + ii, j0 + jj, k0 + kk);
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void ChunkedVolumeStore::write(const std::string& dir,
|
||||
const geopro::core::BuiltI16& b, int brick) {
|
||||
if (brick <= 0) throw std::invalid_argument("ChunkedVolumeStore: brick must be > 0");
|
||||
|
||||
const auto& vol = b.vol;
|
||||
const int nx = vol.nx(), ny = vol.ny(), nz = vol.nz();
|
||||
|
||||
fs::create_directories(fs::path(dir));
|
||||
|
||||
const int bX = ceilDiv(nx, brick);
|
||||
const int bY = ceilDiv(ny, brick);
|
||||
const int bZ = ceilDiv(nz, brick);
|
||||
|
||||
std::ofstream data((fs::path(dir) / kDataFile).string(),
|
||||
std::ios::binary | std::ios::trunc);
|
||||
if (!data) throw std::runtime_error("ChunkedVolumeStore: cannot open data.bin for write");
|
||||
|
||||
json bricks = json::array();
|
||||
std::int64_t offset = 0;
|
||||
|
||||
// 固定遍历顺序：bz 最慢、bx 最快（与 brickIndex 一致）。
|
||||
for (int bz = 0; bz < bZ; ++bz) {
|
||||
for (int by = 0; by < bY; ++by) {
|
||||
for (int bx = 0; bx < bX; ++bx) {
|
||||
const int bw = extent(nx, bx, brick);
|
||||
const int bh = extent(ny, by, brick);
|
||||
const int bd = extent(nz, bz, brick);
|
||||
auto raw = sliceBrick(vol, bx, by, bz, brick, bw, bh, bd);
|
||||
|
||||
const int rawBytes =
|
||||
static_cast<int>(raw.size() * sizeof(std::int16_t));
|
||||
const QByteArray compressed = qCompress(
|
||||
reinterpret_cast<const uchar*>(raw.data()), rawBytes);
|
||||
const std::int64_t clen = compressed.size();
|
||||
|
||||
data.write(compressed.constData(), clen);
|
||||
if (!data) throw std::runtime_error("ChunkedVolumeStore: data.bin write failed");
|
||||
|
||||
bricks.push_back(json{{"offset", offset},
|
||||
{"compressedLen", clen},
|
||||
{"bw", bw},
|
||||
{"bh", bh},
|
||||
{"bd", bd}});
|
||||
offset += clen;
|
||||
}
|
||||
}
|
||||
}
|
||||
data.close();
|
||||
|
||||
json meta;
|
||||
meta["nx"] = nx;
|
||||
meta["ny"] = ny;
|
||||
meta["nz"] = nz;
|
||||
meta["brick"] = brick;
|
||||
meta["origin"] = {b.origin[0], b.origin[1], b.origin[2]};
|
||||
meta["spacing"] = {b.spacing[0], b.spacing[1], b.spacing[2]};
|
||||
meta["quant"] = {{"scale", b.quant.scale}, {"offset", b.quant.offset}};
|
||||
meta["vminPhys"] = b.vminPhys;
|
||||
meta["vmaxPhys"] = b.vmaxPhys;
|
||||
meta["bricks"] = std::move(bricks);
|
||||
|
||||
std::ofstream metaOut((fs::path(dir) / kMetaFile).string(),
|
||||
std::ios::trunc);
|
||||
if (!metaOut) throw std::runtime_error("ChunkedVolumeStore: cannot open meta.json for write");
|
||||
metaOut << meta.dump(2);
|
||||
if (!metaOut) throw std::runtime_error("ChunkedVolumeStore: meta.json write failed");
|
||||
}
|
||||
|
||||
StoreMeta ChunkedVolumeStore::readMeta(const std::string& dir) {
|
||||
std::ifstream in((fs::path(dir) / kMetaFile).string());
|
||||
if (!in) throw std::runtime_error("ChunkedVolumeStore: cannot open meta.json");
|
||||
json meta;
|
||||
in >> meta;
|
||||
|
||||
StoreMeta m;
|
||||
m.nx = meta.at("nx").get<int>();
|
||||
m.ny = meta.at("ny").get<int>();
|
||||
m.nz = meta.at("nz").get<int>();
|
||||
m.brick = meta.at("brick").get<int>();
|
||||
const auto& o = meta.at("origin");
|
||||
m.origin = {o[0].get<double>(), o[1].get<double>(), o[2].get<double>()};
|
||||
const auto& s = meta.at("spacing");
|
||||
m.spacing = {s[0].get<double>(), s[1].get<double>(), s[2].get<double>()};
|
||||
m.quant.scale = meta.at("quant").at("scale").get<double>();
|
||||
m.quant.offset = meta.at("quant").at("offset").get<double>();
|
||||
m.vminPhys = meta.at("vminPhys").get<double>();
|
||||
m.vmaxPhys = meta.at("vmaxPhys").get<double>();
|
||||
return m;
|
||||
}
|
||||
|
||||
ChunkedVolumeStore::ChunkedVolumeStore(const std::string& dir) : dir_(dir) {
|
||||
std::ifstream in((fs::path(dir) / kMetaFile).string());
|
||||
if (!in) throw std::runtime_error("ChunkedVolumeStore: cannot open meta.json");
|
||||
json meta;
|
||||
in >> meta;
|
||||
|
||||
meta_.nx = meta.at("nx").get<int>();
|
||||
meta_.ny = meta.at("ny").get<int>();
|
||||
meta_.nz = meta.at("nz").get<int>();
|
||||
meta_.brick = meta.at("brick").get<int>();
|
||||
const auto& o = meta.at("origin");
|
||||
meta_.origin = {o[0].get<double>(), o[1].get<double>(), o[2].get<double>()};
|
||||
const auto& sp = meta.at("spacing");
|
||||
meta_.spacing = {sp[0].get<double>(), sp[1].get<double>(), sp[2].get<double>()};
|
||||
meta_.quant.scale = meta.at("quant").at("scale").get<double>();
|
||||
meta_.quant.offset = meta.at("quant").at("offset").get<double>();
|
||||
meta_.vminPhys = meta.at("vminPhys").get<double>();
|
||||
meta_.vmaxPhys = meta.at("vmaxPhys").get<double>();
|
||||
|
||||
bricksX_ = ceilDiv(meta_.nx, meta_.brick);
|
||||
bricksY_ = ceilDiv(meta_.ny, meta_.brick);
|
||||
bricksZ_ = ceilDiv(meta_.nz, meta_.brick);
|
||||
|
||||
const auto& arr = meta.at("bricks");
|
||||
bricks_.reserve(arr.size());
|
||||
for (const auto& e : arr) {
|
||||
BrickEntry be;
|
||||
be.offset = e.at("offset").get<std::int64_t>();
|
||||
be.compressedLen = e.at("compressedLen").get<std::int64_t>();
|
||||
be.bw = e.at("bw").get<int>();
|
||||
be.bh = e.at("bh").get<int>();
|
||||
be.bd = e.at("bd").get<int>();
|
||||
bricks_.push_back(be);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::int16_t> ChunkedVolumeStore::readBrick(int bx, int by,
|
||||
int bz) const {
|
||||
if (bx < 0 || by < 0 || bz < 0 || bx >= bricksX_ || by >= bricksY_ ||
|
||||
bz >= bricksZ_) {
|
||||
throw std::out_of_range("ChunkedVolumeStore::readBrick: brick index out of range");
|
||||
}
|
||||
const BrickEntry& be = bricks_.at(static_cast<std::size_t>(brickIndex(bx, by, bz)));
|
||||
|
||||
std::ifstream data((fs::path(dir_) / kDataFile).string(), std::ios::binary);
|
||||
if (!data) throw std::runtime_error("ChunkedVolumeStore: cannot open data.bin");
|
||||
data.seekg(static_cast<std::streamoff>(be.offset), std::ios::beg);
|
||||
|
||||
QByteArray compressed;
|
||||
compressed.resize(static_cast<int>(be.compressedLen));
|
||||
data.read(compressed.data(), be.compressedLen);
|
||||
if (!data) throw std::runtime_error("ChunkedVolumeStore: data.bin read failed");
|
||||
|
||||
const QByteArray raw = qUncompress(compressed);
|
||||
const std::size_t count =
|
||||
static_cast<std::size_t>(be.bw) * be.bh * be.bd;
|
||||
if (static_cast<std::size_t>(raw.size()) != count * sizeof(std::int16_t)) {
|
||||
throw std::runtime_error("ChunkedVolumeStore: decompressed size mismatch");
|
||||
}
|
||||
|
||||
std::vector<std::int16_t> out(count);
|
||||
std::memcpy(out.data(), raw.constData(), raw.size());
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace geopro::data
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
#ifndef GEOPRO_DATA_STORE_CHUNKEDVOLUMESTORE_HPP
|
||||
#define GEOPRO_DATA_STORE_CHUNKEDVOLUMESTORE_HPP
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "model/ScalarVolumeI16.hpp" // geopro::core::Quant
|
||||
|
||||
namespace geopro::core {
|
||||
struct BuiltI16; // src/core/algo/GprVolumeBuilder.hpp
|
||||
}
|
||||
|
||||
namespace geopro::data {
|
||||
|
||||
// 分块存储的 sidecar 元数据（meta.json 反序列化结果，不含逐块索引）。
|
||||
struct StoreMeta {
|
||||
int nx = 0, ny = 0, nz = 0;
|
||||
int brick = 64;
|
||||
std::array<double, 3> origin{{0, 0, 0}};
|
||||
std::array<double, 3> spacing{{0, 0, 0}};
|
||||
geopro::core::Quant quant; // scale/offset
|
||||
double vminPhys = 0, vmaxPhys = 0;
|
||||
};
|
||||
|
||||
// GPR 三维体的分块压缩落盘（B/C 共用基座）。
|
||||
// 格式：dir/meta.json（几何 + 量化 + 逐块索引）+ dir/data.bin（逐块 qCompress 流）。
|
||||
// 块布局与体一致（块内 i 最快、k 最慢）；边缘块尺寸 < brick。
|
||||
// 偏移/长度全程 64 位（块偏移可能 > 2GB）。
|
||||
class ChunkedVolumeStore {
|
||||
public:
|
||||
// 落盘：dir/meta.json + dir/data.bin。逐块 int16 → qCompress 压缩流，
|
||||
// 块索引/偏移/压缩长度记入 meta.json。dir 不存在则创建。
|
||||
static void write(const std::string& dir, const geopro::core::BuiltI16& b,
|
||||
int brick = 64);
|
||||
|
||||
// 只读 meta.json（不打开 data.bin）。
|
||||
static StoreMeta readMeta(const std::string& dir);
|
||||
|
||||
// 读 meta + 打开 data.bin。
|
||||
explicit ChunkedVolumeStore(const std::string& dir);
|
||||
|
||||
const StoreMeta& meta() const { return meta_; }
|
||||
|
||||
int bricksX() const { return bricksX_; }
|
||||
int bricksY() const { return bricksY_; }
|
||||
int bricksZ() const { return bricksZ_; }
|
||||
|
||||
// 读单块 → 还原 int16 vector。返回长度 = bw*bh*bd（内部块 = brick³，边缘块更小）。
|
||||
std::vector<std::int16_t> readBrick(int bx, int by, int bz) const;
|
||||
|
||||
private:
|
||||
// 单块在 data.bin 中的位置与未压缩尺寸。
|
||||
struct BrickEntry {
|
||||
std::int64_t offset = 0;
|
||||
std::int64_t compressedLen = 0;
|
||||
int bw = 0, bh = 0, bd = 0;
|
||||
};
|
||||
|
||||
int brickIndex(int bx, int by, int bz) const {
|
||||
return (bz * bricksY_ + by) * bricksX_ + bx;
|
||||
}
|
||||
|
||||
std::string dir_;
|
||||
StoreMeta meta_;
|
||||
int bricksX_ = 0, bricksY_ = 0, bricksZ_ = 0;
|
||||
std::vector<BrickEntry> bricks_;
|
||||
};
|
||||
|
||||
} // namespace geopro::data
|
||||
|
||||
#endif // GEOPRO_DATA_STORE_CHUNKEDVOLUMESTORE_HPP
|
||||
|
|
@ -54,6 +54,10 @@ target_sources(geopro_tests PRIVATE data/test_async_repo_dispatch.cpp)
|
|||
target_sources(geopro_tests PRIVATE data/test_nav_request.cpp)
|
||||
target_link_libraries(geopro_tests PRIVATE geopro_data)
|
||||
|
||||
# store 层：ChunkedVolumeStore（GPR 三维体分块压缩落盘 round-trip + 边缘块 + 压缩生效）。
|
||||
target_sources(geopro_tests PRIVATE data/store/test_chunked_volume_store.cpp)
|
||||
target_link_libraries(geopro_tests PRIVATE geopro_store)
|
||||
|
||||
# net 层：RSA 加密器。测试需直接用 OpenSSL 生成/解密密钥，故显式 find_package
|
||||
# 并链接 OpenSSL（geopro_net 的 PUBLIC 链接通常已传递，这里显式以防头文件找不到）。
|
||||
find_package(OpenSSL REQUIRED)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,43 @@
|
|||
#include "data/store/ChunkedVolumeStore.hpp"
|
||||
#include "core/algo/GprVolumeBuilder.hpp"
|
||||
#include <gtest/gtest.h>
|
||||
#include <filesystem>
|
||||
#include <cstdint>
|
||||
using namespace geopro::data;
|
||||
namespace {
|
||||
geopro::core::BuiltI16 makeBuilt(int n) {
|
||||
geopro::core::BuiltI16 b;
|
||||
b.vol = geopro::core::ScalarVolumeI16(n,n,n);
|
||||
for (auto& v : b.vol.data()) v = 7; // 常量→高压缩比
|
||||
b.quant = {1.0, 0.0}; b.origin={{0,0,0}}; b.spacing={{1,1,1}};
|
||||
b.vminPhys=0; b.vmaxPhys=7;
|
||||
return b;
|
||||
}
|
||||
}
|
||||
TEST(ChunkedVolumeStore, RoundTripBrickAndCompresses) {
|
||||
auto dir = (std::filesystem::temp_directory_path() / "gpr_store_test").string();
|
||||
std::filesystem::remove_all(dir);
|
||||
auto b = makeBuilt(128);
|
||||
ChunkedVolumeStore::write(dir, b, 64);
|
||||
auto m = ChunkedVolumeStore::readMeta(dir);
|
||||
EXPECT_EQ(m.nx, 128); EXPECT_EQ(m.brick, 64);
|
||||
ChunkedVolumeStore s(dir);
|
||||
EXPECT_EQ(s.bricksX(), 2);
|
||||
auto blk = s.readBrick(0,0,0);
|
||||
EXPECT_EQ(blk.size(), 64u*64*64);
|
||||
EXPECT_EQ(blk[0], 7);
|
||||
auto dataSize = std::filesystem::file_size(std::filesystem::path(dir)/"data.bin");
|
||||
EXPECT_LT(dataSize, 128u*128*128*2); // 压缩生效
|
||||
std::filesystem::remove_all(dir);
|
||||
}
|
||||
TEST(ChunkedVolumeStore, EdgeBrickPartial) {
|
||||
auto dir = (std::filesystem::temp_directory_path() / "gpr_store_edge").string();
|
||||
std::filesystem::remove_all(dir);
|
||||
auto b = makeBuilt(100); // 100 不整除 64 → 边缘块 36
|
||||
ChunkedVolumeStore::write(dir, b, 64);
|
||||
ChunkedVolumeStore s(dir);
|
||||
EXPECT_EQ(s.bricksX(), 2); // ceil(100/64)=2
|
||||
auto edge = s.readBrick(1,0,0); // x 方向边缘块,bw=36
|
||||
EXPECT_EQ(edge.size(), 36u*64*64);
|
||||
std::filesystem::remove_all(dir);
|
||||
}
|
||||
Loading…
Reference in New Issue