Skip to content

Commit 7315c31

Browse files
BigVanBigVan
committed
save digest of zfile's header/trailer and index
Signed-off-by: Yifan Yuan <tuji.yyf@alibaba-inc.com>
1 parent cc42223 commit 7315c31

3 files changed

Lines changed: 110 additions & 28 deletions

File tree

‎src/overlaybd/zfile/format_spec.md‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,15 @@ The format of header is described as below. All fields are little-endian.
2121
| :---: | :----: | :----: | :--- |
2222
| magic0 | 0 | 8 | "ZFile\0\1" (and an implicit '\0') |
2323
| magic1 | 8 | 16 | 74 75 6A 69, 2E 79 79 66, 40 41 6C 69, 62 61 62 61 |
24-
| size | 24 | uint32_t | size of the header struct (108), excluding the tail padding |
25-
| reserved| 28 | 4 | reserved space, should be 0 |
24+
| size | 24 | uint32_t | size of the header structure, excluding the tail padding |
25+
| digest | 28 | uint32_t | checksum for the range 28-511 bytes in header |
2626
| flags | 32 | uint64_t | bits for flags* (see later for details) |
2727
| index_offset | 40 | uint64_t | index offset |
28-
| index_size | 48 | uint64_t | size of the index section, possibly compressed|
28+
| index_size | 48 | uint64_t | size of the index section, possibly compressed base on flags |
2929
| original_file_size | 56 | uint64_t | size of the orignal file before compression |
30-
| reserved| 64 | 8 | reserved space, should be 0 |
31-
| block_size | 72 | uint32_t | size of each compression block |
30+
| index_crc | 64 | uint32_t | checksum value of index |
31+
| reserved| 68 | 4 | reserved space, should be 0 |
32+
| block_size| 72 | uint32_t | size of each compression block |
3233
| algo | 76 | uint8_t | compression algorithm |
3334
| level | 77 | uint8_t | compression level |
3435
| use_dict| 78 | bool | whether use dictionary |
@@ -45,7 +46,9 @@ The format of header is described as below. All fields are little-endian.
4546
| type | 1 | this is a data file (1) or index file (0) |
4647
| sealed | 2 | this file is sealed (1) or not (0) |
4748
| info_valid | 3 | information validity of the fields *after* flags (they were initially invalid (0) after creation; and readers must resort to trailer when they meet such headers) |
48-
| reserved | 4~63 | reserved for future use; must be 0s |
49+
| digest | 4 | the digest of this header/trailer has been recorded in the digest field |
50+
| index_comperssion | 5 | whether the index has been compressed(1) or not(0) |
51+
| reserved | 6~63 | reserved for future use; must be 0s |
4952

5053

5154
## index

‎src/overlaybd/zfile/test/test.cpp‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,38 @@ TEST_F(ZFileTest, validation_check) {
227227
EXPECT_NE(zfile_validation_check(fdst.get()), 0);
228228
}
229229

230+
TEST_F(ZFileTest, ht_check) {
231+
// log_output_level = 1;
232+
auto fn_src = "verify.data";
233+
auto fn_zfile = "verify.zfile";
234+
auto src = lfs->open(fn_src, O_CREAT | O_TRUNC | O_RDWR /*| O_DIRECT */, 0644);
235+
unique_ptr<IFile> fsrc(src);
236+
if (!fsrc) {
237+
LOG_ERROR("err: `(`)", errno, strerror(errno));
238+
}
239+
randwrite(fsrc.get(), 1024);
240+
struct stat _st;
241+
if (fsrc->fstat(&_st) != 0) {
242+
LOG_ERROR("err: `(`)", errno, strerror(errno));
243+
return;
244+
}
245+
auto dst = lfs->open(fn_zfile, O_CREAT | O_TRUNC | O_RDWR /*| O_DIRECT */, 0644);
246+
unique_ptr<IFile> fdst(dst);
247+
if (!fdst) {
248+
LOG_ERROR("err: `(`)", errno, strerror(errno));
249+
}
250+
CompressOptions opt;
251+
opt.algo = CompressOptions::LZ4;
252+
opt.verify = 1;
253+
CompressArgs args(opt);
254+
int ret = zfile_compress(fsrc.get(), fdst.get(), &args);
255+
EXPECT_EQ(ret, 0);
256+
auto x=2324;
257+
dst->pwrite(&x, sizeof(x), 400);
258+
EXPECT_NE(zfile_validation_check(fdst.get()), 0);
259+
EXPECT_EQ(is_zfile(dst), -1);
260+
}
261+
230262
TEST_F(ZFileTest, dsa) {
231263
const int buf_size = 1024;
232264
const int crc_count = 3000;

‎src/overlaybd/zfile/zfile.cpp‎

Lines changed: 69 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,16 @@ class CompressionFile : public VirtualReadOnlyFile {
8686

8787
// offset 24, 28, 32
8888
uint32_t size = sizeof(HeaderTrailer);
89-
uint32_t __padding = 0;
89+
// uint32_t __padding = 0;
90+
uint32_t digest = 0;
9091
uint64_t flags;
9192

9293
static const uint32_t FLAG_SHIFT_HEADER = 0; // 1:header 0:trailer
9394
static const uint32_t FLAG_SHIFT_TYPE = 1; // 1:data file, 0:index file
9495
static const uint32_t FLAG_SHIFT_SEALED = 2; // 1:YES, 0:NO
95-
static const uint32_t FLAG_SHIFT_HEADER_OVERWRITE = 3;
96+
static const uint32_t FLAG_SHIFT_HEADER_OVERWRITE = 3; // overwrite trailer info to header
97+
static const uint32_t FLAG_SHIFT_CALC_DIGEST = 4; // caculate digest for zfile header/trailer and jumptable
98+
static const uint32_t FLAG_SHIFT_IDX_COMP = 5; // compress zfile index(jumptable)
9699

97100
uint32_t get_flag_bit(uint32_t shift) const {
98101
return flags & (1 << shift);
@@ -121,6 +124,17 @@ class CompressionFile : public VirtualReadOnlyFile {
121124
bool is_sealed() const {
122125
return get_flag_bit(FLAG_SHIFT_SEALED);
123126
}
127+
bool is_digest_enabled() {
128+
return get_flag_bit(FLAG_SHIFT_CALC_DIGEST);
129+
}
130+
bool is_valid() {
131+
if (!is_digest_enabled()) return true;
132+
auto saved_crc = this->digest;
133+
this->digest = 0;
134+
DEFER(this->digest = saved_crc;);
135+
auto crc = crc32::crc32c(this, CompressionFile::HeaderTrailer::SPACE);
136+
return crc == saved_crc;
137+
}
124138
void set_header() {
125139
set_flag_bit(FLAG_SHIFT_HEADER);
126140
}
@@ -143,6 +157,14 @@ class CompressionFile : public VirtualReadOnlyFile {
143157
set_flag_bit(FLAG_SHIFT_HEADER_OVERWRITE);
144158
}
145159

160+
void set_digest_enable() {
161+
set_flag_bit(FLAG_SHIFT_CALC_DIGEST);
162+
}
163+
164+
void set_compress_index() {
165+
set_flag_bit(FLAG_SHIFT_IDX_COMP);
166+
}
167+
146168
void set_compress_option(const CompressOptions &opt) {
147169
this->opt = opt;
148170
}
@@ -151,7 +173,8 @@ class CompressionFile : public VirtualReadOnlyFile {
151173
uint64_t index_offset; // in bytes
152174
uint64_t index_size; // # of SegmentMappings
153175
uint64_t original_file_size;
154-
uint64_t reserved_0;
176+
uint32_t index_crc;
177+
uint32_t reserved_0;
155178
// offset 72
156179
CompressOptions opt;
157180

@@ -181,7 +204,8 @@ class CompressionFile : public VirtualReadOnlyFile {
181204
return deltas.size();
182205
}
183206

184-
int build(const uint32_t *ibuf, size_t n, off_t offset_begin, uint32_t block_size) {
207+
int build(const uint32_t *ibuf, size_t n, off_t offset_begin, uint32_t block_size,
208+
bool enable_crc) {
185209
partial_offset.clear();
186210
deltas.clear();
187211
group_size = (uinttype_max + 1) / block_size;
@@ -190,7 +214,11 @@ class CompressionFile : public VirtualReadOnlyFile {
190214
auto raw_offset = offset_begin;
191215
partial_offset.push_back(raw_offset);
192216
deltas.push_back(0);
217+
size_t min_blksize = (enable_crc ? sizeof(uint32_t) : 0);
193218
for (ssize_t i = 1; i < (ssize_t)n + 1; i++) {
219+
if (ibuf[i - 1] <= min_blksize) {
220+
LOG_ERRNO_RETURN(EIO, -1, "unexpected block size(id: `):", i - 1, ibuf[i - 1]);
221+
}
194222
raw_offset += ibuf[i - 1];
195223
if ((i % group_size) == 0) {
196224
partial_offset.push_back(raw_offset);
@@ -199,7 +227,7 @@ class CompressionFile : public VirtualReadOnlyFile {
199227
}
200228
if ((uint64_t)deltas[i - 1] + ibuf[i - 1] >= (uint64_t)uinttype_max) {
201229
LOG_ERROR_RETURN(ERANGE, -1, "build block[`] length failed `+` > ` (exceed)",
202-
i-1, deltas[i-1], ibuf[i-1], (uint64_t)uinttype_max);
230+
i - 1, deltas[i - 1], ibuf[i - 1], (uint64_t)uinttype_max);
203231
}
204232
deltas.push_back(deltas[i - 1] + ibuf[i - 1]);
205233
}
@@ -260,13 +288,14 @@ class CompressionFile : public VirtualReadOnlyFile {
260288
LOG_WARN("trim and reload. (idx: `, offset: `, len: `)", idx, begin_offset, read_size);
261289
int trim_res = m_zfile->m_file->trim(begin_offset, read_size);
262290
if (trim_res < 0) {
263-
LOG_ERRNO_RETURN(0, -1, "trim block failed. (idx: `, offset: `, len: `)",
264-
idx, begin_offset, read_size);
291+
LOG_ERRNO_RETURN(0, -1, "trim block failed. (idx: `, offset: `, len: `)", idx,
292+
begin_offset, read_size);
265293
}
266294
auto readn = m_zfile->m_file->pread(m_buf + m_buf_offset, read_size, begin_offset);
267295
if (readn != (ssize_t)read_size) {
268-
LOG_ERRNO_RETURN(0, -1, "read compressed blocks failed. (idx: `, offset: `, len: `)",
269-
idx, begin_offset, read_size);
296+
LOG_ERRNO_RETURN(0, -1,
297+
"read compressed blocks failed. (idx: `, offset: `, len: `)", idx,
298+
begin_offset, read_size);
270299
}
271300
return 0;
272301
}
@@ -351,8 +380,9 @@ class CompressionFile : public VirtualReadOnlyFile {
351380
compressed_size = m_reader->compressed_size();
352381
if ((size_t)(m_reader->m_buf_offset) + compressed_size > sizeof(m_buf)) {
353382
m_reader->m_eno = ERANGE;
354-
LOG_ERRNO_RETURN(0, -1, "inner buffer offset (`) + compressed size (`) overflow.",
355-
m_reader->m_buf_offset, compressed_size);
383+
LOG_ERRNO_RETURN(0, -1,
384+
"inner buffer offset (`) + compressed size (`) overflow.",
385+
m_reader->m_buf_offset, compressed_size);
356386
}
357387

358388
if (blk_idx == m_reader->m_begin_idx) {
@@ -439,15 +469,15 @@ class CompressionFile : public VirtualReadOnlyFile {
439469
if (count <= 0)
440470
return 0;
441471
if (offset + count > m_ht.original_file_size) {
442-
LOG_ERRNO_RETURN(ERANGE, -1, "pread range exceed (` > `)",
443-
offset + count, m_ht.original_file_size);
472+
LOG_ERRNO_RETURN(ERANGE, -1, "pread range exceed (` > `)", offset + count,
473+
m_ht.original_file_size);
444474
}
445475
ssize_t readn = 0; // final will equal to count
446476
unsigned char raw[MAX_READ_SIZE];
447477
BlockReader br(this, offset, count);
448478
for (auto &block : br) {
449479
if (buf == nullptr) {
450-
//used for prefetch; no copy, no decompress;
480+
// used for prefetch; no copy, no decompress;
451481
readn += block.cp_len;
452482
continue;
453483
}
@@ -506,7 +536,7 @@ static int write_header_trailer(IFile *file, bool is_header, bool is_sealed, boo
506536
CompressionFile::HeaderTrailer *pht, off_t offset = -1);
507537

508538
ssize_t compress_data(ICompressor *compressor, const unsigned char *buf, size_t count,
509-
unsigned char *dest_buf, size_t dest_len, bool gen_crc) {
539+
unsigned char *dest_buf, size_t dest_len, bool gen_crc) {
510540

511541
ssize_t compressed_len = 0;
512542
auto ret = compressor->compress((const unsigned char *)buf, count, dest_buf, dest_len);
@@ -584,6 +614,7 @@ class ZFileBuilder : public VirtualReadOnlyFile {
584614
LOG_ERRNO_RETURN(0, -1, "failed to write index.");
585615
}
586616
auto pht = (CompressionFile::HeaderTrailer *)m_ht;
617+
pht->index_crc = crc32::crc32c(&m_block_len[0], index_bytes);
587618
pht->index_offset = index_offset;
588619
pht->index_size = index_size;
589620
pht->original_file_size = raw_data_size;
@@ -674,7 +705,9 @@ bool load_jump_table(IFile *file, CompressionFile::HeaderTrailer *pheader_traile
674705
if (!pht->verify_magic() || !pht->is_header()) {
675706
LOG_ERROR_RETURN(0, false, "header magic/type don't match");
676707
}
677-
708+
if (pht->is_valid() == false) {
709+
LOG_ERROR_RETURN(0, false, "digest verification failed.");
710+
}
678711
struct stat stat;
679712
ret = file->fstat(&stat);
680713
if (ret < 0) {
@@ -720,12 +753,20 @@ bool load_jump_table(IFile *file, CompressionFile::HeaderTrailer *pheader_traile
720753
if (ret < (ssize_t)index_bytes) {
721754
LOG_ERRNO_RETURN(0, false, "failed to read index");
722755
}
756+
if (pht->is_digest_enabled()) {
757+
LOG_INFO("check jumptable CRC32 (` expected)", pht->index_crc);
758+
auto crc = crc32::crc32c(ibuf.get(), index_bytes);
759+
if (crc != pht->index_crc) {
760+
LOG_ERRNO_RETURN(0, false, "checksum of jumptable is incorrect");
761+
}
762+
}
723763
ret = jump_table.build(ibuf.get(), pht->index_size,
724-
CompressionFile::HeaderTrailer::SPACE + pht->opt.dict_size,
725-
pht->opt.block_size);
764+
CompressionFile::HeaderTrailer::SPACE + pht->opt.dict_size,
765+
pht->opt.block_size, pht->opt.verify);
726766
if (ret != 0) {
727767
LOG_ERRNO_RETURN(0, false, "failed to build jump table");
728768
}
769+
729770
if (pheader_trailer)
730771
*pheader_trailer = *pht;
731772
return true;
@@ -745,8 +786,7 @@ IFile *zfile_open_ro(IFile *file, bool verify, bool ownership) {
745786
auto res = file->fallocate(0, 0, -1);
746787
LOG_ERROR("failed to load jump table, fallocate result: `", res);
747788
if (res < 0) {
748-
LOG_ERRNO_RETURN(0, nullptr,
749-
"failed to load jump table and failed to evict");
789+
LOG_ERRNO_RETURN(0, nullptr, "failed to load jump table and failed to evict");
750790
}
751791
if (retry--) {
752792
LOG_INFO("retry loading jump table");
@@ -787,7 +827,10 @@ static int write_header_trailer(IFile *file, bool is_header, bool is_sealed, boo
787827
if (offset != -1)
788828
pht->set_header_overwrite();
789829

790-
LOG_INFO("pht->opt.dict_size: `", pht->opt.dict_size);
830+
pht->set_digest_enable(); // by default
831+
pht->digest = 0;
832+
pht->digest = crc32::crc32c(pht, CompressionFile::HeaderTrailer::SPACE);
833+
LOG_INFO("save header/trailer with digest: `", pht->digest);
791834
if (offset == -1) {
792835
return (int)file->write(pht, CompressionFile::HeaderTrailer::SPACE);
793836
}
@@ -812,7 +855,6 @@ int zfile_compress(IFile *file, IFile *as, const CompressArgs *args) {
812855
char buf[CompressionFile::HeaderTrailer::SPACE] = {};
813856
auto pht = new (buf) CompressionFile::HeaderTrailer;
814857
pht->set_compress_option(opt);
815-
816858
LOG_INFO("write header.");
817859
auto ret = write_header_trailer(as, true, false, true, pht);
818860
if (ret < 0) {
@@ -884,6 +926,7 @@ int zfile_compress(IFile *file, IFile *as, const CompressArgs *args) {
884926
if (as->write(&block_len[0], index_bytes) != index_bytes) {
885927
LOG_ERRNO_RETURN(0, -1, "failed to write index.");
886928
}
929+
pht->index_crc = crc32::crc32c(&block_len[0], index_bytes);
887930
pht->index_offset = index_offset;
888931
pht->index_size = index_size;
889932
pht->original_file_size = raw_data_size;
@@ -962,6 +1005,10 @@ int is_zfile(IFile *file) {
9621005
LOG_DEBUG("file: ` is not a zfile object", file);
9631006
return 0;
9641007
}
1008+
if (!pht->is_valid()) {
1009+
LOG_ERRNO_RETURN(0, -1,
1010+
"file: ` is a zfile object but verify digest failed.", file);
1011+
}
9651012
LOG_DEBUG("file: ` is a zfile object", file);
9661013
return 1;
9671014
}

0 commit comments

Comments
 (0)