Skip to content

Commit 2fb5fbf

Browse files
yuchen0ccyuchen0cc
authored
Merge pull request #262 from BigVan/zfile_digest
save digest of zfile's header/trailer and index
2 parents 3a3b2c7 + 7315c31 commit 2fb5fbf

3 files changed

Lines changed: 88 additions & 12 deletions

File tree

‎src/overlaybd/zfile/format_spec.md‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,15 @@ The format of header is described as below. All fields are little-endian.
2121
| :---: | :----: | :----: | :--- |
2222
| magic0 | 0 | 8 | "ZFile\0\1" (and an implicit '\0') |
2323
| magic1 | 8 | 16 | 74 75 6A 69, 2E 79 79 66, 40 41 6C 69, 62 61 62 61 |
24-
| size | 24 | uint32_t | size of the header struct (108), excluding the tail padding |
25-
| reserved| 28 | 4 | reserved space, should be 0 |
24+
| size | 24 | uint32_t | size of the header structure, excluding the tail padding |
25+
| digest | 28 | uint32_t | checksum for the range 28-511 bytes in header |
2626
| flags | 32 | uint64_t | bits for flags* (see later for details) |
2727
| index_offset | 40 | uint64_t | index offset |
28-
| index_size | 48 | uint64_t | size of the index section, possibly compressed|
28+
| index_size | 48 | uint64_t | size of the index section, possibly compressed base on flags |
2929
| original_file_size | 56 | uint64_t | size of the orignal file before compression |
30-
| reserved| 64 | 8 | reserved space, should be 0 |
31-
| block_size | 72 | uint32_t | size of each compression block |
30+
| index_crc | 64 | uint32_t | checksum value of index |
31+
| reserved| 68 | 4 | reserved space, should be 0 |
32+
| block_size| 72 | uint32_t | size of each compression block |
3233
| algo | 76 | uint8_t | compression algorithm |
3334
| level | 77 | uint8_t | compression level |
3435
| use_dict| 78 | bool | whether use dictionary |
@@ -45,7 +46,9 @@ The format of header is described as below. All fields are little-endian.
4546
| type | 1 | this is a data file (1) or index file (0) |
4647
| sealed | 2 | this file is sealed (1) or not (0) |
4748
| info_valid | 3 | information validity of the fields *after* flags (they were initially invalid (0) after creation; and readers must resort to trailer when they meet such headers) |
48-
| reserved | 4~63 | reserved for future use; must be 0s |
49+
| digest | 4 | the digest of this header/trailer has been recorded in the digest field |
50+
| index_comperssion | 5 | whether the index has been compressed(1) or not(0) |
51+
| reserved | 6~63 | reserved for future use; must be 0s |
4952

5053

5154
## index

‎src/overlaybd/zfile/test/test.cpp‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,38 @@ TEST_F(ZFileTest, validation_check) {
227227
EXPECT_NE(zfile_validation_check(fdst.get()), 0);
228228
}
229229

230+
TEST_F(ZFileTest, ht_check) {
231+
// log_output_level = 1;
232+
auto fn_src = "verify.data";
233+
auto fn_zfile = "verify.zfile";
234+
auto src = lfs->open(fn_src, O_CREAT | O_TRUNC | O_RDWR /*| O_DIRECT */, 0644);
235+
unique_ptr<IFile> fsrc(src);
236+
if (!fsrc) {
237+
LOG_ERROR("err: `(`)", errno, strerror(errno));
238+
}
239+
randwrite(fsrc.get(), 1024);
240+
struct stat _st;
241+
if (fsrc->fstat(&_st) != 0) {
242+
LOG_ERROR("err: `(`)", errno, strerror(errno));
243+
return;
244+
}
245+
auto dst = lfs->open(fn_zfile, O_CREAT | O_TRUNC | O_RDWR /*| O_DIRECT */, 0644);
246+
unique_ptr<IFile> fdst(dst);
247+
if (!fdst) {
248+
LOG_ERROR("err: `(`)", errno, strerror(errno));
249+
}
250+
CompressOptions opt;
251+
opt.algo = CompressOptions::LZ4;
252+
opt.verify = 1;
253+
CompressArgs args(opt);
254+
int ret = zfile_compress(fsrc.get(), fdst.get(), &args);
255+
EXPECT_EQ(ret, 0);
256+
auto x=2324;
257+
dst->pwrite(&x, sizeof(x), 400);
258+
EXPECT_NE(zfile_validation_check(fdst.get()), 0);
259+
EXPECT_EQ(is_zfile(dst), -1);
260+
}
261+
230262
TEST_F(ZFileTest, dsa) {
231263
const int buf_size = 1024;
232264
const int crc_count = 3000;

‎src/overlaybd/zfile/zfile.cpp‎

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,16 @@ class CompressionFile : public VirtualReadOnlyFile {
8686

8787
// offset 24, 28, 32
8888
uint32_t size = sizeof(HeaderTrailer);
89-
uint32_t __padding = 0;
89+
// uint32_t __padding = 0;
90+
uint32_t digest = 0;
9091
uint64_t flags;
9192

9293
static const uint32_t FLAG_SHIFT_HEADER = 0; // 1:header 0:trailer
9394
static const uint32_t FLAG_SHIFT_TYPE = 1; // 1:data file, 0:index file
9495
static const uint32_t FLAG_SHIFT_SEALED = 2; // 1:YES, 0:NO
95-
static const uint32_t FLAG_SHIFT_HEADER_OVERWRITE = 3;
96+
static const uint32_t FLAG_SHIFT_HEADER_OVERWRITE = 3; // overwrite trailer info to header
97+
static const uint32_t FLAG_SHIFT_CALC_DIGEST = 4; // caculate digest for zfile header/trailer and jumptable
98+
static const uint32_t FLAG_SHIFT_IDX_COMP = 5; // compress zfile index(jumptable)
9699

97100
uint32_t get_flag_bit(uint32_t shift) const {
98101
return flags & (1 << shift);
@@ -121,6 +124,17 @@ class CompressionFile : public VirtualReadOnlyFile {
121124
bool is_sealed() const {
122125
return get_flag_bit(FLAG_SHIFT_SEALED);
123126
}
127+
bool is_digest_enabled() {
128+
return get_flag_bit(FLAG_SHIFT_CALC_DIGEST);
129+
}
130+
bool is_valid() {
131+
if (!is_digest_enabled()) return true;
132+
auto saved_crc = this->digest;
133+
this->digest = 0;
134+
DEFER(this->digest = saved_crc;);
135+
auto crc = crc32::crc32c(this, CompressionFile::HeaderTrailer::SPACE);
136+
return crc == saved_crc;
137+
}
124138
void set_header() {
125139
set_flag_bit(FLAG_SHIFT_HEADER);
126140
}
@@ -143,6 +157,14 @@ class CompressionFile : public VirtualReadOnlyFile {
143157
set_flag_bit(FLAG_SHIFT_HEADER_OVERWRITE);
144158
}
145159

160+
void set_digest_enable() {
161+
set_flag_bit(FLAG_SHIFT_CALC_DIGEST);
162+
}
163+
164+
void set_compress_index() {
165+
set_flag_bit(FLAG_SHIFT_IDX_COMP);
166+
}
167+
146168
void set_compress_option(const CompressOptions &opt) {
147169
this->opt = opt;
148170
}
@@ -151,7 +173,8 @@ class CompressionFile : public VirtualReadOnlyFile {
151173
uint64_t index_offset; // in bytes
152174
uint64_t index_size; // # of SegmentMappings
153175
uint64_t original_file_size;
154-
uint64_t reserved_0;
176+
uint32_t index_crc;
177+
uint32_t reserved_0;
155178
// offset 72
156179
CompressOptions opt;
157180

@@ -591,6 +614,7 @@ class ZFileBuilder : public VirtualReadOnlyFile {
591614
LOG_ERRNO_RETURN(0, -1, "failed to write index.");
592615
}
593616
auto pht = (CompressionFile::HeaderTrailer *)m_ht;
617+
pht->index_crc = crc32::crc32c(&m_block_len[0], index_bytes);
594618
pht->index_offset = index_offset;
595619
pht->index_size = index_size;
596620
pht->original_file_size = raw_data_size;
@@ -681,7 +705,9 @@ bool load_jump_table(IFile *file, CompressionFile::HeaderTrailer *pheader_traile
681705
if (!pht->verify_magic() || !pht->is_header()) {
682706
LOG_ERROR_RETURN(0, false, "header magic/type don't match");
683707
}
684-
708+
if (pht->is_valid() == false) {
709+
LOG_ERROR_RETURN(0, false, "digest verification failed.");
710+
}
685711
struct stat stat;
686712
ret = file->fstat(&stat);
687713
if (ret < 0) {
@@ -727,12 +753,20 @@ bool load_jump_table(IFile *file, CompressionFile::HeaderTrailer *pheader_traile
727753
if (ret < (ssize_t)index_bytes) {
728754
LOG_ERRNO_RETURN(0, false, "failed to read index");
729755
}
756+
if (pht->is_digest_enabled()) {
757+
LOG_INFO("check jumptable CRC32 (` expected)", pht->index_crc);
758+
auto crc = crc32::crc32c(ibuf.get(), index_bytes);
759+
if (crc != pht->index_crc) {
760+
LOG_ERRNO_RETURN(0, false, "checksum of jumptable is incorrect");
761+
}
762+
}
730763
ret = jump_table.build(ibuf.get(), pht->index_size,
731764
CompressionFile::HeaderTrailer::SPACE + pht->opt.dict_size,
732765
pht->opt.block_size, pht->opt.verify);
733766
if (ret != 0) {
734767
LOG_ERRNO_RETURN(0, false, "failed to build jump table");
735768
}
769+
736770
if (pheader_trailer)
737771
*pheader_trailer = *pht;
738772
return true;
@@ -793,7 +827,10 @@ static int write_header_trailer(IFile *file, bool is_header, bool is_sealed, boo
793827
if (offset != -1)
794828
pht->set_header_overwrite();
795829

796-
LOG_INFO("pht->opt.dict_size: `", pht->opt.dict_size);
830+
pht->set_digest_enable(); // by default
831+
pht->digest = 0;
832+
pht->digest = crc32::crc32c(pht, CompressionFile::HeaderTrailer::SPACE);
833+
LOG_INFO("save header/trailer with digest: `", pht->digest);
797834
if (offset == -1) {
798835
return (int)file->write(pht, CompressionFile::HeaderTrailer::SPACE);
799836
}
@@ -818,7 +855,6 @@ int zfile_compress(IFile *file, IFile *as, const CompressArgs *args) {
818855
char buf[CompressionFile::HeaderTrailer::SPACE] = {};
819856
auto pht = new (buf) CompressionFile::HeaderTrailer;
820857
pht->set_compress_option(opt);
821-
822858
LOG_INFO("write header.");
823859
auto ret = write_header_trailer(as, true, false, true, pht);
824860
if (ret < 0) {
@@ -890,6 +926,7 @@ int zfile_compress(IFile *file, IFile *as, const CompressArgs *args) {
890926
if (as->write(&block_len[0], index_bytes) != index_bytes) {
891927
LOG_ERRNO_RETURN(0, -1, "failed to write index.");
892928
}
929+
pht->index_crc = crc32::crc32c(&block_len[0], index_bytes);
893930
pht->index_offset = index_offset;
894931
pht->index_size = index_size;
895932
pht->original_file_size = raw_data_size;
@@ -968,6 +1005,10 @@ int is_zfile(IFile *file) {
9681005
LOG_DEBUG("file: ` is not a zfile object", file);
9691006
return 0;
9701007
}
1008+
if (!pht->is_valid()) {
1009+
LOG_ERRNO_RETURN(0, -1,
1010+
"file: ` is a zfile object but verify digest failed.", file);
1011+
}
9711012
LOG_DEBUG("file: ` is a zfile object", file);
9721013
return 1;
9731014
}

0 commit comments

Comments
 (0)