Skip to content

Commit 62a7031

Browse files
committed
Merge branch 'master' of github.com:openvanilla/McBopomofo
2 parents 225977b + 00f6143 commit 62a7031

9 files changed

Lines changed: 309 additions & 74 deletions

Source/Data/BPMFMappings.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3116,6 +3116,9 @@
31163116
七小時後 ㄑㄧ ㄒㄧㄠˇ ㄕˊ ㄏㄡˋ
31173117
七尺 ㄑㄧ ㄔˇ
31183118
七尺之驅 ㄑㄧ ㄔˇ ㄓ ㄑㄩ
3119+
七局 ㄑㄧ ㄐㄩˊ
3120+
七局上半 ㄑㄧ ㄐㄩˊ ㄕㄤˋ ㄅㄢˋ
3121+
七局下半 ㄑㄧ ㄐㄩˊ ㄒㄧㄚˋ ㄅㄢˋ
31193122
七層 ㄑㄧ ㄘㄥˊ
31203123
七層樓 ㄑㄧ ㄘㄥˊ ㄌㄡˊ
31213124
七巧板 ㄑㄧ ㄑㄧㄠˇ ㄅㄢˇ
@@ -3474,6 +3477,8 @@
34743477
三小片 ㄙㄢ ㄒㄧㄠˇ ㄆㄧㄢˋ
34753478
三尺 ㄙㄢ ㄔˇ
34763479
三局 ㄙㄢ ㄐㄩˊ
3480+
三局上半 ㄙㄢ ㄐㄩˊ ㄕㄤˋ ㄅㄢˋ
3481+
三局下半 ㄙㄢ ㄐㄩˊ ㄒㄧㄚˋ ㄅㄢˋ
34773482
三屆 ㄙㄢ ㄐㄧㄝˋ
34783483
三層 ㄙㄢ ㄘㄥˊ
34793484
三層樓 ㄙㄢ ㄘㄥˊ ㄌㄡˊ

Source/Data/phrase.occ

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1705,6 +1705,9 @@
17051705
七小時後 0
17061706
七尺 1
17071707
七尺之驅 0
1708+
七局 1
1709+
七局上半 1
1710+
七局下半 1
17081711
七層 8
17091712
七層樓 5
17101713
七巧板 3
@@ -2063,6 +2066,8 @@
20632066
三小片 0
20642067
三尺 17
20652068
三局 13
2069+
三局上半 1
2070+
三局下半 1
20662071
三屆 57
20672072
三層 31
20682073
三層樓 8
@@ -5974,8 +5979,8 @@
59745979
九寨溝 3
59755980
九小時後 0
59765981
九局 14
5977-
九局上半 0
5978-
九局下半 0
5982+
九局上半 1
5983+
九局下半 1
59795984
九屆 20
59805985
九層塔 11
59815986
九層雲外 0
@@ -6736,8 +6741,8 @@
67366741
二專制 0
67376742
二尖瓣 0
67386743
二局 8
6739-
二局上半 0
6740-
二局下半 0
6744+
二局上半 1
6745+
二局下半 1
67416746
二屆 104
67426747
二屆國代 1
67436748
二屆國代選舉 0
@@ -7237,8 +7242,8 @@
72377242
五小時後 0
72387243
五尺 4
72397244
五局 9
7240-
五局上半 0
7241-
五局下半 0
7245+
五局上半 1
7246+
五局下半 1
72427247
五屆 60
72437248
五層 22
72447249
五層樓 8
@@ -15687,8 +15692,8 @@
1568715692
八小時後 0
1568815693
八尺 2
1568915694
八局 9
15690-
八局上半 0
15691-
八局下半 0
15695+
八局上半 1
15696+
八局下半 1
1569215697
八屆 19
1569315698
八層 8
1569415699
八層樓 3
@@ -16304,8 +16309,8 @@
1630416309
六尺褌 0
1630516310
六尾 0
1630616311
六局 5
16307-
六局上半 0
16308-
六局下半 0
16312+
六局上半 1
16313+
六局下半 1
1630916314
六屆 23
1631016315
六巷 11
1631116316
六師 3

Source/Engine/CMakeLists.txt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,5 +143,15 @@ if (ENABLE_TEST)
143143
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/ParselessLMBenchmark
144144
)
145145
add_dependencies(runParselessLMBenchmark ParselessLMBenchmark)
146+
147+
add_executable(ParselessPhraseDBBenchmark
148+
ParselessPhraseDBBenchmark.cpp)
149+
target_link_libraries(ParselessPhraseDBBenchmark McBopomofoLMLib benchmark::benchmark)
150+
151+
add_custom_target(
152+
runParselessPhraseDBBenchmark
153+
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/ParselessPhraseDBBenchmark
154+
)
155+
add_dependencies(runParselessPhraseDBBenchmark ParselessPhraseDBBenchmark)
146156
endif ()
147157
endif ()

Source/Engine/MemoryMappedFile.cpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ namespace McBopomofo {
3535
MemoryMappedFile::MemoryMappedFile(MemoryMappedFile&& other) noexcept
3636
: fd_(std::exchange(other.fd_, -1)),
3737
data_(std::exchange(other.data_, nullptr)),
38-
length_(std::exchange(other.length_, -1)) {}
38+
length_(std::exchange(other.length_, 0)) {}
3939

4040
MemoryMappedFile& MemoryMappedFile::operator=(
4141
MemoryMappedFile&& other) noexcept {
@@ -49,7 +49,7 @@ MemoryMappedFile& MemoryMappedFile::operator=(
4949
MemoryMappedFile::~MemoryMappedFile() { close(); }
5050

5151
bool MemoryMappedFile::open(const char* path) {
52-
if (data_) {
52+
if (fd_ != -1) {
5353
return false;
5454
}
5555

@@ -67,6 +67,7 @@ bool MemoryMappedFile::open(const char* path) {
6767

6868
length_ = static_cast<size_t>(sb.st_size);
6969

70+
// No need to check if length_ is 0; mmmap fails on empty files.
7071
data_ = mmap(nullptr, length_, PROT_READ, MAP_SHARED, fd_, 0);
7172
if (data_ == MAP_FAILED) {
7273
::close(fd_);
@@ -80,7 +81,7 @@ bool MemoryMappedFile::open(const char* path) {
8081
}
8182

8283
void MemoryMappedFile::close() {
83-
if (data_ == nullptr) {
84+
if (fd_ == -1) {
8485
return;
8586
}
8687
munmap(data_, length_);

Source/Engine/MemoryMappedFile.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ class MemoryMappedFile {
4848
bool open(const char* path);
4949
void close();
5050

51+
bool isOpen() const { return fd_ != -1; }
52+
5153
[[nodiscard]] const char* data() const {
5254
return static_cast<const char*>(data_);
5355
}

Source/Engine/MemoryMappedFileTest.cpp

Lines changed: 94 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
2222
// OTHER DEALINGS IN THE SOFTWARE.
2323

24+
#include <cassert>
2425
#include <cstdio>
2526
#include <filesystem>
2627
#include <fstream>
@@ -33,48 +34,89 @@
3334

3435
namespace McBopomofo {
3536

37+
class TempFile {
38+
public:
39+
TempFile(const char* initData, size_t length) {
40+
std::filesystem::path p = std::filesystem::temp_directory_path() /
41+
"org.openvanilla.mcbopomofo.XXXXXX";
42+
path_ = p.native();
43+
44+
int fd = mkstemp(path_.data());
45+
assert(fd != -1);
46+
47+
ssize_t written = write(fd, initData, length);
48+
assert(static_cast<size_t>(written) == length);
49+
(void)written;
50+
51+
close(fd);
52+
}
53+
54+
~TempFile() {
55+
std::error_code ec;
56+
bool result = std::filesystem::remove(path_, ec);
57+
assert(result);
58+
(void)result;
59+
}
60+
61+
const char* path() const { return path_.c_str(); }
62+
63+
protected:
64+
std::string path_;
65+
};
66+
67+
class TempDir {
68+
public:
69+
TempDir() {
70+
std::filesystem::path p = std::filesystem::temp_directory_path() /
71+
"org.openvanilla.mcbopomofo.XXXXXX";
72+
path_ = p.native();
73+
char* result = mkdtemp(path_.data());
74+
assert(result != nullptr);
75+
fprintf(stderr, "TEMPDIR: %s\n", result);
76+
}
77+
78+
~TempDir() {
79+
std::error_code ec;
80+
bool result = std::filesystem::remove(path_, ec);
81+
assert(result);
82+
(void)result;
83+
}
84+
85+
const char* path() const { return path_.c_str(); }
86+
87+
protected:
88+
std::string path_;
89+
};
90+
3691
TEST(MemoryMappedFileTest, UnopenedInstance) {
3792
MemoryMappedFile mf;
93+
EXPECT_FALSE(mf.isOpen());
3894
EXPECT_EQ(mf.length(), 0);
3995
EXPECT_EQ(mf.data(), nullptr);
4096
}
4197

4298
TEST(MemoryMappedFileTest, BasicFunctionalities) {
43-
std::random_device rd;
44-
std::default_random_engine re(rd());
45-
std::uniform_int_distribution<unsigned int> suffix_gen(0);
46-
47-
std::string prefix("org.openvanilla.mcbopomofo.memorymappedfiletest-");
48-
std::filesystem::path tmp_file_path;
49-
50-
constexpr int kMaxRetry = 10;
51-
for (int i = 0; i < kMaxRetry; ++i) {
52-
std::string filename = prefix + std::to_string(suffix_gen(re));
53-
std::filesystem::path p = std::filesystem::temp_directory_path() / filename;
54-
if (!std::filesystem::exists(p)) {
55-
tmp_file_path = p;
56-
break;
57-
}
58-
}
59-
ASSERT_FALSE(tmp_file_path.empty()) << "Must form a temp filename";
60-
6199
constexpr size_t kBufSize = 4 * 1024 * 1024;
62100
uint8_t* buf = new uint8_t[kBufSize];
63101

102+
std::random_device rd;
103+
std::default_random_engine re(rd());
104+
64105
std::uniform_int_distribution<unsigned int> randchar(0, 255);
65106
for (size_t i = 0; i < kBufSize; ++i) {
66107
buf[i] = static_cast<uint8_t>(randchar(re) & 0xff);
67108
}
68109

69-
std::ofstream out(tmp_file_path, std::ios::binary);
70-
out.write(reinterpret_cast<char*>(buf), kBufSize);
71-
out.close();
110+
std::unique_ptr<TempFile> temp =
111+
std::make_unique<TempFile>(reinterpret_cast<char*>(buf), kBufSize);
72112

73113
MemoryMappedFile mf;
74-
ASSERT_EQ(mf.length(), 0);
75-
ASSERT_EQ(mf.data(), nullptr);
76-
bool open_result = mf.open(tmp_file_path.c_str());
77-
ASSERT_TRUE(open_result);
114+
EXPECT_EQ(mf.length(), 0);
115+
EXPECT_EQ(mf.data(), nullptr);
116+
EXPECT_FALSE(mf.isOpen());
117+
bool open_result = mf.open(temp->path());
118+
EXPECT_TRUE(open_result);
119+
EXPECT_TRUE(mf.isOpen());
78120

79121
EXPECT_EQ(mf.length(), kBufSize);
80122
EXPECT_TRUE(mf.data() != nullptr);
@@ -83,25 +125,36 @@ TEST(MemoryMappedFileTest, BasicFunctionalities) {
83125
mf.close();
84126
EXPECT_EQ(mf.length(), 0);
85127
EXPECT_EQ(mf.data(), nullptr);
128+
EXPECT_FALSE(mf.isOpen());
86129

87130
// Should be a no-op.
88131
mf.close();
89132

90133
MemoryMappedFile mf2;
91-
open_result = mf2.open(tmp_file_path.c_str());
134+
open_result = mf2.open(temp->path());
92135
EXPECT_TRUE(open_result);
136+
EXPECT_EQ(mf2.length(), kBufSize);
93137
EXPECT_TRUE(mf2.data() != nullptr);
138+
EXPECT_TRUE(mf2.isOpen());
94139

95140
MemoryMappedFile mf3(std::move(mf2));
141+
EXPECT_EQ(mf2.length(), 0);
96142
EXPECT_TRUE(mf2.data() == nullptr);
143+
EXPECT_EQ(mf3.length(), kBufSize);
97144
EXPECT_TRUE(mf3.data() != nullptr);
145+
EXPECT_FALSE(mf2.isOpen());
146+
EXPECT_TRUE(mf3.isOpen());
98147

99148
MemoryMappedFile mf4 = std::move(mf3);
149+
EXPECT_EQ(mf3.length(), 0);
100150
EXPECT_TRUE(mf3.data() == nullptr);
151+
EXPECT_EQ(mf4.length(), kBufSize);
101152
EXPECT_TRUE(mf4.data() != nullptr);
153+
EXPECT_FALSE(mf3.isOpen());
154+
EXPECT_TRUE(mf4.isOpen());
102155

103156
// Flip a byte of the underlying file. The map should reflect that.
104-
FILE* f = fopen(tmp_file_path.c_str(), "r+b");
157+
FILE* f = fopen(temp->path(), "r+b");
105158
EXPECT_NE(f, nullptr);
106159
EXPECT_NE(fputc(~buf[0], f), EOF);
107160
EXPECT_EQ(fclose(f), 0);
@@ -112,19 +165,23 @@ TEST(MemoryMappedFileTest, BasicFunctionalities) {
112165

113166
mf4.close();
114167
EXPECT_TRUE(mf4.data() == nullptr);
168+
EXPECT_FALSE(mf4.isOpen());
115169

116-
std::filesystem::remove(tmp_file_path);
117170
delete[] buf;
118171

172+
std::string oldPath = temp->path();
173+
temp = nullptr;
174+
119175
// Opening a non-existence file.
120176
MemoryMappedFile mf5;
121-
open_result = mf5.open(tmp_file_path.c_str());
177+
open_result = mf5.open(oldPath.c_str());
122178
EXPECT_FALSE(open_result);
123179
EXPECT_EQ(mf5.length(), 0);
124180
EXPECT_EQ(mf5.data(), nullptr);
181+
EXPECT_FALSE(mf5.isOpen());
125182
}
126183

127-
TEST(MemoryMappedFileTest, EmptyFile) {
184+
TEST(MemoryMappedFileTest, OpenFailureOnEmptyFile) {
128185
std::filesystem::path tmp_file_path =
129186
std::filesystem::temp_directory_path() /
130187
"org.openvanilla.mcbopomofo.memorymappedfiletest-empty-file";
@@ -134,33 +191,30 @@ TEST(MemoryMappedFileTest, EmptyFile) {
134191

135192
MemoryMappedFile mf;
136193
EXPECT_FALSE(mf.open(tmp_file_path.c_str()));
194+
EXPECT_FALSE(mf.isOpen());
137195
EXPECT_EQ(mf.length(), 0);
138196
EXPECT_EQ(mf.data(), nullptr);
139197

140198
mf.close();
141199
EXPECT_EQ(mf.length(), 0);
142200
EXPECT_EQ(mf.data(), nullptr);
201+
EXPECT_FALSE(mf.isOpen());
143202

144203
std::filesystem::remove(tmp_file_path);
145204
}
146205

147-
TEST(MemoryMappedFileTest, MmapFailure) {
148-
std::filesystem::path tmp_dir_path =
149-
std::filesystem::temp_directory_path() /
150-
"org.openvanilla.mcbopomofo.memorymappedfiletest-mmap-failure";
151-
std::filesystem::remove_all(tmp_dir_path);
152-
ASSERT_TRUE(std::filesystem::create_directory(tmp_dir_path));
153-
206+
TEST(MemoryMappedFileTest, OpenFailureOnDirectory) {
207+
TempDir dir;
154208
MemoryMappedFile mf;
155-
EXPECT_FALSE(mf.open(tmp_dir_path.c_str()));
209+
EXPECT_FALSE(mf.open(dir.path()));
210+
EXPECT_FALSE(mf.isOpen());
156211
EXPECT_EQ(mf.length(), 0);
157212
EXPECT_EQ(mf.data(), nullptr);
158213

159214
mf.close();
160215
EXPECT_EQ(mf.length(), 0);
161216
EXPECT_EQ(mf.data(), nullptr);
162-
163-
std::filesystem::remove_all(tmp_dir_path);
217+
EXPECT_FALSE(mf.isOpen());
164218
}
165219

166220
} // namespace McBopomofo

0 commit comments

Comments
 (0)