Skip to content

Commit

Permalink
feat,refactor(dict): tweak to reduce peak memory usage
Browse files Browse the repository at this point in the history
- Use smart pointer
- Release memory in time

This may save about 12% memory usage when compiling dict.
  • Loading branch information
WhiredPlanck committed Jun 10, 2023
1 parent 8bf2cf7 commit b05f11b
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 10 deletions.
10 changes: 6 additions & 4 deletions src/rime/dict/dict_compiler.cc
Original file line number Diff line number Diff line change
Expand Up @@ -236,9 +236,9 @@ bool DictCompiler::BuildTable(int table_index,
for (const auto& s : collector.syllabary) {
syllable_to_id[s] = syllable_id++;
}
for (RawDictEntry& r : collector.entries) {
for (auto r : collector.entries) {
Code code;
for (const auto& s : r.raw_code) {
for (const auto& s : r->raw_code) {
code.push_back(syllable_to_id[s]);
}
DictEntryList* ls = vocabulary.LocateEntries(code);
Expand All @@ -248,10 +248,12 @@ bool DictCompiler::BuildTable(int table_index,
}
auto e = New<DictEntry>();
e->code.swap(code);
e->text.swap(r.text);
e->weight = log(r.weight > 0 ? r.weight : DBL_EPSILON);
e->text.swap(r->text);
e->weight = log(r->weight > 0 ? r->weight : DBL_EPSILON);
ls->push_back(e);
}
// release memory in time to reduce peak memory usage
vector<of<RawDictEntry>>().swap(collector.entries);
if (settings->sort_order() != "original") {
vocabulary.SortHomophones();
}
Expand Down
10 changes: 5 additions & 5 deletions src/rime/dict/entry_collector.cc
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ void EntryCollector::CreateEntry(const string &word,
words[e.text][code_str] += e.weight;
total_weight[e.text] += e.weight;
}
entries.push_back(e);
entries.emplace_back(New<RawDictEntry>(e));
++num_entries;
}

Expand Down Expand Up @@ -240,10 +240,10 @@ void EntryCollector::Dump(const string& file_name) const {
out << "# - " << syllable << std::endl;
}
out << std::endl;
for (const RawDictEntry& e : entries) {
out << e.text << '\t'
<< e.raw_code.ToString() << '\t'
<< e.weight << std::endl;
for (const auto &e : entries) {
out << e->text << '\t'
<< e->raw_code.ToString() << '\t'
<< e->weight << std::endl;
}
out.close();
}
Expand Down
2 changes: 1 addition & 1 deletion src/rime/dict/entry_collector.h
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class EntryCollector : public PhraseCollector {
public:
Syllabary syllabary;
bool build_syllabary = true;
vector<RawDictEntry> entries;
vector<of<RawDictEntry>> entries;
size_t num_entries = 0;
ReverseLookupTable stems;

Expand Down

0 comments on commit b05f11b

Please # to comment.