From d3fdc6cc59773ab5aef38e1a361b65d6a6205aa7 Mon Sep 17 00:00:00 2001 From: FeralChild64 Date: Tue, 23 Jun 2020 17:28:46 +0200 Subject: [PATCH] more work towards dictionary encoding --- tools/generate_strings.cc | 317 +++++++++++++++++++++++++++++++++----- 1 file changed, 278 insertions(+), 39 deletions(-) diff --git a/tools/generate_strings.cc b/tools/generate_strings.cc index 24316cc1..9a97792f 100644 --- a/tools/generate_strings.cc +++ b/tools/generate_strings.cc @@ -2,9 +2,6 @@ // Utility to generate compressed messages and BASIC tokens // -// XXX print somewhere configuration file used to generate strings -// XXX use DualStream class for logging - #include "common.h" #include @@ -52,14 +49,8 @@ typedef struct StringEntryList std::vector list; } StringEntryList; -typedef std::vector StringFreqEncoded; -typedef std::vector StringDictEncoded; - -typedef struct StringEncodedList -{ - std::vector byFreq; - std::vector byDict; -} StringEncodedList; +typedef std::vector StringEncoded; +typedef std::vector StringEncodedList; // http://www.classic-games.com/commodore64/cbmtoken.html // https://www.c64-wiki.com/wiki/BASIC_token @@ -462,16 +453,21 @@ class DictEncoder { public: - void addString(const std::string &inString, StringDictEncoded *outPtr); + void addString(const std::string &inString, StringEncoded *outPtr); void process(StringEntryList &outDictionary); private: bool optimizeSplit(); + bool optimizeJoin(); + void optimizeOrder(); - std::vector encodings; - std::vector dictionary; + void extractWords(std::vector &candidateList); + int32_t evaluateCandidate(std::string &candidate); + + std::vector encodings; + std::vector dictionary; }; class DataSet @@ -492,7 +488,7 @@ private: void encodeStringsDict(); void encodeStringsFreq(); - void encodeByFreq(const std::string &plain, StringFreqEncoded &encoded) const; + void encodeByFreq(const std::string &plain, StringEncoded &encoded) const; void prepareOutput(); @@ -538,7 +534,7 @@ class DataSetX16 : public DataSet // Work class implementation // -void DictEncoder::addString(const std::string &inString, StringDictEncoded *outPtr) +void DictEncoder::addString(const std::string &inString, StringEncoded *outPtr) { // Store the pointer to encoding @@ -566,18 +562,244 @@ void DictEncoder::addString(const std::string &inString, StringDictEncoded *outP } } + +void DictEncoder::extractWords(std::vector &candidateList) +{ + for (const auto &dictionaryEntry : dictionary) + { + // Split the dictionary entry into words + + std::istringstream entryStream(dictionaryEntry); + while (entryStream) + { + // XXX probably not the best way + std::string word_1; + entryStream >> word_1; + + // Create possible variants with spaces + + std::string word_2 = " " + word_1; + std::string word_3 = word_1 + " "; + std::string word_4 = word_2 + " "; + + // If necessary, add the word to candidate list + + auto addToList = [&dictionaryEntry, &candidateList](std::string &word) + { + // Before dding, make sure the word variant exists in the string, + // is not already on the list, and is long enough it makes sense to try + + if (word.size() < 1) return; + if (dictionaryEntry.find(word) == std::string::npos) return; + if (std::find(candidateList.begin(), candidateList.end(), word) == candidateList.end()) return; + + candidateList.push_back(word); + }; + + addToList(word_1); + addToList(word_2); + addToList(word_3); + addToList(word_4); + } + } +} + +int32_t DictEncoder::evaluateCandidate(std::string &candidate) +{ + // Build the score - find how much bytes can be spared by extracting this particular candidate + // Limited size of the dictionary is taken into consideration + + int32_t score = -(candidate.size() + 1); + uint32_t targetSize = dictionary.size() + 1; + + // First check for situation when the candidate equals the currently existing dictionary entry + + for (const auto &dictionaryEntry : dictionary) + { + if (dictionaryEntry == candidate) + { + // Candidate equals the string + + score += candidate.size() + 1; + targetSize--; + continue; + } + } + + // Now check the remaining cases + + for (const auto &dictionaryEntry : dictionary) + { + if (dictionaryEntry == candidate) continue; // already handled + + // Find all the occurences of the candidate in the current string + + std::vector occurences; + for (auto iter = dictionaryEntry.begin(); iter < (dictionaryEntry.end() - candidate.size() + 1); iter++) + { + if (candidate != std::string(iter, iter + candidate.size())) continue; + + occurences.push_back(iter - dictionaryEntry.begin()); + iter += candidate.size() - 1; // to make sure we have no occurence overlapping + } + + if (occurences.empty()) continue; + + // Now summarize what we are going to gain when doing replacement + + if (occurences[0] != 0) + { + // For the substring before the first occurence we need to create a separate entry + // in the dictionary - unless it is already there, it brings some additional cost + + // XXX, targetSize + } + + for (auto iter = occurences.begin(); iter < occurences.end(); iter++) + { + // With a replacement, we should gain some bytes + score += candidate.size() - 1; + + // If the occurence ends with the last byte of dictionary string - nothing more to do + if (*iter + candidate.size() == dictionaryEntry.size()) break; + + // If next occurence starts immediately after this one - next iteration + if (iter + 1 != occurences.end() && (*(iter + 1) - *iter) == (uint8_t) candidate.size()) continue; + + // For the substring between this occurence and the next one (or the end of the string) + // we need to create a separate entry in the dictionary - unless it is already there, + // it brings some additional cost + + // XXX, targetSize + } + + // We crossed the maximum number of strings, do not consider this candidate + // XXX improve the behaviour, maybe only some substrings can be replaced + + if (targetSize > 255) return -1; + } + + return score; +} + bool DictEncoder::optimizeSplit() { + if (dictionary.size() >= 255) return false; // up to 255 entries in the dictionary are possible + + // First select words, which can potentially be extracted to new substrings + + std::vector candidateList; + extractWords(candidateList); + + // XXX find a couple of largest common substrings and add them as candidates too + + // Now find the best word to be extracted + + auto bestCandidate = candidateList.end(); + int32_t bestCandidateScore = 0; + + for (auto iter = candidateList.begin(); iter < candidateList.end(); iter++) + { + int32_t candidateScore = evaluateCandidate(*iter); + if (candidateScore > bestCandidateScore) + { + bestCandidate = iter; + bestCandidateScore = bestCandidateScore; + } + } + + if (bestCandidateScore < 1) return false; + + // Perform the extraction + // XXX return false; } +bool DictEncoder::optimizeJoin() +{ + // Try to optimize by joining two substrings + + // XXX + + return false; +} + +void DictEncoder::optimizeOrder() +{ + // Try to optimize order of strings in the dictionary for fastest display + // - put the shorter substrings first + // - for strings with roughly the same length put more frequently used first + + typedef struct Entry + { + uint32_t penalty; + std::string word; + } Entry; + + // Create copy of the dictionary with optimization info + + std::vector optimizedOrder; + for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++) + { + const auto &dictionaryStr = *iter; + + uint8_t sizePenalty = dictionaryStr.size() / 3; + uint16_t freqPenalty = 65535; + + // Calculate penalty for low occurence frequency + + for (auto &encoding : encodings) + { + for (auto &byte : *encoding) + { + if (byte == (iter - dictionary.begin())) freqPenalty--; + } + } + + // Create new entry + + Entry newEntry; + newEntry.word = dictionaryStr; + newEntry.penalty = sizePenalty * 65536 + freqPenalty; + + optimizedOrder.push_back(newEntry); + } + + // Determine ordering - sort from smallest penalty to largest + + std::sort(optimizedOrder.begin(), optimizedOrder.end(), + [](const Entry &e1, const Entry &e2) { return e1.penalty < e2.penalty; }); + + // Create helper table for reordering + + std::vector reorderTable; + + for (uint8_t idx1 = 0; idx1 < dictionary.size(); idx1++) + { + uint8_t idx2 = 0; + while (dictionary[idx1] != optimizedOrder[idx2].word) idx2++; + + reorderTable.push_back(idx2); + } + + // Perform the reordering + + for (auto &encoding : encodings) + { + for (auto &byte : *encoding) byte = reorderTable[byte]; + } + + for (uint8_t idx = 0; idx < dictionary.size(); idx++) dictionary[idx] = optimizedOrder[idx].word; +} + void DictEncoder::process(StringEntryList &outDictionary) { // Optimize as long as it brings any improvement - while (optimizeSplit()) ; + while (optimizeSplit() || optimizeJoin()) ; + optimizeOrder(); // Export the dictionary to external format @@ -589,6 +811,14 @@ void DictEncoder::process(StringEntryList &outDictionary) StringEntry newEntry = { true, true, true, "", dictionaryStr }; outDictionary.list.push_back(newEntry); } + + // Adapt the encoding to external format (0 = end of string) + + for (auto &encoding : encodings) + { + for (auto &byte : *encoding) byte++; + encoding->push_back(0); + } } bool DataSet::isCompressionLvl2(const StringEntryList &list) const @@ -623,7 +853,7 @@ void DataSet::addStrings(const StringEntryList &stringList) void DataSet::process() { - std::cout << "Processing layout '" << layoutName() << "'" << std::endl; + std::cout << "Processing file '" << CMD_cnfFile << "', layout '" << layoutName() << "'" << std::endl; generateConfigDepStrings(); validateLists(); @@ -782,11 +1012,11 @@ void DataSet::encodeStringsDict() // Skip lists not to be encoded using the dictionary if (!isCompressionLvl2(stringEntryList)) continue; - stringEncodedList.byDict.resize(stringEntryList.list.size()); + stringEncodedList.resize(stringEntryList.list.size()); for (uint8_t idxEntry = 0; idxEntry < stringEntryList.list.size(); idxEntry++) { dictEncoder.addString(stringEntryList.list[idxEntry].string, - &stringEncodedList.byDict[idxEntry]); + &stringEncodedList[idxEntry]); } } @@ -885,7 +1115,7 @@ void DataSet::calculateFrequencies() } } -void DataSet::encodeByFreq(const std::string &plain, StringFreqEncoded &encoded) const +void DataSet::encodeByFreq(const std::string &plain, StringEncoded &encoded) const { bool fullByte = true; @@ -958,12 +1188,12 @@ void DataSet::encodeStringsFreq() // Skip lists encoded by the dictionary if (isCompressionLvl2(stringEntryList)) continue; - // Perform encoding of the list + // Perform frequency encoding of the list for (const auto &stringEntry : stringEntryList.list) { - stringEncodedList.byFreq.emplace_back(); - auto &stringEncoded = stringEncodedList.byFreq.back(); + stringEncodedList.emplace_back(); + auto &stringEncoded = stringEncodedList.back(); if (isRelevant(stringEntry)) { @@ -1042,20 +1272,20 @@ void DataSet::prepareOutput() stream << std::endl << ".label TK__PACKED_AS_3N = $" << std::hex << +tk__packed_as_3n << std::endl << ".label TK__MAX_KEYWORD_LEN = " << std::dec << +tk__max_keyword_len << std::endl; - // Export frequency-encoded strings + // Export encoded strings for (uint8_t idxList = 0; idxList < stringEntryLists.size(); idxList++) { - const auto &stringEntryList = stringEntryLists[idxList]; - const auto &stringFreqEncodedList = stringEncodedLists[idxList].byFreq; + const auto &stringEntryList = stringEntryLists[idxList]; + const auto &stringEncodedList = stringEncodedLists[idxList]; stream << std::endl; - for (uint8_t idxString = 0; idxString < stringFreqEncodedList.size(); idxString++) + for (uint8_t idxString = 0; idxString < stringEncodedList.size(); idxString++) { - const auto &stringEntry = stringEntryList.list[idxString]; - const auto &stringFreqEncoded = stringFreqEncodedList[idxString]; + const auto &stringEntry = stringEntryList.list[idxString]; + const auto &stringEncoded = stringEncodedList[idxString]; - if (!stringFreqEncoded.empty()) + if (!stringEncoded.empty()) { stream << ".label IDX__" << stringEntry.alias << std::string(maxAliasLen - stringEntry.alias.length(), ' ') << " = $" << std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +idxString << std::endl; @@ -1067,18 +1297,27 @@ void DataSet::prepareOutput() if (stringEntryList.type == ListType::KEYWORDS) { stream << std::endl << ".label TK__MAXTOKEN_" << stringEntryList.name << " = " << - std::dec << stringFreqEncodedList.size() << std::endl; + std::dec << stringEncodedList.size() << std::endl; } - stream << std::endl << ".macro put_freq_packed_" << stringEntryList.name << "()" << std::endl << "{" << std::endl; + if (isCompressionLvl2(stringEntryList)) + { + stream << std::endl << ".macro put_dict_packed_"; + } + else + { + stream << std::endl << ".macro put_freq_packed_"; + } + + stream << stringEntryList.name << "()" << std::endl << "{" << std::endl; enum LastStr { NONE, SKIPPED, WRITTEN } lastStr = LastStr::NONE; - for (uint8_t idxString = 0; idxString < stringFreqEncodedList.size(); idxString++) + for (uint8_t idxString = 0; idxString < stringEncodedList.size(); idxString++) { - const auto &stringEntry = stringEntryList.list[idxString]; - const auto &stringFreqEncoded = stringFreqEncodedList[idxString]; + const auto &stringEntry = stringEntryList.list[idxString]; + const auto &stringEncoded = stringEncodedList[idxString]; - if (stringFreqEncoded.empty()) + if (stringEncoded.empty()) { if (lastStr == LastStr::WRITTEN) stream << std::endl; stream << "\t.byte $00 // skipped " << stringEntry.alias << std::endl; @@ -1092,7 +1331,7 @@ void DataSet::prepareOutput() stream << "\t.byte "; bool first = true; - for (const auto &charEncoded : stringFreqEncoded) + for (const auto &charEncoded : stringEncoded) { if (first) { @@ -1191,7 +1430,7 @@ void printUsage() void printBanner() { printBannerLineTop(); - std::cout << "// Generating compressed messages and BASIC tokens" << "\n"; + std::cout << "// Generating compressed messages and BASIC tokens\n"; printBannerLineBottom(); }