From 74284d39b8b109f927848f6b6fdf934fd11a922a Mon Sep 17 00:00:00 2001 From: FeralChild64 Date: Tue, 23 Jun 2020 22:49:55 +0200 Subject: [PATCH] tested dictionary compression - not worth the effort, work stopped --- c64/,,config_custom.s | 5 ++- tools/generate_strings.cc | 71 ++++++++++++++++++++++++++++++++------- 2 files changed, 63 insertions(+), 13 deletions(-) diff --git a/c64/,,config_custom.s b/c64/,,config_custom.s index e43a3a58..f2ec3464 100644 --- a/c64/,,config_custom.s +++ b/c64/,,config_custom.s @@ -126,7 +126,6 @@ #define CONFIG_TAPE_WEDGE // #define CONFIG_TAPE_HEAD_ALIGN #define CONFIG_BCD_SAFE_INTERRUPTS -// #define CONFIG_COMPRESSION_LVL_2 // work-in-progress // Eye candy @@ -141,3 +140,7 @@ // #define CONFIG_DBG_STUBS_BRK // #define CONFIG_DBG_PRINTF + +// Other + +#define CONFIG_COMPRESSION_LVL_2 // work on this option stopped - it would not give any improvement diff --git a/tools/generate_strings.cc b/tools/generate_strings.cc index 782ab2ff..6207451a 100644 --- a/tools/generate_strings.cc +++ b/tools/generate_strings.cc @@ -579,9 +579,10 @@ void DictEncoder::extractWords(std::vector &candidateList) std::istringstream entryStream(dictionaryEntry); while (entryStream) { - // XXX probably not the best way std::string word_1; entryStream >> word_1; + + if (word_1.empty()) continue; // Create possible variants with spaces @@ -598,8 +599,8 @@ void DictEncoder::extractWords(std::vector &candidateList) if (word.size() < 1) return; if (dictionaryEntry.find(word) == std::string::npos) return; - if (std::find(candidateList.begin(), candidateList.end(), word) == candidateList.end()) return; - + if (std::find(candidateList.begin(), candidateList.end(), word) != candidateList.end()) return; + candidateList.push_back(word); }; @@ -659,7 +660,19 @@ int32_t DictEncoder::evaluateCandidate(std::string &candidate) // For the substring before the first occurence we need to create a separate entry // in the dictionary - unless it is already there, it brings some additional cost - // XXX, targetSize + std::string otherStr = std::string(dictionaryEntry.begin(), dictionaryEntry.end() + occurences[0]); + + if (std::find(dictionary.begin(), dictionary.end(), otherStr) != dictionary.end()) + { + // This extra string is already in the dictionary - that gives extra saving + score += otherStr.size() - 1; + } + else + { + // This extra string is not in the dictionary, it has to be added + score -= 2; + targetSize++; + } } for (auto iter = occurences.begin(); iter < occurences.end(); iter++) @@ -676,12 +689,33 @@ int32_t DictEncoder::evaluateCandidate(std::string &candidate) // For the substring between this occurence and the next one (or the end of the string) // we need to create a separate entry in the dictionary - unless it is already there, // it brings some additional cost - - // XXX, targetSize + + std::string otherStr; + if (iter + 1 == occurences.end()) + { + otherStr = std::string(dictionaryEntry.begin() + *iter + candidate.size(), + dictionaryEntry.end()); + } + else + { + otherStr = std::string(dictionaryEntry.begin() + *iter + candidate.size(), + dictionaryEntry.begin() + *(iter + 1)); + } + + if (std::find(dictionary.begin(), dictionary.end(), otherStr) != dictionary.end()) + { + // This extra string is already in the dictionary - that gives extra saving + score += otherStr.size() - 1; + } + else + { + // This extra string is not in the dictionary, it has to be added + score -= 2; + targetSize++; + } } // We crossed the maximum number of strings, do not consider this candidate - // XXX improve the behaviour, maybe only some substrings can be replaced if (targetSize > 255) return -1; } @@ -697,8 +731,8 @@ bool DictEncoder::optimizeSplit() std::vector candidateList; extractWords(candidateList); - - // XXX find a couple of largest common substrings and add them as candidates too + + // XXX possible future improvement: find a couple of largest common substrings and add them as candidates too // Now find the best word to be extracted @@ -708,6 +742,9 @@ bool DictEncoder::optimizeSplit() for (auto iter = candidateList.begin(); iter < candidateList.end(); iter++) { int32_t candidateScore = evaluateCandidate(*iter); + + std::cout << "XXX candidate '" << *iter << "' score: " << candidateScore << std::endl; + if (candidateScore > bestCandidateScore) { bestCandidate = iter; @@ -717,10 +754,19 @@ bool DictEncoder::optimizeSplit() if (bestCandidateScore < 1) return false; - // Perform the extraction + // Extract the best candidate to a separate string + std::cout << "XXX best is '" << *bestCandidate << "' score: " << bestCandidateScore << std::endl; + // XXX + std::cout << std::endl << "NOTE: 'CONFIG_COMPRESSION_LVL_2' support is not finished - as of writing" << + std::endl << " this text, it would not give any space improvement, so I've decided" << + std::endl << " to stop working on it for now." << std::endl << + std::endl << " Please disable this option in your configuration file." << std::endl << std::endl; + + exit(1); + return false; } @@ -728,7 +774,7 @@ bool DictEncoder::optimizeJoin() { // Try to optimize by joining two substrings - // XXX + // XXX - possible future improvement return false; } @@ -1434,7 +1480,8 @@ void parseConfigFile() // Read a single line, remove leading spaces and tabs - std::getline(cnfFile, workStr); // XXX check for errors here + std::getline(cnfFile, workStr); + if (cnfFile.bad()) ERROR("error reading configuration file"); workStr = std::regex_replace(workStr, std::regex("^[ \t]+"), ""); // Skip lines which are not preprocessor definitions