made dictionary compression working - it stays optional, currently even worsens the situation

This commit is contained in:
FeralChild64 committed 2020-06-25 17:32:42 +02:00
1 parent 07c36cdd4b
commit 473660be80
1 file changed
+189 -15
+189 -15
View File
@@ -463,6 +463,8 @@ private:
bool optimizeJoin(); bool optimizeJoin();
void optimizeOrder(); void optimizeOrder();
void cleanupDictionary();
void extractWords(std::vector<std::string> &candidateList); void extractWords(std::vector<std::string> &candidateList);
int32_t evaluateCandidate(std::string &candidate); int32_t evaluateCandidate(std::string &candidate);
@@ -561,7 +563,7 @@ void DictEncoder::addString(const std::string &inString, StringEncoded *outPtr)
{ {
if (dictionary.size() > 255) if (dictionary.size() > 255)
{ {
ERROR("max 255 strings for dictionary compression"); ERROR("max 255 strings allowed for dictionary compression");
} }
dictionary.push_back(inString); dictionary.push_back(inString);
@@ -715,16 +717,54 @@ int32_t DictEncoder::evaluateCandidate(std::string &candidate)
} }
} }
// We crossed the maximum number of strings, do not consider this candidate // We crossed the maximum number of strings (leave one free for the work buffer),
// do not consider this candidate
if (targetSize > 255) return -1; if (targetSize > 254) return -1;
} }
return score; return score;
} }
void DictEncoder::cleanupDictionary()
{
// Get rid of dictionary entries which are not needed anymore
bool isClean = false;
while (!isClean)
{
isClean = true;
for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++)
{
if (!iter->empty()) continue;
// Found an empty string in the dictionary - remove it and adapt encoding
isClean = false;
uint8_t obsoleteStrIdx = iter - dictionary.begin();
dictionary.erase(iter);
for (auto &encoding : encodings) for (auto &byte : *encoding)
{
if (byte >= obsoleteStrIdx) byte--;
}
break;
}
}
}
bool DictEncoder::optimizeSplit() bool DictEncoder::optimizeSplit()
{ {
// XXX - The code here is not the best one - redesign and algorithm improvements
// (like - consider optimizing as a game and apply alpha-beta scheme to select the best
// candidate, incorporate frequency encoding into candidate evaluation) could result in
// a slightly better compression. But currently we have very few strings, and the dictionary
// compression is disabled by default - so right now this is not worth the effort.
if (dictionary.size() >= 255) return false; // up to 255 entries in the dictionary are possible if (dictionary.size() >= 255) return false; // up to 255 entries in the dictionary are possible
// First select words, which can potentially be extracted to new substrings // First select words, which can potentially be extracted to new substrings
@@ -743,8 +783,6 @@ bool DictEncoder::optimizeSplit()
{ {
int32_t candidateScore = evaluateCandidate(*iter); int32_t candidateScore = evaluateCandidate(*iter);
std::cout << "XXX candidate '" << *iter << "' score: " << candidateScore << std::endl;
if (candidateScore > bestCandidateScore) if (candidateScore > bestCandidateScore)
{ {
bestCandidate = iter; bestCandidate = iter;
@@ -752,22 +790,157 @@ bool DictEncoder::optimizeSplit()
} }
} }
// Only allow for replacements that brings some size benefit
if (bestCandidateScore < 1) return false; if (bestCandidateScore < 1) return false;
// Extract the best candidate to a separate string // Extract the best candidate to a separate string
std::cout << "XXX best is '" << *bestCandidate << "' score: " << bestCandidateScore << std::endl; // First add our selected candidate to the dictionary and replace all strings which are equal to it
// XXX auto &selectedStr = *bestCandidate;
uint8_t selectedStrIdx = dictionary.size();
std::cout << std::endl << "NOTE: 'CONFIG_COMPRESSION_LVL_2' support is not finished - as of writing" << dictionary.push_back(selectedStr);
std::endl << " this text, it would not give any space improvement, so I've decided" <<
std::endl << " to stop working on it for now." << std::endl <<
std::endl << " Please disable this option in your configuration file." << std::endl << std::endl;
exit(1); for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++)
{
uint8_t currentStrIdx = iter - dictionary.begin();
return false; if (*iter != selectedStr || selectedStrIdx == currentStrIdx) continue;
// Replace the current string with the new one
for (auto &encoding : encodings) for (auto &byte : *encoding)
{
if (byte == currentStrIdx) byte = selectedStrIdx;
}
// Mark obsolete dictionary entry as free for removal
iter->clear();
}
cleanupDictionary();
// Now handle normal cases, when the string needs to be split
// XXX - this is not time-effective, optimize this in the future
bool optimizeAgain = true;
bool nexIteration = true;
while (nexIteration)
{
nexIteration = false;
for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++)
{
auto &currentStr = *iter;
uint8_t currentStrIdx = iter - dictionary.begin();
auto selectedStrIter = std::find(dictionary.begin(), dictionary.end(), selectedStr);
selectedStrIdx = selectedStrIter - dictionary.begin();
if (currentStrIdx == selectedStrIdx) continue;
// Check if the current string contains the selected string
auto pos = currentStr.find(selectedStr);
if (pos == std::string::npos) continue;
nexIteration = true;
optimizeAgain = true;
// Prepare replacement sequence for 'selectedStrIdx'
std::vector<uint8_t> replacement;
if (pos == 0)
{
// The current dictionary string starts with our selected substring
replacement.push_back(selectedStrIdx);
currentStr = currentStr.substr(selectedStr.size(), currentStr.size() - selectedStr.size());
if (!currentStr.empty())
{
// Check if the remaining string is present somewhere else in the dictionary; if so, reuse it
uint8_t pos2 = currentStrIdx;
for (auto iter2 = dictionary.begin(); iter2 < dictionary.end(); iter2++)
{
if (iter2 == iter) continue;
if (currentStr == *iter2)
{
dictionary[pos2].clear();
pos2 = iter2 - dictionary.begin();
break;
}
}
replacement.push_back(pos2);
}
}
else
{
// The current dictionary string does not start with our selected substring
std::string newStr = currentStr.substr(0, pos);
currentStr = currentStr.substr(pos, currentStr.size() - pos);
// Check if the newStr is present somewhere else in the dictionary; if so, reuse it
bool found = false;
uint8_t pos2 = 0;
for (auto iter2 = dictionary.begin(); iter2 < dictionary.end(); iter2++)
{
if (*iter2 == newStr)
{
found = true;
pos2 = iter2 - dictionary.begin();
break;
}
}
if (found)
{
replacement.push_back(pos2);
}
else
{
dictionary.push_back(newStr);
replacement.push_back(dictionary.size() - 1);
}
replacement.push_back(currentStrIdx);
}
// Now replace all the occurences of 'selectedStrIdx' with a 'replacement' vector
for (auto &encoding : encodings)
{
for (uint8_t idx = 0; idx < encoding->size(); idx++)
{
if ((*encoding)[idx] != currentStrIdx) continue;
// Perform replacement
encoding->erase(encoding->begin() + idx);
for (const auto replacementIdx : replacement)
{
encoding->insert(encoding->begin() + idx, replacementIdx);
idx++;
}
idx--;
}
}
cleanupDictionary();
}
}
return optimizeAgain;
} }
bool DictEncoder::optimizeJoin() bool DictEncoder::optimizeJoin()
@@ -820,6 +993,7 @@ void DictEncoder::optimizeOrder()
optimizedOrder.push_back(newEntry); optimizedOrder.push_back(newEntry);
} }
// Determine ordering - sort from smallest penalty to largest // Determine ordering - sort from smallest penalty to largest
std::sort(optimizedOrder.begin(), optimizedOrder.end(), std::sort(optimizedOrder.begin(), optimizedOrder.end(),
@@ -839,9 +1013,9 @@ void DictEncoder::optimizeOrder()
// Perform the reordering // Perform the reordering
for (auto &encoding : encodings) for (auto &encoding : encodings) for (auto &byte : *encoding)
{ {
for (auto &byte : *encoding) byte = reorderTable[byte]; byte = reorderTable[byte];
} }
for (uint8_t idx = 0; idx < dictionary.size(); idx++) dictionary[idx] = optimizedOrder[idx].word; for (uint8_t idx = 0; idx < dictionary.size(); idx++) dictionary[idx] = optimizedOrder[idx].word;