diff --git a/Makefile b/Makefile index e4928f32..596794e9 100644 --- a/Makefile +++ b/Makefile @@ -99,6 +99,14 @@ SRC_ACME = $(DIR_ACME)/acme.c $(DIR_ACME)/platform.c $(DIR_ACME)/alu.c $(D $(DIR_ACME)/dynabuf.c $(DIR_ACME)/mnemo.c $(DIR_ACME)/input.c $(DIR_ACME)/section.c \ $(DIR_ACME)/symbol.c $(DIR_ACME)/tree.c +HDR_COMMON = tools/common.h +SRC_COMMON = tools/common.cc + +DIR_GENSTR = tools/generate_strings +HDR_GENSTR = $(HDR_COMMON) $(DIR_GENSTR)/dataset.h $(DIR_GENSTR)/dictencoder.h $(DIR_GENSTR)/global.h +SRC_GENSTR = $(SRC_COMMON) $(DIR_GENSTR)/dataset.cc $(DIR_GENSTR)/dictencoder.cc $(DIR_GENSTR)/global.cc \ + $(DIR_GENSTR)/main.cc + CRT_BIN_LIST = assets/cartridge/header-cart.bin \ assets/cartridge/header-seg0.bin \ assets/cartridge/header-seg1.bin \ @@ -316,6 +324,12 @@ $(TOOL_ASSEMBLER): $(DIR_ACME) $(SRC_ACME) $(HDR_ACME) @mkdir -p build/tools @$(CC) -o $(TOOL_ASSEMBLER) $(SRC_ACME) -lm -w -I./3rdparty/acme/src +$(TOOL_GENERATE_STRINGS): $(SRC_GENSTR) $(HDR_GENSTR) + @echo + @echo Compiling tool $@ ... + @mkdir -p build/tools + @$(CXX) -O2 -Wall -o $(TOOL_GENERATE_STRINGS) $(SRC_GENSTR) -lm -I./tools + $(TOOL_PNGPREPARE): tools/pngprepare.c @echo @echo Compiling tool $@ ... @@ -328,11 +342,11 @@ build/tools/%: tools/%.c @mkdir -p build/tools @$(CC) -O2 -Wall -o $@ $< -build/tools/%: tools/%.cc tools/common.h +build/tools/%: tools/%.cc $(SRC_COMMON) $(HDR_COMMON) @echo @echo Compiling tool $@ ... @mkdir -p build/tools - @$(CXX) -O2 -Wall -o $@ $< + @$(CXX) -O2 -Wall -o $@ $< $(SRC_COMMON) # Rules - CHARGEN diff --git a/tools/common.cc b/tools/common.cc new file mode 100644 index 00000000..8800504a --- /dev/null +++ b/tools/common.cc @@ -0,0 +1,38 @@ +// +// Common parts of utilities written in C++ +// for providing uniform user experience +// + +#include "common.h" + +#include + + +const std::string BANNER_LINE = "//-------------------------------------------------------------------------------------------"; + + +void ERROR() +{ + exit(-1); +} + +void ERROR(const std::string &message) +{ + std::cout << "\n" << "ERROR: " << message << "\n\n"; + exit(-1); +} + +void printBannerLineTop() +{ + std::cout << "\n\n\n" << BANNER_LINE << "\n"; +} + +void printBannerLineBottom() +{ + std::cout << BANNER_LINE << "\n\n"; +} + + +DualStream::DualStream(std::ostream& str1, std::ostream& str2) : str1(str1), str2(str2) +{ +} diff --git a/tools/common.h b/tools/common.h index d8f2ee6d..fcbf8b6b 100644 --- a/tools/common.h +++ b/tools/common.h @@ -3,6 +3,10 @@ // for providing uniform user experience // + +#ifndef COMMON_H +#define COMMON_H + #include #include #include @@ -15,35 +19,17 @@ #endif -const std::string BANNER_LINE = "//-------------------------------------------------------------------------------------------"; +void ERROR(); +void ERROR(const std::string &message); - -void ERROR() -{ - exit(-1); -} - -void ERROR(const std::string &message) -{ - std::cout << "\n" << "ERROR: " << message << "\n\n"; - exit(-1); -} - -void printBannerLineTop() -{ - std::cout << "\n\n\n" << BANNER_LINE << "\n"; -} - -void printBannerLineBottom() -{ - std::cout << BANNER_LINE << "\n\n"; -} +void printBannerLineTop(); +void printBannerLineBottom(); class DualStream { public: - DualStream(std::ostream& str1, std::ostream& str2) : str1(str1), str2(str2) {} + DualStream(std::ostream& str1, std::ostream& str2); template DualStream &operator<<(const T& x) { @@ -57,3 +43,6 @@ private: std::ostream& str1; std::ostream& str2; }; + + +#endif // COMMON_H diff --git a/tools/generate_strings.cc b/tools/generate_strings.cc deleted file mode 100644 index 7e4b76b2..00000000 --- a/tools/generate_strings.cc +++ /dev/null @@ -1,1956 +0,0 @@ -// -// Utility to generate compressed messages and BASIC keywords -// - -#include "common.h" - -#include - -#include -#include -#include -#include -#include -#include -#include - -// -// Command line settings -// - -std::string CMD_outFile = "out.s"; -std::string CMD_cnfFile = ""; - -// -// Type definition for strings/keywords to generate -// - -typedef struct StringEntry -{ - bool enabledSTD; // whether enabled for standard build - bool enabledCRT; // whether enabled for standard build with extra ROM cartridge - bool enabledM65; // whether enabled for Mega 65 build - bool enabledU64; // whether enabled for Ultimate 64 build - bool enabledX16; // whether enabled for Commander X16 build - std::string alias; // alias, for the assembler - std::string string; // string/token - uint8_t abbrevLen = 0; // length of token abbreviation -} StringEntry; - -enum class ListType -{ - KEYWORDS, - STRINGS_BASIC, - DICTIONARY // for internal use only -}; - -typedef struct StringEntryList -{ - ListType type; - std::string name; - std::vector list; -} StringEntryList; - -typedef std::vector StringEncoded; -typedef std::vector StringEncodedList; - -// http://www.classic-games.com/commodore64/cbmtoken.html -// https://www.c64-wiki.com/wiki/BASIC_token - -// BASIC keywords - V2 dialect - -/* - The URLs here provide examples of other BASIC dialects that include the specified commands - or keywords. - All except VERIFY are known with certainty to be part of other BASIC dialects, and thus - cannot be subject to copyright infringement when implemented on a C64. - - The VERIFY Command however appears in other programming languages, and as a single word, - is the obvious complement to LOAD and SAVE when considering the following storage functions: - - 1. Load a file from storage into memory. - 2. Save a file from memory to storage. - 3. Verify that a file has been correctly written from memory to storage. - - Note that verify is the key verb in the behaviour of this function. Therefore it is the - obvious choice for the name for such a function, taking into account the convention of - LOAD and SAVE. Indeed, that this is so is highlighted by the fact that it was used in the - C64 ROM. Therefore there is no exercise of creativity here, and a reasonable person would - likely select this name independently. Also, single words cannot be copyright. - - Further, the list is a list of facts, and thus not copyrightable. - - We sourced the list from a public source, https://www.c64-wiki.com/wiki/BASIC, which has - been public on the internet for a long time, as have many other similar sources. - That website offers the page under the GFDL license, and we can therefore take the list - under those license terms if we wish, should the need arise. - - Note that in providing these justifications above, we do not in any way acknowledge that - without these conditions that the BASIC keyword list could be copyright, but provide this - information as a further layer of defence against any claims that it could somehow - infringe on the copyrights of the C64/C65/C128 etc ROMs. -*/ - -const StringEntryList GLOBAL_Keywords_V2 = { ListType::KEYWORDS, "keywords_V2", -{ - // STD CRT M65 U64 X16 - { true, true, true, true, true, "KV2_80", "END", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_81", "FOR", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_82", "NEXT", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_83", "DATA", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_84", "INPUT#", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php - { true, true, true, true, true, "KV2_85", "INPUT", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_86", "DIM", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_87", "READ", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_88", "LET", 2 }, // https://en.wikipedia.org/wiki/Atari_BASIC - { true, true, true, true, true, "KV2_89", "GOTO", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_8A", "RUN", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_8B", "IF", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_8C", "RESTORE", 3 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_8D", "GOSUB", 3 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_8E", "RETURN", 3 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_8F", "REM", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_90", "STOP", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_91", "ON", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_92", "WAIT", 2 }, // http://www.picaxe.com/BASIC-Commands/Time-Delays/wait/ - { true, true, true, true, true, "KV2_93", "LOAD", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_94", "SAVE", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_95", "VERIFY", 2 }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - { true, true, true, true, true, "KV2_96", "DEF", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_97", "POKE", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_98", "PRINT#", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php - { true, true, true, true, true, "KV2_99", "PRINT", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_9A", "CONT", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_9B", "LIST", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_9C", "CLR", 2 }, // Apple I Replica Creation: Back to the Garage, p125 - { true, true, true, true, true, "KV2_9D", "CMD", 2 }, // https://en.wikipedia.org/wiki/List_of_DOS_commands - { true, true, true, true, true, "KV2_9E", "SYS", 2 }, // https://www.lifewire.com/dos-commands-4070427 - { true, true, true, true, true, "KV2_9F", "OPEN", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php - // STD CRT M65 U64 X16 - { true, true, true, true, true, "KV2_A0", "CLOSE", 3 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php - { true, true, true, true, true, "KV2_A1", "GET", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_A2", "NEW", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_A3", "TAB(", 2 }, // http://www.antonis.de/qbebooks/gwbasman/tab.html - { true, true, true, true, true, "KV2_A4", "TO", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_A5", "FN", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_A6", "SPC(", 2 }, // http://www.antonis.de/qbebooks/gwbasman/spc.html - { true, true, true, true, true, "KV2_A7", "THEN", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_A8", "NOT", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_A9", "STEP", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_AA", "+", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_AB", "-", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_AC", "*", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_AD", "/", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_AE", "^", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_AF", "AND", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_B0", "OR", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_B1", ">", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_B2", "=", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_B3", "<", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_B4", "SGN", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_B5", "INT", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_B6", "ABS", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_B7", "USR", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_B8", "FRE", 2 }, // http://www.antonis.de/qbebooks/gwbasman/fre.html - { true, true, true, true, true, "KV2_B9", "POS", 0 }, // http://www.antonis.de/qbebooks/gwbasman/pos.html - { true, true, true, true, true, "KV2_BA", "SQR", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_BB", "RND", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_BC", "LOG", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_BD", "EXP", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_BE", "COS", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_BF", "SIN", 2 }, // https://www.landsnail.com/a2ref.htm - // STD CRT M65 U64 X16 - { true, true, true, true, true, "KV2_C0", "TAN", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_C1", "ATN", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_C2", "PEEK", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_C3", "LEN", 0 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_C4", "STR$", 3 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_C5", "VAL", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_C6", "ASC", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_C7", "CHR$", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_C8", "LEFT$", 3 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_C9", "RIGHT$", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_CA", "MID$", 2 }, // https://www.landsnail.com/a2ref.htm - { true, true, true, true, true, "KV2_CB", "GO", 0 }, // https://en.wikipedia.org/wiki/Goto -} }; - -// extended BASIC keywords - list reserved for small BASIC commands, suitable for inclusion in all machines with extended ROM - -const StringEntryList GLOBAL_Keywords_01 = { ListType::KEYWORDS, "keywords_01", -{ - // STD CRT M65 U64 X16 - { true, true, true, true, true, "K01_01", "SLOW", }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant - { true, true, true, true, true, "K01_02", "FAST", }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant - { true, true, true, true, true, "K01_03", "OLD", }, // Not present in CBM BASIC dialects, but common to some extensions (*) - { false, true, true, true, false, "K01_04", "CLEAR", }, // Not present in CBM BASIC dialects, Open ROMs specific - { false, true, true, true, false, "K01_05", "DISPOSE", }, // Not present in CBM BASIC dialects, Open ROMs specific - { false, true, true, true, false, "K01_06", "MERGE", }, // Not present in CBM BASIC dialects, but common to some extensions (*) - { false, true, true, true, false, "K01_07", "BLOAD", }, // http://www.antonis.de/qbebooks/gwbasman/bload.html - { false, true, true, true, false, "K01_08", "BSAVE", }, // http://www.antonis.de/qbebooks/gwbasman/bsave.html - { false, true, true, true, false, "K01_09", "BVERIFY", }, - { false, true, true, true, false, "K01_0A", "COLD", }, // Not present in CBM BASIC dialects, but common to some extensions (*) - { false, true, true, true, false, "K01_0B", "MEM", }, // Not present in CBM BASIC dialects, Open ROMs specific - - // (*) see https://sourceforge.net/p/vice-emu/code/HEAD/tree/trunk/vice/src/petcat.c -} }; - -// extended BASIC keywords - list reserved for hardware dependent BASIC commands - -const StringEntryList GLOBAL_Keywords_04 = { ListType::KEYWORDS, "keywords_04", -{ - // STD CRT M65 U64 X16 - { false, false, true, false, false, "K04_01", "SYSINFO", }, // Not present in CBM BASIC dialects, Open ROMs specific - { false, false, true, false, false, "K04_02", "MONITOR", }, // Amos Professional Manual, Command Index - { false, false, true, false, false, "K04_03", "BOOTCPM", }, // Not present in CBM BASIC dialects, Open ROMs specific - { false, false, true, false, false, "K04_04", "JOYCRSR", }, // Not present in CBM BASIC dialects, Open ROMs specific - { false, false, true, false, false, "K04_05", "COLORSET", }, // Not present in CBM BASIC dialects, Open ROMs specific - -} }; - -// extended BASIC keywords - list reserved for functions - -const StringEntryList GLOBAL_Keywords_06 = { ListType::KEYWORDS, "keywords_06", -{ - // STD CRT M65 U64 X16 - { false, false, true, false, false, "K06_01", "TEST", }, // Not present in CBM BASIC dialects, Open ROMs specific - -} }; - - -// BASIC errors - all dialects - -/* - These error messages are generally so well known, that it seems silly to have to even point out that they are - so widely used and known, if not completely genericised. - The most of them are inherited from the MICROSOFT BASIC on which the C64's BASIC is based, so for those, there - cannot even be any claim to copyright over them in any exclusive way. - - However, copyright of such short phrases that express ideas in short form cannot be copyrighted: - https://fairuse.stanford.edu/2003/09/09/copyright_protection_for_short/ - */ - -const StringEntryList GLOBAL_Errors = { ListType::STRINGS_BASIC, "errors", -{ - // STD CRT M65 U64 X16 --- error strings compatible with CBM BASIC V2 - { true, true, true, true, true, "EV2_01", "TOO MANY FILES" }, - { true, true, true, true, true, "EV2_02", "FILE OPEN" }, - { true, true, true, true, true, "EV2_03", "FILE NOT OPEN" }, - { true, true, true, true, true, "EV2_04", "FILE NOT FOUND" }, - { true, true, true, true, true, "EV2_05", "DEVICE NOT PRESENT" }, - { true, true, true, true, true, "EV2_06", "NOT INPUT FILE" }, - { true, true, true, true, true, "EV2_07", "NOT OUTPUT FILE" }, - { true, true, true, true, true, "EV2_08", "MISSING FILENAME" }, - { true, true, true, true, true, "EV2_09", "ILLEGAL DEVICE NUMBER" }, - { true, true, true, true, true, "EV2_0A", "NEXT WITHOUT FOR" }, - { true, true, true, true, true, "EV2_0B", "SYNTAX" }, - { true, true, true, true, true, "EV2_0C", "RETURN WITHOUT GOSUB" }, - { true, true, true, true, true, "EV2_0D", "OUT OF DATA" }, - { true, true, true, true, true, "EV2_0E", "ILLEGAL QUANTITY" }, - { true, true, true, true, true, "EV2_0F", "OVERFLOW" }, - { true, true, true, true, true, "EV2_10", "OUT OF MEMORY" }, - { true, true, true, true, true, "EV2_11", "UNDEF\'D STATEMENT" }, - { true, true, true, true, true, "EV2_12", "BAD SUBSCRIPT" }, - { true, true, true, true, true, "EV2_13", "REDIM\'D ARRAY" }, - { true, true, true, true, true, "EV2_14", "DIVISION BY ZERO" }, - { true, true, true, true, true, "EV2_15", "ILLEGAL DIRECT" }, - { true, true, true, true, true, "EV2_16", "TYPE MISMATCH" }, - { true, true, true, true, true, "EV2_17", "STRING TOO LONG" }, - { true, true, true, true, true, "EV2_18", "FILE DATA" }, - { true, true, true, true, true, "EV2_19", "FORMULA TOO COMPLEX" }, - { true, true, true, true, true, "EV2_1A", "CAN\'T CONTINUE" }, - { true, true, true, true, true, "EV2_1B", "UNDEF\'D FUNCTION" }, - { true, true, true, true, true, "EV2_1C", "VERIFY" }, - { true, true, true, true, true, "EV2_1D", "LOAD" }, - { true, true, true, true, true, "EV2_1E", "BREAK" }, - // STD CRT M65 U64 X16 --- error strings compatible with CBM BASIC V7 - { false, false, false, false, false, "EV7_1F", "CAN'T RESUME" }, // not used for now - { false, false, false, false, false, "EV7_20", "LOOP NOT FOUND" }, // not used for now - { false, false, false, false, false, "EV7_21", "LOOP WITHOUT DO" }, // not used for now - { true, true, true, true, true, "EV7_22", "DIRECT MODE ONLY" }, - { false, false, false, false, false, "EV7_23", "NO GRAPHICS AREA" }, // not used for now - { false, false, false, false, false, "EV7_24", "BAD DISK" }, // not used for now - { false, false, false, false, false, "EV7_25", "BEND NOT FOUND" }, // not used for now - { true, true, true, true, true, "EV7_26", "LINE NUMBER TOO LARGE" }, - { false, false, false, false, false, "EV7_27", "UNRESOLVED REFERENCE" }, // not used for now - { true, true, true, true, true, "EV7_28", "NOT IMPLEMENTED" }, // this message actually differs from the CBM one - { false, false, false, false, false, "EV7_29", "FILE READ" }, // not used for now - // STD CRT M65 U64 X16 --- error strings specific to Open ROMs, not present in CBM BASIC dialects - { true, true, true, true, true, "EOR_2A", "MEMORY CORRUPT" }, - { false, false, true, false, false, "EOR_2B", "NATIVE MODE ONLY" }, - { false, false, true, false, false, "EOR_2C", "LEGACY MODE ONLY" }, -} }; - -// BASIC errors - miscelaneous strings - -const StringEntryList GLOBAL_MiscStrings = { ListType::STRINGS_BASIC, "misc", -{ - // STD CRT M65 U64 X16 --- misc strings as on CBM machines - { true, true, true, true, true, "STR_RET_QM", "\r?" }, - { true, true, true, true, true, "STR_BYTES", " BASIC BYTES FREE" }, // https://github.com/stefanhaustein/expressionparser - { true, true, true, true, true, "STR_READY", "\rREADY.\r" }, // https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconc_whatistsonative.htm https://github.com/stefanhaustein/expressionparser - { true, true, true, true, true, "STR_ERROR", " ERROR" }, // simply the word error that is attached to the other parts of messages https://fjkraan.home.xs4all.nl/comp/apple2faq/app2asoftfaq.html - { true, true, true, true, true, "STR_IN", " IN " }, - { false, true, true, false, false, "STR_IF_SURE", "\rARE YOU SURE? " }, // https://docs.microsoft.com/en-us/windows/win32/uxguide/mess-confirm - // STD CRT M65 U64 X16 --- misc strings specific to Open ROMs, not present in CBM ROMs - { true, true, true, true, true, "STR_BRK_AT", "\rBRK AT $" }, - - { false, false, true, false, false, "STR_ORS", "OPEN ROMS BASIC & KERNAL\r" }, - { false, false, true, false, false, "STR_ORS_LEGACY", "(LEGACY MODE) " }, - - { false, true, true, false, false, "STR_MEM_HDR", "\r\x12 AREA START SIZE \r" }, - { false, true, true, false, false, "STR_MEM_1", " $" }, - { false, true, true, false, false, "STR_MEM_2", " " }, - { false, true, true, false, false, "STR_MEM_TEXT", "\r TEXT" }, - { false, true, true, false, false, "STR_MEM_VARS", "\r VARS" }, - { false, true, true, false, false, "STR_MEM_ARRS", "\r ARRS" }, - { false, true, true, false, false, "STR_MEM_STRS", "\r STRS" }, - { false, true, true, false, false, "STR_MEM_FREE", "\r\r FREE" }, - - { false, false, true, false, false, "STR_DOS_FD", "\rFLOPPY : " }, - { false, false, true, false, false, "STR_DOS_SEPAR", " / " }, - { false, false, true, false, false, "STR_DOS_RD", "\rRAM DISK : " }, - - { false, false, true, false, false, "STR_SI_HEADER", "OPEN ROMS, " }, - { false, false, true, false, false, "STR_SI_MODE64", "LEGACY MODE" }, - { false, false, true, false, false, "STR_SI_MODE65", "NATIVE MODE" }, - { false, false, true, false, false, "STR_SI_HDR_REL", "\rRELEASE " }, - { false, false, true, false, false, "STR_SI_HDR_HW", "BOARD : " }, - { false, false, true, false, false, "STR_SI_HW_01", "MEGA65 R1" }, - { false, false, true, false, false, "STR_SI_HW_02", "MEGA65 R2" }, - { false, false, true, false, false, "STR_SI_HW_03", "MEGA65 R3" }, - { false, false, true, false, false, "STR_SI_HW_21", "MEGAPHONE R1" }, - { false, false, true, false, false, "STR_SI_HW_40", "NEXYS4 PSRAM" }, - { false, false, true, false, false, "STR_SI_HW_41", "NEXYS4 DDR" }, - { false, false, true, false, false, "STR_SI_HW_42", "NEXYS4 DDR WIDGET" }, - { false, false, true, false, false, "STR_SI_HW_FD", "WUKONG A100T" }, - { false, false, true, false, false, "STR_SI_HW_FE", "SIMULATION VHDL" }, - { false, false, true, false, false, "STR_SI_HW_XX", "UNKNOWN ID $" }, - { false, false, true, false, false, "STR_SI_HDR_VID", "\rVIDEO : " }, - - { false, false, true, false, false, "STR_RD_NAME", "RAMDISK.DNP " }, - { false, false, true, false, false, "STR_RD_OK", "LOADED OK\r" }, - { false, false, true, false, false, "STR_RD_ERR1", "SIZE WRONG\r" }, - { false, false, true, false, false, "STR_RD_ERR2", "TOO LARGE\r" }, - { false, false, true, false, false, "STR_RD_ERR3", "NOT FOUND\r" }, - - // Note: depending on configuration, additional strings will be added here -} }; - - -/* NOTE: below are the BASIC V7/V10 keywords, see: - - https://sites.google.com/site/h2obsession/CBM/basic/tokens - - http://www.zimmers.net/anonftp/pub/cbm/programming/cbm-basic-tokens.txt - - "RGR" // $CC - "RCLR" // $CD - - "JOY" // $CF // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html - "RDOT" // $D0 - "DEC" // $D1 // Amos Professional Manual, Command Index - "HEX$" // $D2 // http://www.antonis.de/qbebooks/gwbasman/hexs.html - "ERR$" // $D3 // Amos Professional Manual, Command Index - "INSTR" // $D4 // Amos Professional Manual, Command Index - "ELSE" // $D5 // Amos Professional Manual, Command Index - "RESUME" // $D6 // Amos Professional Manual, Command Index - "TRAP" // $D7 // Amos Professional Manual, Command Index - "TRON" // $D8 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html - "TROFF" // $D9 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html - "SOUND" // $DA // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html - "VOL" // $DB - "AUTO" // $DC // http://www.antonis.de/qbebooks/gwbasman/auto.html - "PUDEF" // $DD - "GRAPHIC" // $DE - "PAINT" // $DF // Amos Professional Manual, Command Index - "CHAR" // $E0 // https://www.c64-wiki.com/wiki/Screen_Graphics_64 - "BOX" // $E1 // Amos Professional Manual, Command Index - "CIRCLE" // $E2 // https://en.wikipedia.org/wiki/Sinclair_BASIC - "PASTE" // $E3 // Amos Professional Manual, Command Index - "CUT" // $E4 - "LINE" // $E5 // https://en.wikipedia.org/wiki/Sinclair_BASIC - "LOCATE" // $E6 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html - "COLOR" // $E7 - "SCNCLR" // $E8 - "SCALE" // $E9 // https://www.c64-wiki.com/wiki/Super_Expander_64 - "HELP" // $EA // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7 - "DO" // $EB // Amos Professional Manual, Command Index - "LOOP" // $EC // Amos Professional Manual, Command Index - "EXIT" // $ED // Amos Professional Manual, Command Index - "DIR" // $EE // Amos Professional Manual, Command Index - "DSAVE" // $EF // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "DLOAD" // $F0 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "HEADER" // $F1 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "SCRATCH" // $F2 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "COLLECT" // $F3 - "COPY" // $F4 // https://en.wikipedia.org/wiki/Sinclair_BASIC - "RENAME" // $F5 // Amos Professional Manual, Command Index - "BACKUP" // $F6 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "DELETE" // $F7 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "RENUMBER" // $F8 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "KEY" // $F9 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html - "MONITOR" // $FA // Amos Professional Manual, Command Index - "USING" // $FB // http://www.antonis.de/qbebooks/gwbasman/printusing.html - "UNTIL" // $FC // Amos Professional Manual, Command Index - "WHILE" // $FD // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html - - "POT" // $CE $02 - "BUMP" // $CE $03 - "PEN" // $CE $04 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html - "RSPPOS" // $CE $05 // https://www.c64-wiki.com/wiki/Super_Expander_64 - "RSPRITE" // $CE $06 - "RSPCOLOR" // $CE $07 - "XOR" // $CE $08 - "RWINDOW" // $CE $09 - "POINTER" // $CE $0A - - "BANK" // $FE $02 // Amos Professional Manual, Command Index - "FILTER" // $FE $03 // https://www.c64-wiki.com/wiki/Super_Expander_64 - "PLAY" // $FE $04 // Amos Professional Manual, Command Index - "TEMPO" // $FE $05 // Amos Professional Manual, Command Index - "MOVSPR" // $FE $06 - "SPRITE" // $FE $07 // Amos Professional Manual, Command Index - "SPRCOLOR" // $FE $08 - "RREG" // $FE $09 - "ENVELOPE" // $FE $0A - "SLEEP" // $FE $0B - "CATALOG" // $FE $0C // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "DOPEN" // $FE $0D - "APPEND" // $FE $0E // Amos Professional Manual, Command Index - "DCLOSE" // $FE $0F - "BSAVE" // $FE $10 // http://www.antonis.de/qbebooks/gwbasman/bsave.html - "BLOAD" // $FE $11 // http://www.antonis.de/qbebooks/gwbasman/bload.html - "RECORD" // $FE $12 - "CONCAT" // $FE $13 - "DVERIFY" // $FE $14 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "DCLEAR" // $FE $15 - "SPRSAV" // $FE $16 // https://www.c64-wiki.com/wiki/Super_Expander_64 - "COLLISION" // $FE $17 - "BEGIN" // $FE $18 - "BEND" // $FE $19 - "WINDOW" // $FE $1A // http://www.antonis.de/qbebooks/gwbasman/window.html - "BOOT" // $FE $1B - "WIDTH" // $FE $1C // http://www.antonis.de/qbebooks/gwbasman/width.html - "SPRDEF" // $FE $1D // https://www.c64-wiki.com/wiki/Super_Expander_64 - "QUIT" // $FE $1E // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7 - - "STASH" // $FE $1F - v7 - "FETCH" // $FE $21 - v7 - "SWAP" // $FE $23 - v7 // http://www.antonis.de/qbebooks/gwbasman/swap.html - - "DMA" // $FE $17/$21/$23 - v10 - - "OFF" // $FE $24 // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7 - "FAST" // $FE $25 // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant - "SLOW" // $FE $26 // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant - "TYPE" // $FE $27 - "BVERIFY" // $FE $28 - "DIRECTORY" // $FE $29 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "ERASE" // $FE $2A // https://en.wikipedia.org/wiki/Sinclair_BASIC - "FIND" // $FE $2B // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) - "CHANGE" // $FE $2C - "SET" // $FE $2D - "SCREEN" // $FE $2E // http://www.antonis.de/qbebooks/gwbasman/screens.html - "POLYGON" // $FE $2F // Amos Professional Manual, Command Index - "ELLIPSE" // $FE $30 // Amos Professional Manual, Command Index - "VIEWPORT" // $FE $31 - "GCOPY" // $FE $32 - "PEN" // $FE $33 // Amos Professional Manual, Command Index - "PALETTE" // $FE $34 // Amos Professional Manual, Command Index - "DMODE" // $FE $35 - "DPAT" // $FE $36 - "PIC" // $FE $37 - "GENLOCK" // $FE $38 - "FOREGROUND" // $FE $39 - - "BACKGROUND" // $FE $3B - "BORDER" // $FE $3C // https://en.wikipedia.org/wiki/Sinclair_BASIC - "HIGHLIGHT" // $FE $3D -*/ - -std::map GLOBAL_ConfigOptions; - -// -// Work class definitions -// - -// This class encapsulates the optional dictionary encoding of selected string lists -class DictEncoder -{ -public: - - void addString(const std::string &inString, StringEncoded *outPtr); - - void process(StringEntryList &outDictionary); - -private: - - bool optimizeSplit(); - bool optimizeJoin(); - void optimizeOrder(); - - void cleanupDictionary(); - - void extractWords(std::vector &candidateList); - int32_t evaluateCandidate(std::string &candidate); - - std::vector encodings; - std::vector dictionary; -}; - -// Main class to encode strings based on character frequency -class DataSet -{ -public: - - void addStrings(const StringEntryList &stringList); - - const std::string &getOutput(); - -private: - - void process(); - - void generateConfigDepStrings(); - void validateLists(); - void calculateFrequencies(); - void encodeStringsDict(); - void encodeStringsFreq(); - - void encodeByFreq(const std::string &plain, StringEncoded &encoded) const; - - void prepareOutput(); - void prepareOutput_1n_3n(std::ostringstream &stream); - void prepareOutput_labels(std::ostringstream &stream, - const StringEntryList &stringEntryList, - const StringEncodedList &stringEncodedList); - void prepareOutput_packed(std::ostringstream &stream, - const StringEntryList &stringEntryList, - const StringEncodedList &stringEncodedList); - - void putCharEncoding(std::ostringstream &stream, uint8_t idx, char character, bool is3n); - - bool isCompressionLvl2(const StringEntryList &list) const; - - virtual bool isRelevant(const StringEntry &entry) const = 0; - virtual std::string layoutName() const = 0; - - std::vector stringEntryLists; - std::vector stringEncodedLists; - - std::vector as1n; // list of bytes to be encoded as 1 nibble - std::vector as3n; // list of bytes to be encoded as 3 nibbles - - uint8_t tk__packed_as_3n = 0; - uint8_t tk__max_keyword_len = 0; - - size_t maxAliasLen = 0; - std::string outFileContent; -}; - -class DataSetSTD : public DataSet -{ - bool isRelevant(const StringEntry &entry) const { return entry.enabledSTD; } - std::string layoutName() const { return "STD"; } -}; - -class DataSetCRT : public DataSet -{ - bool isRelevant(const StringEntry &entry) const { return entry.enabledCRT; } - std::string layoutName() const { return "CRT"; } -}; - -class DataSetM65 : public DataSet -{ - bool isRelevant(const StringEntry &entry) const { return entry.enabledM65; } - std::string layoutName() const { return "M65"; } -}; - -class DataSetU64 : public DataSet -{ - bool isRelevant(const StringEntry &entry) const { return entry.enabledU64; } - std::string layoutName() const { return "STD"; } // XXX to be changed for 'U64' once the layout is finished -}; - -class DataSetX16 : public DataSet -{ - bool isRelevant(const StringEntry &entry) const { return entry.enabledX16; } - std::string layoutName() const { return "X16"; } -}; - -// -// Work class implementation -// - -void DictEncoder::addString(const std::string &inString, StringEncoded *outPtr) -{ - // Store the pointer to encoding - - encodings.push_back(outPtr); - - // Check if string is already present in the dictionary - - auto pos = std::find(dictionary.begin(), dictionary.end(), inString); - - // Create initial encoding - - if (pos != dictionary.end()) - { - encodings.back()->push_back(pos - dictionary.begin()); - } - else - { - if (dictionary.size() > 255) - { - ERROR("max 255 strings allowed for dictionary compression"); - } - - dictionary.push_back(inString); - encodings.back()->push_back(dictionary.size() - 1); - } -} - - -void DictEncoder::extractWords(std::vector &candidateList) -{ - for (const auto &dictionaryEntry : dictionary) - { - // Split the dictionary entry into words - - std::istringstream entryStream(dictionaryEntry); - while (entryStream) - { - std::string word_1; - entryStream >> word_1; - - if (word_1.empty()) continue; - - // Create possible variants with spaces - - std::string word_2 = " " + word_1; - std::string word_3 = word_1 + " "; - std::string word_4 = word_2 + " "; - - // If necessary, add the word to candidate list - - auto addToList = [&dictionaryEntry, &candidateList](std::string &word) - { - // Before dding, make sure the word variant exists in the string, - // is not already on the list, and is long enough it makes sense to try - - if (word.size() < 1) return; - if (dictionaryEntry.find(word) == std::string::npos) return; - if (std::find(candidateList.begin(), candidateList.end(), word) != candidateList.end()) return; - - candidateList.push_back(word); - }; - - addToList(word_1); - addToList(word_2); - addToList(word_3); - addToList(word_4); - } - } -} - -int32_t DictEncoder::evaluateCandidate(std::string &candidate) -{ - // XXX it looks like this is not 100% right - debug the algorithm if dictionary compression is reintroduced - - // Build the score - find how much bytes can be spared by extracting this particular candidate - // Limited size of the dictionary is taken into consideration - - int32_t score = -(candidate.size() + 1); - uint32_t targetSize = dictionary.size() + 1; - - // First check for situation when the candidate equals the currently existing dictionary entry - - for (const auto &dictionaryEntry : dictionary) - { - if (dictionaryEntry == candidate) - { - // Candidate equals the string - - score += candidate.size() + 1; - targetSize--; - continue; - } - } - - // Now check the remaining cases - - for (const auto &dictionaryEntry : dictionary) - { - if (dictionaryEntry == candidate) continue; // already handled - - // Find all the occurences of the candidate in the current string - - std::vector occurences; - for (auto iter = dictionaryEntry.begin(); iter < (dictionaryEntry.end() - candidate.size() + 1); iter++) - { - if (candidate != std::string(iter, iter + candidate.size())) continue; - - occurences.push_back(iter - dictionaryEntry.begin()); - iter += candidate.size() - 1; // to make sure we have no occurence overlapping - } - - if (occurences.empty()) continue; - - // Now summarize what we are going to gain when doing replacement - - if (occurences[0] != 0) - { - // For the substring before the first occurence we need to create a separate entry - // in the dictionary - unless it is already there, it brings some additional cost - - std::string otherStr = std::string(dictionaryEntry.begin(), dictionaryEntry.end() + occurences[0]); - - if (std::find(dictionary.begin(), dictionary.end(), otherStr) != dictionary.end()) - { - // This extra string is already in the dictionary - that gives extra saving - score += otherStr.size() - 1; - } - else - { - // This extra string is not in the dictionary, it has to be added - score -= 2; - targetSize++; - } - } - - for (auto iter = occurences.begin(); iter < occurences.end(); iter++) - { - // With a replacement, we should gain some bytes - score += candidate.size() - 1; - - // If the occurence ends with the last byte of dictionary string - nothing more to do - if (*iter + candidate.size() == dictionaryEntry.size()) break; - - // If next occurence starts immediately after this one - next iteration - if (iter + 1 != occurences.end() && (*(iter + 1) - *iter) == (uint8_t) candidate.size()) continue; - - // For the substring between this occurence and the next one (or the end of the string) - // we need to create a separate entry in the dictionary - unless it is already there, - // it brings some additional cost - - std::string otherStr; - if (iter + 1 == occurences.end()) - { - otherStr = std::string(dictionaryEntry.begin() + *iter + candidate.size(), - dictionaryEntry.end()); - } - else - { - otherStr = std::string(dictionaryEntry.begin() + *iter + candidate.size(), - dictionaryEntry.begin() + *(iter + 1)); - } - - if (std::find(dictionary.begin(), dictionary.end(), otherStr) != dictionary.end()) - { - // This extra string is already in the dictionary - that gives extra saving - score += otherStr.size() - 1; - } - else - { - // This extra string is not in the dictionary, it has to be added - score -= 2; - targetSize++; - } - } - - // We crossed the maximum number of strings (leave one free for the work buffer), - // do not consider this candidate - - if (targetSize > 254) return -1; - } - - return score; -} - -void DictEncoder::cleanupDictionary() -{ - // Get rid of dictionary entries which are not needed anymore - - bool isClean = false; - - while (!isClean) - { - isClean = true; - - for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++) - { - if (!iter->empty()) continue; - - // Found an empty string in the dictionary - remove it and adapt encoding - - isClean = false; - uint8_t obsoleteStrIdx = iter - dictionary.begin(); - - dictionary.erase(iter); - - for (auto &encoding : encodings) for (auto &byte : *encoding) - { - if (byte >= obsoleteStrIdx) byte--; - } - - break; - } - } -} - -bool DictEncoder::optimizeSplit() -{ - // XXX - The code here is not the best one - redesign and algorithm improvements - // (like - consider optimizing as a game and apply alpha-beta scheme to select the best - // candidate, incorporate frequency encoding into candidate evaluation) could result in - // a slightly better compression. But currently we have very few strings, and the dictionary - // compression is disabled by default - so right now this is not worth the effort. - - if (dictionary.size() >= 255) return false; // up to 255 entries in the dictionary are possible - - // First select words, which can potentially be extracted to new substrings - - std::vector candidateList; - extractWords(candidateList); - - // XXX possible future improvement: find a couple of largest common substrings and add them as candidates too - - // Now find the best word to be extracted - - auto bestCandidate = candidateList.end(); - int32_t bestCandidateScore = 0; - - for (auto iter = candidateList.begin(); iter < candidateList.end(); iter++) - { - int32_t candidateScore = evaluateCandidate(*iter); - - if (candidateScore > bestCandidateScore) - { - bestCandidate = iter; - bestCandidateScore = candidateScore; - } - } - - // Only allow for replacements that brings some size benefit - - if (bestCandidateScore < 1) return false; - - // Extract the best candidate to a separate string - - // First add our selected candidate to the dictionary and replace all strings which are equal to it - - auto &selectedStr = *bestCandidate; - uint8_t selectedStrIdx = dictionary.size(); - - dictionary.push_back(selectedStr); - - for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++) - { - uint8_t currentStrIdx = iter - dictionary.begin(); - - if (*iter != selectedStr || selectedStrIdx == currentStrIdx) continue; - - // Replace the current string with the new one - - for (auto &encoding : encodings) for (auto &byte : *encoding) - { - if (byte == currentStrIdx) byte = selectedStrIdx; - } - - // Mark obsolete dictionary entry as free for removal - - iter->clear(); - } - - cleanupDictionary(); - - // Now handle normal cases, when the string needs to be split - // XXX - this is not time-effective, optimize this in the future - - bool optimizeAgain = true; - bool nexIteration = true; - while (nexIteration) - { - nexIteration = false; - - for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++) - { - auto ¤tStr = *iter; - uint8_t currentStrIdx = iter - dictionary.begin(); - auto selectedStrIter = std::find(dictionary.begin(), dictionary.end(), selectedStr); - selectedStrIdx = selectedStrIter - dictionary.begin(); - - if (currentStrIdx == selectedStrIdx) continue; - - // Check if the current string contains the selected string - - auto pos = currentStr.find(selectedStr); - if (pos == std::string::npos) continue; - - nexIteration = true; - optimizeAgain = true; - - // Prepare replacement sequence for 'selectedStrIdx' - - std::vector replacement; - - if (pos == 0) - { - // The current dictionary string starts with our selected substring - - replacement.push_back(selectedStrIdx); - currentStr = currentStr.substr(selectedStr.size(), currentStr.size() - selectedStr.size()); - - if (!currentStr.empty()) - { - // Check if the remaining string is present somewhere else in the dictionary; if so, reuse it - - uint8_t pos2 = currentStrIdx; - for (auto iter2 = dictionary.begin(); iter2 < dictionary.end(); iter2++) - { - if (iter2 == iter) continue; - if (currentStr == *iter2) - { - dictionary[pos2].clear(); - pos2 = iter2 - dictionary.begin(); - break; - } - } - - replacement.push_back(pos2); - } - } - else - { - // The current dictionary string does not start with our selected substring - - std::string newStr = currentStr.substr(0, pos); - currentStr = currentStr.substr(pos, currentStr.size() - pos); - - // Check if the newStr is present somewhere else in the dictionary; if so, reuse it - - bool found = false; - uint8_t pos2 = 0; - for (auto iter2 = dictionary.begin(); iter2 < dictionary.end(); iter2++) - { - if (*iter2 == newStr) - { - found = true; - pos2 = iter2 - dictionary.begin(); - - break; - } - } - - if (found) - { - replacement.push_back(pos2); - } - else - { - dictionary.push_back(newStr); - replacement.push_back(dictionary.size() - 1); - } - - replacement.push_back(currentStrIdx); - } - - // Now replace all the occurences of 'selectedStrIdx' with a 'replacement' vector - - for (auto &encoding : encodings) - { - for (uint8_t idx = 0; idx < encoding->size(); idx++) - { - if ((*encoding)[idx] != currentStrIdx) continue; - - // Perform replacement - - encoding->erase(encoding->begin() + idx); - - for (const auto replacementIdx : replacement) - { - encoding->insert(encoding->begin() + idx, replacementIdx); - idx++; - } - - idx--; - } - } - - cleanupDictionary(); - } - } - - return optimizeAgain; -} - -bool DictEncoder::optimizeJoin() -{ - // Try to optimize by joining two substrings - - // XXX - possible future improvement - - return false; -} - -void DictEncoder::optimizeOrder() -{ - // XXX - it would be better to do this after the dictionary is compressed - - // Try to optimize order of strings in the dictionary for fastest display - // - put the shorter substrings first - // - for strings with roughly the same length put more frequently used first - - typedef struct Entry - { - uint32_t penalty; - std::string word; - } Entry; - - // Create copy of the dictionary with optimization info - - std::vector optimizedOrder; - for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++) - { - const auto &dictionaryStr = *iter; - - uint8_t sizePenalty = dictionaryStr.size() / 3; - uint16_t freqPenalty = 65535; - - // Calculate penalty for low occurence frequency - - for (auto &encoding : encodings) - { - for (auto &byte : *encoding) - { - if (byte == (iter - dictionary.begin())) freqPenalty--; - } - } - - // Create new entry - - Entry newEntry; - newEntry.word = dictionaryStr; - newEntry.penalty = sizePenalty * 65536 + freqPenalty; - - optimizedOrder.push_back(newEntry); - } - - - // Determine ordering - sort from smallest penalty to largest - - std::sort(optimizedOrder.begin(), optimizedOrder.end(), - [](const Entry &e1, const Entry &e2) { return e1.penalty < e2.penalty; }); - - // Create helper table for reordering - - std::vector reorderTable; - - for (uint8_t idx1 = 0; idx1 < dictionary.size(); idx1++) - { - uint8_t idx2 = 0; - while (dictionary[idx1] != optimizedOrder[idx2].word) idx2++; - - reorderTable.push_back(idx2); - } - - // Perform the reordering - - for (auto &encoding : encodings) for (auto &byte : *encoding) - { - byte = reorderTable[byte]; - } - - for (uint8_t idx = 0; idx < dictionary.size(); idx++) dictionary[idx] = optimizedOrder[idx].word; -} - -void DictEncoder::process(StringEntryList &outDictionary) -{ - if (dictionary.empty()) return; - // Optimize as long as it brings any improvement - - while (optimizeSplit() || optimizeJoin()) ; - optimizeOrder(); - - // Export the dictionary to external format - - outDictionary.type = ListType::DICTIONARY; - outDictionary.name = "DICTIONARY"; - - for (const auto &dictionaryStr : dictionary) - { - StringEntry newEntry = { true, true, true, true, true, "", dictionaryStr }; - outDictionary.list.push_back(newEntry); - } - - // Adapt the encoding to external format (0 = end of string) - - for (auto &encoding : encodings) - { - for (auto &byte : *encoding) byte++; - encoding->push_back(0); - } -} - -bool DataSet::isCompressionLvl2(const StringEntryList &list) const -{ - return (GLOBAL_ConfigOptions["COMPRESSION_LVL_2"] && list.type == ListType::STRINGS_BASIC); -} - -void DataSet::addStrings(const StringEntryList &stringList) -{ - // Import the new list of strings - - stringEntryLists.push_back(stringList); - stringEncodedLists.emplace_back(); - - // Clear strings not relevant for the current configuration - - while (1) - { - if (isRelevant(stringEntryLists.back().list.back())) break; - - stringEntryLists.back().list.pop_back(); - if (stringEntryLists.back().list.empty()) - { - ERROR(std::string("no valid strings in layout '") + layoutName() + "', list'" + stringEntryLists.back().name + "'"); - } - } - - // Clear the output content - make sure it is not valid anymore - - outFileContent.clear(); -} - -void DataSet::process() -{ - std::cout << "processing file '" << CMD_cnfFile << "', layout '" << layoutName() << "'" << std::endl; - - generateConfigDepStrings(); - validateLists(); - encodeStringsDict(); - calculateFrequencies(); - encodeStringsFreq(); - prepareOutput(); -} - -const std::string &DataSet::getOutput() -{ - if (outFileContent.empty()) - { - process(); - } - - return outFileContent; -} - -void DataSet::generateConfigDepStrings() -{ - // Generate string to show the build features - - std::string featureStr; - std::string featureStrM65 = "\r"; - - // Tape support features - - if (GLOBAL_ConfigOptions["TAPE_NORMAL"] && GLOBAL_ConfigOptions["TAPE_TURBO"]) - { - featureStr += "TAPE LOAD NORMAL TURBO\r"; - featureStrM65 += "TAPE : LOAD NORMAL TURBO\r"; - } - else if (GLOBAL_ConfigOptions["TAPE_NORMAL"]) - { - featureStr += "TAPE LOAD NORMAL\r"; - featureStrM65 += "TAPE : LOAD NORMAL\r"; - } - else if (GLOBAL_ConfigOptions["TAPE_TURBO"]) - { - featureStr += "TAPE LOAD TURBO\r"; - featureStrM65 += "TAPE : LOAD TURBO\r"; - } - - // IEC support features - - if (GLOBAL_ConfigOptions["IEC"]) - { - featureStr += "IEC"; - featureStrM65 += "IEC :"; - - bool extendedIEC = false; - - if (GLOBAL_ConfigOptions["IEC_BURST_CIA1"]) - { - featureStr += " BURST1"; - extendedIEC = true; - } - if (GLOBAL_ConfigOptions["IEC_BURST_CIA2"]) - { - featureStr += " BURST2"; - extendedIEC = true; - } - if (GLOBAL_ConfigOptions["IEC_BURST_MEGA65"]) - { - featureStr += " BURST"; - featureStrM65 += " BURST"; - extendedIEC = true; - } - - if (GLOBAL_ConfigOptions["IEC_DOLPHINDOS"]) - { - featureStr += " DOLPHIN"; - featureStrM65 += " DOLPHIN"; - extendedIEC = true; - } - - if (GLOBAL_ConfigOptions["IEC_JIFFYDOS"]) - { - featureStr += " JIFFY"; - featureStrM65 += " JIFFY"; - extendedIEC = true; - } - - if (!extendedIEC) - { - featureStr += " NORMAL ONLY"; - featureStrM65 += " NORMAL ONLY"; - } - - featureStr += "\r"; - featureStrM65 += "\r"; - } - else - { - featureStrM65 += "IEC : -\r"; - } - - // RS-232 support features - - if (GLOBAL_ConfigOptions["RS232_ACIA"]) featureStr += "ACIA 6551\r"; - if (GLOBAL_ConfigOptions["RS232_UP2400"]) featureStr += "UP2400\r"; - if (GLOBAL_ConfigOptions["RS232_UP9600"]) featureStr += "UP9600\r"; - - featureStrM65 += "RS-232 : -\r"; - - // CBDOS features - - featureStrM65 += "\rSD CARD : "; - - // Keyboard support features - - if (GLOBAL_ConfigOptions["KEYBOARD_C128"]) featureStr += "KBD 128\r"; - - // Add strings to appropriate list - - for (auto &stringEntryList : stringEntryLists) - { - if (stringEntryList.name != std::string("misc")) continue; - - // List found - - if (GLOBAL_ConfigOptions["SHOW_FEATURES"] || GLOBAL_ConfigOptions["MB_M65"]) - { - StringEntry newEntry1 = { true, true, true, true, true, "STR_PAL", "PAL\r" }; - StringEntry newEntry2 = { true, true, true, true, true, "STR_NTSC", "NTSC\r" }; - - stringEntryList.list.push_back(newEntry1); - stringEntryList.list.push_back(newEntry2); - } - - if (GLOBAL_ConfigOptions["SHOW_FEATURES"]) - { - StringEntry newEntry = { true, true, true, true, true, "STR_FEATURES", featureStr }; - stringEntryList.list.push_back(newEntry); - } - - if (GLOBAL_ConfigOptions["MB_M65"]) - { - StringEntry newEntry = { false, true, true, false, false, "STR_SI_FEATURES", featureStrM65 }; - stringEntryList.list.push_back(newEntry); - } - - if (!GLOBAL_ConfigOptions["BRAND_CUSTOM_BUILD"] || GLOBAL_ConfigOptions["MB_M65"]) - { - StringEntry newEntry = { true, true, true, true, true, "STR_PRE_REV", "RELEASE " }; - stringEntryList.list.push_back(newEntry); - } - - break; - } -} - -void DataSet::validateLists() -{ - for (const auto &stringEntryList : stringEntryLists) - { - for (const auto &stringEntry : stringEntryList.list) - { - // Check for maximum allowed string length - if (stringEntry.string.length() > 255) ERROR("string cannot be longer than 255 characters"); - - // Check mor maximum keyword length - if (stringEntryList.type == ListType::KEYWORDS) - { - if (stringEntry.string.length() > 16) ERROR("keyword cannot be longer than 16 characters"); - tk__max_keyword_len = std::max(tk__max_keyword_len, (uint8_t) stringEntry.string.length()); - } - - // Update maximum alias length too - maxAliasLen = std::max(maxAliasLen, stringEntry.alias.length()); - - for (const auto &character : stringEntry.string) - { - if ((unsigned char) character >= 0x80) - { - ERROR(std::string("character above 0x80 in string '") + stringEntry.string + "'"); - } - } - } - } -} - -void DataSet::encodeStringsDict() -{ - DictEncoder dictEncoder; - - // Add strings for dictionary compresssion - - for (uint8_t idx = 0; idx < stringEntryLists.size(); idx++) - { - const auto &stringEntryList = stringEntryLists[idx]; - auto &stringEncodedList = stringEncodedLists[idx]; - - // Skip lists not to be encoded using the dictionary - if (!isCompressionLvl2(stringEntryList)) continue; - - stringEncodedList.resize(stringEntryList.list.size()); - for (uint8_t idxEntry = 0; idxEntry < stringEntryList.list.size(); idxEntry++) - { - dictEncoder.addString(stringEntryList.list[idxEntry].string, - &stringEncodedList[idxEntry]); - } - } - - // Perform the compression - - StringEntryList dictionary; - dictEncoder.process(dictionary); - - // Add new lists - - stringEntryLists.push_back(dictionary); - stringEncodedLists.emplace_back(); -} - -void DataSet::calculateFrequencies() -{ - as1n.clear(); - as3n.clear(); - - std::map freqMapGeneral; // general character frequency map - std::map freqMapKeywords; // frequency map for keywords - - // Calculate frequencies of characters in the strings - - for (const auto &stringEntryList : stringEntryLists) - { - // Skip lists encoded by the dictionary - if (isCompressionLvl2(stringEntryList)) continue; - - for (const auto &stringEntry : stringEntryList.list) - { - if (!isRelevant(stringEntry)) continue; - - for (const auto &character : stringEntry.string) - { - freqMapGeneral[character]++; - if (stringEntryList.type == ListType::KEYWORDS) freqMapKeywords[character]++; - } - } - } - - // Sort characters by frequency - - std::vector freqVector1; - - for (auto iter = freqMapGeneral.begin(); iter != freqMapGeneral.end(); ++iter) - { - freqVector1.push_back(iter->first); - } - - std::sort(freqVector1.begin(), freqVector1.end(), [&freqMapGeneral](char e1, char e2) - { return freqMapGeneral[e2] < freqMapGeneral[e1]; }); - - // Check if minimal amount of characters needed, below 15 is not supported by the 6502 side code - - if (freqVector1.size() < 15) ERROR(std::string("not enough distinct characters in layout '") + layoutName() + "', at least 15 needed"); - - // Extract 14 most frequent characters to be encoded as 1 nibble - - for (uint8_t idx = 0; idx < 14; idx++) - { - as1n.push_back(freqVector1[idx]); - } - - // Now sort them by frequency in keywords, in descending order - this will speed up the tokenizer a little - - std::sort(as1n.begin(), as1n.end(), [&freqMapKeywords](char e1, char e2) { return freqMapKeywords[e2] > freqMapKeywords[e1]; }); - - // Extract characters to be encoded as 3 nibbles, which actually exist in keywords - - std::vector freqVector2; - - tk__packed_as_3n = 0; - for (uint8_t idx = 14; idx < freqVector1.size(); idx++) - { - const auto &character = freqVector1[idx]; - - if (freqMapKeywords[character] > 0) - { - as3n.push_back(character); - tk__packed_as_3n++; - } - else - { - freqVector2.push_back(character); - } - } - tk__packed_as_3n = std::max(tk__packed_as_3n, (uint8_t) 1); - - // Again, sort them by frequency in keywords, in descending order - this will speed up the tokenizer a little - - std::sort(as3n.begin(), as3n.end(), [&freqMapKeywords](char e1, char e2) { return freqMapKeywords[e2] > freqMapKeywords[e1]; }); - - // Finally extract the remaining characters to be encoded as 3 nibbles - - for (const auto &character : freqVector2) - { - as3n.push_back(character); - } -} - -void DataSet::encodeByFreq(const std::string &plain, StringEncoded &encoded) const -{ - bool fullByte = true; - - auto push1n = [&](uint8_t val) // push 1 nibble - encoded character or 0xF mark - { - if (fullByte) - { - encoded.push_back(val); - fullByte = false; - } - else - { - encoded.back() += val * 0x10; - fullByte = true; - } - }; - - auto push2n = [&](uint8_t val) // push the remaining nibbles for 3-niobble encoded characters - { - if (fullByte) - { - encoded.push_back(val); - } - else - { - // Encode byte in a way to be easily decoded by 6502 - encoded.back() += (val / 0x10) * 0x10; - encoded.push_back(val % 0x10); - } - }; - - // Encode every single character by frequency, put them in the output vector - - for (const char &character : plain) - { - auto iterEncoding1 = std::find(as1n.begin(), as1n.end(), character); - if (iterEncoding1 != as1n.end()) - { - push1n(std::distance(as1n.begin(), iterEncoding1) + 1); - } - else - { - push1n(0x0F); - - auto iterEncoding3 = std::find(as3n.begin(), as3n.end(), character); - if (iterEncoding3 == as3n.end()) - { - ERROR("internal error in 'encodeByFreq'"); - } - push2n(std::distance(as3n.begin(), iterEncoding3) + 1); - } - } - - // Make sure the last byte of encoded stream is 0 - - if (encoded.size() == 0 || encoded.back() != 0) encoded.push_back(0); -} - -void DataSet::encodeStringsFreq() -{ - // Encode every relevant string from every list - by character frequency - - for (uint8_t idx = 0; idx < stringEntryLists.size(); idx++) - { - const auto &stringEntryList = stringEntryLists[idx]; - auto &stringEncodedList = stringEncodedLists[idx]; - - // Skip lists encoded by the dictionary - if (isCompressionLvl2(stringEntryList)) continue; - - // Perform frequency encoding of the list - - for (const auto &stringEntry : stringEntryList.list) - { - stringEncodedList.emplace_back(); - auto &stringEncoded = stringEncodedList.back(); - - if (isRelevant(stringEntry)) - { - encodeByFreq(stringEntry.string, stringEncoded); - } - } - } -} - -void DataSet::putCharEncoding(std::ostringstream &stream, uint8_t idx, char character, bool is3n) -{ - stream << "\t!byte $" << std::uppercase << std::hex << - std::setfill('0') << std::setw(2) << +character << - " ; " << std::setfill(is3n ? '0' : ' ') << std::setw(2) << +idx; - - std::string petscii; - - if (character == 0x20) - { - petscii = " = SPACE"; - } - else if (character == 0x27) - { - petscii = " = APOSTROPHE"; - } - else if (character == 0x0D) - { - petscii = " = RETURN"; - } - else if (character > ' ' && character < 'a') - { - petscii = std::string(" = '") + character + "'"; - } - - stream << petscii << std::endl; -} - -void DataSet::prepareOutput_1n_3n(std::ostringstream &stream) -{ - uint8_t idx; - - // Export all nibble-encoded characters - - stream << std::endl << "!macro PUT_PACKED_AS_1N { ; characters encoded as 1 nibble" << std::endl << std::endl; - - idx = 0; - for (const auto& encoding : as1n) - { - putCharEncoding(stream, ++idx, encoding, false); - } - - stream << "}" << std::endl; - - // Export all byte-encoded characters - - stream << std::endl << "!macro PUT_PACKED_AS_3N { ; characters encoded as 3 nibbles" << std::endl << std::endl; - - idx = 0; - for (const auto& encoding : as3n) - { - if (idx == tk__packed_as_3n) stream << std::endl << - "\t; Characters below are not used by any BASIC keyword" << std::endl << std::endl; - - putCharEncoding(stream, ++idx, encoding, true); - } - - stream << "}" << std::endl; -} - -void DataSet::prepareOutput_labels(std::ostringstream &stream, - const StringEntryList &stringEntryList, - const StringEncodedList &stringEncodedList) -{ - // For the dictionary we do not have any labels - if (stringEntryList.type == ListType::DICTIONARY) return; - - stream << std::endl; - for (uint8_t idx = 0; idx < stringEncodedList.size(); idx++) - { - const auto &stringEntry = stringEntryList.list[idx]; - const auto &stringEncoded = stringEncodedList[idx]; - - if (!stringEncoded.empty()) - { - stream << "!set IDX__" << stringEntry.alias << - std::string(maxAliasLen - stringEntry.alias.length(), ' ') << " = $" << - std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +idx << std::endl; - } - } -} - -void DataSet::prepareOutput_packed(std::ostringstream &stream, - const StringEntryList &stringEntryList, - const StringEncodedList &stringEncodedList) -{ - if (isCompressionLvl2(stringEntryList)) - { - stream << std::endl << "!macro PUT_PACKED_DICT_"; - } - else - { - stream << std::endl << "!macro PUT_PACKED_FREQ_"; - } - - stream << stringEntryList.name << " {" << std::endl << std::endl; - - enum LastStr { NONE, SKIPPED, WRITTEN } lastStr = LastStr::NONE; - for (uint8_t idxString = 0; idxString < stringEncodedList.size(); idxString++) - { - const auto &stringEncoded = stringEncodedList[idxString]; - - if (stringEncoded.empty()) - { - if (stringEntryList.type == ListType::DICTIONARY) ERROR("internal error"); // should never happen - - if (lastStr == LastStr::WRITTEN) stream << std::endl; - stream << "\t!byte $00 ; skipped " << stringEntryList.list[idxString].alias << std::endl; - lastStr = LastStr::SKIPPED; - } - else - { - if (lastStr != LastStr::NONE) stream << std::endl; - - // Output the label - as a comment - - if (stringEntryList.type != ListType::DICTIONARY) - { - stream << "\t; IDX__" << stringEntryList.list[idxString].alias << std::endl; - } - - // Output the source string - as a comment - - stream << "\t; '"; - for (auto &character : stringEntryList.list[idxString].string) - { - if (character >= 32 && character <= 132 && character != 39 && character != 34) - { - stream << character; - } - else if (character == 13) - { - stream << ""; - } - else - { - stream << "_"; - } - } - stream << "'" << std::endl; - - // Output the encoding - - stream << "\t!byte "; - - bool first = true; - for (const auto &charEncoded : stringEncoded) - { - if (first) - { - first = false; - } - else - { - stream << ", "; - } - - stream << "$" << std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +charEncoded; - } - - stream << std::endl; - lastStr = LastStr::WRITTEN; - } - } - - // For the token list - put the 'end of keywords' mark - - if (stringEntryList.type == ListType::KEYWORDS) - { - stream << std::endl << "\t; Marker - end of the keyword list" << std::endl; - stream << "\t!byte $FF, $FF" << std::endl; - } - - stream << "}" << std::endl; -} - -void DataSet::prepareOutput() -{ - // Convert our encoded strings to a KickAssembler source - - std::ostringstream stream; - - // Export 1-nibble and 3-nibble encoding data - - prepareOutput_1n_3n(stream); - - // Export additional data for the tokenizer - - stream << std::endl << "!set TK__PACKED_AS_3N = $" << std::hex << +tk__packed_as_3n << - std::endl << "!set TK__MAX_KEYWORD_LEN = " << std::dec << +tk__max_keyword_len << std::endl; - - // Export encoded strings - - for (uint8_t idx = 0; idx < stringEntryLists.size(); idx++) - { - const auto &stringEntryList = stringEntryLists[idx]; - const auto &stringEncodedList = stringEncodedLists[idx]; - - if (stringEncodedList.empty()) continue; - - // Export labels for the current list - - prepareOutput_labels(stream, stringEntryList, stringEncodedList); - - // For the token list - put the number of tokens available - - if (stringEntryList.type == ListType::KEYWORDS) - { - stream << std::endl << "!set TK__MAXTOKEN_" << stringEntryList.name << " = " << - std::dec << stringEncodedList.size() << std::endl; - } - - // Export the packed data - - prepareOutput_packed(stream, stringEntryList, stringEncodedList); - } - - // Finalize the file stream - - stream << std::endl; - outFileContent = stream.str(); -} - - -// -// Common helper functions -// - -void parseConfigFile() -{ - GLOBAL_ConfigOptions.clear(); - - // Open the configuration file - - std::ifstream cnfFile; - cnfFile.open(CMD_cnfFile); - if (!cnfFile.good()) ERROR("unable to open config file"); - - // Parse the file - - size_t lineNum = 0; - while (!cnfFile.eof()) - { - lineNum++; - std::string workStr; - - // Read a single line, remove leading spaces and tabs - - std::getline(cnfFile, workStr); - if (cnfFile.bad()) ERROR("error reading configuration file"); - workStr = std::regex_replace(workStr, std::regex("^[ \t]+"), ""); - - // Split the line into tokens - - std::vector tokens; - - std::istringstream workStream(workStr); - std::string token; - while(std::getline(workStream, token, ' ')) - { - if (token.empty()) continue; - tokens.push_back(token); - } - if (tokens.empty()) continue; - - // Only accept lines which are boolean config options set to YES - - if (tokens.size() < 4 || - tokens[0].compare(";;") != 0 || - tokens[1].compare("#CONFIG#") != 0 || - tokens[3].compare("YES") != 0) - { - continue; - } - - // Add definitin to config option map - - if (tokens[2].empty()) ERROR(std::string("error parsing config file - line ") + std::to_string(lineNum)); - - GLOBAL_ConfigOptions[tokens[2]] = true; - } - - cnfFile.close(); -} - -void printUsage() -{ - std::cout << "\n" << - "usage: generate_strings [-o ] [-c ]" << "\n\n"; -} - -void printBanner() -{ - printBannerLineTop(); - std::cout << "// Generating compressed messages and BASIC tokens\n"; - printBannerLineBottom(); -} - -void parseCommandLine(int argc, char **argv) -{ - int opt; - - // Retrieve command line options - - while ((opt = getopt(argc, argv, "o:c:")) != -1) - { - switch(opt) - { - case 'o': CMD_outFile = optarg; break; - case 'c': CMD_cnfFile = optarg; break; - default: printUsage(); ERROR(); - } - } -} - -void writeStrings() -{ - std::string outputString; - - if (GLOBAL_ConfigOptions["PLATFORM_COMMANDER_X16"]) - { - DataSetX16 dataSetX16; - - // Add input data to computation objects - - dataSetX16.addStrings(GLOBAL_Keywords_V2); - dataSetX16.addStrings(GLOBAL_Keywords_01); - dataSetX16.addStrings(GLOBAL_Errors); - dataSetX16.addStrings(GLOBAL_MiscStrings); - - // Retrieve the results - - outputString = dataSetX16.getOutput(); - } - else if (GLOBAL_ConfigOptions["PLATFORM_COMMODORE_64"] && GLOBAL_ConfigOptions["MB_M65"]) - { - DataSetM65 dataSetM65; - - // Add input data to computation objects - - dataSetM65.addStrings(GLOBAL_Keywords_V2); - dataSetM65.addStrings(GLOBAL_Keywords_01); - dataSetM65.addStrings(GLOBAL_Keywords_04); - dataSetM65.addStrings(GLOBAL_Keywords_06); - dataSetM65.addStrings(GLOBAL_Errors); - dataSetM65.addStrings(GLOBAL_MiscStrings); - - // Retrieve the results - - outputString = dataSetM65.getOutput(); - } - else if (GLOBAL_ConfigOptions["PLATFORM_COMMODORE_64"] && GLOBAL_ConfigOptions["ROM_CRT"]) - { - DataSetCRT dataSetCRT; - - // Add input data to computation objects - - dataSetCRT.addStrings(GLOBAL_Keywords_V2); - dataSetCRT.addStrings(GLOBAL_Keywords_01); - dataSetCRT.addStrings(GLOBAL_Errors); - dataSetCRT.addStrings(GLOBAL_MiscStrings); - - // Retrieve the results - - outputString = dataSetCRT.getOutput(); - } - else if (GLOBAL_ConfigOptions["PLATFORM_COMMODORE_64"] && GLOBAL_ConfigOptions["MB_U64"]) - { - DataSetU64 dataSetU64; - - // Add input data to computation objects - - dataSetU64.addStrings(GLOBAL_Keywords_V2); - dataSetU64.addStrings(GLOBAL_Keywords_01); - dataSetU64.addStrings(GLOBAL_Errors); - dataSetU64.addStrings(GLOBAL_MiscStrings); - - // Retrieve the results - - outputString = dataSetU64.getOutput(); - } - else if (GLOBAL_ConfigOptions["PLATFORM_COMMODORE_64"]) - { - DataSetSTD dataSetSTD; - - // Add input data to computation objects - - dataSetSTD.addStrings(GLOBAL_Keywords_V2); - dataSetSTD.addStrings(GLOBAL_Keywords_01); - dataSetSTD.addStrings(GLOBAL_Errors); - dataSetSTD.addStrings(GLOBAL_MiscStrings); - - // Retrieve the results - - outputString = dataSetSTD.getOutput(); - } - else - { - ERROR("unable to determine string set"); - } - - // Remove old file - - unlink(CMD_outFile.c_str()); - - // Open output file for writing - - std::ofstream outFile(CMD_outFile, std::fstream::out | std::fstream::trunc); - if (!outFile.good()) ERROR(std::string("can't open oputput file '") + CMD_outFile + "'"); - - // Write header - - outFile << ";\n; Generated file - do not edit\n;"; - - // Write packed strings - - outFile << std::endl << std::endl; - outFile << outputString; - outFile << std::endl << std::endl; - - // Close the file - - outFile.close(); - - std::cout << std::string("compressed strings written to: '") + CMD_outFile + "'\n\n"; -} - -// -// Main function -// - -int main(int argc, char **argv) -{ - printBanner(); - parseCommandLine(argc, argv); - - parseConfigFile(); - - writeStrings(); - - return 0; -} diff --git a/tools/generate_strings/dataset.cc b/tools/generate_strings/dataset.cc new file mode 100644 index 00000000..6ec5dcae --- /dev/null +++ b/tools/generate_strings/dataset.cc @@ -0,0 +1,654 @@ + +#include "dataset.h" + +#include "dictencoder.h" + +#include +#include +#include + + +bool DataSet::isCompressionLvl2(const StringEntryList &list) const +{ + return (GLOBAL_ConfigOptions["COMPRESSION_LVL_2"] && list.type == ListType::STRINGS_BASIC); +} + +void DataSet::addStrings(const StringEntryList &stringList) +{ + // Import the new list of strings + + stringEntryLists.push_back(stringList); + stringEncodedLists.emplace_back(); + + // Clear strings not relevant for the current configuration + + while (1) + { + if (isRelevant(stringEntryLists.back().list.back())) break; + + stringEntryLists.back().list.pop_back(); + if (stringEntryLists.back().list.empty()) + { + ERROR(std::string("no valid strings in layout '") + layoutName() + "', list'" + stringEntryLists.back().name + "'"); + } + } + + // Clear the output content - make sure it is not valid anymore + + outFileContent.clear(); +} + +void DataSet::process() +{ + std::cout << "processing file '" << CMD_cnfFile << "', layout '" << layoutName() << "'" << std::endl; + + generateConfigDepStrings(); + validateLists(); + encodeStringsDict(); + calculateFrequencies(); + encodeStringsFreq(); + prepareOutput(); +} + +const std::string &DataSet::getOutput() +{ + if (outFileContent.empty()) + { + process(); + } + + return outFileContent; +} + +void DataSet::generateConfigDepStrings() +{ + // Generate string to show the build features + + std::string featureStr; + std::string featureStrM65 = "\r"; + + // Tape support features + + if (GLOBAL_ConfigOptions["TAPE_NORMAL"] && GLOBAL_ConfigOptions["TAPE_TURBO"]) + { + featureStr += "TAPE LOAD NORMAL TURBO\r"; + featureStrM65 += "TAPE : LOAD NORMAL TURBO\r"; + } + else if (GLOBAL_ConfigOptions["TAPE_NORMAL"]) + { + featureStr += "TAPE LOAD NORMAL\r"; + featureStrM65 += "TAPE : LOAD NORMAL\r"; + } + else if (GLOBAL_ConfigOptions["TAPE_TURBO"]) + { + featureStr += "TAPE LOAD TURBO\r"; + featureStrM65 += "TAPE : LOAD TURBO\r"; + } + + // IEC support features + + if (GLOBAL_ConfigOptions["IEC"]) + { + featureStr += "IEC"; + featureStrM65 += "IEC :"; + + bool extendedIEC = false; + + if (GLOBAL_ConfigOptions["IEC_BURST_CIA1"]) + { + featureStr += " BURST1"; + extendedIEC = true; + } + if (GLOBAL_ConfigOptions["IEC_BURST_CIA2"]) + { + featureStr += " BURST2"; + extendedIEC = true; + } + if (GLOBAL_ConfigOptions["IEC_BURST_MEGA65"]) + { + featureStr += " BURST"; + featureStrM65 += " BURST"; + extendedIEC = true; + } + + if (GLOBAL_ConfigOptions["IEC_DOLPHINDOS"]) + { + featureStr += " DOLPHIN"; + featureStrM65 += " DOLPHIN"; + extendedIEC = true; + } + + if (GLOBAL_ConfigOptions["IEC_JIFFYDOS"]) + { + featureStr += " JIFFY"; + featureStrM65 += " JIFFY"; + extendedIEC = true; + } + + if (!extendedIEC) + { + featureStr += " NORMAL ONLY"; + featureStrM65 += " NORMAL ONLY"; + } + + featureStr += "\r"; + featureStrM65 += "\r"; + } + else + { + featureStrM65 += "IEC : -\r"; + } + + // RS-232 support features + + if (GLOBAL_ConfigOptions["RS232_ACIA"]) featureStr += "ACIA 6551\r"; + if (GLOBAL_ConfigOptions["RS232_UP2400"]) featureStr += "UP2400\r"; + if (GLOBAL_ConfigOptions["RS232_UP9600"]) featureStr += "UP9600\r"; + + featureStrM65 += "RS-232 : -\r"; + + // CBDOS features + + featureStrM65 += "\rSD CARD : "; + + // Keyboard support features + + if (GLOBAL_ConfigOptions["KEYBOARD_C128"]) featureStr += "KBD 128\r"; + + // Add strings to appropriate list + + for (auto &stringEntryList : stringEntryLists) + { + if (stringEntryList.name != std::string("misc")) continue; + + // List found + + if (GLOBAL_ConfigOptions["SHOW_FEATURES"] || GLOBAL_ConfigOptions["MB_M65"]) + { + StringEntry newEntry1 = { true, true, true, true, true, "STR_PAL", "PAL\r" }; + StringEntry newEntry2 = { true, true, true, true, true, "STR_NTSC", "NTSC\r" }; + + stringEntryList.list.push_back(newEntry1); + stringEntryList.list.push_back(newEntry2); + } + + if (GLOBAL_ConfigOptions["SHOW_FEATURES"]) + { + StringEntry newEntry = { true, true, true, true, true, "STR_FEATURES", featureStr }; + stringEntryList.list.push_back(newEntry); + } + + if (GLOBAL_ConfigOptions["MB_M65"]) + { + StringEntry newEntry = { false, true, true, false, false, "STR_SI_FEATURES", featureStrM65 }; + stringEntryList.list.push_back(newEntry); + } + + if (!GLOBAL_ConfigOptions["BRAND_CUSTOM_BUILD"] || GLOBAL_ConfigOptions["MB_M65"]) + { + StringEntry newEntry = { true, true, true, true, true, "STR_PRE_REV", "RELEASE " }; + stringEntryList.list.push_back(newEntry); + } + + break; + } +} + +void DataSet::validateLists() +{ + for (const auto &stringEntryList : stringEntryLists) + { + for (const auto &stringEntry : stringEntryList.list) + { + // Check for maximum allowed string length + if (stringEntry.string.length() > 255) ERROR("string cannot be longer than 255 characters"); + + // Check mor maximum keyword length + if (stringEntryList.type == ListType::KEYWORDS) + { + if (stringEntry.string.length() > 16) ERROR("keyword cannot be longer than 16 characters"); + tk__max_keyword_len = std::max(tk__max_keyword_len, (uint8_t) stringEntry.string.length()); + } + + // Update maximum alias length too + maxAliasLen = std::max(maxAliasLen, stringEntry.alias.length()); + + for (const auto &character : stringEntry.string) + { + if ((unsigned char) character >= 0x80) + { + ERROR(std::string("character above 0x80 in string '") + stringEntry.string + "'"); + } + } + } + } +} + +void DataSet::encodeStringsDict() +{ + DictEncoder dictEncoder; + + // Add strings for dictionary compresssion + + for (uint8_t idx = 0; idx < stringEntryLists.size(); idx++) + { + const auto &stringEntryList = stringEntryLists[idx]; + auto &stringEncodedList = stringEncodedLists[idx]; + + // Skip lists not to be encoded using the dictionary + if (!isCompressionLvl2(stringEntryList)) continue; + + stringEncodedList.resize(stringEntryList.list.size()); + for (uint8_t idxEntry = 0; idxEntry < stringEntryList.list.size(); idxEntry++) + { + dictEncoder.addString(stringEntryList.list[idxEntry].string, + &stringEncodedList[idxEntry]); + } + } + + // Perform the compression + + StringEntryList dictionary; + dictEncoder.process(dictionary); + + // Add new lists + + stringEntryLists.push_back(dictionary); + stringEncodedLists.emplace_back(); +} + +void DataSet::calculateFrequencies() +{ + as1n.clear(); + as3n.clear(); + + std::map freqMapGeneral; // general character frequency map + std::map freqMapKeywords; // frequency map for keywords + + // Calculate frequencies of characters in the strings + + for (const auto &stringEntryList : stringEntryLists) + { + // Skip lists encoded by the dictionary + if (isCompressionLvl2(stringEntryList)) continue; + + for (const auto &stringEntry : stringEntryList.list) + { + if (!isRelevant(stringEntry)) continue; + + for (const auto &character : stringEntry.string) + { + freqMapGeneral[character]++; + if (stringEntryList.type == ListType::KEYWORDS) freqMapKeywords[character]++; + } + } + } + + // Sort characters by frequency + + std::vector freqVector1; + + for (auto iter = freqMapGeneral.begin(); iter != freqMapGeneral.end(); ++iter) + { + freqVector1.push_back(iter->first); + } + + std::sort(freqVector1.begin(), freqVector1.end(), [&freqMapGeneral](char e1, char e2) + { return freqMapGeneral[e2] < freqMapGeneral[e1]; }); + + // Check if minimal amount of characters needed, below 15 is not supported by the 6502 side code + + if (freqVector1.size() < 15) ERROR(std::string("not enough distinct characters in layout '") + layoutName() + "', at least 15 needed"); + + // Extract 14 most frequent characters to be encoded as 1 nibble + + for (uint8_t idx = 0; idx < 14; idx++) + { + as1n.push_back(freqVector1[idx]); + } + + // Now sort them by frequency in keywords, in descending order - this will speed up the tokenizer a little + + std::sort(as1n.begin(), as1n.end(), [&freqMapKeywords](char e1, char e2) { return freqMapKeywords[e2] > freqMapKeywords[e1]; }); + + // Extract characters to be encoded as 3 nibbles, which actually exist in keywords + + std::vector freqVector2; + + tk__packed_as_3n = 0; + for (uint8_t idx = 14; idx < freqVector1.size(); idx++) + { + const auto &character = freqVector1[idx]; + + if (freqMapKeywords[character] > 0) + { + as3n.push_back(character); + tk__packed_as_3n++; + } + else + { + freqVector2.push_back(character); + } + } + tk__packed_as_3n = std::max(tk__packed_as_3n, (uint8_t) 1); + + // Again, sort them by frequency in keywords, in descending order - this will speed up the tokenizer a little + + std::sort(as3n.begin(), as3n.end(), [&freqMapKeywords](char e1, char e2) { return freqMapKeywords[e2] > freqMapKeywords[e1]; }); + + // Finally extract the remaining characters to be encoded as 3 nibbles + + for (const auto &character : freqVector2) + { + as3n.push_back(character); + } +} + +void DataSet::encodeByFreq(const std::string &plain, StringEncoded &encoded) const +{ + bool fullByte = true; + + auto push1n = [&](uint8_t val) // push 1 nibble - encoded character or 0xF mark + { + if (fullByte) + { + encoded.push_back(val); + fullByte = false; + } + else + { + encoded.back() += val * 0x10; + fullByte = true; + } + }; + + auto push2n = [&](uint8_t val) // push the remaining nibbles for 3-niobble encoded characters + { + if (fullByte) + { + encoded.push_back(val); + } + else + { + // Encode byte in a way to be easily decoded by 6502 + encoded.back() += (val / 0x10) * 0x10; + encoded.push_back(val % 0x10); + } + }; + + // Encode every single character by frequency, put them in the output vector + + for (const char &character : plain) + { + auto iterEncoding1 = std::find(as1n.begin(), as1n.end(), character); + if (iterEncoding1 != as1n.end()) + { + push1n(std::distance(as1n.begin(), iterEncoding1) + 1); + } + else + { + push1n(0x0F); + + auto iterEncoding3 = std::find(as3n.begin(), as3n.end(), character); + if (iterEncoding3 == as3n.end()) + { + ERROR("internal error in 'encodeByFreq'"); + } + push2n(std::distance(as3n.begin(), iterEncoding3) + 1); + } + } + + // Make sure the last byte of encoded stream is 0 + + if (encoded.size() == 0 || encoded.back() != 0) encoded.push_back(0); +} + +void DataSet::encodeStringsFreq() +{ + // Encode every relevant string from every list - by character frequency + + for (uint8_t idx = 0; idx < stringEntryLists.size(); idx++) + { + const auto &stringEntryList = stringEntryLists[idx]; + auto &stringEncodedList = stringEncodedLists[idx]; + + // Skip lists encoded by the dictionary + if (isCompressionLvl2(stringEntryList)) continue; + + // Perform frequency encoding of the list + + for (const auto &stringEntry : stringEntryList.list) + { + stringEncodedList.emplace_back(); + auto &stringEncoded = stringEncodedList.back(); + + if (isRelevant(stringEntry)) + { + encodeByFreq(stringEntry.string, stringEncoded); + } + } + } +} + +void DataSet::putCharEncoding(std::ostringstream &stream, uint8_t idx, char character, bool is3n) +{ + stream << "\t!byte $" << std::uppercase << std::hex << + std::setfill('0') << std::setw(2) << +character << + " ; " << std::setfill(is3n ? '0' : ' ') << std::setw(2) << +idx; + + std::string petscii; + + if (character == 0x20) + { + petscii = " = SPACE"; + } + else if (character == 0x27) + { + petscii = " = APOSTROPHE"; + } + else if (character == 0x0D) + { + petscii = " = RETURN"; + } + else if (character > ' ' && character < 'a') + { + petscii = std::string(" = '") + character + "'"; + } + + stream << petscii << std::endl; +} + +void DataSet::prepareOutput_1n_3n(std::ostringstream &stream) +{ + uint8_t idx; + + // Export all nibble-encoded characters + + stream << std::endl << "!macro PUT_PACKED_AS_1N { ; characters encoded as 1 nibble" << std::endl << std::endl; + + idx = 0; + for (const auto& encoding : as1n) + { + putCharEncoding(stream, ++idx, encoding, false); + } + + stream << "}" << std::endl; + + // Export all byte-encoded characters + + stream << std::endl << "!macro PUT_PACKED_AS_3N { ; characters encoded as 3 nibbles" << std::endl << std::endl; + + idx = 0; + for (const auto& encoding : as3n) + { + if (idx == tk__packed_as_3n) stream << std::endl << + "\t; Characters below are not used by any BASIC keyword" << std::endl << std::endl; + + putCharEncoding(stream, ++idx, encoding, true); + } + + stream << "}" << std::endl; +} + +void DataSet::prepareOutput_labels(std::ostringstream &stream, + const StringEntryList &stringEntryList, + const StringEncodedList &stringEncodedList) +{ + // For the dictionary we do not have any labels + if (stringEntryList.type == ListType::DICTIONARY) return; + + stream << std::endl; + for (uint8_t idx = 0; idx < stringEncodedList.size(); idx++) + { + const auto &stringEntry = stringEntryList.list[idx]; + const auto &stringEncoded = stringEncodedList[idx]; + + if (!stringEncoded.empty()) + { + stream << "!set IDX__" << stringEntry.alias << + std::string(maxAliasLen - stringEntry.alias.length(), ' ') << " = $" << + std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +idx << std::endl; + } + } +} + +void DataSet::prepareOutput_packed(std::ostringstream &stream, + const StringEntryList &stringEntryList, + const StringEncodedList &stringEncodedList) +{ + if (isCompressionLvl2(stringEntryList)) + { + stream << std::endl << "!macro PUT_PACKED_DICT_"; + } + else + { + stream << std::endl << "!macro PUT_PACKED_FREQ_"; + } + + stream << stringEntryList.name << " {" << std::endl << std::endl; + + enum LastStr { NONE, SKIPPED, WRITTEN } lastStr = LastStr::NONE; + for (uint8_t idxString = 0; idxString < stringEncodedList.size(); idxString++) + { + const auto &stringEncoded = stringEncodedList[idxString]; + + if (stringEncoded.empty()) + { + if (stringEntryList.type == ListType::DICTIONARY) ERROR("internal error"); // should never happen + + if (lastStr == LastStr::WRITTEN) stream << std::endl; + stream << "\t!byte $00 ; skipped " << stringEntryList.list[idxString].alias << std::endl; + lastStr = LastStr::SKIPPED; + } + else + { + if (lastStr != LastStr::NONE) stream << std::endl; + + // Output the label - as a comment + + if (stringEntryList.type != ListType::DICTIONARY) + { + stream << "\t; IDX__" << stringEntryList.list[idxString].alias << std::endl; + } + + // Output the source string - as a comment + + stream << "\t; '"; + for (auto &character : stringEntryList.list[idxString].string) + { + if (character >= 32 && character <= 132 && character != 39 && character != 34) + { + stream << character; + } + else if (character == 13) + { + stream << ""; + } + else + { + stream << "_"; + } + } + stream << "'" << std::endl; + + // Output the encoding + + stream << "\t!byte "; + + bool first = true; + for (const auto &charEncoded : stringEncoded) + { + if (first) + { + first = false; + } + else + { + stream << ", "; + } + + stream << "$" << std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +charEncoded; + } + + stream << std::endl; + lastStr = LastStr::WRITTEN; + } + } + + // For the token list - put the 'end of keywords' mark + + if (stringEntryList.type == ListType::KEYWORDS) + { + stream << std::endl << "\t; Marker - end of the keyword list" << std::endl; + stream << "\t!byte $FF, $FF" << std::endl; + } + + stream << "}" << std::endl; +} + +void DataSet::prepareOutput() +{ + // Convert our encoded strings to a KickAssembler source + + std::ostringstream stream; + + // Export 1-nibble and 3-nibble encoding data + + prepareOutput_1n_3n(stream); + + // Export additional data for the tokenizer + + stream << std::endl << "!set TK__PACKED_AS_3N = $" << std::hex << +tk__packed_as_3n << + std::endl << "!set TK__MAX_KEYWORD_LEN = " << std::dec << +tk__max_keyword_len << std::endl; + + // Export encoded strings + + for (uint8_t idx = 0; idx < stringEntryLists.size(); idx++) + { + const auto &stringEntryList = stringEntryLists[idx]; + const auto &stringEncodedList = stringEncodedLists[idx]; + + if (stringEncodedList.empty()) continue; + + // Export labels for the current list + + prepareOutput_labels(stream, stringEntryList, stringEncodedList); + + // For the token list - put the number of tokens available + + if (stringEntryList.type == ListType::KEYWORDS) + { + stream << std::endl << "!set TK__MAXTOKEN_" << stringEntryList.name << " = " << + std::dec << stringEncodedList.size() << std::endl; + } + + // Export the packed data + + prepareOutput_packed(stream, stringEntryList, stringEncodedList); + } + + // Finalize the file stream + + stream << std::endl; + outFileContent = stream.str(); +} diff --git a/tools/generate_strings/dataset.h b/tools/generate_strings/dataset.h new file mode 100644 index 00000000..69c9d252 --- /dev/null +++ b/tools/generate_strings/dataset.h @@ -0,0 +1,90 @@ + +#ifndef DATASET_H +#define DATASET_H + +#include "global.h" + + +// Main class to encode strings based on character frequency +class DataSet +{ +public: + + void addStrings(const StringEntryList &stringList); + + const std::string &getOutput(); + +private: + + void process(); + + void generateConfigDepStrings(); + void validateLists(); + void calculateFrequencies(); + void encodeStringsDict(); + void encodeStringsFreq(); + + void encodeByFreq(const std::string &plain, StringEncoded &encoded) const; + + void prepareOutput(); + void prepareOutput_1n_3n(std::ostringstream &stream); + void prepareOutput_labels(std::ostringstream &stream, + const StringEntryList &stringEntryList, + const StringEncodedList &stringEncodedList); + void prepareOutput_packed(std::ostringstream &stream, + const StringEntryList &stringEntryList, + const StringEncodedList &stringEncodedList); + + void putCharEncoding(std::ostringstream &stream, uint8_t idx, char character, bool is3n); + + bool isCompressionLvl2(const StringEntryList &list) const; + + virtual bool isRelevant(const StringEntry &entry) const = 0; + virtual std::string layoutName() const = 0; + + std::vector stringEntryLists; + std::vector stringEncodedLists; + + std::vector as1n; // list of bytes to be encoded as 1 nibble + std::vector as3n; // list of bytes to be encoded as 3 nibbles + + uint8_t tk__packed_as_3n = 0; + uint8_t tk__max_keyword_len = 0; + + size_t maxAliasLen = 0; + std::string outFileContent; +}; + + +class DataSetSTD : public DataSet +{ + bool isRelevant(const StringEntry &entry) const { return entry.enabledSTD; } + std::string layoutName() const { return "STD"; } +}; + +class DataSetCRT : public DataSet +{ + bool isRelevant(const StringEntry &entry) const { return entry.enabledCRT; } + std::string layoutName() const { return "CRT"; } +}; + +class DataSetM65 : public DataSet +{ + bool isRelevant(const StringEntry &entry) const { return entry.enabledM65; } + std::string layoutName() const { return "M65"; } +}; + +class DataSetU64 : public DataSet +{ + bool isRelevant(const StringEntry &entry) const { return entry.enabledU64; } + std::string layoutName() const { return "STD"; } // XXX to be changed for 'U64' once the layout is finished +}; + +class DataSetX16 : public DataSet +{ + bool isRelevant(const StringEntry &entry) const { return entry.enabledX16; } + std::string layoutName() const { return "X16"; } +}; + + +#endif // DATASET_H diff --git a/tools/generate_strings/dictencoder.cc b/tools/generate_strings/dictencoder.cc new file mode 100644 index 00000000..130a9971 --- /dev/null +++ b/tools/generate_strings/dictencoder.cc @@ -0,0 +1,516 @@ + + +#include "dictencoder.h" + +#include +#include + + +void DictEncoder::addString(const std::string &inString, StringEncoded *outPtr) +{ + // Store the pointer to encoding + + encodings.push_back(outPtr); + + // Check if string is already present in the dictionary + + auto pos = std::find(dictionary.begin(), dictionary.end(), inString); + + // Create initial encoding + + if (pos != dictionary.end()) + { + encodings.back()->push_back(pos - dictionary.begin()); + } + else + { + if (dictionary.size() > 255) + { + ERROR("max 255 strings allowed for dictionary compression"); + } + + dictionary.push_back(inString); + encodings.back()->push_back(dictionary.size() - 1); + } +} + +void DictEncoder::extractWords(std::vector &candidateList) +{ + for (const auto &dictionaryEntry : dictionary) + { + // Split the dictionary entry into words + + std::istringstream entryStream(dictionaryEntry); + while (entryStream) + { + std::string word_1; + entryStream >> word_1; + + if (word_1.empty()) continue; + + // Create possible variants with spaces + + std::string word_2 = " " + word_1; + std::string word_3 = word_1 + " "; + std::string word_4 = word_2 + " "; + + // If necessary, add the word to candidate list + + auto addToList = [&dictionaryEntry, &candidateList](std::string &word) + { + // Before dding, make sure the word variant exists in the string, + // is not already on the list, and is long enough it makes sense to try + + if (word.size() < 1) return; + if (dictionaryEntry.find(word) == std::string::npos) return; + if (std::find(candidateList.begin(), candidateList.end(), word) != candidateList.end()) return; + + candidateList.push_back(word); + }; + + addToList(word_1); + addToList(word_2); + addToList(word_3); + addToList(word_4); + } + } +} + +int32_t DictEncoder::evaluateCandidate(std::string &candidate) +{ + // XXX it looks like this is not 100% right - debug the algorithm if dictionary compression is reintroduced + + // Build the score - find how much bytes can be spared by extracting this particular candidate + // Limited size of the dictionary is taken into consideration + + int32_t score = -(candidate.size() + 1); + uint32_t targetSize = dictionary.size() + 1; + + // First check for situation when the candidate equals the currently existing dictionary entry + + for (const auto &dictionaryEntry : dictionary) + { + if (dictionaryEntry == candidate) + { + // Candidate equals the string + + score += candidate.size() + 1; + targetSize--; + continue; + } + } + + // Now check the remaining cases + + for (const auto &dictionaryEntry : dictionary) + { + if (dictionaryEntry == candidate) continue; // already handled + + // Find all the occurences of the candidate in the current string + + std::vector occurences; + for (auto iter = dictionaryEntry.begin(); iter < (dictionaryEntry.end() - candidate.size() + 1); iter++) + { + if (candidate != std::string(iter, iter + candidate.size())) continue; + + occurences.push_back(iter - dictionaryEntry.begin()); + iter += candidate.size() - 1; // to make sure we have no occurence overlapping + } + + if (occurences.empty()) continue; + + // Now summarize what we are going to gain when doing replacement + + if (occurences[0] != 0) + { + // For the substring before the first occurence we need to create a separate entry + // in the dictionary - unless it is already there, it brings some additional cost + + std::string otherStr = std::string(dictionaryEntry.begin(), dictionaryEntry.end() + occurences[0]); + + if (std::find(dictionary.begin(), dictionary.end(), otherStr) != dictionary.end()) + { + // This extra string is already in the dictionary - that gives extra saving + score += otherStr.size() - 1; + } + else + { + // This extra string is not in the dictionary, it has to be added + score -= 2; + targetSize++; + } + } + + for (auto iter = occurences.begin(); iter < occurences.end(); iter++) + { + // With a replacement, we should gain some bytes + score += candidate.size() - 1; + + // If the occurence ends with the last byte of dictionary string - nothing more to do + if (*iter + candidate.size() == dictionaryEntry.size()) break; + + // If next occurence starts immediately after this one - next iteration + if (iter + 1 != occurences.end() && (*(iter + 1) - *iter) == (uint8_t) candidate.size()) continue; + + // For the substring between this occurence and the next one (or the end of the string) + // we need to create a separate entry in the dictionary - unless it is already there, + // it brings some additional cost + + std::string otherStr; + if (iter + 1 == occurences.end()) + { + otherStr = std::string(dictionaryEntry.begin() + *iter + candidate.size(), + dictionaryEntry.end()); + } + else + { + otherStr = std::string(dictionaryEntry.begin() + *iter + candidate.size(), + dictionaryEntry.begin() + *(iter + 1)); + } + + if (std::find(dictionary.begin(), dictionary.end(), otherStr) != dictionary.end()) + { + // This extra string is already in the dictionary - that gives extra saving + score += otherStr.size() - 1; + } + else + { + // This extra string is not in the dictionary, it has to be added + score -= 2; + targetSize++; + } + } + + // We crossed the maximum number of strings (leave one free for the work buffer), + // do not consider this candidate + + if (targetSize > 254) return -1; + } + + return score; +} + +void DictEncoder::cleanupDictionary() +{ + // Get rid of dictionary entries which are not needed anymore + + bool isClean = false; + + while (!isClean) + { + isClean = true; + + for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++) + { + if (!iter->empty()) continue; + + // Found an empty string in the dictionary - remove it and adapt encoding + + isClean = false; + uint8_t obsoleteStrIdx = iter - dictionary.begin(); + + dictionary.erase(iter); + + for (auto &encoding : encodings) for (auto &byte : *encoding) + { + if (byte >= obsoleteStrIdx) byte--; + } + + break; + } + } +} + +bool DictEncoder::optimizeSplit() +{ + // XXX - The code here is not the best one - redesign and algorithm improvements + // (like - consider optimizing as a game and apply alpha-beta scheme to select the best + // candidate, incorporate frequency encoding into candidate evaluation) could result in + // a slightly better compression. But currently we have very few strings, and the dictionary + // compression is disabled by default - so right now this is not worth the effort. + + if (dictionary.size() >= 255) return false; // up to 255 entries in the dictionary are possible + + // First select words, which can potentially be extracted to new substrings + + std::vector candidateList; + extractWords(candidateList); + + // XXX possible future improvement: find a couple of largest common substrings and add them as candidates too + + // Now find the best word to be extracted + + auto bestCandidate = candidateList.end(); + int32_t bestCandidateScore = 0; + + for (auto iter = candidateList.begin(); iter < candidateList.end(); iter++) + { + int32_t candidateScore = evaluateCandidate(*iter); + + if (candidateScore > bestCandidateScore) + { + bestCandidate = iter; + bestCandidateScore = candidateScore; + } + } + + // Only allow for replacements that brings some size benefit + + if (bestCandidateScore < 1) return false; + + // Extract the best candidate to a separate string + + // First add our selected candidate to the dictionary and replace all strings which are equal to it + + auto &selectedStr = *bestCandidate; + uint8_t selectedStrIdx = dictionary.size(); + + dictionary.push_back(selectedStr); + + for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++) + { + uint8_t currentStrIdx = iter - dictionary.begin(); + + if (*iter != selectedStr || selectedStrIdx == currentStrIdx) continue; + + // Replace the current string with the new one + + for (auto &encoding : encodings) for (auto &byte : *encoding) + { + if (byte == currentStrIdx) byte = selectedStrIdx; + } + + // Mark obsolete dictionary entry as free for removal + + iter->clear(); + } + + cleanupDictionary(); + + // Now handle normal cases, when the string needs to be split + // XXX - this is not time-effective, optimize this in the future + + bool optimizeAgain = true; + bool nexIteration = true; + while (nexIteration) + { + nexIteration = false; + + for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++) + { + auto ¤tStr = *iter; + uint8_t currentStrIdx = iter - dictionary.begin(); + auto selectedStrIter = std::find(dictionary.begin(), dictionary.end(), selectedStr); + selectedStrIdx = selectedStrIter - dictionary.begin(); + + if (currentStrIdx == selectedStrIdx) continue; + + // Check if the current string contains the selected string + + auto pos = currentStr.find(selectedStr); + if (pos == std::string::npos) continue; + + nexIteration = true; + optimizeAgain = true; + + // Prepare replacement sequence for 'selectedStrIdx' + + std::vector replacement; + + if (pos == 0) + { + // The current dictionary string starts with our selected substring + + replacement.push_back(selectedStrIdx); + currentStr = currentStr.substr(selectedStr.size(), currentStr.size() - selectedStr.size()); + + if (!currentStr.empty()) + { + // Check if the remaining string is present somewhere else in the dictionary; if so, reuse it + + uint8_t pos2 = currentStrIdx; + for (auto iter2 = dictionary.begin(); iter2 < dictionary.end(); iter2++) + { + if (iter2 == iter) continue; + if (currentStr == *iter2) + { + dictionary[pos2].clear(); + pos2 = iter2 - dictionary.begin(); + break; + } + } + + replacement.push_back(pos2); + } + } + else + { + // The current dictionary string does not start with our selected substring + + std::string newStr = currentStr.substr(0, pos); + currentStr = currentStr.substr(pos, currentStr.size() - pos); + + // Check if the newStr is present somewhere else in the dictionary; if so, reuse it + + bool found = false; + uint8_t pos2 = 0; + for (auto iter2 = dictionary.begin(); iter2 < dictionary.end(); iter2++) + { + if (*iter2 == newStr) + { + found = true; + pos2 = iter2 - dictionary.begin(); + + break; + } + } + + if (found) + { + replacement.push_back(pos2); + } + else + { + dictionary.push_back(newStr); + replacement.push_back(dictionary.size() - 1); + } + + replacement.push_back(currentStrIdx); + } + + // Now replace all the occurences of 'selectedStrIdx' with a 'replacement' vector + + for (auto &encoding : encodings) + { + for (uint8_t idx = 0; idx < encoding->size(); idx++) + { + if ((*encoding)[idx] != currentStrIdx) continue; + + // Perform replacement + + encoding->erase(encoding->begin() + idx); + + for (const auto replacementIdx : replacement) + { + encoding->insert(encoding->begin() + idx, replacementIdx); + idx++; + } + + idx--; + } + } + + cleanupDictionary(); + } + } + + return optimizeAgain; +} + +bool DictEncoder::optimizeJoin() +{ + // Try to optimize by joining two substrings + + // XXX - possible future improvement + + return false; +} + +void DictEncoder::optimizeOrder() +{ + // XXX - it would be better to do this after the dictionary is compressed + + // Try to optimize order of strings in the dictionary for fastest display + // - put the shorter substrings first + // - for strings with roughly the same length put more frequently used first + + typedef struct Entry + { + uint32_t penalty; + std::string word; + } Entry; + + // Create copy of the dictionary with optimization info + + std::vector optimizedOrder; + for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++) + { + const auto &dictionaryStr = *iter; + + uint8_t sizePenalty = dictionaryStr.size() / 3; + uint16_t freqPenalty = 65535; + + // Calculate penalty for low occurence frequency + + for (auto &encoding : encodings) + { + for (auto &byte : *encoding) + { + if (byte == (iter - dictionary.begin())) freqPenalty--; + } + } + + // Create new entry + + Entry newEntry; + newEntry.word = dictionaryStr; + newEntry.penalty = sizePenalty * 65536 + freqPenalty; + + optimizedOrder.push_back(newEntry); + } + + + // Determine ordering - sort from smallest penalty to largest + + std::sort(optimizedOrder.begin(), optimizedOrder.end(), + [](const Entry &e1, const Entry &e2) { return e1.penalty < e2.penalty; }); + + // Create helper table for reordering + + std::vector reorderTable; + + for (uint8_t idx1 = 0; idx1 < dictionary.size(); idx1++) + { + uint8_t idx2 = 0; + while (dictionary[idx1] != optimizedOrder[idx2].word) idx2++; + + reorderTable.push_back(idx2); + } + + // Perform the reordering + + for (auto &encoding : encodings) for (auto &byte : *encoding) + { + byte = reorderTable[byte]; + } + + for (uint8_t idx = 0; idx < dictionary.size(); idx++) dictionary[idx] = optimizedOrder[idx].word; +} + +void DictEncoder::process(StringEntryList &outDictionary) +{ + if (dictionary.empty()) return; + // Optimize as long as it brings any improvement + + while (optimizeSplit() || optimizeJoin()) ; + optimizeOrder(); + + // Export the dictionary to external format + + outDictionary.type = ListType::DICTIONARY; + outDictionary.name = "DICTIONARY"; + + for (const auto &dictionaryStr : dictionary) + { + StringEntry newEntry = { true, true, true, true, true, "", dictionaryStr }; + outDictionary.list.push_back(newEntry); + } + + // Adapt the encoding to external format (0 = end of string) + + for (auto &encoding : encodings) + { + for (auto &byte : *encoding) byte++; + encoding->push_back(0); + } +} diff --git a/tools/generate_strings/dictencoder.h b/tools/generate_strings/dictencoder.h new file mode 100644 index 00000000..f04b8fd3 --- /dev/null +++ b/tools/generate_strings/dictencoder.h @@ -0,0 +1,33 @@ + +#ifndef DICTENCODER_H +#define DICTENCODER_H + +#include "global.h" + + +// This class encapsulates the optional dictionary encoding of selected string lists +class DictEncoder +{ +public: + + void addString(const std::string &inString, StringEncoded *outPtr); + + void process(StringEntryList &outDictionary); + +private: + + bool optimizeSplit(); + bool optimizeJoin(); + void optimizeOrder(); + + void cleanupDictionary(); + + void extractWords(std::vector &candidateList); + int32_t evaluateCandidate(std::string &candidate); + + std::vector encodings; + std::vector dictionary; +}; + + +#endif // DICTENCODER_H diff --git a/tools/generate_strings/global.cc b/tools/generate_strings/global.cc new file mode 100644 index 00000000..346a8c14 --- /dev/null +++ b/tools/generate_strings/global.cc @@ -0,0 +1,432 @@ + +#include "global.h" + + +// +// Command line settings +// + +std::string CMD_outFile = "out.s"; +std::string CMD_cnfFile = ""; + +// +// Options from the configuration file +// + +std::map GLOBAL_ConfigOptions; + +// +// Open ROMs string definitions +// + +// http://www.classic-games.com/commodore64/cbmtoken.html +// https://www.c64-wiki.com/wiki/BASIC_token + +// BASIC keywords - V2 dialect + +/* + The URLs here provide examples of other BASIC dialects that include the specified commands + or keywords. + All except VERIFY are known with certainty to be part of other BASIC dialects, and thus + cannot be subject to copyright infringement when implemented on a C64. + + The VERIFY Command however appears in other programming languages, and as a single word, + is the obvious complement to LOAD and SAVE when considering the following storage functions: + + 1. Load a file from storage into memory. + 2. Save a file from memory to storage. + 3. Verify that a file has been correctly written from memory to storage. + + Note that verify is the key verb in the behaviour of this function. Therefore it is the + obvious choice for the name for such a function, taking into account the convention of + LOAD and SAVE. Indeed, that this is so is highlighted by the fact that it was used in the + C64 ROM. Therefore there is no exercise of creativity here, and a reasonable person would + likely select this name independently. Also, single words cannot be copyright. + + Further, the list is a list of facts, and thus not copyrightable. + + We sourced the list from a public source, https://www.c64-wiki.com/wiki/BASIC, which has + been public on the internet for a long time, as have many other similar sources. + That website offers the page under the GFDL license, and we can therefore take the list + under those license terms if we wish, should the need arise. + + Note that in providing these justifications above, we do not in any way acknowledge that + without these conditions that the BASIC keyword list could be copyright, but provide this + information as a further layer of defence against any claims that it could somehow + infringe on the copyrights of the C64/C65/C128 etc ROMs. +*/ + +const StringEntryList GLOBAL_Keywords_V2 = { ListType::KEYWORDS, "keywords_V2", +{ + // STD CRT M65 U64 X16 + { true, true, true, true, true, "KV2_80", "END", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_81", "FOR", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_82", "NEXT", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_83", "DATA", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_84", "INPUT#", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php + { true, true, true, true, true, "KV2_85", "INPUT", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_86", "DIM", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_87", "READ", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_88", "LET", 2 }, // https://en.wikipedia.org/wiki/Atari_BASIC + { true, true, true, true, true, "KV2_89", "GOTO", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_8A", "RUN", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_8B", "IF", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_8C", "RESTORE", 3 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_8D", "GOSUB", 3 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_8E", "RETURN", 3 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_8F", "REM", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_90", "STOP", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_91", "ON", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_92", "WAIT", 2 }, // http://www.picaxe.com/BASIC-Commands/Time-Delays/wait/ + { true, true, true, true, true, "KV2_93", "LOAD", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_94", "SAVE", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_95", "VERIFY", 2 }, // https://en.wikipedia.org/wiki/Sinclair_BASIC + { true, true, true, true, true, "KV2_96", "DEF", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_97", "POKE", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_98", "PRINT#", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php + { true, true, true, true, true, "KV2_99", "PRINT", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_9A", "CONT", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_9B", "LIST", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_9C", "CLR", 2 }, // Apple I Replica Creation: Back to the Garage, p125 + { true, true, true, true, true, "KV2_9D", "CMD", 2 }, // https://en.wikipedia.org/wiki/List_of_DOS_commands + { true, true, true, true, true, "KV2_9E", "SYS", 2 }, // https://www.lifewire.com/dos-commands-4070427 + { true, true, true, true, true, "KV2_9F", "OPEN", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php + // STD CRT M65 U64 X16 + { true, true, true, true, true, "KV2_A0", "CLOSE", 3 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php + { true, true, true, true, true, "KV2_A1", "GET", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_A2", "NEW", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_A3", "TAB(", 2 }, // http://www.antonis.de/qbebooks/gwbasman/tab.html + { true, true, true, true, true, "KV2_A4", "TO", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_A5", "FN", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_A6", "SPC(", 2 }, // http://www.antonis.de/qbebooks/gwbasman/spc.html + { true, true, true, true, true, "KV2_A7", "THEN", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_A8", "NOT", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_A9", "STEP", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_AA", "+", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_AB", "-", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_AC", "*", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_AD", "/", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_AE", "^", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_AF", "AND", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_B0", "OR", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_B1", ">", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_B2", "=", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_B3", "<", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_B4", "SGN", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_B5", "INT", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_B6", "ABS", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_B7", "USR", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_B8", "FRE", 2 }, // http://www.antonis.de/qbebooks/gwbasman/fre.html + { true, true, true, true, true, "KV2_B9", "POS", 0 }, // http://www.antonis.de/qbebooks/gwbasman/pos.html + { true, true, true, true, true, "KV2_BA", "SQR", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_BB", "RND", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_BC", "LOG", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_BD", "EXP", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_BE", "COS", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_BF", "SIN", 2 }, // https://www.landsnail.com/a2ref.htm + // STD CRT M65 U64 X16 + { true, true, true, true, true, "KV2_C0", "TAN", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_C1", "ATN", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_C2", "PEEK", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_C3", "LEN", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_C4", "STR$", 3 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_C5", "VAL", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_C6", "ASC", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_C7", "CHR$", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_C8", "LEFT$", 3 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_C9", "RIGHT$", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_CA", "MID$", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, true, true, "KV2_CB", "GO", 0 }, // https://en.wikipedia.org/wiki/Goto +} }; + +// extended BASIC keywords - list reserved for small BASIC commands, suitable for inclusion in all machines with extended ROM + +const StringEntryList GLOBAL_Keywords_01 = { ListType::KEYWORDS, "keywords_01", +{ + // STD CRT M65 U64 X16 + { true, true, true, true, true, "K01_01", "SLOW", }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant + { true, true, true, true, true, "K01_02", "FAST", }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant + { true, true, true, true, true, "K01_03", "OLD", }, // Not present in CBM BASIC dialects, but common to some extensions (*) + { false, true, true, true, false, "K01_04", "CLEAR", }, // Not present in CBM BASIC dialects, Open ROMs specific + { false, true, true, true, false, "K01_05", "DISPOSE", }, // Not present in CBM BASIC dialects, Open ROMs specific + { false, true, true, true, false, "K01_06", "MERGE", }, // Not present in CBM BASIC dialects, but common to some extensions (*) + { false, true, true, true, false, "K01_07", "BLOAD", }, // http://www.antonis.de/qbebooks/gwbasman/bload.html + { false, true, true, true, false, "K01_08", "BSAVE", }, // http://www.antonis.de/qbebooks/gwbasman/bsave.html + { false, true, true, true, false, "K01_09", "BVERIFY", }, + { false, true, true, true, false, "K01_0A", "COLD", }, // Not present in CBM BASIC dialects, but common to some extensions (*) + { false, true, true, true, false, "K01_0B", "MEM", }, // Not present in CBM BASIC dialects, Open ROMs specific + + // (*) see https://sourceforge.net/p/vice-emu/code/HEAD/tree/trunk/vice/src/petcat.c +} }; + +// extended BASIC keywords - list reserved for hardware dependent BASIC commands + +const StringEntryList GLOBAL_Keywords_04 = { ListType::KEYWORDS, "keywords_04", +{ + // STD CRT M65 U64 X16 + { false, false, true, false, false, "K04_01", "SYSINFO", }, // Not present in CBM BASIC dialects, Open ROMs specific + { false, false, true, false, false, "K04_02", "MONITOR", }, // Amos Professional Manual, Command Index + { false, false, true, false, false, "K04_03", "BOOTCPM", }, // Not present in CBM BASIC dialects, Open ROMs specific + { false, false, true, false, false, "K04_04", "JOYCRSR", }, // Not present in CBM BASIC dialects, Open ROMs specific + { false, false, true, false, false, "K04_05", "COLORSET", }, // Not present in CBM BASIC dialects, Open ROMs specific + +} }; + +// extended BASIC keywords - list reserved for functions + +const StringEntryList GLOBAL_Keywords_06 = { ListType::KEYWORDS, "keywords_06", +{ + // STD CRT M65 U64 X16 + { false, false, true, false, false, "K06_01", "TEST", }, // Not present in CBM BASIC dialects, Open ROMs specific + +} }; + + +// BASIC errors - all dialects + +/* + These error messages are generally so well known, that it seems silly to have to even point out that they are + so widely used and known, if not completely genericised. + The most of them are inherited from the MICROSOFT BASIC on which the C64's BASIC is based, so for those, there + cannot even be any claim to copyright over them in any exclusive way. + + However, copyright of such short phrases that express ideas in short form cannot be copyrighted: + https://fairuse.stanford.edu/2003/09/09/copyright_protection_for_short/ + */ + +const StringEntryList GLOBAL_Errors = { ListType::STRINGS_BASIC, "errors", +{ + // STD CRT M65 U64 X16 --- error strings compatible with CBM BASIC V2 + { true, true, true, true, true, "EV2_01", "TOO MANY FILES" }, + { true, true, true, true, true, "EV2_02", "FILE OPEN" }, + { true, true, true, true, true, "EV2_03", "FILE NOT OPEN" }, + { true, true, true, true, true, "EV2_04", "FILE NOT FOUND" }, + { true, true, true, true, true, "EV2_05", "DEVICE NOT PRESENT" }, + { true, true, true, true, true, "EV2_06", "NOT INPUT FILE" }, + { true, true, true, true, true, "EV2_07", "NOT OUTPUT FILE" }, + { true, true, true, true, true, "EV2_08", "MISSING FILENAME" }, + { true, true, true, true, true, "EV2_09", "ILLEGAL DEVICE NUMBER" }, + { true, true, true, true, true, "EV2_0A", "NEXT WITHOUT FOR" }, + { true, true, true, true, true, "EV2_0B", "SYNTAX" }, + { true, true, true, true, true, "EV2_0C", "RETURN WITHOUT GOSUB" }, + { true, true, true, true, true, "EV2_0D", "OUT OF DATA" }, + { true, true, true, true, true, "EV2_0E", "ILLEGAL QUANTITY" }, + { true, true, true, true, true, "EV2_0F", "OVERFLOW" }, + { true, true, true, true, true, "EV2_10", "OUT OF MEMORY" }, + { true, true, true, true, true, "EV2_11", "UNDEF\'D STATEMENT" }, + { true, true, true, true, true, "EV2_12", "BAD SUBSCRIPT" }, + { true, true, true, true, true, "EV2_13", "REDIM\'D ARRAY" }, + { true, true, true, true, true, "EV2_14", "DIVISION BY ZERO" }, + { true, true, true, true, true, "EV2_15", "ILLEGAL DIRECT" }, + { true, true, true, true, true, "EV2_16", "TYPE MISMATCH" }, + { true, true, true, true, true, "EV2_17", "STRING TOO LONG" }, + { true, true, true, true, true, "EV2_18", "FILE DATA" }, + { true, true, true, true, true, "EV2_19", "FORMULA TOO COMPLEX" }, + { true, true, true, true, true, "EV2_1A", "CAN\'T CONTINUE" }, + { true, true, true, true, true, "EV2_1B", "UNDEF\'D FUNCTION" }, + { true, true, true, true, true, "EV2_1C", "VERIFY" }, + { true, true, true, true, true, "EV2_1D", "LOAD" }, + { true, true, true, true, true, "EV2_1E", "BREAK" }, + // STD CRT M65 U64 X16 --- error strings compatible with CBM BASIC V7 + { false, false, false, false, false, "EV7_1F", "CAN'T RESUME" }, // not used for now + { false, false, false, false, false, "EV7_20", "LOOP NOT FOUND" }, // not used for now + { false, false, false, false, false, "EV7_21", "LOOP WITHOUT DO" }, // not used for now + { true, true, true, true, true, "EV7_22", "DIRECT MODE ONLY" }, + { false, false, false, false, false, "EV7_23", "NO GRAPHICS AREA" }, // not used for now + { false, false, false, false, false, "EV7_24", "BAD DISK" }, // not used for now + { false, false, false, false, false, "EV7_25", "BEND NOT FOUND" }, // not used for now + { true, true, true, true, true, "EV7_26", "LINE NUMBER TOO LARGE" }, + { false, false, false, false, false, "EV7_27", "UNRESOLVED REFERENCE" }, // not used for now + { true, true, true, true, true, "EV7_28", "NOT IMPLEMENTED" }, // this message actually differs from the CBM one + { false, false, false, false, false, "EV7_29", "FILE READ" }, // not used for now + // STD CRT M65 U64 X16 --- error strings specific to Open ROMs, not present in CBM BASIC dialects + { true, true, true, true, true, "EOR_2A", "MEMORY CORRUPT" }, + { false, false, true, false, false, "EOR_2B", "NATIVE MODE ONLY" }, + { false, false, true, false, false, "EOR_2C", "LEGACY MODE ONLY" }, +} }; + +// BASIC errors - miscelaneous strings + +const StringEntryList GLOBAL_MiscStrings = { ListType::STRINGS_BASIC, "misc", +{ + // STD CRT M65 U64 X16 --- misc strings as on CBM machines + { true, true, true, true, true, "STR_RET_QM", "\r?" }, + { true, true, true, true, true, "STR_BYTES", " BASIC BYTES FREE" }, // https://github.com/stefanhaustein/expressionparser + { true, true, true, true, true, "STR_READY", "\rREADY.\r" }, // https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconc_whatistsonative.htm https://github.com/stefanhaustein/expressionparser + { true, true, true, true, true, "STR_ERROR", " ERROR" }, // simply the word error that is attached to the other parts of messages https://fjkraan.home.xs4all.nl/comp/apple2faq/app2asoftfaq.html + { true, true, true, true, true, "STR_IN", " IN " }, + { false, true, true, false, false, "STR_IF_SURE", "\rARE YOU SURE? " }, // https://docs.microsoft.com/en-us/windows/win32/uxguide/mess-confirm + // STD CRT M65 U64 X16 --- misc strings specific to Open ROMs, not present in CBM ROMs + { true, true, true, true, true, "STR_BRK_AT", "\rBRK AT $" }, + + { false, false, true, false, false, "STR_ORS", "OPEN ROMS BASIC & KERNAL\r" }, + { false, false, true, false, false, "STR_ORS_LEGACY", "(LEGACY MODE) " }, + + { false, true, true, false, false, "STR_MEM_HDR", "\r\x12 AREA START SIZE \r" }, + { false, true, true, false, false, "STR_MEM_1", " $" }, + { false, true, true, false, false, "STR_MEM_2", " " }, + { false, true, true, false, false, "STR_MEM_TEXT", "\r TEXT" }, + { false, true, true, false, false, "STR_MEM_VARS", "\r VARS" }, + { false, true, true, false, false, "STR_MEM_ARRS", "\r ARRS" }, + { false, true, true, false, false, "STR_MEM_STRS", "\r STRS" }, + { false, true, true, false, false, "STR_MEM_FREE", "\r\r FREE" }, + + { false, false, true, false, false, "STR_DOS_FD", "\rFLOPPY : " }, + { false, false, true, false, false, "STR_DOS_SEPAR", " / " }, + { false, false, true, false, false, "STR_DOS_RD", "\rRAM DISK : " }, + + { false, false, true, false, false, "STR_SI_HEADER", "OPEN ROMS, " }, + { false, false, true, false, false, "STR_SI_MODE64", "LEGACY MODE" }, + { false, false, true, false, false, "STR_SI_MODE65", "NATIVE MODE" }, + { false, false, true, false, false, "STR_SI_HDR_REL", "\rRELEASE " }, + { false, false, true, false, false, "STR_SI_HDR_HW", "BOARD : " }, + { false, false, true, false, false, "STR_SI_HW_01", "MEGA65 R1" }, + { false, false, true, false, false, "STR_SI_HW_02", "MEGA65 R2" }, + { false, false, true, false, false, "STR_SI_HW_03", "MEGA65 R3" }, + { false, false, true, false, false, "STR_SI_HW_21", "MEGAPHONE R1" }, + { false, false, true, false, false, "STR_SI_HW_40", "NEXYS4 PSRAM" }, + { false, false, true, false, false, "STR_SI_HW_41", "NEXYS4 DDR" }, + { false, false, true, false, false, "STR_SI_HW_42", "NEXYS4 DDR WIDGET" }, + { false, false, true, false, false, "STR_SI_HW_FD", "WUKONG A100T" }, + { false, false, true, false, false, "STR_SI_HW_FE", "SIMULATION VHDL" }, + { false, false, true, false, false, "STR_SI_HW_XX", "UNKNOWN ID $" }, + { false, false, true, false, false, "STR_SI_HDR_VID", "\rVIDEO : " }, + + { false, false, true, false, false, "STR_RD_NAME", "RAMDISK.DNP " }, + { false, false, true, false, false, "STR_RD_OK", "LOADED OK\r" }, + { false, false, true, false, false, "STR_RD_ERR1", "SIZE WRONG\r" }, + { false, false, true, false, false, "STR_RD_ERR2", "TOO LARGE\r" }, + { false, false, true, false, false, "STR_RD_ERR3", "NOT FOUND\r" }, + + // Note: depending on configuration, additional strings will be added here +} }; + + +/* NOTE: below are the BASIC V7/V10 keywords, see: + - https://sites.google.com/site/h2obsession/CBM/basic/tokens + - http://www.zimmers.net/anonftp/pub/cbm/programming/cbm-basic-tokens.txt + + "RGR" // $CC + "RCLR" // $CD + + "JOY" // $CF // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "RDOT" // $D0 + "DEC" // $D1 // Amos Professional Manual, Command Index + "HEX$" // $D2 // http://www.antonis.de/qbebooks/gwbasman/hexs.html + "ERR$" // $D3 // Amos Professional Manual, Command Index + "INSTR" // $D4 // Amos Professional Manual, Command Index + "ELSE" // $D5 // Amos Professional Manual, Command Index + "RESUME" // $D6 // Amos Professional Manual, Command Index + "TRAP" // $D7 // Amos Professional Manual, Command Index + "TRON" // $D8 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "TROFF" // $D9 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "SOUND" // $DA // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "VOL" // $DB + "AUTO" // $DC // http://www.antonis.de/qbebooks/gwbasman/auto.html + "PUDEF" // $DD + "GRAPHIC" // $DE + "PAINT" // $DF // Amos Professional Manual, Command Index + "CHAR" // $E0 // https://www.c64-wiki.com/wiki/Screen_Graphics_64 + "BOX" // $E1 // Amos Professional Manual, Command Index + "CIRCLE" // $E2 // https://en.wikipedia.org/wiki/Sinclair_BASIC + "PASTE" // $E3 // Amos Professional Manual, Command Index + "CUT" // $E4 + "LINE" // $E5 // https://en.wikipedia.org/wiki/Sinclair_BASIC + "LOCATE" // $E6 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "COLOR" // $E7 + "SCNCLR" // $E8 + "SCALE" // $E9 // https://www.c64-wiki.com/wiki/Super_Expander_64 + "HELP" // $EA // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7 + "DO" // $EB // Amos Professional Manual, Command Index + "LOOP" // $EC // Amos Professional Manual, Command Index + "EXIT" // $ED // Amos Professional Manual, Command Index + "DIR" // $EE // Amos Professional Manual, Command Index + "DSAVE" // $EF // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "DLOAD" // $F0 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "HEADER" // $F1 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "SCRATCH" // $F2 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "COLLECT" // $F3 + "COPY" // $F4 // https://en.wikipedia.org/wiki/Sinclair_BASIC + "RENAME" // $F5 // Amos Professional Manual, Command Index + "BACKUP" // $F6 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "DELETE" // $F7 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "RENUMBER" // $F8 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "KEY" // $F9 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "MONITOR" // $FA // Amos Professional Manual, Command Index + "USING" // $FB // http://www.antonis.de/qbebooks/gwbasman/printusing.html + "UNTIL" // $FC // Amos Professional Manual, Command Index + "WHILE" // $FD // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + + "POT" // $CE $02 + "BUMP" // $CE $03 + "PEN" // $CE $04 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "RSPPOS" // $CE $05 // https://www.c64-wiki.com/wiki/Super_Expander_64 + "RSPRITE" // $CE $06 + "RSPCOLOR" // $CE $07 + "XOR" // $CE $08 + "RWINDOW" // $CE $09 + "POINTER" // $CE $0A + + "BANK" // $FE $02 // Amos Professional Manual, Command Index + "FILTER" // $FE $03 // https://www.c64-wiki.com/wiki/Super_Expander_64 + "PLAY" // $FE $04 // Amos Professional Manual, Command Index + "TEMPO" // $FE $05 // Amos Professional Manual, Command Index + "MOVSPR" // $FE $06 + "SPRITE" // $FE $07 // Amos Professional Manual, Command Index + "SPRCOLOR" // $FE $08 + "RREG" // $FE $09 + "ENVELOPE" // $FE $0A + "SLEEP" // $FE $0B + "CATALOG" // $FE $0C // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "DOPEN" // $FE $0D + "APPEND" // $FE $0E // Amos Professional Manual, Command Index + "DCLOSE" // $FE $0F + "BSAVE" // $FE $10 // http://www.antonis.de/qbebooks/gwbasman/bsave.html + "BLOAD" // $FE $11 // http://www.antonis.de/qbebooks/gwbasman/bload.html + "RECORD" // $FE $12 + "CONCAT" // $FE $13 + "DVERIFY" // $FE $14 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "DCLEAR" // $FE $15 + "SPRSAV" // $FE $16 // https://www.c64-wiki.com/wiki/Super_Expander_64 + "COLLISION" // $FE $17 + "BEGIN" // $FE $18 + "BEND" // $FE $19 + "WINDOW" // $FE $1A // http://www.antonis.de/qbebooks/gwbasman/window.html + "BOOT" // $FE $1B + "WIDTH" // $FE $1C // http://www.antonis.de/qbebooks/gwbasman/width.html + "SPRDEF" // $FE $1D // https://www.c64-wiki.com/wiki/Super_Expander_64 + "QUIT" // $FE $1E // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7 + + "STASH" // $FE $1F - v7 + "FETCH" // $FE $21 - v7 + "SWAP" // $FE $23 - v7 // http://www.antonis.de/qbebooks/gwbasman/swap.html + + "DMA" // $FE $17/$21/$23 - v10 + + "OFF" // $FE $24 // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7 + "FAST" // $FE $25 // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant + "SLOW" // $FE $26 // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant + "TYPE" // $FE $27 + "BVERIFY" // $FE $28 + "DIRECTORY" // $FE $29 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "ERASE" // $FE $2A // https://en.wikipedia.org/wiki/Sinclair_BASIC + "FIND" // $FE $2B // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "CHANGE" // $FE $2C + "SET" // $FE $2D + "SCREEN" // $FE $2E // http://www.antonis.de/qbebooks/gwbasman/screens.html + "POLYGON" // $FE $2F // Amos Professional Manual, Command Index + "ELLIPSE" // $FE $30 // Amos Professional Manual, Command Index + "VIEWPORT" // $FE $31 + "GCOPY" // $FE $32 + "PEN" // $FE $33 // Amos Professional Manual, Command Index + "PALETTE" // $FE $34 // Amos Professional Manual, Command Index + "DMODE" // $FE $35 + "DPAT" // $FE $36 + "PIC" // $FE $37 + "GENLOCK" // $FE $38 + "FOREGROUND" // $FE $39 + + "BACKGROUND" // $FE $3B + "BORDER" // $FE $3C // https://en.wikipedia.org/wiki/Sinclair_BASIC + "HIGHLIGHT" // $FE $3D +*/ diff --git a/tools/generate_strings/global.h b/tools/generate_strings/global.h new file mode 100644 index 00000000..a0168fa3 --- /dev/null +++ b/tools/generate_strings/global.h @@ -0,0 +1,69 @@ + +#ifndef GLOBAL_H +#define GLOBAL_H + +#include +#include + +#include "common.h" + + +// +// Command line settings +// + +extern std::string CMD_outFile; +extern std::string CMD_cnfFile; + +// +// Options from the configuration file +// + +extern std::map GLOBAL_ConfigOptions; + +// +// Type definition for strings/keywords to generate +// + +typedef struct StringEntry +{ + bool enabledSTD; // whether enabled for standard build + bool enabledCRT; // whether enabled for standard build with extra ROM cartridge + bool enabledM65; // whether enabled for Mega 65 build + bool enabledU64; // whether enabled for Ultimate 64 build + bool enabledX16; // whether enabled for Commander X16 build + std::string alias; // alias, for the assembler + std::string string; // string/token + uint8_t abbrevLen = 0; // length of token abbreviation +} StringEntry; + +enum class ListType +{ + KEYWORDS, + STRINGS_BASIC, + DICTIONARY // for internal use only +}; + +typedef struct StringEntryList +{ + ListType type; + std::string name; + std::vector list; +} StringEntryList; + +typedef std::vector StringEncoded; +typedef std::vector StringEncodedList; + +// +// Open ROMs string definitions +// + +extern const StringEntryList GLOBAL_Keywords_V2; +extern const StringEntryList GLOBAL_Keywords_01; +extern const StringEntryList GLOBAL_Keywords_04; +extern const StringEntryList GLOBAL_Keywords_06; +extern const StringEntryList GLOBAL_Errors; +extern const StringEntryList GLOBAL_MiscStrings; + + +#endif // GLOBAL_H diff --git a/tools/generate_strings/main.cc b/tools/generate_strings/main.cc new file mode 100644 index 00000000..619d88a2 --- /dev/null +++ b/tools/generate_strings/main.cc @@ -0,0 +1,227 @@ +// +// Utility to generate compressed messages and BASIC keywords +// + +#include "global.h" + +#include "dataset.h" + +#include +#include +#include + + +void printUsage() +{ + std::cout << "\n" << + "usage: generate_strings [-o ] [-c ]" << "\n\n"; +} + +void printBanner() +{ + printBannerLineTop(); + std::cout << "// Generating compressed messages and BASIC tokens\n"; + printBannerLineBottom(); +} + +void parseCommandLine(int argc, char **argv) +{ + int opt; + + // Retrieve command line options + + while ((opt = getopt(argc, argv, "o:c:")) != -1) + { + switch(opt) + { + case 'o': CMD_outFile = optarg; break; + case 'c': CMD_cnfFile = optarg; break; + default: printUsage(); ERROR(); + } + } +} + +void parseConfigFile() +{ + GLOBAL_ConfigOptions.clear(); + + // Open the configuration file + + std::ifstream cnfFile; + cnfFile.open(CMD_cnfFile); + if (!cnfFile.good()) ERROR("unable to open config file"); + + // Parse the file + + size_t lineNum = 0; + while (!cnfFile.eof()) + { + lineNum++; + std::string workStr; + + // Read a single line, remove leading spaces and tabs + + std::getline(cnfFile, workStr); + if (cnfFile.bad()) ERROR("error reading configuration file"); + workStr = std::regex_replace(workStr, std::regex("^[ \t]+"), ""); + + // Split the line into tokens + + std::vector tokens; + + std::istringstream workStream(workStr); + std::string token; + while(std::getline(workStream, token, ' ')) + { + if (token.empty()) continue; + tokens.push_back(token); + } + if (tokens.empty()) continue; + + // Only accept lines which are boolean config options set to YES + + if (tokens.size() < 4 || + tokens[0].compare(";;") != 0 || + tokens[1].compare("#CONFIG#") != 0 || + tokens[3].compare("YES") != 0) + { + continue; + } + + // Add definitin to config option map + + if (tokens[2].empty()) ERROR(std::string("error parsing config file - line ") + std::to_string(lineNum)); + + GLOBAL_ConfigOptions[tokens[2]] = true; + } + + cnfFile.close(); +} + +void writeStrings() +{ + std::string outputString; + + if (GLOBAL_ConfigOptions["PLATFORM_COMMANDER_X16"]) + { + DataSetX16 dataSetX16; + + // Add input data to computation objects + + dataSetX16.addStrings(GLOBAL_Keywords_V2); + dataSetX16.addStrings(GLOBAL_Keywords_01); + dataSetX16.addStrings(GLOBAL_Errors); + dataSetX16.addStrings(GLOBAL_MiscStrings); + + // Retrieve the results + + outputString = dataSetX16.getOutput(); + } + else if (GLOBAL_ConfigOptions["PLATFORM_COMMODORE_64"] && GLOBAL_ConfigOptions["MB_M65"]) + { + DataSetM65 dataSetM65; + + // Add input data to computation objects + + dataSetM65.addStrings(GLOBAL_Keywords_V2); + dataSetM65.addStrings(GLOBAL_Keywords_01); + dataSetM65.addStrings(GLOBAL_Keywords_04); + dataSetM65.addStrings(GLOBAL_Keywords_06); + dataSetM65.addStrings(GLOBAL_Errors); + dataSetM65.addStrings(GLOBAL_MiscStrings); + + // Retrieve the results + + outputString = dataSetM65.getOutput(); + } + else if (GLOBAL_ConfigOptions["PLATFORM_COMMODORE_64"] && GLOBAL_ConfigOptions["ROM_CRT"]) + { + DataSetCRT dataSetCRT; + + // Add input data to computation objects + + dataSetCRT.addStrings(GLOBAL_Keywords_V2); + dataSetCRT.addStrings(GLOBAL_Keywords_01); + dataSetCRT.addStrings(GLOBAL_Errors); + dataSetCRT.addStrings(GLOBAL_MiscStrings); + + // Retrieve the results + + outputString = dataSetCRT.getOutput(); + } + else if (GLOBAL_ConfigOptions["PLATFORM_COMMODORE_64"] && GLOBAL_ConfigOptions["MB_U64"]) + { + DataSetU64 dataSetU64; + + // Add input data to computation objects + + dataSetU64.addStrings(GLOBAL_Keywords_V2); + dataSetU64.addStrings(GLOBAL_Keywords_01); + dataSetU64.addStrings(GLOBAL_Errors); + dataSetU64.addStrings(GLOBAL_MiscStrings); + + // Retrieve the results + + outputString = dataSetU64.getOutput(); + } + else if (GLOBAL_ConfigOptions["PLATFORM_COMMODORE_64"]) + { + DataSetSTD dataSetSTD; + + // Add input data to computation objects + + dataSetSTD.addStrings(GLOBAL_Keywords_V2); + dataSetSTD.addStrings(GLOBAL_Keywords_01); + dataSetSTD.addStrings(GLOBAL_Errors); + dataSetSTD.addStrings(GLOBAL_MiscStrings); + + // Retrieve the results + + outputString = dataSetSTD.getOutput(); + } + else + { + ERROR("unable to determine string set"); + } + + // Remove old file + + unlink(CMD_outFile.c_str()); + + // Open output file for writing + + std::ofstream outFile(CMD_outFile, std::fstream::out | std::fstream::trunc); + if (!outFile.good()) ERROR(std::string("can't open oputput file '") + CMD_outFile + "'"); + + // Write header + + outFile << ";\n; Generated file - do not edit\n;"; + + // Write packed strings + + outFile << std::endl << std::endl; + outFile << outputString; + outFile << std::endl << std::endl; + + // Close the file + + outFile.close(); + + std::cout << std::string("compressed strings written to: '") + CMD_outFile + "'\n\n"; +} + +// +// Main function +// + +int main(int argc, char **argv) +{ + printBanner(); + parseCommandLine(argc, argv); + + parseConfigFile(); + + writeStrings(); + + return 0; +}