1594 lines
55 KiB
C++
1594 lines
55 KiB
C++
//
|
|
// Utility to generate compressed messages and BASIC tokens
|
|
//
|
|
|
|
#include "common.h"
|
|
|
|
#include <unistd.h>
|
|
|
|
#include <algorithm>
|
|
#include <fstream>
|
|
#include <iomanip>
|
|
#include <regex>
|
|
#include <sstream>
|
|
#include <map>
|
|
#include <vector>
|
|
|
|
//
|
|
// Command line settings
|
|
//
|
|
|
|
std::string CMD_outFile = "out.s";
|
|
std::string CMD_cnfFile = "";
|
|
|
|
//
|
|
// Type definition for text/tokens to generate
|
|
//
|
|
|
|
typedef struct StringEntry
|
|
{
|
|
bool enabledSTD; // whether enabled for basic build
|
|
bool enabledM65; // whether enabled for Mega65 build
|
|
bool enabledX16; // whether enabled for Commander X16 build
|
|
std::string alias; // alias, for the assembler
|
|
std::string string; // string/token
|
|
uint8_t abbrevLen = 0; // length of token abbreviation
|
|
} StringEntry;
|
|
|
|
enum class ListType
|
|
{
|
|
KEYWORDS,
|
|
STRINGS_BASIC,
|
|
DICTIONARY // for internal use only
|
|
};
|
|
|
|
typedef struct StringEntryList
|
|
{
|
|
ListType type;
|
|
std::string name;
|
|
std::vector<StringEntry> list;
|
|
} StringEntryList;
|
|
|
|
typedef std::vector<uint8_t> StringEncoded;
|
|
typedef std::vector<StringEncoded> StringEncodedList;
|
|
|
|
// http://www.classic-games.com/commodore64/cbmtoken.html
|
|
// https://www.c64-wiki.com/wiki/BASIC_token
|
|
|
|
// BASIC keywords - V2 dialect
|
|
|
|
/*
|
|
The URLs here provide examples of other BASIC dialects that include the specified commands
|
|
or keywords.
|
|
All except VERIFY are known with certainty to be part of other BASIC dialects, and thus
|
|
cannot be subject to copyright infringement when implemented on a C64.
|
|
|
|
The VERIFY Command however appears in other programming languages, and as a single word,
|
|
is the obvious complement to LOAD and SAVE when considering the following storage functions:
|
|
|
|
1. Load a file from storage into memory.
|
|
2. Save a file from memory to storage.
|
|
3. Verify that a file has been correctly written from memory to storage.
|
|
|
|
Note that verify is the key verb in the behaviour of this function. Therefore it is the
|
|
obvious choice for the name for such a function, taking into account the convention of
|
|
LOAD and SAVE. Indeed, that this is so is highlighted by the fact that it was used in the
|
|
C64 ROM. Therefore there is no exercise of creativity here, and a reasonable person would
|
|
likely select this name independently. Also, single words cannot be copyright.
|
|
|
|
Further, the list is a list of facts, and thus not copyrightable.
|
|
|
|
We sourced the list from a public source, https://www.c64-wiki.com/wiki/BASIC, which has
|
|
been public on the internet for a long time, as have many other similar sources.
|
|
That website offers the page under the GFDL license, and we can therefore take the list
|
|
under those license terms if we wish, should the need arise.
|
|
|
|
Note that in providing these justifications above, we do not in any way acknowledge that
|
|
without these conditions that the BASIC keyword list could be copyright, but provide this
|
|
information as a further layer of defence against any claims that it could somehow
|
|
infringe on the copyrights of the C64/C65/C128 etc ROMs.
|
|
*/
|
|
|
|
const StringEntryList GLOBAL_Keywords_V2 = { ListType::KEYWORDS, "keywords_V2",
|
|
{
|
|
// STD M65 X16
|
|
{ true, true, true, "KV2_80", "END", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_81", "FOR", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_82", "NEXT", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_83", "DATA", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_84", "INPUT#", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php
|
|
{ true, true, true, "KV2_85", "INPUT", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_86", "DIM", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_87", "READ", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_88", "LET", 2 }, // https://en.wikipedia.org/wiki/Atari_BASIC
|
|
{ true, true, true, "KV2_89", "GOTO", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_8A", "RUN", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_8B", "IF", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_8C", "RESTORE", 3 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_8D", "GOSUB", 3 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_8E", "RETURN", 3 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_8F", "REM", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_90", "STOP", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_91", "ON", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_92", "WAIT", 2 }, // http://www.picaxe.com/BASIC-Commands/Time-Delays/wait/
|
|
{ true, true, true, "KV2_93", "LOAD", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_94", "SAVE", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_95", "VERIFY", 2 }, // https://en.wikipedia.org/wiki/Sinclair_BASIC
|
|
{ true, true, true, "KV2_96", "DEF", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_97", "POKE", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_98", "PRINT#", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php
|
|
{ true, true, true, "KV2_99", "PRINT", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_9A", "CONT", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_9B", "LIST", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_9C", "CLR", 2 }, // Apple I Replica Creation: Back to the Garage, p125
|
|
{ true, true, true, "KV2_9D", "CMD", 2 }, // https://en.wikipedia.org/wiki/List_of_DOS_commands
|
|
{ true, true, true, "KV2_9E", "SYS", 2 }, // https://www.lifewire.com/dos-commands-4070427
|
|
{ true, true, true, "KV2_9F", "OPEN", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php
|
|
// STD M65 X16
|
|
{ true, true, true, "KV2_A0", "CLOSE", 3 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php
|
|
{ true, true, true, "KV2_A1", "GET", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_A2", "NEW", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_A3", "TAB(", 2 }, // http://www.antonis.de/qbebooks/gwbasman/tab.html
|
|
{ true, true, true, "KV2_A4", "TO", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_A5", "FN", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_A6", "SPC(", 2 }, // http://www.antonis.de/qbebooks/gwbasman/spc.html
|
|
{ true, true, true, "KV2_A7", "THEN", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_A8", "NOT", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_A9", "STEP", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_AA", "+", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_AB", "-", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_AC", "*", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_AD", "/", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_AE", "^", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_AF", "AND", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_B0", "OR", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_B1", ">", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_B2", "=", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_B3", "<", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_B4", "SGN", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_B5", "INT", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_B6", "ABS", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_B7", "USR", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_B8", "FRE", 2 }, // http://www.antonis.de/qbebooks/gwbasman/fre.html
|
|
{ true, true, true, "KV2_B9", "POS", 0 }, // http://www.antonis.de/qbebooks/gwbasman/pos.html
|
|
{ true, true, true, "KV2_BA", "SQR", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_BB", "RND", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_BC", "LOG", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_BD", "EXP", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_BE", "COS", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_BF", "SIN", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
// STD M65 X16
|
|
{ true, true, true, "KV2_C0", "TAN", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_C1", "ATN", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_C2", "PEEK", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_C3", "LEN", 0 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_C4", "STR$", 3 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_C5", "VAL", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_C6", "ASC", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_C7", "CHR$", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_C8", "LEFT$", 3 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_C9", "RIGHT$", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_CA", "MID$", 2 }, // https://www.landsnail.com/a2ref.htm
|
|
{ true, true, true, "KV2_CB", "GO", 0 }, // https://en.wikipedia.org/wiki/Goto
|
|
} };
|
|
|
|
// extended BASIC keywords - reserved for generic (hardware independent) BASIC commands
|
|
|
|
const StringEntryList GLOBAL_Keywords_CC = { ListType::KEYWORDS, "keywords_CC",
|
|
{
|
|
// STD M65 X16
|
|
{ true, true, true, "KCC_01", "MERGE", }, // OpenROMs specific, not present in CBM BASIC dialects
|
|
{ true, true, true, "KCC_02", "BLOAD", }, // http://www.antonis.de/qbebooks/gwbasman/bload.html
|
|
{ true, true, true, "KCC_03", "BSAVE", }, // http://www.antonis.de/qbebooks/gwbasman/bsave.html
|
|
{ true, true, true, "KCC_04", "BVERIFY", },
|
|
} };
|
|
|
|
// extended BASIC keywords - reserved for BASIC commands likely making sense on most C64-compatible machines
|
|
|
|
const StringEntryList GLOBAL_Keywords_CD = { ListType::KEYWORDS, "keywords_CD",
|
|
{
|
|
// STD M65 X16
|
|
{ true, true, true, "KCD_01", "SLOW", }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant
|
|
{ true, true, true, "KCD_02", "FAST", }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant
|
|
} };
|
|
|
|
// extended BASIC keywords - reserved for board-specific BASIC commands
|
|
|
|
const StringEntryList GLOBAL_Keywords_CE = { ListType::KEYWORDS, "keywords_CE",
|
|
{
|
|
// STD M65 X16
|
|
{ false, false, false, "KCE_01", "TESTCMD", },
|
|
} };
|
|
|
|
const StringEntryList GLOBAL_Keywords_CF = { ListType::KEYWORDS, "keywords_CF",
|
|
{
|
|
// STD M65 X16
|
|
{ false, false, false, "KCF_01", "TESTCMD", },
|
|
} };
|
|
|
|
// extended BASIC keywords - reserved for generic (hardware independent) BASIC functions
|
|
|
|
const StringEntryList GLOBAL_Keywords_D0 = { ListType::KEYWORDS, "keywords_D0",
|
|
{
|
|
// STD M65 X16
|
|
{ false, false, false, "KD0_01", "TESTFUN", },
|
|
} };
|
|
|
|
// extended BASIC keywords - reserved for BASIC functions likely making sense on most C64-compatible machines
|
|
|
|
const StringEntryList GLOBAL_Keywords_D1 = { ListType::KEYWORDS, "keywords_D1",
|
|
{
|
|
// STD M65 X16
|
|
{ false, false, false, "KD1_01", "TESTFUN", },
|
|
} };
|
|
|
|
// extended BASIC keywords - reserved for board-specific BASIC functions
|
|
|
|
const StringEntryList GLOBAL_Keywords_D2 = { ListType::KEYWORDS, "keywords_D2",
|
|
{
|
|
// STD M65 X16
|
|
{ false, false, false, "KD2_01", "TESTFUN", },
|
|
} };
|
|
|
|
const StringEntryList GLOBAL_Keywords_D3 = { ListType::KEYWORDS, "keywords_D3",
|
|
{
|
|
// STD M65 X16
|
|
{ false, false, false, "KD3_01", "TESTFUN", },
|
|
} };
|
|
|
|
|
|
// BASIC errors - all dialects
|
|
|
|
/*
|
|
These error messages are generally so well known, that it seems silly to have to even point out that they are
|
|
so widely used and known, if not completely genericised.
|
|
The most of them are inherited from the MICROSOFT BASIC on which the C64's BASIC is based, so for those, there
|
|
cannot even be any claim to copyright over them in any exclusive way.
|
|
|
|
However, copyright of such short phrases that express ideas in short form cannot be copyrighted:
|
|
https://fairuse.stanford.edu/2003/09/09/copyright_protection_for_short/
|
|
*/
|
|
|
|
const StringEntryList GLOBAL_Errors = { ListType::STRINGS_BASIC, "errors",
|
|
{
|
|
// STD M65 X16 --- error strings compatible with CBM BASIC V2
|
|
{ true, true, true, "EV2_01", "TOO MANY FILES" },
|
|
{ true, true, true, "EV2_02", "FILE OPEN" },
|
|
{ true, true, true, "EV2_03", "FILE NOT OPEN" },
|
|
{ true, true, true, "EV2_04", "FILE NOT FOUND" },
|
|
{ true, true, true, "EV2_05", "DEVICE NOT PRESENT" },
|
|
{ true, true, true, "EV2_06", "NOT INPUT FILE" },
|
|
{ true, true, true, "EV2_07", "NOT OUTPUT FILE" },
|
|
{ true, true, true, "EV2_08", "MISSING FILENAME" },
|
|
{ true, true, true, "EV2_09", "ILLEGAL DEVICE NUMBER" },
|
|
{ true, true, true, "EV2_0A", "NEXT WITHOUT FOR" },
|
|
{ true, true, true, "EV2_0B", "SYNTAX" },
|
|
{ true, true, true, "EV2_0C", "RETURN WITHOUT GOSUB" },
|
|
{ true, true, true, "EV2_0D", "OUT OF DATA" },
|
|
{ true, true, true, "EV2_0E", "ILLEGAL QUANTITY" },
|
|
{ true, true, true, "EV2_0F", "OVERFLOW" },
|
|
{ true, true, true, "EV2_10", "OUT OF MEMORY" },
|
|
{ true, true, true, "EV2_11", "UNDEF\'D STATEMENT" },
|
|
{ true, true, true, "EV2_12", "BAD SUBSCRIPT" },
|
|
{ true, true, true, "EV2_13", "REDIM\'D ARRAY" },
|
|
{ true, true, true, "EV2_14", "DIVISION BY ZERO" },
|
|
{ true, true, true, "EV2_15", "ILLEGAL DIRECT" },
|
|
{ true, true, true, "EV2_16", "TYPE MISMATCH" },
|
|
{ true, true, true, "EV2_17", "STRING TOO LONG" },
|
|
{ true, true, true, "EV2_18", "FILE DATA" },
|
|
{ true, true, true, "EV2_19", "FORMULA TOO COMPLEX" },
|
|
{ true, true, true, "EV2_1A", "CAN\'T CONTINUE" },
|
|
{ true, true, true, "EV2_1B", "UNDEF\'D FUNCTION" },
|
|
{ true, true, true, "EV2_1C", "VERIFY" },
|
|
{ true, true, true, "EV2_1D", "LOAD" },
|
|
{ true, true, true, "EV2_1E", "BREAK" },
|
|
// STD M65 X16 --- error strings compatible with CBM BASIC V7
|
|
{ false, false, false, "EV7_1F", "CAN'T RESUME" }, // not used for now
|
|
{ false, false, false, "EV7_20", "LOOP NOT FOUND" }, // not used for now
|
|
{ false, false, false, "EV7_21", "LOOP WITHOUT DO" }, // not used for now
|
|
{ true, true, true, "EV7_22", "DIRECT MODE ONLY" },
|
|
{ false, false, false, "EV7_23", "NO GRAPHICS AREA" }, // not used for now
|
|
{ false, false, false, "EV7_24", "BAD DISK" }, // not used for now
|
|
{ false, false, false, "EV7_25", "BEND NOT FOUND" }, // not used for now
|
|
{ false, false, false, "EV7_26", "LINE NUMBER TOO LARGE" }, // not used for now
|
|
{ false, false, false, "EV7_27", "UNRESOLVED REFERENCE" }, // not used for now
|
|
{ true, true, true, "EV7_28", "NOT IMPLEMENTED" }, // this message actually differs from the CBM one
|
|
{ false, false, false, "EV7_29", "FILE READ" }, // not used for now
|
|
// STD M65 X16 --- error strings specific to OpenROMs, not present in CBM BASIC dialects
|
|
{ true, true, true, "EOR_2A", "MEMORY CORRUPT" },
|
|
} };
|
|
|
|
// BASIC errors - miscelaneous strings
|
|
|
|
const StringEntryList GLOBAL_MiscStrings = { ListType::STRINGS_BASIC, "misc",
|
|
{
|
|
// STD M65 X16 --- misc strings as on CBM machines
|
|
{ true, true, true, "STR_RET_QM", "\r?" },
|
|
{ true, true, true, "STR_BYTES", " BASIC BYTES FREE" }, // https://github.com/stefanhaustein/expressionparser
|
|
{ true, true, true, "STR_READY", "\rREADY.\r" }, // https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconc_whatistsonative.htm https://github.com/stefanhaustein/expressionparser
|
|
{ true, true, true, "STR_ERROR", " ERROR" }, // simply the word error that is attached to the other parts of messages https://fjkraan.home.xs4all.nl/comp/apple2faq/app2asoftfaq.html
|
|
{ true, true, true, "STR_IN", " IN " },
|
|
// STD M65 X16 --- misc strings specific to OpenROMs, not present in CBM ROMs
|
|
{ true, true, true, "STR_BRK_AT", "\rBRK AT $" },
|
|
// Note: depending on configuration, additional strings will be added here
|
|
} };
|
|
|
|
|
|
/* NOTE: below are the BASIC V7/V10 keywords, see:
|
|
- https://sites.google.com/site/h2obsession/CBM/basic/tokens
|
|
- http://www.zimmers.net/anonftp/pub/cbm/programming/cbm-basic-tokens.txt
|
|
|
|
"RGR" // $CC
|
|
"RCLR" // $CD
|
|
|
|
"JOY" // $CF // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
|
|
"RDOT" // $D0
|
|
"DEC" // $D1 // Amos Professional Manual, Command Index
|
|
"HEX$" // $D2 // http://www.antonis.de/qbebooks/gwbasman/hexs.html
|
|
"ERR$" // $D3 // Amos Professional Manual, Command Index
|
|
"INSTR" // $D4 // Amos Professional Manual, Command Index
|
|
"ELSE" // $D5 // Amos Professional Manual, Command Index
|
|
"RESUME" // $D6 // Amos Professional Manual, Command Index
|
|
"TRAP" // $D7 // Amos Professional Manual, Command Index
|
|
"TRON" // $D8 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
|
|
"TROFF" // $D9 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
|
|
"SOUND" // $DA // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
|
|
"VOL" // $DB
|
|
"AUTO" // $DC // http://www.antonis.de/qbebooks/gwbasman/auto.html
|
|
"PUDEF" // $DD
|
|
"GRAPHIC" // $DE
|
|
"PAINT" // $DF // Amos Professional Manual, Command Index
|
|
"CHAR" // $E0 // https://www.c64-wiki.com/wiki/Screen_Graphics_64
|
|
"BOX" // $E1 // Amos Professional Manual, Command Index
|
|
"CIRCLE" // $E2 // https://en.wikipedia.org/wiki/Sinclair_BASIC
|
|
"PASTE" // $E3 // Amos Professional Manual, Command Index
|
|
"CUT" // $E4
|
|
"LINE" // $E5 // https://en.wikipedia.org/wiki/Sinclair_BASIC
|
|
"LOCATE" // $E6 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
|
|
"COLOR" // $E7
|
|
"SCNCLR" // $E8
|
|
"SCALE" // $E9 // https://www.c64-wiki.com/wiki/Super_Expander_64
|
|
"HELP" // $EA // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7
|
|
"DO" // $EB // Amos Professional Manual, Command Index
|
|
"LOOP" // $EC // Amos Professional Manual, Command Index
|
|
"EXIT" // $ED // Amos Professional Manual, Command Index
|
|
"DIR" // $EE // Amos Professional Manual, Command Index
|
|
"DSAVE" // $EF // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"DLOAD" // $F0 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"HEADER" // $F1 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"SCRATCH" // $F2 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"COLLECT" // $F3
|
|
"COPY" // $F4 // https://en.wikipedia.org/wiki/Sinclair_BASIC
|
|
"RENAME" // $F5 // Amos Professional Manual, Command Index
|
|
"BACKUP" // $F6 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"DELETE" // $F7 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"RENUMBER" // $F8 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"KEY" // $F9 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
|
|
"MONITOR" // $FA // Amos Professional Manual, Command Index
|
|
"USING" // $FB // http://www.antonis.de/qbebooks/gwbasman/printusing.html
|
|
"UNTIL" // $FC // Amos Professional Manual, Command Index
|
|
"WHILE" // $FD // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
|
|
|
|
"POT" // $CE $02
|
|
"BUMP" // $CE $03
|
|
"PEN" // $CE $04 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
|
|
"RSPPOS" // $CE $05 // https://www.c64-wiki.com/wiki/Super_Expander_64
|
|
"RSPRITE" // $CE $06
|
|
"RSPCOLOR" // $CE $07
|
|
"XOR" // $CE $08
|
|
"RWINDOW" // $CE $09
|
|
"POINTER" // $CE $0A
|
|
|
|
"BANK" // $FE $02 // Amos Professional Manual, Command Index
|
|
"FILTER" // $FE $03 // https://www.c64-wiki.com/wiki/Super_Expander_64
|
|
"PLAY" // $FE $04 // Amos Professional Manual, Command Index
|
|
"TEMPO" // $FE $05 // Amos Professional Manual, Command Index
|
|
"MOVSPR" // $FE $06
|
|
"SPRITE" // $FE $07 // Amos Professional Manual, Command Index
|
|
"SPRCOLOR" // $FE $08
|
|
"RREG" // $FE $09
|
|
"ENVELOPE" // $FE $0A
|
|
"SLEEP" // $FE $0B
|
|
"CATALOG" // $FE $0C // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"DOPEN" // $FE $0D
|
|
"APPEND" // $FE $0E // Amos Professional Manual, Command Index
|
|
"DCLOSE" // $FE $0F
|
|
"BSAVE" // $FE $10 // http://www.antonis.de/qbebooks/gwbasman/bsave.html
|
|
"BLOAD" // $FE $11 // http://www.antonis.de/qbebooks/gwbasman/bload.html
|
|
"RECORD" // $FE $12
|
|
"CONCAT" // $FE $13
|
|
"DVERIFY" // $FE $14 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"DCLEAR" // $FE $15
|
|
"SPRSAV" // $FE $16 // https://www.c64-wiki.com/wiki/Super_Expander_64
|
|
"COLLISION" // $FE $17
|
|
"BEGIN" // $FE $18
|
|
"BEND" // $FE $19
|
|
"WINDOW" // $FE $1A // http://www.antonis.de/qbebooks/gwbasman/window.html
|
|
"BOOT" // $FE $1B
|
|
"WIDTH" // $FE $1C // http://www.antonis.de/qbebooks/gwbasman/width.html
|
|
"SPRDEF" // $FE $1D // https://www.c64-wiki.com/wiki/Super_Expander_64
|
|
"QUIT" // $FE $1E // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7
|
|
|
|
"STASH" // $FE $1F - v7
|
|
"FETCH" // $FE $21 - v7
|
|
"SWAP" // $FE $23 - v7 // http://www.antonis.de/qbebooks/gwbasman/swap.html
|
|
|
|
"DMA" // $FE $17/$21/$23 - v10
|
|
|
|
"OFF" // $FE $24 // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7
|
|
"FAST" // $FE $25 // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant
|
|
"SLOW" // $FE $26 // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant
|
|
"TYPE" // $FE $27
|
|
"BVERIFY" // $FE $28
|
|
"DIRECTORY" // $FE $29 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"ERASE" // $FE $2A // https://en.wikipedia.org/wiki/Sinclair_BASIC
|
|
"FIND" // $FE $2B // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
|
|
"CHANGE" // $FE $2C
|
|
"SET" // $FE $2D
|
|
"SCREEN" // $FE $2E // http://www.antonis.de/qbebooks/gwbasman/screens.html
|
|
"POLYGON" // $FE $2F // Amos Professional Manual, Command Index
|
|
"ELLIPSE" // $FE $30 // Amos Professional Manual, Command Index
|
|
"VIEWPORT" // $FE $31
|
|
"GCOPY" // $FE $32
|
|
"PEN" // $FE $33 // Amos Professional Manual, Command Index
|
|
"PALETTE" // $FE $34 // Amos Professional Manual, Command Index
|
|
"DMODE" // $FE $35
|
|
"DPAT" // $FE $36
|
|
"PIC" // $FE $37
|
|
"GENLOCK" // $FE $38
|
|
"FOREGROUND" // $FE $39
|
|
|
|
"BACKGROUND" // $FE $3B
|
|
"BORDER" // $FE $3C // https://en.wikipedia.org/wiki/Sinclair_BASIC
|
|
"HIGHLIGHT" // $FE $3D
|
|
*/
|
|
|
|
std::map<std::string, bool> GLOBAL_ConfigOptions;
|
|
|
|
//
|
|
// Work class definitions
|
|
//
|
|
|
|
class DictEncoder
|
|
{
|
|
public:
|
|
|
|
void addString(const std::string &inString, StringEncoded *outPtr);
|
|
|
|
void process(StringEntryList &outDictionary);
|
|
|
|
private:
|
|
|
|
bool optimizeSplit();
|
|
bool optimizeJoin();
|
|
void optimizeOrder();
|
|
|
|
void extractWords(std::vector<std::string> &candidateList);
|
|
int32_t evaluateCandidate(std::string &candidate);
|
|
|
|
std::vector<StringEncoded *> encodings;
|
|
std::vector<std::string> dictionary;
|
|
};
|
|
|
|
class DataSet
|
|
{
|
|
public:
|
|
|
|
void addStrings(const StringEntryList &stringList);
|
|
|
|
const std::string &getOutput();
|
|
|
|
private:
|
|
|
|
void process();
|
|
|
|
void generateConfigDepStrings();
|
|
void validateLists();
|
|
void calculateFrequencies();
|
|
void encodeStringsDict();
|
|
void encodeStringsFreq();
|
|
|
|
void encodeByFreq(const std::string &plain, StringEncoded &encoded) const;
|
|
|
|
void prepareOutput();
|
|
void prepareOutput_1n_3n(std::ostringstream &stream);
|
|
void prepareOutput_labels(std::ostringstream &stream,
|
|
const StringEntryList &stringEntryList,
|
|
const StringEncodedList &stringEncodedList);
|
|
void prepareOutput_packed(std::ostringstream &stream,
|
|
const StringEntryList &stringEntryList,
|
|
const StringEncodedList &stringEncodedList);
|
|
|
|
void putCharEncoding(std::ostringstream &stream, uint8_t idx, char character, bool is3n);
|
|
|
|
bool isCompressionLvl2(const StringEntryList &list) const;
|
|
|
|
virtual bool isRelevant(const StringEntry &entry) const = 0;
|
|
virtual std::string layoutName() const = 0;
|
|
|
|
std::vector<StringEntryList> stringEntryLists;
|
|
std::vector<StringEncodedList> stringEncodedLists;
|
|
|
|
std::vector<char> as1n; // list of bytes to be encoded as 1 nibble
|
|
std::vector<char> as3n; // list of bytes to be encoded as 3 nibbles
|
|
|
|
uint8_t tk__packed_as_3n = 0;
|
|
uint8_t tk__max_keyword_len = 0;
|
|
|
|
size_t maxAliasLen = 0;
|
|
std::string outFileContent;
|
|
};
|
|
|
|
class DataSetSTD : public DataSet
|
|
{
|
|
bool isRelevant(const StringEntry &entry) const { return entry.enabledSTD; }
|
|
std::string layoutName() const { return "STD"; }
|
|
};
|
|
|
|
class DataSetM65 : public DataSet
|
|
{
|
|
bool isRelevant(const StringEntry &entry) const { return entry.enabledM65; }
|
|
std::string layoutName() const { return "M65"; }
|
|
};
|
|
|
|
class DataSetX16 : public DataSet
|
|
{
|
|
bool isRelevant(const StringEntry &entry) const { return entry.enabledX16; }
|
|
std::string layoutName() const { return "X16"; }
|
|
};
|
|
|
|
//
|
|
// Work class implementation
|
|
//
|
|
|
|
void DictEncoder::addString(const std::string &inString, StringEncoded *outPtr)
|
|
{
|
|
// Store the pointer to encoding
|
|
|
|
encodings.push_back(outPtr);
|
|
|
|
// Check if string is already present in the dictionary
|
|
|
|
auto pos = std::find(dictionary.begin(), dictionary.end(), inString);
|
|
|
|
// Create initial encoding
|
|
|
|
if (pos != dictionary.end())
|
|
{
|
|
encodings.back()->push_back(pos - dictionary.begin());
|
|
}
|
|
else
|
|
{
|
|
if (dictionary.size() > 255)
|
|
{
|
|
ERROR("max 255 strings for dictionary compression");
|
|
}
|
|
|
|
dictionary.push_back(inString);
|
|
encodings.back()->push_back(dictionary.size() - 1);
|
|
}
|
|
}
|
|
|
|
|
|
void DictEncoder::extractWords(std::vector<std::string> &candidateList)
|
|
{
|
|
for (const auto &dictionaryEntry : dictionary)
|
|
{
|
|
// Split the dictionary entry into words
|
|
|
|
std::istringstream entryStream(dictionaryEntry);
|
|
while (entryStream)
|
|
{
|
|
// XXX probably not the best way
|
|
std::string word_1;
|
|
entryStream >> word_1;
|
|
|
|
// Create possible variants with spaces
|
|
|
|
std::string word_2 = " " + word_1;
|
|
std::string word_3 = word_1 + " ";
|
|
std::string word_4 = word_2 + " ";
|
|
|
|
// If necessary, add the word to candidate list
|
|
|
|
auto addToList = [&dictionaryEntry, &candidateList](std::string &word)
|
|
{
|
|
// Before dding, make sure the word variant exists in the string,
|
|
// is not already on the list, and is long enough it makes sense to try
|
|
|
|
if (word.size() < 1) return;
|
|
if (dictionaryEntry.find(word) == std::string::npos) return;
|
|
if (std::find(candidateList.begin(), candidateList.end(), word) == candidateList.end()) return;
|
|
|
|
candidateList.push_back(word);
|
|
};
|
|
|
|
addToList(word_1);
|
|
addToList(word_2);
|
|
addToList(word_3);
|
|
addToList(word_4);
|
|
}
|
|
}
|
|
}
|
|
|
|
int32_t DictEncoder::evaluateCandidate(std::string &candidate)
|
|
{
|
|
// Build the score - find how much bytes can be spared by extracting this particular candidate
|
|
// Limited size of the dictionary is taken into consideration
|
|
|
|
int32_t score = -(candidate.size() + 1);
|
|
uint32_t targetSize = dictionary.size() + 1;
|
|
|
|
// First check for situation when the candidate equals the currently existing dictionary entry
|
|
|
|
for (const auto &dictionaryEntry : dictionary)
|
|
{
|
|
if (dictionaryEntry == candidate)
|
|
{
|
|
// Candidate equals the string
|
|
|
|
score += candidate.size() + 1;
|
|
targetSize--;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
// Now check the remaining cases
|
|
|
|
for (const auto &dictionaryEntry : dictionary)
|
|
{
|
|
if (dictionaryEntry == candidate) continue; // already handled
|
|
|
|
// Find all the occurences of the candidate in the current string
|
|
|
|
std::vector<uint8_t> occurences;
|
|
for (auto iter = dictionaryEntry.begin(); iter < (dictionaryEntry.end() - candidate.size() + 1); iter++)
|
|
{
|
|
if (candidate != std::string(iter, iter + candidate.size())) continue;
|
|
|
|
occurences.push_back(iter - dictionaryEntry.begin());
|
|
iter += candidate.size() - 1; // to make sure we have no occurence overlapping
|
|
}
|
|
|
|
if (occurences.empty()) continue;
|
|
|
|
// Now summarize what we are going to gain when doing replacement
|
|
|
|
if (occurences[0] != 0)
|
|
{
|
|
// For the substring before the first occurence we need to create a separate entry
|
|
// in the dictionary - unless it is already there, it brings some additional cost
|
|
|
|
// XXX, targetSize
|
|
}
|
|
|
|
for (auto iter = occurences.begin(); iter < occurences.end(); iter++)
|
|
{
|
|
// With a replacement, we should gain some bytes
|
|
score += candidate.size() - 1;
|
|
|
|
// If the occurence ends with the last byte of dictionary string - nothing more to do
|
|
if (*iter + candidate.size() == dictionaryEntry.size()) break;
|
|
|
|
// If next occurence starts immediately after this one - next iteration
|
|
if (iter + 1 != occurences.end() && (*(iter + 1) - *iter) == (uint8_t) candidate.size()) continue;
|
|
|
|
// For the substring between this occurence and the next one (or the end of the string)
|
|
// we need to create a separate entry in the dictionary - unless it is already there,
|
|
// it brings some additional cost
|
|
|
|
// XXX, targetSize
|
|
}
|
|
|
|
// We crossed the maximum number of strings, do not consider this candidate
|
|
// XXX improve the behaviour, maybe only some substrings can be replaced
|
|
|
|
if (targetSize > 255) return -1;
|
|
}
|
|
|
|
return score;
|
|
}
|
|
|
|
bool DictEncoder::optimizeSplit()
|
|
{
|
|
if (dictionary.size() >= 255) return false; // up to 255 entries in the dictionary are possible
|
|
|
|
// First select words, which can potentially be extracted to new substrings
|
|
|
|
std::vector<std::string> candidateList;
|
|
extractWords(candidateList);
|
|
|
|
// XXX find a couple of largest common substrings and add them as candidates too
|
|
|
|
// Now find the best word to be extracted
|
|
|
|
auto bestCandidate = candidateList.end();
|
|
int32_t bestCandidateScore = 0;
|
|
|
|
for (auto iter = candidateList.begin(); iter < candidateList.end(); iter++)
|
|
{
|
|
int32_t candidateScore = evaluateCandidate(*iter);
|
|
if (candidateScore > bestCandidateScore)
|
|
{
|
|
bestCandidate = iter;
|
|
bestCandidateScore = candidateScore;
|
|
}
|
|
}
|
|
|
|
if (bestCandidateScore < 1) return false;
|
|
|
|
// Perform the extraction
|
|
|
|
// XXX
|
|
|
|
return false;
|
|
}
|
|
|
|
bool DictEncoder::optimizeJoin()
|
|
{
|
|
// Try to optimize by joining two substrings
|
|
|
|
// XXX
|
|
|
|
return false;
|
|
}
|
|
|
|
void DictEncoder::optimizeOrder()
|
|
{
|
|
// Try to optimize order of strings in the dictionary for fastest display
|
|
// - put the shorter substrings first
|
|
// - for strings with roughly the same length put more frequently used first
|
|
|
|
typedef struct Entry
|
|
{
|
|
uint32_t penalty;
|
|
std::string word;
|
|
} Entry;
|
|
|
|
// Create copy of the dictionary with optimization info
|
|
|
|
std::vector<Entry> optimizedOrder;
|
|
for (auto iter = dictionary.begin(); iter < dictionary.end(); iter++)
|
|
{
|
|
const auto &dictionaryStr = *iter;
|
|
|
|
uint8_t sizePenalty = dictionaryStr.size() / 3;
|
|
uint16_t freqPenalty = 65535;
|
|
|
|
// Calculate penalty for low occurence frequency
|
|
|
|
for (auto &encoding : encodings)
|
|
{
|
|
for (auto &byte : *encoding)
|
|
{
|
|
if (byte == (iter - dictionary.begin())) freqPenalty--;
|
|
}
|
|
}
|
|
|
|
// Create new entry
|
|
|
|
Entry newEntry;
|
|
newEntry.word = dictionaryStr;
|
|
newEntry.penalty = sizePenalty * 65536 + freqPenalty;
|
|
|
|
optimizedOrder.push_back(newEntry);
|
|
}
|
|
|
|
// Determine ordering - sort from smallest penalty to largest
|
|
|
|
std::sort(optimizedOrder.begin(), optimizedOrder.end(),
|
|
[](const Entry &e1, const Entry &e2) { return e1.penalty < e2.penalty; });
|
|
|
|
// Create helper table for reordering
|
|
|
|
std::vector<uint8_t> reorderTable;
|
|
|
|
for (uint8_t idx1 = 0; idx1 < dictionary.size(); idx1++)
|
|
{
|
|
uint8_t idx2 = 0;
|
|
while (dictionary[idx1] != optimizedOrder[idx2].word) idx2++;
|
|
|
|
reorderTable.push_back(idx2);
|
|
}
|
|
|
|
// Perform the reordering
|
|
|
|
for (auto &encoding : encodings)
|
|
{
|
|
for (auto &byte : *encoding) byte = reorderTable[byte];
|
|
}
|
|
|
|
for (uint8_t idx = 0; idx < dictionary.size(); idx++) dictionary[idx] = optimizedOrder[idx].word;
|
|
}
|
|
|
|
void DictEncoder::process(StringEntryList &outDictionary)
|
|
{
|
|
if (dictionary.empty()) return;
|
|
// Optimize as long as it brings any improvement
|
|
|
|
while (optimizeSplit() || optimizeJoin()) ;
|
|
optimizeOrder();
|
|
|
|
// Export the dictionary to external format
|
|
|
|
outDictionary.type = ListType::DICTIONARY;
|
|
outDictionary.name = "DICTIONARY";
|
|
|
|
for (const auto &dictionaryStr : dictionary)
|
|
{
|
|
StringEntry newEntry = { true, true, true, "", dictionaryStr };
|
|
outDictionary.list.push_back(newEntry);
|
|
}
|
|
|
|
// Adapt the encoding to external format (0 = end of string)
|
|
|
|
for (auto &encoding : encodings)
|
|
{
|
|
for (auto &byte : *encoding) byte++;
|
|
encoding->push_back(0);
|
|
}
|
|
}
|
|
|
|
bool DataSet::isCompressionLvl2(const StringEntryList &list) const
|
|
{
|
|
return (GLOBAL_ConfigOptions["CONFIG_COMPRESSION_LVL_2"] && list.type == ListType::STRINGS_BASIC);
|
|
}
|
|
|
|
void DataSet::addStrings(const StringEntryList &stringList)
|
|
{
|
|
// Import the new list of strings
|
|
|
|
stringEntryLists.push_back(stringList);
|
|
stringEncodedLists.emplace_back();
|
|
|
|
// Clear strings not relevant for the current configuration
|
|
|
|
while (1)
|
|
{
|
|
if (isRelevant(stringEntryLists.back().list.back())) break;
|
|
|
|
stringEntryLists.back().list.pop_back();
|
|
if (stringEntryLists.back().list.empty())
|
|
{
|
|
ERROR(std::string("no valid strings in layout '") + layoutName() + "', list'" + stringEntryLists.back().name + "'");
|
|
}
|
|
}
|
|
|
|
// Clear the output content - make sure it is not valid anymore
|
|
|
|
outFileContent.clear();
|
|
}
|
|
|
|
void DataSet::process()
|
|
{
|
|
std::cout << "Processing file '" << CMD_cnfFile << "', layout '" << layoutName() << "'" << std::endl;
|
|
|
|
generateConfigDepStrings();
|
|
validateLists();
|
|
encodeStringsDict();
|
|
calculateFrequencies();
|
|
encodeStringsFreq();
|
|
prepareOutput();
|
|
}
|
|
|
|
const std::string &DataSet::getOutput()
|
|
{
|
|
if (outFileContent.empty())
|
|
{
|
|
process();
|
|
}
|
|
|
|
return outFileContent;
|
|
}
|
|
|
|
void DataSet::generateConfigDepStrings()
|
|
{
|
|
// Generate string to show the build features
|
|
|
|
std::string featureStr;
|
|
|
|
// Tape support features
|
|
|
|
if (GLOBAL_ConfigOptions["CONFIG_TAPE_NORMAL"] && GLOBAL_ConfigOptions["CONFIG_TAPE_TURBO"])
|
|
{
|
|
featureStr = "TAPE LOAD NORMAL TURBO\r";
|
|
}
|
|
else if (GLOBAL_ConfigOptions["CONFIG_TAPE_NORMAL"])
|
|
{
|
|
featureStr = "TAPE LOAD NORMAL\r";
|
|
}
|
|
else if (GLOBAL_ConfigOptions["CONFIG_TAPE_TURBO"])
|
|
{
|
|
featureStr = "TAPE LOAD TURBO\r";
|
|
}
|
|
|
|
// IEC support features
|
|
|
|
if (GLOBAL_ConfigOptions["CONFIG_IEC"])
|
|
{
|
|
featureStr += "IEC";
|
|
|
|
bool extendedIEC = false;
|
|
|
|
if (GLOBAL_ConfigOptions["CONFIG_IEC_BURST_CIA1"])
|
|
{
|
|
featureStr += " BURST1";
|
|
extendedIEC = true;
|
|
}
|
|
if (GLOBAL_ConfigOptions["CONFIG_IEC_BURST_CIA2"])
|
|
{
|
|
featureStr += " BURST2";
|
|
extendedIEC = true;
|
|
}
|
|
if (GLOBAL_ConfigOptions["CONFIG_IEC_BURST_MEGA65"])
|
|
{
|
|
featureStr += " BURST";
|
|
extendedIEC = true;
|
|
}
|
|
|
|
if (GLOBAL_ConfigOptions["CONFIG_IEC_DOLPHINDOS"])
|
|
{
|
|
featureStr += " DOLPHIN";
|
|
extendedIEC = true;
|
|
}
|
|
|
|
if (GLOBAL_ConfigOptions["CONFIG_IEC_JIFFYDOS"])
|
|
{
|
|
featureStr += " JIFFY";
|
|
extendedIEC = true;
|
|
}
|
|
|
|
if (!extendedIEC) featureStr += " NORMAL";
|
|
|
|
featureStr += "\r";
|
|
}
|
|
|
|
// RS-232 support features
|
|
|
|
if (GLOBAL_ConfigOptions["CONFIG_RS232_UP2400"]) featureStr += "UP2400\r";
|
|
if (GLOBAL_ConfigOptions["CONFIG_RS232_UP9600"]) featureStr += "UP9600\r";
|
|
|
|
// Keyboard support features
|
|
|
|
if (GLOBAL_ConfigOptions["CONFIG_KEYBOARD_C128"]) featureStr += "KBD 128\r";
|
|
if (GLOBAL_ConfigOptions["CONFIG_KEYBOARD_C65"]) featureStr += "KBD 65\r";
|
|
|
|
// Add strings to appropriate list
|
|
|
|
for (auto &stringEntryList : stringEntryLists)
|
|
{
|
|
if (stringEntryList.name != std::string("misc")) continue;
|
|
|
|
// List found
|
|
|
|
if (GLOBAL_ConfigOptions["CONFIG_SHOW_FEATURES"])
|
|
{
|
|
StringEntry newEntry1 = { true, true, true, "STR_PAL", "PAL\r" };
|
|
StringEntry newEntry2 = { true, true, true, "STR_NTSC", "NTSC\r" };
|
|
StringEntry newEntry3 = { true, true, true, "STR_FEATURES", featureStr };
|
|
|
|
stringEntryList.list.push_back(newEntry1);
|
|
stringEntryList.list.push_back(newEntry2);
|
|
stringEntryList.list.push_back(newEntry3);
|
|
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
void DataSet::validateLists()
|
|
{
|
|
for (const auto &stringEntryList : stringEntryLists)
|
|
{
|
|
for (const auto &stringEntry : stringEntryList.list)
|
|
{
|
|
// Check for maximum allowed string length
|
|
if (stringEntry.string.length() > 255) ERROR("string cannot be longer than 255 characters");
|
|
|
|
// Check mor maximum keyword length
|
|
if (stringEntryList.type == ListType::KEYWORDS)
|
|
{
|
|
if (stringEntry.string.length() > 8) ERROR("keyword cannot be longer than 8 characters");
|
|
tk__max_keyword_len = std::max(tk__max_keyword_len, (uint8_t) stringEntry.string.length());
|
|
}
|
|
|
|
// Update maximum alias length too
|
|
maxAliasLen = std::max(maxAliasLen, stringEntry.alias.length());
|
|
|
|
for (const auto &character : stringEntry.string)
|
|
{
|
|
if ((unsigned char) character >= 0x80)
|
|
{
|
|
ERROR(std::string("character above 0x80 in string '") + stringEntry.string + "'");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void DataSet::encodeStringsDict()
|
|
{
|
|
DictEncoder dictEncoder;
|
|
|
|
// Add strings for dictionary compresssion
|
|
|
|
for (uint8_t idx = 0; idx < stringEntryLists.size(); idx++)
|
|
{
|
|
const auto &stringEntryList = stringEntryLists[idx];
|
|
auto &stringEncodedList = stringEncodedLists[idx];
|
|
|
|
// Skip lists not to be encoded using the dictionary
|
|
if (!isCompressionLvl2(stringEntryList)) continue;
|
|
|
|
stringEncodedList.resize(stringEntryList.list.size());
|
|
for (uint8_t idxEntry = 0; idxEntry < stringEntryList.list.size(); idxEntry++)
|
|
{
|
|
dictEncoder.addString(stringEntryList.list[idxEntry].string,
|
|
&stringEncodedList[idxEntry]);
|
|
}
|
|
}
|
|
|
|
// Perform the compression
|
|
|
|
StringEntryList dictionary;
|
|
dictEncoder.process(dictionary);
|
|
|
|
// Add new lists
|
|
|
|
stringEntryLists.push_back(dictionary);
|
|
stringEncodedLists.emplace_back();
|
|
}
|
|
|
|
void DataSet::calculateFrequencies()
|
|
{
|
|
as1n.clear();
|
|
as3n.clear();
|
|
|
|
std::map<char, uint16_t> freqMapGeneral; // general character frequency map
|
|
std::map<char, uint16_t> freqMapKeywords; // frequency map for keywords
|
|
|
|
// Calculate frequencies of characters in the strings
|
|
|
|
for (const auto &stringEntryList : stringEntryLists)
|
|
{
|
|
// Skip lists encoded by the dictionary
|
|
if (isCompressionLvl2(stringEntryList)) continue;
|
|
|
|
for (const auto &stringEntry : stringEntryList.list)
|
|
{
|
|
if (!isRelevant(stringEntry)) continue;
|
|
|
|
for (const auto &character : stringEntry.string)
|
|
{
|
|
freqMapGeneral[character]++;
|
|
if (stringEntryList.type == ListType::KEYWORDS) freqMapKeywords[character]++;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Sort characters by frequency
|
|
|
|
std::vector<char> freqVector1;
|
|
|
|
for (auto iter = freqMapGeneral.begin(); iter != freqMapGeneral.end(); ++iter)
|
|
{
|
|
freqVector1.push_back(iter->first);
|
|
}
|
|
|
|
std::sort(freqVector1.begin(), freqVector1.end(), [&freqMapGeneral](char e1, char e2)
|
|
{ return freqMapGeneral[e2] < freqMapGeneral[e1]; });
|
|
|
|
// Check if minimal amount of characters needed, below 15 is not supported by the 6502 side code
|
|
|
|
if (freqVector1.size() < 15) ERROR(std::string("not enough distinct characters in layout '") + layoutName() + "', at least 15 needed");
|
|
|
|
// Extract 14 most frequent characters to be encoded as 1 nibble
|
|
|
|
for (uint8_t idx = 0; idx < 14; idx++)
|
|
{
|
|
as1n.push_back(freqVector1[idx]);
|
|
}
|
|
|
|
// Now sort them by frequency in keywords, in descending order - this will speed up the tokenizer a little
|
|
|
|
std::sort(as1n.begin(), as1n.end(), [&freqMapKeywords](char e1, char e2) { return freqMapKeywords[e2] > freqMapKeywords[e1]; });
|
|
|
|
// Extract characters to be encoded as 3 nibbles, which actually exist in keywords
|
|
|
|
std::vector<char> freqVector2;
|
|
|
|
tk__packed_as_3n = 0;
|
|
for (uint8_t idx = 14; idx < freqVector1.size(); idx++)
|
|
{
|
|
const auto &character = freqVector1[idx];
|
|
|
|
if (freqMapKeywords[character] > 0)
|
|
{
|
|
as3n.push_back(character);
|
|
tk__packed_as_3n++;
|
|
}
|
|
else
|
|
{
|
|
freqVector2.push_back(character);
|
|
}
|
|
}
|
|
tk__packed_as_3n = std::max(tk__packed_as_3n, (uint8_t) 1);
|
|
|
|
// Again, sort them by frequency in keywords, in descending order - this will speed up the tokenizer a little
|
|
|
|
std::sort(as3n.begin(), as3n.end(), [&freqMapKeywords](char e1, char e2) { return freqMapKeywords[e2] > freqMapKeywords[e1]; });
|
|
|
|
// Finally extract the remaining characters to be encoded as 3 nibbles
|
|
|
|
for (const auto &character : freqVector2)
|
|
{
|
|
as3n.push_back(character);
|
|
}
|
|
}
|
|
|
|
void DataSet::encodeByFreq(const std::string &plain, StringEncoded &encoded) const
|
|
{
|
|
bool fullByte = true;
|
|
|
|
auto push1n = [&](uint8_t val) // push 1 nibble - encoded character or 0xF mark
|
|
{
|
|
if (fullByte)
|
|
{
|
|
encoded.push_back(val);
|
|
fullByte = false;
|
|
}
|
|
else
|
|
{
|
|
encoded.back() += val * 0x10;
|
|
fullByte = true;
|
|
}
|
|
};
|
|
|
|
auto push2n = [&](uint8_t val) // push the remaining nibbles for 3-niobble encoded characters
|
|
{
|
|
if (fullByte)
|
|
{
|
|
encoded.push_back(val);
|
|
}
|
|
else
|
|
{
|
|
// Encode byte in a way to be easily decoded by 6502
|
|
encoded.back() += (val / 0x10) * 0x10;
|
|
encoded.push_back(val % 0x10);
|
|
}
|
|
};
|
|
|
|
// Encode every single character by frequency, put them in the output vector
|
|
|
|
for (const char &character : plain)
|
|
{
|
|
auto iterEncoding1 = std::find(as1n.begin(), as1n.end(), character);
|
|
if (iterEncoding1 != as1n.end())
|
|
{
|
|
push1n(std::distance(as1n.begin(), iterEncoding1) + 1);
|
|
}
|
|
else
|
|
{
|
|
push1n(0x0F);
|
|
|
|
auto iterEncoding3 = std::find(as3n.begin(), as3n.end(), character);
|
|
if (iterEncoding3 == as3n.end())
|
|
{
|
|
ERROR("internal error in 'encodeByFreq'");
|
|
}
|
|
push2n(std::distance(as3n.begin(), iterEncoding3) + 1);
|
|
}
|
|
}
|
|
|
|
// Make sure the last byte of encoded stream is 0
|
|
|
|
if (encoded.size() == 0 || encoded.back() != 0) encoded.push_back(0);
|
|
}
|
|
|
|
void DataSet::encodeStringsFreq()
|
|
{
|
|
// Encode every relevant string from every list - by character frequency
|
|
|
|
for (uint8_t idx = 0; idx < stringEntryLists.size(); idx++)
|
|
{
|
|
const auto &stringEntryList = stringEntryLists[idx];
|
|
auto &stringEncodedList = stringEncodedLists[idx];
|
|
|
|
// Skip lists encoded by the dictionary
|
|
if (isCompressionLvl2(stringEntryList)) continue;
|
|
|
|
// Perform frequency encoding of the list
|
|
|
|
for (const auto &stringEntry : stringEntryList.list)
|
|
{
|
|
stringEncodedList.emplace_back();
|
|
auto &stringEncoded = stringEncodedList.back();
|
|
|
|
if (isRelevant(stringEntry))
|
|
{
|
|
encodeByFreq(stringEntry.string, stringEncoded);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void DataSet::putCharEncoding(std::ostringstream &stream, uint8_t idx, char character, bool is3n)
|
|
{
|
|
stream << "\t.byte $" << std::uppercase << std::hex <<
|
|
std::setfill('0') << std::setw(2) << +character <<
|
|
" // " << std::setfill(is3n ? '0' : ' ') << std::setw(2) << +idx;
|
|
|
|
std::string petscii;
|
|
|
|
if (character == 0x20)
|
|
{
|
|
petscii = " = SPACE";
|
|
}
|
|
else if (character == 0x27)
|
|
{
|
|
petscii = " = APOSTROPHE";
|
|
}
|
|
else if (character == 0x0D)
|
|
{
|
|
petscii = " = RETURN";
|
|
}
|
|
else if (character > ' ' && character < 'a')
|
|
{
|
|
petscii = std::string(" = '") + character + "'";
|
|
}
|
|
|
|
stream << petscii << std::endl;
|
|
}
|
|
|
|
void DataSet::prepareOutput_1n_3n(std::ostringstream &stream)
|
|
{
|
|
uint8_t idx;
|
|
|
|
// Export all nibble-encoded characters
|
|
|
|
stream << std::endl << ".macro put_packed_as_1n() // characters encoded as 1 nibble" << std::endl << "{" << std::endl;
|
|
|
|
idx = 0;
|
|
for (const auto& encoding : as1n)
|
|
{
|
|
putCharEncoding(stream, ++idx, encoding, false);
|
|
}
|
|
|
|
stream << "}" << std::endl;
|
|
|
|
// Export all byte-encoded characters
|
|
|
|
stream << std::endl << ".macro put_packed_as_3n() // characters encoded as 3 nibbles" << std::endl << "{" << std::endl;
|
|
|
|
idx = 0;
|
|
for (const auto& encoding : as3n)
|
|
{
|
|
if (idx == tk__packed_as_3n) stream << std::endl <<
|
|
"\t// Characters below are not used by any BASIC keyword" << std::endl << std::endl;
|
|
|
|
putCharEncoding(stream, ++idx, encoding, true);
|
|
}
|
|
|
|
stream << "}" << std::endl;
|
|
}
|
|
|
|
void DataSet::prepareOutput_labels(std::ostringstream &stream,
|
|
const StringEntryList &stringEntryList,
|
|
const StringEncodedList &stringEncodedList)
|
|
{
|
|
// For the dictionary we do not have any labels
|
|
if (stringEntryList.type == ListType::DICTIONARY) return;
|
|
|
|
stream << std::endl;
|
|
for (uint8_t idx = 0; idx < stringEncodedList.size(); idx++)
|
|
{
|
|
const auto &stringEntry = stringEntryList.list[idx];
|
|
const auto &stringEncoded = stringEncodedList[idx];
|
|
|
|
if (!stringEncoded.empty())
|
|
{
|
|
stream << ".label IDX__" << stringEntry.alias <<
|
|
std::string(maxAliasLen - stringEntry.alias.length(), ' ') << " = $" <<
|
|
std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +idx << std::endl;
|
|
}
|
|
}
|
|
}
|
|
|
|
void DataSet::prepareOutput_packed(std::ostringstream &stream,
|
|
const StringEntryList &stringEntryList,
|
|
const StringEncodedList &stringEncodedList)
|
|
{
|
|
if (isCompressionLvl2(stringEntryList))
|
|
{
|
|
stream << std::endl << ".macro put_packed_dict_";
|
|
}
|
|
else
|
|
{
|
|
stream << std::endl << ".macro put_packed_freq_";
|
|
}
|
|
|
|
stream << stringEntryList.name << "()" << std::endl << "{" << std::endl;
|
|
|
|
enum LastStr { NONE, SKIPPED, WRITTEN } lastStr = LastStr::NONE;
|
|
for (uint8_t idxString = 0; idxString < stringEncodedList.size(); idxString++)
|
|
{
|
|
const auto &stringEncoded = stringEncodedList[idxString];
|
|
|
|
if (stringEncoded.empty())
|
|
{
|
|
if (stringEntryList.type == ListType::DICTIONARY) ERROR("internal error"); // should never happen
|
|
|
|
if (lastStr == LastStr::WRITTEN) stream << std::endl;
|
|
stream << "\t.byte $00 // skipped " << stringEntryList.list[idxString].alias << std::endl;
|
|
lastStr = LastStr::SKIPPED;
|
|
}
|
|
else
|
|
{
|
|
if (lastStr != LastStr::NONE) stream << std::endl;
|
|
|
|
if (stringEntryList.type != ListType::DICTIONARY)
|
|
{
|
|
stream << "\t// IDX__" << stringEntryList.list[idxString].alias << std::endl;
|
|
}
|
|
stream << "\t.byte ";
|
|
|
|
bool first = true;
|
|
for (const auto &charEncoded : stringEncoded)
|
|
{
|
|
if (first)
|
|
{
|
|
first = false;
|
|
}
|
|
else
|
|
{
|
|
stream << ", ";
|
|
}
|
|
|
|
stream << "$" << std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +charEncoded;
|
|
}
|
|
|
|
stream << std::endl;
|
|
lastStr = LastStr::WRITTEN;
|
|
}
|
|
}
|
|
|
|
// For the token list - put the 'end of keywords' mark
|
|
|
|
if (stringEntryList.type == ListType::KEYWORDS)
|
|
{
|
|
stream << std::endl << "\t// Marker - end of the keyword list" << std::endl;
|
|
stream << "\t.byte $FF, $FF" << std::endl;
|
|
}
|
|
|
|
stream << "}" << std::endl;
|
|
}
|
|
|
|
void DataSet::prepareOutput()
|
|
{
|
|
// Convert our encoded strings to a KickAssembler source
|
|
|
|
std::ostringstream stream;
|
|
stream << std::endl << "#if ROM_LAYOUT_" << layoutName() << std::endl;
|
|
|
|
// Export 1-nibble and 3-nibble encoding data
|
|
|
|
prepareOutput_1n_3n(stream);
|
|
|
|
// Export additional data for the tokenizer
|
|
|
|
stream << std::endl << ".label TK__PACKED_AS_3N = $" << std::hex << +tk__packed_as_3n <<
|
|
std::endl << ".label TK__MAX_KEYWORD_LEN = " << std::dec << +tk__max_keyword_len << std::endl;
|
|
|
|
// Export encoded strings
|
|
|
|
for (uint8_t idx = 0; idx < stringEntryLists.size(); idx++)
|
|
{
|
|
const auto &stringEntryList = stringEntryLists[idx];
|
|
const auto &stringEncodedList = stringEncodedLists[idx];
|
|
|
|
// Export labels for the current list
|
|
|
|
prepareOutput_labels(stream, stringEntryList, stringEncodedList);
|
|
|
|
// For the token list - put the number of tokens available
|
|
|
|
if (stringEntryList.type == ListType::KEYWORDS)
|
|
{
|
|
stream << std::endl << ".label TK__MAXTOKEN_" << stringEntryList.name << " = " <<
|
|
std::dec << stringEncodedList.size() << std::endl;
|
|
}
|
|
|
|
// Export the packed data
|
|
|
|
prepareOutput_packed(stream, stringEntryList, stringEncodedList);
|
|
}
|
|
|
|
// Finalize the file stream
|
|
|
|
stream << std::endl << "#endif // ROM_LAYOUT_" << layoutName() << std::endl;
|
|
outFileContent = stream.str();
|
|
}
|
|
|
|
|
|
//
|
|
// Common helper functions
|
|
//
|
|
|
|
void parseConfigFile()
|
|
{
|
|
GLOBAL_ConfigOptions.clear();
|
|
|
|
// Open the configuration file
|
|
|
|
std::ifstream cnfFile;
|
|
cnfFile.open(CMD_cnfFile);
|
|
if (!cnfFile.good()) ERROR("unable to open config file");
|
|
|
|
// Parse the file
|
|
|
|
size_t lineNum = 0;
|
|
while (!cnfFile.eof())
|
|
{
|
|
lineNum++;
|
|
std::string workStr;
|
|
|
|
// Read a single line, remove leading spaces and tabs
|
|
|
|
std::getline(cnfFile, workStr); // XXX check for errors here
|
|
workStr = std::regex_replace(workStr, std::regex("^[ \t]+"), "");
|
|
|
|
// Skip lines which are not preprocessor definitions
|
|
|
|
if (workStr.empty() || workStr[0] != '#') continue;
|
|
|
|
// Make sure this is '#define '
|
|
|
|
if (!std::regex_match(workStr, std::regex("^#define[ \t].*")))
|
|
{
|
|
ERROR(std::string("only '#define' preprocessor directives allowed in config files - line ") + std::to_string(lineNum));
|
|
}
|
|
|
|
// Get rid of the directive and trailing spaces/comments
|
|
|
|
workStr = std::regex_replace(workStr, std::regex("^#define[ \t]+"), "");
|
|
workStr = std::regex_replace(workStr, std::regex("[ \t]+.*"), "");
|
|
|
|
// Add definitin to config option map
|
|
|
|
if (workStr.empty()) ERROR(std::string("error parsing config file - line ") + std::to_string(lineNum));
|
|
|
|
GLOBAL_ConfigOptions[workStr] = true;
|
|
}
|
|
|
|
cnfFile.close();
|
|
}
|
|
|
|
void printUsage()
|
|
{
|
|
std::cout << "\n" <<
|
|
"usage: generate_strings [-o <out file>] [-c <configuration file>]" << "\n\n";
|
|
}
|
|
|
|
void printBanner()
|
|
{
|
|
printBannerLineTop();
|
|
std::cout << "// Generating compressed messages and BASIC tokens\n";
|
|
printBannerLineBottom();
|
|
}
|
|
|
|
void parseCommandLine(int argc, char **argv)
|
|
{
|
|
int opt;
|
|
|
|
// Retrieve command line options
|
|
|
|
while ((opt = getopt(argc, argv, "o:c:")) != -1)
|
|
{
|
|
switch(opt)
|
|
{
|
|
case 'o': CMD_outFile = optarg; break;
|
|
case 'c': CMD_cnfFile = optarg; break;
|
|
default: printUsage(); ERROR();
|
|
}
|
|
}
|
|
}
|
|
|
|
void writeStrings()
|
|
{
|
|
std::string outputString;
|
|
|
|
if (GLOBAL_ConfigOptions["CONFIG_PLATFORM_COMMANDER_X16"])
|
|
{
|
|
DataSetX16 dataSetX16;
|
|
|
|
// Add input data to computation objects
|
|
|
|
dataSetX16.addStrings(GLOBAL_Keywords_V2);
|
|
dataSetX16.addStrings(GLOBAL_Keywords_CC);
|
|
dataSetX16.addStrings(GLOBAL_Keywords_CD);
|
|
dataSetX16.addStrings(GLOBAL_Errors);
|
|
dataSetX16.addStrings(GLOBAL_MiscStrings);
|
|
|
|
// Retrieve the results
|
|
|
|
outputString = dataSetX16.getOutput();
|
|
}
|
|
else if (GLOBAL_ConfigOptions["CONFIG_PLATFORM_COMMODORE_64"] && GLOBAL_ConfigOptions["CONFIG_MB_MEGA_65"])
|
|
{
|
|
DataSetM65 dataSetM65;
|
|
|
|
// Add input data to computation objects
|
|
|
|
dataSetM65.addStrings(GLOBAL_Keywords_V2);
|
|
dataSetM65.addStrings(GLOBAL_Keywords_CC);
|
|
dataSetM65.addStrings(GLOBAL_Keywords_CD);
|
|
dataSetM65.addStrings(GLOBAL_Errors);
|
|
dataSetM65.addStrings(GLOBAL_MiscStrings);
|
|
|
|
// Retrieve the results
|
|
|
|
outputString = dataSetM65.getOutput();
|
|
}
|
|
else if (GLOBAL_ConfigOptions["CONFIG_PLATFORM_COMMODORE_64"])
|
|
{
|
|
DataSetSTD dataSetSTD;
|
|
|
|
// Add input data to computation objects
|
|
|
|
dataSetSTD.addStrings(GLOBAL_Keywords_V2);
|
|
dataSetSTD.addStrings(GLOBAL_Keywords_CC);
|
|
dataSetSTD.addStrings(GLOBAL_Keywords_CD);
|
|
dataSetSTD.addStrings(GLOBAL_Errors);
|
|
dataSetSTD.addStrings(GLOBAL_MiscStrings);
|
|
|
|
// Retrieve the results
|
|
|
|
outputString = dataSetSTD.getOutput();
|
|
}
|
|
else
|
|
{
|
|
ERROR("unable to determine string set");
|
|
}
|
|
|
|
// Remove old file
|
|
|
|
unlink(CMD_outFile.c_str());
|
|
|
|
// Open output file for writing
|
|
|
|
std::ofstream outFile(CMD_outFile, std::fstream::out | std::fstream::trunc);
|
|
if (!outFile.good()) ERROR(std::string("can't open oputput file '") + CMD_outFile + "'");
|
|
|
|
// Write header
|
|
|
|
outFile << "//\n// Generated file - do not edit\n//";
|
|
|
|
// Write packed strings
|
|
|
|
outFile << std::endl << std::endl;
|
|
outFile << outputString;
|
|
outFile << std::endl << std::endl;
|
|
|
|
// Close the file
|
|
|
|
outFile.close();
|
|
|
|
std::cout << std::string("Compressed strings written to: ") + CMD_outFile + "\n\n";
|
|
}
|
|
|
|
//
|
|
// Main function
|
|
//
|
|
|
|
int main(int argc, char **argv)
|
|
{
|
|
printBanner();
|
|
parseCommandLine(argc, argv);
|
|
|
|
parseConfigFile();
|
|
|
|
writeStrings();
|
|
|
|
return 0;
|
|
}
|