chris/open-roms
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

new text compression and tokeniser - faster, about 100 bytes saved

Browse Source
This commit is contained in:
FeralChild64 committed 2020-06-21 18:16:38 +02:00
1 parent 51e45e9983
commit 94170f156c
24 files changed
+1848 -872
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Makefile
+15 -18
View File
@@ -25,6 +25,7 @@ SRCDIR_BASIC = $(SRCDIR_COMMON) \
c64/basic/math_mov \
c64/basic/print \
c64/basic/rom_revision \
c64/basic/tokenizer \
c64/basic/wedge
SRCDIR_KERNAL = $(SRCDIR_COMMON) \
@@ -54,7 +55,7 @@ SRC_TOOLS = $(wildcard tools/*.c,tools/*.cc)
# Generated files
GEN_BASIC = build/,generated/packed_messages.s build/,generated/float_constants.s
GEN_BASIC = build/,generated/packed_strings.s build/,generated/float_constants.s
GEN_KERNAL =
# List of build directories
@@ -68,12 +69,12 @@ DIR_X16 = build/target_cx16
# List of config files
CFG_CUS = c64/,,config_custom.s
CFG_GEN = c64/,,config_generic.s
CFG_TST = c64/,,config_testing.s
CFG_M65 = c64/,,config_mega65.s
CFG_U64 = c64/,,config_ultimate64.s
CFG_X16 = c64/,,config_cx16.s
CFG_CUS = c64/,,config_custom.s
CFG_GEN = c64/,,config_generic.s
CFG_TST = c64/,,config_testing.s
CFG_M65 = c64/,,config_mega65.s
CFG_U64 = c64/,,config_ultimate64.s
CFG_X16 = c64/,,config_cx16.s
# Dependencies - helper variables
@@ -83,8 +84,8 @@ DEP_KERNAL = $(SRC_KERNAL) $(SRCDIR_KERNAL) $(GEN_KERNAL)
# List of tools
TOOL_COLLECT_DATA = build/tools/collect_data
TOOL_COMPRESS_TEXT = build/tools/compress_text
TOOL_GENERATE_CONSTANTS = build/tools/generate_constants
TOOL_GENERATE_STRINGS = build/tools/generate_strings
TOOL_PATCH_CHARGEN = build/tools/patch_chargen
TOOL_PNGPREPARE = build/tools/pngprepare
TOOL_BUILD_SEGMENT = build/tools/build_segment
@@ -166,19 +167,15 @@ updatebin:
$(TOOL_PNGPREPARE): tools/pngprepare.c
@mkdir -p build/tools
$(CC) -O2 -Wall -I/usr/local/include -L/usr/local/lib -o $@ $< -lpng
$(TOOL_COMPRESS_TEXT): tools/compress_text.c
@mkdir -p build/tools
$(CC) -O2 -Wall -I/usr/local/include -L/usr/local/lib -o $@ $< -lm
$(CC) -O2 -g -Wall -I/usr/local/include -L/usr/local/lib -o $@ $< -lpng
build/tools/%: tools/%.c
@mkdir -p build/tools
$(CC) -O2 -Wall -o $@ $<
$(CC) -O2 -g -Wall -o $@ $<
build/tools/%: tools/%.cc tools/common.h
@mkdir -p build/tools
$(CXX) -O2 -Wall -o $@ $<
$(CXX) -O2 -g -Wall -o $@ $<
# Rules - CHARGEN
@@ -258,9 +255,9 @@ build/symbols_ultimate64.vs: $(DIR_U64)/BASIC_combined.vs $(DIR_U64)/KER
# Rules - BASIC and KERNAL intermediate files
build/,generated/packed_messages.s: $(TOOL_COMPRESS_TEXT)
build/,generated/packed_strings.s: $(TOOL_GENERATE_STRINGS)
@mkdir -p build/,generated
$(TOOL_COMPRESS_TEXT) > build/,generated/packed_messages.s
$(TOOL_GENERATE_STRINGS) -o build/,generated/packed_strings.s
build/,generated/float_constants.s: $(TOOL_GENERATE_CONSTANTS)
@mkdir -p build/,generated
@@ -441,7 +438,7 @@ test_ultimate64: build/kernal_ultimate64.rom build/basic_ultimate64.rom build/sy
x64 -kernal build/kernal_ultimate64.rom -basic build/basic_ultimate64.rom -moncommands build/symbols_ultimate64.vs -1 $(TESTTAPE) -8 $(TESTDISK)
test_mega65: $(TARGET_M65_x) $(TARGET_M65_x_PXL)
../xemu/build/bin/xmega65.native -dmarev 2 -forcerom -loadrom $(TARGET_M65_x_PXL)
../xemu/build/bin/xmega65.native -dmarev 2 -besure -fontrefresh -forcerom -loadrom $(TARGET_M65_x_PXL)
test_cx16: $(TARGET_X16_x)
../x16-emulator/x16emu -rom $(TARGET_X16_x)
c64/,,config_custom.s
+3 -3
View File
@@ -46,8 +46,8 @@
// I/O devices
#define CONFIG_IEC
// #define CONFIG_IEC_DOLPHINDOS
// #define CONFIG_IEC_DOLPHINDOS_FAST
#define CONFIG_IEC_DOLPHINDOS
#define CONFIG_IEC_DOLPHINDOS_FAST
#define CONFIG_IEC_JIFFYDOS
// #define CONFIG_IEC_JIFFYDOS_BLANK
// #define CONFIG_IEC_BURST_CIA1 // please keep disabled for now
@@ -124,7 +124,7 @@
#define CONFIG_PANIC_SCREEN
#define CONFIG_DOS_WEDGE
#define CONFIG_TAPE_WEDGE
#define CONFIG_TAPE_HEAD_ALIGN
// #define CONFIG_TAPE_HEAD_ALIGN
#define CONFIG_BCD_SAFE_INTERRUPTS
c64/basic/assets/packed_keywords.s → c64/basic/assets/packed_as_1n.s
+3 -2
View File
@@ -3,6 +3,7 @@
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
packed_keywords:
put_packed_keywords()
packed_as_1n:
put_packed_as_1n()
c64/basic/assets/packed_chars.s → c64/basic/assets/packed_as_3n.s
+3 -2
View File
@@ -3,6 +3,7 @@
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
packed_message_chars:
put_packed_chars()
packed_as_3n:
put_packed_as_3n()
c64/basic/assets/packed_tokens.s → c64/basic/assets/packed_str_errors.s
+3 -2
View File
@@ -3,6 +3,7 @@
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
packed_message_tokens:
put_packed_tokens()
packed_str_errors:
put_freq_packed_errors()
c64/basic/assets/packed_str_keywords_V2.s
+9
View File
@@ -0,0 +1,9 @@
// #LAYOUT# STD * #TAKE
// #LAYOUT# M65 BASIC_1 #TAKE
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
packed_str_keywords_V2:
put_freq_packed_keywords_V2()
c64/basic/assets/packed_words.s → c64/basic/assets/packed_str_misc.s
+3 -2
View File
@@ -3,6 +3,7 @@
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
packed_message_words:
put_packed_words()
packed_str_misc:
put_freq_packed_misc()
c64/basic/basic_commands/cmd_stop_end.s
+4 -4
View File
@@ -4,8 +4,8 @@
cmd_stop:
ldx #30 // "BREAK"
jsr print_packed_message
ldx #IDX__EV2_1E // "BREAK"
jsr print_packed_error
// Are we in direct mode
lda CURLIN+1
@@ -13,8 +13,8 @@ cmd_stop:
beq !+
// Not direct mode
ldx #31
jsr print_packed_message
ldx #IDX__STR_IN
jsr print_packed_misc_str
lda CURLIN+1
ldx CURLIN+0
c64/basic/basic_main_loop.s
+2 -2
View File
@@ -6,8 +6,8 @@
basic_main_loop:
// Tell user we are ready
ldx #30
jsr print_packed_message
ldx #IDX__STR_READY
jsr print_packed_misc_str
// Enable Kernal messages
lda #$80
c64/basic/board_m65/4000.mega65_vectable_basic_1.s
+12 -8
View File
@@ -11,13 +11,15 @@
// Label definitions
.label VB1__INITMSG = $4000 + 2 * 0
.label VB1__LINKPRG = $4000 + 2 * 1
.label VB1__tokenise_line = $4000 + 2 * 2
.label VB1__list_single_line = $4000 + 2 * 3
.label VB1__print_packed_message = $4000 + 2 * 4
.label VB1__basic_do_clr = $4000 + 2 * 5
.label VB1__basic_do_new = $4000 + 2 * 6
.label VB1__INITMSG = $4000 + 2 * 0
.label VB1__LINKPRG = $4000 + 2 * 1
.label VB1__tokenise_line = $4000 + 2 * 2
.label VB1__list_single_line = $4000 + 2 * 3
.label VB1__print_packed_error = $4000 + 2 * 4
.label VB1__print_packed_misc_str = $4000 + 2 * 5
.label VB1__print_packed_keyword_V2 = $4000 + 2 * 6
.label VB1__basic_do_clr = $4000 + 2 * 7
.label VB1__basic_do_new = $4000 + 2 * 8
#else
@@ -27,7 +29,9 @@
.word LINKPRG
.word tokenise_line
.word list_single_line
.word print_packed_message
.word print_packed_error
.word print_packed_misc_str
.word print_packed_keyword_V2
.word basic_do_clr
.word basic_do_new
c64/basic/engine/do_error.s
+5 -5
View File
@@ -151,10 +151,10 @@ do_basic_error: // error code in .X
// Error message text + " ERROR"
jsr print_packed_message
jsr print_packed_error
ldx #33
jsr print_packed_message
ldx #IDX__STR_ERROR
jsr print_packed_misc_str
// Check if direct mode
@@ -164,8 +164,8 @@ do_basic_error: // error code in .X
// We were in a program, so show IN <line>
ldx #31
jsr print_packed_message
ldx #IDX__STR_IN
jsr print_packed_misc_str
lda CURLIN+1
ldx CURLIN+0
c64/basic/init/basic_warm_start_internal.s
+2 -2
View File
@@ -19,8 +19,8 @@ basic_warm_start_real:
ora CMP0+1
beq !+
ldx #32
jsr print_packed_message
ldx #IDX__STR_BRK_AT
jsr print_packed_misc_str
lda CMP0+1
jsr print_hex_byte
c64/basic/init/initmsg_bytes_free.s
+2 -2
View File
@@ -21,5 +21,5 @@ initmsg_bytes_free:
jsr print_integer
// Print rest of the start up message
ldx #34
jmp print_packed_message
ldx #IDX__STR_BYTES
jmp print_packed_misc_str
c64/basic/pack_word.s
-260
View File
@@ -1,260 +0,0 @@
// #LAYOUT# STD * #TAKE
// #LAYOUT# M65 BASIC_1 #TAKE
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
// Pack a word using the simple compression scheme.
// This has to be the same implementation as in compress_text.c
// because we compress input lines of BASIC to do the tokenisation.
// This allows us to search through the compressed tokens directly
// using the compressed fragment of BASIC, and thus work out the
// token for a given piece of string.
// This also helps ensure that we have an implementation that is totally
// different to that of the original C64 BASIC, as well as likely
// saving a bit of space in the ROM, especially once we start implementing
// larger keyword vocabs for BASIC 7 or BASIC 10 compatibility, since the
// compression and decompression machinery is of fixed size.
pack_word:
// Inputs:
// X = offset of string to pack. Address is $0200 + X
// __tokenise_work1 = number of bytes to pack
// __tokenise_work2 = packed length of string
// Outputs:
// Output is written to $0100 (bottom of stack).
// Memory modified:
// __tokenise_work4 = temporary storage
// __tokenise_work3 = flag for whether we have a nybl prepared or not.
// (This can also be checked on exit, to know whether the last byte
// is incomplete, which is important for searching for abbreviated
// key words.)
// Make sure we have a sane request
lda __tokenise_work1
bne !+
sec
rts
!:
// Initialise internal variables
lda #$00
sta __tokenise_work2
sta __tokenise_work3
pack_char_loop:
// Get the next char
lda $0200,x
// Work out what type of character we have, and thus
// how we need to pack it.
jsr get_packed_char_index
cmp #$00
bne not_exception_char
// Char is an exception char.
// We need to either finish the previous byte with $xF,
// Or if there is no existing nybl, then we need to output
// either $FE or $FF, depending on whether this is the last
// byte or not
lda __tokenise_work3
bne have_nybl_before_exception_char
// No nybl exists before the exception char, so prefix with $FE or $FF
// based on whether we are at the end of the input string or not
inx
lda __tokenise_work1
cmp #$01
beq at_end
lda #$FF
.byte $2C
at_end:
lda #$FE
dex
jsr write_unpacked_char
// Fall through
output_exception_byte:
lda $0200,x
jsr write_unpacked_char
jmp consider_next_char
have_nybl_before_exception_char:
// Or existing byte with #$0F, and clear nybl flag
ldy __tokenise_work2
lda $0100,y
ora #$0f
sta $0100,y
inc __tokenise_work2
// FALL THROUGH
write_literal_and_terminate_if_required:
// Clear have nybl flag
lda #$00
sta __tokenise_work3
lda $0200,x
jsr write_unpacked_char
jmp end_string_if_required
end_string_if_required:
// Check if X points to last char of string to be packed
inx
lda __tokenise_work1
cmp #$01
beq !+
dex
jmp consider_next_char
!:
dex
// Fall through
end_string:
// Write end of string sequence if required
lda #$00
jsr write_unpacked_char
rts
write_unpacked_char:
ldy __tokenise_work2
sta $0100,y
iny
sty __tokenise_work2
rts
not_exception_char:
// It wasn't an exception char, so write the symbol out
cmp #$10
bcs whole_byte_symbol
// Nybl encoded value: Put in upper half or lower half of a byte?
ldy __tokenise_work3
bne store_low_nybl
// Store in high nybl -- so shift it into the hi nybl
asl
asl
asl
asl
ldy __tokenise_work2
sta $0100,y
bne stored_nybl // Must be taken, as A cannot be $00
store_low_nybl:
// Low nybl gets added to high nybl already stored in the byte
ldy __tokenise_work2
ora $0100,y
sta $0100,y
stored_nybl:
// Now toggle the nybl flag
lda __tokenise_work3
eor #$ff
sta __tokenise_work3
// And advance the offset if we filled this byte up
bne consider_next_char // Taken if byte has a nybl free, i.e., don't advance pointer while half byte remains free
// Count the filled up byte
inc __tokenise_work2
// Add termination byte if required
// (This re-writes the same byte again, which we don't care about)
jmp end_string_if_required
whole_byte_symbol:
// It's a $Fx symbol.
// But if we have a nybl to flush, then we handle it like
// a full exception character
ldy __tokenise_work3
beq nothing_to_flush
// Set low nybl to $F to mark next byte as literal
ldy __tokenise_work2
lda $0100,y
ora #$0f
sta $0100,y
inc __tokenise_work2
jmp write_literal_and_terminate_if_required
nothing_to_flush:
// Write the symbol as it is, or the literal
// if it has been passed to us via fall-through
jsr write_unpacked_char
jmp end_string_if_required
consider_next_char:
// Pack next char
inx
dec __tokenise_work1
beq !+
jmp pack_char_loop
!:
// Add 1 to length if nybl waiting to be flushed
// (but leave flag set so the caller knows if the
// bottom nybl might be different if the string were
// longer).
lda __tokenise_work3
beq !+
inc __tokenise_work2
rts
!:
// Last byte was full, so we need at $00 on the end
// (unless a $FE token was written 2 bytes ago)
ldy __tokenise_work2
iny
lda #$00
sta $0100,y
inc __tokenise_work2
rts
get_packed_char_index:
// Find char in A in the packed char list, and return
// $1 - $E, $F1 - $FD if the charactar is abbreviable,
// or else return $00 if the character has to be stored literally.
// Y register and A are modified, as are processor flags.
ldy #$00
find_char_loop:
cmp packed_message_chars,y
bne not_a_match
enumerate:
// Is it a nybl abbreviable char?
cpy #14
bcs not_nybl_char
// Yes, so add 1 and return it
iny
tya
rts
not_nybl_char:
cpy #27
bcs not_a_match
// It's a $Fx character.
// Since the lowest index is $E, and it should be encoded as $F1
// we need to add $E3
tya
clc
adc #$E3
rts
not_a_match:
iny
// Loop until we find it, or we discover it isn't
// a character with a special encoding
cpy #27
bne find_char_loop
// Not a char that we can pack, so return 0
lda #$00
rts
c64/basic/print/list_single_line.s
+4 -10
View File
@@ -86,22 +86,16 @@ list_not_quote:
pha
phy_trash_a
// Get pointer to compressed keyword list
lda #<packed_keywords
sta FRESPC+0
lda #>packed_keywords
sta FRESPC+1
// Subtract $80 from token to get offset in word
// list
// Subtract $80 from token to get offset in token list
txa
and #$7f
tax
// Now ask for it to be printed
ldy #$ff
jsr packed_word_search
jsr print_packed_keyword_V2
// Restore registers
ply_trash_a
pla
c64/basic/print/print_packed_message.s
-244
View File
@@ -1,244 +0,0 @@
// #LAYOUT# STD * #TAKE
// #LAYOUT# M65 * #TAKE
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
// This routine prints messages that have been packed
// using the make_error_tables program.
// The general idea is to save space in the ROM.
// The output of make_error_tables is written to packed_messages.s, and
// contains the following:
//
// packed_message_words - List of nybl packed words.
// packed_message_tokens - List of tokens that get joined to make messages
// packed_message_chars - List of characters in messages sorted by frequency.
print_packed_message:
// .X = message number
#if (ROM_LAYOUT_M65 && SEGMENT_BASIC_0)
jsr map_BASIC_1
jsr_ind VB1__print_packed_message
jmp map_NORMAL
#else
lda #$ff
ldy #$ff
packed_message_search:
iny
cpx #$00
beq found_message_in_token_stream
lda packed_message_tokens,y
cmp #$ff
bne !+
dex
!:
cpy #$ff
bne packed_message_search
// Could not find message
sec
rts
found_message_in_token_stream: // XXX size-optimize this
// Now print each word in the message
phy_trash_a
lda packed_message_tokens,y
cmp #$FF
beq found_message_in_token_stream_done
jsr print_packed_word
ply_trash_a
iny
beq found_message_in_token_stream_done
!:
phy_trash_a
lda packed_message_tokens,y
cmp #$FF
beq found_message_in_token_stream_done
pha
jsr print_space
pla
jsr print_packed_word
ply_trash_a
iny
bne !-
found_message_in_token_stream_done:
pla
rts
print_packed_word:
// A = the word to print
tax
// Setup pointer to packed words
lda #<packed_message_words
sta FRESPC+0
lda #>packed_message_words
sta FRESPC+1
// Find the word in the list
ldy #$ff
packed_word_search:
iny
cpx #$00
beq found_packed_word
// $x0 is an end of work marker
lda (FRESPC),y
and #$0f
cmp #$00
bne !+
dex
!:
// $FE $xx is an end of work marker
lda (FRESPC),y
cmp #$fe
bne !+
dex
iny // skip next byte
!:
cmp #$ff
bne !+
iny // skip next byte
!:
// Advance to next page if required
cpy #$ff
bne packed_word_search
inc FRESPC+1
jmp packed_word_search
found_packed_word:
// Get pointer to start of packed word
// Make sure we do not wrap on a page boundary
tya
clc
adc FRESPC+0
sta FRESPC+0
lda FRESPC+1
adc #$00
sta FRESPC+1
print_packed_string:
ldy #$ff
next_packed_word_char:
// .Y = offset into packed word data - 1
iny
// Check for end of word
lda (FRESPC),y
cmp #$00
beq end_of_packed_word
cmp #$fe
beq last_literal
cmp #$ff
beq next_is_literal_char
and #$f0
beq end_of_packed_word
// Check if it is an uncommon char
lda (FRESPC),y
and #$f0
cmp #$f0
beq is_uncommon_char
// Is not an uncommon char, so get char of upper nybl
lda (FRESPC),y
// Put high nybl into low bits of X
lsr
lsr
lsr
lsr
tax
// X=1-14 = first 14 chars, so subtract one
lda packed_message_chars-1,x
jsr JCHROUT // preserves .Y
// See if there is a char in the low nybl to print
lda (FRESPC),y
and #$0f
beq end_of_packed_word
cmp #$0f
beq no_lo_nybl_char
has_lo_nybl_char:
// We need to print the low nybl char as well
tax
phy_trash_a
// Skip lower nybls of $F or $0 in nybl packed
// bytes, as these do not encode characters
cpx #$f
beq !+
has_nybl:
cpx #0
beq !+
// X=1-14 = first 14 chars, so subtract one
lda packed_message_chars-1,x
jsr JCHROUT
!:
ply_trash_a
// FALLTHROUGH
no_lo_nybl_char:
// If current byte has $0F in low nybl, then the next
// byte is a whole byte.
lda (FRESPC),y
and #$0f
cmp #$0f
beq next_is_literal_char
ready_for_next_char:
cpy #$ff
bne next_packed_word_char
end_of_packed_word:
// Hit end of packed data
rts
last_literal:
jsr unpack_literal_char
rts
next_is_literal_char:
jsr unpack_literal_char
jmp ready_for_next_char
unpack_literal_char:
// Output the literal char, and ready for next char
iny
phy_trash_a
lda (FRESPC),y
jsr JCHROUT
pla
tay
rts
is_uncommon_char:
// Lower nybl has offset into low-frequency part of the table
lda (FRESPC),y
and #$0f
clc
adc #14
// We need to print the low nybl char as well
tax
phy_trash_a
jmp has_nybl
#endif // ROM layout
c64/basic/print/print_packed_string.s
+182
View File
@@ -0,0 +1,182 @@
// #LAYOUT# STD * #TAKE
// #LAYOUT# M65 * #TAKE
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
// This routine prints strings that have been packed using the 'generate_trings' tool.
// The general idea is to save space in the ROM.
#if (ROM_LAYOUT_M65 && SEGMENT_BASIC_0)
print_packed_error:
jsr map_BASIC_1
jsr_ind VB1__print_packed_error
jmp map_NORMAL
print_packed_misc_str:
jsr map_BASIC_1
jsr_ind VB1__print_packed_misc_str
jmp map_NORMAL
print_packed_keyword_V2:
jsr map_BASIC_1
jsr_ind VB1__print_packed_keyword_V2
jmp map_NORMAL
#else
print_packed_error: // .X - error string index
lda #<packed_str_errors
ldy #>packed_str_errors
bne print_freq_packed_string // branch always
print_packed_misc_str: // .X - misc string index
lda #<packed_str_misc
ldy #>packed_str_misc
bne print_freq_packed_string // branch always
print_packed_keyword_V2: // .X - token number
lda #<packed_str_keywords_V2
ldy #>packed_str_keywords_V2
// FALLTROUGH
print_freq_packed_string: // not to be used directly XXX rename to print_packed_string
// First store the initial address of the packed strings list
sta FRESPC+0
sty FRESPC+1
// Now find the address of the string to print out
// The implementation assumes no packed string is longer than 255 bytes, including trailing zero
!:
cpx #$00
beq print_freq_packed_string_found // branch if no need to advance anymore
dex
ldy #$00
!:
lda (FRESPC), y
iny
cmp #$00
bne !- // branch if not the end of packed string
clc // advance FREESPC pointer
tya
adc FRESPC+0
sta FRESPC+0
lda #$00
adc FRESPC+1
sta FRESPC+1
jmp !--
print_freq_packed_string_found:
// At this point FRESPC contains a pointer to the string to display
// and we should start from the lower nibble
ldy #$00
// FALLTROUGH
print_freq_packed_string_nibble_lo:
// Get the low nibble and check for the end of packed string
lda (FRESPC), y
and #$0F
beq print_freq_packed_string_end
// Not the end - check if this is a common character, encoded by a single nibble
cmp #$0F
beq print_freq_packed_string_3n_split
// Character is encoded by a single nibble - fetch it and display
tax
lda packed_as_1n-1, x
jsr JCHROUT // preserves .Y
// FALLTROUGH
print_freq_packed_string_nibble_hi:
// Get the high nibble and check for the end of packed string
lda (FRESPC), y
and #$F0
beq print_freq_packed_string_end
// Not the end - check if this is a common character, encoded by a single nibble
cmp #$F0
beq print_freq_packed_string_3n_single
// Character is encoded by a single nibble - fetch it and display
lsr
lsr
lsr
lsr
tax
lda packed_as_1n-1, x
jmp !+ // XXX should be bra on some CPUs
print_freq_packed_string_3n_single:
// Character is encoded using 3 nibbles (1st is a mark, already interpreted)
iny
lda (FRESPC), y
tax
lda packed_as_3n-1, x
!:
jsr JCHROUT // preserves .Y
// Advance to next nibble
iny
bne print_freq_packed_string_nibble_lo // branch always
print_freq_packed_string_3n_split:
// Character is encoded using 3 nibbles (1st is a mark, already interpreted),
// but the 2 remaining nibbles are in two neighbour bytes; use a stack to combine them
lda (FRESPC), y
and #$F0
tsx
pha
iny
lda (FRESPC), y
and #$0F
clc
adc STACK, x
txs // one cycle faster than PLA
tax
lda packed_as_3n-1, x
jsr JCHROUT // preserves .Y
jmp print_freq_packed_string_nibble_hi
print_freq_packed_string_end:
rts
#endif // ROM layout
c64/basic/tokenise_line.s
-306
View File
@@ -1,306 +0,0 @@
// #LAYOUT# STD * #TAKE
// #LAYOUT# M65 * #TAKE
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
// Tokenise a line of BASIC
// Stored at $0200
// Length in __tokenise_work1
tokenise_line:
#if (ROM_LAYOUT_M65 && SEGMENT_BASIC_0)
jsr map_BASIC_1
jsr_ind VB1__tokenise_line
jmp map_NORMAL
#else
// We need to consider each offset in the string to see if it matches
// a token. No BASIC keyword is > 7 bytes, so we need to start with that
// length (or remaining length of string) and count our way back down.
// This means we need separate start and end points to keep track of.
// $0110 = offset into string
// $0111 = number of bytes to pack
// $0112 = offset into tokenised output
// $0113 = length of raw string
lda #$00
sta $0110
sta $0112
sta QTSW
lda __tokenise_work1
sta $0113
tokenise_char:
// Check if we have reached end of the line?
lda $0110
cmp $0113
bne !+
// End of line reached
// Finished tokenising line
// Update __tokenise_work1 and terminate with null
ldx $0112
lda #$00
sta $0200,x
stx __tokenise_work1
// Clear quote mode flag that is also used by KERNAL for screen display
lda #$00
sta QTSW
rts
!:
// More to do
// Maximum keyword length
lda #7
sta $0111
tokenise_char_loop:
// Pack the word from the start (start = X, raw len = __tokenise_work1)
// Returns packed length in __tokenise_work2 and writes the packed
// word at $0100+
ldx $0110
lda $0200,x
cmp #$22
beq tk_quote
cmp #$DE
beq tk_pi
cmp #$3F
bne tk_not_special
// FALLTROUGH
tk_question_mark:
lda #$99
skip_2_bytes_trash_nvz
// FALLTROUGH
tk_pi:
lda #$FF
ldx $0110
sta $0200,x
jmp tk_literal_char
tk_quote:
// Quote
lda QTSW
eor #$ff
sta QTSW
jmp tk_literal_char
tk_not_special:
// Check for literal characters in various ranges
// (This just speeds the tokeniser up, by avoiding the packed token search
// for such characters).
// SPACE and punctuation before +,- and *
cmp #$2a
bcc tk_literal_char
// Don't try to tokenise digits
cmp #$30
bcc tk_might_be_keyword
cmp #$3a
bcs tk_might_be_keyword
jmp tk_literal_char
tk_might_be_keyword:
// Don't tokenise in quote mode
lda QTSW
bne tk_literal_char
// Pack string
ldx $0110
lda $0111
sta __tokenise_work1
// XXX - We should implement an optimisation where we
// trim the last symbol of successively, instead of re-packing the word
// again with one less char each time.
// XXX - We should limit the word length to the length of interesting chars
// that are available.
jsr pack_word
lda __tokenise_work2
cmp #$10
bcc !+
// Packed string too long
jmp do_STRING_TOO_LONG_error
!:
// Search for it in keywords
jsr keyword_search
// Did we find a keyword?
bcc found_token_in_line
// Nope, so try shorter
dec $0111
bne tokenise_char_loop
tk_literal_char:
// Is not a token, so copy to output verbatim
ldx $0110
ldy $0112
lda $0200,x
sta $0200,y
inc $0112
inc $0110
// Tokenise next char
jmp tokenise_char
found_token_in_line:
// Got a token, so write it out
ldx $0112
sta $0200,x
inx
stx $0112
// If it is REM, then lock us in quote mode to the end of the line
cmp #$8f
bne not_rem
sta QTSW
not_rem:
// Now skip over the length of the token
lda $0110
clc
adc $0111
sta $0110
// See if there is more of the line left to process
jmp tokenise_char
// Search for word in keyword list
// The packed form is canonical, so we can search for the keyword while
// it remains compressed. This also saves a little time.
// Algorithm is simply to find matching first bytes, and then look for
// the rest being a match.
keyword_search:
// Search for the compressed keyword stored at $0100 in the list
// of compressed keywords
// __tokenise_work1 = offset in line of input
// __tokenise_work2 = length of token we are matching
// __tokenise_work3 = offset in token list
// __tokenise_work5 = temporary work space
// X and Y registers used as temporary space.
// returns C=1 if no matching token, else C=0 and A=token
lda #$00
sta __tokenise_work1
sta __tokenise_work3
next_kw_offset:
// Advance offset in compressed keyword list, to see if a match
// BEGINS here
// Get count of bytes to compare
lda __tokenise_work2
sta __tokenise_work5
// Load current position
ldy __tokenise_work3
ldx __tokenise_work1
// Advance offset in compressed token list, and stop at the end
inc __tokenise_work3
lda __tokenise_work3
cmp #packed_keyword_table_len
bne next_in_match
jmp done_searching_for_token
next_in_match:
lda $0100,x
cmp packed_keywords,y
bne next_kw_offset
// Advance to next chars in source and target
inx
iny
dec __tokenise_work5 // Have we compared all bytes yet?
bne next_in_match
// Keyword matches!
// Now make sure we started on a keyword boundary
// (we know we finished on one, because of the structure of the compressed
// keyword data).
// Offset 0
// previous byte is $x0
// two byte prior is $FE
// Get offset of first byte of this token
ldx __tokenise_work3
dex // pointer points one late, so rewind it
// At start of list?
cpx #$00
beq word_boundary
dex
// Previous byte is $x0 end token?
lda packed_keywords,x
and #$0f
beq word_boundary
dex
// Uber-previouss byte is $FE final literal token?
lda packed_keywords,x
cmp #$fe
beq word_boundary
// Not a word boundary, so ignore
jmp next_kw_offset
word_boundary:
// Found a token whose offset is in __tokenise_work3
// Now trace back to the start of the compressed keyword list
// to work out what keyword number we are, so that we can return
// the token number.
ldy #$80
token_count_loop:
lda packed_keywords,x
cmp #$fe
bne !+
iny
!:
and #$0f
cmp #$00
bne !+
iny
!:
cpx #$00
beq !+
dex
jmp token_count_loop
!:
tya
clc
rts
done_searching_for_token:
// No token found
sec
lda #$00
rts
#endif // ROM layout
c64/basic/tokenizer/tk_pack.s
+228
View File
@@ -0,0 +1,228 @@
// #LAYOUT# STD * #TAKE
// #LAYOUT# M65 BASIC_1 #TAKE
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
//
// Packs a keyword candidate using a simple compression scheme,
// same as in 'generate_strings.cc'.
//
// This allows us to search through the compressed tokens directly
// using the compressed fragment of BASIC, and thus work out the
// token for a given piece of string.
//
// This also helps ensure that we have an implementation that is totally
// different to that of the original C64 BASIC, as well as likely
// saving a bit of space in the ROM, especially once we start implementing
// larger keyword vocabs for extended BASIC, since the compression and
// decompression machinery is of fixed size.
//
// Input:
// - .X = offset of string to pack, address is BUF ($0200) + .X
// Output:
// - see labels in 'tk_pack'
//
tk_pack:
// Reuse the CPU stack - addresses below $102 are used by 'tokenise_line.s'
.label tk__len_unpacked = $103 // length of unpacked data; it could replace 'tk__nibble_flag', but at the cost of code size/performance
.label tk__shorten_bits = $104 // for quick shortening of packed candidate
.label tk__nibble_flag = $105 // $00 = start from new byte, $FF = start from high nibble
.label tk__byte_offset = $106 // offset of the current byte (to place new data) in tk__packed
.label tk__packed = $107 // packed candidate, 13 bytes is enough for worst case - 8 byte keyword
// Initialize variables
lda #$00
ldy #(tk__packed - tk__len_unpacked + 13) // XXX maybe do not clear that much, just put 0 at the end while compressing
!:
sta tk__len_unpacked, y
dey
bpl !-
// FALLTROUGH
tk_pack_loop:
// Get the next character
lda BUF, x
// If character cannot be a part of a keyword - do not pack anything more
cmp #$23 // '#' - first sane character for a keyword
bcc tk_pack_done // branch if below '#'
cmp #$7B // 'Z' + 1
bcs tk_pack_done // branch if above 'Z'
// Keywords should not contain digits, colons, or semicolons
cmp #$30
bcc !+ // branch if below '0'
cmp #$3C
bcc tk_pack_done // branch if below '<'
!:
// Try to find a nibble to encode the character
ldy #$0E
!:
cmp packed_as_1n-1, y
beq tk_pack_1n
dey
bne !-
// Try to find a 3-nibble sequence to encode the character (1st nibble is always 0xF in this case)
ldy #TK__PACKED_AS_3N
!:
cmp packed_as_3n-1, y
beq tk_pack_3n
dey
bne !-
// If it is not on the list, it cannot be a part of token - quit
// FALLTROUGH
tk_pack_done:
rts
tk_pack_1n:
// .Y contains our nibble; check whether we should start a new encoded byte
bit tk__nibble_flag
bpl tk_pack_1n_new_byte
// Store in the high nibble of the current byte
tya
asl
asl
asl
asl
ldy tk__byte_offset
ora tk__packed, y
sta tk__packed, y
// Adjust remaining counters / flags
inc tk__byte_offset
inc tk__nibble_flag // $FF -> $00
// FALLTROUGH
tk_pack_1n_done:
// Put the bit mark - to indicate 1-nibble encoding
clc
ror tk__shorten_bits
// FALLTROUGH
tk_pack_loop_next:
// Prepare for the next loop iteration - increment counters
inx
inc tk__len_unpacked
// Check length, if it is OK to pack one more byte; quit if not
lda tk__len_unpacked
cmp #TK__MAX_KEYWORD_LEN
bne tk_pack_loop
rts
tk_pack_1n_new_byte:
// Store in the low nibble of the next byte
tya
ldy tk__byte_offset
sta tk__packed, y
// Adjust remaining counters / flags
dec tk__nibble_flag // $00 -> $FF
bmi tk_pack_1n_done
tk_pack_3n:
// .Y contains 2 nibbles to encode; check whether we should start a new encoded byte
bit tk__nibble_flag
bpl tk_pack_3n_new_byte
// Store 1st nibble (alwys 0xF) in the high nibble of the current byte
phy_trash_a
lda #$F0
ldy tk__byte_offset
ora tk__packed, y
sta tk__packed, y
// Store byte in the next byte
iny
pla
sta tk__packed, y
// Adjust remaining counters / flags
iny
sty tk__byte_offset
inc tk__nibble_flag // $FF -> $00
// FALLTROUGH
tk_pack_3n_done:
// Put the bit mark - to indicate 3-nibble encoding
sec
ror tk__shorten_bits
bcc tk_pack_loop_next // branch always
tk_pack_3n_new_byte:
// First store everything in the current byte, we will set '0xF' mark later
tya
ldy tk__byte_offset
sta tk__packed, y
// Now store just the low nibble in the next byte
iny
and #$0F
sta tk__packed, y
// Go back for a moment and set the '0xF' mark in the low nibble
dey
lda tk__packed, y
ora #$0F
sta tk__packed, y
iny
// Adjust remaining counters / flags
iny
sty tk__byte_offset
dec tk__nibble_flag // $00 -> $FF
bmi tk_pack_3n_done // branch always
c64/basic/tokenizer/tk_search.s
+93
View File
@@ -0,0 +1,93 @@
// #LAYOUT# STD * #TAKE
// #LAYOUT# M65 BASIC_1 #TAKE
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
//
// Search for a token on the list pointed by FRESPC
//
// Input:
// - list address in FRESPC, has to be refreshed after routine call
// - uses same variables as 'tk_pack'
// Output:
// - Carry set = not found
// - .X = index of token found
//
tk_search:
ldx #$00 // initial token index
// FALLTROUGH
tk_search_loop:
// Check if end of the list
ldy #$00
lda (FRESPC), y
cmp #$FF
bne tk_search_compare
iny
lda (FRESPC), y
dey
cmp #$FF
bne tk_search_compare
// End of the list
sec
rts
tk_search_compare:
// Compare the packed candidate with the current keyword list entry
lda (FRESPC), y
cmp tk__packed, y
bne tk_search_not_matching
cmp #$00
beq tk_search_found
// For now everything matches, try the next byte
iny
bne tk_search_compare // branch always
tk_search_not_matching:
inx // increment token index
// We need to find the end of the keyword on the list
// and proceed one byte further
cmp #$00
beq tk_search_next
!:
iny
lda (FRESPC), y
bne !-
// FALLTROUGH
tk_search_next:
// Update FRESPC - advance by .Y
iny
tya
clc
adc FRESPC+0
sta FRESPC+0
lda FRESPC+1
adc #$00
sta FRESPC+1
bcc tk_search_loop // branch always
tk_search_found:
clc
rts
c64/basic/tokenizer/tk_shorten.s
+112
View File
@@ -0,0 +1,112 @@
// #LAYOUT# STD * #TAKE
// #LAYOUT# M65 BASIC_1 #TAKE
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
//
// Shortens a keyword candidate previously packed by tk_pack by 1 character
//
// Input:
// - uses same variables as 'tk_pack'
//
tk_shorten:
// First make sure we actually can shorten anything
dec tk__len_unpacked
beq tk_shorten_end
// Now check if the last character was encoded as 1 nibble or 3 nibbles
asl tk__nibble_flag
bcs tk_shorten_3n
// FALLTROUGH
tk_shorten_1n:
ldy tk__byte_offset // will be needed in both cases
// Now we need to check which nibble to cut away
bit tk__nibble_flag
bcs tk_shorten_1n_lo
// FALLTROUGH
tk_shorten_1n_hi: // tk__nibble_flag = $00, both nibbles free
// Go back one byte and clear the high nibble only
dey
sty tk__byte_offset
lda tk__packed, y
and #$F0
sta tk__packed, y
// Adjust remaining counters / flags, and quit
dec tk__nibble_flag // $00 -> $FF
// FALLTROUGH
tk_shorten_end:
rts
tk_shorten_1n_lo: // tk__nibble_flag = $FF, only the high nibble is free
// Clear the whole byte, it will be faster
lda #$00
sta tk__packed, y
// Adjust remaining counters / flags, and quit
inc tk__nibble_flag // $FF -> $00
rts
tk_shorten_3n:
ldy tk__byte_offset // will be needed in both cases
lda #$00
// Now we need to check which nibble to cut away
bit tk__nibble_flag
bcs tk_shorten_3n_2bytes
// FALLTROUGH
tk_shorten_3n_byte_nibble: // tk__nibble_flag = $00, both nibbles free
// Go back one byte, clear it, go back and clear the high nibble only
dey
sta tk__packed, y
// Reuse 'tk_shorten_1n_hi'
sty tk__byte_offset
#if HAS_OPCODES_65C02
bra tk_shorten_1n_hi
#else
jmp tk_shorten_1n_hi
#endif
tk_shorten_3n_2bytes: // tk__nibble_flag = $FF, only the high nibble is free
// Clear the byte, go back, clear one more byte
sta tk__packed, y
dey
sta tk__packed, y
// Adjust remaining counters / flags, and quit
sty tk__byte_offset
inc tk__nibble_flag // $FF -> $00
rts
c64/basic/tokenizer/tokenise_line.s
+176
View File
@@ -0,0 +1,176 @@
// #LAYOUT# STD * #TAKE
// #LAYOUT# M65 * #TAKE
// #LAYOUT# X16 BASIC_0 #TAKE
// #LAYOUT# * * #IGNORE
//
// Tokenise a line of BASIC stored at $0200
//
// Input:
// - length in __tokenise_work1
//
tokenise_line:
#if (ROM_LAYOUT_M65 && SEGMENT_BASIC_0)
jsr map_BASIC_1
jsr_ind VB1__tokenise_line
jmp map_NORMAL
#else
// Reuse the CPU stack - addresses above $101 are used by 'tk_pack.s'
.label tk__offset = $100 // offset into string
.label tk__length = $101 // length of the raw string XXX reuse __tokenise_work1?
// Initialize variables
lda #$00
sta tk__offset
ldx __tokenise_work1
stx tk__length
// Terminate input string with $00
// XXX do we need this?
sta BUF, x
// FALLTROUGH
tokenise_line_loop:
// Check if we have reached end of the line
lda tk__offset
cmp tk__length
beq tokenise_line_done
// Check for some special characters
ldx tk__offset
lda BUF, x
cmp #$22
beq tokenise_line_quote
cmp #$DE
beq tokenise_line_pi
cmp #$3F
beq tokenise_line_question_mark // shortcut for PRINT command
// Try to tokenise
jsr tk_pack
!:
lda tk__len_unpacked
beq tokenise_line_char // branch if attempt to tokenise failed
lda #<packed_str_keywords_V2
sta FRESPC+0
lda #>packed_str_keywords_V2
sta FRESPC+1
jsr tk_search
bcc tokenise_line_keyword_V2 // branch if keyword identified
// XXX add support for additional keyword lists here
// Shorten packed keyword candidate and try again
jsr tk_shorten
#if HAS_OPCODES_65C02
bra !-
#else
jmp !-
#endif
tokenise_line_keyword_V2:
// .X contains a token ID, starting from 0
txa
pha // store the token on the stack, we will need it for REM support
// Store the token
ldx tk__offset
clc
adc #$80
sta BUF, x
// Cut away unnecessary bytes
inc tk__offset
ldx tk__offset
dec tk__len_unpacked // keyword length, afterwards number of bytes to cut away
lda tk__len_unpacked
clc
adc tk__offset
tay
!:
lda BUF, y
sta BUF, x
beq !+
inx
iny
bne !-
!:
// Special handling for REM command - after this one nothing more should be tokenised
pla
cmp #$0F // REM token index
bne tokenise_line_loop
// FALLTROUGH
tokenise_line_done:
// Update line length and quit
lda tk__length
sta __tokenise_work1
rts
tokenise_line_quote:
// For the quote, we advance without tokenizing, until the next quote is found
inc tk__offset
ldx tk__offset
lda BUF, x
beq tokenise_line_done
cmp #$22
beq tokenise_line_char
bne tokenise_line_quote // branch always
tokenise_line_pi:
lda #$FF // token for PI
skip_2_bytes_trash_nvz
// FALLTROUGH
tokenise_line_question_mark:
lda #$99 // token for PRINT
sta BUF, x
// FALLTROUGH
tokenise_line_char:
inc tk__offset
jmp tokenise_line_loop
#endif // ROM layout
tools/generate_strings.cc
+987
View File
@@ -0,0 +1,987 @@
//
// Utility to generate compressed messages and BASIC tokens
//
// XXX parse the config file to select the string set (STD, M65, X16) and generate feature string
// XXX use DualStream class for logging
// XXX use dictionary compression by finding the set of non-overlapping strings that can be concatenated
// to produce full set of strings; some idea for the algorithm (not sure if proper one) is available here
// https://stackoverflow.com/questions/9195676/finding-the-smallest-number-of-substrings-to-represent-a-set-of-strings
// needs some investigation...
#include "common.h"
#include <unistd.h>
#include <algorithm>
#include <fstream>
#include <iomanip>
#include <sstream>
#include <map>
#include <vector>
//
// Command line settings
//
std::string CMD_outFile = "out.s";
//
// Type definition for text/tokens to generate
//
typedef struct StringEntry
{
bool enabledSTD; // whether enabled for basic build
bool enabledM65; // whether enabled for Mega65 build
bool enabledX16; // whether enabled for Commander X16 build
std::string alias; // alias, for the assembler
std::string string; // string/token
uint8_t abbrevLen = 0; // length of token abbreviation
} StringEntry;
enum class ListType
{
KEYWORDS,
STRINGS_BASIC
};
typedef struct StringEntryList
{
ListType type;
std::string name;
std::vector<StringEntry> list;
} StringEntryList;
typedef std::vector<uint8_t> StringEncoded;
typedef std::vector<StringEncoded> StringEncodedList;
// http://www.classic-games.com/commodore64/cbmtoken.html
// https://www.c64-wiki.com/wiki/BASIC_token
// BASIC keywords - V2 dialect
/*
The URLs here provide examples of other BASIC dialects that include the specified commands
or keywords.
All except VERIFY are known with certainty to be part of other BASIC dialects, and thus
cannot be subject to copyright infringement when implemented on a C64.
The VERIFY Command however appears in other programming languages, and as a single word,
is the obvious complement to LOAD and SAVE when considering the following storage functions:
1. Load a file from storage into memory.
2. Save a file from memory to storage.
3. Verify that a file has been correctly written from memory to storage.
Note that verify is the key verb in the behaviour of this function. Therefore it is the
obvious choice for the name for such a function, taking into account the convention of
LOAD and SAVE. Indeed, that this is so is highlighted by the fact that it was used in the
C64 ROM. Therefore there is no exercise of creativity here, and a reasonable person would
likely select this name independently. Also, single words cannot be copyright.
Further, the list is a list of facts, and thus not copyrightable.
We sourced the list from a public source, https://www.c64-wiki.com/wiki/BASIC, which has
been public on the internet for a long time, as have many other similar sources.
That website offers the page under the GFDL license, and we can therefore take the list
under those license terms if we wish, should the need arise.
Note that in providing these justifications above, we do not in any way acknowledge that
without these conditions that the BASIC keyword list could be copyright, but provide this
information as a further layer of defence against any claims that it could somehow
infringe on the copyrights of the C64/C65/C128 etc ROMs.
*/
const StringEntryList GLOBAL_Keywords_V2 = { ListType::KEYWORDS, "keywords_V2",
{
// STD M65 X16
{ true, true, true, "KV2_80", "END", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_81", "FOR", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_82", "NEXT", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_83", "DATA", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_84", "INPUT#", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php
{ true, true, true, "KV2_85", "INPUT", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_86", "DIM", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_87", "READ", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_88", "LET", 2 }, // https://en.wikipedia.org/wiki/Atari_BASIC
{ true, true, true, "KV2_89", "GOTO", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_8A", "RUN", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_8B", "IF", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_8C", "RESTORE", 3 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_8D", "GOSUB", 3 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_8E", "RETURN", 3 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_8F", "REM", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_90", "STOP", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_91", "ON", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_92", "WAIT", 2 }, // http://www.picaxe.com/BASIC-Commands/Time-Delays/wait/
{ true, true, true, "KV2_93", "LOAD", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_94", "SAVE", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_95", "VERIFY", 2 }, // https://en.wikipedia.org/wiki/Sinclair_BASIC
{ true, true, true, "KV2_96", "DEF", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_97", "POKE", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_98", "PRINT#", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php
{ true, true, true, "KV2_99", "PRINT", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_9A", "CONT", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_9B", "LIST", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_9C", "CLR", 2 }, // Apple I Replica Creation: Back to the Garage, p125
{ true, true, true, "KV2_9D", "CMD", 2 }, // https://en.wikipedia.org/wiki/List_of_DOS_commands
{ true, true, true, "KV2_9E", "SYS", 2 }, // https://www.lifewire.com/dos-commands-4070427
{ true, true, true, "KV2_9F", "OPEN", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php
// STD M65 X16
{ true, true, true, "KV2_A0", "CLOSE", 3 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php
{ true, true, true, "KV2_A1", "GET", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_A2", "NEW", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_A3", "TAB(", 2 }, // http://www.antonis.de/qbebooks/gwbasman/tab.html
{ true, true, true, "KV2_A4", "TO", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_A5", "FN", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_A6", "SPC(", 2 }, // http://www.antonis.de/qbebooks/gwbasman/spc.html
{ true, true, true, "KV2_A7", "THEN", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_A8", "NOT", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_A9", "STEP", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_AA", "+", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_AB", "-", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_AC", "*", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_AD", "/", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_AE", "^", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_AF", "AND", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_B0", "OR", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_B1", ">", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_B2", "=", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_B3", "<", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_B4", "SGN", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_B5", "INT", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_B6", "ABS", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_B7", "USR", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_B8", "FRE", 2 }, // http://www.antonis.de/qbebooks/gwbasman/fre.html
{ true, true, true, "KV2_B9", "POS", 0 }, // http://www.antonis.de/qbebooks/gwbasman/pos.html
{ true, true, true, "KV2_BA", "SQR", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_BB", "RND", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_BC", "LOG", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_BD", "EXP", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_BE", "COS", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_BF", "SIN", 2 }, // https://www.landsnail.com/a2ref.htm
// STD M65 X16
{ true, true, true, "KV2_C0", "TAN", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_C1", "ATN", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_C2", "PEEK", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_C3", "LEN", 0 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_C4", "STR$", 3 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_C5", "VAL", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_C6", "ASC", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_C7", "CHR$", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_C8", "LEFT$", 3 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_C9", "RIGHT$", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_CA", "MID$", 2 }, // https://www.landsnail.com/a2ref.htm
{ true, true, true, "KV2_CB", "GO", 0 }, // https://en.wikipedia.org/wiki/Goto
} };
// extended BASIC keywords - reserved for generic (hardware independent) BASIC commands
const StringEntryList GLOBAL_Keywords_CC = { ListType::KEYWORDS, "keywords_CC",
{
// STD M65 X16
{ false, false, false, "KCC_01", "BLOAD", }, // http://www.antonis.de/qbebooks/gwbasman/bload.html
{ false, false, false, "KCC_02", "BSAVE", }, // http://www.antonis.de/qbebooks/gwbasman/bload.html
{ false, false, false, "KCC_03", "BVERIFY", },
} };
// extended BASIC keywords - reserved for BASIC commands likely making sense on most C64-compatible machines
const StringEntryList GLOBAL_Keywords_CD = { ListType::KEYWORDS, "keywords_CD",
{
// STD M65 X16
{ false, false, false, "KCD_01", "SLOW", }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant
{ false, false, false, "KCD_02", "FAST", }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant
} };
// extended BASIC keywords - reserved for board-specific BASIC commands
const StringEntryList GLOBAL_Keywords_CE = { ListType::KEYWORDS, "keywords_CE",
{
// STD M65 X16
{ false, false, false, "KCE_01", "TESTCMD", },
} };
const StringEntryList GLOBAL_Keywords_CF = { ListType::KEYWORDS, "keywords_CF",
{
// STD M65 X16
{ false, false, false, "KCF_01", "TESTCMD", },
} };
// extended BASIC keywords - reserved for generic (hardware independent) BASIC functions
const StringEntryList GLOBAL_Keywords_D0 = { ListType::KEYWORDS, "keywords_D0",
{
// STD M65 X16
{ false, false, false, "KD0_01", "TESTFUN", },
} };
// extended BASIC keywords - reserved for BASIC functions likely making sense on most C64-compatible machines
const StringEntryList GLOBAL_Keywords_D1 = { ListType::KEYWORDS, "keywords_D1",
{
// STD M65 X16
{ false, false, false, "KD1_01", "TESTFUN", },
} };
// extended BASIC keywords - reserved for board-specific BASIC functions
const StringEntryList GLOBAL_Keywords_D2 = { ListType::KEYWORDS, "keywords_D2",
{
// STD M65 X16
{ false, false, false, "KD2_01", "TESTFUN", },
} };
const StringEntryList GLOBAL_Keywords_D3 = { ListType::KEYWORDS, "keywords_D3",
{
// STD M65 X16
{ false, false, false, "KD3_01", "TESTFUN", },
} };
// BASIC errors - all dialects
/*
These error messages are generally so well known, that it seems silly to have to even point out that they are
so widely used and known, if not completely genericised.
The most of them are inherited from the MICROSOFT BASIC on which the C64's BASIC is based, so for those, there
cannot even be any claim to copyright over them in any exclusive way.
However, copyright of such short phrases that express ideas in short form cannot be copyrighted:
https://fairuse.stanford.edu/2003/09/09/copyright_protection_for_short/
*/
const StringEntryList GLOBAL_Errors = { ListType::STRINGS_BASIC, "errors",
{
// STD M65 X16 --- error strings compatible with CBM BASIC V2
{ true, true, true, "EV2_01", "TOO MANY FILES" },
{ true, true, true, "EV2_02", "FILE OPEN" },
{ true, true, true, "EV2_03", "FILE NOT OPEN" },
{ true, true, true, "EV2_04", "FILE NOT FOUND" },
{ true, true, true, "EV2_05", "DEVICE NOT PRESENT" },
{ true, true, true, "EV2_06", "NOT INPUT FILE" },
{ true, true, true, "EV2_07", "NOT OUTPUT FILE" },
{ true, true, true, "EV2_08", "MISSING FILENAME" },
{ true, true, true, "EV2_09", "ILLEGAL DEVICE NUMBER" },
{ true, true, true, "EV2_0A", "NEXT WITHOUT FOR" },
{ true, true, true, "EV2_0B", "SYNTAX" },
{ true, true, true, "EV2_0C", "RETURN WITHOUT GOSUB" },
{ true, true, true, "EV2_0D", "OUT OF DATA" },
{ true, true, true, "EV2_0E", "ILLEGAL QUANTITY" },
{ true, true, true, "EV2_0F", "OVERFLOW" },
{ true, true, true, "EV2_10", "OUT OF MEMORY" },
{ true, true, true, "EV2_11", "UNDEF\'D STATEMENT" },
{ true, true, true, "EV2_12", "BAD SUBSCRIPT" },
{ true, true, true, "EV2_13", "REDIM\'D ARRAY" },
{ true, true, true, "EV2_14", "DIVISION BY ZERO" },
{ true, true, true, "EV2_15", "ILLEGAL DIRECT" },
{ true, true, true, "EV2_16", "TYPE MISMATCH" },
{ true, true, true, "EV2_17", "STRING TOO LONG" },
{ true, true, true, "EV2_18", "FILE DATA" },
{ true, true, true, "EV2_19", "FORMULA TOO COMPLEX" },
{ true, true, true, "EV2_1A", "CAN\'T CONTINUE" },
{ true, true, true, "EV2_1B", "UNDEF\'D FUNCTION" },
{ true, true, true, "EV2_1C", "VERIFY" },
{ true, true, true, "EV2_1D", "LOAD" },
{ true, true, true, "EV2_1E", "BREAK" },
// STD M65 X16 --- error strings compatible with CBM BASIC V7
{ false, false, false, "EV7_1F", "CAN'T RESUME" }, // not used for now
{ false, false, false, "EV7_20", "LOOP NOT FOUND" }, // not used for now
{ false, false, false, "EV7_21", "LOOP WITHOUT DO" }, // not used for now
{ false, false, false, "EV7_22", "DIRECT MODE ONLY" }, // not used for now
{ false, false, false, "EV7_23", "NO GRAPHICS AREA" }, // not used for now
{ false, false, false, "EV7_24", "BAD DISK" }, // not used for now
{ false, false, false, "EV7_25", "BEND NOT FOUND" }, // not used for now
{ false, false, false, "EV7_26", "LINE NUMBER TOO LARGE" }, // not used for now
{ false, false, false, "EV7_27", "UNRESOLVED REFERENCE" }, // not used for now
{ true, true, true, "EV7_28", "NOT IMPLEMENTED" }, // this message actually differs rom CBM one
{ false, false, false, "EV7_29", "FILE READ" }, // not used for now
// STD M65 X16 --- error strings specific to OpenROMs, not present in CBM BASIC dialects
{ true, true, true, "EOR_2A", "MEMORY CORRUPT" },
} };
// BASIC errors - miscelaneous strings
const StringEntryList GLOBAL_MiscStrings = { ListType::STRINGS_BASIC, "misc",
{
// STD M65 X16 --- misc strings as on CBM machines
{ true, true, true, "STR_BYTES", " BASIC BYTES FREE" }, // https://github.com/stefanhaustein/expressionparser
{ true, true, true, "STR_READY", "READY.\r" }, // https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconc_whatistsonative.htm https://github.com/stefanhaustein/expressionparser
{ true, true, true, "STR_ERROR", " ERROR" }, // simply the word error that is attached to the other parts of messages https://fjkraan.home.xs4all.nl/comp/apple2faq/app2asoftfaq.html
{ true, true, true, "STR_IN", " IN " },
// STD M65 X16 --- misc strings specific to OpenROMs, not present in CBM ROMs
{ true, true, true, "STR_BRK_AT", "BRK AT $" },
} };
/* NOTE: below are the BASIC V7/V10 keywords, see:
- https://sites.google.com/site/h2obsession/CBM/basic/tokens
- http://www.zimmers.net/anonftp/pub/cbm/programming/cbm-basic-tokens.txt
"RGR" // $CC
"RCLR" // $CD
"JOY" // $CF // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
"RDOT" // $D0
"DEC" // $D1 // Amos Professional Manual, Command Index
"HEX$" // $D2 // http://www.antonis.de/qbebooks/gwbasman/hexs.html
"ERR$" // $D3 // Amos Professional Manual, Command Index
"INSTR" // $D4 // Amos Professional Manual, Command Index
"ELSE" // $D5 // Amos Professional Manual, Command Index
"RESUME" // $D6 // Amos Professional Manual, Command Index
"TRAP" // $D7 // Amos Professional Manual, Command Index
"TRON" // $D8 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
"TROFF" // $D9 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
"SOUND" // $DA // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
"VOL" // $DB
"AUTO" // $DC // http://www.antonis.de/qbebooks/gwbasman/auto.html
"PUDEF" // $DD
"GRAPHIC" // $DE
"PAINT" // $DF // Amos Professional Manual, Command Index
"CHAR" // $E0 // https://www.c64-wiki.com/wiki/Screen_Graphics_64
"BOX" // $E1 // Amos Professional Manual, Command Index
"CIRCLE" // $E2 // https://en.wikipedia.org/wiki/Sinclair_BASIC
"PASTE" // $E3 // Amos Professional Manual, Command Index
"CUT" // $E4
"LINE" // $E5 // https://en.wikipedia.org/wiki/Sinclair_BASIC
"LOCATE" // $E6 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
"COLOR" // $E7
"SCNCLR" // $E8
"SCALE" // $E9 // https://www.c64-wiki.com/wiki/Super_Expander_64
"HELP" // $EA // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7
"DO" // $EB // Amos Professional Manual, Command Index
"LOOP" // $EC // Amos Professional Manual, Command Index
"EXIT" // $ED // Amos Professional Manual, Command Index
"DIR" // $EE // Amos Professional Manual, Command Index
"DSAVE" // $EF // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"DLOAD" // $F0 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"HEADER" // $F1 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"SCRATCH" // $F2 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"COLLECT" // $F3
"COPY" // $F4 // https://en.wikipedia.org/wiki/Sinclair_BASIC
"RENAME" // $F5 // Amos Professional Manual, Command Index
"BACKUP" // $F6 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"DELETE" // $F7 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"RENUMBER" // $F8 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"KEY" // $F9 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
"MONITOR" // $FA // Amos Professional Manual, Command Index
"USING" // $FB // http://www.antonis.de/qbebooks/gwbasman/printusing.html
"UNTIL" // $FC // Amos Professional Manual, Command Index
"WHILE" // $FD // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
"POT" // $CE $02
"BUMP" // $CE $03
"PEN" // $CE $04 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html
"RSPPOS" // $CE $05 // https://www.c64-wiki.com/wiki/Super_Expander_64
"RSPRITE" // $CE $06
"RSPCOLOR" // $CE $07
"XOR" // $CE $08
"RWINDOW" // $CE $09
"POINTER" // $CE $0A
"BANK" // $FE $02 // Amos Professional Manual, Command Index
"FILTER" // $FE $03 // https://www.c64-wiki.com/wiki/Super_Expander_64
"PLAY" // $FE $04 // Amos Professional Manual, Command Index
"TEMPO" // $FE $05 // Amos Professional Manual, Command Index
"MOVSPR" // $FE $06
"SPRITE" // $FE $07 // Amos Professional Manual, Command Index
"SPRCOLOR" // $FE $08
"RREG" // $FE $09
"ENVELOPE" // $FE $0A
"SLEEP" // $FE $0B
"CATALOG" // $FE $0C // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"DOPEN" // $FE $0D
"APPEND" // $FE $0E // Amos Professional Manual, Command Index
"DCLOSE" // $FE $0F
"BSAVE" // $FE $10 // http://www.antonis.de/qbebooks/gwbasman/bsave.html
"BLOAD" // $FE $11 // http://www.antonis.de/qbebooks/gwbasman/bload.html
"RECORD" // $FE $12
"CONCAT" // $FE $13
"DVERIFY" // $FE $14 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"DCLEAR" // $FE $15
"SPRSAV" // $FE $16 // https://www.c64-wiki.com/wiki/Super_Expander_64
"COLLISION" // $FE $17
"BEGIN" // $FE $18
"BEND" // $FE $19
"WINDOW" // $FE $1A // http://www.antonis.de/qbebooks/gwbasman/window.html
"BOOT" // $FE $1B
"WIDTH" // $FE $1C // http://www.antonis.de/qbebooks/gwbasman/width.html
"SPRDEF" // $FE $1D // https://www.c64-wiki.com/wiki/Super_Expander_64
"QUIT" // $FE $1E // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7
"STASH" // $FE $1F - v7
"FETCH" // $FE $21 - v7
"SWAP" // $FE $23 - v7 // http://www.antonis.de/qbebooks/gwbasman/swap.html
"DMA" // $FE $17/$21/$23 - v10
"OFF" // $FE $24 // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7
"FAST" // $FE $25 // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant
"SLOW" // $FE $26 // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant
"TYPE" // $FE $27
"BVERIFY" // $FE $28
"DIRECTORY" // $FE $29 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"ERASE" // $FE $2A // https://en.wikipedia.org/wiki/Sinclair_BASIC
"FIND" // $FE $2B // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic)
"CHANGE" // $FE $2C
"SET" // $FE $2D
"SCREEN" // $FE $2E // http://www.antonis.de/qbebooks/gwbasman/screens.html
"POLYGON" // $FE $2F // Amos Professional Manual, Command Index
"ELLIPSE" // $FE $30 // Amos Professional Manual, Command Index
"VIEWPORT" // $FE $31
"GCOPY" // $FE $32
"PEN" // $FE $33 // Amos Professional Manual, Command Index
"PALETTE" // $FE $34 // Amos Professional Manual, Command Index
"DMODE" // $FE $35
"DPAT" // $FE $36
"PIC" // $FE $37
"GENLOCK" // $FE $38
"FOREGROUND" // $FE $39
"BACKGROUND" // $FE $3B
"BORDER" // $FE $3C // https://en.wikipedia.org/wiki/Sinclair_BASIC
"HIGHLIGHT" // $FE $3D
*/
//
// Work class definitions
//
class DataSet
{
public:
void addStrings(const StringEntryList &stringList);
const std::string &getOutput();
private:
void process();
void calculateFrequencies();
void encodeStrings();
void encodeByFreq(const std::string &plain, StringEncoded &encoded) const;
void prepareOutput();
void putCharEncoding(std::ostringstream &stream, uint8_t idx, char character, bool is3n);
virtual bool isRelevant(const StringEntry &entry) const = 0;
virtual std::string layoutName() const = 0;
std::vector<StringEntryList> stringEntryLists;
std::vector<StringEncodedList> stringEncodedLists;
std::vector<char> as1n; // list of bytes to be encoded as 1 nibble
std::vector<char> as3n; // list of bytes to be encoded as 3 nibbles
uint8_t tk__packed_as_3n = 0;
uint8_t tk__max_keyword_len = 0;
size_t maxAliasLen = 0;
std::string outFileContent;
};
class DataSetSTD : public DataSet
{
bool isRelevant(const StringEntry &entry) const { return entry.enabledSTD; }
std::string layoutName() const { return "STD"; }
};
class DataSetM65 : public DataSet
{
bool isRelevant(const StringEntry &entry) const { return entry.enabledM65; }
std::string layoutName() const { return "M65"; }
};
class DataSetX16 : public DataSet
{
bool isRelevant(const StringEntry &entry) const { return entry.enabledX16; }
std::string layoutName() const { return "X16"; }
};
//
// Work class implementation
//
void DataSet::addStrings(const StringEntryList &stringList)
{
// Import the new list of strings
stringEntryLists.push_back(stringList);
// Clear strings not relevant for the current configuration
while (1)
{
if (isRelevant(stringEntryLists.back().list.back())) break;
stringEntryLists.back().list.pop_back();
if (stringEntryLists.back().list.empty())
{
ERROR(std::string("no valid strings in layout '") + layoutName() + "', list'" + stringEntryLists.back().name + "'");
}
}
// Clear the output content - make sure it is not valid anymore
outFileContent.clear();
}
void DataSet::process()
{
std::cout << "Processing layout '" << layoutName() << "'" << std::endl;
calculateFrequencies();
encodeStrings();
prepareOutput();
}
const std::string &DataSet::getOutput()
{
if (outFileContent.empty())
{
process();
}
return outFileContent;
}
void DataSet::calculateFrequencies()
{
as1n.clear();
as3n.clear();
std::map<char, uint16_t> freqMapGeneral; // general character frequency map
std::map<char, uint16_t> freqMapKeywords; // frequency map for keywords
// Calculate frequencies of characters in the strings
for (const auto &stringEntryList : stringEntryLists)
{
for (const auto &stringEntry : stringEntryList.list)
{
if (!isRelevant(stringEntry)) continue;
// Check for maximum allowed string length
if (stringEntry.string.length() > 255) ERROR("string cannot be longer than 255 characters");
// Check mor maximum keyword length
if (stringEntryList.type == ListType::KEYWORDS)
{
if (stringEntry.string.length() > 8) ERROR("keyword cannot be longer than 8 characters");
tk__max_keyword_len = std::max(tk__max_keyword_len, (uint8_t) stringEntry.string.length());
}
// Update maximum alias length too
maxAliasLen = std::max(maxAliasLen, stringEntry.alias.length());
for (const auto &character : stringEntry.string)
{
if (character >= 0x80)
{
ERROR(std::string("character above 0x80 in string '") + stringEntry.string + "'");
}
freqMapGeneral[character]++;
if (stringEntryList.type == ListType::KEYWORDS) freqMapKeywords[character]++;
}
}
}
// Sort characters by frequency
std::vector<char> freqVector1;
for (auto iter = freqMapGeneral.begin(); iter != freqMapGeneral.end(); ++iter)
{
freqVector1.push_back(iter->first);
}
std::sort(freqVector1.begin(), freqVector1.end(), [&freqMapGeneral](char e1, char e2)
{ return freqMapGeneral[e2] < freqMapGeneral[e1]; });
// Check if minimal amount of characters needed, below 15 is not supported by the 6502 side code
if (freqVector1.size() < 15) ERROR(std::string("not enough distinct characters in layout '") + layoutName() + "', at least 15 needed");
// Extract 14 most frequent characters to be encoded as 1 nibble
for (uint8_t idx = 0; idx < 14; idx++)
{
as1n.push_back(freqVector1[idx]);
}
// Now sort them by frequency in keywords, in descending order - this will speed up the tokenizer a little
std::sort(as1n.begin(), as1n.end(), [&freqMapKeywords](char e1, char e2) { return freqMapKeywords[e2] > freqMapKeywords[e1]; });
// Extract characters to be encoded as 3 nibbles, which actually exist in keywords
std::vector<char> freqVector2;
tk__packed_as_3n = 0;
for (uint8_t idx = 14; idx < freqVector1.size(); idx++)
{
const auto &character = freqVector1[idx];
if (freqMapKeywords[character] > 0)
{
as3n.push_back(character);
tk__packed_as_3n++;
}
else
{
freqVector2.push_back(character);
}
}
tk__packed_as_3n = std::max(tk__packed_as_3n, (uint8_t) 1);
// Again, sort them by frequency in keywords, in descending order - this will speed up the tokenizer a little
std::sort(as3n.begin(), as3n.end(), [&freqMapKeywords](char e1, char e2) { return freqMapKeywords[e2] > freqMapKeywords[e1]; });
// Finally extract the remaining characters to be encoded as 3 nibbles
for (const auto &character : freqVector2)
{
as3n.push_back(character);
}
}
void DataSet::encodeByFreq(const std::string &plain, StringEncoded &encoded) const
{
bool fullByte = true;
auto push1n = [&](uint8_t val) // push 1 nibble - encoded character or 0xF mark
{
if (fullByte)
{
encoded.push_back(val);
fullByte = false;
}
else
{
encoded.back() += val * 0x10;
fullByte = true;
}
};
auto push2n = [&](uint8_t val) // push the remaining nibbles for 3-niobble encoded characters
{
if (fullByte)
{
encoded.push_back(val);
}
else
{
// Encode byte in a way to be easily decoded by 6502
encoded.back() += (val / 0x10) * 0x10;
encoded.push_back(val % 0x10);
}
};
// Encode every single character by frequency, put them in the output vector
for (const char &character : plain)
{
auto iterEncoding1 = std::find(as1n.begin(), as1n.end(), character);
if (iterEncoding1 != as1n.end())
{
push1n(std::distance(as1n.begin(), iterEncoding1) + 1);
}
else
{
push1n(0x0F);
auto iterEncoding3 = std::find(as3n.begin(), as3n.end(), character);
if (iterEncoding3 == as3n.end())
{
ERROR("internal error in 'encodeByFreq'");
}
push2n(std::distance(as3n.begin(), iterEncoding3) + 1);
}
}
// Make sure the last byte of encoded stream is 0
if (encoded.size() == 0 || encoded.back() != 0) encoded.push_back(0);
}
void DataSet::encodeStrings()
{
stringEncodedLists.clear();
// Encode every relevant string from every list - by character frequency
for (const auto &stringEntryList : stringEntryLists)
{
stringEncodedLists.emplace_back();
auto &stringEncodedList = stringEncodedLists.back();
for (const auto &stringEntry : stringEntryList.list)
{
stringEncodedList.emplace_back();
auto &stringEncoded = stringEncodedList.back();
if (isRelevant(stringEntry))
{
encodeByFreq(stringEntry.string, stringEncoded);
}
}
}
}
void DataSet::putCharEncoding(std::ostringstream &stream, uint8_t idx, char character, bool is3n)
{
stream << "\t.byte $" << std::uppercase << std::hex <<
std::setfill('0') << std::setw(2) << +character <<
" // " << std::setfill(is3n ? '0' : ' ') << std::setw(2) << +idx;
std::string petscii;
if (character == 0x20)
{
petscii = " = SPACE";
}
else if (character == 0x27)
{
petscii = " = APOSTROPHE";
}
else if (character == 0x0D)
{
petscii = " = RETURN";
}
else if (character > ' ' && character < 'a')
{
petscii = std::string(" = '") + character + "'";
}
stream << petscii << std::endl;
}
void DataSet::prepareOutput()
{
// Convert our encoded strings to a KickAssembler source
std::ostringstream stream;
stream << std::endl << "#if ROM_LAYOUT_" << layoutName() << std::endl;
uint8_t idx;
// Export all nibble-encoded characters
stream << std::endl << ".macro put_packed_as_1n() // characters encoded as 1 nibble" << std::endl << "{" << std::endl;
idx = 0;
for (const auto& encoding : as1n)
{
putCharEncoding(stream, ++idx, encoding, false);
}
stream << "}" << std::endl;
// Export all byte-encoded characters
stream << std::endl << ".macro put_packed_as_3n() // characters encoded as 3 nibbles" << std::endl << "{" << std::endl;
idx = 0;
for (const auto& encoding : as3n)
{
if (idx == tk__packed_as_3n) stream << std::endl <<
"\t// Characters below are not used by any BASIC keyword" << std::endl << std::endl;
putCharEncoding(stream, ++idx, encoding, true);
}
stream << "}" << std::endl;
// Export additional data for the tokenizer
stream << std::endl << ".label TK__PACKED_AS_3N = $" << std::hex << +tk__packed_as_3n <<
std::endl << ".label TK__MAX_KEYWORD_LEN = " << std::dec << +tk__max_keyword_len << std::endl;
// Export frequency-encoded strings
for (uint8_t idxList = 0; idxList < stringEntryLists.size(); idxList++)
{
const auto &stringEntryList = stringEntryLists[idxList];
const auto &stringEncodedList = stringEncodedLists[idxList];
stream << std::endl;
for (uint8_t idxString = 0; idxString < stringEncodedList.size(); idxString++)
{
const auto &stringEntry = stringEntryList.list[idxString];
const auto &stringEncoded = stringEncodedList[idxString];
if (!stringEncoded.empty())
{
stream << ".label IDX__" << stringEntry.alias << std::string(maxAliasLen - stringEntry.alias.length(), ' ') <<
" = $" << std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +idxString << std::endl;
}
}
stream << std::endl << ".macro put_freq_packed_" << stringEntryList.name << "()" << std::endl << "{" << std::endl;
enum LastStr { NONE, SKIPPED, WRITTEN } lastStr = LastStr::NONE;
for (uint8_t idxString = 0; idxString < stringEncodedList.size(); idxString++)
{
const auto &stringEntry = stringEntryList.list[idxString];
const auto &stringEncoded = stringEncodedList[idxString];
if (stringEncoded.empty())
{
if (lastStr == LastStr::WRITTEN) stream << std::endl;
stream << "\t.byte $00 // skipped " << stringEntry.alias << std::endl;
lastStr = LastStr::SKIPPED;
}
else
{
if (lastStr != LastStr::NONE) stream << std::endl;
stream << "\t// IDX__" << stringEntry.alias << std::endl;
stream << "\t.byte ";
bool first = true;
for (const auto &charEncoded : stringEncoded)
{
if (first)
{
first = false;
}
else
{
stream << ", ";
}
stream << "$" << std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +charEncoded;
}
stream << std::endl;
lastStr = LastStr::WRITTEN;
}
}
// For the token list - pu the 'end of keywords' mark
if (stringEntryList.type == ListType::KEYWORDS)
{
stream << std::endl << "\t// Marker - end of the keyword list" << std::endl;
stream << "\t.byte $FF, $FF" << std::endl;
}
stream << "}" << std::endl;
}
// Finalize the file stream
stream << std::endl << "#endif // ROM_LAYOUT_" << layoutName() << std::endl;
outFileContent = stream.str();
}
//
// Common helper functions
//
void printUsage()
{
std::cout << "\n" <<
"usage: generate_strings [-o <out file>]" << "\n\n";
}
void printBanner()
{
printBannerLineTop();
std::cout << "// Generating compressed messages and BASIC tokens" << "\n";
printBannerLineBottom();
}
void parseCommandLine(int argc, char **argv)
{
int opt;
// Retrieve command line options
while ((opt = getopt(argc, argv, "o:")) != -1)
{
switch(opt)
{
case 'o': CMD_outFile = optarg; break;
default: printUsage(); ERROR();
}
}
}
void writeStrings()
{
DataSetSTD dataSetSTD;
DataSetM65 dataSetM65;
DataSetX16 dataSetX16;
// Add input data to computation objects
dataSetSTD.addStrings(GLOBAL_Keywords_V2);
dataSetSTD.addStrings(GLOBAL_Errors);
dataSetSTD.addStrings(GLOBAL_MiscStrings);
dataSetM65.addStrings(GLOBAL_Keywords_V2);
dataSetM65.addStrings(GLOBAL_Errors);
dataSetM65.addStrings(GLOBAL_MiscStrings);
dataSetX16.addStrings(GLOBAL_Keywords_V2);
dataSetX16.addStrings(GLOBAL_Errors);
dataSetX16.addStrings(GLOBAL_MiscStrings);
// Retrieve the results
std::string outputSTD = dataSetSTD.getOutput();
std::string outputM65 = dataSetM65.getOutput();
std::string outputX16 = dataSetX16.getOutput();
// Remove old file
unlink(CMD_outFile.c_str());
// Open output file for writing
std::ofstream outFile(CMD_outFile, std::fstream::out | std::fstream::trunc);
if (!outFile.good()) ERROR(std::string("can't open oputput file '") + CMD_outFile + "'");
// Write header
outFile << "//\n// Generated file - do not edit\n//";
// Write packed strings
outFile << std::endl << std::endl;
outFile << outputSTD;
outFile << std::endl << std::endl;
outFile << outputM65;
outFile << std::endl << std::endl;
outFile << outputX16;
outFile << std::endl << std::endl;
// Close the file
outFile.close();
std::cout << std::string("Compressed strings written to: ") + CMD_outFile + "\n\n";
}
//
// Main function
//
int main(int argc, char **argv)
{
printBanner();
parseCommandLine(argc, argv);
writeStrings();
return 0;
}
tools/compress_text.c → tools/outdated/compress_text.c
View File
File renamed without changes.