diff --git a/Makefile b/Makefile index 7f0b7ccb..d65bbff7 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ SRCDIR_BASIC = $(SRCDIR_COMMON) \ c64/basic/math_mov \ c64/basic/print \ c64/basic/rom_revision \ + c64/basic/tokenizer \ c64/basic/wedge SRCDIR_KERNAL = $(SRCDIR_COMMON) \ @@ -54,7 +55,7 @@ SRC_TOOLS = $(wildcard tools/*.c,tools/*.cc) # Generated files -GEN_BASIC = build/,generated/packed_messages.s build/,generated/float_constants.s +GEN_BASIC = build/,generated/packed_strings.s build/,generated/float_constants.s GEN_KERNAL = # List of build directories @@ -68,12 +69,12 @@ DIR_X16 = build/target_cx16 # List of config files -CFG_CUS = c64/,,config_custom.s -CFG_GEN = c64/,,config_generic.s -CFG_TST = c64/,,config_testing.s -CFG_M65 = c64/,,config_mega65.s -CFG_U64 = c64/,,config_ultimate64.s -CFG_X16 = c64/,,config_cx16.s +CFG_CUS = c64/,,config_custom.s +CFG_GEN = c64/,,config_generic.s +CFG_TST = c64/,,config_testing.s +CFG_M65 = c64/,,config_mega65.s +CFG_U64 = c64/,,config_ultimate64.s +CFG_X16 = c64/,,config_cx16.s # Dependencies - helper variables @@ -83,8 +84,8 @@ DEP_KERNAL = $(SRC_KERNAL) $(SRCDIR_KERNAL) $(GEN_KERNAL) # List of tools TOOL_COLLECT_DATA = build/tools/collect_data -TOOL_COMPRESS_TEXT = build/tools/compress_text TOOL_GENERATE_CONSTANTS = build/tools/generate_constants +TOOL_GENERATE_STRINGS = build/tools/generate_strings TOOL_PATCH_CHARGEN = build/tools/patch_chargen TOOL_PNGPREPARE = build/tools/pngprepare TOOL_BUILD_SEGMENT = build/tools/build_segment @@ -166,19 +167,15 @@ updatebin: $(TOOL_PNGPREPARE): tools/pngprepare.c @mkdir -p build/tools - $(CC) -O2 -Wall -I/usr/local/include -L/usr/local/lib -o $@ $< -lpng - -$(TOOL_COMPRESS_TEXT): tools/compress_text.c - @mkdir -p build/tools - $(CC) -O2 -Wall -I/usr/local/include -L/usr/local/lib -o $@ $< -lm + $(CC) -O2 -g -Wall -I/usr/local/include -L/usr/local/lib -o $@ $< -lpng build/tools/%: tools/%.c @mkdir -p build/tools - $(CC) -O2 -Wall -o $@ $< + $(CC) -O2 -g -Wall -o $@ $< build/tools/%: tools/%.cc tools/common.h @mkdir -p build/tools - $(CXX) -O2 -Wall -o $@ $< + $(CXX) -O2 -g -Wall -o $@ $< # Rules - CHARGEN @@ -258,9 +255,9 @@ build/symbols_ultimate64.vs: $(DIR_U64)/BASIC_combined.vs $(DIR_U64)/KER # Rules - BASIC and KERNAL intermediate files -build/,generated/packed_messages.s: $(TOOL_COMPRESS_TEXT) +build/,generated/packed_strings.s: $(TOOL_GENERATE_STRINGS) @mkdir -p build/,generated - $(TOOL_COMPRESS_TEXT) > build/,generated/packed_messages.s + $(TOOL_GENERATE_STRINGS) -o build/,generated/packed_strings.s build/,generated/float_constants.s: $(TOOL_GENERATE_CONSTANTS) @mkdir -p build/,generated @@ -441,7 +438,7 @@ test_ultimate64: build/kernal_ultimate64.rom build/basic_ultimate64.rom build/sy x64 -kernal build/kernal_ultimate64.rom -basic build/basic_ultimate64.rom -moncommands build/symbols_ultimate64.vs -1 $(TESTTAPE) -8 $(TESTDISK) test_mega65: $(TARGET_M65_x) $(TARGET_M65_x_PXL) - ../xemu/build/bin/xmega65.native -dmarev 2 -forcerom -loadrom $(TARGET_M65_x_PXL) + ../xemu/build/bin/xmega65.native -dmarev 2 -besure -fontrefresh -forcerom -loadrom $(TARGET_M65_x_PXL) test_cx16: $(TARGET_X16_x) ../x16-emulator/x16emu -rom $(TARGET_X16_x) diff --git a/c64/,,config_custom.s b/c64/,,config_custom.s index 84c60ab8..e023c716 100644 --- a/c64/,,config_custom.s +++ b/c64/,,config_custom.s @@ -46,8 +46,8 @@ // I/O devices #define CONFIG_IEC -// #define CONFIG_IEC_DOLPHINDOS -// #define CONFIG_IEC_DOLPHINDOS_FAST +#define CONFIG_IEC_DOLPHINDOS +#define CONFIG_IEC_DOLPHINDOS_FAST #define CONFIG_IEC_JIFFYDOS // #define CONFIG_IEC_JIFFYDOS_BLANK // #define CONFIG_IEC_BURST_CIA1 // please keep disabled for now @@ -124,7 +124,7 @@ #define CONFIG_PANIC_SCREEN #define CONFIG_DOS_WEDGE #define CONFIG_TAPE_WEDGE -#define CONFIG_TAPE_HEAD_ALIGN +// #define CONFIG_TAPE_HEAD_ALIGN #define CONFIG_BCD_SAFE_INTERRUPTS diff --git a/c64/basic/assets/packed_keywords.s b/c64/basic/assets/packed_as_1n.s similarity index 75% rename from c64/basic/assets/packed_keywords.s rename to c64/basic/assets/packed_as_1n.s index 82229cde..ba42d331 100644 --- a/c64/basic/assets/packed_keywords.s +++ b/c64/basic/assets/packed_as_1n.s @@ -3,6 +3,7 @@ // #LAYOUT# X16 BASIC_0 #TAKE // #LAYOUT# * * #IGNORE -packed_keywords: - put_packed_keywords() +packed_as_1n: + + put_packed_as_1n() diff --git a/c64/basic/assets/packed_chars.s b/c64/basic/assets/packed_as_3n.s similarity index 74% rename from c64/basic/assets/packed_chars.s rename to c64/basic/assets/packed_as_3n.s index 985ca369..27e7f9e3 100644 --- a/c64/basic/assets/packed_chars.s +++ b/c64/basic/assets/packed_as_3n.s @@ -3,6 +3,7 @@ // #LAYOUT# X16 BASIC_0 #TAKE // #LAYOUT# * * #IGNORE -packed_message_chars: - put_packed_chars() +packed_as_3n: + + put_packed_as_3n() diff --git a/c64/basic/assets/packed_tokens.s b/c64/basic/assets/packed_str_errors.s similarity index 72% rename from c64/basic/assets/packed_tokens.s rename to c64/basic/assets/packed_str_errors.s index 0a2709a2..4582fb54 100644 --- a/c64/basic/assets/packed_tokens.s +++ b/c64/basic/assets/packed_str_errors.s @@ -3,6 +3,7 @@ // #LAYOUT# X16 BASIC_0 #TAKE // #LAYOUT# * * #IGNORE -packed_message_tokens: - put_packed_tokens() +packed_str_errors: + + put_freq_packed_errors() diff --git a/c64/basic/assets/packed_str_keywords_V2.s b/c64/basic/assets/packed_str_keywords_V2.s new file mode 100644 index 00000000..5247b32e --- /dev/null +++ b/c64/basic/assets/packed_str_keywords_V2.s @@ -0,0 +1,9 @@ +// #LAYOUT# STD * #TAKE +// #LAYOUT# M65 BASIC_1 #TAKE +// #LAYOUT# X16 BASIC_0 #TAKE +// #LAYOUT# * * #IGNORE + + +packed_str_keywords_V2: + + put_freq_packed_keywords_V2() diff --git a/c64/basic/assets/packed_words.s b/c64/basic/assets/packed_str_misc.s similarity index 74% rename from c64/basic/assets/packed_words.s rename to c64/basic/assets/packed_str_misc.s index 515892e6..bceadcdf 100644 --- a/c64/basic/assets/packed_words.s +++ b/c64/basic/assets/packed_str_misc.s @@ -3,6 +3,7 @@ // #LAYOUT# X16 BASIC_0 #TAKE // #LAYOUT# * * #IGNORE -packed_message_words: - put_packed_words() +packed_str_misc: + + put_freq_packed_misc() diff --git a/c64/basic/basic_commands/cmd_stop_end.s b/c64/basic/basic_commands/cmd_stop_end.s index ee1a5768..95fd5091 100644 --- a/c64/basic/basic_commands/cmd_stop_end.s +++ b/c64/basic/basic_commands/cmd_stop_end.s @@ -4,8 +4,8 @@ cmd_stop: - ldx #30 // "BREAK" - jsr print_packed_message + ldx #IDX__EV2_1E // "BREAK" + jsr print_packed_error // Are we in direct mode lda CURLIN+1 @@ -13,8 +13,8 @@ cmd_stop: beq !+ // Not direct mode - ldx #31 - jsr print_packed_message + ldx #IDX__STR_IN + jsr print_packed_misc_str lda CURLIN+1 ldx CURLIN+0 diff --git a/c64/basic/basic_main_loop.s b/c64/basic/basic_main_loop.s index fd4dede5..06179538 100644 --- a/c64/basic/basic_main_loop.s +++ b/c64/basic/basic_main_loop.s @@ -6,8 +6,8 @@ basic_main_loop: // Tell user we are ready - ldx #30 - jsr print_packed_message + ldx #IDX__STR_READY + jsr print_packed_misc_str // Enable Kernal messages lda #$80 diff --git a/c64/basic/board_m65/4000.mega65_vectable_basic_1.s b/c64/basic/board_m65/4000.mega65_vectable_basic_1.s index e69e0332..25c233d0 100644 --- a/c64/basic/board_m65/4000.mega65_vectable_basic_1.s +++ b/c64/basic/board_m65/4000.mega65_vectable_basic_1.s @@ -11,13 +11,15 @@ // Label definitions - .label VB1__INITMSG = $4000 + 2 * 0 - .label VB1__LINKPRG = $4000 + 2 * 1 - .label VB1__tokenise_line = $4000 + 2 * 2 - .label VB1__list_single_line = $4000 + 2 * 3 - .label VB1__print_packed_message = $4000 + 2 * 4 - .label VB1__basic_do_clr = $4000 + 2 * 5 - .label VB1__basic_do_new = $4000 + 2 * 6 + .label VB1__INITMSG = $4000 + 2 * 0 + .label VB1__LINKPRG = $4000 + 2 * 1 + .label VB1__tokenise_line = $4000 + 2 * 2 + .label VB1__list_single_line = $4000 + 2 * 3 + .label VB1__print_packed_error = $4000 + 2 * 4 + .label VB1__print_packed_misc_str = $4000 + 2 * 5 + .label VB1__print_packed_keyword_V2 = $4000 + 2 * 6 + .label VB1__basic_do_clr = $4000 + 2 * 7 + .label VB1__basic_do_new = $4000 + 2 * 8 #else @@ -27,7 +29,9 @@ .word LINKPRG .word tokenise_line .word list_single_line - .word print_packed_message + .word print_packed_error + .word print_packed_misc_str + .word print_packed_keyword_V2 .word basic_do_clr .word basic_do_new diff --git a/c64/basic/engine/do_error.s b/c64/basic/engine/do_error.s index 6947ca0b..1d0c153e 100644 --- a/c64/basic/engine/do_error.s +++ b/c64/basic/engine/do_error.s @@ -151,10 +151,10 @@ do_basic_error: // error code in .X // Error message text + " ERROR" - jsr print_packed_message + jsr print_packed_error - ldx #33 - jsr print_packed_message + ldx #IDX__STR_ERROR + jsr print_packed_misc_str // Check if direct mode @@ -164,8 +164,8 @@ do_basic_error: // error code in .X // We were in a program, so show IN - ldx #31 - jsr print_packed_message + ldx #IDX__STR_IN + jsr print_packed_misc_str lda CURLIN+1 ldx CURLIN+0 diff --git a/c64/basic/init/basic_warm_start_internal.s b/c64/basic/init/basic_warm_start_internal.s index f9ad47c0..79257cd1 100644 --- a/c64/basic/init/basic_warm_start_internal.s +++ b/c64/basic/init/basic_warm_start_internal.s @@ -19,8 +19,8 @@ basic_warm_start_real: ora CMP0+1 beq !+ - ldx #32 - jsr print_packed_message + ldx #IDX__STR_BRK_AT + jsr print_packed_misc_str lda CMP0+1 jsr print_hex_byte diff --git a/c64/basic/init/initmsg_bytes_free.s b/c64/basic/init/initmsg_bytes_free.s index 0093c12a..b277340f 100644 --- a/c64/basic/init/initmsg_bytes_free.s +++ b/c64/basic/init/initmsg_bytes_free.s @@ -21,5 +21,5 @@ initmsg_bytes_free: jsr print_integer // Print rest of the start up message - ldx #34 - jmp print_packed_message + ldx #IDX__STR_BYTES + jmp print_packed_misc_str diff --git a/c64/basic/pack_word.s b/c64/basic/pack_word.s deleted file mode 100644 index d975ca38..00000000 --- a/c64/basic/pack_word.s +++ /dev/null @@ -1,260 +0,0 @@ -// #LAYOUT# STD * #TAKE -// #LAYOUT# M65 BASIC_1 #TAKE -// #LAYOUT# X16 BASIC_0 #TAKE -// #LAYOUT# * * #IGNORE - -// Pack a word using the simple compression scheme. -// This has to be the same implementation as in compress_text.c -// because we compress input lines of BASIC to do the tokenisation. -// This allows us to search through the compressed tokens directly -// using the compressed fragment of BASIC, and thus work out the -// token for a given piece of string. -// This also helps ensure that we have an implementation that is totally -// different to that of the original C64 BASIC, as well as likely -// saving a bit of space in the ROM, especially once we start implementing -// larger keyword vocabs for BASIC 7 or BASIC 10 compatibility, since the -// compression and decompression machinery is of fixed size. - -pack_word: - // Inputs: - // X = offset of string to pack. Address is $0200 + X - // __tokenise_work1 = number of bytes to pack - // __tokenise_work2 = packed length of string - - // Outputs: - // Output is written to $0100 (bottom of stack). - - // Memory modified: - // __tokenise_work4 = temporary storage - // __tokenise_work3 = flag for whether we have a nybl prepared or not. - // (This can also be checked on exit, to know whether the last byte - // is incomplete, which is important for searching for abbreviated - // key words.) - - - // Make sure we have a sane request - lda __tokenise_work1 - bne !+ - sec - rts -!: - // Initialise internal variables - lda #$00 - sta __tokenise_work2 - sta __tokenise_work3 - -pack_char_loop: - - // Get the next char - lda $0200,x - - // Work out what type of character we have, and thus - // how we need to pack it. - jsr get_packed_char_index - - cmp #$00 - bne not_exception_char - - // Char is an exception char. - // We need to either finish the previous byte with $xF, - // Or if there is no existing nybl, then we need to output - // either $FE or $FF, depending on whether this is the last - // byte or not - - lda __tokenise_work3 - bne have_nybl_before_exception_char - - // No nybl exists before the exception char, so prefix with $FE or $FF - // based on whether we are at the end of the input string or not - inx - lda __tokenise_work1 - cmp #$01 - beq at_end - lda #$FF - .byte $2C -at_end: - lda #$FE - dex - jsr write_unpacked_char - // Fall through - -output_exception_byte: - lda $0200,x - jsr write_unpacked_char - jmp consider_next_char - -have_nybl_before_exception_char: - // Or existing byte with #$0F, and clear nybl flag - ldy __tokenise_work2 - lda $0100,y - ora #$0f - sta $0100,y - inc __tokenise_work2 - - // FALL THROUGH - -write_literal_and_terminate_if_required: - - // Clear have nybl flag - lda #$00 - sta __tokenise_work3 - - lda $0200,x - jsr write_unpacked_char - - jmp end_string_if_required - -end_string_if_required: - // Check if X points to last char of string to be packed - inx - lda __tokenise_work1 - cmp #$01 - beq !+ - dex - jmp consider_next_char -!: - dex - // Fall through - -end_string: - // Write end of string sequence if required - lda #$00 - jsr write_unpacked_char - rts - -write_unpacked_char: - ldy __tokenise_work2 - sta $0100,y - iny - sty __tokenise_work2 - rts - -not_exception_char: - // It wasn't an exception char, so write the symbol out - cmp #$10 - bcs whole_byte_symbol - - // Nybl encoded value: Put in upper half or lower half of a byte? - ldy __tokenise_work3 - bne store_low_nybl - - // Store in high nybl -- so shift it into the hi nybl - asl - asl - asl - asl - ldy __tokenise_work2 - sta $0100,y - bne stored_nybl // Must be taken, as A cannot be $00 - -store_low_nybl: - // Low nybl gets added to high nybl already stored in the byte - ldy __tokenise_work2 - ora $0100,y - sta $0100,y - -stored_nybl: - // Now toggle the nybl flag - lda __tokenise_work3 - eor #$ff - sta __tokenise_work3 - // And advance the offset if we filled this byte up - bne consider_next_char // Taken if byte has a nybl free, i.e., don't advance pointer while half byte remains free - - // Count the filled up byte - inc __tokenise_work2 - - // Add termination byte if required - // (This re-writes the same byte again, which we don't care about) - jmp end_string_if_required - -whole_byte_symbol: - // It's a $Fx symbol. - // But if we have a nybl to flush, then we handle it like - // a full exception character - - ldy __tokenise_work3 - beq nothing_to_flush - - // Set low nybl to $F to mark next byte as literal - ldy __tokenise_work2 - lda $0100,y - ora #$0f - sta $0100,y - inc __tokenise_work2 - - jmp write_literal_and_terminate_if_required - -nothing_to_flush: - // Write the symbol as it is, or the literal - // if it has been passed to us via fall-through - jsr write_unpacked_char - - jmp end_string_if_required - -consider_next_char: - // Pack next char - inx - dec __tokenise_work1 - beq !+ - jmp pack_char_loop -!: - // Add 1 to length if nybl waiting to be flushed - // (but leave flag set so the caller knows if the - // bottom nybl might be different if the string were - // longer). - lda __tokenise_work3 - beq !+ - inc __tokenise_work2 - rts -!: - // Last byte was full, so we need at $00 on the end - // (unless a $FE token was written 2 bytes ago) - ldy __tokenise_work2 - iny - lda #$00 - sta $0100,y - inc __tokenise_work2 - - rts - -get_packed_char_index: - // Find char in A in the packed char list, and return - // $1 - $E, $F1 - $FD if the charactar is abbreviable, - // or else return $00 if the character has to be stored literally. - - // Y register and A are modified, as are processor flags. - - ldy #$00 -find_char_loop: - cmp packed_message_chars,y - bne not_a_match -enumerate: - // Is it a nybl abbreviable char? - cpy #14 - bcs not_nybl_char - // Yes, so add 1 and return it - iny - tya - rts - -not_nybl_char: - cpy #27 - bcs not_a_match - // It's a $Fx character. - // Since the lowest index is $E, and it should be encoded as $F1 - // we need to add $E3 - tya - clc - adc #$E3 - rts - -not_a_match: - iny - // Loop until we find it, or we discover it isn't - // a character with a special encoding - cpy #27 - bne find_char_loop - // Not a char that we can pack, so return 0 - lda #$00 - rts diff --git a/c64/basic/print/list_single_line.s b/c64/basic/print/list_single_line.s index 64e6b2b4..cf13147b 100644 --- a/c64/basic/print/list_single_line.s +++ b/c64/basic/print/list_single_line.s @@ -86,22 +86,16 @@ list_not_quote: pha phy_trash_a - // Get pointer to compressed keyword list - lda #packed_keywords - sta FRESPC+1 - - // Subtract $80 from token to get offset in word - // list + + // Subtract $80 from token to get offset in token list txa and #$7f tax // Now ask for it to be printed - ldy #$ff - jsr packed_word_search + jsr print_packed_keyword_V2 + // Restore registers ply_trash_a pla diff --git a/c64/basic/print/print_packed_message.s b/c64/basic/print/print_packed_message.s deleted file mode 100644 index 7a062e6a..00000000 --- a/c64/basic/print/print_packed_message.s +++ /dev/null @@ -1,244 +0,0 @@ -// #LAYOUT# STD * #TAKE -// #LAYOUT# M65 * #TAKE -// #LAYOUT# X16 BASIC_0 #TAKE -// #LAYOUT# * * #IGNORE - -// This routine prints messages that have been packed -// using the make_error_tables program. -// The general idea is to save space in the ROM. - -// The output of make_error_tables is written to packed_messages.s, and -// contains the following: -// -// packed_message_words - List of nybl packed words. -// packed_message_tokens - List of tokens that get joined to make messages -// packed_message_chars - List of characters in messages sorted by frequency. - - -print_packed_message: - // .X = message number - -#if (ROM_LAYOUT_M65 && SEGMENT_BASIC_0) - - jsr map_BASIC_1 - jsr_ind VB1__print_packed_message - jmp map_NORMAL - -#else - - lda #$ff - ldy #$ff -packed_message_search: - iny - cpx #$00 - beq found_message_in_token_stream - lda packed_message_tokens,y - cmp #$ff - bne !+ - dex -!: - cpy #$ff - bne packed_message_search - // Could not find message - sec - rts - -found_message_in_token_stream: // XXX size-optimize this - // Now print each word in the message - phy_trash_a - lda packed_message_tokens,y - cmp #$FF - beq found_message_in_token_stream_done - - jsr print_packed_word - ply_trash_a - iny - beq found_message_in_token_stream_done -!: - phy_trash_a - lda packed_message_tokens,y - cmp #$FF - beq found_message_in_token_stream_done - - pha - jsr print_space - pla - - jsr print_packed_word - ply_trash_a - iny - bne !- - -found_message_in_token_stream_done: - pla - rts - -print_packed_word: - // A = the word to print - tax - - // Setup pointer to packed words - lda #packed_message_words - sta FRESPC+1 - - // Find the word in the list - ldy #$ff -packed_word_search: - iny - cpx #$00 - beq found_packed_word - - // $x0 is an end of work marker - lda (FRESPC),y - and #$0f - cmp #$00 - bne !+ - dex -!: - // $FE $xx is an end of work marker - lda (FRESPC),y - cmp #$fe - bne !+ - dex - iny // skip next byte -!: - cmp #$ff - bne !+ - iny // skip next byte -!: - - // Advance to next page if required - cpy #$ff - bne packed_word_search - inc FRESPC+1 - jmp packed_word_search - -found_packed_word: - - // Get pointer to start of packed word - // Make sure we do not wrap on a page boundary - tya - clc - adc FRESPC+0 - sta FRESPC+0 - lda FRESPC+1 - adc #$00 - sta FRESPC+1 - -print_packed_string: - ldy #$ff - -next_packed_word_char: - // .Y = offset into packed word data - 1 - iny - - // Check for end of word - lda (FRESPC),y - cmp #$00 - beq end_of_packed_word - cmp #$fe - beq last_literal - cmp #$ff - beq next_is_literal_char - and #$f0 - beq end_of_packed_word - - // Check if it is an uncommon char - lda (FRESPC),y - and #$f0 - cmp #$f0 - beq is_uncommon_char - - // Is not an uncommon char, so get char of upper nybl - lda (FRESPC),y - - // Put high nybl into low bits of X - lsr - lsr - lsr - lsr - tax - - // X=1-14 = first 14 chars, so subtract one - lda packed_message_chars-1,x - jsr JCHROUT // preserves .Y - - // See if there is a char in the low nybl to print - lda (FRESPC),y - and #$0f - beq end_of_packed_word - cmp #$0f - beq no_lo_nybl_char - -has_lo_nybl_char: - // We need to print the low nybl char as well - tax - phy_trash_a - - // Skip lower nybls of $F or $0 in nybl packed - // bytes, as these do not encode characters - cpx #$f - beq !+ -has_nybl: - cpx #0 - beq !+ - - // X=1-14 = first 14 chars, so subtract one - lda packed_message_chars-1,x - jsr JCHROUT -!: - ply_trash_a - // FALLTHROUGH - -no_lo_nybl_char: - - // If current byte has $0F in low nybl, then the next - // byte is a whole byte. - lda (FRESPC),y - and #$0f - cmp #$0f - beq next_is_literal_char - -ready_for_next_char: - cpy #$ff - bne next_packed_word_char -end_of_packed_word: - // Hit end of packed data - rts - -last_literal: - jsr unpack_literal_char - rts - -next_is_literal_char: - jsr unpack_literal_char - jmp ready_for_next_char - -unpack_literal_char: - // Output the literal char, and ready for next char - iny - phy_trash_a - lda (FRESPC),y - jsr JCHROUT - pla - tay - rts - -is_uncommon_char: - // Lower nybl has offset into low-frequency part of the table - - lda (FRESPC),y - and #$0f - clc - adc #14 - - // We need to print the low nybl char as well - tax - phy_trash_a - - jmp has_nybl - - -#endif // ROM layout diff --git a/c64/basic/print/print_packed_string.s b/c64/basic/print/print_packed_string.s new file mode 100644 index 00000000..984a7d6b --- /dev/null +++ b/c64/basic/print/print_packed_string.s @@ -0,0 +1,182 @@ +// #LAYOUT# STD * #TAKE +// #LAYOUT# M65 * #TAKE +// #LAYOUT# X16 BASIC_0 #TAKE +// #LAYOUT# * * #IGNORE + +// This routine prints strings that have been packed using the 'generate_trings' tool. +// The general idea is to save space in the ROM. + + +#if (ROM_LAYOUT_M65 && SEGMENT_BASIC_0) + +print_packed_error: + + jsr map_BASIC_1 + jsr_ind VB1__print_packed_error + jmp map_NORMAL + +print_packed_misc_str: + + jsr map_BASIC_1 + jsr_ind VB1__print_packed_misc_str + jmp map_NORMAL + +print_packed_keyword_V2: + + jsr map_BASIC_1 + jsr_ind VB1__print_packed_keyword_V2 + jmp map_NORMAL + +#else + +print_packed_error: // .X - error string index + + lda #packed_str_errors + bne print_freq_packed_string // branch always + +print_packed_misc_str: // .X - misc string index + + lda #packed_str_misc + bne print_freq_packed_string // branch always + +print_packed_keyword_V2: // .X - token number + + lda #packed_str_keywords_V2 + + // FALLTROUGH + +print_freq_packed_string: // not to be used directly XXX rename to print_packed_string + + // First store the initial address of the packed strings list + + sta FRESPC+0 + sty FRESPC+1 + + // Now find the address of the string to print out + // The implementation assumes no packed string is longer than 255 bytes, including trailing zero +!: + cpx #$00 + beq print_freq_packed_string_found // branch if no need to advance anymore + dex + + ldy #$00 +!: + lda (FRESPC), y + iny + cmp #$00 + bne !- // branch if not the end of packed string + + clc // advance FREESPC pointer + tya + adc FRESPC+0 + sta FRESPC+0 + lda #$00 + adc FRESPC+1 + sta FRESPC+1 + + jmp !-- + +print_freq_packed_string_found: + + // At this point FRESPC contains a pointer to the string to display + // and we should start from the lower nibble + + ldy #$00 + + // FALLTROUGH + +print_freq_packed_string_nibble_lo: + + // Get the low nibble and check for the end of packed string + + lda (FRESPC), y + and #$0F + beq print_freq_packed_string_end + + // Not the end - check if this is a common character, encoded by a single nibble + + cmp #$0F + beq print_freq_packed_string_3n_split + + // Character is encoded by a single nibble - fetch it and display + + tax + lda packed_as_1n-1, x + jsr JCHROUT // preserves .Y + + // FALLTROUGH + +print_freq_packed_string_nibble_hi: + + // Get the high nibble and check for the end of packed string + + lda (FRESPC), y + and #$F0 + beq print_freq_packed_string_end + + // Not the end - check if this is a common character, encoded by a single nibble + + cmp #$F0 + beq print_freq_packed_string_3n_single + + // Character is encoded by a single nibble - fetch it and display + + lsr + lsr + lsr + lsr + + tax + lda packed_as_1n-1, x + jmp !+ // XXX should be bra on some CPUs + +print_freq_packed_string_3n_single: + + // Character is encoded using 3 nibbles (1st is a mark, already interpreted) + + iny + lda (FRESPC), y + tax + lda packed_as_3n-1, x +!: + jsr JCHROUT // preserves .Y + + // Advance to next nibble + + iny + bne print_freq_packed_string_nibble_lo // branch always + +print_freq_packed_string_3n_split: + + // Character is encoded using 3 nibbles (1st is a mark, already interpreted), + // but the 2 remaining nibbles are in two neighbour bytes; use a stack to combine them + + lda (FRESPC), y + and #$F0 + tsx + pha + + iny + lda (FRESPC), y + and #$0F + + clc + adc STACK, x + + txs // one cycle faster than PLA + + tax + lda packed_as_3n-1, x + jsr JCHROUT // preserves .Y + + jmp print_freq_packed_string_nibble_hi + +print_freq_packed_string_end: + + rts + + +#endif // ROM layout diff --git a/c64/basic/tokenise_line.s b/c64/basic/tokenise_line.s deleted file mode 100644 index 5e00a443..00000000 --- a/c64/basic/tokenise_line.s +++ /dev/null @@ -1,306 +0,0 @@ -// #LAYOUT# STD * #TAKE -// #LAYOUT# M65 * #TAKE -// #LAYOUT# X16 BASIC_0 #TAKE -// #LAYOUT# * * #IGNORE - -// Tokenise a line of BASIC -// Stored at $0200 -// Length in __tokenise_work1 - -tokenise_line: - -#if (ROM_LAYOUT_M65 && SEGMENT_BASIC_0) - - jsr map_BASIC_1 - jsr_ind VB1__tokenise_line - jmp map_NORMAL - -#else - - // We need to consider each offset in the string to see if it matches - // a token. No BASIC keyword is > 7 bytes, so we need to start with that - // length (or remaining length of string) and count our way back down. - // This means we need separate start and end points to keep track of. - - // $0110 = offset into string - // $0111 = number of bytes to pack - // $0112 = offset into tokenised output - // $0113 = length of raw string - - lda #$00 - sta $0110 - sta $0112 - sta QTSW - - lda __tokenise_work1 - sta $0113 - -tokenise_char: - - // Check if we have reached end of the line? - lda $0110 - cmp $0113 - bne !+ - - // End of line reached - - // Finished tokenising line - // Update __tokenise_work1 and terminate with null - ldx $0112 - lda #$00 - sta $0200,x - stx __tokenise_work1 - - // Clear quote mode flag that is also used by KERNAL for screen display - lda #$00 - sta QTSW - rts -!: - // More to do - - // Maximum keyword length - lda #7 - sta $0111 - -tokenise_char_loop: - // Pack the word from the start (start = X, raw len = __tokenise_work1) - // Returns packed length in __tokenise_work2 and writes the packed - // word at $0100+ - - ldx $0110 - lda $0200,x - cmp #$22 - beq tk_quote - cmp #$DE - beq tk_pi - cmp #$3F - bne tk_not_special - - // FALLTROUGH - -tk_question_mark: - - lda #$99 - skip_2_bytes_trash_nvz - - // FALLTROUGH - -tk_pi: - - lda #$FF - ldx $0110 - sta $0200,x - - jmp tk_literal_char - -tk_quote: - - // Quote - lda QTSW - eor #$ff - sta QTSW - jmp tk_literal_char - -tk_not_special: - // Check for literal characters in various ranges - // (This just speeds the tokeniser up, by avoiding the packed token search - // for such characters). - - // SPACE and punctuation before +,- and * - cmp #$2a - bcc tk_literal_char - // Don't try to tokenise digits - cmp #$30 - bcc tk_might_be_keyword - cmp #$3a - bcs tk_might_be_keyword - jmp tk_literal_char - -tk_might_be_keyword: - // Don't tokenise in quote mode - lda QTSW - bne tk_literal_char - - // Pack string - ldx $0110 - lda $0111 - sta __tokenise_work1 - - // XXX - We should implement an optimisation where we - // trim the last symbol of successively, instead of re-packing the word - // again with one less char each time. - // XXX - We should limit the word length to the length of interesting chars - // that are available. - - jsr pack_word - lda __tokenise_work2 - cmp #$10 - bcc !+ - // Packed string too long - jmp do_STRING_TOO_LONG_error -!: - - // Search for it in keywords - jsr keyword_search - - // Did we find a keyword? - bcc found_token_in_line - - // Nope, so try shorter - dec $0111 - bne tokenise_char_loop - -tk_literal_char: - // Is not a token, so copy to output verbatim - ldx $0110 - ldy $0112 - lda $0200,x - sta $0200,y - inc $0112 - inc $0110 - - // Tokenise next char - jmp tokenise_char - -found_token_in_line: - - // Got a token, so write it out - ldx $0112 - sta $0200,x - inx - stx $0112 - - // If it is REM, then lock us in quote mode to the end of the line - cmp #$8f - bne not_rem - sta QTSW -not_rem: - - // Now skip over the length of the token - lda $0110 - clc - adc $0111 - sta $0110 - - // See if there is more of the line left to process - jmp tokenise_char - - - - - // Search for word in keyword list - // The packed form is canonical, so we can search for the keyword while - // it remains compressed. This also saves a little time. - // Algorithm is simply to find matching first bytes, and then look for - // the rest being a match. - -keyword_search: - // Search for the compressed keyword stored at $0100 in the list - // of compressed keywords - - // __tokenise_work1 = offset in line of input - // __tokenise_work2 = length of token we are matching - // __tokenise_work3 = offset in token list - // __tokenise_work5 = temporary work space - // X and Y registers used as temporary space. - // returns C=1 if no matching token, else C=0 and A=token - - lda #$00 - sta __tokenise_work1 - sta __tokenise_work3 -next_kw_offset: - // Advance offset in compressed keyword list, to see if a match - // BEGINS here - - // Get count of bytes to compare - lda __tokenise_work2 - sta __tokenise_work5 - - // Load current position - ldy __tokenise_work3 - ldx __tokenise_work1 - - // Advance offset in compressed token list, and stop at the end - inc __tokenise_work3 - lda __tokenise_work3 - cmp #packed_keyword_table_len - bne next_in_match - jmp done_searching_for_token -next_in_match: - lda $0100,x - cmp packed_keywords,y - bne next_kw_offset - // Advance to next chars in source and target - inx - iny - - dec __tokenise_work5 // Have we compared all bytes yet? - - bne next_in_match - - // Keyword matches! - - // Now make sure we started on a keyword boundary - // (we know we finished on one, because of the structure of the compressed - // keyword data). - // Offset 0 - // previous byte is $x0 - // two byte prior is $FE - - // Get offset of first byte of this token - ldx __tokenise_work3 - dex // pointer points one late, so rewind it - - // At start of list? - cpx #$00 - beq word_boundary - dex - // Previous byte is $x0 end token? - lda packed_keywords,x - and #$0f - beq word_boundary - dex - // Uber-previouss byte is $FE final literal token? - lda packed_keywords,x - cmp #$fe - beq word_boundary - - // Not a word boundary, so ignore - jmp next_kw_offset - -word_boundary: - // Found a token whose offset is in __tokenise_work3 - // Now trace back to the start of the compressed keyword list - // to work out what keyword number we are, so that we can return - // the token number. - - ldy #$80 - -token_count_loop: - lda packed_keywords,x - cmp #$fe - bne !+ - iny -!: - and #$0f - cmp #$00 - bne !+ - iny -!: - cpx #$00 - beq !+ - dex - jmp token_count_loop -!: - - tya - clc - rts - -done_searching_for_token: - // No token found - sec - lda #$00 - rts - -#endif // ROM layout diff --git a/c64/basic/tokenizer/tk_pack.s b/c64/basic/tokenizer/tk_pack.s new file mode 100644 index 00000000..bf4ee935 --- /dev/null +++ b/c64/basic/tokenizer/tk_pack.s @@ -0,0 +1,228 @@ +// #LAYOUT# STD * #TAKE +// #LAYOUT# M65 BASIC_1 #TAKE +// #LAYOUT# X16 BASIC_0 #TAKE +// #LAYOUT# * * #IGNORE + +// +// Packs a keyword candidate using a simple compression scheme, +// same as in 'generate_strings.cc'. +// +// This allows us to search through the compressed tokens directly +// using the compressed fragment of BASIC, and thus work out the +// token for a given piece of string. +// +// This also helps ensure that we have an implementation that is totally +// different to that of the original C64 BASIC, as well as likely +// saving a bit of space in the ROM, especially once we start implementing +// larger keyword vocabs for extended BASIC, since the compression and +// decompression machinery is of fixed size. +// +// Input: +// - .X = offset of string to pack, address is BUF ($0200) + .X +// Output: +// - see labels in 'tk_pack' +// + + +tk_pack: + + // Reuse the CPU stack - addresses below $102 are used by 'tokenise_line.s' + + .label tk__len_unpacked = $103 // length of unpacked data; it could replace 'tk__nibble_flag', but at the cost of code size/performance + .label tk__shorten_bits = $104 // for quick shortening of packed candidate + .label tk__nibble_flag = $105 // $00 = start from new byte, $FF = start from high nibble + .label tk__byte_offset = $106 // offset of the current byte (to place new data) in tk__packed + + .label tk__packed = $107 // packed candidate, 13 bytes is enough for worst case - 8 byte keyword + + // Initialize variables + + lda #$00 + ldy #(tk__packed - tk__len_unpacked + 13) // XXX maybe do not clear that much, just put 0 at the end while compressing +!: + sta tk__len_unpacked, y + dey + bpl !- + + // FALLTROUGH + +tk_pack_loop: + + // Get the next character + + lda BUF, x + + // If character cannot be a part of a keyword - do not pack anything more + + cmp #$23 // '#' - first sane character for a keyword + bcc tk_pack_done // branch if below '#' + cmp #$7B // 'Z' + 1 + bcs tk_pack_done // branch if above 'Z' + + // Keywords should not contain digits, colons, or semicolons + + cmp #$30 + bcc !+ // branch if below '0' + cmp #$3C + bcc tk_pack_done // branch if below '<' +!: + // Try to find a nibble to encode the character + + ldy #$0E +!: + cmp packed_as_1n-1, y + beq tk_pack_1n + + dey + bne !- + + // Try to find a 3-nibble sequence to encode the character (1st nibble is always 0xF in this case) + + ldy #TK__PACKED_AS_3N +!: + cmp packed_as_3n-1, y + beq tk_pack_3n + + dey + bne !- + + // If it is not on the list, it cannot be a part of token - quit + + // FALLTROUGH + +tk_pack_done: + + rts + +tk_pack_1n: + + // .Y contains our nibble; check whether we should start a new encoded byte + + bit tk__nibble_flag + bpl tk_pack_1n_new_byte + + // Store in the high nibble of the current byte + + tya + + asl + asl + asl + asl + + ldy tk__byte_offset + ora tk__packed, y + sta tk__packed, y + + // Adjust remaining counters / flags + + inc tk__byte_offset + inc tk__nibble_flag // $FF -> $00 + + // FALLTROUGH + +tk_pack_1n_done: + + // Put the bit mark - to indicate 1-nibble encoding + + clc + ror tk__shorten_bits + + // FALLTROUGH + +tk_pack_loop_next: + + // Prepare for the next loop iteration - increment counters + + inx + inc tk__len_unpacked + + // Check length, if it is OK to pack one more byte; quit if not + + lda tk__len_unpacked + cmp #TK__MAX_KEYWORD_LEN + + bne tk_pack_loop + rts + +tk_pack_1n_new_byte: + + // Store in the low nibble of the next byte + + tya + ldy tk__byte_offset + sta tk__packed, y + + // Adjust remaining counters / flags + + dec tk__nibble_flag // $00 -> $FF + bmi tk_pack_1n_done + +tk_pack_3n: + + // .Y contains 2 nibbles to encode; check whether we should start a new encoded byte + + bit tk__nibble_flag + bpl tk_pack_3n_new_byte + + // Store 1st nibble (alwys 0xF) in the high nibble of the current byte + + phy_trash_a + + lda #$F0 + ldy tk__byte_offset + ora tk__packed, y + sta tk__packed, y + + // Store byte in the next byte + + iny + pla + sta tk__packed, y + + // Adjust remaining counters / flags + + iny + sty tk__byte_offset + inc tk__nibble_flag // $FF -> $00 + + // FALLTROUGH + +tk_pack_3n_done: + + // Put the bit mark - to indicate 3-nibble encoding + + sec + ror tk__shorten_bits + + bcc tk_pack_loop_next // branch always + +tk_pack_3n_new_byte: + + // First store everything in the current byte, we will set '0xF' mark later + + tya + ldy tk__byte_offset + sta tk__packed, y + + // Now store just the low nibble in the next byte + + iny + and #$0F + sta tk__packed, y + + // Go back for a moment and set the '0xF' mark in the low nibble + + dey + lda tk__packed, y + ora #$0F + sta tk__packed, y + iny + + // Adjust remaining counters / flags + + iny + sty tk__byte_offset + dec tk__nibble_flag // $00 -> $FF + + bmi tk_pack_3n_done // branch always diff --git a/c64/basic/tokenizer/tk_search.s b/c64/basic/tokenizer/tk_search.s new file mode 100644 index 00000000..fb847b24 --- /dev/null +++ b/c64/basic/tokenizer/tk_search.s @@ -0,0 +1,93 @@ +// #LAYOUT# STD * #TAKE +// #LAYOUT# M65 BASIC_1 #TAKE +// #LAYOUT# X16 BASIC_0 #TAKE +// #LAYOUT# * * #IGNORE + +// +// Search for a token on the list pointed by FRESPC +// +// Input: +// - list address in FRESPC, has to be refreshed after routine call +// - uses same variables as 'tk_pack' +// Output: +// - Carry set = not found +// - .X = index of token found +// + + +tk_search: + + ldx #$00 // initial token index + + // FALLTROUGH + +tk_search_loop: + + // Check if end of the list + + ldy #$00 + lda (FRESPC), y + cmp #$FF + bne tk_search_compare + iny + lda (FRESPC), y + dey + cmp #$FF + bne tk_search_compare + + // End of the list + + sec + rts + +tk_search_compare: + + // Compare the packed candidate with the current keyword list entry + + lda (FRESPC), y + cmp tk__packed, y + bne tk_search_not_matching + + cmp #$00 + beq tk_search_found + + // For now everything matches, try the next byte + + iny + bne tk_search_compare // branch always + +tk_search_not_matching: + + inx // increment token index + + // We need to find the end of the keyword on the list + // and proceed one byte further + + cmp #$00 + beq tk_search_next +!: + iny + lda (FRESPC), y + bne !- + + // FALLTROUGH + +tk_search_next: + + // Update FRESPC - advance by .Y + + iny + tya + clc + adc FRESPC+0 + sta FRESPC+0 + lda FRESPC+1 + adc #$00 + sta FRESPC+1 + + bcc tk_search_loop // branch always + +tk_search_found: + + clc + rts diff --git a/c64/basic/tokenizer/tk_shorten.s b/c64/basic/tokenizer/tk_shorten.s new file mode 100644 index 00000000..9b14833f --- /dev/null +++ b/c64/basic/tokenizer/tk_shorten.s @@ -0,0 +1,112 @@ +// #LAYOUT# STD * #TAKE +// #LAYOUT# M65 BASIC_1 #TAKE +// #LAYOUT# X16 BASIC_0 #TAKE +// #LAYOUT# * * #IGNORE + +// +// Shortens a keyword candidate previously packed by tk_pack by 1 character +// +// Input: +// - uses same variables as 'tk_pack' +// + + +tk_shorten: + + // First make sure we actually can shorten anything + + dec tk__len_unpacked + beq tk_shorten_end + + // Now check if the last character was encoded as 1 nibble or 3 nibbles + + asl tk__nibble_flag + bcs tk_shorten_3n + + // FALLTROUGH + +tk_shorten_1n: + + ldy tk__byte_offset // will be needed in both cases + + // Now we need to check which nibble to cut away + + bit tk__nibble_flag + bcs tk_shorten_1n_lo + + // FALLTROUGH + +tk_shorten_1n_hi: // tk__nibble_flag = $00, both nibbles free + + // Go back one byte and clear the high nibble only + + dey + sty tk__byte_offset + + lda tk__packed, y + and #$F0 + sta tk__packed, y + + // Adjust remaining counters / flags, and quit + + dec tk__nibble_flag // $00 -> $FF + + // FALLTROUGH + +tk_shorten_end: + + rts + +tk_shorten_1n_lo: // tk__nibble_flag = $FF, only the high nibble is free + + // Clear the whole byte, it will be faster + + lda #$00 + sta tk__packed, y + + // Adjust remaining counters / flags, and quit + + inc tk__nibble_flag // $FF -> $00 + rts + +tk_shorten_3n: + + ldy tk__byte_offset // will be needed in both cases + lda #$00 + + // Now we need to check which nibble to cut away + + bit tk__nibble_flag + bcs tk_shorten_3n_2bytes + + // FALLTROUGH + +tk_shorten_3n_byte_nibble: // tk__nibble_flag = $00, both nibbles free + + // Go back one byte, clear it, go back and clear the high nibble only + + dey + sta tk__packed, y + + // Reuse 'tk_shorten_1n_hi' + + sty tk__byte_offset +#if HAS_OPCODES_65C02 + bra tk_shorten_1n_hi +#else + jmp tk_shorten_1n_hi +#endif + +tk_shorten_3n_2bytes: // tk__nibble_flag = $FF, only the high nibble is free + + // Clear the byte, go back, clear one more byte + + sta tk__packed, y + dey + sta tk__packed, y + + // Adjust remaining counters / flags, and quit + + sty tk__byte_offset + inc tk__nibble_flag // $FF -> $00 + rts diff --git a/c64/basic/tokenizer/tokenise_line.s b/c64/basic/tokenizer/tokenise_line.s new file mode 100644 index 00000000..912e3832 --- /dev/null +++ b/c64/basic/tokenizer/tokenise_line.s @@ -0,0 +1,176 @@ +// #LAYOUT# STD * #TAKE +// #LAYOUT# M65 * #TAKE +// #LAYOUT# X16 BASIC_0 #TAKE +// #LAYOUT# * * #IGNORE + +// +// Tokenise a line of BASIC stored at $0200 +// +// Input: +// - length in __tokenise_work1 +// + + +tokenise_line: + +#if (ROM_LAYOUT_M65 && SEGMENT_BASIC_0) + + jsr map_BASIC_1 + jsr_ind VB1__tokenise_line + jmp map_NORMAL + +#else + + // Reuse the CPU stack - addresses above $101 are used by 'tk_pack.s' + + .label tk__offset = $100 // offset into string + .label tk__length = $101 // length of the raw string XXX reuse __tokenise_work1? + + // Initialize variables + + lda #$00 + sta tk__offset + + ldx __tokenise_work1 + stx tk__length + + // Terminate input string with $00 + + // XXX do we need this? + sta BUF, x + + // FALLTROUGH + +tokenise_line_loop: + + // Check if we have reached end of the line + + lda tk__offset + cmp tk__length + beq tokenise_line_done + + // Check for some special characters + + ldx tk__offset + lda BUF, x + + cmp #$22 + beq tokenise_line_quote + + cmp #$DE + beq tokenise_line_pi + + cmp #$3F + beq tokenise_line_question_mark // shortcut for PRINT command + + // Try to tokenise + + jsr tk_pack +!: + lda tk__len_unpacked + beq tokenise_line_char // branch if attempt to tokenise failed + + lda #packed_str_keywords_V2 + sta FRESPC+1 + + jsr tk_search + bcc tokenise_line_keyword_V2 // branch if keyword identified + + // XXX add support for additional keyword lists here + + // Shorten packed keyword candidate and try again + + jsr tk_shorten + +#if HAS_OPCODES_65C02 + bra !- +#else + jmp !- +#endif + +tokenise_line_keyword_V2: + + // .X contains a token ID, starting from 0 + + txa + pha // store the token on the stack, we will need it for REM support + + // Store the token + + ldx tk__offset + clc + adc #$80 + sta BUF, x + + // Cut away unnecessary bytes + + inc tk__offset + ldx tk__offset + + dec tk__len_unpacked // keyword length, afterwards number of bytes to cut away + lda tk__len_unpacked + clc + adc tk__offset + tay +!: + lda BUF, y + sta BUF, x + beq !+ + + inx + iny + bne !- +!: + // Special handling for REM command - after this one nothing more should be tokenised + + pla + cmp #$0F // REM token index + bne tokenise_line_loop + + // FALLTROUGH + +tokenise_line_done: + + // Update line length and quit + + lda tk__length + sta __tokenise_work1 + + rts + +tokenise_line_quote: + + // For the quote, we advance without tokenizing, until the next quote is found + + inc tk__offset + ldx tk__offset + + lda BUF, x + beq tokenise_line_done + + cmp #$22 + beq tokenise_line_char + bne tokenise_line_quote // branch always + +tokenise_line_pi: + + lda #$FF // token for PI + skip_2_bytes_trash_nvz + + // FALLTROUGH + +tokenise_line_question_mark: + + lda #$99 // token for PRINT + sta BUF, x + + // FALLTROUGH + +tokenise_line_char: + + inc tk__offset + jmp tokenise_line_loop + +#endif // ROM layout diff --git a/tools/generate_strings.cc b/tools/generate_strings.cc new file mode 100644 index 00000000..cc5f3586 --- /dev/null +++ b/tools/generate_strings.cc @@ -0,0 +1,987 @@ +// +// Utility to generate compressed messages and BASIC tokens +// + +// XXX parse the config file to select the string set (STD, M65, X16) and generate feature string +// XXX use DualStream class for logging +// XXX use dictionary compression by finding the set of non-overlapping strings that can be concatenated +// to produce full set of strings; some idea for the algorithm (not sure if proper one) is available here +// https://stackoverflow.com/questions/9195676/finding-the-smallest-number-of-substrings-to-represent-a-set-of-strings +// needs some investigation... + +#include "common.h" + +#include + +#include +#include +#include +#include +#include +#include + +// +// Command line settings +// + +std::string CMD_outFile = "out.s"; + +// +// Type definition for text/tokens to generate +// + +typedef struct StringEntry +{ + bool enabledSTD; // whether enabled for basic build + bool enabledM65; // whether enabled for Mega65 build + bool enabledX16; // whether enabled for Commander X16 build + std::string alias; // alias, for the assembler + std::string string; // string/token + uint8_t abbrevLen = 0; // length of token abbreviation +} StringEntry; + +enum class ListType +{ + KEYWORDS, + STRINGS_BASIC +}; + +typedef struct StringEntryList +{ + ListType type; + std::string name; + std::vector list; +} StringEntryList; + +typedef std::vector StringEncoded; +typedef std::vector StringEncodedList; + +// http://www.classic-games.com/commodore64/cbmtoken.html +// https://www.c64-wiki.com/wiki/BASIC_token + +// BASIC keywords - V2 dialect + +/* + The URLs here provide examples of other BASIC dialects that include the specified commands + or keywords. + All except VERIFY are known with certainty to be part of other BASIC dialects, and thus + cannot be subject to copyright infringement when implemented on a C64. + + The VERIFY Command however appears in other programming languages, and as a single word, + is the obvious complement to LOAD and SAVE when considering the following storage functions: + + 1. Load a file from storage into memory. + 2. Save a file from memory to storage. + 3. Verify that a file has been correctly written from memory to storage. + + Note that verify is the key verb in the behaviour of this function. Therefore it is the + obvious choice for the name for such a function, taking into account the convention of + LOAD and SAVE. Indeed, that this is so is highlighted by the fact that it was used in the + C64 ROM. Therefore there is no exercise of creativity here, and a reasonable person would + likely select this name independently. Also, single words cannot be copyright. + + Further, the list is a list of facts, and thus not copyrightable. + + We sourced the list from a public source, https://www.c64-wiki.com/wiki/BASIC, which has + been public on the internet for a long time, as have many other similar sources. + That website offers the page under the GFDL license, and we can therefore take the list + under those license terms if we wish, should the need arise. + + Note that in providing these justifications above, we do not in any way acknowledge that + without these conditions that the BASIC keyword list could be copyright, but provide this + information as a further layer of defence against any claims that it could somehow + infringe on the copyrights of the C64/C65/C128 etc ROMs. +*/ + +const StringEntryList GLOBAL_Keywords_V2 = { ListType::KEYWORDS, "keywords_V2", +{ + // STD M65 X16 + { true, true, true, "KV2_80", "END", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_81", "FOR", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_82", "NEXT", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_83", "DATA", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_84", "INPUT#", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php + { true, true, true, "KV2_85", "INPUT", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_86", "DIM", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_87", "READ", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_88", "LET", 2 }, // https://en.wikipedia.org/wiki/Atari_BASIC + { true, true, true, "KV2_89", "GOTO", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_8A", "RUN", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_8B", "IF", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_8C", "RESTORE", 3 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_8D", "GOSUB", 3 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_8E", "RETURN", 3 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_8F", "REM", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_90", "STOP", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_91", "ON", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_92", "WAIT", 2 }, // http://www.picaxe.com/BASIC-Commands/Time-Delays/wait/ + { true, true, true, "KV2_93", "LOAD", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_94", "SAVE", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_95", "VERIFY", 2 }, // https://en.wikipedia.org/wiki/Sinclair_BASIC + { true, true, true, "KV2_96", "DEF", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_97", "POKE", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_98", "PRINT#", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php + { true, true, true, "KV2_99", "PRINT", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_9A", "CONT", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_9B", "LIST", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_9C", "CLR", 2 }, // Apple I Replica Creation: Back to the Garage, p125 + { true, true, true, "KV2_9D", "CMD", 2 }, // https://en.wikipedia.org/wiki/List_of_DOS_commands + { true, true, true, "KV2_9E", "SYS", 2 }, // https://www.lifewire.com/dos-commands-4070427 + { true, true, true, "KV2_9F", "OPEN", 2 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php + // STD M65 X16 + { true, true, true, "KV2_A0", "CLOSE", 3 }, // https://www.atariarchives.org/creativeatari/Using_Disks_With_Atari_Basic.php + { true, true, true, "KV2_A1", "GET", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_A2", "NEW", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_A3", "TAB(", 2 }, // http://www.antonis.de/qbebooks/gwbasman/tab.html + { true, true, true, "KV2_A4", "TO", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_A5", "FN", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_A6", "SPC(", 2 }, // http://www.antonis.de/qbebooks/gwbasman/spc.html + { true, true, true, "KV2_A7", "THEN", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_A8", "NOT", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_A9", "STEP", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_AA", "+", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_AB", "-", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_AC", "*", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_AD", "/", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_AE", "^", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_AF", "AND", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_B0", "OR", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_B1", ">", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_B2", "=", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_B3", "<", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_B4", "SGN", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_B5", "INT", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_B6", "ABS", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_B7", "USR", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_B8", "FRE", 2 }, // http://www.antonis.de/qbebooks/gwbasman/fre.html + { true, true, true, "KV2_B9", "POS", 0 }, // http://www.antonis.de/qbebooks/gwbasman/pos.html + { true, true, true, "KV2_BA", "SQR", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_BB", "RND", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_BC", "LOG", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_BD", "EXP", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_BE", "COS", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_BF", "SIN", 2 }, // https://www.landsnail.com/a2ref.htm + // STD M65 X16 + { true, true, true, "KV2_C0", "TAN", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_C1", "ATN", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_C2", "PEEK", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_C3", "LEN", 0 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_C4", "STR$", 3 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_C5", "VAL", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_C6", "ASC", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_C7", "CHR$", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_C8", "LEFT$", 3 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_C9", "RIGHT$", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_CA", "MID$", 2 }, // https://www.landsnail.com/a2ref.htm + { true, true, true, "KV2_CB", "GO", 0 }, // https://en.wikipedia.org/wiki/Goto +} }; + +// extended BASIC keywords - reserved for generic (hardware independent) BASIC commands + +const StringEntryList GLOBAL_Keywords_CC = { ListType::KEYWORDS, "keywords_CC", +{ + // STD M65 X16 + { false, false, false, "KCC_01", "BLOAD", }, // http://www.antonis.de/qbebooks/gwbasman/bload.html + { false, false, false, "KCC_02", "BSAVE", }, // http://www.antonis.de/qbebooks/gwbasman/bload.html + { false, false, false, "KCC_03", "BVERIFY", }, +} }; + +// extended BASIC keywords - reserved for BASIC commands likely making sense on most C64-compatible machines + +const StringEntryList GLOBAL_Keywords_CD = { ListType::KEYWORDS, "keywords_CD", +{ + // STD M65 X16 + { false, false, false, "KCD_01", "SLOW", }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant + { false, false, false, "KCD_02", "FAST", }, // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant +} }; + +// extended BASIC keywords - reserved for board-specific BASIC commands + +const StringEntryList GLOBAL_Keywords_CE = { ListType::KEYWORDS, "keywords_CE", +{ + // STD M65 X16 + { false, false, false, "KCE_01", "TESTCMD", }, +} }; + +const StringEntryList GLOBAL_Keywords_CF = { ListType::KEYWORDS, "keywords_CF", +{ + // STD M65 X16 + { false, false, false, "KCF_01", "TESTCMD", }, +} }; + +// extended BASIC keywords - reserved for generic (hardware independent) BASIC functions + +const StringEntryList GLOBAL_Keywords_D0 = { ListType::KEYWORDS, "keywords_D0", +{ + // STD M65 X16 + { false, false, false, "KD0_01", "TESTFUN", }, +} }; + +// extended BASIC keywords - reserved for BASIC functions likely making sense on most C64-compatible machines + +const StringEntryList GLOBAL_Keywords_D1 = { ListType::KEYWORDS, "keywords_D1", +{ + // STD M65 X16 + { false, false, false, "KD1_01", "TESTFUN", }, +} }; + +// extended BASIC keywords - reserved for board-specific BASIC functions + +const StringEntryList GLOBAL_Keywords_D2 = { ListType::KEYWORDS, "keywords_D2", +{ + // STD M65 X16 + { false, false, false, "KD2_01", "TESTFUN", }, +} }; + +const StringEntryList GLOBAL_Keywords_D3 = { ListType::KEYWORDS, "keywords_D3", +{ + // STD M65 X16 + { false, false, false, "KD3_01", "TESTFUN", }, +} }; + + +// BASIC errors - all dialects + +/* + These error messages are generally so well known, that it seems silly to have to even point out that they are + so widely used and known, if not completely genericised. + The most of them are inherited from the MICROSOFT BASIC on which the C64's BASIC is based, so for those, there + cannot even be any claim to copyright over them in any exclusive way. + + However, copyright of such short phrases that express ideas in short form cannot be copyrighted: + https://fairuse.stanford.edu/2003/09/09/copyright_protection_for_short/ + */ + +const StringEntryList GLOBAL_Errors = { ListType::STRINGS_BASIC, "errors", +{ + // STD M65 X16 --- error strings compatible with CBM BASIC V2 + { true, true, true, "EV2_01", "TOO MANY FILES" }, + { true, true, true, "EV2_02", "FILE OPEN" }, + { true, true, true, "EV2_03", "FILE NOT OPEN" }, + { true, true, true, "EV2_04", "FILE NOT FOUND" }, + { true, true, true, "EV2_05", "DEVICE NOT PRESENT" }, + { true, true, true, "EV2_06", "NOT INPUT FILE" }, + { true, true, true, "EV2_07", "NOT OUTPUT FILE" }, + { true, true, true, "EV2_08", "MISSING FILENAME" }, + { true, true, true, "EV2_09", "ILLEGAL DEVICE NUMBER" }, + { true, true, true, "EV2_0A", "NEXT WITHOUT FOR" }, + { true, true, true, "EV2_0B", "SYNTAX" }, + { true, true, true, "EV2_0C", "RETURN WITHOUT GOSUB" }, + { true, true, true, "EV2_0D", "OUT OF DATA" }, + { true, true, true, "EV2_0E", "ILLEGAL QUANTITY" }, + { true, true, true, "EV2_0F", "OVERFLOW" }, + { true, true, true, "EV2_10", "OUT OF MEMORY" }, + { true, true, true, "EV2_11", "UNDEF\'D STATEMENT" }, + { true, true, true, "EV2_12", "BAD SUBSCRIPT" }, + { true, true, true, "EV2_13", "REDIM\'D ARRAY" }, + { true, true, true, "EV2_14", "DIVISION BY ZERO" }, + { true, true, true, "EV2_15", "ILLEGAL DIRECT" }, + { true, true, true, "EV2_16", "TYPE MISMATCH" }, + { true, true, true, "EV2_17", "STRING TOO LONG" }, + { true, true, true, "EV2_18", "FILE DATA" }, + { true, true, true, "EV2_19", "FORMULA TOO COMPLEX" }, + { true, true, true, "EV2_1A", "CAN\'T CONTINUE" }, + { true, true, true, "EV2_1B", "UNDEF\'D FUNCTION" }, + { true, true, true, "EV2_1C", "VERIFY" }, + { true, true, true, "EV2_1D", "LOAD" }, + { true, true, true, "EV2_1E", "BREAK" }, + // STD M65 X16 --- error strings compatible with CBM BASIC V7 + { false, false, false, "EV7_1F", "CAN'T RESUME" }, // not used for now + { false, false, false, "EV7_20", "LOOP NOT FOUND" }, // not used for now + { false, false, false, "EV7_21", "LOOP WITHOUT DO" }, // not used for now + { false, false, false, "EV7_22", "DIRECT MODE ONLY" }, // not used for now + { false, false, false, "EV7_23", "NO GRAPHICS AREA" }, // not used for now + { false, false, false, "EV7_24", "BAD DISK" }, // not used for now + { false, false, false, "EV7_25", "BEND NOT FOUND" }, // not used for now + { false, false, false, "EV7_26", "LINE NUMBER TOO LARGE" }, // not used for now + { false, false, false, "EV7_27", "UNRESOLVED REFERENCE" }, // not used for now + { true, true, true, "EV7_28", "NOT IMPLEMENTED" }, // this message actually differs rom CBM one + { false, false, false, "EV7_29", "FILE READ" }, // not used for now + // STD M65 X16 --- error strings specific to OpenROMs, not present in CBM BASIC dialects + { true, true, true, "EOR_2A", "MEMORY CORRUPT" }, +} }; + +// BASIC errors - miscelaneous strings + +const StringEntryList GLOBAL_MiscStrings = { ListType::STRINGS_BASIC, "misc", +{ + // STD M65 X16 --- misc strings as on CBM machines + { true, true, true, "STR_BYTES", " BASIC BYTES FREE" }, // https://github.com/stefanhaustein/expressionparser + { true, true, true, "STR_READY", "READY.\r" }, // https://www.ibm.com/support/knowledgecenter/zosbasics/com.ibm.zos.zconcepts/zconc_whatistsonative.htm https://github.com/stefanhaustein/expressionparser + { true, true, true, "STR_ERROR", " ERROR" }, // simply the word error that is attached to the other parts of messages https://fjkraan.home.xs4all.nl/comp/apple2faq/app2asoftfaq.html + { true, true, true, "STR_IN", " IN " }, + // STD M65 X16 --- misc strings specific to OpenROMs, not present in CBM ROMs + { true, true, true, "STR_BRK_AT", "BRK AT $" }, +} }; + + +/* NOTE: below are the BASIC V7/V10 keywords, see: + - https://sites.google.com/site/h2obsession/CBM/basic/tokens + - http://www.zimmers.net/anonftp/pub/cbm/programming/cbm-basic-tokens.txt + + "RGR" // $CC + "RCLR" // $CD + + "JOY" // $CF // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "RDOT" // $D0 + "DEC" // $D1 // Amos Professional Manual, Command Index + "HEX$" // $D2 // http://www.antonis.de/qbebooks/gwbasman/hexs.html + "ERR$" // $D3 // Amos Professional Manual, Command Index + "INSTR" // $D4 // Amos Professional Manual, Command Index + "ELSE" // $D5 // Amos Professional Manual, Command Index + "RESUME" // $D6 // Amos Professional Manual, Command Index + "TRAP" // $D7 // Amos Professional Manual, Command Index + "TRON" // $D8 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "TROFF" // $D9 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "SOUND" // $DA // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "VOL" // $DB + "AUTO" // $DC // http://www.antonis.de/qbebooks/gwbasman/auto.html + "PUDEF" // $DD + "GRAPHIC" // $DE + "PAINT" // $DF // Amos Professional Manual, Command Index + "CHAR" // $E0 // https://www.c64-wiki.com/wiki/Screen_Graphics_64 + "BOX" // $E1 // Amos Professional Manual, Command Index + "CIRCLE" // $E2 // https://en.wikipedia.org/wiki/Sinclair_BASIC + "PASTE" // $E3 // Amos Professional Manual, Command Index + "CUT" // $E4 + "LINE" // $E5 // https://en.wikipedia.org/wiki/Sinclair_BASIC + "LOCATE" // $E6 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "COLOR" // $E7 + "SCNCLR" // $E8 + "SCALE" // $E9 // https://www.c64-wiki.com/wiki/Super_Expander_64 + "HELP" // $EA // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7 + "DO" // $EB // Amos Professional Manual, Command Index + "LOOP" // $EC // Amos Professional Manual, Command Index + "EXIT" // $ED // Amos Professional Manual, Command Index + "DIR" // $EE // Amos Professional Manual, Command Index + "DSAVE" // $EF // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "DLOAD" // $F0 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "HEADER" // $F1 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "SCRATCH" // $F2 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "COLLECT" // $F3 + "COPY" // $F4 // https://en.wikipedia.org/wiki/Sinclair_BASIC + "RENAME" // $F5 // Amos Professional Manual, Command Index + "BACKUP" // $F6 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "DELETE" // $F7 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "RENUMBER" // $F8 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "KEY" // $F9 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "MONITOR" // $FA // Amos Professional Manual, Command Index + "USING" // $FB // http://www.antonis.de/qbebooks/gwbasman/printusing.html + "UNTIL" // $FC // Amos Professional Manual, Command Index + "WHILE" // $FD // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + + "POT" // $CE $02 + "BUMP" // $CE $03 + "PEN" // $CE $04 // https://www.qsl.net/hb9xch/computer/amstrad/locomotivebasic.html + "RSPPOS" // $CE $05 // https://www.c64-wiki.com/wiki/Super_Expander_64 + "RSPRITE" // $CE $06 + "RSPCOLOR" // $CE $07 + "XOR" // $CE $08 + "RWINDOW" // $CE $09 + "POINTER" // $CE $0A + + "BANK" // $FE $02 // Amos Professional Manual, Command Index + "FILTER" // $FE $03 // https://www.c64-wiki.com/wiki/Super_Expander_64 + "PLAY" // $FE $04 // Amos Professional Manual, Command Index + "TEMPO" // $FE $05 // Amos Professional Manual, Command Index + "MOVSPR" // $FE $06 + "SPRITE" // $FE $07 // Amos Professional Manual, Command Index + "SPRCOLOR" // $FE $08 + "RREG" // $FE $09 + "ENVELOPE" // $FE $0A + "SLEEP" // $FE $0B + "CATALOG" // $FE $0C // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "DOPEN" // $FE $0D + "APPEND" // $FE $0E // Amos Professional Manual, Command Index + "DCLOSE" // $FE $0F + "BSAVE" // $FE $10 // http://www.antonis.de/qbebooks/gwbasman/bsave.html + "BLOAD" // $FE $11 // http://www.antonis.de/qbebooks/gwbasman/bload.html + "RECORD" // $FE $12 + "CONCAT" // $FE $13 + "DVERIFY" // $FE $14 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "DCLEAR" // $FE $15 + "SPRSAV" // $FE $16 // https://www.c64-wiki.com/wiki/Super_Expander_64 + "COLLISION" // $FE $17 + "BEGIN" // $FE $18 + "BEND" // $FE $19 + "WINDOW" // $FE $1A // http://www.antonis.de/qbebooks/gwbasman/window.html + "BOOT" // $FE $1B + "WIDTH" // $FE $1C // http://www.antonis.de/qbebooks/gwbasman/width.html + "SPRDEF" // $FE $1D // https://www.c64-wiki.com/wiki/Super_Expander_64 + "QUIT" // $FE $1E // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7 + + "STASH" // $FE $1F - v7 + "FETCH" // $FE $21 - v7 + "SWAP" // $FE $23 - v7 // http://www.antonis.de/qbebooks/gwbasman/swap.html + + "DMA" // $FE $17/$21/$23 - v10 + + "OFF" // $FE $24 // http://www.classic-games.com/commodore64/cbmtoken.html, Speech Basic 2.7 + "FAST" // $FE $25 // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant + "SLOW" // $FE $26 // https://en.wikipedia.org/wiki/Sinclair_BASIC - ZX81 variant + "TYPE" // $FE $27 + "BVERIFY" // $FE $28 + "DIRECTORY" // $FE $29 // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "ERASE" // $FE $2A // https://en.wikipedia.org/wiki/Sinclair_BASIC + "FIND" // $FE $2B // http://www.classic-games.com/commodore64/cbmtoken.html, AtBasic (@Basic) + "CHANGE" // $FE $2C + "SET" // $FE $2D + "SCREEN" // $FE $2E // http://www.antonis.de/qbebooks/gwbasman/screens.html + "POLYGON" // $FE $2F // Amos Professional Manual, Command Index + "ELLIPSE" // $FE $30 // Amos Professional Manual, Command Index + "VIEWPORT" // $FE $31 + "GCOPY" // $FE $32 + "PEN" // $FE $33 // Amos Professional Manual, Command Index + "PALETTE" // $FE $34 // Amos Professional Manual, Command Index + "DMODE" // $FE $35 + "DPAT" // $FE $36 + "PIC" // $FE $37 + "GENLOCK" // $FE $38 + "FOREGROUND" // $FE $39 + + "BACKGROUND" // $FE $3B + "BORDER" // $FE $3C // https://en.wikipedia.org/wiki/Sinclair_BASIC + "HIGHLIGHT" // $FE $3D +*/ + +// +// Work class definitions +// + +class DataSet +{ +public: + + void addStrings(const StringEntryList &stringList); + + const std::string &getOutput(); + +private: + + void process(); + + void calculateFrequencies(); + void encodeStrings(); + + void encodeByFreq(const std::string &plain, StringEncoded &encoded) const; + + void prepareOutput(); + + void putCharEncoding(std::ostringstream &stream, uint8_t idx, char character, bool is3n); + + virtual bool isRelevant(const StringEntry &entry) const = 0; + virtual std::string layoutName() const = 0; + + std::vector stringEntryLists; + std::vector stringEncodedLists; + + std::vector as1n; // list of bytes to be encoded as 1 nibble + std::vector as3n; // list of bytes to be encoded as 3 nibbles + + uint8_t tk__packed_as_3n = 0; + uint8_t tk__max_keyword_len = 0; + + size_t maxAliasLen = 0; + std::string outFileContent; +}; + +class DataSetSTD : public DataSet +{ + bool isRelevant(const StringEntry &entry) const { return entry.enabledSTD; } + std::string layoutName() const { return "STD"; } +}; + +class DataSetM65 : public DataSet +{ + bool isRelevant(const StringEntry &entry) const { return entry.enabledM65; } + std::string layoutName() const { return "M65"; } +}; + +class DataSetX16 : public DataSet +{ + bool isRelevant(const StringEntry &entry) const { return entry.enabledX16; } + std::string layoutName() const { return "X16"; } +}; + +// +// Work class implementation +// + +void DataSet::addStrings(const StringEntryList &stringList) +{ + // Import the new list of strings + + stringEntryLists.push_back(stringList); + + // Clear strings not relevant for the current configuration + + while (1) + { + if (isRelevant(stringEntryLists.back().list.back())) break; + + stringEntryLists.back().list.pop_back(); + if (stringEntryLists.back().list.empty()) + { + ERROR(std::string("no valid strings in layout '") + layoutName() + "', list'" + stringEntryLists.back().name + "'"); + } + } + + // Clear the output content - make sure it is not valid anymore + + outFileContent.clear(); +} + +void DataSet::process() +{ + std::cout << "Processing layout '" << layoutName() << "'" << std::endl; + + calculateFrequencies(); + encodeStrings(); + prepareOutput(); +} + +const std::string &DataSet::getOutput() +{ + if (outFileContent.empty()) + { + process(); + } + + return outFileContent; +} + +void DataSet::calculateFrequencies() +{ + as1n.clear(); + as3n.clear(); + + std::map freqMapGeneral; // general character frequency map + std::map freqMapKeywords; // frequency map for keywords + + // Calculate frequencies of characters in the strings + + for (const auto &stringEntryList : stringEntryLists) + { + for (const auto &stringEntry : stringEntryList.list) + { + if (!isRelevant(stringEntry)) continue; + + // Check for maximum allowed string length + if (stringEntry.string.length() > 255) ERROR("string cannot be longer than 255 characters"); + + // Check mor maximum keyword length + if (stringEntryList.type == ListType::KEYWORDS) + { + if (stringEntry.string.length() > 8) ERROR("keyword cannot be longer than 8 characters"); + tk__max_keyword_len = std::max(tk__max_keyword_len, (uint8_t) stringEntry.string.length()); + } + + // Update maximum alias length too + maxAliasLen = std::max(maxAliasLen, stringEntry.alias.length()); + + for (const auto &character : stringEntry.string) + { + if (character >= 0x80) + { + ERROR(std::string("character above 0x80 in string '") + stringEntry.string + "'"); + } + + freqMapGeneral[character]++; + if (stringEntryList.type == ListType::KEYWORDS) freqMapKeywords[character]++; + } + } + } + + // Sort characters by frequency + + std::vector freqVector1; + + for (auto iter = freqMapGeneral.begin(); iter != freqMapGeneral.end(); ++iter) + { + freqVector1.push_back(iter->first); + } + + std::sort(freqVector1.begin(), freqVector1.end(), [&freqMapGeneral](char e1, char e2) + { return freqMapGeneral[e2] < freqMapGeneral[e1]; }); + + // Check if minimal amount of characters needed, below 15 is not supported by the 6502 side code + + if (freqVector1.size() < 15) ERROR(std::string("not enough distinct characters in layout '") + layoutName() + "', at least 15 needed"); + + // Extract 14 most frequent characters to be encoded as 1 nibble + + for (uint8_t idx = 0; idx < 14; idx++) + { + as1n.push_back(freqVector1[idx]); + } + + // Now sort them by frequency in keywords, in descending order - this will speed up the tokenizer a little + + std::sort(as1n.begin(), as1n.end(), [&freqMapKeywords](char e1, char e2) { return freqMapKeywords[e2] > freqMapKeywords[e1]; }); + + // Extract characters to be encoded as 3 nibbles, which actually exist in keywords + + std::vector freqVector2; + + tk__packed_as_3n = 0; + for (uint8_t idx = 14; idx < freqVector1.size(); idx++) + { + const auto &character = freqVector1[idx]; + + if (freqMapKeywords[character] > 0) + { + as3n.push_back(character); + tk__packed_as_3n++; + } + else + { + freqVector2.push_back(character); + } + } + tk__packed_as_3n = std::max(tk__packed_as_3n, (uint8_t) 1); + + // Again, sort them by frequency in keywords, in descending order - this will speed up the tokenizer a little + + std::sort(as3n.begin(), as3n.end(), [&freqMapKeywords](char e1, char e2) { return freqMapKeywords[e2] > freqMapKeywords[e1]; }); + + // Finally extract the remaining characters to be encoded as 3 nibbles + + for (const auto &character : freqVector2) + { + as3n.push_back(character); + } +} + +void DataSet::encodeByFreq(const std::string &plain, StringEncoded &encoded) const +{ + bool fullByte = true; + + auto push1n = [&](uint8_t val) // push 1 nibble - encoded character or 0xF mark + { + if (fullByte) + { + encoded.push_back(val); + fullByte = false; + } + else + { + encoded.back() += val * 0x10; + fullByte = true; + } + }; + + auto push2n = [&](uint8_t val) // push the remaining nibbles for 3-niobble encoded characters + { + if (fullByte) + { + encoded.push_back(val); + } + else + { + // Encode byte in a way to be easily decoded by 6502 + encoded.back() += (val / 0x10) * 0x10; + encoded.push_back(val % 0x10); + } + }; + + // Encode every single character by frequency, put them in the output vector + + for (const char &character : plain) + { + auto iterEncoding1 = std::find(as1n.begin(), as1n.end(), character); + if (iterEncoding1 != as1n.end()) + { + push1n(std::distance(as1n.begin(), iterEncoding1) + 1); + } + else + { + push1n(0x0F); + + auto iterEncoding3 = std::find(as3n.begin(), as3n.end(), character); + if (iterEncoding3 == as3n.end()) + { + ERROR("internal error in 'encodeByFreq'"); + } + push2n(std::distance(as3n.begin(), iterEncoding3) + 1); + } + } + + // Make sure the last byte of encoded stream is 0 + + if (encoded.size() == 0 || encoded.back() != 0) encoded.push_back(0); +} + +void DataSet::encodeStrings() +{ + stringEncodedLists.clear(); + + // Encode every relevant string from every list - by character frequency + + for (const auto &stringEntryList : stringEntryLists) + { + stringEncodedLists.emplace_back(); + auto &stringEncodedList = stringEncodedLists.back(); + + for (const auto &stringEntry : stringEntryList.list) + { + stringEncodedList.emplace_back(); + auto &stringEncoded = stringEncodedList.back(); + + if (isRelevant(stringEntry)) + { + encodeByFreq(stringEntry.string, stringEncoded); + } + } + } +} + +void DataSet::putCharEncoding(std::ostringstream &stream, uint8_t idx, char character, bool is3n) +{ + stream << "\t.byte $" << std::uppercase << std::hex << + std::setfill('0') << std::setw(2) << +character << + " // " << std::setfill(is3n ? '0' : ' ') << std::setw(2) << +idx; + + std::string petscii; + + if (character == 0x20) + { + petscii = " = SPACE"; + } + else if (character == 0x27) + { + petscii = " = APOSTROPHE"; + } + else if (character == 0x0D) + { + petscii = " = RETURN"; + } + else if (character > ' ' && character < 'a') + { + petscii = std::string(" = '") + character + "'"; + } + + stream << petscii << std::endl; +} + +void DataSet::prepareOutput() +{ + // Convert our encoded strings to a KickAssembler source + + std::ostringstream stream; + stream << std::endl << "#if ROM_LAYOUT_" << layoutName() << std::endl; + + uint8_t idx; + + // Export all nibble-encoded characters + + stream << std::endl << ".macro put_packed_as_1n() // characters encoded as 1 nibble" << std::endl << "{" << std::endl; + + idx = 0; + for (const auto& encoding : as1n) + { + putCharEncoding(stream, ++idx, encoding, false); + } + + stream << "}" << std::endl; + + // Export all byte-encoded characters + + stream << std::endl << ".macro put_packed_as_3n() // characters encoded as 3 nibbles" << std::endl << "{" << std::endl; + + idx = 0; + for (const auto& encoding : as3n) + { + if (idx == tk__packed_as_3n) stream << std::endl << + "\t// Characters below are not used by any BASIC keyword" << std::endl << std::endl; + + putCharEncoding(stream, ++idx, encoding, true); + } + + stream << "}" << std::endl; + + // Export additional data for the tokenizer + + stream << std::endl << ".label TK__PACKED_AS_3N = $" << std::hex << +tk__packed_as_3n << + std::endl << ".label TK__MAX_KEYWORD_LEN = " << std::dec << +tk__max_keyword_len << std::endl; + + // Export frequency-encoded strings + + for (uint8_t idxList = 0; idxList < stringEntryLists.size(); idxList++) + { + const auto &stringEntryList = stringEntryLists[idxList]; + const auto &stringEncodedList = stringEncodedLists[idxList]; + + stream << std::endl; + for (uint8_t idxString = 0; idxString < stringEncodedList.size(); idxString++) + { + const auto &stringEntry = stringEntryList.list[idxString]; + const auto &stringEncoded = stringEncodedList[idxString]; + + if (!stringEncoded.empty()) + { + stream << ".label IDX__" << stringEntry.alias << std::string(maxAliasLen - stringEntry.alias.length(), ' ') << + " = $" << std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +idxString << std::endl; + } + } + + stream << std::endl << ".macro put_freq_packed_" << stringEntryList.name << "()" << std::endl << "{" << std::endl; + + enum LastStr { NONE, SKIPPED, WRITTEN } lastStr = LastStr::NONE; + for (uint8_t idxString = 0; idxString < stringEncodedList.size(); idxString++) + { + const auto &stringEntry = stringEntryList.list[idxString]; + const auto &stringEncoded = stringEncodedList[idxString]; + + if (stringEncoded.empty()) + { + if (lastStr == LastStr::WRITTEN) stream << std::endl; + stream << "\t.byte $00 // skipped " << stringEntry.alias << std::endl; + lastStr = LastStr::SKIPPED; + } + else + { + if (lastStr != LastStr::NONE) stream << std::endl; + + stream << "\t// IDX__" << stringEntry.alias << std::endl; + stream << "\t.byte "; + + bool first = true; + for (const auto &charEncoded : stringEncoded) + { + if (first) + { + first = false; + } + else + { + stream << ", "; + } + + stream << "$" << std::uppercase << std::hex << std::setfill('0') << std::setw(2) << +charEncoded; + } + + stream << std::endl; + lastStr = LastStr::WRITTEN; + } + } + + // For the token list - pu the 'end of keywords' mark + + if (stringEntryList.type == ListType::KEYWORDS) + { + stream << std::endl << "\t// Marker - end of the keyword list" << std::endl; + stream << "\t.byte $FF, $FF" << std::endl; + } + + stream << "}" << std::endl; + } + + // Finalize the file stream + + stream << std::endl << "#endif // ROM_LAYOUT_" << layoutName() << std::endl; + outFileContent = stream.str(); +} + + +// +// Common helper functions +// + +void printUsage() +{ + std::cout << "\n" << + "usage: generate_strings [-o ]" << "\n\n"; +} + +void printBanner() +{ + printBannerLineTop(); + std::cout << "// Generating compressed messages and BASIC tokens" << "\n"; + printBannerLineBottom(); +} + +void parseCommandLine(int argc, char **argv) +{ + int opt; + + // Retrieve command line options + + while ((opt = getopt(argc, argv, "o:")) != -1) + { + switch(opt) + { + case 'o': CMD_outFile = optarg; break; + default: printUsage(); ERROR(); + } + } +} + +void writeStrings() +{ + DataSetSTD dataSetSTD; + DataSetM65 dataSetM65; + DataSetX16 dataSetX16; + + // Add input data to computation objects + + dataSetSTD.addStrings(GLOBAL_Keywords_V2); + dataSetSTD.addStrings(GLOBAL_Errors); + dataSetSTD.addStrings(GLOBAL_MiscStrings); + + dataSetM65.addStrings(GLOBAL_Keywords_V2); + dataSetM65.addStrings(GLOBAL_Errors); + dataSetM65.addStrings(GLOBAL_MiscStrings); + + dataSetX16.addStrings(GLOBAL_Keywords_V2); + dataSetX16.addStrings(GLOBAL_Errors); + dataSetX16.addStrings(GLOBAL_MiscStrings); + + // Retrieve the results + + std::string outputSTD = dataSetSTD.getOutput(); + std::string outputM65 = dataSetM65.getOutput(); + std::string outputX16 = dataSetX16.getOutput(); + + // Remove old file + + unlink(CMD_outFile.c_str()); + + // Open output file for writing + + std::ofstream outFile(CMD_outFile, std::fstream::out | std::fstream::trunc); + if (!outFile.good()) ERROR(std::string("can't open oputput file '") + CMD_outFile + "'"); + + // Write header + + outFile << "//\n// Generated file - do not edit\n//"; + + // Write packed strings + + outFile << std::endl << std::endl; + outFile << outputSTD; + outFile << std::endl << std::endl; + outFile << outputM65; + outFile << std::endl << std::endl; + outFile << outputX16; + outFile << std::endl << std::endl; + + // Close the file + + outFile.close(); + + std::cout << std::string("Compressed strings written to: ") + CMD_outFile + "\n\n"; +} + +// +// Main function +// + +int main(int argc, char **argv) +{ + printBanner(); + parseCommandLine(argc, argv); + + writeStrings(); + + return 0; +} diff --git a/tools/compress_text.c b/tools/outdated/compress_text.c similarity index 100% rename from tools/compress_text.c rename to tools/outdated/compress_text.c