new text compression and tokeniser - faster, about 100 bytes saved
This commit is contained in:
FeralChild64 committed
1 parent
51e45e9983
commit
94170f156c
24 files changed
+1848
-872
No files matched your search
@@ -0,0 +1,228 @@
|
||||
// #LAYOUT# STD * #TAKE
|
||||
// #LAYOUT# M65 BASIC_1 #TAKE
|
||||
// #LAYOUT# X16 BASIC_0 #TAKE
|
||||
// #LAYOUT# * * #IGNORE
|
||||
|
||||
//
|
||||
// Packs a keyword candidate using a simple compression scheme,
|
||||
// same as in 'generate_strings.cc'.
|
||||
//
|
||||
// This allows us to search through the compressed tokens directly
|
||||
// using the compressed fragment of BASIC, and thus work out the
|
||||
// token for a given piece of string.
|
||||
//
|
||||
// This also helps ensure that we have an implementation that is totally
|
||||
// different to that of the original C64 BASIC, as well as likely
|
||||
// saving a bit of space in the ROM, especially once we start implementing
|
||||
// larger keyword vocabs for extended BASIC, since the compression and
|
||||
// decompression machinery is of fixed size.
|
||||
//
|
||||
// Input:
|
||||
// - .X = offset of string to pack, address is BUF ($0200) + .X
|
||||
// Output:
|
||||
// - see labels in 'tk_pack'
|
||||
//
|
||||
|
||||
|
||||
tk_pack:
|
||||
|
||||
// Reuse the CPU stack - addresses below $102 are used by 'tokenise_line.s'
|
||||
|
||||
.label tk__len_unpacked = $103 // length of unpacked data; it could replace 'tk__nibble_flag', but at the cost of code size/performance
|
||||
.label tk__shorten_bits = $104 // for quick shortening of packed candidate
|
||||
.label tk__nibble_flag = $105 // $00 = start from new byte, $FF = start from high nibble
|
||||
.label tk__byte_offset = $106 // offset of the current byte (to place new data) in tk__packed
|
||||
|
||||
.label tk__packed = $107 // packed candidate, 13 bytes is enough for worst case - 8 byte keyword
|
||||
|
||||
// Initialize variables
|
||||
|
||||
lda #$00
|
||||
ldy #(tk__packed - tk__len_unpacked + 13) // XXX maybe do not clear that much, just put 0 at the end while compressing
|
||||
!:
|
||||
sta tk__len_unpacked, y
|
||||
dey
|
||||
bpl !-
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_pack_loop:
|
||||
|
||||
// Get the next character
|
||||
|
||||
lda BUF, x
|
||||
|
||||
// If character cannot be a part of a keyword - do not pack anything more
|
||||
|
||||
cmp #$23 // '#' - first sane character for a keyword
|
||||
bcc tk_pack_done // branch if below '#'
|
||||
cmp #$7B // 'Z' + 1
|
||||
bcs tk_pack_done // branch if above 'Z'
|
||||
|
||||
// Keywords should not contain digits, colons, or semicolons
|
||||
|
||||
cmp #$30
|
||||
bcc !+ // branch if below '0'
|
||||
cmp #$3C
|
||||
bcc tk_pack_done // branch if below '<'
|
||||
!:
|
||||
// Try to find a nibble to encode the character
|
||||
|
||||
ldy #$0E
|
||||
!:
|
||||
cmp packed_as_1n-1, y
|
||||
beq tk_pack_1n
|
||||
|
||||
dey
|
||||
bne !-
|
||||
|
||||
// Try to find a 3-nibble sequence to encode the character (1st nibble is always 0xF in this case)
|
||||
|
||||
ldy #TK__PACKED_AS_3N
|
||||
!:
|
||||
cmp packed_as_3n-1, y
|
||||
beq tk_pack_3n
|
||||
|
||||
dey
|
||||
bne !-
|
||||
|
||||
// If it is not on the list, it cannot be a part of token - quit
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_pack_done:
|
||||
|
||||
rts
|
||||
|
||||
tk_pack_1n:
|
||||
|
||||
// .Y contains our nibble; check whether we should start a new encoded byte
|
||||
|
||||
bit tk__nibble_flag
|
||||
bpl tk_pack_1n_new_byte
|
||||
|
||||
// Store in the high nibble of the current byte
|
||||
|
||||
tya
|
||||
|
||||
asl
|
||||
asl
|
||||
asl
|
||||
asl
|
||||
|
||||
ldy tk__byte_offset
|
||||
ora tk__packed, y
|
||||
sta tk__packed, y
|
||||
|
||||
// Adjust remaining counters / flags
|
||||
|
||||
inc tk__byte_offset
|
||||
inc tk__nibble_flag // $FF -> $00
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_pack_1n_done:
|
||||
|
||||
// Put the bit mark - to indicate 1-nibble encoding
|
||||
|
||||
clc
|
||||
ror tk__shorten_bits
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_pack_loop_next:
|
||||
|
||||
// Prepare for the next loop iteration - increment counters
|
||||
|
||||
inx
|
||||
inc tk__len_unpacked
|
||||
|
||||
// Check length, if it is OK to pack one more byte; quit if not
|
||||
|
||||
lda tk__len_unpacked
|
||||
cmp #TK__MAX_KEYWORD_LEN
|
||||
|
||||
bne tk_pack_loop
|
||||
rts
|
||||
|
||||
tk_pack_1n_new_byte:
|
||||
|
||||
// Store in the low nibble of the next byte
|
||||
|
||||
tya
|
||||
ldy tk__byte_offset
|
||||
sta tk__packed, y
|
||||
|
||||
// Adjust remaining counters / flags
|
||||
|
||||
dec tk__nibble_flag // $00 -> $FF
|
||||
bmi tk_pack_1n_done
|
||||
|
||||
tk_pack_3n:
|
||||
|
||||
// .Y contains 2 nibbles to encode; check whether we should start a new encoded byte
|
||||
|
||||
bit tk__nibble_flag
|
||||
bpl tk_pack_3n_new_byte
|
||||
|
||||
// Store 1st nibble (alwys 0xF) in the high nibble of the current byte
|
||||
|
||||
phy_trash_a
|
||||
|
||||
lda #$F0
|
||||
ldy tk__byte_offset
|
||||
ora tk__packed, y
|
||||
sta tk__packed, y
|
||||
|
||||
// Store byte in the next byte
|
||||
|
||||
iny
|
||||
pla
|
||||
sta tk__packed, y
|
||||
|
||||
// Adjust remaining counters / flags
|
||||
|
||||
iny
|
||||
sty tk__byte_offset
|
||||
inc tk__nibble_flag // $FF -> $00
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_pack_3n_done:
|
||||
|
||||
// Put the bit mark - to indicate 3-nibble encoding
|
||||
|
||||
sec
|
||||
ror tk__shorten_bits
|
||||
|
||||
bcc tk_pack_loop_next // branch always
|
||||
|
||||
tk_pack_3n_new_byte:
|
||||
|
||||
// First store everything in the current byte, we will set '0xF' mark later
|
||||
|
||||
tya
|
||||
ldy tk__byte_offset
|
||||
sta tk__packed, y
|
||||
|
||||
// Now store just the low nibble in the next byte
|
||||
|
||||
iny
|
||||
and #$0F
|
||||
sta tk__packed, y
|
||||
|
||||
// Go back for a moment and set the '0xF' mark in the low nibble
|
||||
|
||||
dey
|
||||
lda tk__packed, y
|
||||
ora #$0F
|
||||
sta tk__packed, y
|
||||
iny
|
||||
|
||||
// Adjust remaining counters / flags
|
||||
|
||||
iny
|
||||
sty tk__byte_offset
|
||||
dec tk__nibble_flag // $00 -> $FF
|
||||
|
||||
bmi tk_pack_3n_done // branch always
|
||||
@@ -0,0 +1,93 @@
|
||||
// #LAYOUT# STD * #TAKE
|
||||
// #LAYOUT# M65 BASIC_1 #TAKE
|
||||
// #LAYOUT# X16 BASIC_0 #TAKE
|
||||
// #LAYOUT# * * #IGNORE
|
||||
|
||||
//
|
||||
// Search for a token on the list pointed by FRESPC
|
||||
//
|
||||
// Input:
|
||||
// - list address in FRESPC, has to be refreshed after routine call
|
||||
// - uses same variables as 'tk_pack'
|
||||
// Output:
|
||||
// - Carry set = not found
|
||||
// - .X = index of token found
|
||||
//
|
||||
|
||||
|
||||
tk_search:
|
||||
|
||||
ldx #$00 // initial token index
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_search_loop:
|
||||
|
||||
// Check if end of the list
|
||||
|
||||
ldy #$00
|
||||
lda (FRESPC), y
|
||||
cmp #$FF
|
||||
bne tk_search_compare
|
||||
iny
|
||||
lda (FRESPC), y
|
||||
dey
|
||||
cmp #$FF
|
||||
bne tk_search_compare
|
||||
|
||||
// End of the list
|
||||
|
||||
sec
|
||||
rts
|
||||
|
||||
tk_search_compare:
|
||||
|
||||
// Compare the packed candidate with the current keyword list entry
|
||||
|
||||
lda (FRESPC), y
|
||||
cmp tk__packed, y
|
||||
bne tk_search_not_matching
|
||||
|
||||
cmp #$00
|
||||
beq tk_search_found
|
||||
|
||||
// For now everything matches, try the next byte
|
||||
|
||||
iny
|
||||
bne tk_search_compare // branch always
|
||||
|
||||
tk_search_not_matching:
|
||||
|
||||
inx // increment token index
|
||||
|
||||
// We need to find the end of the keyword on the list
|
||||
// and proceed one byte further
|
||||
|
||||
cmp #$00
|
||||
beq tk_search_next
|
||||
!:
|
||||
iny
|
||||
lda (FRESPC), y
|
||||
bne !-
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_search_next:
|
||||
|
||||
// Update FRESPC - advance by .Y
|
||||
|
||||
iny
|
||||
tya
|
||||
clc
|
||||
adc FRESPC+0
|
||||
sta FRESPC+0
|
||||
lda FRESPC+1
|
||||
adc #$00
|
||||
sta FRESPC+1
|
||||
|
||||
bcc tk_search_loop // branch always
|
||||
|
||||
tk_search_found:
|
||||
|
||||
clc
|
||||
rts
|
||||
@@ -0,0 +1,112 @@
|
||||
// #LAYOUT# STD * #TAKE
|
||||
// #LAYOUT# M65 BASIC_1 #TAKE
|
||||
// #LAYOUT# X16 BASIC_0 #TAKE
|
||||
// #LAYOUT# * * #IGNORE
|
||||
|
||||
//
|
||||
// Shortens a keyword candidate previously packed by tk_pack by 1 character
|
||||
//
|
||||
// Input:
|
||||
// - uses same variables as 'tk_pack'
|
||||
//
|
||||
|
||||
|
||||
tk_shorten:
|
||||
|
||||
// First make sure we actually can shorten anything
|
||||
|
||||
dec tk__len_unpacked
|
||||
beq tk_shorten_end
|
||||
|
||||
// Now check if the last character was encoded as 1 nibble or 3 nibbles
|
||||
|
||||
asl tk__nibble_flag
|
||||
bcs tk_shorten_3n
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_shorten_1n:
|
||||
|
||||
ldy tk__byte_offset // will be needed in both cases
|
||||
|
||||
// Now we need to check which nibble to cut away
|
||||
|
||||
bit tk__nibble_flag
|
||||
bcs tk_shorten_1n_lo
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_shorten_1n_hi: // tk__nibble_flag = $00, both nibbles free
|
||||
|
||||
// Go back one byte and clear the high nibble only
|
||||
|
||||
dey
|
||||
sty tk__byte_offset
|
||||
|
||||
lda tk__packed, y
|
||||
and #$F0
|
||||
sta tk__packed, y
|
||||
|
||||
// Adjust remaining counters / flags, and quit
|
||||
|
||||
dec tk__nibble_flag // $00 -> $FF
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_shorten_end:
|
||||
|
||||
rts
|
||||
|
||||
tk_shorten_1n_lo: // tk__nibble_flag = $FF, only the high nibble is free
|
||||
|
||||
// Clear the whole byte, it will be faster
|
||||
|
||||
lda #$00
|
||||
sta tk__packed, y
|
||||
|
||||
// Adjust remaining counters / flags, and quit
|
||||
|
||||
inc tk__nibble_flag // $FF -> $00
|
||||
rts
|
||||
|
||||
tk_shorten_3n:
|
||||
|
||||
ldy tk__byte_offset // will be needed in both cases
|
||||
lda #$00
|
||||
|
||||
// Now we need to check which nibble to cut away
|
||||
|
||||
bit tk__nibble_flag
|
||||
bcs tk_shorten_3n_2bytes
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tk_shorten_3n_byte_nibble: // tk__nibble_flag = $00, both nibbles free
|
||||
|
||||
// Go back one byte, clear it, go back and clear the high nibble only
|
||||
|
||||
dey
|
||||
sta tk__packed, y
|
||||
|
||||
// Reuse 'tk_shorten_1n_hi'
|
||||
|
||||
sty tk__byte_offset
|
||||
#if HAS_OPCODES_65C02
|
||||
bra tk_shorten_1n_hi
|
||||
#else
|
||||
jmp tk_shorten_1n_hi
|
||||
#endif
|
||||
|
||||
tk_shorten_3n_2bytes: // tk__nibble_flag = $FF, only the high nibble is free
|
||||
|
||||
// Clear the byte, go back, clear one more byte
|
||||
|
||||
sta tk__packed, y
|
||||
dey
|
||||
sta tk__packed, y
|
||||
|
||||
// Adjust remaining counters / flags, and quit
|
||||
|
||||
sty tk__byte_offset
|
||||
inc tk__nibble_flag // $FF -> $00
|
||||
rts
|
||||
@@ -0,0 +1,176 @@
|
||||
// #LAYOUT# STD * #TAKE
|
||||
// #LAYOUT# M65 * #TAKE
|
||||
// #LAYOUT# X16 BASIC_0 #TAKE
|
||||
// #LAYOUT# * * #IGNORE
|
||||
|
||||
//
|
||||
// Tokenise a line of BASIC stored at $0200
|
||||
//
|
||||
// Input:
|
||||
// - length in __tokenise_work1
|
||||
//
|
||||
|
||||
|
||||
tokenise_line:
|
||||
|
||||
#if (ROM_LAYOUT_M65 && SEGMENT_BASIC_0)
|
||||
|
||||
jsr map_BASIC_1
|
||||
jsr_ind VB1__tokenise_line
|
||||
jmp map_NORMAL
|
||||
|
||||
#else
|
||||
|
||||
// Reuse the CPU stack - addresses above $101 are used by 'tk_pack.s'
|
||||
|
||||
.label tk__offset = $100 // offset into string
|
||||
.label tk__length = $101 // length of the raw string XXX reuse __tokenise_work1?
|
||||
|
||||
// Initialize variables
|
||||
|
||||
lda #$00
|
||||
sta tk__offset
|
||||
|
||||
ldx __tokenise_work1
|
||||
stx tk__length
|
||||
|
||||
// Terminate input string with $00
|
||||
|
||||
// XXX do we need this?
|
||||
sta BUF, x
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tokenise_line_loop:
|
||||
|
||||
// Check if we have reached end of the line
|
||||
|
||||
lda tk__offset
|
||||
cmp tk__length
|
||||
beq tokenise_line_done
|
||||
|
||||
// Check for some special characters
|
||||
|
||||
ldx tk__offset
|
||||
lda BUF, x
|
||||
|
||||
cmp #$22
|
||||
beq tokenise_line_quote
|
||||
|
||||
cmp #$DE
|
||||
beq tokenise_line_pi
|
||||
|
||||
cmp #$3F
|
||||
beq tokenise_line_question_mark // shortcut for PRINT command
|
||||
|
||||
// Try to tokenise
|
||||
|
||||
jsr tk_pack
|
||||
!:
|
||||
lda tk__len_unpacked
|
||||
beq tokenise_line_char // branch if attempt to tokenise failed
|
||||
|
||||
lda #<packed_str_keywords_V2
|
||||
sta FRESPC+0
|
||||
lda #>packed_str_keywords_V2
|
||||
sta FRESPC+1
|
||||
|
||||
jsr tk_search
|
||||
bcc tokenise_line_keyword_V2 // branch if keyword identified
|
||||
|
||||
// XXX add support for additional keyword lists here
|
||||
|
||||
// Shorten packed keyword candidate and try again
|
||||
|
||||
jsr tk_shorten
|
||||
|
||||
#if HAS_OPCODES_65C02
|
||||
bra !-
|
||||
#else
|
||||
jmp !-
|
||||
#endif
|
||||
|
||||
tokenise_line_keyword_V2:
|
||||
|
||||
// .X contains a token ID, starting from 0
|
||||
|
||||
txa
|
||||
pha // store the token on the stack, we will need it for REM support
|
||||
|
||||
// Store the token
|
||||
|
||||
ldx tk__offset
|
||||
clc
|
||||
adc #$80
|
||||
sta BUF, x
|
||||
|
||||
// Cut away unnecessary bytes
|
||||
|
||||
inc tk__offset
|
||||
ldx tk__offset
|
||||
|
||||
dec tk__len_unpacked // keyword length, afterwards number of bytes to cut away
|
||||
lda tk__len_unpacked
|
||||
clc
|
||||
adc tk__offset
|
||||
tay
|
||||
!:
|
||||
lda BUF, y
|
||||
sta BUF, x
|
||||
beq !+
|
||||
|
||||
inx
|
||||
iny
|
||||
bne !-
|
||||
!:
|
||||
// Special handling for REM command - after this one nothing more should be tokenised
|
||||
|
||||
pla
|
||||
cmp #$0F // REM token index
|
||||
bne tokenise_line_loop
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tokenise_line_done:
|
||||
|
||||
// Update line length and quit
|
||||
|
||||
lda tk__length
|
||||
sta __tokenise_work1
|
||||
|
||||
rts
|
||||
|
||||
tokenise_line_quote:
|
||||
|
||||
// For the quote, we advance without tokenizing, until the next quote is found
|
||||
|
||||
inc tk__offset
|
||||
ldx tk__offset
|
||||
|
||||
lda BUF, x
|
||||
beq tokenise_line_done
|
||||
|
||||
cmp #$22
|
||||
beq tokenise_line_char
|
||||
bne tokenise_line_quote // branch always
|
||||
|
||||
tokenise_line_pi:
|
||||
|
||||
lda #$FF // token for PI
|
||||
skip_2_bytes_trash_nvz
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tokenise_line_question_mark:
|
||||
|
||||
lda #$99 // token for PRINT
|
||||
sta BUF, x
|
||||
|
||||
// FALLTROUGH
|
||||
|
||||
tokenise_line_char:
|
||||
|
||||
inc tk__offset
|
||||
jmp tokenise_line_loop
|
||||
|
||||
#endif // ROM layout
|
||||
Reference in new issue
Block a user