From 06c6bd9c204b6457fd824bbf8f7be9dd00939b04 Mon Sep 17 00:00:00 2001 From: Paul Gardner-Stephen Date: Fri, 26 Apr 2019 22:02:56 +0930 Subject: [PATCH] work on basic tokeniser --- c64/basic/,aliases.s | 1 + c64/basic/basic_main_loop.s | 94 +----------------- c64/basic/packed_messages.s | 122 +++++++++++++++++++++++ c64/basic/tokenise_line.s | 190 ++++++++++++++++++++++++++++++++++++ 4 files changed, 315 insertions(+), 92 deletions(-) create mode 100644 c64/basic/packed_messages.s create mode 100644 c64/basic/tokenise_line.s diff --git a/c64/basic/,aliases.s b/c64/basic/,aliases.s index 26174a2f..c2e3bf27 100644 --- a/c64/basic/,aliases.s +++ b/c64/basic/,aliases.s @@ -7,6 +7,7 @@ .alias tokenise_work3 $B .alias tokenise_work4 $F .alias temp_string_ptr $35 + .alias load_or_verify_or_tokenise_work5 $C .alias TXTTAB $2B .alias IOSTATUS $90 diff --git a/c64/basic/basic_main_loop.s b/c64/basic/basic_main_loop.s index 87abdaad..825a03ba 100644 --- a/c64/basic/basic_main_loop.s +++ b/c64/basic/basic_main_loop.s @@ -27,103 +27,13 @@ got_line_of_input: ;; Store length of input buffer ready for tokenising stx tokenise_work1 - + ;; Do printing of the new line lda #$0d jsr $ffd2 - ;; Pack the word from the start - ldx #$00 - jsr pack_word - - ;; Search for word in keyword list - ;; The packed form is canonical, so we can search for the keyword while - ;; it remains compressed. This also saves a little time. - ;; Algorithm is simply to find matching first bytes, and then look for - ;; the rest being a match. - -token_search: - ;; tokenise_work1 = offset in line of input - ;; tokenise_work2 = length of token we are matching - ;; tokenise_work3 = offset in token list - lda #$00 - sta tokenise_work1 - sta tokenise_work3 -next_kw_offset: - ;; Advance offset in compressed keyword list, to see if a match - ;; BEGINS here - - ;; Get count of bytes to compare - lda tokenise_work2 - sta tokenise_work4 + jsr tokenise_line - ;; Load current position - ldy tokenise_work3 - ldx tokenise_work1 - - ;; Advance offset in compressed token list, and stop at the end - inc tokenise_work3 - lda tokenise_work3 - cmp #$ff - bne next_in_match - jmp done_searching_for_token -next_in_match: - lda $0100,x - cmp packed_keywords,y - bne next_kw_offset - ;; Advance to next chars in source and target - inx - iny - - dec tokenise_work4 ; Have we compared all bytes yet? - - bne next_in_match - - ;; Keyword matches! - - ;; Now make sure we are on a keyword boundary. - ;; Offset 0 - ;; previous byte is $x0 - ;; two byte prior is $FE - - ;; Get offset of first byte of this token - ldx tokenise_work3 - dex ; pointer points one late, so rewind it - - ;; At start of list? - cpx #$00 - beq word_boundary - dex - ;; Previous byte is $x0 end token? - lda packed_keywords,x - and #$0f - beq word_boundary - dex - ;; Uber-previouss byte is $FE final literal token? - lda packed_keywords,x - cmp #$fe - beq word_boundary - - ;; Not a word boundary, so ignore - jmp next_kw_offset - -word_boundary: - inc $d020 - - ;; We actually point one by late, so rewind to find start of token - lda tokenise_work3 - sec - sbc #$01 - clc - adc #packed_keywords - adc #$00 - sta temp_string_ptr+1 - ;; Print the token we found - jsr print_packed_string - -done_searching_for_token: jsr ready_message jmp basic_main_loop diff --git a/c64/basic/packed_messages.s b/c64/basic/packed_messages.s new file mode 100644 index 00000000..93e95c47 --- /dev/null +++ b/c64/basic/packed_messages.s @@ -0,0 +1,122 @@ +packed_message_chars: + .byte $45 ; 'E' = $1 (nybl) + .byte $54 ; 'T' = $2 (nybl) + .byte $4F ; 'O' = $3 (nybl) + .byte $4E ; 'N' = $4 (nybl) + .byte $49 ; 'I' = $5 (nybl) + .byte $52 ; 'R' = $6 (nybl) + .byte $41 ; 'A' = $7 (nybl) + .byte $53 ; 'S' = $8 (nybl) + .byte $4C ; 'L' = $9 (nybl) + .byte $46 ; 'F' = $a (nybl) + .byte $44 ; 'D' = $b (nybl) + .byte $55 ; 'U' = $c (nybl) + .byte $50 ; 'P' = $d (nybl) + .byte $43 ; 'C' = $e (nybl) + .byte $47 ; 'G' = $F1 / $xF $47 + .byte $4D ; 'M' = $F2 / $xF $4D + .byte $59 ; 'Y' = $F3 / $xF $59 + .byte $56 ; 'V' = $F4 / $xF $56 + .byte $42 ; 'B' = $F5 / $xF $42 + .byte $48 ; 'H' = $F6 / $xF $48 + .byte $58 ; 'X' = $F7 / $xF $58 + .byte $57 ; 'W' = $F8 / $xF $57 + .byte $24 ; '$' = $F9 / $xF $24 + .byte $27 ; ''' = $Fa / $xF $27 + .byte $0D ; ' ' = $Fb / $xF $0D + .byte $51 ; 'Q' = $Fc / $xF $51 + .byte $2E ; '.' = $Fd / $xF $2E + .byte $23 ; '#' = $FE/F + $23 + .byte $4B ; 'K' = $FE/F + $4B + .byte $28 ; '(' = $FE/F + $28 + .byte $5A ; 'Z' = $FE/F + $5A + .byte $2B ; '+' = $FE/F + $2B + .byte $2D ; '-' = $FE/F + $2D + .byte $2A ; '*' = $FE/F + $2A + .byte $2F ; '/' = $FE/F + $2F + .byte $5E ; '^' = $FE/F + $5E + .byte $3E ; '>' = $FE/F + $3E + .byte $3D ; '=' = $FE/F + $3D + .byte $3C ; '<' = $FE/F + $3C +packed_message_tokens: + .byte $00,$01,$02,$ff,$03,$04,$ff,$03 + .byte $05,$04,$ff,$03,$05,$06,$ff,$07 + .byte $05,$08,$ff,$05,$09,$03,$ff,$05 + .byte $0a,$03,$ff,$0b,$0c,$ff,$0d,$07 + .byte $0e,$ff,$0f,$10,$11,$ff,$12,$ff + .byte $13,$10,$14,$ff,$15,$16,$17,$ff + .byte $0d,$18,$ff,$19,$ff,$15,$16,$1a + .byte $ff,$1b,$1c,$ff,$1d,$1e,$ff,$1f + .byte $20,$ff,$21,$22,$23,$ff,$0d,$24 + .byte $ff,$25,$26,$ff,$27,$00,$28,$ff + .byte $03,$17,$ff,$29,$00,$2a,$ff,$2b + .byte $2c,$ff,$1b,$2d,$ff,$2e,$ff,$2f + .byte $ff,$30,$ff,$31,$ff,$32,$ff,$33 + .byte $ff + .byte $34 + .byte $ff + .byte $35 + .byte $36 + .byte $ff +packed_message_words: + .byte $23,$30,$f2,$74,$f3,$00,$a5,$91 + .byte $80,$a5,$91,$00,$3d,$14,$00,$43 + .byte $20,$a3,$c4,$b0,$b1,$f4,$5e,$10 + .byte $d6,$18,$14,$20,$54,$dc,$20,$3c + .byte $2d,$c2,$00,$f2,$58,$85,$4f,$47 + .byte $00,$a5,$91,$47,$f2,$10,$59,$91 + .byte $f1,$79,$00,$4c,$f2,$f5,$16,$00 + .byte $41,$f7,$20,$f8,$52,$f6,$3c,$20 + .byte $a3,$60,$8f,$59,$42,$7f,$58,$00 + .byte $61,$2c,$64,$00,$f1,$38,$cf,$42 + .byte $00,$3c,$20,$3a,$00,$b7,$27,$00 + .byte $fc,$c7,$42,$52,$f3,$00,$3f,$56 + .byte $16,$a9,$3f,$57,$00,$f2,$1f,$4d + .byte $36,$f3,$00,$c4,$b1,$af,$27,$b0 + .byte $82,$72,$1f,$4d,$14,$20,$f5,$7b + .byte $00,$8c,$f5,$8e,$65,$d2,$00,$61 + .byte $b5,$f2,$fa,$b0,$76,$67,$f3,$00 + .byte $b5,$f4,$58,$53,$40,$f5,$f3,$00 + .byte $ff,$5a,$16,$30,$b5,$61,$e2,$00 + .byte $2f,$59,$d1,$00,$f2,$58,$f2,$72 + .byte $ef,$48,$00,$82,$65,$4f,$47,$00 + .byte $93,$4f,$47,$00,$a3,$6f,$4d,$c9 + .byte $70,$e3,$f2,$d9,$1f,$58,$00,$e7 + .byte $4f,$27,$20,$e3,$42,$54,$c1,$00 + .byte $ac,$4e,$25,$34,$00,$f4,$16,$5a + .byte $f3,$00,$93,$7b,$00,$61,$7b,$f3 + .byte $fd,$fb,$00,$93,$7b,$54,$f1,$00 + .byte $f4,$16,$5a,$f3,$54,$f1,$00,$87 + .byte $f4,$54,$f1,$00,$16,$63,$60,$f5 + .byte $f3,$21,$80,$a6,$11,$fd,$fb,$fb + .byte $00 +packed_keywords: + .byte $14,$b0,$a3,$60,$41,$f7,$20,$b7 + .byte $27,$00,$54,$dc,$2f,$23,$00,$54 + .byte $dc,$20,$b5,$f2,$00,$61,$7b,$00 + .byte $91,$20,$f1,$32,$30,$6c,$40,$5a + .byte $00,$61,$82,$36,$10,$f1,$38,$cf + .byte $42,$00,$61,$2c,$64,$00,$61,$f2 + .byte $00,$82,$3d,$00,$34,$00,$f8,$75 + .byte $20,$93,$7b,$00,$87,$f4,$10,$f4 + .byte $16,$5a,$f3,$00,$b1,$a0,$d3,$ff + .byte $4b,$10,$d6,$54,$2f,$23,$00,$d6 + .byte $54,$20,$e3,$42,$00,$95,$82,$00 + .byte $e9,$60,$ef,$4d,$b0,$8f,$59,$80 + .byte $3d,$14,$00,$e9,$38,$10,$f1,$12 + .byte $00,$41,$f8,$00,$27,$f5,$fe,$28 + .byte $23,$00,$a4,$00,$8d,$ef,$28,$00 + .byte $2f,$48,$14,$00,$43,$20,$82,$1d + .byte $00,$fe,$2b,$fe,$2d,$fe,$2a,$fe + .byte $2f,$fe,$5e,$74,$b0,$36,$00,$fe + .byte $3e,$fe,$3d,$fe,$3c,$8f,$47,$40 + .byte $54,$20,$7f,$42,$80,$c8,$60,$a6 + .byte $10,$d3,$80,$8f,$51,$60,$64,$b0 + .byte $93,$f1,$00,$1f,$58,$d0,$e3,$80 + .byte $85,$40,$27,$40,$72,$40,$d1,$1f + .byte $4b,$00,$91,$40,$82,$6f,$24,$00 + .byte $f4,$79,$00,$78,$e0,$ef,$48,$6f + .byte $24,$00,$91,$a2,$f9,$00,$65,$f1 + .byte $f6,$2f,$24,$00,$f2,$5b,$f9,$00 + .byte $f1 + .byte $30 diff --git a/c64/basic/tokenise_line.s b/c64/basic/tokenise_line.s new file mode 100644 index 00000000..415ee05a --- /dev/null +++ b/c64/basic/tokenise_line.s @@ -0,0 +1,190 @@ + ;; Tokenise a line of BASIC + ;; Stored at $0200 + ;; Length in tokenise_work1 + +tokenise_line: + + ;; We need to consider each offset in the string to see if it matches + ;; a token. No BASIC keyword is > 7 bytes, so we need to start with that + ;; length (or remaining length of string) and count our way back down. + ;; This means we need separate start and end points to keep track of. + + ;; $0110 = offset into string + ;; $0111 = number of bytes to pack + ;; $0112 = offset into tokenised output + ;; $0113 = length of raw string + + lda #$00 + sta $0110 + sta $0112 + lda tokenise_work1 + sta $0113 + +tokenise_char: + + ;; Check if we have reached end of the line? + lda $0110 + cmp $0113 + bne + + + ;; Finished tokenising line + ;; Update tokenise_work1 and terminate with null + ldx $0112 + lda #$00 + sta $0200,x + stx tokenise_work1 + rts +* + ;; More to do + + ;; Maximum keyword length + lda #7 + sta $0111 + +tokenise_char_loop: + ;; Pack the word from the start (start = X, raw len = tokenise_work1) + ;; Returns packed length in tokenise_work2 and writes the packed + ;; word at $0100+ + + ;; Pack string + ldx $0110 + lda $0111 + sta tokenise_work1 + jsr pack_word + + ;; Search for it in keywords + jsr keyword_search + + ;; Did we find a keyword? + bcc found_token_in_line + + ;; Nope, so try shorter + dec $0111 + bne tokenise_char_loop + + ;; Is not a token, so copy to output verbatim + ldx $0110 + ldy $0112 + lda $0200,x + sta $0200,y + inc $0112 + + ;; Tokenise next char + jmp tokenise_char + +found_token_in_line: + + ;; Got a token, so write it out + ldx $0112 + sta $0200,x + inx + stx $0112 + ;; Now skip over the length of the token + lda $0110 + clc + adc $0111 + sta $0110 + + ;; See if there is more of the line left to process + jmp tokenise_char + + + + + ;; Search for word in keyword list + ;; The packed form is canonical, so we can search for the keyword while + ;; it remains compressed. This also saves a little time. + ;; Algorithm is simply to find matching first bytes, and then look for + ;; the rest being a match. + +keyword_search: + ;; Search for the compressed keyword stored at $0100 in the list + ;; of compressed keywords + + ;; tokenise_work1 = offset in line of input + ;; tokenise_work2 = length of token we are matching + ;; tokenise_work3 = offset in token list + ;; load_or_verify_or_tokenise_work5 = temporary work space + ;; X and Y registers used as temporary space. + ;; returns C=1 if no matching token, else C=0 and A=token + + lda #$00 + sta tokenise_work1 + sta tokenise_work3 +next_kw_offset: + ;; Advance offset in compressed keyword list, to see if a match + ;; BEGINS here + + ;; Get count of bytes to compare + lda tokenise_work2 + sta load_or_verify_or_tokenise_work5 + + ;; Load current position + ldy tokenise_work3 + ldx tokenise_work1 + + ;; Advance offset in compressed token list, and stop at the end + inc tokenise_work3 + lda tokenise_work3 + cmp #$ff + bne next_in_match + jmp done_searching_for_token +next_in_match: + lda $0100,x + cmp packed_keywords,y + bne next_kw_offset + ;; Advance to next chars in source and target + inx + iny + + dec load_or_verify_or_tokenise_work5 ; Have we compared all bytes yet? + + bne next_in_match + + ;; Keyword matches! + + ;; Now make sure we started on a keyword boundary + ;; (we know we finished on one, because of the structure of the compressed + ;; keyword data). + ;; Offset 0 + ;; previous byte is $x0 + ;; two byte prior is $FE + + ;; Get offset of first byte of this token + ldx tokenise_work3 + dex ; pointer points one late, so rewind it + + ;; At start of list? + cpx #$00 + beq word_boundary + dex + ;; Previous byte is $x0 end token? + lda packed_keywords,x + and #$0f + beq word_boundary + dex + ;; Uber-previouss byte is $FE final literal token? + lda packed_keywords,x + cmp #$fe + beq word_boundary + + ;; Not a word boundary, so ignore + jmp next_kw_offset + +word_boundary: + ;; Found a token whose offset is in tokenise_work3 + ;; No trace back to the start of the compressed keyword list + ;; to work out what word number we are, so that we can return + ;; the token number. + + inc $d020 + lda #$80 ; Pretend token + clc + rts + +done_searching_for_token: + ;; No token found + sec + lda #$00 + rts +