work on basic tokeniser
This commit is contained in:
Paul Gardner-Stephen committed
1 parent
736fbca535
commit
06c6bd9c20
4 files changed
+315
-92
No files matched your search
@@ -7,6 +7,7 @@
|
||||
.alias tokenise_work3 $B
|
||||
.alias tokenise_work4 $F
|
||||
.alias temp_string_ptr $35
|
||||
.alias load_or_verify_or_tokenise_work5 $C
|
||||
|
||||
.alias TXTTAB $2B
|
||||
.alias IOSTATUS $90
|
||||
|
||||
@@ -27,103 +27,13 @@ got_line_of_input:
|
||||
|
||||
;; Store length of input buffer ready for tokenising
|
||||
stx tokenise_work1
|
||||
|
||||
|
||||
;; Do printing of the new line
|
||||
lda #$0d
|
||||
jsr $ffd2
|
||||
|
||||
;; Pack the word from the start
|
||||
ldx #$00
|
||||
jsr pack_word
|
||||
|
||||
;; Search for word in keyword list
|
||||
;; The packed form is canonical, so we can search for the keyword while
|
||||
;; it remains compressed. This also saves a little time.
|
||||
;; Algorithm is simply to find matching first bytes, and then look for
|
||||
;; the rest being a match.
|
||||
|
||||
token_search:
|
||||
;; tokenise_work1 = offset in line of input
|
||||
;; tokenise_work2 = length of token we are matching
|
||||
;; tokenise_work3 = offset in token list
|
||||
lda #$00
|
||||
sta tokenise_work1
|
||||
sta tokenise_work3
|
||||
next_kw_offset:
|
||||
;; Advance offset in compressed keyword list, to see if a match
|
||||
;; BEGINS here
|
||||
|
||||
;; Get count of bytes to compare
|
||||
lda tokenise_work2
|
||||
sta tokenise_work4
|
||||
jsr tokenise_line
|
||||
|
||||
;; Load current position
|
||||
ldy tokenise_work3
|
||||
ldx tokenise_work1
|
||||
|
||||
;; Advance offset in compressed token list, and stop at the end
|
||||
inc tokenise_work3
|
||||
lda tokenise_work3
|
||||
cmp #$ff
|
||||
bne next_in_match
|
||||
jmp done_searching_for_token
|
||||
next_in_match:
|
||||
lda $0100,x
|
||||
cmp packed_keywords,y
|
||||
bne next_kw_offset
|
||||
;; Advance to next chars in source and target
|
||||
inx
|
||||
iny
|
||||
|
||||
dec tokenise_work4 ; Have we compared all bytes yet?
|
||||
|
||||
bne next_in_match
|
||||
|
||||
;; Keyword matches!
|
||||
|
||||
;; Now make sure we are on a keyword boundary.
|
||||
;; Offset 0
|
||||
;; previous byte is $x0
|
||||
;; two byte prior is $FE
|
||||
|
||||
;; Get offset of first byte of this token
|
||||
ldx tokenise_work3
|
||||
dex ; pointer points one late, so rewind it
|
||||
|
||||
;; At start of list?
|
||||
cpx #$00
|
||||
beq word_boundary
|
||||
dex
|
||||
;; Previous byte is $x0 end token?
|
||||
lda packed_keywords,x
|
||||
and #$0f
|
||||
beq word_boundary
|
||||
dex
|
||||
;; Uber-previouss byte is $FE final literal token?
|
||||
lda packed_keywords,x
|
||||
cmp #$fe
|
||||
beq word_boundary
|
||||
|
||||
;; Not a word boundary, so ignore
|
||||
jmp next_kw_offset
|
||||
|
||||
word_boundary:
|
||||
inc $d020
|
||||
|
||||
;; We actually point one by late, so rewind to find start of token
|
||||
lda tokenise_work3
|
||||
sec
|
||||
sbc #$01
|
||||
clc
|
||||
adc #<packed_keywords
|
||||
sta temp_string_ptr+0
|
||||
lda #>packed_keywords
|
||||
adc #$00
|
||||
sta temp_string_ptr+1
|
||||
;; Print the token we found
|
||||
jsr print_packed_string
|
||||
|
||||
done_searching_for_token:
|
||||
jsr ready_message
|
||||
|
||||
jmp basic_main_loop
|
||||
@@ -0,0 +1,122 @@
|
||||
packed_message_chars:
|
||||
.byte $45 ; 'E' = $1 (nybl)
|
||||
.byte $54 ; 'T' = $2 (nybl)
|
||||
.byte $4F ; 'O' = $3 (nybl)
|
||||
.byte $4E ; 'N' = $4 (nybl)
|
||||
.byte $49 ; 'I' = $5 (nybl)
|
||||
.byte $52 ; 'R' = $6 (nybl)
|
||||
.byte $41 ; 'A' = $7 (nybl)
|
||||
.byte $53 ; 'S' = $8 (nybl)
|
||||
.byte $4C ; 'L' = $9 (nybl)
|
||||
.byte $46 ; 'F' = $a (nybl)
|
||||
.byte $44 ; 'D' = $b (nybl)
|
||||
.byte $55 ; 'U' = $c (nybl)
|
||||
.byte $50 ; 'P' = $d (nybl)
|
||||
.byte $43 ; 'C' = $e (nybl)
|
||||
.byte $47 ; 'G' = $F1 / $xF $47
|
||||
.byte $4D ; 'M' = $F2 / $xF $4D
|
||||
.byte $59 ; 'Y' = $F3 / $xF $59
|
||||
.byte $56 ; 'V' = $F4 / $xF $56
|
||||
.byte $42 ; 'B' = $F5 / $xF $42
|
||||
.byte $48 ; 'H' = $F6 / $xF $48
|
||||
.byte $58 ; 'X' = $F7 / $xF $58
|
||||
.byte $57 ; 'W' = $F8 / $xF $57
|
||||
.byte $24 ; '$' = $F9 / $xF $24
|
||||
.byte $27 ; ''' = $Fa / $xF $27
|
||||
.byte $0D ; '␍' = $Fb / $xF $0D
|
||||
.byte $51 ; 'Q' = $Fc / $xF $51
|
||||
.byte $2E ; '.' = $Fd / $xF $2E
|
||||
.byte $23 ; '#' = $FE/F + $23
|
||||
.byte $4B ; 'K' = $FE/F + $4B
|
||||
.byte $28 ; '(' = $FE/F + $28
|
||||
.byte $5A ; 'Z' = $FE/F + $5A
|
||||
.byte $2B ; '+' = $FE/F + $2B
|
||||
.byte $2D ; '-' = $FE/F + $2D
|
||||
.byte $2A ; '*' = $FE/F + $2A
|
||||
.byte $2F ; '/' = $FE/F + $2F
|
||||
.byte $5E ; '^' = $FE/F + $5E
|
||||
.byte $3E ; '>' = $FE/F + $3E
|
||||
.byte $3D ; '=' = $FE/F + $3D
|
||||
.byte $3C ; '<' = $FE/F + $3C
|
||||
packed_message_tokens:
|
||||
.byte $00,$01,$02,$ff,$03,$04,$ff,$03
|
||||
.byte $05,$04,$ff,$03,$05,$06,$ff,$07
|
||||
.byte $05,$08,$ff,$05,$09,$03,$ff,$05
|
||||
.byte $0a,$03,$ff,$0b,$0c,$ff,$0d,$07
|
||||
.byte $0e,$ff,$0f,$10,$11,$ff,$12,$ff
|
||||
.byte $13,$10,$14,$ff,$15,$16,$17,$ff
|
||||
.byte $0d,$18,$ff,$19,$ff,$15,$16,$1a
|
||||
.byte $ff,$1b,$1c,$ff,$1d,$1e,$ff,$1f
|
||||
.byte $20,$ff,$21,$22,$23,$ff,$0d,$24
|
||||
.byte $ff,$25,$26,$ff,$27,$00,$28,$ff
|
||||
.byte $03,$17,$ff,$29,$00,$2a,$ff,$2b
|
||||
.byte $2c,$ff,$1b,$2d,$ff,$2e,$ff,$2f
|
||||
.byte $ff,$30,$ff,$31,$ff,$32,$ff,$33
|
||||
.byte $ff
|
||||
.byte $34
|
||||
.byte $ff
|
||||
.byte $35
|
||||
.byte $36
|
||||
.byte $ff
|
||||
packed_message_words:
|
||||
.byte $23,$30,$f2,$74,$f3,$00,$a5,$91
|
||||
.byte $80,$a5,$91,$00,$3d,$14,$00,$43
|
||||
.byte $20,$a3,$c4,$b0,$b1,$f4,$5e,$10
|
||||
.byte $d6,$18,$14,$20,$54,$dc,$20,$3c
|
||||
.byte $2d,$c2,$00,$f2,$58,$85,$4f,$47
|
||||
.byte $00,$a5,$91,$47,$f2,$10,$59,$91
|
||||
.byte $f1,$79,$00,$4c,$f2,$f5,$16,$00
|
||||
.byte $41,$f7,$20,$f8,$52,$f6,$3c,$20
|
||||
.byte $a3,$60,$8f,$59,$42,$7f,$58,$00
|
||||
.byte $61,$2c,$64,$00,$f1,$38,$cf,$42
|
||||
.byte $00,$3c,$20,$3a,$00,$b7,$27,$00
|
||||
.byte $fc,$c7,$42,$52,$f3,$00,$3f,$56
|
||||
.byte $16,$a9,$3f,$57,$00,$f2,$1f,$4d
|
||||
.byte $36,$f3,$00,$c4,$b1,$af,$27,$b0
|
||||
.byte $82,$72,$1f,$4d,$14,$20,$f5,$7b
|
||||
.byte $00,$8c,$f5,$8e,$65,$d2,$00,$61
|
||||
.byte $b5,$f2,$fa,$b0,$76,$67,$f3,$00
|
||||
.byte $b5,$f4,$58,$53,$40,$f5,$f3,$00
|
||||
.byte $ff,$5a,$16,$30,$b5,$61,$e2,$00
|
||||
.byte $2f,$59,$d1,$00,$f2,$58,$f2,$72
|
||||
.byte $ef,$48,$00,$82,$65,$4f,$47,$00
|
||||
.byte $93,$4f,$47,$00,$a3,$6f,$4d,$c9
|
||||
.byte $70,$e3,$f2,$d9,$1f,$58,$00,$e7
|
||||
.byte $4f,$27,$20,$e3,$42,$54,$c1,$00
|
||||
.byte $ac,$4e,$25,$34,$00,$f4,$16,$5a
|
||||
.byte $f3,$00,$93,$7b,$00,$61,$7b,$f3
|
||||
.byte $fd,$fb,$00,$93,$7b,$54,$f1,$00
|
||||
.byte $f4,$16,$5a,$f3,$54,$f1,$00,$87
|
||||
.byte $f4,$54,$f1,$00,$16,$63,$60,$f5
|
||||
.byte $f3,$21,$80,$a6,$11,$fd,$fb,$fb
|
||||
.byte $00
|
||||
packed_keywords:
|
||||
.byte $14,$b0,$a3,$60,$41,$f7,$20,$b7
|
||||
.byte $27,$00,$54,$dc,$2f,$23,$00,$54
|
||||
.byte $dc,$20,$b5,$f2,$00,$61,$7b,$00
|
||||
.byte $91,$20,$f1,$32,$30,$6c,$40,$5a
|
||||
.byte $00,$61,$82,$36,$10,$f1,$38,$cf
|
||||
.byte $42,$00,$61,$2c,$64,$00,$61,$f2
|
||||
.byte $00,$82,$3d,$00,$34,$00,$f8,$75
|
||||
.byte $20,$93,$7b,$00,$87,$f4,$10,$f4
|
||||
.byte $16,$5a,$f3,$00,$b1,$a0,$d3,$ff
|
||||
.byte $4b,$10,$d6,$54,$2f,$23,$00,$d6
|
||||
.byte $54,$20,$e3,$42,$00,$95,$82,$00
|
||||
.byte $e9,$60,$ef,$4d,$b0,$8f,$59,$80
|
||||
.byte $3d,$14,$00,$e9,$38,$10,$f1,$12
|
||||
.byte $00,$41,$f8,$00,$27,$f5,$fe,$28
|
||||
.byte $23,$00,$a4,$00,$8d,$ef,$28,$00
|
||||
.byte $2f,$48,$14,$00,$43,$20,$82,$1d
|
||||
.byte $00,$fe,$2b,$fe,$2d,$fe,$2a,$fe
|
||||
.byte $2f,$fe,$5e,$74,$b0,$36,$00,$fe
|
||||
.byte $3e,$fe,$3d,$fe,$3c,$8f,$47,$40
|
||||
.byte $54,$20,$7f,$42,$80,$c8,$60,$a6
|
||||
.byte $10,$d3,$80,$8f,$51,$60,$64,$b0
|
||||
.byte $93,$f1,$00,$1f,$58,$d0,$e3,$80
|
||||
.byte $85,$40,$27,$40,$72,$40,$d1,$1f
|
||||
.byte $4b,$00,$91,$40,$82,$6f,$24,$00
|
||||
.byte $f4,$79,$00,$78,$e0,$ef,$48,$6f
|
||||
.byte $24,$00,$91,$a2,$f9,$00,$65,$f1
|
||||
.byte $f6,$2f,$24,$00,$f2,$5b,$f9,$00
|
||||
.byte $f1
|
||||
.byte $30
|
||||
@@ -0,0 +1,190 @@
|
||||
;; Tokenise a line of BASIC
|
||||
;; Stored at $0200
|
||||
;; Length in tokenise_work1
|
||||
|
||||
tokenise_line:
|
||||
|
||||
;; We need to consider each offset in the string to see if it matches
|
||||
;; a token. No BASIC keyword is > 7 bytes, so we need to start with that
|
||||
;; length (or remaining length of string) and count our way back down.
|
||||
;; This means we need separate start and end points to keep track of.
|
||||
|
||||
;; $0110 = offset into string
|
||||
;; $0111 = number of bytes to pack
|
||||
;; $0112 = offset into tokenised output
|
||||
;; $0113 = length of raw string
|
||||
|
||||
lda #$00
|
||||
sta $0110
|
||||
sta $0112
|
||||
lda tokenise_work1
|
||||
sta $0113
|
||||
|
||||
tokenise_char:
|
||||
|
||||
;; Check if we have reached end of the line?
|
||||
lda $0110
|
||||
cmp $0113
|
||||
bne +
|
||||
|
||||
;; Finished tokenising line
|
||||
;; Update tokenise_work1 and terminate with null
|
||||
ldx $0112
|
||||
lda #$00
|
||||
sta $0200,x
|
||||
stx tokenise_work1
|
||||
rts
|
||||
*
|
||||
;; More to do
|
||||
|
||||
;; Maximum keyword length
|
||||
lda #7
|
||||
sta $0111
|
||||
|
||||
tokenise_char_loop:
|
||||
;; Pack the word from the start (start = X, raw len = tokenise_work1)
|
||||
;; Returns packed length in tokenise_work2 and writes the packed
|
||||
;; word at $0100+
|
||||
|
||||
;; Pack string
|
||||
ldx $0110
|
||||
lda $0111
|
||||
sta tokenise_work1
|
||||
jsr pack_word
|
||||
|
||||
;; Search for it in keywords
|
||||
jsr keyword_search
|
||||
|
||||
;; Did we find a keyword?
|
||||
bcc found_token_in_line
|
||||
|
||||
;; Nope, so try shorter
|
||||
dec $0111
|
||||
bne tokenise_char_loop
|
||||
|
||||
;; Is not a token, so copy to output verbatim
|
||||
ldx $0110
|
||||
ldy $0112
|
||||
lda $0200,x
|
||||
sta $0200,y
|
||||
inc $0112
|
||||
|
||||
;; Tokenise next char
|
||||
jmp tokenise_char
|
||||
|
||||
found_token_in_line:
|
||||
|
||||
;; Got a token, so write it out
|
||||
ldx $0112
|
||||
sta $0200,x
|
||||
inx
|
||||
stx $0112
|
||||
;; Now skip over the length of the token
|
||||
lda $0110
|
||||
clc
|
||||
adc $0111
|
||||
sta $0110
|
||||
|
||||
;; See if there is more of the line left to process
|
||||
jmp tokenise_char
|
||||
|
||||
|
||||
|
||||
|
||||
;; Search for word in keyword list
|
||||
;; The packed form is canonical, so we can search for the keyword while
|
||||
;; it remains compressed. This also saves a little time.
|
||||
;; Algorithm is simply to find matching first bytes, and then look for
|
||||
;; the rest being a match.
|
||||
|
||||
keyword_search:
|
||||
;; Search for the compressed keyword stored at $0100 in the list
|
||||
;; of compressed keywords
|
||||
|
||||
;; tokenise_work1 = offset in line of input
|
||||
;; tokenise_work2 = length of token we are matching
|
||||
;; tokenise_work3 = offset in token list
|
||||
;; load_or_verify_or_tokenise_work5 = temporary work space
|
||||
;; X and Y registers used as temporary space.
|
||||
;; returns C=1 if no matching token, else C=0 and A=token
|
||||
|
||||
lda #$00
|
||||
sta tokenise_work1
|
||||
sta tokenise_work3
|
||||
next_kw_offset:
|
||||
;; Advance offset in compressed keyword list, to see if a match
|
||||
;; BEGINS here
|
||||
|
||||
;; Get count of bytes to compare
|
||||
lda tokenise_work2
|
||||
sta load_or_verify_or_tokenise_work5
|
||||
|
||||
;; Load current position
|
||||
ldy tokenise_work3
|
||||
ldx tokenise_work1
|
||||
|
||||
;; Advance offset in compressed token list, and stop at the end
|
||||
inc tokenise_work3
|
||||
lda tokenise_work3
|
||||
cmp #$ff
|
||||
bne next_in_match
|
||||
jmp done_searching_for_token
|
||||
next_in_match:
|
||||
lda $0100,x
|
||||
cmp packed_keywords,y
|
||||
bne next_kw_offset
|
||||
;; Advance to next chars in source and target
|
||||
inx
|
||||
iny
|
||||
|
||||
dec load_or_verify_or_tokenise_work5 ; Have we compared all bytes yet?
|
||||
|
||||
bne next_in_match
|
||||
|
||||
;; Keyword matches!
|
||||
|
||||
;; Now make sure we started on a keyword boundary
|
||||
;; (we know we finished on one, because of the structure of the compressed
|
||||
;; keyword data).
|
||||
;; Offset 0
|
||||
;; previous byte is $x0
|
||||
;; two byte prior is $FE
|
||||
|
||||
;; Get offset of first byte of this token
|
||||
ldx tokenise_work3
|
||||
dex ; pointer points one late, so rewind it
|
||||
|
||||
;; At start of list?
|
||||
cpx #$00
|
||||
beq word_boundary
|
||||
dex
|
||||
;; Previous byte is $x0 end token?
|
||||
lda packed_keywords,x
|
||||
and #$0f
|
||||
beq word_boundary
|
||||
dex
|
||||
;; Uber-previouss byte is $FE final literal token?
|
||||
lda packed_keywords,x
|
||||
cmp #$fe
|
||||
beq word_boundary
|
||||
|
||||
;; Not a word boundary, so ignore
|
||||
jmp next_kw_offset
|
||||
|
||||
word_boundary:
|
||||
;; Found a token whose offset is in tokenise_work3
|
||||
;; No trace back to the start of the compressed keyword list
|
||||
;; to work out what word number we are, so that we can return
|
||||
;; the token number.
|
||||
|
||||
inc $d020
|
||||
lda #$80 ; Pretend token
|
||||
clc
|
||||
rts
|
||||
|
||||
done_searching_for_token:
|
||||
;; No token found
|
||||
sec
|
||||
lda #$00
|
||||
rts
|
||||
|
||||
Reference in new issue
Block a user