begin implementing word compression routine for tokeniser

c64/basic/,aliases.s

@@ -2,8 +2,11 @@
 ;	(Only those for now that we feel the need to implement initially)
 ;	(https://www.c64-wiki.com/wiki/Zeropage)
 
+	.alias tokenise_work1 $7
+	.alias tokenise_work2 $8
+	.alias tokenise_work3 $B
+	.alias tokenise_work4 $F
 	.alias temp_string_ptr $35
-
 	
 	.alias TXTTAB $2B
 	.alias IOSTATUS $90

c64/basic/basic_main_loop.s

@@ -25,9 +25,24 @@ read_line_loop:
 
 got_line_of_input:
 
+	;; Store length of input buffer ready for tokenising
+	stx tokenise_work1
+	
 	;; Do printing of the new line
 	lda #$0d
 	jsr $ffd2
+
+	;; Pack the word from the start
+	ldx #$00
+	jsr pack_word
+
+	;; Print the packed word
+	ldx tokenise_work2
+	dex
+*	lda $0100,x
+	sta $0608,x
+	dex
+	bpl -
 	
 	jsr ready_message
 

c64/basic/pack_word.s

@@ -0,0 +1,88 @@
+	;; Pack a word using the simple compression scheme.
+	;; This has to be the same implementation as in compress_text.c
+	;; because we compress input lines of BASIC to do the tokenisation.
+	;; This allows us to search through the compressed tokens directly
+	;; using the compressed fragment of BASIC, and thus work out the
+	;; token for a given piece of string.
+	;; This also helps ensure that we have an implementation that is totally
+	;; different to that of the original C64 BASIC, as well as likely
+	;; saving a bit of space in the ROM, especially once we start implementing
+	;; larger keyword vocabs for BASIC 7 or BASIC 10 compatibility, since the
+	;; compression and decompression machinery is of fixed size.
+
+pack_word:
+	;; Inputs:
+	;; X = offset of string to pack. Address is $0200 + X
+	;; tokenise_work1 = number of bytes to pack
+	;; tokenise_work2 = packed length of string
+
+	;; Outputs:
+	;; Output is written to $0100 (bottom of stack).
+
+	;; Memory modified:
+	;; tokenise_work4 = temporary storage
+	;; tokenise_work3 = flag for whether we have a nybl prepared or not.
+	;; (This can also be checked on exit, to know whether the last byte
+	;; is incomplete, which is important for searching for abbreviated
+	;; key words.)
+
+	;; Initialise internal variables
+	lda #$00
+	sta tokenise_work2
+	sta tokenise_work3
+
+pack_char_loop:
+	;; Get the next char
+*	lda $0200,x
+	;; Work out what type of character we have, and thus
+	;; how we need to pack it.
+	jsr get_packed_char_index
+	sta $0100,x
+	inx 
+	cpx tokenise_work1
+	bne -
+	lda tokenise_work1
+	sta tokenise_work2
+	
+	rts
+
+get_packed_char_index:
+	;; Find char in A in the packed char list, and return
+	;; $1 - $E, $F1 - $FD if the charactar is abbreviable,
+	;; or else return $00 if the character has to be stored literally.
+
+	;; Y register and A are modified, as are processor flags.
+	
+	ldy #$00
+find_char_loop:	
+	cmp packed_message_chars,y
+	bne not_a_match
+
+	;; Is it a nybl abbreviable char?
+	cpy #13
+	bcs not_nybl_char
+	;; Yes, so add 1 and return it
+	iny
+	tya
+	rts
+	
+not_nybl_char:
+	cpy #27
+	bcs not_a_match
+	;; It's a $Fx character.
+	;; Since the lowest index is $F, and it should be encoded as $F1
+	;; we need to add $E2
+	tya
+	clc
+	adc #$E2
+	rts
+	
+not_a_match:
+	iny
+	;; Loop until we find it, or we discover it isn't
+	;; a character with a special encoding
+	cpy #27
+	bne find_char_loop
+	;; Not a char that we can pack, so return 0
+	lda #$00
+	rts