fixed VARTAB handling, tokenizer now needs sloghtly less memory

doc/Extended-BASIC.md

@@ -15,7 +15,7 @@ Tries to switch the machine back to 1 MHz operation mode.
 
 ### `MERGE "file_name" [, device_number]`
 
-Appends the BASIC program from a storage medium to the one currently present in memory. NOTE: due to bug in BASIC editor (improper VARTAB handling) it currently works only with programs loaded from the storage medium, not the ones typed in!
+Appends the BASIC program from a storage medium to the one currently present in memory.
 
 ### `BLOAD "file_name", device_number, start_address`
 

src/basic/basic_commands_CC/cmd_merge.s

@@ -18,8 +18,6 @@ cmd_merge:
 
 	// Perform loading - just for a different address
 
-	// XXX sometimes works, sometimes does not - probably improper handling of VARTAB somewhere
-
 	sec
 	lda VARTAB+0
 	sbc #$02

src/basic/editor/insert_line.s

@@ -43,7 +43,7 @@ insert_line:
 	adc #5
 	pha
 	tax
-	
+
 	// Make the space
 	jsr basic_shift_mem_up_and_relink
 

src/basic/editor/shell.s

@@ -154,7 +154,6 @@ shell_add_delete_line:
 	bcs !+
 
 	// Line already present - delete it
-.break
 	jsr delete_line
 
 !:

src/basic/editor/tk_cut_away.s

@@ -18,6 +18,15 @@ tk_cut_away_1: // for single-byte tokenized keywords
 
 	dec tk__len_unpacked                         // keyword length, afterwards number of bytes to cut away
 
+	// Correct the line length
+
+	sec
+	lda tk__length
+	sbc tk__len_unpacked
+	sta tk__length
+
+	// Copy the remaining,. untokenized part in the proper place
+
 	inc tk__offset
 	ldy tk__offset
 

src/basic/editor/tk_pack.s

@@ -26,14 +26,14 @@
 
 tk_pack:
 
-	// Reuse the CPU stack - addresses below $102 are used by 'tokenise_line.s'
+	// Reuse the CPU stack - address $100 is used by 'tokenise_line.s'
 
-	.label tk__len_unpacked = $103 // length of unpacked data; it could replace 'tk__nibble_flag', but at the cost of code size/performance
+	.label tk__len_unpacked = $101 // length of unpacked data; it could replace 'tk__nibble_flag', but at the cost of code size/performance
-	.label tk__shorten_bits = $104 // 2 bytes for quick shortening of packed candidate
+	.label tk__shorten_bits = $102 // 2 bytes for quick shortening of packed candidate
-	.label tk__nibble_flag  = $106 // $00 = start from new byte, $FF = start from high nibble
+	.label tk__nibble_flag  = $104 // $00 = start from new byte, $FF = start from high nibble
-	.label tk__byte_offset  = $107 // offset of the current byte (to place new data) in tk__packed
+	.label tk__byte_offset  = $105 // offset of the current byte (to place new data) in tk__packed
 
-	.label tk__packed       = $108 // packed candidate, 25 bytes is enough for worst case - a 16 byte keyword
+	.label tk__packed       = $106 // packed candidate, 25 bytes is enough for worst case - a 16 byte keyword
 
 	// Initialize variables
 

src/basic/editor/tokenise_line.s

@@ -21,22 +21,20 @@ tokenise_line:
 
 #else
 
-	// Reuse the CPU stack - addresses above $101 are used by 'tk_pack.s'
+	// Reuse the CPU stack - addresses above $100 are used by 'tk_pack.s'
 
 	.label tk__offset       = $100               // offset into string
-	.label tk__length       = $101               // length of the raw string  XXX reuse __tokenise_work1?
+	.label tk__length       = __tokenise_work1   // length of the raw string
 
 	// Initialize variables
 
 	lda #$00
 	sta tk__offset
 
-	ldx __tokenise_work1
-	stx tk__length
-
 	// Terminate input string with $00
 
 	// XXX do we need this?
+	ldx tk__length
 	sta BUF, x
 
 	// FALLTROUGH
@@ -123,7 +121,6 @@ tokenise_line_keyword_V2:
 	sta BUF, x
 
 	// Cut away unnecessary bytes
-
 	jsr tk_cut_away_1
 
 	// Special handling for REM command - after this one nothing more should be tokenised
@@ -136,10 +133,7 @@ tokenise_line_keyword_V2:
 
 tokenise_line_done:
 
-	// Update line length and quit
+	// Quit
-
-	lda tk__length
-	sta __tokenise_work1
 
 	rts