Merge pull request #428 from boriel/feature/add_prepro_concatenation

Feature/add prepro concatenation

Author: boriel

src/libzxbpp/zxbpplex.py

@@ -10,8 +10,11 @@
 # This is the Lexer for the ZXBpp (ZXBasic Preprocessor)
 # ----------------------------------------------------------------------
 
+import re
 import sys
+
 from src.ply import lex
+from src.api import global_
 from src.libzxbpp.base_pplex import BaseLexer
 
 
@@ -102,24 +105,13 @@ def t_asm_CONTINUE(self, t):
         return t
 
     def t_asm_COMMENT(self, t):
-        r';'
-        t.lexer.push_state('asmcomment')
-
-    def t_asmcomment_skip(self, t):
-        r'.'
+        r';.*'
 
-    def t_asmcomment_NEWLINE(self, t):
-        r'\r?\n'
-        # New line => remove whatever state in top of the stack and replace it with INITIAL
-        t.lexer.lineno += 1
-        t.lexer.pop_state()
-        return t
-
-    def t_asm_NEWLINE(self, t):
-        r'\r?\n'
-        # New line => remove whatever state in top of the stack and replace it with INITIAL
-        t.lexer.lineno += 1
-        return t
+    def t_asm_PREPROCLINE(self, t):
+        r'\#[ \t]*[Ll][Ii][Nn][Ee][ \t]+([0-9]+)(?:[ \t]+"((?:[^"]|"")*)")?[ \t]*\r?\n'
+        match = re.match('#[ \t]*[Ll][Ii][Nn][Ee][ \t]+([0-9]+)(?:[ \t]+"((?:[^"]|"")*)")?[ \t]*\r?\n', t.value)
+        t.lexer.lineno = int(match.groups()[0])
+        global_.FILENAME = match.groups()[1] or global_.FILENAME
 
     def t_asm_ID(self, t):
         r'[_A-Za-z][_A-Za-z0-9]*'
@@ -149,7 +141,7 @@ def t_prepro_define_defargs_defargsopt_defexpr_pragma_if_NEWLINE(self, t):
         t.lexer.pop_state()
         return t
 
-    def t_INITIAL_NEWLINE(self, t):
+    def t_INITIAL_asm_NEWLINE(self, t):
         r'\r?\n'
         t.lexer.lineno += 1
         return t

tests/functional/line_asm.bi

@@ -0,0 +1,6 @@
+ASM
+NOP
+#line 25
+END ASM
+#warning this should be line 26
+

tests/functional/line_asm.out

@@ -0,0 +1,5 @@
+#line 1 "line_asm.bi"
+ASM
+NOP
+END ASM
+

tests/functional/line_err.bas

@@ -0,0 +1,7 @@
+DIM q as UByte = 1
+ASM
+NOP
+#line 25
+END ASM
+DIM q = 1: REM this error should show at line 26, not line 6!
+

tests/functional/test.py

@@ -240,7 +240,7 @@ def updateTest(tfname: str, pattern_, strip_blanks: bool = True):
     lines = get_file_lines(tfname, replace_regexp=pattern_, replace_what=ZXBASIC_ROOT,
                            replace_with=_original_root, strip_blanks=strip_blanks)
     with src.api.utils.open_file(tfname, 'wt', encoding='utf-8') as f:
-        f.write('\n'.join(lines))
+        f.write('\n'.join(lines) + '\n')
 
 
 @src.api.utils.timeout(_timeout)
@@ -506,7 +506,7 @@ def normalizeDiff(diff: List[str]) -> List[str]:
         lines = get_file_lines(tfname, replace_regexp=FILTER, replace_what=ZXBASIC_ROOT,
                                replace_with=_original_root)
         with src.api.utils.open_file(fname1, 'wt', encoding='utf-8') as f:
-            f.write('\n'.join(lines))
+            f.write('\n'.join(lines) + '\n')
 
         os.unlink(tfname)
         _msg("\rTest: %s (%s) updated\n" % (fname, fname1))

tests/functional/test_errmsg.txt

@@ -172,4 +172,7 @@ due_par.bas:5: error: Syntax error. Unexpected end of file
 error_array.bas:3: error: Invalid assignment. Variable z$() is an array
 >>> process_file('prepro76.bi')
 prepro76.bi:2: error: this is an intended error
-
+>>> process_file('line_asm.bi')
+line_asm.bi:26: warning: this should be line 26
+>>> process_file('line_err.bas')
+line_err.bas:5: error: Variable 'q' already declared at line_err.bas:1