Implement token-pasting zxbbpp operator

Tests included

Author: boriel

src/libzxbpp/prepro/macrocall.py

@@ -5,6 +5,8 @@
 import re
 import copy
 
+from typing import Union
+
 from .exceptions import PreprocError
 from src.api.debug import __DEBUG__
 
@@ -22,12 +24,12 @@ class MacroCall:
     """
     __slots__ = 'table', 'id_', 'callargs', 'lineno'
 
-    def __init__(self, lineno: int, table: 'prepro.DefinesTable', id_: str, args=None):
+    def __init__(self, lineno: int, table: 'prepro.DefinesTable', id_: Union['MacroCall', str], args=None):
         """ Initializes the object with the ID table, the ID name and
         optionally, the passed args.
         """
         self.table: 'prepro.DefinesTable' = table
-        self.id_: str = id_
+        self.id_ = id_
         self.callargs = args
         self.lineno: int = lineno
 
@@ -108,4 +110,5 @@ def is_defined(self, symbolTable: 'prepro.DefinesTable' = None) -> bool:
         if symbolTable is None:
             symbolTable = self.table
 
+        assert isinstance(self.id_, str)
         return symbolTable.defined(self.id_)

src/libzxbpp/prepro/operators.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+# Operators implemented in the preprocessor
+
+
+from src.libzxbpp import prepro
+from src.libzxbpp.prepro.macrocall import MacroCall
+
+
+class Concatenation(MacroCall):
+    """ Implements the concatenation (a.k.a. token-paste) (##) operator.
+    When in a macro body, ID1 ## ID2 becomes
+    <expanded(ID1)><expanded(ID2)> (concatenated without spaces).
+    Out of a macro body, ID1 and ID2 are expanded normally and "##" is
+    also output as is.
+    """
+    def __init__(self, lineno: int, table: 'prepro.DefinesTable', left: MacroCall, right: MacroCall):
+        super().__init__(lineno=lineno, table=table, id_='')
+        self.left = left
+        self.right = right
+
+    def __call__(self, symbolTable: 'prepro.DefinesTable' = None) -> str:
+        return self.left(symbolTable).rstrip() + self.right(symbolTable).lstrip()

src/libzxbpp/zxbpp.py

@@ -30,6 +30,7 @@
 from .prepro.output import error
 from .prepro import DefinesTable, ID, MacroCall, Arg, ArgList
 from .prepro.exceptions import PreprocError
+from .prepro.operators import Concatenation
 
 from src import arch
 
@@ -63,6 +64,7 @@ class IfDef(NamedTuple):
     ('nonassoc', 'DUMMY'),
     ('left', 'EQ', 'NE', 'LT', 'LE', 'GT', 'GE'),
     ('right', 'LLP'),
+    ('left', 'PASTE'),
 )
 
 
@@ -684,6 +686,12 @@ def p_macrocall_args(p):
     p[0] = MacroCall(p[2].end_lineno, ID_TABLE, p[1], p[2])
 
 
+def p_macrocall_paste(p):
+    """ macrocall : macrocall PASTE macrocall
+    """
+    p[0] = Concatenation(p[1].lineno, ID_TABLE, p[1], p[3])
+
+
 def p_args(p):
     """ args : LLP arglist RRP
     """

src/libzxbpp/zxbpplex.py

@@ -42,8 +42,8 @@
 
 _tokens = ('STRING', 'TOKEN', 'NEWLINE', '_ENDFILE_', 'FILENAME', 'ID',
            'INTEGER', 'EQ', 'PUSH', 'POP', 'LP', 'LLP', 'RRP', 'RP', 'COMMA',
-           'CONTINUE', 'NUMBER', 'SEPARATOR', 'GT', 'GE', 'LT', 'LE', 'NE'
-           )
+           'CONTINUE', 'NUMBER', 'SEPARATOR', 'GT', 'GE', 'LT', 'LE', 'NE',
+           'PASTE')
 
 reserved_directives = {
     'include': 'INCLUDE',
@@ -340,6 +340,10 @@ def t_INITIAL_defexpr_TOKEN(self, t):
         r'=>|<=|>=|<>|[$!&|~@:;{}.<>^=+*/%-]'
         return t
 
+    def t_defexpr_PASTE(self, t):
+        r'[ \t]*\#\#[ \t]*'
+        return t
+
     def t_INITIAL_defexpr_asm_SEPARATOR(self, t):
         r'[ \t]+'
         return t

src/parsetab/tabs.dbm.bak

@@ -1,4 +1,4 @@
-'asmparse', (0, 254311)
-'zxnext_asmparse', (254464, 285299)
-'zxbparser', (540160, 712099)
-'zxbpp', (1252352, 69831)
+'zxbpp', (0, 70950)
+'asmparse', (71168, 254311)
+'zxnext_asmparse', (325632, 285299)
+'zxbparser', (611328, 712099)

src/parsetab/tabs.dbm.dat

@@ -1,4 +1,4 @@
-'asmparse', (0, 254311)
-'zxnext_asmparse', (254464, 285299)
-'zxbparser', (540160, 712099)
-'zxbpp', (1252352, 69831)
+'zxbpp', (0, 70950)
+'asmparse', (71168, 254311)
+'zxnext_asmparse', (325632, 285299)
+'zxbparser', (611328, 712099)

src/parsetab/tabs.dbm.dir

@@ -0,0 +1,9 @@
+
+
+#define x(a, b) a ## b
+
+x( A ,  B  )
+
+REM Should not be concatenated out of define body
+A ## B
+

tests/functional/token-paste0.bi

@@ -0,0 +1,10 @@
+#line 1 "token-paste0.bi"
+
+
+
+
+AB
+
+
+A ## B
+

tests/functional/token-paste0.out

@@ -0,0 +1,8 @@
+
+
+#define aa(x) (AA x)
+
+#define x(a, b, c) a##b(c)
+
+x(a, a, 1)
+