Change from str to lists for building strings during parsing

1 files changed, 60 insertions, 41 deletions

diff --git a/src/smp/macro_processor.py b/src/smp/macro_processor.py
index b95dfd1..a595c36 100644
--- a/src/smp/macro_processor.py
+++ b/src/smp/macro_processor.py
@@ -51,6 +51,14 @@ def macro_name_clean(macro_name: str) -> str:
     return macro_name
 
 
+def strip_list(l, chars=[" "]):
+    while len(l) > 0 and l[0] in chars:
+        l.pop(0)
+    while len(l) > 0 and l[-1] in chars:
+        l.pop()
+    return l
+
+
 class MacroProcessor:
     source_file_path: str
     """All currently defined macros in this MacroProcessor"""
@@ -171,27 +179,31 @@ class MacroProcessor:
     ) -> str:
         if len(args) == 0:
             return macro_name
-        out = f"{macro_name}("
+        out: list[str] = []
+        out.extend(f"{macro_name}(")
         for i, arg in enumerate(args):
             if process_args:
-                out += self.process_input(arg)
+                out.extend(self.process_input(arg))
             else:
-                out += arg
+                out.extend(arg)
             if i < (len(args) - 1):
-                out += ","
-        out += ")"
-        return out
+                out.append(",")
+        out.append(")")
+        return "".join(out)
 
     def expand_macro(self, macro_name: str, args: list[str] = list()) -> str:
         # Ignore trailing underscore in macro name, the parser will pop a space in front if
         # present, but we should ignore it for finding the macro.
+        if isinstance(macro_name, list):
+            macro_name = "".join(macro_name)
+
         macro_name = macro_name_clean(macro_name)
         if macro_name not in self.macros:
             return self._expand_unknown_macro(macro_name, args, process_args=True)
 
         # Strip leading whitespace from arguments
         for arg in args:
-            arg = arg.strip()
+            arg = strip_list(arg)
 
         # Log macro invokation
         # The fact that we are here, does not ensure that the macro is actually expanded into
@@ -282,10 +294,10 @@ class MacroProcessor:
 
         state = ParserState.NORMAL
         state_start = 0
-        macro_name = ""
+        macro_name: list[str] = []
         macro_args = []
-        argument = ""
-        py_expr = ""
+        argument: list[str] = []
+        py_expr: list[str] = []
 
         skip_next_line_ending = False
 
@@ -387,7 +399,7 @@ class MacroProcessor:
                 elif c.isalnum():
                     state = ParserState.IN_MACRO
                     state_start = i
-                    macro_name += c
+                    macro_name.append(c)
                 else:
                     output.append(c)
 
@@ -409,7 +421,7 @@ class MacroProcessor:
 
             elif state == ParserState.IN_MACRO:
                 if c.isalnum() or c == "_":
-                    macro_name += c
+                    macro_name.append(c)
                 elif c == "(":
                     hi_range()
                     parens_level += 1
@@ -417,28 +429,29 @@ class MacroProcessor:
                     state_start = i
                 else:
                     hi_range()
-                    if macro_is_whitespace_deleting(macro_name):
+                    _macro_name = "".join(macro_name)
+                    if macro_is_whitespace_deleting(_macro_name):
                         if output[-1] == " ":
                             output = output[:-1]
-                        macro_name = macro_name_clean(macro_name)
+                        _macro_name = macro_name_clean(_macro_name)
 
-                    self._enter_frame(macro_name, file, linenr, input)
+                    self._enter_frame(_macro_name, file, linenr, input)
 
-                    if macro_name == "SNNL":
+                    if _macro_name == "SNNL":
                         skip_next_line_ending = c != "\n"
-                    elif macro_name == "DNL":
+                    elif _macro_name == "DNL":
                         if c != "\n":
                             state = ParserState.DNL
                             state_start = i
-                        macro_name = ""
+                        macro_name.clear()
                         i += 1
                         self._pop_frame()
                         continue
                     else:
-                        expanded = self.expand_macro(macro_name)
+                        expanded = self.expand_macro(_macro_name)
                         output.extend(expanded)
                         output.append(c)
-                    macro_name = ""
+                    macro_name.clear()
 
                     self._pop_frame()
 
@@ -448,53 +461,58 @@ class MacroProcessor:
                 if c == "%" and peek == '"':
                     quote_level += 1
                     i += 2
-                    argument += '%"'
+                    argument.extend(["%", '"'])
                     continue
                 elif c == '"' and peek == "%":
                     quote_level -= 1
                     i += 2
-                    argument += '"%'
+                    argument.extend(['"', "%"])
                     continue
                 elif quote_level > 0:
-                    argument += c
+                    argument.append(c)
                     i += 1
                     continue
 
                 if (c == ")") and (parens_level == 1):
                     hi_range()
-                    if macro_is_whitespace_deleting(macro_name):
+                    _macro_name = "".join(macro_name)
+                    if macro_is_whitespace_deleting(_macro_name):
                         if output[-1] == " ":
                             output = output[:-1]
-                        macro_name = macro_name_clean(macro_name)
+                        _macro_name = macro_name_clean(_macro_name)
                     parens_level = 0
-                    macro_args.append(argument.strip())
-                    self._enter_frame(macro_name, file, linenr, input)
-                    expanded = self.expand_macro(macro_name, macro_args)
+                    macro_args.append("".join(strip_list(argument)))
+                    self._enter_frame(_macro_name, file, linenr, input)
+                    expanded = self.expand_macro(_macro_name, macro_args)
                     output.extend(expanded)
                     state = ParserState.NORMAL
                     state_start = i
-                    macro_name = ""
-                    macro_args = []
-                    argument = ""
+                    macro_name.clear()
+                    macro_args.clear()
+                    argument.clear()
                     self._pop_frame()
                 elif (c == ",") and (parens_level == 1):
-                    macro_args.append(argument.strip())
+                    macro_args.append("".join(strip_list(argument)))
                     state_start = i
                     hi_range()
-                    argument = ""
+                    argument.clear()
                 else:
                     if c == "(":
                         parens_level += 1
                     if c == ")":
                         parens_level -= 1
-                    argument += c
+                    argument.append(c)
             elif state == ParserState.IN_CODE:
                 if c == ")" and peek == "%":
                     try:
                         self._enter_frame("inline_code", file, linenr, input)
                         f = StringIO()
                         with redirect_stdout(f):
-                            exec(py_expr, self.py_global_env, self.py_local_env_current)
+                            exec(
+                                "".join(py_expr),
+                                self.py_global_env,
+                                self.py_local_env_current,
+                            )
                         s = f.getvalue()
                         if s != "":
                             output.extend(s)
@@ -502,22 +520,23 @@ class MacroProcessor:
                         traceback.print_exc()
                     finally:
                         self._pop_frame()
-                    py_expr = ""
+                    py_expr.clear()
                     state = ParserState.NORMAL
                     i += 1
                     state_start = i
                 else:
-                    py_expr += c
+                    py_expr.append(c)
             i += 1
 
         # Handle cases where the text ends with a macro without arguments
-        if macro_name != "":
-            if macro_is_whitespace_deleting(macro_name):
+        if len(macro_name) > 0:
+            _macro_name = "".join(macro_name)
+            if macro_is_whitespace_deleting(_macro_name):
                 if len(output) > 0 and output[-1] == " ":
                     output = output[:-1]
-                macro_name = macro_name_clean(macro_name)
+                _macro_name = macro_name_clean(_macro_name)
             hi_range()
-            output.extend(self.expand_macro(macro_name))
+            output.extend(self.expand_macro(_macro_name))
 
         return "".join(output)