aboutsummaryrefslogtreecommitdiff
path: root/src/smp/macro_processor.py
diff options
context:
space:
mode:
Diffstat (limited to 'src/smp/macro_processor.py')
-rw-r--r--src/smp/macro_processor.py154
1 files changed, 146 insertions, 8 deletions
diff --git a/src/smp/macro_processor.py b/src/smp/macro_processor.py
index a31f040..77641e7 100644
--- a/src/smp/macro_processor.py
+++ b/src/smp/macro_processor.py
@@ -1,6 +1,7 @@
import smp.builtins
import traceback
import inspect
+import sys
from typing import Any
from enum import Enum
@@ -19,6 +20,25 @@ class ParserState(Enum):
DNL = 8
+def debug_hi(input, linestart, pos, color="\u001b[7m"):
+ return debug_hi_range(input, linestart, pos, pos + 1, color=color)
+
+
+def debug_hi_range(input, linestart, start, end, color="\u001b[7m"):
+ lineend = input.find("\n", linestart)
+ line = input[linestart:] if lineend == -1 else input[linestart:lineend]
+
+ out = ""
+ for j, _c in enumerate(line):
+ c_pos = linestart + j
+ if (c_pos >= start) and (c_pos < end):
+ out += f"{color}{_c}\u001b[0m"
+ else:
+ out += f"{_c}"
+
+ return out
+
+
def macro_is_whitespace_deleting(s: str) -> bool:
if len(s) == 0:
return False
@@ -49,6 +69,11 @@ class MacroProcessor:
end_quote: str = '"%'
prefix: str = ""
+ file = None
+
+ expansion_stack: list[Any]
+ file_stack: list[Any]
+
def __init__(self, prefix=""):
self.macros = dict()
self.macro_invocations = list()
@@ -58,6 +83,11 @@ class MacroProcessor:
self.py_local_env_current = self.macros
self.prefix = prefix
+ self.expansion_stack = (
+ list()
+ ) # This probably needs some magic at some point later
+ self.file_stack = list() # This probably needs some magic at some point later
+
self._define_builtins(self.macros)
self._define_builtins(self.py_local_env_alt)
@@ -75,6 +105,8 @@ class MacroProcessor:
env[f"{self.prefix}macro_processor"] = self
env[f"{self.prefix}template_stack"] = []
+ env[f"{self.prefix}template_stack_content"] = []
+ env[f"{self.prefix}template_prefix"] = "templates/"
# If true, include-macros will parse yaml in beginning of content
env[f"{self.prefix}parse_file_yaml"] = True
@@ -82,6 +114,7 @@ class MacroProcessor:
# meaning they will skip steps that are slow.
env[f"{self.prefix}draft"] = False
env[f"{self.prefix}metadata_prefix"] = "METADATA_"
+ env[f"{self.prefix}smp_debug"] = "1"
def define_macro_string(self, macro_name, macro_value):
self.define_macro(macro_name, str(macro_value))
@@ -122,14 +155,17 @@ class MacroProcessor:
macro_args.append(self)
macro_args.extend(args)
# This should maybe be processed as well(?)
- return str(macro(*macro_args))
+ res = str(macro(*macro_args))
+ return res
def _expand_string_macro(self, macro: Any, args: list[str] = list()) -> str:
expanded = macro
for i, arg in enumerate(args):
placeholder = f"${i}"
expanded = macro.replace(placeholder, arg)
- return self.process_input(expanded)
+
+ res = self.process_input(expanded)
+ return res
def _expand_unknown_macro(
self, macro_name: str, args: list[str] = list(), process_args: bool = True
@@ -170,9 +206,17 @@ class MacroProcessor:
return self._expand_callable_macro(macro, args)
except Exception as e:
s = self._expand_unknown_macro(macro_name, args, process_args=False)
- self.log_warning(
- f"Error expanding macro {s} ({e})\n{traceback.format_exc()}"
- )
+
+ e2 = ""
+ if self._debug_on(1):
+ e2 += "\n"
+ e2 += self._stack()
+
+ if self._debug_on(2):
+ e2 += "\n- Python stack ---\n"
+ e2 += traceback.format_exc()
+
+ self.log_warning(f"Error expanding macro {s} ({e}){e2}")
return s
if isinstance(macro, str):
@@ -180,7 +224,42 @@ class MacroProcessor:
return f"{repr(macro)}"
- def process_input(self, input: str):
+ def _enter_frame(self, frame_type, filename, position, content):
+ self.expansion_stack.append((frame_type, filename, position, ""))
+
+ def _pop_frame(self):
+ self.expansion_stack.pop()
+
+ def _enter_file_frame(self, filename, linenr, template):
+ self.file_stack.append([filename, linenr, 0])
+
+ def _pop_file_frame(self):
+ self.file_stack.pop()
+
+ def _stack(self):
+ out = "- File stack ------\n"
+ for i, frame in enumerate(self.file_stack):
+ out += f" {' ' * i} {frame[0]}:{frame[1]}\n"
+ out += "\n- Macro stack -----\n"
+ for i, frame in enumerate(self.expansion_stack):
+ out += f" {' ' * i} [{frame[0]}] in file {frame[1]}:{frame[2]}\n"
+ return out
+
+ def _debug_on(self, level):
+ try:
+ if level > int(self._get_macro_with_prefix("smp_debug")):
+ return False
+ except:
+ return False
+
+ return True
+
+ def _debug(self, level, message):
+ if self._debug_on(level):
+ return
+ print(message, file=sys.stderr)
+
+ def process_input(self, input: str, file: str | None = None):
"""
I also want to add special syntax for "special blocks",
I am thinking of two main options, either some macro_names are intercepted, _or_ a special kind of macro can exist like
@@ -198,8 +277,13 @@ class MacroProcessor:
by pushing the expanded macros back to the input string, this may be more confusing,
but may also be faster (stream or mutable string)
"""
+ if file is not None:
+ self.file = file
+ if (file is None) and (self.file is not None):
+ file = self.file
output = ""
state = ParserState.NORMAL
+ state_start = 0
macro_name = ""
macro_args = []
argument = ""
@@ -210,6 +294,8 @@ class MacroProcessor:
# We should keep track of filename, linenumber, and character number on line here
# So we can give sensible error messages
# Probably add to python stack trace?
+ linestart = 0
+ linenr = 0
quote_level = 0
parens_level = 0
@@ -218,8 +304,33 @@ class MacroProcessor:
while i < len(input):
c = input[i]
peek = None if i + 1 >= len(input) else input[i + 1]
- # import sys
- # print(f"[{i:4}] {repr(c):4} -> {repr(peek):4} [{state}] = {repr(output)}", file=sys.stderr)
+
+ def hi_range():
+ if not self._debug_on(5):
+ return
+ _linestart = linestart
+ _linenr = linenr
+ if state_start < linestart:
+ _linestart = state_start
+ _linenr = f"~{linenr-1}"
+ range = debug_hi_range(
+ input, _linestart, state_start, i, color="\u001b[42m"
+ )
+ print(
+ f"[{file}:{_linenr}] {range} \t\u001b[33m({state})\u001b[0m",
+ file=sys.stderr,
+ )
+
+ if c == "\n":
+ linenr += 1
+ linestart = i + 1
+ self.file_stack[-1][1] = linenr
+
+ if self._debug_on(5):
+ print(
+ f"[{file}:{linenr}] {debug_hi(input, linestart, i)} \t\u001b[33m({state})\u001b[0m",
+ file=sys.stderr,
+ )
if state == ParserState.DNL:
if c == "\n":
@@ -233,14 +344,17 @@ class MacroProcessor:
if c == "%" and peek == "(":
state = ParserState.IN_CODE
i += 2
+ state_start = i
continue
if c == "%" and peek == '"':
state = ParserState.IN_QUOTES
quote_level += 1
i += 1
+ state_start = i
elif c.isalnum():
state = ParserState.IN_MACRO
+ state_start = i
macro_name += c
else:
output += c
@@ -254,37 +368,50 @@ class MacroProcessor:
quote_level -= 1
if quote_level == 0:
state = ParserState.NORMAL
+ state_start = i
else:
output += '"%'
i += 1
else:
output += c
+
elif state == ParserState.IN_MACRO:
if c.isalnum() or c == "_":
macro_name += c
elif c == "(":
+ hi_range()
parens_level += 1
state = ParserState.IN_MACRO_ARGS
+ state_start = i
else:
+ hi_range()
if macro_is_whitespace_deleting(macro_name):
if output[-1] == " ":
output = output[:-1]
macro_name = macro_name_clean(macro_name)
+ self._enter_frame(macro_name, file, linenr, input)
+
if macro_name == "SNNL":
skip_next_line_ending = c != "\n"
elif macro_name == "DNL":
if c != "\n":
state = ParserState.DNL
+ state_start = i
macro_name = ""
i += 1
+ self._pop_frame()
continue
else:
expanded = self.expand_macro(macro_name)
output += expanded
output += c
macro_name = ""
+
+ self._pop_frame()
+
state = ParserState.NORMAL
+ state_start = i
elif state == ParserState.IN_MACRO_ARGS:
if c == "%" and peek == '"':
quote_level += 1
@@ -302,20 +429,26 @@ class MacroProcessor:
continue
if (c == ")") and (parens_level == 1):
+ hi_range()
if macro_is_whitespace_deleting(macro_name):
if output[-1] == " ":
output = output[:-1]
macro_name = macro_name_clean(macro_name)
parens_level = 0
macro_args.append(argument.strip())
+ self._enter_frame(macro_name, file, linenr, input)
expanded = self.expand_macro(macro_name, macro_args)
output += expanded
state = ParserState.NORMAL
+ state_start = i
macro_name = ""
macro_args = []
argument = ""
+ self._pop_frame()
elif (c == ",") and (parens_level == 1):
macro_args.append(argument.strip())
+ state_start = i
+ hi_range()
argument = ""
else:
if c == "(":
@@ -326,6 +459,7 @@ class MacroProcessor:
elif state == ParserState.IN_CODE:
if c == ")" and peek == "%":
try:
+ self._enter_frame("inline_code", file, linenr, input)
f = StringIO()
with redirect_stdout(f):
exec(py_expr, self.py_global_env, self.py_local_env_current)
@@ -334,9 +468,12 @@ class MacroProcessor:
output += s
except Exception:
traceback.print_exc()
+ finally:
+ self._pop_frame()
py_expr = ""
state = ParserState.NORMAL
i += 1
+ state_start = i
else:
py_expr += c
i += 1
@@ -347,6 +484,7 @@ class MacroProcessor:
if len(output) > 0 and output[-1] == " ":
output = output[:-1]
macro_name = macro_name_clean(macro_name)
+ hi_range()
output += self.expand_macro(macro_name)
return output
generated by cgit v1.2.3 (git 2.39.1) at 2025-04-26 21:47:14 +0000