Rename some macros, add stack trace system

10 files changed, 200 insertions, 46 deletions

diff --git a/src/skaldpress/macros.py b/src/skaldpress/macros.py
index 33aac86..9d06505 100644
--- a/src/skaldpress/macros.py
+++ b/src/skaldpress/macros.py
@@ -1,6 +1,6 @@
 from copy import deepcopy
 from smp.builtins import (
-    smp_builtin_read,
+    _smp_builtin_read,
 )
 
 
@@ -42,6 +42,6 @@ def sp_all_tagged_by(
 
         smp_local.macros.update(file["stored_data"])
         smp_builtin_undefine(smp_local, "METADATA_template")
-        out += smp_builtin_read(smp_local, template, template_content=file["content"])
+        out += _smp_builtin_read(smp_local, template, template_content=file["content"])
         macro_processor.warnings.extend(smp_local.warnings)
     return out
diff --git a/src/skaldpress/main.py b/src/skaldpress/main.py
index 8fd823a..b0b1784 100644
--- a/src/skaldpress/main.py
+++ b/src/skaldpress/main.py
@@ -14,7 +14,7 @@ from skaldpress.filelist import (
     file_filtered,
 )
 from smp.builtins import (
-    smp_builtin_read,
+    _smp_builtin_read,
     smp_builtin_add_metadata,
 )
 import skaldpress.macros
@@ -63,7 +63,7 @@ def compile_file(smps: smp.macro_processor.MacroProcessorState, file_path, opts)
     macro_processor.source_file_path = file_path
     macro_processor_initialize(opts.metadata, macro_processor, stored_smp_state)
 
-    content = smp_builtin_read(macro_processor, file_path)
+    content = _smp_builtin_read(macro_processor, file_path)
 
     print_warnings(macro_processor)
     return macro_processor.store(content=content)
diff --git a/src/smp/__init__.py b/src/smp/__init__.py
index 51dce90..898e683 100644
--- a/src/smp/__init__.py
+++ b/src/smp/__init__.py
@@ -22,7 +22,8 @@ def read_stdin():
     data = sys.stdin.read()
     macro_processor_state = smp.macro_processor.MacroProcessorState()
     macro_processor = macro_processor_state.macro_processor()
-    res = macro_processor.process_input(data)
+    macro_processor._enter_file_frame("[stdin]", 0, None)
+    res = macro_processor.process_input(data, file="[stdin]")
     print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━", file=sys.stderr)
     print(res)
 
@@ -42,11 +43,11 @@ def main():
         repl()
         sys.exit(0)
 
-    with open(sys.argv[1], "r") as f:
-        file_content = f.read()
-
     macro_processor_state = smp.macro_processor.MacroProcessorState()
     macro_processor = macro_processor_state.macro_processor()
-    res = macro_processor.process_input(file_content)
+    res = smp.builtins._smp_builtin_read(macro_processor, sys.argv[1])
     print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━", file=sys.stderr)
     print(res)
+
+    for warning in macro_processor.warnings:
+        print(f"\u001b[33m{warning}\u001b[0m", file=sys.stderr)
diff --git a/src/smp/builtins.py b/src/smp/builtins.py
index b44cc23..a9e1977 100644
--- a/src/smp/builtins.py
+++ b/src/smp/builtins.py
@@ -92,8 +92,8 @@ def smp_builtin_add_metadata(macro_processor, metadata: dict[str, Any], overwrit
             macro_processor.define_macro(macro_name, macro_value)
 
 
-def smp_builtin_include(macro_processor, filename):
-    return smp_builtin_read(macro_processor, filename, template_content=None)
+def smp_builtin_include_file(macro_processor, filename):
+    return _smp_builtin_read(macro_processor, filename, template_content=None)
 
 
 def smp_builtin_parse_leading_yaml(macro_processor, content):
@@ -171,9 +171,7 @@ def smp_builtin_format_time(macro_processor, format, time):
 
 
 def smp_builtin_html_from_markdown(macro_processor, text, extensions=list()):
-    # Get rid of quoting, I don't remember why, but the rust implementation does it like this.
-    for _ in range(2):
-        text = macro_processor.process_input(text)
+    text = macro_processor.process_input(text)
     extensions.append(TableExtension())
     extensions.append(FencedCodeExtension())
     extensions.append(AutolinkExtension())
@@ -181,24 +179,30 @@ def smp_builtin_html_from_markdown(macro_processor, text, extensions=list()):
     return markdown.markdown(text, extensions=extensions)
 
 
-def _smp_builtin_template_content(content):
+def _smp_builtin_template_content():
     def inner(macro_processor):
-        """
-        This should do some kind of stack thing, so we can track which file we are processing.
-        entering the CONTENT is fine, the question is how to handle exiting it.
+        filename, content, extension = macro_processor._get_macro_with_prefix(
+            "template_stack_content"
+        )[0]
+        macro_processor._enter_file_frame(f"[part]{filename}", 0, "")
+        res = macro_processor.process_input(content, file=filename)
 
-        could have a "once" macro or something, that is added to the end of the content.
-        """
-        return content
+        if extension == "md":
+            res = smp_builtin_html_from_markdown(macro_processor, res)
+
+        macro_processor._pop_file_frame()
+        macro_processor._get_macro_with_prefix("template_stack_content").pop(0)
+        return res
 
     return inner
 
 
 def smp_builtin_template(macro_processor, template, content):
-    return smp_builtin_read(macro_processor, template, template_content=content)
+    return _smp_builtin_read(macro_processor, template, template_content=content)
 
 
-def smp_builtin_read(macro_processor, filename, template_content=None):
+def _smp_builtin_read(macro_processor, filename, template_content=None):
+    macro_processor._enter_file_frame(filename, 0, template_content)
     with open(filename, "r") as f:
         file_content = f.read()
 
@@ -214,14 +218,27 @@ def smp_builtin_read(macro_processor, filename, template_content=None):
         macro_processor._get_macro_with_prefix("template_stack").append(filename)
         macro_processor.macros["CONTENT"] = template_content
 
-    content = macro_processor.process_input(file_content)
-
-    if extension == "md":
-        content = smp_builtin_html_from_markdown(macro_processor, content)
+    # content = macro_processor.process_input(file_content, file=filename)
+    content = file_content
 
     if (template := macro_processor._get_metadata("template")) is not None:
+
+        template_prefix = macro_processor._get_macro_with_prefix("template_prefix")
+        if not os.path.exists(template):
+            template = os.path.join(template_prefix, template)
+
         if template not in macro_processor._get_macro_with_prefix("template_stack"):
-            return smp_builtin_read(macro_processor, template, content)
+            macro_processor._get_macro_with_prefix("template_stack_content").append(
+                (filename, content, extension)
+            )
+            return _smp_builtin_read(
+                macro_processor, template, _smp_builtin_template_content()
+            )
+
+    content = macro_processor.process_input(file_content, file=filename)
+
+    if extension == "md":
+        content = smp_builtin_html_from_markdown(macro_processor, content)
 
     return content
 
@@ -258,12 +275,12 @@ def smp_builtin_wodl(macro_processor, link, timeout_seconds=5):
         cache[link] = (working_link, r.status, r.reason)
         if not working_link:
             macro_processor.log_warning(f"Dead link {link} ({r.status} {r.reason})!")
-    except urllib.error.URLError as e:
+    except Exception as e:
         macro_processor.log_warning(f"Dead link {link} ({e})!")
     return ""
 
 
-def smp_builtin_once(macro_processor, content):
+def smp_builtin_expand_once(macro_processor, content):
     if (cache := macro_processor._get_macro_with_prefix("once_cache")) is not None:
         if (exp := cache.get(content)) is not None:
             return exp
diff --git a/src/smp/macro_processor.py b/src/smp/macro_processor.py
index a31f040..77641e7 100644
--- a/src/smp/macro_processor.py
+++ b/src/smp/macro_processor.py
@@ -1,6 +1,7 @@
 import smp.builtins
 import traceback
 import inspect
+import sys
 
 from typing import Any
 from enum import Enum
@@ -19,6 +20,25 @@ class ParserState(Enum):
     DNL = 8
 
 
+def debug_hi(input, linestart, pos, color="\u001b[7m"):
+    return debug_hi_range(input, linestart, pos, pos + 1, color=color)
+
+
+def debug_hi_range(input, linestart, start, end, color="\u001b[7m"):
+    lineend = input.find("\n", linestart)
+    line = input[linestart:] if lineend == -1 else input[linestart:lineend]
+
+    out = ""
+    for j, _c in enumerate(line):
+        c_pos = linestart + j
+        if (c_pos >= start) and (c_pos < end):
+            out += f"{color}{_c}\u001b[0m"
+        else:
+            out += f"{_c}"
+
+    return out
+
+
 def macro_is_whitespace_deleting(s: str) -> bool:
     if len(s) == 0:
         return False
@@ -49,6 +69,11 @@ class MacroProcessor:
     end_quote: str = '"%'
     prefix: str = ""
 
+    file = None
+
+    expansion_stack: list[Any]
+    file_stack: list[Any]
+
     def __init__(self, prefix=""):
         self.macros = dict()
         self.macro_invocations = list()
@@ -58,6 +83,11 @@ class MacroProcessor:
         self.py_local_env_current = self.macros
         self.prefix = prefix
 
+        self.expansion_stack = (
+            list()
+        )  # This probably needs some magic at some point later
+        self.file_stack = list()  # This probably needs some magic at some point later
+
         self._define_builtins(self.macros)
         self._define_builtins(self.py_local_env_alt)
 
@@ -75,6 +105,8 @@ class MacroProcessor:
 
         env[f"{self.prefix}macro_processor"] = self
         env[f"{self.prefix}template_stack"] = []
+        env[f"{self.prefix}template_stack_content"] = []
+        env[f"{self.prefix}template_prefix"] = "templates/"
 
         # If true, include-macros will parse yaml in beginning of content
         env[f"{self.prefix}parse_file_yaml"] = True
@@ -82,6 +114,7 @@ class MacroProcessor:
         # meaning they will skip steps that are slow.
         env[f"{self.prefix}draft"] = False
         env[f"{self.prefix}metadata_prefix"] = "METADATA_"
+        env[f"{self.prefix}smp_debug"] = "1"
 
     def define_macro_string(self, macro_name, macro_value):
         self.define_macro(macro_name, str(macro_value))
@@ -122,14 +155,17 @@ class MacroProcessor:
             macro_args.append(self)
         macro_args.extend(args)
         # This should maybe be processed as well(?)
-        return str(macro(*macro_args))
+        res = str(macro(*macro_args))
+        return res
 
     def _expand_string_macro(self, macro: Any, args: list[str] = list()) -> str:
         expanded = macro
         for i, arg in enumerate(args):
             placeholder = f"${i}"
             expanded = macro.replace(placeholder, arg)
-        return self.process_input(expanded)
+
+        res = self.process_input(expanded)
+        return res
 
     def _expand_unknown_macro(
         self, macro_name: str, args: list[str] = list(), process_args: bool = True
@@ -170,9 +206,17 @@ class MacroProcessor:
                 return self._expand_callable_macro(macro, args)
             except Exception as e:
                 s = self._expand_unknown_macro(macro_name, args, process_args=False)
-                self.log_warning(
-                    f"Error expanding macro {s} ({e})\n{traceback.format_exc()}"
-                )
+
+                e2 = ""
+                if self._debug_on(1):
+                    e2 += "\n"
+                    e2 += self._stack()
+
+                if self._debug_on(2):
+                    e2 += "\n- Python stack ---\n"
+                    e2 += traceback.format_exc()
+
+                self.log_warning(f"Error expanding macro {s} ({e}){e2}")
                 return s
 
         if isinstance(macro, str):
@@ -180,7 +224,42 @@ class MacroProcessor:
 
         return f"{repr(macro)}"
 
-    def process_input(self, input: str):
+    def _enter_frame(self, frame_type, filename, position, content):
+        self.expansion_stack.append((frame_type, filename, position, ""))
+
+    def _pop_frame(self):
+        self.expansion_stack.pop()
+
+    def _enter_file_frame(self, filename, linenr, template):
+        self.file_stack.append([filename, linenr, 0])
+
+    def _pop_file_frame(self):
+        self.file_stack.pop()
+
+    def _stack(self):
+        out = "- File stack ------\n"
+        for i, frame in enumerate(self.file_stack):
+            out += f" {' ' * i} {frame[0]}:{frame[1]}\n"
+        out += "\n- Macro stack -----\n"
+        for i, frame in enumerate(self.expansion_stack):
+            out += f" {' ' * i} [{frame[0]}] in file {frame[1]}:{frame[2]}\n"
+        return out
+
+    def _debug_on(self, level):
+        try:
+            if level > int(self._get_macro_with_prefix("smp_debug")):
+                return False
+        except:
+            return False
+
+        return True
+
+    def _debug(self, level, message):
+        if self._debug_on(level):
+            return
+        print(message, file=sys.stderr)
+
+    def process_input(self, input: str, file: str | None = None):
         """
         I also want to add special syntax for "special blocks",
         I am thinking of two main options, either some macro_names are intercepted, _or_ a special kind of macro can exist like
@@ -198,8 +277,13 @@ class MacroProcessor:
         by pushing the expanded macros back to the input string, this may be more confusing,
         but may also be faster (stream or mutable string)
         """
+        if file is not None:
+            self.file = file
+        if (file is None) and (self.file is not None):
+            file = self.file
         output = ""
         state = ParserState.NORMAL
+        state_start = 0
         macro_name = ""
         macro_args = []
         argument = ""
@@ -210,6 +294,8 @@ class MacroProcessor:
         # We should keep track of filename, linenumber, and character number on line here
         # So we can give sensible error messages
         # Probably add to python stack trace?
+        linestart = 0
+        linenr = 0
 
         quote_level = 0
         parens_level = 0
@@ -218,8 +304,33 @@ class MacroProcessor:
         while i < len(input):
             c = input[i]
             peek = None if i + 1 >= len(input) else input[i + 1]
-            # import sys
-            # print(f"[{i:4}] {repr(c):4} -> {repr(peek):4} [{state}] = {repr(output)}", file=sys.stderr)
+
+            def hi_range():
+                if not self._debug_on(5):
+                    return
+                _linestart = linestart
+                _linenr = linenr
+                if state_start < linestart:
+                    _linestart = state_start
+                    _linenr = f"~{linenr-1}"
+                range = debug_hi_range(
+                    input, _linestart, state_start, i, color="\u001b[42m"
+                )
+                print(
+                    f"[{file}:{_linenr}] {range} \t\u001b[33m({state})\u001b[0m",
+                    file=sys.stderr,
+                )
+
+            if c == "\n":
+                linenr += 1
+                linestart = i + 1
+                self.file_stack[-1][1] = linenr
+
+            if self._debug_on(5):
+                print(
+                    f"[{file}:{linenr}] {debug_hi(input, linestart, i)} \t\u001b[33m({state})\u001b[0m",
+                    file=sys.stderr,
+                )
 
             if state == ParserState.DNL:
                 if c == "\n":
@@ -233,14 +344,17 @@ class MacroProcessor:
                 if c == "%" and peek == "(":
                     state = ParserState.IN_CODE
                     i += 2
+                    state_start = i
                     continue
 
                 if c == "%" and peek == '"':
                     state = ParserState.IN_QUOTES
                     quote_level += 1
                     i += 1
+                    state_start = i
                 elif c.isalnum():
                     state = ParserState.IN_MACRO
+                    state_start = i
                     macro_name += c
                 else:
                     output += c
@@ -254,37 +368,50 @@ class MacroProcessor:
                     quote_level -= 1
                     if quote_level == 0:
                         state = ParserState.NORMAL
+                        state_start = i
                     else:
                         output += '"%'
                     i += 1
                 else:
                     output += c
+
             elif state == ParserState.IN_MACRO:
                 if c.isalnum() or c == "_":
                     macro_name += c
                 elif c == "(":
+                    hi_range()
                     parens_level += 1
                     state = ParserState.IN_MACRO_ARGS
+                    state_start = i
                 else:
+                    hi_range()
                     if macro_is_whitespace_deleting(macro_name):
                         if output[-1] == " ":
                             output = output[:-1]
                         macro_name = macro_name_clean(macro_name)
 
+                    self._enter_frame(macro_name, file, linenr, input)
+
                     if macro_name == "SNNL":
                         skip_next_line_ending = c != "\n"
                     elif macro_name == "DNL":
                         if c != "\n":
                             state = ParserState.DNL
+                            state_start = i
                         macro_name = ""
                         i += 1
+                        self._pop_frame()
                         continue
                     else:
                         expanded = self.expand_macro(macro_name)
                         output += expanded
                         output += c
                     macro_name = ""
+
+                    self._pop_frame()
+
                     state = ParserState.NORMAL
+                    state_start = i
             elif state == ParserState.IN_MACRO_ARGS:
                 if c == "%" and peek == '"':
                     quote_level += 1
@@ -302,20 +429,26 @@ class MacroProcessor:
                     continue
 
                 if (c == ")") and (parens_level == 1):
+                    hi_range()
                     if macro_is_whitespace_deleting(macro_name):
                         if output[-1] == " ":
                             output = output[:-1]
                         macro_name = macro_name_clean(macro_name)
                     parens_level = 0
                     macro_args.append(argument.strip())
+                    self._enter_frame(macro_name, file, linenr, input)
                     expanded = self.expand_macro(macro_name, macro_args)
                     output += expanded
                     state = ParserState.NORMAL
+                    state_start = i
                     macro_name = ""
                     macro_args = []
                     argument = ""
+                    self._pop_frame()
                 elif (c == ",") and (parens_level == 1):
                     macro_args.append(argument.strip())
+                    state_start = i
+                    hi_range()
                     argument = ""
                 else:
                     if c == "(":
@@ -326,6 +459,7 @@ class MacroProcessor:
             elif state == ParserState.IN_CODE:
                 if c == ")" and peek == "%":
                     try:
+                        self._enter_frame("inline_code", file, linenr, input)
                         f = StringIO()
                         with redirect_stdout(f):
                             exec(py_expr, self.py_global_env, self.py_local_env_current)
@@ -334,9 +468,12 @@ class MacroProcessor:
                             output += s
                     except Exception:
                         traceback.print_exc()
+                    finally:
+                        self._pop_frame()
                     py_expr = ""
                     state = ParserState.NORMAL
                     i += 1
+                    state_start = i
                 else:
                     py_expr += c
             i += 1
@@ -347,6 +484,7 @@ class MacroProcessor:
                 if len(output) > 0 and output[-1] == " ":
                     output = output[:-1]
                 macro_name = macro_name_clean(macro_name)
+            hi_range()
             output += self.expand_macro(macro_name)
         return output
 
diff --git a/tests/skaldpress/content/article.md b/tests/skaldpress/content/article.md
index b20468c..7d99e15 100644
--- a/tests/skaldpress/content/article.md
+++ b/tests/skaldpress/content/article.md
@@ -9,3 +9,4 @@ tags:
 ---
 
 This is a example article
+
diff --git a/tests/skaldpress/templates/article.html b/tests/skaldpress/templates/article.html
index cb3b58b..c47a114 100644
--- a/tests/skaldpress/templates/article.html
+++ b/tests/skaldpress/templates/article.html
@@ -1,10 +1,7 @@
 ---
 template: base.html
 table_of_contents: true
-keep_states:
-  - TOC_ITEMS
 ---
-DNL include(templates/common_macros.smp)DNL
 <header>
     <h1>METADATA_title</h1>
 </header>
@@ -17,6 +14,5 @@ DNL include(templates/common_macros.smp)DNL
         )
     </div>
 
-    CONTENT
+    indent(4, CONTENT)
 </main>
-
diff --git a/tests/skaldpress/templates/base.html b/tests/skaldpress/templates/base.html
index aa73ffc..b66a00b 100644
--- a/tests/skaldpress/templates/base.html
+++ b/tests/skaldpress/templates/base.html
@@ -5,10 +5,11 @@
             <a href="/">Home</a>
         </nav>
 
-        CONTENT
+        indent(8, CONTENT)
 
         <footer>
         </footer>
     </body>
 </html>
 
+dumpenv
diff --git a/tests/smp/include_1 b/tests/smp/include_1
index 1621de2..9664507 100644
--- a/tests/smp/include_1
+++ b/tests/smp/include_1
@@ -1,3 +1,3 @@
-include(tests/example_include.smp)
+include_file(tests/example_include.smp)
 ---
 
diff --git a/tests/smp/include_2 b/tests/smp/include_2
index 2714eff..b69fe13 100644
--- a/tests/smp/include_2
+++ b/tests/smp/include_2
@@ -1,4 +1,4 @@
-include(tests/example_include.smp)SNNL
+include_file(tests/example_include.smp)SNNL
 ifdef(SMP, SMP_ISDEF, SMP_ISNDEF)
 ---
 SMP_ISDEF