Approach malleable system

5 files changed, 146 insertions, 140 deletions

diff --git a/src/skaldpress/macros.py b/src/skaldpress/macros.py
new file mode 100644
index 0000000..33aac86
--- /dev/null
+++ b/src/skaldpress/macros.py
@@ -0,0 +1,47 @@
+from copy import deepcopy
+from smp.builtins import (
+    smp_builtin_read,
+)
+
+
+def sp_all_tagged_by(
+    macro_processor, tag: str, template: str, field=None, reversed=""
+) -> str:
+    """
+    SMP Macro for getting all files with specific tag, this is only _really_ effective the second run
+
+    Usage in files:
+    all_tagged_by(<tag name>, <template> [, <field to sort by>] [, reversed])
+    """
+    tagged_files = [
+        k
+        for k, v in macro_processor.py_global_env["macro_processor_state"].items()
+        if f"{macro_processor._get_macro_with_prefix('metadata_prefix')}tags"
+        in v["stored_data"]
+    ]
+
+    if len(tagged_files) == 0:
+        macro_processor.log_warning(f"No tags for {tag}\u001b[0m")
+        return ""
+
+    if field is not None:
+        tagged_files.sort(
+            key=lambda fname: macro_processor.py_global_env["macro_processor_state"][
+                fname
+            ]["stored_data"][
+                f"{macro_processor._get_macro_with_prefix('metadata_prefix')}{field}"
+            ],
+            reverse=(reversed != ""),
+        )
+
+    out = ""
+    for filename in tagged_files:
+        file = macro_processor.py_global_env["macro_processor_state"][filename]
+        smp_local = deepcopy(macro_processor)
+        from smp.builtins import smp_builtin_undefine
+
+        smp_local.macros.update(file["stored_data"])
+        smp_builtin_undefine(smp_local, "METADATA_template")
+        out += smp_builtin_read(smp_local, template, template_content=file["content"])
+        macro_processor.warnings.extend(smp_local.warnings)
+    return out
diff --git a/src/skaldpress/main.py b/src/skaldpress/main.py
index cc6dc94..8fd823a 100644
--- a/src/skaldpress/main.py
+++ b/src/skaldpress/main.py
@@ -7,7 +7,6 @@ from functools import partial
 from itertools import chain
 from collections import deque
 import smp.macro_processor
-from copy import deepcopy
 from skaldpress.filelist import (
     make_filelist_iter,
     FileList,
@@ -18,47 +17,10 @@ from smp.builtins import (
     smp_builtin_read,
     smp_builtin_add_metadata,
 )
+import skaldpress.macros
 from time import perf_counter
 
 
-def sp_all_tagged_by(
-    macro_processor, tag: str, template: str, field=None, reversed=""
-) -> str:
-    """
-    SMP Macro for getting all files with specific tag, this is only _really_ effective the second run
-
-    Usage in files:
-    all_tagged_by(<tag name>, <template> [, <field to sort by>] [, reversed])
-    """
-    tagged_files = [
-        k
-        for k, v in macro_processor.py_global_env["macro_processor_state"].items()
-        if "METADATA_tags" in v["stored_data"]
-    ]
-
-    if len(tagged_files) == 0:
-        macro_processor.log_warning(f"No tags for {tag}\u001b[0m")
-        return ""
-
-    # FRAGILE
-    if field is not None:
-        tagged_files.sort(
-            key=lambda fname: macro_processor.py_global_env["macro_processor_state"][
-                fname
-            ]["stored_data"][f"METADATA_{field}"],
-            reverse=(reversed != ""),
-        )
-
-    out = ""
-    for filename in tagged_files:
-        file = macro_processor.py_global_env["macro_processor_state"][filename]
-        smp_local = deepcopy(macro_processor)
-        smp_local.macros.update(file["stored_data"])
-        out += smp_builtin_read(smp_local, template, template_content=file["content"])
-        print_warnings(smp_local)
-    return out
-
-
 class SkaldpressError(Exception):
     def __init__(self, code, error, path=None):
         self.code = code
@@ -73,7 +35,9 @@ def print_warnings(macro_processor):
 
 def macro_processor_initialize(metadata, old_macro_processor, additional_state=None):
     macro_processor = old_macro_processor
-    macro_processor.define_macro("all_tagged_by", sp_all_tagged_by)
+    macro_processor._import_symbols(
+        skaldpress.macros, macro_processor.macros, function_prefix="sp_"
+    )
     smp_builtin_add_metadata(macro_processor, metadata, overwrite=False)
     if additional_state:
         for key, value in additional_state.items():
@@ -86,12 +50,13 @@ def compile_file(smps: smp.macro_processor.MacroProcessorState, file_path, opts)
         raise SkaldpressError(3, None)
 
     stored_smp_state = None
-    if file_path in smps.global_state:
-        stored_smp_state = smps.global_state[file_path]["stored_data"]
+    # if file_path in smps.global_state:
+    #    stored_smp_state = smps.global_state[file_path]["stored_data"]
+    #    print(stored_smp_state)
 
     macro_processor = smps.macro_processor()
-    macro_processor.define_macro_string(
-        "METADATA_filename",
+    macro_processor._define_metadata(
+        "filename",
         os.path.splitext(os.path.relpath(file_path, opts.content_dir))[0],
     )
 
diff --git a/src/smp/__init__.py b/src/smp/__init__.py
index d6e5d52..51dce90 100644
--- a/src/smp/__init__.py
+++ b/src/smp/__init__.py
@@ -48,7 +48,5 @@ def main():
     macro_processor_state = smp.macro_processor.MacroProcessorState()
     macro_processor = macro_processor_state.macro_processor()
     res = macro_processor.process_input(file_content)
-    macro_processor.store("", "", "")
-    breakpoint()
     print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━", file=sys.stderr)
     print(res)
diff --git a/src/smp/builtins.py b/src/smp/builtins.py
index afa153d..2fde231 100644
--- a/src/smp/builtins.py
+++ b/src/smp/builtins.py
@@ -201,7 +201,7 @@ def smp_builtin_read(macro_processor, filename, template_content=None):
     metadata = {}
     if macro_processor._get_macro_with_prefix("parse_file_yaml"):
         metadata, file_content = extract_parse_yaml_metadata(file_content)
-        smp_builtin_add_metadata(macro_processor, metadata, overwrite=False)
+        smp_builtin_add_metadata(macro_processor, metadata, overwrite=True)
 
     extension = os.path.splitext(filename)[1][1:] or ""
     macro_processor._define_macro_with_prefix("target_file_extension", extension)
@@ -215,13 +215,22 @@ def smp_builtin_read(macro_processor, filename, template_content=None):
     if extension == "md":
         content = smp_builtin_html_from_markdown(macro_processor, content)
 
-    if (template := macro_processor.macros.get("METADATA_template")) is not None:
+    if (template := macro_processor._get_metadata("template")) is not None:
         if template not in macro_processor._get_macro_with_prefix("template_stack"):
             return smp_builtin_read(macro_processor, template, content)
 
     return content
 
 
+def smp_builtin_indent(macro_processor, indent: int, content: str):
+    indent = int(indent)
+    content = macro_processor.process_input(content)
+    return "\n".join(
+        f"{' ' * (indent if i > 0 else 0)}{x}"
+        for i, x in enumerate(content.split("\n"))
+    )
+
+
 def smp_builtin_wodl(macro_processor, link, timeout_seconds=5):
     if (macro_processor._get_macro_with_prefix("wodl_cache")) is None:
         macro_processor._define_macro_with_prefix("wodl_cache", {})
diff --git a/src/smp/macro_processor.py b/src/smp/macro_processor.py
index 2af26dd..a31f040 100644
--- a/src/smp/macro_processor.py
+++ b/src/smp/macro_processor.py
@@ -31,48 +31,20 @@ def macro_name_clean(macro_name: str) -> str:
     return macro_name
 
 
-def seek(input: str, start: int, target: str) -> int | None:
-    """Seek for a value in a string, consider using startswith instead"""
-    from warnings import warn
-
-    warn(
-        "seek should be considered replaced with str.startswith",
-        DeprecationWarning,
-        stacklevel=2,
-    )
-    input_end = len(input)
-    target_end = len(target)
-
-    if input_end < start + target_end:
-        return None
-
-    i = 0
-    while i < len(target):
-        if input[start + i] != target[i]:
-            return None
-        i += 1
-
-    return start + target_end
-
-
 class MacroProcessor:
     source_file_path: str
-
     """All currently defined macros in this MacroProcessor"""
     macros: dict[str, Any]
-    """ All macro invocations that has happened """
-    macro_invocations: list[tuple[str, list[str]]]
-    warnings: list[Any]
-
+    """ Macros which are @directives """
+    special_macros: dict[str, tuple[Any, Any]]
     """ Global environment for python execution """
-
     py_global_env: dict
-
+    """ Local environment for python execution """
     py_local_env_alt: dict
     py_local_env_current: dict
-
-    special_macros: dict[str, tuple[Any, Any]]
-
+    """ All macro invocations that has happened """
+    macro_invocations: list[tuple[str, list[str]]]
+    warnings: list[Any]
     start_quote: str = '%"'
     end_quote: str = '"%'
     prefix: str = ""
@@ -84,39 +56,24 @@ class MacroProcessor:
         self.py_global_env = dict()
         self.py_local_env_alt = dict()
         self.py_local_env_current = self.macros
-        self.indent_level = ""
         self.prefix = prefix
 
         self._define_builtins(self.macros)
         self._define_builtins(self.py_local_env_alt)
 
+    def _import_symbols(self, module, env, function_prefix=""):
+        for name, fun in inspect.getmembers(module, inspect.isfunction):
+            if name.startswith("_"):
+                continue
+            if not name.startswith(function_prefix):
+                continue
+            name = name.replace(function_prefix, "")
+            env[f"{self.prefix}{name}"] = fun
+
     def _define_builtins(self, env):
+        self._import_symbols(smp.builtins, env, function_prefix="smp_builtin_")
+
         env[f"{self.prefix}macro_processor"] = self
-        env[f"{self.prefix}define"] = smp.builtins.smp_builtin_define
-        env[f"{self.prefix}undefine"] = smp.builtins.smp_builtin_undefine
-        env[f"{self.prefix}define_array"] = smp.builtins.smp_builtin_define_array
-        env[f"{self.prefix}ifdef"] = smp.builtins.smp_builtin_ifdef
-        env[f"{self.prefix}ifndef"] = smp.builtins.smp_builtin_ifndef
-        env[f"{self.prefix}ifeq"] = smp.builtins.smp_builtin_ifeq
-        env[f"{self.prefix}ifneq"] = smp.builtins.smp_builtin_ifneq
-        env[f"{self.prefix}once"] = smp.builtins.smp_builtin_once
-        env[f"{self.prefix}include"] = smp.builtins.smp_builtin_include
-        env[f"{self.prefix}include_verbatim"] = (
-            smp.builtins.smp_builtin_include_verbatim
-        )
-        env[f"{self.prefix}shell"] = smp.builtins.smp_builtin_shell
-        env[f"{self.prefix}dumpenv"] = smp.builtins.smp_builtin_dumpenv
-        env[f"{self.prefix}eval"] = smp.builtins.smp_builtin_eval
-        env[f"{self.prefix}array_push"] = smp.builtins.smp_builtin_array_push
-        env[f"{self.prefix}array_each"] = smp.builtins.smp_builtin_array_each
-        env[f"{self.prefix}array_size"] = smp.builtins.smp_builtin_array_size
-        env[f"{self.prefix}explode"] = smp.builtins.smp_builtin_explode
-        env[f"{self.prefix}format_time"] = smp.builtins.smp_builtin_format_time
-        env[f"{self.prefix}html_from_markdown"] = (
-            smp.builtins.smp_builtin_html_from_markdown
-        )
-        env[f"{self.prefix}wodl"] = smp.builtins.smp_builtin_wodl
-        env[f"{self.prefix}template"] = smp.builtins.smp_builtin_template
         env[f"{self.prefix}template_stack"] = []
 
         # If true, include-macros will parse yaml in beginning of content
@@ -138,26 +95,64 @@ class MacroProcessor:
     def _get_macro_with_prefix(self, macro_name, sub_prefix: str = "", default=None):
         return self.macros.get(f"{self.prefix}{sub_prefix}{macro_name}", default)
 
+    def _get_metadata(self, macro_name, default=None):
+        return self._get_macro_with_prefix(
+            macro_name,
+            sub_prefix=self._get_macro_with_prefix("metadata_prefix"),
+            default=default,
+        )
+
+    def _define_metadata(self, macro_name, macro_value):
+        return self._define_macro_with_prefix(
+            macro_name,
+            macro_value,
+            sub_prefix=self._get_macro_with_prefix("metadata_prefix"),
+        )
+
     def log_warning(self, message):
         """
         Here we should add some more information, line number, file etc, when that is available
         """
         self.warnings.append(message)
 
+    def _expand_callable_macro(self, macro: Any, args: list[str] = list()) -> str:
+        signature = inspect.signature(macro)
+        macro_args: list[Any] = []
+        if "macro_processor" in signature.parameters or "smp" in signature.parameters:
+            macro_args.append(self)
+        macro_args.extend(args)
+        # This should maybe be processed as well(?)
+        return str(macro(*macro_args))
+
+    def _expand_string_macro(self, macro: Any, args: list[str] = list()) -> str:
+        expanded = macro
+        for i, arg in enumerate(args):
+            placeholder = f"${i}"
+            expanded = macro.replace(placeholder, arg)
+        return self.process_input(expanded)
+
+    def _expand_unknown_macro(
+        self, macro_name: str, args: list[str] = list(), process_args: bool = True
+    ) -> str:
+        if len(args) == 0:
+            return macro_name
+        out = f"{macro_name}("
+        for i, arg in enumerate(args):
+            if process_args:
+                out += self.process_input(arg)
+            else:
+                out += arg
+            if i < (len(args) - 1):
+                out += ","
+        out += ")"
+        return out
+
     def expand_macro(self, macro_name: str, args: list[str] = list()) -> str:
         # Ignore trailing underscore in macro name, the parser will pop a space in front if
         # present, but we should ignore it for finding the macro.
         macro_name = macro_name_clean(macro_name)
         if macro_name not in self.macros:
-            if len(args) == 0:
-                return macro_name
-            out = f"{macro_name}("
-            for i, arg in enumerate(args):
-                out += self.process_input(arg)
-                if i < (len(args) - 1):
-                    out += ","
-            out += ")"
-            return out
+            return self._expand_unknown_macro(macro_name, args, process_args=True)
 
         # Strip leading whitespace from arguments
         for arg in args:
@@ -171,28 +166,18 @@ class MacroProcessor:
         macro = self.macros.get(macro_name)
 
         if callable(macro):
-            signature = inspect.signature(macro)
-            macro_args: list[Any] = []
-            if (
-                "macro_processor" in signature.parameters
-                or "smp" in signature.parameters
-            ):
-                macro_args.append(self)
-            macro_args.extend(args)
             try:
-                return str(macro(*macro_args))
+                return self._expand_callable_macro(macro, args)
             except Exception as e:
-                s = f"{macro_name}({','.join([repr(x) for x in args])})"
+                s = self._expand_unknown_macro(macro_name, args, process_args=False)
                 self.log_warning(
                     f"Error expanding macro {s} ({e})\n{traceback.format_exc()}"
                 )
                 return s
+
         if isinstance(macro, str):
-            expanded = macro
-            for i, arg in enumerate(args):
-                placeholder = f"${i}"
-                expanded = macro.replace(placeholder, arg)
-            return self.process_input(expanded)
+            return self._expand_string_macro(macro, args)
+
         return f"{repr(macro)}"
 
     def process_input(self, input: str):
@@ -366,21 +351,23 @@ class MacroProcessor:
         return output
 
     def store(self, **xargs):
-        requested_keys = self.macros.get("METADATA_keep_states", self.macros.keys())
+        requested_keys = self._get_metadata("keep_states", [])
         for key in self.macros.keys():
-            if key.startswith("METADATA_") and key not in requested_keys:
+            if (
+                key.startswith(self._get_macro_with_prefix("metadata_prefix"))
+                and key not in requested_keys
+            ):
                 requested_keys.append(key)
 
         if isinstance(requested_keys, str):
             requested_keys = [str(requested_keys)]
 
-        needs_recompilation = ("METADATA_keep_states" in self.macros) or (
-            "all_tagged_by" in [x[0] for x in self.macro_invocations]
-        )
+        needs_recompilation = (
+            f"{self._get_macro_with_prefix('metadata_prefix')}keep_states"
+            in self.macros
+        ) or ("all_tagged_by" in [x[0] for x in self.macro_invocations])
 
-        target_filename = self._get_macro_with_prefix(
-            "target_filename", sub_prefix="METADATA_"
-        )
+        target_filename = self._get_metadata("target_filename")
 
         self.py_global_env["macro_processor_state"][self.source_file_path] = dict(
             {