diff options
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | src/skaldpress/macros.py | 47 | ||||
| -rw-r--r-- | src/skaldpress/main.py | 53 | ||||
| -rw-r--r-- | src/smp/__init__.py | 2 | ||||
| -rw-r--r-- | src/smp/builtins.py | 13 | ||||
| -rw-r--r-- | src/smp/macro_processor.py | 171 |
6 files changed, 149 insertions, 141 deletions
@@ -34,5 +34,7 @@ test: mypy src/ && \ pyflakes src/ && \ ./tests/test_macro_processor.sh && \ - ./tests/test_skaldpress.sh + ./tests/test_skaldpress.sh && \ + ./tests/test_unittests.sh + diff --git a/src/skaldpress/macros.py b/src/skaldpress/macros.py new file mode 100644 index 0000000..33aac86 --- /dev/null +++ b/src/skaldpress/macros.py @@ -0,0 +1,47 @@ +from copy import deepcopy +from smp.builtins import ( + smp_builtin_read, +) + + +def sp_all_tagged_by( + macro_processor, tag: str, template: str, field=None, reversed="" +) -> str: + """ + SMP Macro for getting all files with specific tag, this is only _really_ effective the second run + + Usage in files: + all_tagged_by(<tag name>, <template> [, <field to sort by>] [, reversed]) + """ + tagged_files = [ + k + for k, v in macro_processor.py_global_env["macro_processor_state"].items() + if f"{macro_processor._get_macro_with_prefix('metadata_prefix')}tags" + in v["stored_data"] + ] + + if len(tagged_files) == 0: + macro_processor.log_warning(f"No tags for {tag}\u001b[0m") + return "" + + if field is not None: + tagged_files.sort( + key=lambda fname: macro_processor.py_global_env["macro_processor_state"][ + fname + ]["stored_data"][ + f"{macro_processor._get_macro_with_prefix('metadata_prefix')}{field}" + ], + reverse=(reversed != ""), + ) + + out = "" + for filename in tagged_files: + file = macro_processor.py_global_env["macro_processor_state"][filename] + smp_local = deepcopy(macro_processor) + from smp.builtins import smp_builtin_undefine + + smp_local.macros.update(file["stored_data"]) + smp_builtin_undefine(smp_local, "METADATA_template") + out += smp_builtin_read(smp_local, template, template_content=file["content"]) + macro_processor.warnings.extend(smp_local.warnings) + return out diff --git a/src/skaldpress/main.py b/src/skaldpress/main.py index cc6dc94..8fd823a 100644 --- a/src/skaldpress/main.py +++ b/src/skaldpress/main.py @@ -7,7 +7,6 @@ from functools import partial from itertools import chain from collections import deque import smp.macro_processor -from copy import deepcopy from skaldpress.filelist import ( make_filelist_iter, FileList, @@ -18,47 +17,10 @@ from smp.builtins import ( smp_builtin_read, smp_builtin_add_metadata, ) +import skaldpress.macros from time import perf_counter -def sp_all_tagged_by( - macro_processor, tag: str, template: str, field=None, reversed="" -) -> str: - """ - SMP Macro for getting all files with specific tag, this is only _really_ effective the second run - - Usage in files: - all_tagged_by(<tag name>, <template> [, <field to sort by>] [, reversed]) - """ - tagged_files = [ - k - for k, v in macro_processor.py_global_env["macro_processor_state"].items() - if "METADATA_tags" in v["stored_data"] - ] - - if len(tagged_files) == 0: - macro_processor.log_warning(f"No tags for {tag}\u001b[0m") - return "" - - # FRAGILE - if field is not None: - tagged_files.sort( - key=lambda fname: macro_processor.py_global_env["macro_processor_state"][ - fname - ]["stored_data"][f"METADATA_{field}"], - reverse=(reversed != ""), - ) - - out = "" - for filename in tagged_files: - file = macro_processor.py_global_env["macro_processor_state"][filename] - smp_local = deepcopy(macro_processor) - smp_local.macros.update(file["stored_data"]) - out += smp_builtin_read(smp_local, template, template_content=file["content"]) - print_warnings(smp_local) - return out - - class SkaldpressError(Exception): def __init__(self, code, error, path=None): self.code = code @@ -73,7 +35,9 @@ def print_warnings(macro_processor): def macro_processor_initialize(metadata, old_macro_processor, additional_state=None): macro_processor = old_macro_processor - macro_processor.define_macro("all_tagged_by", sp_all_tagged_by) + macro_processor._import_symbols( + skaldpress.macros, macro_processor.macros, function_prefix="sp_" + ) smp_builtin_add_metadata(macro_processor, metadata, overwrite=False) if additional_state: for key, value in additional_state.items(): @@ -86,12 +50,13 @@ def compile_file(smps: smp.macro_processor.MacroProcessorState, file_path, opts) raise SkaldpressError(3, None) stored_smp_state = None - if file_path in smps.global_state: - stored_smp_state = smps.global_state[file_path]["stored_data"] + # if file_path in smps.global_state: + # stored_smp_state = smps.global_state[file_path]["stored_data"] + # print(stored_smp_state) macro_processor = smps.macro_processor() - macro_processor.define_macro_string( - "METADATA_filename", + macro_processor._define_metadata( + "filename", os.path.splitext(os.path.relpath(file_path, opts.content_dir))[0], ) diff --git a/src/smp/__init__.py b/src/smp/__init__.py index d6e5d52..51dce90 100644 --- a/src/smp/__init__.py +++ b/src/smp/__init__.py @@ -48,7 +48,5 @@ def main(): macro_processor_state = smp.macro_processor.MacroProcessorState() macro_processor = macro_processor_state.macro_processor() res = macro_processor.process_input(file_content) - macro_processor.store("", "", "") - breakpoint() print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━", file=sys.stderr) print(res) diff --git a/src/smp/builtins.py b/src/smp/builtins.py index afa153d..2fde231 100644 --- a/src/smp/builtins.py +++ b/src/smp/builtins.py @@ -201,7 +201,7 @@ def smp_builtin_read(macro_processor, filename, template_content=None): metadata = {} if macro_processor._get_macro_with_prefix("parse_file_yaml"): metadata, file_content = extract_parse_yaml_metadata(file_content) - smp_builtin_add_metadata(macro_processor, metadata, overwrite=False) + smp_builtin_add_metadata(macro_processor, metadata, overwrite=True) extension = os.path.splitext(filename)[1][1:] or "" macro_processor._define_macro_with_prefix("target_file_extension", extension) @@ -215,13 +215,22 @@ def smp_builtin_read(macro_processor, filename, template_content=None): if extension == "md": content = smp_builtin_html_from_markdown(macro_processor, content) - if (template := macro_processor.macros.get("METADATA_template")) is not None: + if (template := macro_processor._get_metadata("template")) is not None: if template not in macro_processor._get_macro_with_prefix("template_stack"): return smp_builtin_read(macro_processor, template, content) return content +def smp_builtin_indent(macro_processor, indent: int, content: str): + indent = int(indent) + content = macro_processor.process_input(content) + return "\n".join( + f"{' ' * (indent if i > 0 else 0)}{x}" + for i, x in enumerate(content.split("\n")) + ) + + def smp_builtin_wodl(macro_processor, link, timeout_seconds=5): if (macro_processor._get_macro_with_prefix("wodl_cache")) is None: macro_processor._define_macro_with_prefix("wodl_cache", {}) diff --git a/src/smp/macro_processor.py b/src/smp/macro_processor.py index 2af26dd..a31f040 100644 --- a/src/smp/macro_processor.py +++ b/src/smp/macro_processor.py @@ -31,48 +31,20 @@ def macro_name_clean(macro_name: str) -> str: return macro_name -def seek(input: str, start: int, target: str) -> int | None: - """Seek for a value in a string, consider using startswith instead""" - from warnings import warn - - warn( - "seek should be considered replaced with str.startswith", - DeprecationWarning, - stacklevel=2, - ) - input_end = len(input) - target_end = len(target) - - if input_end < start + target_end: - return None - - i = 0 - while i < len(target): - if input[start + i] != target[i]: - return None - i += 1 - - return start + target_end - - class MacroProcessor: source_file_path: str - """All currently defined macros in this MacroProcessor""" macros: dict[str, Any] - """ All macro invocations that has happened """ - macro_invocations: list[tuple[str, list[str]]] - warnings: list[Any] - + """ Macros which are @directives """ + special_macros: dict[str, tuple[Any, Any]] """ Global environment for python execution """ - py_global_env: dict - + """ Local environment for python execution """ py_local_env_alt: dict py_local_env_current: dict - - special_macros: dict[str, tuple[Any, Any]] - + """ All macro invocations that has happened """ + macro_invocations: list[tuple[str, list[str]]] + warnings: list[Any] start_quote: str = '%"' end_quote: str = '"%' prefix: str = "" @@ -84,39 +56,24 @@ class MacroProcessor: self.py_global_env = dict() self.py_local_env_alt = dict() self.py_local_env_current = self.macros - self.indent_level = "" self.prefix = prefix self._define_builtins(self.macros) self._define_builtins(self.py_local_env_alt) + def _import_symbols(self, module, env, function_prefix=""): + for name, fun in inspect.getmembers(module, inspect.isfunction): + if name.startswith("_"): + continue + if not name.startswith(function_prefix): + continue + name = name.replace(function_prefix, "") + env[f"{self.prefix}{name}"] = fun + def _define_builtins(self, env): + self._import_symbols(smp.builtins, env, function_prefix="smp_builtin_") + env[f"{self.prefix}macro_processor"] = self - env[f"{self.prefix}define"] = smp.builtins.smp_builtin_define - env[f"{self.prefix}undefine"] = smp.builtins.smp_builtin_undefine - env[f"{self.prefix}define_array"] = smp.builtins.smp_builtin_define_array - env[f"{self.prefix}ifdef"] = smp.builtins.smp_builtin_ifdef - env[f"{self.prefix}ifndef"] = smp.builtins.smp_builtin_ifndef - env[f"{self.prefix}ifeq"] = smp.builtins.smp_builtin_ifeq - env[f"{self.prefix}ifneq"] = smp.builtins.smp_builtin_ifneq - env[f"{self.prefix}once"] = smp.builtins.smp_builtin_once - env[f"{self.prefix}include"] = smp.builtins.smp_builtin_include - env[f"{self.prefix}include_verbatim"] = ( - smp.builtins.smp_builtin_include_verbatim - ) - env[f"{self.prefix}shell"] = smp.builtins.smp_builtin_shell - env[f"{self.prefix}dumpenv"] = smp.builtins.smp_builtin_dumpenv - env[f"{self.prefix}eval"] = smp.builtins.smp_builtin_eval - env[f"{self.prefix}array_push"] = smp.builtins.smp_builtin_array_push - env[f"{self.prefix}array_each"] = smp.builtins.smp_builtin_array_each - env[f"{self.prefix}array_size"] = smp.builtins.smp_builtin_array_size - env[f"{self.prefix}explode"] = smp.builtins.smp_builtin_explode - env[f"{self.prefix}format_time"] = smp.builtins.smp_builtin_format_time - env[f"{self.prefix}html_from_markdown"] = ( - smp.builtins.smp_builtin_html_from_markdown - ) - env[f"{self.prefix}wodl"] = smp.builtins.smp_builtin_wodl - env[f"{self.prefix}template"] = smp.builtins.smp_builtin_template env[f"{self.prefix}template_stack"] = [] # If true, include-macros will parse yaml in beginning of content @@ -138,26 +95,64 @@ class MacroProcessor: def _get_macro_with_prefix(self, macro_name, sub_prefix: str = "", default=None): return self.macros.get(f"{self.prefix}{sub_prefix}{macro_name}", default) + def _get_metadata(self, macro_name, default=None): + return self._get_macro_with_prefix( + macro_name, + sub_prefix=self._get_macro_with_prefix("metadata_prefix"), + default=default, + ) + + def _define_metadata(self, macro_name, macro_value): + return self._define_macro_with_prefix( + macro_name, + macro_value, + sub_prefix=self._get_macro_with_prefix("metadata_prefix"), + ) + def log_warning(self, message): """ Here we should add some more information, line number, file etc, when that is available """ self.warnings.append(message) + def _expand_callable_macro(self, macro: Any, args: list[str] = list()) -> str: + signature = inspect.signature(macro) + macro_args: list[Any] = [] + if "macro_processor" in signature.parameters or "smp" in signature.parameters: + macro_args.append(self) + macro_args.extend(args) + # This should maybe be processed as well(?) + return str(macro(*macro_args)) + + def _expand_string_macro(self, macro: Any, args: list[str] = list()) -> str: + expanded = macro + for i, arg in enumerate(args): + placeholder = f"${i}" + expanded = macro.replace(placeholder, arg) + return self.process_input(expanded) + + def _expand_unknown_macro( + self, macro_name: str, args: list[str] = list(), process_args: bool = True + ) -> str: + if len(args) == 0: + return macro_name + out = f"{macro_name}(" + for i, arg in enumerate(args): + if process_args: + out += self.process_input(arg) + else: + out += arg + if i < (len(args) - 1): + out += "," + out += ")" + return out + def expand_macro(self, macro_name: str, args: list[str] = list()) -> str: # Ignore trailing underscore in macro name, the parser will pop a space in front if # present, but we should ignore it for finding the macro. macro_name = macro_name_clean(macro_name) if macro_name not in self.macros: - if len(args) == 0: - return macro_name - out = f"{macro_name}(" - for i, arg in enumerate(args): - out += self.process_input(arg) - if i < (len(args) - 1): - out += "," - out += ")" - return out + return self._expand_unknown_macro(macro_name, args, process_args=True) # Strip leading whitespace from arguments for arg in args: @@ -171,28 +166,18 @@ class MacroProcessor: macro = self.macros.get(macro_name) if callable(macro): - signature = inspect.signature(macro) - macro_args: list[Any] = [] - if ( - "macro_processor" in signature.parameters - or "smp" in signature.parameters - ): - macro_args.append(self) - macro_args.extend(args) try: - return str(macro(*macro_args)) + return self._expand_callable_macro(macro, args) except Exception as e: - s = f"{macro_name}({','.join([repr(x) for x in args])})" + s = self._expand_unknown_macro(macro_name, args, process_args=False) self.log_warning( f"Error expanding macro {s} ({e})\n{traceback.format_exc()}" ) return s + if isinstance(macro, str): - expanded = macro - for i, arg in enumerate(args): - placeholder = f"${i}" - expanded = macro.replace(placeholder, arg) - return self.process_input(expanded) + return self._expand_string_macro(macro, args) + return f"{repr(macro)}" def process_input(self, input: str): @@ -366,21 +351,23 @@ class MacroProcessor: return output def store(self, **xargs): - requested_keys = self.macros.get("METADATA_keep_states", self.macros.keys()) + requested_keys = self._get_metadata("keep_states", []) for key in self.macros.keys(): - if key.startswith("METADATA_") and key not in requested_keys: + if ( + key.startswith(self._get_macro_with_prefix("metadata_prefix")) + and key not in requested_keys + ): requested_keys.append(key) if isinstance(requested_keys, str): requested_keys = [str(requested_keys)] - needs_recompilation = ("METADATA_keep_states" in self.macros) or ( - "all_tagged_by" in [x[0] for x in self.macro_invocations] - ) + needs_recompilation = ( + f"{self._get_macro_with_prefix('metadata_prefix')}keep_states" + in self.macros + ) or ("all_tagged_by" in [x[0] for x in self.macro_invocations]) - target_filename = self._get_macro_with_prefix( - "target_filename", sub_prefix="METADATA_" - ) + target_filename = self._get_metadata("target_filename") self.py_global_env["macro_processor_state"][self.source_file_path] = dict( { |