diff options
| -rw-r--r-- | src/skaldpress/macros.py | 4 | ||||
| -rw-r--r-- | src/skaldpress/main.py | 4 | ||||
| -rw-r--r-- | src/smp/__init__.py | 11 | ||||
| -rw-r--r-- | src/smp/builtins.py | 59 | ||||
| -rw-r--r-- | src/smp/macro_processor.py | 154 | ||||
| -rw-r--r-- | tests/skaldpress/content/article.md | 1 | ||||
| -rw-r--r-- | tests/skaldpress/templates/article.html | 6 | ||||
| -rw-r--r-- | tests/skaldpress/templates/base.html | 3 | ||||
| -rw-r--r-- | tests/smp/include_1 | 2 | ||||
| -rw-r--r-- | tests/smp/include_2 | 2 |
10 files changed, 200 insertions, 46 deletions
diff --git a/src/skaldpress/macros.py b/src/skaldpress/macros.py index 33aac86..9d06505 100644 --- a/src/skaldpress/macros.py +++ b/src/skaldpress/macros.py @@ -1,6 +1,6 @@ from copy import deepcopy from smp.builtins import ( - smp_builtin_read, + _smp_builtin_read, ) @@ -42,6 +42,6 @@ def sp_all_tagged_by( smp_local.macros.update(file["stored_data"]) smp_builtin_undefine(smp_local, "METADATA_template") - out += smp_builtin_read(smp_local, template, template_content=file["content"]) + out += _smp_builtin_read(smp_local, template, template_content=file["content"]) macro_processor.warnings.extend(smp_local.warnings) return out diff --git a/src/skaldpress/main.py b/src/skaldpress/main.py index 8fd823a..b0b1784 100644 --- a/src/skaldpress/main.py +++ b/src/skaldpress/main.py @@ -14,7 +14,7 @@ from skaldpress.filelist import ( file_filtered, ) from smp.builtins import ( - smp_builtin_read, + _smp_builtin_read, smp_builtin_add_metadata, ) import skaldpress.macros @@ -63,7 +63,7 @@ def compile_file(smps: smp.macro_processor.MacroProcessorState, file_path, opts) macro_processor.source_file_path = file_path macro_processor_initialize(opts.metadata, macro_processor, stored_smp_state) - content = smp_builtin_read(macro_processor, file_path) + content = _smp_builtin_read(macro_processor, file_path) print_warnings(macro_processor) return macro_processor.store(content=content) diff --git a/src/smp/__init__.py b/src/smp/__init__.py index 51dce90..898e683 100644 --- a/src/smp/__init__.py +++ b/src/smp/__init__.py @@ -22,7 +22,8 @@ def read_stdin(): data = sys.stdin.read() macro_processor_state = smp.macro_processor.MacroProcessorState() macro_processor = macro_processor_state.macro_processor() - res = macro_processor.process_input(data) + macro_processor._enter_file_frame("[stdin]", 0, None) + res = macro_processor.process_input(data, file="[stdin]") print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━", file=sys.stderr) print(res) @@ -42,11 +43,11 @@ def main(): repl() sys.exit(0) - with open(sys.argv[1], "r") as f: - file_content = f.read() - macro_processor_state = smp.macro_processor.MacroProcessorState() macro_processor = macro_processor_state.macro_processor() - res = macro_processor.process_input(file_content) + res = smp.builtins._smp_builtin_read(macro_processor, sys.argv[1]) print("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━", file=sys.stderr) print(res) + + for warning in macro_processor.warnings: + print(f"\u001b[33m{warning}\u001b[0m", file=sys.stderr) diff --git a/src/smp/builtins.py b/src/smp/builtins.py index b44cc23..a9e1977 100644 --- a/src/smp/builtins.py +++ b/src/smp/builtins.py @@ -92,8 +92,8 @@ def smp_builtin_add_metadata(macro_processor, metadata: dict[str, Any], overwrit macro_processor.define_macro(macro_name, macro_value) -def smp_builtin_include(macro_processor, filename): - return smp_builtin_read(macro_processor, filename, template_content=None) +def smp_builtin_include_file(macro_processor, filename): + return _smp_builtin_read(macro_processor, filename, template_content=None) def smp_builtin_parse_leading_yaml(macro_processor, content): @@ -171,9 +171,7 @@ def smp_builtin_format_time(macro_processor, format, time): def smp_builtin_html_from_markdown(macro_processor, text, extensions=list()): - # Get rid of quoting, I don't remember why, but the rust implementation does it like this. - for _ in range(2): - text = macro_processor.process_input(text) + text = macro_processor.process_input(text) extensions.append(TableExtension()) extensions.append(FencedCodeExtension()) extensions.append(AutolinkExtension()) @@ -181,24 +179,30 @@ def smp_builtin_html_from_markdown(macro_processor, text, extensions=list()): return markdown.markdown(text, extensions=extensions) -def _smp_builtin_template_content(content): +def _smp_builtin_template_content(): def inner(macro_processor): - """ - This should do some kind of stack thing, so we can track which file we are processing. - entering the CONTENT is fine, the question is how to handle exiting it. + filename, content, extension = macro_processor._get_macro_with_prefix( + "template_stack_content" + )[0] + macro_processor._enter_file_frame(f"[part]{filename}", 0, "") + res = macro_processor.process_input(content, file=filename) - could have a "once" macro or something, that is added to the end of the content. - """ - return content + if extension == "md": + res = smp_builtin_html_from_markdown(macro_processor, res) + + macro_processor._pop_file_frame() + macro_processor._get_macro_with_prefix("template_stack_content").pop(0) + return res return inner def smp_builtin_template(macro_processor, template, content): - return smp_builtin_read(macro_processor, template, template_content=content) + return _smp_builtin_read(macro_processor, template, template_content=content) -def smp_builtin_read(macro_processor, filename, template_content=None): +def _smp_builtin_read(macro_processor, filename, template_content=None): + macro_processor._enter_file_frame(filename, 0, template_content) with open(filename, "r") as f: file_content = f.read() @@ -214,14 +218,27 @@ def smp_builtin_read(macro_processor, filename, template_content=None): macro_processor._get_macro_with_prefix("template_stack").append(filename) macro_processor.macros["CONTENT"] = template_content - content = macro_processor.process_input(file_content) - - if extension == "md": - content = smp_builtin_html_from_markdown(macro_processor, content) + # content = macro_processor.process_input(file_content, file=filename) + content = file_content if (template := macro_processor._get_metadata("template")) is not None: + + template_prefix = macro_processor._get_macro_with_prefix("template_prefix") + if not os.path.exists(template): + template = os.path.join(template_prefix, template) + if template not in macro_processor._get_macro_with_prefix("template_stack"): - return smp_builtin_read(macro_processor, template, content) + macro_processor._get_macro_with_prefix("template_stack_content").append( + (filename, content, extension) + ) + return _smp_builtin_read( + macro_processor, template, _smp_builtin_template_content() + ) + + content = macro_processor.process_input(file_content, file=filename) + + if extension == "md": + content = smp_builtin_html_from_markdown(macro_processor, content) return content @@ -258,12 +275,12 @@ def smp_builtin_wodl(macro_processor, link, timeout_seconds=5): cache[link] = (working_link, r.status, r.reason) if not working_link: macro_processor.log_warning(f"Dead link {link} ({r.status} {r.reason})!") - except urllib.error.URLError as e: + except Exception as e: macro_processor.log_warning(f"Dead link {link} ({e})!") return "" -def smp_builtin_once(macro_processor, content): +def smp_builtin_expand_once(macro_processor, content): if (cache := macro_processor._get_macro_with_prefix("once_cache")) is not None: if (exp := cache.get(content)) is not None: return exp diff --git a/src/smp/macro_processor.py b/src/smp/macro_processor.py index a31f040..77641e7 100644 --- a/src/smp/macro_processor.py +++ b/src/smp/macro_processor.py @@ -1,6 +1,7 @@ import smp.builtins import traceback import inspect +import sys from typing import Any from enum import Enum @@ -19,6 +20,25 @@ class ParserState(Enum): DNL = 8 +def debug_hi(input, linestart, pos, color="\u001b[7m"): + return debug_hi_range(input, linestart, pos, pos + 1, color=color) + + +def debug_hi_range(input, linestart, start, end, color="\u001b[7m"): + lineend = input.find("\n", linestart) + line = input[linestart:] if lineend == -1 else input[linestart:lineend] + + out = "" + for j, _c in enumerate(line): + c_pos = linestart + j + if (c_pos >= start) and (c_pos < end): + out += f"{color}{_c}\u001b[0m" + else: + out += f"{_c}" + + return out + + def macro_is_whitespace_deleting(s: str) -> bool: if len(s) == 0: return False @@ -49,6 +69,11 @@ class MacroProcessor: end_quote: str = '"%' prefix: str = "" + file = None + + expansion_stack: list[Any] + file_stack: list[Any] + def __init__(self, prefix=""): self.macros = dict() self.macro_invocations = list() @@ -58,6 +83,11 @@ class MacroProcessor: self.py_local_env_current = self.macros self.prefix = prefix + self.expansion_stack = ( + list() + ) # This probably needs some magic at some point later + self.file_stack = list() # This probably needs some magic at some point later + self._define_builtins(self.macros) self._define_builtins(self.py_local_env_alt) @@ -75,6 +105,8 @@ class MacroProcessor: env[f"{self.prefix}macro_processor"] = self env[f"{self.prefix}template_stack"] = [] + env[f"{self.prefix}template_stack_content"] = [] + env[f"{self.prefix}template_prefix"] = "templates/" # If true, include-macros will parse yaml in beginning of content env[f"{self.prefix}parse_file_yaml"] = True @@ -82,6 +114,7 @@ class MacroProcessor: # meaning they will skip steps that are slow. env[f"{self.prefix}draft"] = False env[f"{self.prefix}metadata_prefix"] = "METADATA_" + env[f"{self.prefix}smp_debug"] = "1" def define_macro_string(self, macro_name, macro_value): self.define_macro(macro_name, str(macro_value)) @@ -122,14 +155,17 @@ class MacroProcessor: macro_args.append(self) macro_args.extend(args) # This should maybe be processed as well(?) - return str(macro(*macro_args)) + res = str(macro(*macro_args)) + return res def _expand_string_macro(self, macro: Any, args: list[str] = list()) -> str: expanded = macro for i, arg in enumerate(args): placeholder = f"${i}" expanded = macro.replace(placeholder, arg) - return self.process_input(expanded) + + res = self.process_input(expanded) + return res def _expand_unknown_macro( self, macro_name: str, args: list[str] = list(), process_args: bool = True @@ -170,9 +206,17 @@ class MacroProcessor: return self._expand_callable_macro(macro, args) except Exception as e: s = self._expand_unknown_macro(macro_name, args, process_args=False) - self.log_warning( - f"Error expanding macro {s} ({e})\n{traceback.format_exc()}" - ) + + e2 = "" + if self._debug_on(1): + e2 += "\n" + e2 += self._stack() + + if self._debug_on(2): + e2 += "\n- Python stack ---\n" + e2 += traceback.format_exc() + + self.log_warning(f"Error expanding macro {s} ({e}){e2}") return s if isinstance(macro, str): @@ -180,7 +224,42 @@ class MacroProcessor: return f"{repr(macro)}" - def process_input(self, input: str): + def _enter_frame(self, frame_type, filename, position, content): + self.expansion_stack.append((frame_type, filename, position, "")) + + def _pop_frame(self): + self.expansion_stack.pop() + + def _enter_file_frame(self, filename, linenr, template): + self.file_stack.append([filename, linenr, 0]) + + def _pop_file_frame(self): + self.file_stack.pop() + + def _stack(self): + out = "- File stack ------\n" + for i, frame in enumerate(self.file_stack): + out += f" {' ' * i} {frame[0]}:{frame[1]}\n" + out += "\n- Macro stack -----\n" + for i, frame in enumerate(self.expansion_stack): + out += f" {' ' * i} [{frame[0]}] in file {frame[1]}:{frame[2]}\n" + return out + + def _debug_on(self, level): + try: + if level > int(self._get_macro_with_prefix("smp_debug")): + return False + except: + return False + + return True + + def _debug(self, level, message): + if self._debug_on(level): + return + print(message, file=sys.stderr) + + def process_input(self, input: str, file: str | None = None): """ I also want to add special syntax for "special blocks", I am thinking of two main options, either some macro_names are intercepted, _or_ a special kind of macro can exist like @@ -198,8 +277,13 @@ class MacroProcessor: by pushing the expanded macros back to the input string, this may be more confusing, but may also be faster (stream or mutable string) """ + if file is not None: + self.file = file + if (file is None) and (self.file is not None): + file = self.file output = "" state = ParserState.NORMAL + state_start = 0 macro_name = "" macro_args = [] argument = "" @@ -210,6 +294,8 @@ class MacroProcessor: # We should keep track of filename, linenumber, and character number on line here # So we can give sensible error messages # Probably add to python stack trace? + linestart = 0 + linenr = 0 quote_level = 0 parens_level = 0 @@ -218,8 +304,33 @@ class MacroProcessor: while i < len(input): c = input[i] peek = None if i + 1 >= len(input) else input[i + 1] - # import sys - # print(f"[{i:4}] {repr(c):4} -> {repr(peek):4} [{state}] = {repr(output)}", file=sys.stderr) + + def hi_range(): + if not self._debug_on(5): + return + _linestart = linestart + _linenr = linenr + if state_start < linestart: + _linestart = state_start + _linenr = f"~{linenr-1}" + range = debug_hi_range( + input, _linestart, state_start, i, color="\u001b[42m" + ) + print( + f"[{file}:{_linenr}] {range} \t\u001b[33m({state})\u001b[0m", + file=sys.stderr, + ) + + if c == "\n": + linenr += 1 + linestart = i + 1 + self.file_stack[-1][1] = linenr + + if self._debug_on(5): + print( + f"[{file}:{linenr}] {debug_hi(input, linestart, i)} \t\u001b[33m({state})\u001b[0m", + file=sys.stderr, + ) if state == ParserState.DNL: if c == "\n": @@ -233,14 +344,17 @@ class MacroProcessor: if c == "%" and peek == "(": state = ParserState.IN_CODE i += 2 + state_start = i continue if c == "%" and peek == '"': state = ParserState.IN_QUOTES quote_level += 1 i += 1 + state_start = i elif c.isalnum(): state = ParserState.IN_MACRO + state_start = i macro_name += c else: output += c @@ -254,37 +368,50 @@ class MacroProcessor: quote_level -= 1 if quote_level == 0: state = ParserState.NORMAL + state_start = i else: output += '"%' i += 1 else: output += c + elif state == ParserState.IN_MACRO: if c.isalnum() or c == "_": macro_name += c elif c == "(": + hi_range() parens_level += 1 state = ParserState.IN_MACRO_ARGS + state_start = i else: + hi_range() if macro_is_whitespace_deleting(macro_name): if output[-1] == " ": output = output[:-1] macro_name = macro_name_clean(macro_name) + self._enter_frame(macro_name, file, linenr, input) + if macro_name == "SNNL": skip_next_line_ending = c != "\n" elif macro_name == "DNL": if c != "\n": state = ParserState.DNL + state_start = i macro_name = "" i += 1 + self._pop_frame() continue else: expanded = self.expand_macro(macro_name) output += expanded output += c macro_name = "" + + self._pop_frame() + state = ParserState.NORMAL + state_start = i elif state == ParserState.IN_MACRO_ARGS: if c == "%" and peek == '"': quote_level += 1 @@ -302,20 +429,26 @@ class MacroProcessor: continue if (c == ")") and (parens_level == 1): + hi_range() if macro_is_whitespace_deleting(macro_name): if output[-1] == " ": output = output[:-1] macro_name = macro_name_clean(macro_name) parens_level = 0 macro_args.append(argument.strip()) + self._enter_frame(macro_name, file, linenr, input) expanded = self.expand_macro(macro_name, macro_args) output += expanded state = ParserState.NORMAL + state_start = i macro_name = "" macro_args = [] argument = "" + self._pop_frame() elif (c == ",") and (parens_level == 1): macro_args.append(argument.strip()) + state_start = i + hi_range() argument = "" else: if c == "(": @@ -326,6 +459,7 @@ class MacroProcessor: elif state == ParserState.IN_CODE: if c == ")" and peek == "%": try: + self._enter_frame("inline_code", file, linenr, input) f = StringIO() with redirect_stdout(f): exec(py_expr, self.py_global_env, self.py_local_env_current) @@ -334,9 +468,12 @@ class MacroProcessor: output += s except Exception: traceback.print_exc() + finally: + self._pop_frame() py_expr = "" state = ParserState.NORMAL i += 1 + state_start = i else: py_expr += c i += 1 @@ -347,6 +484,7 @@ class MacroProcessor: if len(output) > 0 and output[-1] == " ": output = output[:-1] macro_name = macro_name_clean(macro_name) + hi_range() output += self.expand_macro(macro_name) return output diff --git a/tests/skaldpress/content/article.md b/tests/skaldpress/content/article.md index b20468c..7d99e15 100644 --- a/tests/skaldpress/content/article.md +++ b/tests/skaldpress/content/article.md @@ -9,3 +9,4 @@ tags: --- This is a example article + diff --git a/tests/skaldpress/templates/article.html b/tests/skaldpress/templates/article.html index cb3b58b..c47a114 100644 --- a/tests/skaldpress/templates/article.html +++ b/tests/skaldpress/templates/article.html @@ -1,10 +1,7 @@ --- template: base.html table_of_contents: true -keep_states: - - TOC_ITEMS --- -DNL include(templates/common_macros.smp)DNL <header> <h1>METADATA_title</h1> </header> @@ -17,6 +14,5 @@ DNL include(templates/common_macros.smp)DNL ) </div> - CONTENT + indent(4, CONTENT) </main> - diff --git a/tests/skaldpress/templates/base.html b/tests/skaldpress/templates/base.html index aa73ffc..b66a00b 100644 --- a/tests/skaldpress/templates/base.html +++ b/tests/skaldpress/templates/base.html @@ -5,10 +5,11 @@ <a href="/">Home</a> </nav> - CONTENT + indent(8, CONTENT) <footer> </footer> </body> </html> +dumpenv diff --git a/tests/smp/include_1 b/tests/smp/include_1 index 1621de2..9664507 100644 --- a/tests/smp/include_1 +++ b/tests/smp/include_1 @@ -1,3 +1,3 @@ -include(tests/example_include.smp) +include_file(tests/example_include.smp) --- diff --git a/tests/smp/include_2 b/tests/smp/include_2 index 2714eff..b69fe13 100644 --- a/tests/smp/include_2 +++ b/tests/smp/include_2 @@ -1,4 +1,4 @@ -include(tests/example_include.smp)SNNL +include_file(tests/example_include.smp)SNNL ifdef(SMP, SMP_ISDEF, SMP_ISNDEF) --- SMP_ISDEF |