diff options
| -rw-r--r-- | debian/control | 2 | ||||
| -rw-r--r-- | pyproject.toml | 2 | ||||
| -rw-r--r-- | skaldpress.1 | 45 | ||||
| -rw-r--r-- | smp.1 | 157 | ||||
| -rw-r--r-- | src/skaldpress/macros.py | 10 | ||||
| -rw-r--r-- | src/skaldpress/main.py | 12 | ||||
| -rw-r--r-- | src/smp/__init__.py | 9 | ||||
| -rw-r--r-- | src/smp/builtins.py | 20 | ||||
| -rw-r--r-- | src/smp/macro_processor.py | 188 | ||||
| -rwxr-xr-x | tests/test_macro_processor.sh | 2 |
10 files changed, 294 insertions, 153 deletions
diff --git a/debian/control b/debian/control index d7d8c3e..a2566be 100644 --- a/debian/control +++ b/debian/control @@ -8,5 +8,5 @@ Rules-Requires-Root: no Package: skaldpress Architecture: any -Depends: python3, python3-markdown-it +Depends: python3, python3-markdown-it python3-mdit-py-plugins Description: Skaldpress diff --git a/pyproject.toml b/pyproject.toml index 88737b4..6a1ccfb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3 :: Only", ] -dependencies = ["markdown-it-py"] +dependencies = ["markdown-it-py", "mdit-py-plugins"] [project.optional-dependencies] dev = ["check-manifest", "black", "pyflakes", "mypy", "types-Markdown", "pyinstrument"] diff --git a/skaldpress.1 b/skaldpress.1 index a0cf224..a9b7355 100644 --- a/skaldpress.1 +++ b/skaldpress.1 @@ -6,7 +6,8 @@ Skaldpress \- Templating engine .SH DESCRIPTION .B smp -smp is a macro processor, made specifically for a usecase when doing templating of websites. +is a macro processor, made specifically for a usecase when doing templating of websites. +But it is useful for many other things as well. Macros which are available using skaldpress, in addition to the builtin smp(1)-macros @@ -16,34 +17,6 @@ Will output all documents which had the specified tag, using the specified templ If a field to sort by is specified, it will output ascending based on that field, or reversed if a fourth argument \fBreversed\fR is specified. -.PP -All input files can have a metadata-block at the beginning. -Any keys will be defined as macros as \fBMETADATA_<key>\fR. -There are a few special keys which has a specific effect: - -.IP "\fBtags\fR" -This is a list, and all files with entries here will be registered during the first compilation round. -During the second compilation, macros like \fBall_tagged_by\fR will then -make all files with the relevant content available and compile it in. - -.IP "\fBtarget_filename\fR" -By default, a file will have the same name in the \fIoutput\fR-directory (unless a template has a different extension). -This overrides the output-filename, but it keeps the file in the same directory, and it will not affect the extension. - -.IP "\fBtemplate\fR" -If specified, the file will be compiled using a template. -In the template, all \fBMETADATA_<key>\fR macros will be available, -additionally templates can use the \fBCONTENT\fR-macro to get the expanded content of the file. -This is recursive, meaning that templates can use other templates, with their own metadata-block, -since it is recursive, any metadata in templates will overwrite any metadata from their children, -but will keep metadata that is not overwritten. -This means templates can add additional context. - -.IP "\fBkeep_states\fR" -List or string where every listed state/variable will be kept for subsequenct compilations. -Meaning that you can e.g. construct a array in the first compilation, which is then used the second time around. -Setting this, means that the file will always be recompiled, regardeless of other instances (unless a \fB--filter\fR is set). - .SH OPTIONS .IP "\fB-o, --out, --output\fR \fIpath\fR Specifies the directory to output the compiled files to, defaults to \fIbuild\fR. @@ -65,10 +38,24 @@ will be compiled. Comma-separated list of files to exclude, if not specified, all files in \fB--source\fR will be compiled. +.IP "\fB-x, --xclude\fR \fIfilter\fR +Comma-separated list of static files to exclude, if not specified, all files in \fB--static\fR +will be compiled. + .IP "\fB-m, --metadata\fR \fIkey\fR=\fIvalue\fR Metadata to add to compiled file, this can e.g. set a template for all files if you don't want to manually add YAML-blocks. Specify multiple times, to set multiple fields. +.IP "\fB-D, --define\fR \fIkey\fR=\fIvalue\fR +Define macros, these will be interpreted as strings. + +.IP "\fB-P, --prefix-builtins\fR +Prefix all builtin macros with `smp_` + +.IP "\fB-U, --unsafe\fR +If specified, arbitrary code execution will be allowed. +It is most likely possible anyway. + .SH EXAMPLES To run skaldpress on a simple project, simply call it with no arguments @@ -2,7 +2,7 @@ .SH name Skaldpress Macro Processor \- Macro processor .SH SYNOPSIS -.B smp [\fIinput_file\fB] +.B smp [OPTION]... [\fIinput_file\fB] .SH DESCRIPTION .B smp @@ -22,6 +22,9 @@ This defines a macro, optionally with some content. The optional content will be expanded immediately, and later the already processed content will be included in the output. +.IP "\fBundefine(<macro_name>)\fR" +Delete the macro. + .IP "\fBifdef(<macro_name>, <output if defined> [, <output if not defined>])\fR" .IP "\fBifndef(<macro_name>, <output if not defined> [, <output if defined>])\fR" @@ -33,6 +36,9 @@ and later the already processed content will be included in the output. .IP "\fBshell(<command>)\fR" Runs command on shell, and includes the command output in the output +.IP "\fBeval(<python>)\fR" +Run a python command, write return value to output. + .IP "\fBexpr(<arg1>, <arg2>, ..., <argN>)\fR" Shorthand for running the expr command, expands all arguments, and executes it on the shell. @@ -63,43 +69,134 @@ It will process it's input twice before actually doing the conversion. This is t So you should probably always quote arguments to this. .IP "\fBwodl(<url>)\fR" -If compiled with \fBdeadlinks\fR, this macro will always return it's argument. +This macro will always return it's argument. But it will also perform a request to the link, and emit a warning if it it doesn't return HTTP 200 OK. This will significantly slow down compile times! -.SS "\fBCompile-Time Feature Flags\fR" -There are some compile-time flags that can enable/disable features, most are enabled by default, -and exist either because they trigger a external dependency, or because they are unlikely to work without glibc. - -.IP "\fBreadline\fR" 2 -\fBEnabled by default\fR. -Enables the use of libc readline in REPL mode. If not enabled, normal stdio will be used for input. - -Requires readline to be installed on the system. - -.IP "\fBtime\fR" 2 -\fBEnabled by default\fR. -Enables the \fIformat_time\fR-macro, this will compile the chrono dependency to format time. +.IP "\fBtemplate(<filename>, <content>)\fR" +Will read the file in \fIfilename\fR, and process it replacing \fBCONTENT\fR with the \fIcontent\fR. + +.IP "\fBiftruthy(<content>)\fR" +Returns python True if content is evaluated to \fItruthy\fR. + +.IP "\fBindent(<spaces>, <content>)\fR" +Indents each line of the \fIcontent\fR with \fIspaces\R spaces. + +.IP "\fBdumpenv\fR" +Debugging macro for printing smp internals. + +.SS "\fBSpecial macros\fR" +There is a syntax for special macros, which takes more data in once. + +.IP "\fB@template \fI<template>\fR" +Passes the rest of the file as content to the \fBtemplate\fR-macro. + +.IP "\fB@html_from_markdown\fR" +Passes the rest of the file as content to the \fBhtml_from_markdown\fR-macro. + +.SS "\fBStrings\fR" +This macro processor is generally recursive, meaning all macros will process it's arguments normally before using them. +To make this easier to deal with, strings are differentiated between opening and closing. + +Opening is \fB%"\fR, closing is \fB"%\fR. + +.SS "\fBCode\fR" +You can run arbitrary (python) code in the files, which operates in the same environment as the macros. +This means you can from the python code call all builtin macros directly (you will need to manually provide the macro_processor reference). +It also means all definitions you make, are available outside as macro tokens. +Code blocks are opened with \fB%(\fR and closed with \fB)%\fR. + + + +.PP +.nf +.RS +%( +def A(): + return "test" +)% +A +%"A"% +.RE +.fi +.PP +Returns +.PP +.nf +.RS +test +A +.RE +.fi +.PP + +.SS "\fBBuiltin macro options\fR" +There are some macros flags that can enable/disable features, +They will be prefixed by `smp_` if \fB-P\fR is set. + +.IP "\fBparse_file_yaml\fR" 2 +\fB\fItruthy\fR\fB by default\fR. +If set to a truthy value, metadata blocks in the start of files will be parsed, otherwise it will be treated as normal input. + +.IP "\fBdraft\fR" 2 +\fB\fIfalsy\fR\fB by default\fR. +If set to a truthy value, some macros may skip expensive steps to save compilation time (e.g. network calls). + +.IP "\fBtemplate_prefix\fR" 2 +\fB\fItemplates/\fR\fB by default\fR. +The path to search for templates in, this affects metadata wrapping, as well as macros using the template macro. + +.IP "\fBmetadata_prefix\fR" 2 +\fB\fIMETADATA_\fR\fB by default\fR. +The prefix used for defining metadata for macros defined in blocks. + +.IP "\fBsmp_debug\fR" 2 +\fB\fI4\fR\fB by default\fR. +Sets the debug level for the parser, higher number means more verbosity. +Level 1 and 2 is the most basic error messaging, 5 and up will print parser internals. + +.IP "\fBsynclines\fR" 2 +\fB\fIfalsy\fR\fB by default\fR. +If truthy, synclines will be included in output, along with filename. + +.SS Metadata block +All input files can have a metadata-block at the beginning. +Any keys will be defined as macros as \fBMETADATA_<key>\fR. +There are a few special keys which has a specific effect: + +.IP "\fBtarget_filename\fR" +By default, a file will have the same name in the \fIoutput\fR-directory (unless a template has a different extension). +This overrides the output-filename, but it keeps the file in the same directory, and it will not affect the extension. + +.IP "\fBtemplate\fR" +If specified, the file will be compiled using a template. +In the template, all \fBMETADATA_<key>\fR macros will be available, +additionally templates can use the \fBCONTENT\fR-macro to get the expanded content of the file. +This is recursive, meaning that templates can use other templates, with their own metadata-block, +since it is recursive, any metadata in templates will overwrite any metadata from their children, +but will keep metadata that is not overwritten. +This means templates can add additional context. + +.IP "\fBkeep_states\fR" +List or string where every listed state/variable will be kept for subsequenct compilations. +Meaning that you can e.g. construct a array in the first compilation, which is then used the second time around. +Setting this, means that the file will always be recompiled, regardeless of other instances (unless a \fB--filter\fR is set). -.IP "\fBmarkdown\fR" 2 -\fBEnabled by default\fR. -Enabled the \fIhtml_from_markdown\fR-macro, this will compile the markdown dependency. And can convert markdown into html. +.SH OPTIONS -.IP "\fBdeadlinks\fR" 2 -\fBEnabled by default\fR. -Enabled the \fIwodl\fR-macro, this will compile the minreq dependency. -Allows testing the return code of links, and warns when the request is unsuccessful. +.IP "\fBinput_file\fR +If a input file is provided, that will be processed. +If it is `-`, or not specified and something is piped, it will process stdin. -.IP "\fBguile\fR" 2 -If compiled with guile, the macro processor supports running guile-code inline. -This is enabled with \fI%()%\fR in the input, and anything inside the parantheses will be evaluated as guile. +.IP "\fB-D, --define\fR \fIkey\fR=\fIvalue\fR +Define macros, these will be interpreted as strings. -Requires libguile-3.0 to be installed on the system. +.IP "\fB-P, --prefix-builtins\fR +Prefix all builtin macros with `smp_` -\fBExperimental feature!\fR -.SH OPTIONS -If a input file is provided, that will be processed. -If not, a sort of interactive REPL will be started instead. +.IP "\fB-U, --unsafe\fR +If specified, arbitrary code execution will be allowed. +It is most likely possible anyway. .SH SEE ALSO skaldpress(1) diff --git a/src/skaldpress/macros.py b/src/skaldpress/macros.py index c341695..885f803 100644 --- a/src/skaldpress/macros.py +++ b/src/skaldpress/macros.py @@ -15,7 +15,7 @@ def sp_all_tagged_by( """ tagged_files = [ k - for k, v in macro_processor.py_global_env["macro_processor_state"].items() + for k, v in macro_processor._get_macro_builtin("macro_processor_state").items() if f"{macro_processor._get_macro_builtin('metadata_prefix')}tags" in v["stored_data"] and ( @@ -32,9 +32,9 @@ def sp_all_tagged_by( if field is not None: tagged_files.sort( - key=lambda fname: macro_processor.py_global_env["macro_processor_state"][ - fname - ]["stored_data"][ + key=lambda fname: macro_processor._get_macro_builtin( + "macro_processor_state" + )[fname]["stored_data"][ f"{macro_processor._get_macro_builtin('metadata_prefix')}{field}" ], reverse=(reversed != ""), @@ -42,7 +42,7 @@ def sp_all_tagged_by( out = "" for filename in tagged_files: - file = macro_processor.py_global_env["macro_processor_state"][filename] + file = macro_processor._get_macro_builtin("macro_processor_state")[filename] smp_local = deepcopy(macro_processor) from smp.builtins import smp_builtin_undefine diff --git a/src/skaldpress/main.py b/src/skaldpress/main.py index 9efb919..62f5d9c 100644 --- a/src/skaldpress/main.py +++ b/src/skaldpress/main.py @@ -62,6 +62,8 @@ def compile_file(smps: smp.macro_processor.MacroProcessorState, file_path, opts) "filename", os.path.splitext(os.path.relpath(file_path, opts.content_dir))[0], ) + if opts.unsafe: + macro_processor.unsafe_code_execution = True macro_processor.source_file_path = file_path macro_processor_initialize(opts.metadata, macro_processor, stored_smp_state) @@ -196,9 +198,6 @@ def main(): "-m", "--metadata", nargs="+", metavar="key=value", default=[], action="extend" ) parser.add_argument( - "-c", "--compilefilter", metavar="filter", default=[], type=comma_arg - ) - parser.add_argument( "-x", "--xclude", metavar="filter", @@ -216,6 +215,13 @@ def main(): parser.add_argument( "-D", nargs="+", metavar="key=value", default=[], action="extend" ) + parser.add_argument( + "-U", + "--unsafe", + default=False, + action="store_true", + help="Allow unsafe code execution", + ) args = parser.parse_args() args.metadata = parse_keyval_args(args.metadata) args.D = parse_keyval_args(args.D) diff --git a/src/smp/__init__.py b/src/smp/__init__.py index f7cb937..2cdf2fe 100644 --- a/src/smp/__init__.py +++ b/src/smp/__init__.py @@ -42,6 +42,13 @@ def main(): help="Prefix builtins with smp_", ) parser.add_argument( + "-U", + "--unsafe", + default=False, + action="store_true", + help="Allow unsafe code execution", + ) + parser.add_argument( "file", nargs="?", default=None, help='Input file or "-" for stdin' ) args = parser.parse_args() @@ -50,6 +57,8 @@ def main(): macro_processor_state = smp.macro_processor.MacroProcessorState() prefix = "smp_" if args.prefix_builtins else "" macro_processor = macro_processor_state.macro_processor(prefix=prefix) + if args.unsafe: + macro_processor.unsafe_code_execution = True for key, value in args.D.items(): macro_processor.define_macro(key, value) diff --git a/src/smp/builtins.py b/src/smp/builtins.py index f9d6468..00d002b 100644 --- a/src/smp/builtins.py +++ b/src/smp/builtins.py @@ -6,6 +6,7 @@ import urllib.error import urllib.parse import datetime from markdown_it import MarkdownIt +from mdit_py_plugins import tasklists_plugin from skaldpress.metadata_parser import extract_parse_yaml_metadata from typing import Any @@ -113,12 +114,20 @@ def smp_builtin_include_verbatim(macro_processor, filename): def smp_builtin_shell(macro_processor, cmd_args): + if not macro_processor.unsafe_code_execution: + raise Exception("unsafe code execution now allowed!") cmd_args = macro_processor.process_input(cmd_args) return subprocess.check_output(cmd_args, shell=True).decode() def smp_builtin_eval(macro_processor, expression): - r = eval(expression, macro_processor.py_global_env, macro_processor.macros) + if not macro_processor.unsafe_code_execution: + raise Exception("unsafe code execution now allowed!") + r = eval( + expression, + macro_processor.py_local_env_current, + macro_processor.py_local_env_current, + ) return r @@ -179,6 +188,8 @@ def smp_builtin_html_from_markdown(macro_processor, text, extensions=list()): MarkdownIt("commonmark", {"breaks": False, "html": True}) .enable("table") .enable("list") + .enable("strikethrough") + .enable(tasklists_plugin) ) for extension in extensions: @@ -287,7 +298,7 @@ def smp_builtin_wodl(macro_processor, link, timeout_seconds=5): macro_processor.log_warning(f"Dead link {link} ({r.status} {r.reason})!") except Exception as e: macro_processor.log_warning(f"Dead link {link} ({e})!") - return "" + return link def smp_builtin_expand_once(macro_processor, content): @@ -307,10 +318,5 @@ def smp_builtin_dumpenv(macro_processor): out += "━ Macros ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" for key, val in macro_processor.macros.items(): out += f"{repr(key)}: {repr(val)}\n" - out += "━ Globals ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n" - for key, val in macro_processor.py_global_env.items(): - if key == "__builtins__": - continue - out += f"{repr(key)}: {repr(val)}\n" out += "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" return out diff --git a/src/smp/macro_processor.py b/src/smp/macro_processor.py index 19e692e..d5f4e8e 100644 --- a/src/smp/macro_processor.py +++ b/src/smp/macro_processor.py @@ -51,16 +51,21 @@ def macro_name_clean(macro_name: str) -> str: return macro_name +def strip_list(l, chars=[" "]): + while len(l) > 0 and l[0] in chars: + l.pop(0) + while len(l) > 0 and l[-1] in chars: + l.pop() + return l + + class MacroProcessor: source_file_path: str """All currently defined macros in this MacroProcessor""" macros: dict[str, Any] """ Macros which are @directives """ special_macros: dict[str, tuple[Any, Any]] - """ Global environment for python execution """ - py_global_env: dict """ Local environment for python execution """ - py_local_env_alt: dict py_local_env_current: dict """ All macro invocations that has happened """ macro_invocations: list[tuple[str, list[str]]] @@ -69,16 +74,14 @@ class MacroProcessor: end_quote: str = '"%' prefix: str = "" - file = None - expansion_stack: list[Any] - file_stack: list[Any] + + unsafe_code_execution: bool = False def __init__(self, prefix=""): self.macros = dict() self.macro_invocations = list() self.warnings = list() - self.py_global_env = dict() self.py_local_env_alt = dict() self.py_local_env_current = self.macros self.prefix = prefix @@ -86,7 +89,6 @@ class MacroProcessor: self.expansion_stack = ( list() ) # This probably needs some magic at some point later - self.file_stack = list() # This probably needs some magic at some point later self._define_builtins(self.macros) self._define_builtins(self.py_local_env_alt) @@ -108,6 +110,8 @@ class MacroProcessor: def _define_builtins(self, env): self._import_symbols(smp.builtins, env, function_prefix="smp_builtin_") + env[f"{self.prefix}file"] = None + env[f"{self.prefix}file_stack"] = [] env[f"{self.prefix}macro_processor"] = self env[f"{self.prefix}template_stack"] = [] env[f"{self.prefix}template_stack_content"] = [] @@ -138,7 +142,7 @@ class MacroProcessor: return self.macros.get(f"{sub_prefix}{macro_name}", default) def _define_metadata(self, macro_name, macro_value): - sub_prefix = (self._get_macro_builtin("metadata_prefix"),) + sub_prefix = self._get_macro_builtin("metadata_prefix") self.define_macro(f"{sub_prefix}{macro_name}", macro_value) def log_warning(self, message): @@ -171,27 +175,31 @@ class MacroProcessor: ) -> str: if len(args) == 0: return macro_name - out = f"{macro_name}(" + out: list[str] = [] + out.extend(f"{macro_name}(") for i, arg in enumerate(args): if process_args: - out += self.process_input(arg) + out.extend(self.process_input(arg)) else: - out += arg + out.extend(arg) if i < (len(args) - 1): - out += "," - out += ")" - return out + out.append(",") + out.append(")") + return "".join(out) def expand_macro(self, macro_name: str, args: list[str] = list()) -> str: # Ignore trailing underscore in macro name, the parser will pop a space in front if # present, but we should ignore it for finding the macro. + if isinstance(macro_name, list): + macro_name = "".join(macro_name) + macro_name = macro_name_clean(macro_name) if macro_name not in self.macros: return self._expand_unknown_macro(macro_name, args, process_args=True) # Strip leading whitespace from arguments for arg in args: - arg = arg.strip() + arg = strip_list(arg) # Log macro invokation # The fact that we are here, does not ensure that the macro is actually expanded into @@ -230,14 +238,14 @@ class MacroProcessor: self.expansion_stack.pop() def _enter_file_frame(self, filename, linenr, template): - self.file_stack.append([filename, linenr, 0]) + self._get_macro_builtin("file_stack").append([filename, linenr, 0]) def _pop_file_frame(self): - self.file_stack.pop() + self._get_macro_builtin("file_stack").pop() def _stack(self): out = "- File stack ------\n" - for i, frame in enumerate(self.file_stack): + for i, frame in enumerate(self._get_macro_builtin("file_stack")): out += f" {' ' * i} {frame[0]}:{frame[1]}\n" out += "\n- Macro stack -----\n" for i, frame in enumerate(self.expansion_stack): @@ -275,16 +283,17 @@ class MacroProcessor: but may also be faster (stream or mutable string) """ if file is not None: - self.file = file - if (file is None) and (self.file is not None): - file = self.file - output = "" + self._define_macro_builtin("file", file) + if (file is None) and (self._get_macro_builtin("file") is not None): + file = self._get_macro_builtin("file") + output: list[str] = [] + state = ParserState.NORMAL state_start = 0 - macro_name = "" + macro_name: list[str] = [] macro_args = [] - argument = "" - py_expr = "" + argument: list[str] = [] + py_expr: list[str] = [] skip_next_line_ending = False @@ -321,9 +330,13 @@ class MacroProcessor: ) if c == "\n": + if smp.builtins.smp_builtin_iftruthy( + self._get_macro_builtin("synclines", default="false") + ): + output.extend(f"\n#line {linenr} {file}") linenr += 1 linestart = i + 1 - self.file_stack[-1][1] = linenr + self._get_macro_builtin("file_stack")[-1][1] = linenr if self._debug_on(5): print( @@ -340,7 +353,7 @@ class MacroProcessor: i += 1 continue - if c == "%" and peek == "(": + if self.unsafe_code_execution and c == "%" and peek == "(": state = ParserState.IN_CODE i += 2 state_start = i @@ -358,7 +371,7 @@ class MacroProcessor: args = [ x.strip() for x in input[nspace:nline].split(" ") - if x is not "" + if x != "" ] sname = input[i + 1 : end_special] @@ -366,8 +379,10 @@ class MacroProcessor: hi_range(end=end_special) args.append(input[nline + 1 :]) # @TODO: Make try/catch wrapper here as well - output += self._expand_callable_macro( - self.special_macros[sname][0], args + output.extend( + self._expand_callable_macro( + self.special_macros[sname][0], args + ) ) i = len(input) - 1 continue @@ -380,29 +395,29 @@ class MacroProcessor: elif c.isalnum(): state = ParserState.IN_MACRO state_start = i - macro_name += c + macro_name.append(c) else: - output += c + output.append(c) elif state == ParserState.IN_QUOTES: if c == "%" and peek == '"': quote_level += 1 i += 1 - output += '%"' + output.extend(["%", '"']) elif c == '"' and peek == "%": quote_level -= 1 if quote_level == 0: state = ParserState.NORMAL state_start = i else: - output += '"%' + output.extend(['"', "%"]) i += 1 else: - output += c + output.append(c) elif state == ParserState.IN_MACRO: if c.isalnum() or c == "_": - macro_name += c + macro_name.append(c) elif c == "(": hi_range() parens_level += 1 @@ -410,28 +425,29 @@ class MacroProcessor: state_start = i else: hi_range() - if macro_is_whitespace_deleting(macro_name): + _macro_name = "".join(macro_name) + if macro_is_whitespace_deleting(_macro_name): if output[-1] == " ": output = output[:-1] - macro_name = macro_name_clean(macro_name) + _macro_name = macro_name_clean(_macro_name) - self._enter_frame(macro_name, file, linenr, input) + self._enter_frame(_macro_name, file, linenr, input) - if macro_name == "SNNL": + if _macro_name == "SNNL": skip_next_line_ending = c != "\n" - elif macro_name == "DNL": + elif _macro_name == "DNL": if c != "\n": state = ParserState.DNL state_start = i - macro_name = "" + macro_name.clear() i += 1 self._pop_frame() continue else: - expanded = self.expand_macro(macro_name) - output += expanded - output += c - macro_name = "" + expanded = self.expand_macro(_macro_name) + output.extend(expanded) + output.append(c) + macro_name.clear() self._pop_frame() @@ -441,77 +457,95 @@ class MacroProcessor: if c == "%" and peek == '"': quote_level += 1 i += 2 - argument += '%"' + argument.extend(["%", '"']) continue elif c == '"' and peek == "%": quote_level -= 1 i += 2 - argument += '"%' + argument.extend(['"', "%"]) continue elif quote_level > 0: - argument += c + argument.append(c) i += 1 continue if (c == ")") and (parens_level == 1): hi_range() - if macro_is_whitespace_deleting(macro_name): + _macro_name = "".join(macro_name) + if macro_is_whitespace_deleting(_macro_name): if output[-1] == " ": output = output[:-1] - macro_name = macro_name_clean(macro_name) + _macro_name = macro_name_clean(_macro_name) parens_level = 0 - macro_args.append(argument.strip()) - self._enter_frame(macro_name, file, linenr, input) - expanded = self.expand_macro(macro_name, macro_args) - output += expanded + macro_args.append("".join(strip_list(argument))) + self._enter_frame(_macro_name, file, linenr, input) + expanded = self.expand_macro(_macro_name, macro_args) + output.extend(expanded) state = ParserState.NORMAL state_start = i - macro_name = "" - macro_args = [] - argument = "" + macro_name.clear() + macro_args.clear() + argument.clear() self._pop_frame() elif (c == ",") and (parens_level == 1): - macro_args.append(argument.strip()) + macro_args.append("".join(strip_list(argument))) state_start = i hi_range() - argument = "" + argument.clear() else: if c == "(": parens_level += 1 if c == ")": parens_level -= 1 - argument += c + argument.append(c) elif state == ParserState.IN_CODE: if c == ")" and peek == "%": try: + if not self.unsafe_code_execution: + raise Exception("unsafe code execution now allowed!") self._enter_frame("inline_code", file, linenr, input) f = StringIO() with redirect_stdout(f): - exec(py_expr, self.py_global_env, self.py_local_env_current) + exec( + "".join(py_expr), + self.py_local_env_current, + self.py_local_env_current, + ) s = f.getvalue() if s != "": - output += s - except Exception: - traceback.print_exc() + output.extend(s) + except Exception as e: + e2 = "" + if self._debug_on(1): + e2 += "\n" + e2 += self._stack() + + if self._debug_on(2): + e2 += "\n- Python stack ---\n" + e2 += traceback.format_exc() + + self.log_warning(f"Code block exception ({e}){e2}") finally: self._pop_frame() - py_expr = "" + py_expr.clear() state = ParserState.NORMAL i += 1 state_start = i else: - py_expr += c + py_expr.append(c) i += 1 # Handle cases where the text ends with a macro without arguments - if macro_name != "": - if macro_is_whitespace_deleting(macro_name): + if len(macro_name) > 0: + _macro_name = "".join(macro_name) + if macro_is_whitespace_deleting(_macro_name): if len(output) > 0 and output[-1] == " ": output = output[:-1] - macro_name = macro_name_clean(macro_name) + _macro_name = macro_name_clean(_macro_name) hi_range() - output += self.expand_macro(macro_name) - return output + output.extend(self.expand_macro(_macro_name)) + + return "".join(output) def store(self, **xargs): requested_keys = self._get_metadata("keep_states", []) @@ -531,7 +565,7 @@ class MacroProcessor: target_filename = self._get_metadata("target_filename") - self.py_global_env["macro_processor_state"][self.source_file_path] = dict( + self._get_macro_builtin("macro_processor_state")[self.source_file_path] = dict( { # "content": "", "stored_data": { @@ -544,7 +578,7 @@ class MacroProcessor: **xargs, } ) - return self.py_global_env["macro_processor_state"][self.source_file_path] + return self._get_macro_builtin("macro_processor_state")[self.source_file_path] class MacroProcessorState: @@ -557,7 +591,9 @@ class MacroProcessorState: if macro_processor is None: macro_processor = MacroProcessor(prefix) - macro_processor.py_global_env["macro_processor_state"] = self.global_state + macro_processor._define_macro_builtin( + "macro_processor_state", self.global_state + ) return macro_processor def print_state(self): diff --git a/tests/test_macro_processor.sh b/tests/test_macro_processor.sh index daee5d2..96fc04a 100755 --- a/tests/test_macro_processor.sh +++ b/tests/test_macro_processor.sh @@ -9,7 +9,7 @@ test () { var2=$(echo "$file_content" | awk -v RS="\n---\n" 'NR==2') if [ "$ALL_OUTPUT" -eq 0 ]; then - res=$(echo -n "$var1" | smp 2> /dev/null) + res=$(echo -n "$var1" | smp -U 2> /dev/null) else res=$(echo -n "$var1" | smp) fi |