Dynamic syntax: grammar extensions as runtime data
The DSL page above declares its syntax { ... } block as a literal, expanded once before the compiler ever runs — indistinguishable from a macro. This one instead computes its trigger keyword and rules from data at runtime, calls expand_syntax_dsls_with(src, existing_defs) itself to define the grammar extension, keeps only the returned defs list, then makes a SEPARATE, later call — on a string containing only the usage line, no syntax declaration anywhere in it — passing those defs forward. Two independent calls, composing through nothing but an ordinary list value, exactly like passing along any other piece of program state. The assembled result runs through the same tokenize/parse_program/lower_program/run_ir chain that already powers the playground (see self_hosting/examples/playground_main.patlang) — PatLang's compiler has always been callable from PatLang code; this demo is the first to close the loop and use that to synthesize a language construct that couldn't have existed until the program ran. Native transcript only, no "run in browser" button: because this demo calls the compiler pipeline from inside itself, running it live means the playground's own run_ir (a nested interpreter, isolated from the outer compiled binary's functions) has to tokenize/parse/lower the entire ~85KB pipeline-plus-demo bundle a second time, INSIDE that nested interpreter — measured at several orders of magnitude slower than the ~2.5 seconds the same bundle takes through the top-level interpreter directly. The underlying capability is proven correct (this transcript matches the interpreted result exactly, and a compiled-native build agrees too); the live-in-browser path for this specific self-referential case is simply not practical yet — an honest limitation of run_ir's current performance on large inputs, not of the dynamic-syntax feature itself.
PatLang source
# =============================================================================
# Stage 1 self-hosted lexer library (Stage 0 compilable subset).
# Tokens are lists: [type, text, line] with types NUM, IDENT, STR, OP, NL, UNK, EOF.
# =============================================================================
make a function called is_digit_code takes c returns r
return (c >= 48) and (c <= 57)
end
make a function called is_alpha_code takes c returns r
if (c >= 65) and (c <= 90) then
return true
else
if (c >= 97) and (c <= 122) then
return true
else
return c == 95
end
end
end
make a function called is_op_code takes c returns r
if (c == 43) or (c == 45) or (c == 42) or (c == 47) or (c == 37) then
return true
else
if (c == 61) or (c == 60) or (c == 62) or (c == 33) then
return true
else
if (c == 40) or (c == 41) or (c == 91) or (c == 93) or (c == 44) or (c == 46) or (c == 124) then
return true
else
return false
end
end
end
end
make a function called tokenize takes src returns tokens
let tokens = vec_new()
let h = str_intern(src)
let n = sc_len(h)
let i = 0
let line = 1
while i < n do
let c = sc_code(h, i)
if c == 10 then
vec_push(tokens, ["NL", "", line])
let line = line + 1
let i = i + 1
else
if (c == 32) or (c == 9) or (c == 13) then
let i = i + 1
else
if c == 35 then
while (i < n) and (sc_code(h, i) != 10) do
let i = i + 1
end
else
if is_digit_code(c) then
let txt = ""
let dots = 0
let scanning = true
while (i < n) and scanning do
let d = sc_code(h, i)
if is_digit_code(d) then
let txt = txt + sc_char(h, i)
let i = i + 1
else
if (d == 46) and (dots == 0) then
let dots = 1
let txt = txt + sc_char(h, i)
let i = i + 1
else
let scanning = false
end
end
end
vec_push(tokens, ["NUM", txt, line])
else
if is_alpha_code(c) then
let txt = ""
let scanning = true
while (i < n) and scanning do
let d = sc_code(h, i)
if is_alpha_code(d) or is_digit_code(d) then
let txt = txt + sc_char(h, i)
let i = i + 1
else
let scanning = false
end
end
vec_push(tokens, ["IDENT", txt, line])
else
if c == 34 then
let i = i + 1
let txt = ""
let scanning = true
while (i < n) and scanning do
let d = sc_code(h, i)
if d == 34 then
let scanning = false
let i = i + 1
else
if d == 92 then
# escape sequences: n t r quote backslash (unknown kept raw)
let e = sc_code(h, i + 1)
if e == 110 then
let txt = txt + chr(10)
let i = i + 2
else
if e == 116 then
let txt = txt + chr(9)
let i = i + 2
else
if e == 114 then
let txt = txt + chr(13)
let i = i + 2
else
if e == 34 then
let txt = txt + chr(34)
let i = i + 2
else
if e == 92 then
let txt = txt + chr(92)
let i = i + 2
else
let txt = txt + sc_char(h, i)
let i = i + 1
end
end
end
end
end
else
let txt = txt + sc_char(h, i)
let i = i + 1
end
end
end
vec_push(tokens, ["STR", txt, line])
else
if is_op_code(c) then
let txt = sc_char(h, i)
let first = c
let i = i + 1
if i < n then
let d = sc_code(h, i)
if (d == 61) and ((first == 61) or (first == 33) or (first == 60) or (first == 62)) then
let txt = txt + sc_char(h, i)
let i = i + 1
end
end
vec_push(tokens, ["OP", txt, line])
else
vec_push(tokens, ["UNK", sc_char(h, i), line])
let i = i + 1
end
end
end
end
end
end
end
end
vec_push(tokens, ["EOF", "", line])
return tokens
end
make a function called print_tokens takes tokens returns done
let n = vec_len(tokens)
let i = 0
while i < n do
let t = vec_get(tokens, i)
print(t[0] + " '" + t[1] + "' @" + t[2])
let i = i + 1
end
return true
end
# =============================================================================
# Stage 1 self-hosted parser library (Stage 0 compilable subset).
# Consumes tokens from lib/lexer.patlang, produces list-shaped AST nodes.
#
# Statements:
# ["Let", name, expr] let NAME = expr
# ["Expr", expr] call statements, e.g. print(x), emit(e, p)
# ["If", cond, [then], [else]] if expr then ... [else ...] end
# ["While", cond, [body]] while expr do ... end
# ["Func", name, [params], [b]] make a function called N takes a, b returns r ... end
# ["Return", expr] return expr
# ["When", event, [body]] when EVENT do ... end (event handler)
# ["Err", message] parse error placeholder
#
# Expressions:
# ["Num", text] ["Str", text] ["Bool", "true"/"false"] ["Var", name]
# ["Bin", op, lhs, rhs] op: + - * / % == != < <= > >= and or
# ["Un", op, expr] op: not -
# ["Call", name, [args]]
# ["List", [items]]
# ["Index", obj, idx]
# ["Member", obj, prop]
#
# All parse functions return [node, next_pos] pairs; parse_args and
# parse_stmts_until return [list, next_pos].
# =============================================================================
make a function called tok_is takes t, ty, tx returns r
return (t[0] == ty) and (t[1] == tx)
end
make a function called skip_nl takes toks, pos returns p
let p = pos
let looping = true
while looping do
let t = vec_get(toks, p)
if t[0] == "NL" then
let p = p + 1
else
let looping = false
end
end
return p
end
# ---- expressions ----
make a function called parse_args takes toks, pos returns r
# pos points just after '('; returns [args, pos_after_rparen]
# tolerates newlines around '(', ',', and ')' so multi-line calls parse
let args = []
let p = skip_nl(toks, pos)
let t = vec_get(toks, p)
if tok_is(t, "OP", ")") then
return [args, p + 1]
else
let looping = true
while looping do
let e = parse_expr(toks, p)
let args = list_push(args, e[0])
let p = skip_nl(toks, e[1])
let t2 = vec_get(toks, p)
if tok_is(t2, "OP", ",") then
let p = skip_nl(toks, p + 1)
else
let looping = false
end
end
let t3 = vec_get(toks, p)
if tok_is(t3, "OP", ")") then
return [args, p + 1]
else
return [[["Err", "expected ) in argument list"]], p]
end
end
end
make a function called parse_primary takes toks, pos returns r
let t = vec_get(toks, pos)
let ty = t[0]
let tx = t[1]
if ty == "NUM" then
return [["Num", tx], pos + 1]
else
if ty == "STR" then
return [["Str", tx], pos + 1]
else
if ty == "IDENT" then
if (tx == "true") or (tx == "false") then
return [["Bool", tx], pos + 1]
else
let nx = vec_get(toks, pos + 1)
if tok_is(nx, "OP", "(") then
let a = parse_args(toks, pos + 2)
return [["Call", tx, a[0]], a[1]]
else
return [["Var", tx], pos + 1]
end
end
else
if tok_is(t, "OP", "(") then
let inner = parse_expr(toks, pos + 1)
let p = inner[1]
let t2 = vec_get(toks, p)
if tok_is(t2, "OP", ")") then
return [inner[0], p + 1]
else
return [["Err", "expected )"], p]
end
else
if tok_is(t, "OP", "[") then
# tolerates newlines around '[', ',', and ']' for multi-line lists
let items = []
let p = skip_nl(toks, pos + 1)
let t2 = vec_get(toks, p)
if tok_is(t2, "OP", "]") then
return [["List", items], p + 1]
else
let looping = true
while looping do
let e = parse_expr(toks, p)
let items = list_push(items, e[0])
let p = skip_nl(toks, e[1])
let t3 = vec_get(toks, p)
if tok_is(t3, "OP", ",") then
let p = skip_nl(toks, p + 1)
else
let looping = false
end
end
let t4 = vec_get(toks, p)
if tok_is(t4, "OP", "]") then
return [["List", items], p + 1]
else
return [["Err", "expected ] in list"], p]
end
end
else
if tok_is(t, "OP", "|") then
# Closure literal: |params| do body end (Stage 1 uses do/end
# rather than Stage 0's brace-delimited |params| { body },
# consistent with the rest of this dialect's block syntax)
let p = pos + 1
let params = []
let looping = true
while looping do
let pt = vec_get(toks, p)
if tok_is(pt, "OP", "|") then
let p = p + 1
let looping = false
else
if pt[0] == "IDENT" then
let params = list_push(params, pt[1])
let p = p + 1
let nt = vec_get(toks, p)
if tok_is(nt, "OP", ",") then
let p = p + 1
end
else
let looping = false
end
end
end
let p = skip_nl(toks, p)
let dt = vec_get(toks, p)
if (dt[0] == "IDENT") and (dt[1] == "do") then
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
let et = vec_get(toks, p2)
if (et[0] == "IDENT") and (et[1] == "end") then
return [["Closure", params, body[0]], p2 + 1]
else
return [["Err", "expected end after closure body"], p2]
end
else
return [["Err", "expected do after closure params"], p]
end
else
return [["Err", "unexpected " + ty + " '" + tx + "'"], pos + 1]
end
end
end
end
end
end
end
make a function called parse_postfix takes toks, pos returns r
let r1 = parse_primary(toks, pos)
let node = r1[0]
let p = r1[1]
let looping = true
while looping do
let t = vec_get(toks, p)
if tok_is(t, "OP", "[") then
let idx = parse_expr(toks, p + 1)
let p2 = idx[1]
let t2 = vec_get(toks, p2)
if tok_is(t2, "OP", "]") then
let node = ["Index", node, idx[0]]
let p = p2 + 1
else
let node = ["Err", "expected ] after index"]
let looping = false
end
else
if tok_is(t, "OP", ".") then
let nameTok = vec_get(toks, p + 1)
if nameTok[0] == "IDENT" then
let node = ["Member", node, nameTok[1]]
let p = p + 2
else
let node = ["Err", "expected name after ."]
let looping = false
end
else
let looping = false
end
end
end
return [node, p]
end
make a function called parse_unary takes toks, pos returns r
let t = vec_get(toks, pos)
if tok_is(t, "IDENT", "not") then
let e = parse_unary(toks, pos + 1)
return [["Un", "not", e[0]], e[1]]
else
if tok_is(t, "OP", "-") then
let e = parse_unary(toks, pos + 1)
return [["Un", "-", e[0]], e[1]]
else
return parse_postfix(toks, pos)
end
end
end
make a function called parse_mul takes toks, pos returns r
let r1 = parse_unary(toks, pos)
let node = r1[0]
let p = r1[1]
let looping = true
while looping do
# Peek past any newlines for a continuation operator (mirrors the
# native Rust frontend, and parse_args's own newline tolerance below)
# -- without this, a leading `*`/`/`/`%` on the next line is invisible
# here because the token right after `p` is an NL, not an OP, and the
# expression silently truncates instead of continuing.
let p2 = skip_nl(toks, p)
let t = vec_get(toks, p2)
if (t[0] == "OP") and ((t[1] == "*") or (t[1] == "/") or (t[1] == "%")) then
let r2 = parse_unary(toks, p2 + 1)
let node = ["Bin", t[1], node, r2[0]]
let p = r2[1]
else
let looping = false
end
end
return [node, p]
end
make a function called parse_add takes toks, pos returns r
let r1 = parse_mul(toks, pos)
let node = r1[0]
let p = r1[1]
let looping = true
while looping do
# See parse_mul's comment: peek past newlines for a continuation `+`/`-`.
let p2 = skip_nl(toks, p)
let t = vec_get(toks, p2)
if (t[0] == "OP") and ((t[1] == "+") or (t[1] == "-")) then
let r2 = parse_mul(toks, p2 + 1)
let node = ["Bin", t[1], node, r2[0]]
let p = r2[1]
else
let looping = false
end
end
return [node, p]
end
make a function called parse_cmp takes toks, pos returns r
let r1 = parse_add(toks, pos)
let node = r1[0]
let p = r1[1]
let looping = true
while looping do
let t = vec_get(toks, p)
if (t[0] == "OP") and ((t[1] == "==") or (t[1] == "!=") or (t[1] == "<") or (t[1] == "<=") or (t[1] == ">") or (t[1] == ">=")) then
let r2 = parse_add(toks, p + 1)
let node = ["Bin", t[1], node, r2[0]]
let p = r2[1]
else
let looping = false
end
end
return [node, p]
end
make a function called parse_and takes toks, pos returns r
let r1 = parse_cmp(toks, pos)
let node = r1[0]
let p = r1[1]
let looping = true
while looping do
let t = vec_get(toks, p)
if tok_is(t, "IDENT", "and") then
let r2 = parse_cmp(toks, p + 1)
let node = ["Bin", "and", node, r2[0]]
let p = r2[1]
else
let looping = false
end
end
return [node, p]
end
make a function called parse_expr takes toks, pos returns r
let r1 = parse_and(toks, pos)
let node = r1[0]
let p = r1[1]
let looping = true
while looping do
let t = vec_get(toks, p)
if tok_is(t, "IDENT", "or") then
let r2 = parse_and(toks, p + 1)
let node = ["Bin", "or", node, r2[0]]
let p = r2[1]
else
let looping = false
end
end
return [node, p]
end
# ---- statements ----
make a function called at_block_stop takes toks, pos returns r
let t = vec_get(toks, pos)
if t[0] == "EOF" then
return true
else
if (t[0] == "IDENT") and ((t[1] == "end") or (t[1] == "else")) then
return true
else
return false
end
end
end
make a function called parse_stmts_until takes toks, pos returns r
# Collect statements until 'end' / 'else' / EOF (stopper not consumed).
let stmts = []
let p = skip_nl(toks, pos)
let looping = true
while looping do
if at_block_stop(toks, p) then
let looping = false
else
let s = parse_stmt(toks, p)
let stmts = list_push(stmts, s[0])
let p = skip_nl(toks, s[1])
end
end
return [stmts, p]
end
make a function called parse_if takes toks, pos returns r
# pos points at 'if'
let c = parse_expr(toks, pos + 1)
let cond = c[0]
let p = c[1]
let t = vec_get(toks, p)
if tok_is(t, "IDENT", "then") then
let thenPart = parse_stmts_until(toks, p + 1)
let p2 = thenPart[1]
let t2 = vec_get(toks, p2)
if tok_is(t2, "IDENT", "else") then
let elsePart = parse_stmts_until(toks, p2 + 1)
let p3 = elsePart[1]
let t3 = vec_get(toks, p3)
if tok_is(t3, "IDENT", "end") then
return [["If", cond, thenPart[0], elsePart[0]], p3 + 1]
else
return [["Err", "expected end after else block"], p3]
end
else
if tok_is(t2, "IDENT", "end") then
return [["If", cond, thenPart[0], []], p2 + 1]
else
return [["Err", "expected else or end after if block"], p2]
end
end
else
return [["Err", "expected then after if condition"], p]
end
end
make a function called parse_while takes toks, pos returns r
# pos points at 'while'
let c = parse_expr(toks, pos + 1)
let cond = c[0]
let p = c[1]
let t = vec_get(toks, p)
if tok_is(t, "IDENT", "do") then
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
let t2 = vec_get(toks, p2)
if tok_is(t2, "IDENT", "end") then
return [["While", cond, body[0]], p2 + 1]
else
return [["Err", "expected end after while body"], p2]
end
else
return [["Err", "expected do after while condition"], p]
end
end
make a function called parse_function_def takes toks, pos returns r
# pos points at 'make'; expect: make a function called NAME
# [takes a, b] [returns r] NL body end
let p = pos + 1
if tok_is(vec_get(toks, p), "IDENT", "a") then
let p = p + 1
end
if tok_is(vec_get(toks, p), "IDENT", "function") then
let p = p + 1
else
return [["Err", "expected 'function' after make"], p]
end
if tok_is(vec_get(toks, p), "IDENT", "called") then
let p = p + 1
end
let nameTok = vec_get(toks, p)
if nameTok[0] == "IDENT" then
let name = nameTok[1]
let p = p + 1
let params = []
if tok_is(vec_get(toks, p), "IDENT", "takes") then
let p = p + 1
let looping = true
while looping do
let t = vec_get(toks, p)
if t[0] == "IDENT" then
if (t[1] == "returns") then
let looping = false
else
let params = list_push(params, t[1])
let p = p + 1
end
else
if tok_is(t, "OP", ",") then
let p = p + 1
else
let looping = false
end
end
end
end
if tok_is(vec_get(toks, p), "IDENT", "returns") then
# return-var hint: parsed and ignored (Stage 1 requires explicit return)
let p = p + 2
end
let body = parse_stmts_until(toks, p)
let p2 = body[1]
if tok_is(vec_get(toks, p2), "IDENT", "end") then
return [["Func", name, params, body[0]], p2 + 1]
else
return [["Err", "expected end after function body"], p2]
end
else
return [["Err", "expected function name"], p]
end
end
make a function called parse_when takes toks, pos returns r
# pos points at 'when'; expect: when EVENT do body end
let nameTok = vec_get(toks, pos + 1)
if nameTok[0] == "IDENT" then
let ev = nameTok[1]
let p = pos + 2
if tok_is(vec_get(toks, p), "IDENT", "do") then
let body = parse_stmts_until(toks, p + 1)
let p2 = body[1]
if tok_is(vec_get(toks, p2), "IDENT", "end") then
return [["When", ev, body[0]], p2 + 1]
else
return [["Err", "expected end after when body"], p2]
end
else
return [["Err", "expected do after when event"], p]
end
else
return [["Err", "expected event name after when"], pos + 1]
end
end
make a function called parse_stmt takes toks, pos returns r
let t = vec_get(toks, pos)
let ty = t[0]
let tx = t[1]
if ty == "IDENT" then
if tx == "let" then
let nameTok = vec_get(toks, pos + 1)
let name = nameTok[1]
let eqTok = vec_get(toks, pos + 2)
if tok_is(eqTok, "OP", "=") then
let e = parse_expr(toks, pos + 3)
return [["Let", name, e[0]], e[1]]
else
return [["Err", "expected = after let name"], pos + 1]
end
else
if tx == "if" then
return parse_if(toks, pos)
else
if tx == "while" then
return parse_while(toks, pos)
else
if (tx == "require") or (tx == "ensure") or (tx == "assert") then
let e = parse_expr(toks, pos + 1)
return [["Assert", tx, e[0]], e[1]]
else
if tx == "return" then
let e = parse_expr(toks, pos + 1)
return [["Return", e[0]], e[1]]
else
if tx == "make" then
return parse_function_def(toks, pos)
else
if tx == "when" then
return parse_when(toks, pos)
else
# general call statement: NAME(args)
let nx = vec_get(toks, pos + 1)
if tok_is(nx, "OP", "(") then
let a = parse_args(toks, pos + 2)
return [["Expr", ["Call", tx, a[0]]], a[1]]
else
return [["Err", "unexpected token '" + tx + "'"], pos + 1]
end
end
end
end
end
end
end
end
else
return [["Err", "unexpected " + ty + " '" + tx + "'"], pos + 1]
end
end
make a function called parse_program takes toks returns ast
let stmts = []
let pos = skip_nl(toks, 0)
let looping = true
while looping do
let t = vec_get(toks, pos)
if t[0] == "EOF" then
let looping = false
else
let r = parse_stmt(toks, pos)
let stmts = list_push(stmts, r[0])
let pos = skip_nl(toks, r[1])
end
end
return ["Program", stmts]
end
# ---- AST pretty printer ----
make a function called ast_list_to_str takes nodes returns s
let s = ""
let i = 0
while i < nodes.length do
if i > 0 then
let s = s + "; "
end
let s = s + ast_to_str(nodes[i])
let i = i + 1
end
return s
end
# Joins a list of plain strings (e.g. closure param names), not AST nodes
make a function called ast_list_to_str_plain takes items returns s
let s = ""
let i = 0
while i < items.length do
if i > 0 then
let s = s + ", "
end
let s = s + items[i]
let i = i + 1
end
return s
end
make a function called ast_to_str takes node returns s
let ty = node[0]
if ty == "Num" then
return "Num(" + node[1] + ")"
else
if ty == "Str" then
return "Str('" + node[1] + "')"
else
if ty == "Bool" then
return "Bool(" + node[1] + ")"
else
if ty == "Var" then
return "Var(" + node[1] + ")"
else
if ty == "Bin" then
return "(" + ast_to_str(node[2]) + " " + node[1] + " " + ast_to_str(node[3]) + ")"
else
if ty == "Un" then
return "(" + node[1] + " " + ast_to_str(node[2]) + ")"
else
if ty == "Call" then
return node[1] + "(" + ast_list_to_str(node[2]) + ")"
else
if ty == "Closure" then
return "|" + ast_list_to_str_plain(node[1]) + "| do " + ast_list_to_str(node[2]) + " end"
else
if ty == "List" then
return "[" + ast_list_to_str(node[1]) + "]"
else
if ty == "Index" then
return ast_to_str(node[1]) + "[" + ast_to_str(node[2]) + "]"
else
if ty == "Member" then
return ast_to_str(node[1]) + "." + node[2]
else
if ty == "Let" then
return "Let " + node[1] + " = " + ast_to_str(node[2])
else
if ty == "Expr" then
return ast_to_str(node[1])
else
if ty == "If" then
return "If " + ast_to_str(node[1]) + " Then {" + ast_list_to_str(node[2]) + "} Else {" + ast_list_to_str(node[3]) + "}"
else
if ty == "While" then
return "While " + ast_to_str(node[1]) + " {" + ast_list_to_str(node[2]) + "}"
else
if ty == "Func" then
return "Func " + node[1] + " {" + ast_list_to_str(node[3]) + "}"
else
if ty == "Return" then
return "Return " + ast_to_str(node[1])
else
if ty == "When" then
return "When " + node[1] + " {" + ast_list_to_str(node[2]) + "}"
else
if ty == "Assert" then
return node[1] + " " + ast_to_str(node[2])
else
return "Err(" + node[1] + ")"
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
end
# =============================================================================
# Stage 1 self-hosted lowerer (Stage 0 compilable subset).
# Walks the list-shaped AST from lib/parser.patlang and emits list-shaped IR
# instructions. The host's compile_ir only decodes this IR and runs codegen —
# lexing, parsing, lowering, AND (via lib/codegen.patlang) code generation all
# happen in PatLang.
#
# IR shape:
# ["ProgramIR", entry, [functions], [events]]
# ["FuncIR", name, [params], [instrs]]
# ["EventIR", event, handler_name]
# Instructions:
# ["Const", "num"|"str"|"bool", text] ["Load", name] ["Store", name]
# ["Bin", op] ["Un", op] ["Jump", n] ["JumpIfFalse", n]
# ["CallHost", name, argc] ["Call", name, argc]
# ["MakeClosure", func_name, [captured_names]] ["CallValue", argc]
# ["BuildList", n] ["Return"]
#
# Closures: |params| do body end (Stage 1 has no brace-delimited blocks
# anywhere, so closures use the same do/end convention as while-loops rather
# than Stage 0's |params| { body }). A closure captures its ENTIRE enclosing
# locals list (over-capture, not precise free-variable analysis) as leading
# hidden parameters of a synthesized function — simpler than exact capture
# and still correct: the flat per-call locals map means a closure's own
# `let` of a same-named variable just overwrites the pre-bound captured
# value, exactly matching intended shadowing.
# =============================================================================
make a function called contains_str takes xs, s returns r
let i = 0
while i < xs.length do
if xs[i] == s then
return true
end
let i = i + 1
end
return false
end
make a function called vec_contains takes v, s returns r
let i = 0
let n = vec_len(v)
while i < n do
if vec_get(v, i) == s then
return true
end
let i = i + 1
end
return false
end
make a function called next_closure_name returns name
let n = get("__vars", "__closure_seq")
if not n then
let n = 0
end
set_var("__closure_seq", n + 1)
return "__closure_" + n
end
# lower_expr(node, code, fns, locals, pending) -> code
# fns: list of top-level function names (static call vs host call)
# locals: vec of names currently bound in the enclosing scope (for
# disambiguating a call through a closure-valued variable, and
# for over-capturing into new closures)
# pending: vec of synthesized ["FuncIR", ...] nodes from closure literals,
# merged into the program's function list once lowering finishes
make a function called lower_expr takes node, code, fns, locals, pending returns out
let ty = node[0]
if ty == "Num" then
vec_push(code, ["Const", "num", node[1]])
return code
else
if ty == "Str" then
vec_push(code, ["Const", "str", node[1]])
return code
else
if ty == "Bool" then
vec_push(code, ["Const", "bool", node[1]])
return code
else
if ty == "Var" then
vec_push(code, ["Load", node[1]])
return code
else
if ty == "Bin" then
if node[1] == "and" then
# short-circuit: false when lhs falsey, else truthiness of rhs
let code = lower_expr(node[2], code, fns, locals, pending)
vec_push(code, ["Un", "not"])
let jif = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
vec_push(code, ["Const", "bool", "false"])
let jmp = vec_len(code)
vec_push(code, ["Jump", 0])
vec_set(code, jif, ["JumpIfFalse", vec_len(code)])
let code = lower_expr(node[3], code, fns, locals, pending)
vec_push(code, ["Un", "not"])
vec_push(code, ["Un", "not"])
vec_set(code, jmp, ["Jump", vec_len(code)])
return code
else
if node[1] == "or" then
# short-circuit: true when lhs truthy, else truthiness of rhs
let code = lower_expr(node[2], code, fns, locals, pending)
vec_push(code, ["Un", "not"])
let jif = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
let code = lower_expr(node[3], code, fns, locals, pending)
vec_push(code, ["Un", "not"])
vec_push(code, ["Un", "not"])
let jmp = vec_len(code)
vec_push(code, ["Jump", 0])
vec_set(code, jif, ["JumpIfFalse", vec_len(code)])
vec_push(code, ["Const", "bool", "true"])
vec_set(code, jmp, ["Jump", vec_len(code)])
return code
else
let code = lower_expr(node[2], code, fns, locals, pending)
let code = lower_expr(node[3], code, fns, locals, pending)
vec_push(code, ["Bin", node[1]])
return code
end
end
else
if ty == "Un" then
let code = lower_expr(node[2], code, fns, locals, pending)
vec_push(code, ["Un", node[1]])
return code
else
if ty == "Call" then
let callee = node[1]
let args = node[2]
if contains_str(fns, callee) then
let i = 0
while i < args.length do
let code = lower_expr(args[i], code, fns, locals, pending)
let i = i + 1
end
vec_push(code, ["Call", callee, args.length])
return code
else
if vec_contains(locals, callee) then
# dynamic call through a local variable holding a closure
vec_push(code, ["Load", callee])
let i = 0
while i < args.length do
let code = lower_expr(args[i], code, fns, locals, pending)
let i = i + 1
end
vec_push(code, ["CallValue", args.length])
return code
else
let i = 0
while i < args.length do
let code = lower_expr(args[i], code, fns, locals, pending)
let i = i + 1
end
vec_push(code, ["CallHost", callee, args.length])
return code
end
end
else
if ty == "List" then
let items = node[1]
let i = 0
while i < items.length do
let code = lower_expr(items[i], code, fns, locals, pending)
let i = i + 1
end
vec_push(code, ["BuildList", items.length])
return code
else
if ty == "Index" then
let code = lower_expr(node[1], code, fns, locals, pending)
let code = lower_expr(node[2], code, fns, locals, pending)
vec_push(code, ["CallHost", "list_get", 2])
return code
else
if ty == "Member" then
let code = lower_expr(node[1], code, fns, locals, pending)
if (node[2] == "length") or (node[2] == "len") then
vec_push(code, ["CallHost", "len", 1])
return code
else
vec_push(code, ["Const", "str", node[2]])
vec_push(code, ["CallHost", "get", 2])
return code
end
else
if ty == "Closure" then
return lower_closure_literal(node[1], node[2], code, fns, locals, pending)
else
# unknown expression: lower as unit-ish empty string
vec_push(code, ["Const", "str", ""])
return code
end
end
end
end
end
end
end
end
end
end
end
end
# Lowers a closure literal: synthesizes a Function (params = the enclosing
# scope's current locals ++ the closure's own params) queued in `pending`,
# and emits the Load.../MakeClosure sequence at the creation site into `code`.
make a function called lower_closure_literal takes params, body, code, fns, locals, pending returns out
let captured = vec_to_list(locals)
let func_name = next_closure_name()
let all_params = []
let i = 0
while i < captured.length do
let all_params = list_push(all_params, captured[i])
let i = i + 1
end
let i = 0
while i < params.length do
let all_params = list_push(all_params, params[i])
let i = i + 1
end
let inner_locals = vec_new()
let i = 0
while i < all_params.length do
vec_push(inner_locals, all_params[i])
let i = i + 1
end
let inner_code = lower_block(body, vec_new(), fns, func_name, inner_locals, pending)
vec_push(inner_code, ["Return"])
vec_push(pending, ["FuncIR", func_name, all_params, vec_to_list(inner_code)])
# At the creation site: push captured values in the SAME order used for
# all_params's leading section, then bundle them.
let i = 0
while i < captured.length do
vec_push(code, ["Load", captured[i]])
let i = i + 1
end
vec_push(code, ["MakeClosure", func_name, captured])
return code
end
make a function called lower_stmt takes node, code, fns, fname, locals, pending returns out
let ty = node[0]
if ty == "Let" then
let code = lower_expr(node[2], code, fns, locals, pending)
vec_push(code, ["Store", node[1]])
vec_push(locals, node[1])
return code
else
if ty == "Expr" then
return lower_expr(node[1], code, fns, locals, pending)
else
if ty == "Print" then
let code = lower_expr(node[1], code, fns, locals, pending)
vec_push(code, ["CallHost", "print", 1])
return code
else
if ty == "Return" then
let code = lower_expr(node[1], code, fns, locals, pending)
vec_push(code, ["Return"])
return code
else
if ty == "If" then
let code = lower_expr(node[1], code, fns, locals, pending)
let jif = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
let code = lower_block(node[2], code, fns, fname, locals, pending)
let jmp = vec_len(code)
vec_push(code, ["Jump", 0])
vec_set(code, jif, ["JumpIfFalse", vec_len(code)])
let code = lower_block(node[3], code, fns, fname, locals, pending)
vec_set(code, jmp, ["Jump", vec_len(code)])
return code
else
if ty == "While" then
let start = vec_len(code)
let code = lower_expr(node[1], code, fns, locals, pending)
let jif = vec_len(code)
vec_push(code, ["JumpIfFalse", 0])
let code = lower_block(node[2], code, fns, fname, locals, pending)
vec_push(code, ["Jump", start])
vec_set(code, jif, ["JumpIfFalse", vec_len(code)])
return code
else
if ty == "Assert" then
# contract_check(func_name, kind, text, ok) — args pushed in
# order, ok (the evaluated condition) last
vec_push(code, ["Const", "str", fname])
vec_push(code, ["Const", "str", node[1]])
vec_push(code, ["Const", "str", ast_to_str(node[2])])
let code = lower_expr(node[2], code, fns, locals, pending)
vec_push(code, ["CallHost", "contract_check", 4])
return code
else
# unknown statement: no code
return code
end
end
end
end
end
end
end
end
make a function called lower_block takes stmts, code, fns, fname, locals, pending returns out
let i = 0
while i < stmts.length do
let code = lower_stmt(stmts[i], code, fns, fname, locals, pending)
let i = i + 1
end
return code
end
make a function called collect_fns takes stmts returns fns
let fns = []
let i = 0
while i < stmts.length do
let s = stmts[i]
if s[0] == "Func" then
let fns = list_push(fns, s[1])
end
let i = i + 1
end
return fns
end
make a function called lower_program takes ast returns ir
set_var("__closure_seq", 0)
let stmts = ast[1]
let fns = collect_fns(stmts)
let funcs = []
let events = []
let mainCode = vec_new()
let mainLocals = vec_new()
let pending = vec_new()
let handlerCount = 0
let i = 0
while i < stmts.length do
let s = stmts[i]
if s[0] == "Func" then
let flocals = vec_new()
let k = 0
while k < s[2].length do
vec_push(flocals, s[2][k])
let k = k + 1
end
let code = lower_block(s[3], vec_new(), fns, s[1], flocals, pending)
vec_push(code, ["Return"])
let funcs = list_push(funcs, ["FuncIR", s[1], s[2], vec_to_list(code)])
else
if s[0] == "When" then
let handlerCount = handlerCount + 1
let hname = "__when_" + s[1] + "_" + handlerCount
let hlocals = vec_new()
vec_push(hlocals, "event_name")
vec_push(hlocals, "event_data")
let code = lower_block(s[2], vec_new(), fns, hname, hlocals, pending)
vec_push(code, ["Return"])
let funcs = list_push(funcs, ["FuncIR", hname, ["event_name", "event_data"], vec_to_list(code)])
let events = list_push(events, ["EventIR", s[1], hname])
else
let mainCode = lower_stmt(s, mainCode, fns, "main", mainLocals, pending)
end
end
let i = i + 1
end
vec_push(mainCode, ["Return"])
let funcs = list_push(funcs, ["FuncIR", "main", [], vec_to_list(mainCode)])
let pendingList = vec_to_list(pending)
let i = 0
while i < pendingList.length do
let funcs = list_push(funcs, pendingList[i])
let i = i + 1
end
return ["ProgramIR", "main", funcs, events]
end
# =============================================================================
# General-purpose regex engine, Stage 1 self-hosted dialect (concatenate
# after nothing else; only uses host functions available to the self-hosted
# compiler pipeline: char_code/substr/chr/list_push/list_get/list_len/to_num).
#
# This is the self-hosted-dialect twin of the Stage 0 (`fn`/`{}`) engine at
# rust-runtime's self_hosting/lib/regex.patlang, used by the Rust CLI's
# `syntax_dsl.rs` preprocessor. That version cannot be `include`d into
# anything compiled by the self-hosted pipeline (lib/lexer.patlang +
# lib/parser.patlang don't lex `{`/`}` as anything meaningful — this dialect
# is do/end-delimited throughout), so the logic is duplicated here in the
# dialect lib/lexer.patlang and lib/parser.patlang actually understand. Keep
# both in sync if the regex feature set changes.
#
# Supported syntax subset: literals, \d \w \s \b \n \t escapes, '.' (any),
# [abc] / [a-z0-9] / [^abc] classes, (...) grouping, '|' alternation,
# '*' '+' '?' quantifiers, '^' '$' anchors. No capture-group extraction —
# whole-match length only.
#
# AST nodes are tagged lists: ["lit", ch], ["any"], ["class", negate, items],
# ["seq", nodes], ["alt", branches], ["star", node], ["plus", node],
# ["opt", node], ["bol"], ["eol"], ["wordb"], ["group", node].
# =============================================================================
make a function called is_digit_char takes ch returns r
if ch == "0" then
return true
else
if ch == "1" then
return true
else
if ch == "2" then
return true
else
if ch == "3" then
return true
else
if ch == "4" then
return true
else
if ch == "5" then
return true
else
if ch == "6" then
return true
else
if ch == "7" then
return true
else
if ch == "8" then
return true
else
return ch == "9"
end
end
end
end
end
end
end
end
end
end
make a function called is_word_char takes ch returns r
if ch == "_" then
return true
else
if is_digit_char(ch) then
return true
else
let code = char_code(ch, 0)
if (code >= 65) and (code <= 90) then
return true
else
if (code >= 97) and (code <= 122) then
return true
else
return false
end
end
end
end
end
# ---------------------------------------------------------------------
# Parsing: pattern (string) -> AST node. Internal helpers return
# [node, next_pos] pairs; next_pos is -1 on failure.
# ---------------------------------------------------------------------
make a function called regex_atom_for_escape takes ch returns node
if ch == "d" then
return ["class", 0, [["range", "0", "9"]]]
else
if ch == "w" then
return ["class", 0, [["range", "a", "z"], ["range", "A", "Z"], ["range", "0", "9"], ["char", "_"]]]
else
if ch == "s" then
return ["class", 0, [["char", " "], ["char", "\t"], ["char", "\n"], ["char", "\r"]]]
else
if ch == "b" then
return ["wordb"]
else
if ch == "n" then
return ["lit", "\n"]
else
if ch == "t" then
return ["lit", "\t"]
else
return ["lit", ch]
end
end
end
end
end
end
end
make a function called regex_parse_class takes pattern, pos returns r
let plen = to_num(list_len(pattern))
let negate = 0
let p = pos
if (p < plen) and (substr(pattern, p, 1) == "^") then
let negate = 1
let p = p + 1
end
let items = []
let scanning = true
while (p < plen) and scanning do
if substr(pattern, p, 1) == "]" then
let scanning = false
else
let c = substr(pattern, p, 1)
if (c == "\\") and ((p + 1) < plen) then
let c = substr(pattern, p + 1, 1)
let p = p + 1
end
if ((p + 2) < plen) and (substr(pattern, p + 1, 1) == "-") and (substr(pattern, p + 2, 1) != "]") then
let hi = substr(pattern, p + 2, 1)
let items = list_push(items, ["range", c, hi])
let p = p + 3
else
let items = list_push(items, ["char", c])
let p = p + 1
end
end
end
if p >= plen then
return [["class", negate, items], -1]
else
return [["class", negate, items], p + 1]
end
end
make a function called regex_parse_atom takes pattern, pos returns r
let plen = to_num(list_len(pattern))
if pos >= plen then
return [["seq", []], -1]
else
let c = substr(pattern, pos, 1)
if c == "(" then
let inner = regex_parse_alt(pattern, pos + 1)
let node = inner[0]
let p = inner[1]
if p < 0 then
return [node, -1]
else
if (p >= plen) or (substr(pattern, p, 1) != ")") then
return [node, -1]
else
return [["group", node], p + 1]
end
end
else
if c == "." then
return [["any"], pos + 1]
else
if c == "^" then
return [["bol"], pos + 1]
else
if c == "$" then
return [["eol"], pos + 1]
else
if c == "\\" then
if (pos + 1) >= plen then
return [["lit", "\\"], pos + 1]
else
let esc = substr(pattern, pos + 1, 1)
return [regex_atom_for_escape(esc), pos + 2]
end
else
if c == "[" then
return regex_parse_class(pattern, pos + 1)
else
return [["lit", c], pos + 1]
end
end
end
end
end
end
end
end
make a function called regex_parse_quantified takes pattern, pos returns r
let r = regex_parse_atom(pattern, pos)
let node = r[0]
let p = r[1]
if p < 0 then
return [node, -1]
else
let plen = to_num(list_len(pattern))
if p < plen then
let c = substr(pattern, p, 1)
if c == "*" then
return [["star", node], p + 1]
else
if c == "+" then
return [["plus", node], p + 1]
else
if c == "?" then
return [["opt", node], p + 1]
else
return [node, p]
end
end
end
else
return [node, p]
end
end
end
make a function called regex_parse_seq takes pattern, pos returns r
let plen = to_num(list_len(pattern))
let nodes = []
let p = pos
let scanning = true
while (p < plen) and scanning do
if (substr(pattern, p, 1) == "|") or (substr(pattern, p, 1) == ")") then
let scanning = false
else
let r = regex_parse_quantified(pattern, p)
let node = r[0]
let p = r[1]
if p < 0 then
let scanning = false
else
let nodes = list_push(nodes, node)
end
end
end
return [["seq", nodes], p]
end
make a function called regex_parse_alt takes pattern, pos returns r
let plen = to_num(list_len(pattern))
let first = regex_parse_seq(pattern, pos)
let node = first[0]
let p = first[1]
if p < 0 then
return [node, p]
else
let branches = list_push([], node)
let scanning = true
while (p < plen) and scanning and (substr(pattern, p, 1) == "|") do
let nxt = regex_parse_seq(pattern, p + 1)
let branches = list_push(branches, nxt[0])
let p = nxt[1]
if p < 0 then
let scanning = false
end
end
if p < 0 then
return [["alt", branches], -1]
else
let n = to_num(list_len(branches))
if n == 1 then
return [branches[0], p]
else
return [["alt", branches], p]
end
end
end
end
make a function called regex_parse takes pattern returns node
let result = regex_parse_alt(pattern, 0)
return result[0]
end
# ---------------------------------------------------------------------
# Matching: attempts to match `node` in `text` starting at `pos`, calling
# the closure `k(new_pos)` with every position reachable after a successful
# match, until `k` returns a non-negative number (accepted) or every
# possibility is exhausted (-1). This continuation-passing style is what
# makes backtracking over alternation/quantifiers correct: `k` represents
# "the rest of the overall pattern", so a quantifier can retry with fewer
# repetitions if a later part of the pattern needs the characters back.
# ---------------------------------------------------------------------
make a function called regex_class_matches takes items, ch returns r
let n = to_num(list_len(items))
let i = 0
let found = false
while (i < n) and (not found) do
let it = items[i]
if it[0] == "char" then
if ch == it[1] then
let found = true
end
else
if (ch >= it[1]) and (ch <= it[2]) then
let found = true
end
end
let i = i + 1
end
return found
end
make a function called regex_match_seq takes nodes, idx, text, pos, k returns r
let n = to_num(list_len(nodes))
if idx >= n then
return k(pos)
else
let node = nodes[idx]
return regex_match_node(node, text, pos, |p2| do return regex_match_seq(nodes, idx + 1, text, p2, k) end)
end
end
make a function called regex_match_alt takes branches, idx, text, pos, k returns r
let n = to_num(list_len(branches))
if idx >= n then
return -1
else
let r = regex_match_node(branches[idx], text, pos, k)
if r >= 0 then
return r
else
return regex_match_alt(branches, idx + 1, text, pos, k)
end
end
end
# Backtracking repetition: tries the greatest number of repetitions first,
# then backs off, so whatever comes after (represented by `k`) still gets a
# chance to match — standard greedy-with-backtracking behavior.
make a function called regex_match_repeat takes node, text, pos, k, min_count returns r
let r = regex_match_node(node, text, pos, |p2| do
if p2 == pos then
return -1
else
return regex_match_repeat(node, text, p2, k, 0)
end
end)
if r >= 0 then
return r
else
if min_count <= 0 then
return k(pos)
else
return -1
end
end
end
make a function called regex_match_node takes node, text, pos, k returns r
let tag = node[0]
let tlen = to_num(list_len(text))
if tag == "lit" then
if (pos < tlen) and (substr(text, pos, 1) == node[1]) then
return k(pos + 1)
else
return -1
end
else
if tag == "any" then
if (pos < tlen) and (substr(text, pos, 1) != "\n") then
return k(pos + 1)
else
return -1
end
else
if tag == "class" then
if pos >= tlen then
return -1
else
let ch = substr(text, pos, 1)
let hit = regex_class_matches(node[2], ch)
if node[1] == 1 then
let hit = not hit
end
if hit then
return k(pos + 1)
else
return -1
end
end
else
if tag == "bol" then
if pos == 0 then
return k(pos)
else
if substr(text, pos - 1, 1) == "\n" then
return k(pos)
else
return -1
end
end
else
if tag == "eol" then
if pos == tlen then
return k(pos)
else
if substr(text, pos, 1) == "\n" then
return k(pos)
else
return -1
end
end
else
if tag == "wordb" then
let before = false
let after = false
if pos > 0 then
let before = is_word_char(substr(text, pos - 1, 1))
end
if pos < tlen then
let after = is_word_char(substr(text, pos, 1))
end
if before != after then
return k(pos)
else
return -1
end
else
if tag == "group" then
return regex_match_node(node[1], text, pos, k)
else
if tag == "seq" then
return regex_match_seq(node[1], 0, text, pos, k)
else
if tag == "alt" then
return regex_match_alt(node[1], 0, text, pos, k)
else
if tag == "star" then
return regex_match_repeat(node[1], text, pos, k, 0)
else
if tag == "plus" then
return regex_match_repeat(node[1], text, pos, k, 1)
else
if tag == "opt" then
let inner = node[1]
let r = regex_match_node(inner, text, pos, k)
if r >= 0 then
return r
else
return k(pos)
end
else
return -1
end
end
end
end
end
end
end
end
end
end
end
end
end
# Attempts to match `pattern_ast` (already parsed via regex_parse) against
# `text` starting exactly at `start`. Returns the end position of the
# leftmost-greedy match, or -1 if there is no match anchored at `start`.
#
# Note: closures in this dialect must use an explicit `return` for their
# body value to propagate — a bare trailing expression statement (no
# `return`) does not implicitly become the closure's return value.
make a function called regex_match_at takes pattern_ast, text, start returns r
return regex_match_node(pattern_ast, text, start, |p| do return p end)
end
# Convenience combined entry point for callers that only have the raw
# pattern string (no pre-parsed AST cached) — parses then matches.
make a function called regex_match_string_at takes pattern, text, start returns r
let ast = regex_parse(pattern)
return regex_match_at(ast, text, start)
end
# =============================================================================
# Self-hosted-dialect source-to-source preprocessor for user-defined
# `syntax NAME { ... }` DSL blocks (do/end dialect twin of the Rust CLI's
# rust-runtime/src/syntax_dsl.rs). Runs on raw source TEXT before the normal
# lexer/parser pipeline ever sees it, exactly like expand_includes does for
# `include "..."` lines -- so no change to lib/lexer.patlang or
# lib/parser.patlang is needed to support it.
#
# A PatLang program may declare a small grammar extension inline:
#
# syntax RouterDSL {
# trigger: Keyword("routes");
# tokens {
# HttpVerb(verb) = regex("\b(GET|POST|PUT|DELETE)\b", verb);
# UrlPath(path) = regex("/[a-zA-Z0-9_/:-]*", path);
# Arrow = "->";
# }
# rule RouteLine {
# let verb = expect HttpVerb;
# let path = expect UrlPath;
# expect Arrow;
# let controller = expect Identifier;
# expect Symbol(".");
# let action = expect Identifier;
# return AST.RegisterRoute(verb, path, controller, action);
# }
# }
#
# ...and then use it:
#
# routes {
# GET /users -> UserController.index
# }
#
# The `syntax { ... }` block itself is stripped. Every `routes { ... }` block
# is expanded line-by-line into plain PatLang statements built from the
# rule's `return` template, with `expect`-bound captures substituted in as
# string literals -- before the normal lexer/parser ever sees the file.
#
# Per-token `regex(...)` matching is done by the general-purpose regex engine
# at lib/regex_dsl.patlang -- this file assumes regex_match_string_at (and
# nothing else from that file) is already defined in the same program, i.e.
# a driver script should concatenate lib/regex_dsl.patlang's source BEFORE
# this file's source, the same way lib/lexer.patlang/parser.patlang/
# lower.patlang are concatenated in pipeline driver scripts.
#
# NOTE: `{`/`}` are used here only as characters being scanned inside plain
# strings (the *target* syntax being preprocessed) -- never as PatLang block
# syntax. This file's own code is ordinary do/end/if/then/while dialect.
#
# Data shapes (plain tagged lists -- there is no map/struct in this dialect):
# TokenDef = ["token", name, kind, pattern] kind: "regex" | "literal"
# RuleStep = ["step", kind, value, bind] kind: "named" | "identifier" | "symbol"
# RuleDef = ["rule", steps_list, return_template]
# SyntaxDef = ["syntax", trigger_keyword, tokens_list, rules_list]
# MatchResult = ["ok", bindings_list] | ["fail"] bindings_list: [[name, text], ...]
# =============================================================================
make a function called sdsl_is_ws takes ch returns r
if ch == " " then
return true
else
if ch == "\t" then
return true
else
if ch == "\n" then
return true
else
return ch == "\r"
end
end
end
end
make a function called sdsl_is_digit takes ch returns r
return (ch >= "0") and (ch <= "9")
end
make a function called sdsl_is_ident_start takes ch returns r
if ch == "_" then
return true
else
return ((ch >= "a") and (ch <= "z")) or ((ch >= "A") and (ch <= "Z"))
end
end
make a function called sdsl_is_ident_char takes ch returns r
if sdsl_is_ident_start(ch) then
return true
else
return sdsl_is_digit(ch)
end
end
make a function called sdsl_len takes s returns r
return to_num(list_len(s))
end
# Whole-word keyword match: `keyword` occurs at `s[pos..]` and is not glued
# to an identifier character on either side.
make a function called sdsl_starts_keyword_at takes s, pos, keyword returns r
let slen = sdsl_len(s)
let klen = sdsl_len(keyword)
if (pos + klen) > slen then
return false
else
if substr(s, pos, klen) != keyword then
return false
else
let ok = true
if pos > 0 then
if sdsl_is_ident_char(substr(s, pos - 1, 1)) then
let ok = false
end
end
let after = pos + klen
if after < slen then
if sdsl_is_ident_char(substr(s, after, 1)) then
let ok = false
end
end
return ok
end
end
end
make a function called sdsl_skip_ws takes s, pos returns r
let slen = sdsl_len(s)
let i = pos
let scanning = true
while (i < slen) and scanning do
if sdsl_is_ws(substr(s, i, 1)) then
let i = i + 1
else
let scanning = false
end
end
return i
end
make a function called sdsl_skip_ws_and_comments takes s, pos returns r
let slen = sdsl_len(s)
let i = sdsl_skip_ws(s, pos)
let scanning = true
while scanning do
let scanning = false
if (i < slen) and (substr(s, i, 1) == "#") then
while (i < slen) and (substr(s, i, 1) != "\n") do
let i = i + 1
end
let i = sdsl_skip_ws(s, i)
let scanning = true
end
end
return i
end
make a function called sdsl_find_char takes s, from, target returns r
let slen = sdsl_len(s)
let i = from
let found = -1
let scanning = true
while (i < slen) and scanning do
if substr(s, i, 1) == target then
let found = i
let scanning = false
else
let i = i + 1
end
end
return found
end
# Returns [identifier_text, next_pos], or ["", -1] if no identifier starts there.
make a function called sdsl_take_identifier takes s, pos returns r
let slen = sdsl_len(s)
if pos >= slen then
return ["", -1]
else
if not sdsl_is_ident_start(substr(s, pos, 1)) then
return ["", -1]
else
let i = pos + 1
let scanning = true
while (i < slen) and scanning do
if sdsl_is_ident_char(substr(s, i, 1)) then
let i = i + 1
else
let scanning = false
end
end
return [substr(s, pos, i - pos), i]
end
end
end
make a function called sdsl_trim takes s returns r
let slen = sdsl_len(s)
let start = 0
while (start < slen) and sdsl_is_ws(substr(s, start, 1)) do
let start = start + 1
end
let stop = slen
while (stop > start) and sdsl_is_ws(substr(s, stop - 1, 1)) do
let stop = stop - 1
end
return substr(s, start, stop - start)
end
# `"..."` -> `...` (no escape processing needed: the DSL block source is
# scanned as raw text, and quoted strings inside it only ever contain plain
# pattern/literal text in the examples this mechanism targets).
make a function called sdsl_unquote takes s returns r
let t = sdsl_trim(s)
let n = sdsl_len(t)
if (n >= 2) and (substr(t, 0, 1) == "\"") and (substr(t, n - 1, 1) == "\"") then
return substr(t, 1, n - 2)
else
return t
end
end
make a function called sdsl_starts_with takes s, prefix returns r
let n = sdsl_len(prefix)
if sdsl_len(s) < n then
return false
else
return substr(s, 0, n) == prefix
end
end
# `s[open_brace_pos]` must be "{". Returns [inner_body, index_just_past_matching_"}"],
# tracking string literals so braces inside "..." don't confuse the balance count.
make a function called sdsl_take_balanced_block takes s, open_brace_pos returns r
let slen = sdsl_len(s)
let depth = 0
let i = open_brace_pos
let in_string = false
let start_inner = open_brace_pos + 1
let scanning = true
let result_body = ""
let result_pos = -1
while (i < slen) and scanning do
let c = substr(s, i, 1)
if in_string then
if c == "\\" then
let i = i + 2
else
if c == "\"" then
let in_string = false
end
let i = i + 1
end
else
if c == "\"" then
let in_string = true
let i = i + 1
else
if c == "{" then
let depth = depth + 1
let i = i + 1
else
if c == "}" then
let depth = depth - 1
if depth == 0 then
let result_body = substr(s, start_inner, i - start_inner)
let result_pos = i + 1
let scanning = false
else
let i = i + 1
end
else
let i = i + 1
end
end
end
end
end
return [result_body, result_pos]
end
# Splits a `syntax`/`rule`/`tokens` body into ";"-terminated statements,
# respecting string literals and nested parens (so `regex("a;b", x)` isn't
# split in the middle).
make a function called sdsl_split_top_level_statements takes body returns r
let n = sdsl_len(body)
let out = []
let cur = ""
let depth = 0
let in_string = false
let i = 0
while i < n do
let c = substr(body, i, 1)
if in_string then
let cur = cur + c
if (c == "\\") and ((i + 1) < n) then
let cur = cur + substr(body, i + 1, 1)
let i = i + 2
else
if c == "\"" then
let in_string = false
end
let i = i + 1
end
else
if c == "\"" then
let in_string = true
let cur = cur + c
let i = i + 1
else
if c == "(" then
let depth = depth + 1
let cur = cur + c
let i = i + 1
else
if c == ")" then
let depth = depth - 1
let cur = cur + c
let i = i + 1
else
if (c == ";") and (depth == 0) then
let out = list_push(out, cur)
let cur = ""
let i = i + 1
else
let cur = cur + c
let i = i + 1
end
end
end
end
end
end
if sdsl_len(sdsl_trim(cur)) > 0 then
let out = list_push(out, cur)
end
return out
end
# ---------------------------------------------------------------------
# Pass 1: find and remove `syntax NAME { ... }` blocks, building a list of
# SyntaxDefs keyed (by linear scan) on each definition's trigger keyword.
# ---------------------------------------------------------------------
make a function called sdsl_parse_trigger_stmt takes stmt returns r
let s = sdsl_trim(stmt)
if sdsl_starts_with(s, "trigger") then
let s = sdsl_trim(substr(s, 7, sdsl_len(s) - 7))
end
if sdsl_starts_with(s, ":") then
let s = sdsl_trim(substr(s, 1, sdsl_len(s) - 1))
end
if sdsl_starts_with(s, "Keyword") then
let s = sdsl_trim(substr(s, 7, sdsl_len(s) - 7))
end
let n = sdsl_len(s)
if (n >= 2) and (substr(s, 0, 1) == "(") and (substr(s, n - 1, 1) == ")") then
let s = substr(s, 1, n - 2)
end
return sdsl_unquote(s)
end
make a function called sdsl_parse_token_defs takes tbody returns r
let stmts = sdsl_split_top_level_statements(tbody)
let out = []
let i = 0
while i < stmts.length do
let stmt = sdsl_trim(stmts[i])
if sdsl_len(stmt) > 0 then
let eq = sdsl_find_char(stmt, 0, "=")
let lhs = sdsl_trim(substr(stmt, 0, eq))
let rhs = sdsl_trim(substr(stmt, eq + 1, sdsl_len(stmt) - eq - 1))
let op = sdsl_find_char(lhs, 0, "(")
let name = lhs
if op >= 0 then
let name = sdsl_trim(substr(lhs, 0, op))
end
if sdsl_starts_with(rhs, "regex") then
let inner = sdsl_trim(substr(rhs, 5, sdsl_len(rhs) - 5))
let ilen = sdsl_len(inner)
if (ilen >= 2) and (substr(inner, 0, 1) == "(") and (substr(inner, ilen - 1, 1) == ")") then
let inner = substr(inner, 1, ilen - 2)
end
let comma = sdsl_find_char(inner, 0, ",")
let pat_str = inner
if comma >= 0 then
let pat_str = substr(inner, 0, comma)
end
let out = list_push(out, ["token", name, "regex", sdsl_unquote(pat_str)])
else
let out = list_push(out, ["token", name, "literal", sdsl_unquote(rhs)])
end
end
let i = i + 1
end
return out
end
make a function called sdsl_parse_rule_def takes rbody returns r
let stmts = sdsl_split_top_level_statements(rbody)
let steps = []
let return_template = ""
let i = 0
while i < stmts.length do
let stmt = sdsl_trim(stmts[i])
if sdsl_len(stmt) > 0 then
if sdsl_starts_with(stmt, "return") then
let return_template = sdsl_trim(substr(stmt, 6, sdsl_len(stmt) - 6))
else
let bind = ""
let expect_part = stmt
if sdsl_starts_with(stmt, "let") then
let rest = sdsl_trim(substr(stmt, 3, sdsl_len(stmt) - 3))
let eq = sdsl_find_char(rest, 0, "=")
let bind = sdsl_trim(substr(rest, 0, eq))
let expect_part = sdsl_trim(substr(rest, eq + 1, sdsl_len(rest) - eq - 1))
end
if sdsl_starts_with(expect_part, "expect") then
let expect_part = sdsl_trim(substr(expect_part, 6, sdsl_len(expect_part) - 6))
end
if sdsl_starts_with(expect_part, "Symbol") then
let inner = sdsl_trim(substr(expect_part, 6, sdsl_len(expect_part) - 6))
let ilen = sdsl_len(inner)
if (ilen >= 2) and (substr(inner, 0, 1) == "(") and (substr(inner, ilen - 1, 1) == ")") then
let inner = substr(inner, 1, ilen - 2)
end
let steps = list_push(steps, ["step", "symbol", sdsl_unquote(inner), bind])
else
if expect_part == "Identifier" then
let steps = list_push(steps, ["step", "identifier", "", bind])
else
let steps = list_push(steps, ["step", "named", expect_part, bind])
end
end
end
end
let i = i + 1
end
return ["rule", steps, return_template]
end
make a function called sdsl_parse_syntax_def takes body returns r
let n = sdsl_len(body)
let i = sdsl_skip_ws_and_comments(body, 0)
let trigger_keyword = ""
let tokens = []
let rules = []
while i < n do
if sdsl_starts_keyword_at(body, i, "trigger") then
let semi = sdsl_find_char(body, i, ";")
let trigger_keyword = sdsl_parse_trigger_stmt(substr(body, i, semi - i))
let i = semi + 1
else
if sdsl_starts_keyword_at(body, i, "tokens") then
let j = sdsl_skip_ws(body, i + 6)
let br = sdsl_take_balanced_block(body, j)
let tokens = sdsl_parse_token_defs(br[0])
let i = br[1]
else
if sdsl_starts_keyword_at(body, i, "rule") then
let j = sdsl_skip_ws(body, i + 4)
let nameinfo = sdsl_take_identifier(body, j)
let j = sdsl_skip_ws(body, nameinfo[1])
let br = sdsl_take_balanced_block(body, j)
let rules = list_push(rules, sdsl_parse_rule_def(br[0]))
let i = br[1]
else
let i = i + 1
end
end
end
let i = sdsl_skip_ws_and_comments(body, i)
end
return ["syntax", trigger_keyword, tokens, rules]
end
make a function called sdsl_extract_syntax_defs takes src returns r
let n = sdsl_len(src)
let out = ""
let defs = []
let i = 0
while i < n do
if substr(src, i, 1) == "#" then
# Copy the whole comment line through untouched -- a comment may
# itself mention "syntax" or "routes" (e.g. this file's own header,
# or a demo's usage-example doc comment) without being a real block.
let cstart = i
while (i < n) and (substr(src, i, 1) != "\n") do
let i = i + 1
end
let out = out + substr(src, cstart, i - cstart)
else
if sdsl_starts_keyword_at(src, i, "syntax") then
let j = sdsl_skip_ws(src, i + 6)
let nameinfo = sdsl_take_identifier(src, j)
let j = sdsl_skip_ws(src, nameinfo[1])
let br = sdsl_take_balanced_block(src, j)
let def = sdsl_parse_syntax_def(br[0])
let defs = list_push(defs, def)
let i = br[1]
else
let out = out + substr(src, i, 1)
let i = i + 1
end
end
end
return [out, defs]
end
# ---------------------------------------------------------------------
# Pass 2: expand every `<trigger keyword> { ... }` block found in the
# (already syntax-def-stripped) source.
# ---------------------------------------------------------------------
make a function called sdsl_match_literal takes s, pos, lit returns r
let n = sdsl_len(lit)
if (pos + n) > sdsl_len(s) then
return -1
else
if substr(s, pos, n) == lit then
return pos + n
else
return -1
end
end
end
make a function called sdsl_match_identifier takes s, pos returns r
let n = sdsl_len(s)
if pos >= n then
return -1
else
if not sdsl_is_ident_start(substr(s, pos, 1)) then
return -1
else
let i = pos + 1
let scanning = true
while (i < n) and scanning do
if sdsl_is_ident_char(substr(s, i, 1)) then
let i = i + 1
else
let scanning = false
end
end
return i
end
end
end
# Finds `name` inside a TokenDef list; returns the TokenDef or ["token", "", "", ""] if absent.
make a function called sdsl_find_token_def takes tokens, name returns r
let n = tokens.length
let i = 0
let found = ["token", "", "", ""]
let scanning = true
while (i < n) and scanning do
let t = tokens[i]
if t[1] == name then
let found = t
let scanning = false
end
let i = i + 1
end
return found
end
# Finds `value` inside a [[name, value], ...] bindings list; returns "" if absent.
make a function called sdsl_bindings_lookup takes bindings, name returns r
let n = bindings.length
let i = 0
let found = ""
let scanning = true
while (i < n) and scanning do
let b = bindings[i]
if b[0] == name then
let found = b[1]
let scanning = false
end
let i = i + 1
end
return found
end
# Attempts to match every step of `rule` against `line` in order. Returns
# ["ok", bindings] on full success, ["fail"] if any step fails to match.
make a function called sdsl_try_match_rule takes def, rule, line returns r
let steps = rule[1]
let pos = 0
let bindings = []
let n = steps.length
let i = 0
let ok = true
while (i < n) and ok do
let pos = sdsl_skip_ws(line, pos)
let step = steps[i]
let kind = step[1]
let matched = -1
if kind == "identifier" then
let matched = sdsl_match_identifier(line, pos)
else
if kind == "symbol" then
let matched = sdsl_match_literal(line, pos, step[2])
else
let tdef = sdsl_find_token_def(def[2], step[2])
if tdef[2] == "literal" then
let matched = sdsl_match_literal(line, pos, tdef[3])
else
if tdef[2] == "regex" then
let text = substr(line, pos, sdsl_len(line) - pos)
let end_off = regex_match_string_at(tdef[3], text, 0)
if end_off > 0 then
let matched = pos + end_off
end
end
end
end
end
if matched < 0 then
let ok = false
else
let bind = step[3]
if sdsl_len(bind) > 0 then
let bindings = list_push(bindings, [bind, substr(line, pos, matched - pos)])
end
let pos = matched
end
let i = i + 1
end
if ok then
return ["ok", bindings]
else
return ["fail"]
end
end
# Replaces every whole-word occurrence of a bound capture name in `template`
# with its matched text as a PatLang string literal.
make a function called sdsl_substitute_template takes template, bindings returns r
let n = sdsl_len(template)
let out = ""
let i = 0
while i < n do
let c = substr(template, i, 1)
if sdsl_is_ident_start(c) then
let j = i + 1
let scanning = true
while (j < n) and scanning do
if sdsl_is_ident_char(substr(template, j, 1)) then
let j = j + 1
else
let scanning = false
end
end
let word = substr(template, i, j - i)
let val = sdsl_bindings_lookup(bindings, word)
if sdsl_len(val) > 0 then
let out = out + "\"" + val + "\""
else
let out = out + word
end
let i = j
else
let out = out + c
let i = i + 1
end
end
return out
end
# Runs each of the DSL's rules (in declaration order) against `line`,
# returning the first one whose full `expect` sequence matches, with its
# return template's captures substituted in as PatLang string literals.
make a function called sdsl_expand_rule_line takes def, line returns r
let rules = def[3]
let n = rules.length
let i = 0
let result = ""
let found = false
while (i < n) and (not found) do
let m = sdsl_try_match_rule(def, rules[i], line)
if m[0] == "ok" then
# No trailing ";" -- PatLang has no statement-terminating semicolon
# anywhere in its grammar. A stray ";" tokenizes as its own UNK token
# and becomes a parse error; that error was invisible when this output
# only ever went through compile_native (lower_program silently drops
# unparseable Err statements), but is fatal for the playground/IDE
# path, which aborts entirely if any parse error is present.
let result = sdsl_substitute_template(rules[i][2], m[1])
let found = true
end
let i = i + 1
end
return result
end
make a function called sdsl_expand_trigger_blocks takes src, defs returns r
let n = sdsl_len(src)
let out = ""
let i = 0
while i < n do
if substr(src, i, 1) == "#" then
# Same rationale as sdsl_extract_syntax_defs: a comment mentioning a
# trigger keyword (e.g. this file's own doc comments about `routes`)
# is not a real usage block, and must never be scanned into one.
let cstart = i
while (i < n) and (substr(src, i, 1) != "\n") do
let i = i + 1
end
let out = out + substr(src, cstart, i - cstart)
else
let kw_idx = -1
let d = 0
while (d < defs.length) and (kw_idx < 0) do
if sdsl_starts_keyword_at(src, i, defs[d][1]) then
let kw_idx = d
end
let d = d + 1
end
if kw_idx >= 0 then
let def = defs[kw_idx]
let j = sdsl_skip_ws(src, i + sdsl_len(def[1]))
if (j < n) and (substr(src, j, 1) == "{") then
let br = sdsl_take_balanced_block(src, j)
let body = br[0]
let blen = sdsl_len(body)
let start = 0
while start <= blen do
let nl = sdsl_find_char(body, start, "\n")
let line_end = nl
if nl < 0 then
let line_end = blen
end
let line = sdsl_trim(substr(body, start, line_end - start))
if sdsl_len(line) > 0 then
let out = out + sdsl_expand_rule_line(def, line) + "\n"
end
if nl < 0 then
let start = blen + 1
else
let start = nl + 1
end
end
let i = br[1]
else
let out = out + substr(src, i, 1)
let i = i + 1
end
else
let out = out + substr(src, i, 1)
let i = i + 1
end
end
end
return out
end
# Public entry point, composable across multiple separate runtime calls:
# expands every `syntax NAME { ... }` definition and every triggered block
# in `src`, using both `src`'s own definitions AND `existing_defs` (defs
# already extracted from earlier, separate `expand_syntax_dsls_with` calls
# -- e.g. a REPL-style session where one snippet defines a syntax and a
# LATER, separately-submitted snippet uses it). Returns [stripped, all_defs]
# so a caller can thread `all_defs` into its next call and keep
# accumulating, rather than every dynamic snippet needing to redeclare (or
# be concatenated with) every syntax it wants to use.
make a function called expand_syntax_dsls_with takes src, existing_defs returns r
let extracted = sdsl_extract_syntax_defs(src)
let stripped = extracted[0]
let new_defs = extracted[1]
let all_defs = existing_defs
let i = 0
while i < new_defs.length do
let all_defs = list_push(all_defs, new_defs[i])
let i = i + 1
end
if all_defs.length == 0 then
return [stripped, all_defs]
else
return [sdsl_expand_trigger_blocks(stripped, all_defs), all_defs]
end
end
# Public entry point: expands every `syntax NAME { ... }` definition and
# every triggered block in `src`. Thin wrapper over `expand_syntax_dsls_with`
# with no prior defs -- every existing call site (playground_main.patlang,
# build_portfolio.patlang's dsl_driver) is unaffected by the composable
# variant above.
make a function called expand_syntax_dsls takes src returns r
let r = expand_syntax_dsls_with(src, [])
return r[0]
end
# Dynamic syntax demo: proves `syntax NAME { ... }` DSL definitions are a
# genuine RUNTIME capability, not just a compile-time convenience. Every
# other syntax-DSL example in this portfolio (router_dsl_demo.patlang) has
# its `syntax { ... }` block written as a source-code literal, expanded
# exactly once, before the ordinary compiler ever runs -- indistinguishable
# from a macro. This demo instead:
#
# 1. Computes the DSL's trigger keyword from runtime data (not a literal
# in this file) -- the grammar extension genuinely could not have been
# known until the program executed.
# 2. Defines the syntax via `expand_syntax_dsls_with(src, [])` in one call,
# keeping only the returned `defs` -- the definition text itself is
# discarded immediately afterward.
# 3. Uses that syntax in a SEPARATE, later `expand_syntax_dsls_with(usage,
# defs)` call, on a string that contains ONLY the usage line -- no
# syntax declaration anywhere in it. This is the composability case:
# the two calls could be arbitrarily far apart (different functions,
# different point in the program, even a different network request in
# a real system), carrying only `defs` -- a plain list value -- between
# them, exactly like passing any other piece of program state.
# 4. Concatenates both expanded fragments and runs the RESULT through the
# ordinary tokenize/parse_program/lower_program/run_ir pipeline --
# itself just plain PatLang function calls, no special "eval" host
# function required.
#
# NOTE on style: every string built here uses one `let` per concatenation
# step (`let s2 = s1 + "..."`), never a multi-line `+`-continuation
# expression (`"a" + chr(10)` then `+ "b"` starting the NEXT line). The
# self-hosted parser's parse_add/parse_mul (unlike the native Rust frontend,
# and unlike this same file's own list/argument parsing, which explicitly
# skip newlines) do not skip newlines before checking for a continuation
# operator, so a leading `+` on a new line silently truncates the
# expression right there -- a real, previously-latent gap in
# self_hosting/lib/parser.patlang, only surfaced now because this is the
# first self-hosted-pipeline-compiled program to build a string this way.
#
# Concatenate after lib/lexer.patlang + lib/parser.patlang + lib/lower.patlang
# + lib/regex_dsl.patlang + lib/syntax_dsl.patlang when compiling (matches
# playground_main.patlang's own concatenation -- this demo needs
# tokenize/parse_program/lower_program/expand_syntax_dsls_with in scope to
# call them on data it constructs itself, exactly like the live IDE does on
# whatever a user types).
# --- Step 0: pick the trigger keyword from runtime data, not a literal ---
let candidates = ["greet", "salute", "hail"]
let pick = (3 * 5) % candidates.length
let trigger_word = candidates[pick]
print("chosen trigger keyword at runtime: " + trigger_word)
# --- Step 1: define the syntax from a string built with that keyword ---
let d1 = "make a function called say_hello takes name returns done" + chr(10)
let d2 = d1 + " print(" + chr(34) + "Hello, " + chr(34) + " + name + " + chr(34) + "!" + chr(34) + ")" + chr(10)
let d3 = d2 + " return true" + chr(10)
let d4 = d3 + "end" + chr(10)
let d5 = d4 + chr(10)
let d6 = d5 + "syntax GreetDSL {" + chr(10)
let d7 = d6 + " trigger: Keyword(" + chr(34) + trigger_word + chr(34) + ");" + chr(10)
let d8 = d7 + " tokens {" + chr(10)
let d9 = d8 + " }" + chr(10)
let d10 = d9 + " rule GreetLine {" + chr(10)
let d11 = d10 + " let name = expect Identifier;" + chr(10)
let d12 = d11 + " return say_hello(name);" + chr(10)
let d13 = d12 + " }" + chr(10)
let def_src = d13 + "}" + chr(10)
let step1 = expand_syntax_dsls_with(def_src, [])
let program_prefix = step1[0]
let defs = step1[1]
print("syntax defined; " + defs.length + " grammar extension(s) known so far")
# --- Step 2: a LATER, separate call sees ONLY the usage text -- no syntax
# declaration in sight -- yet still expands correctly, because `defs`
# carried forward from step 1. ---
let u1 = trigger_word + " {" + chr(10)
let u2 = u1 + " World" + chr(10)
let usage_src = u2 + "}" + chr(10)
let step2 = expand_syntax_dsls_with(usage_src, defs)
let program_suffix = step2[0]
# --- Step 3: run the assembled program through the ordinary pipeline ---
let p1 = program_prefix + chr(10)
let p2 = p1 + program_suffix
let full_program = p2 + "print(" + chr(34) + "dynamic syntax demo complete." + chr(34) + ")" + chr(10)
let toks = tokenize(full_program)
let ast = parse_program(toks)
let ir = lower_program(ast)
run_ir(ir)
Native run on the build machine:
chosen trigger keyword at runtime: greet syntax defined; 1 grammar extension(s) known so far Hello, World! dynamic syntax demo complete.