lex - syntax

func lex(source: String) -> String

Scans source and renders the frozen token validation format.

Output includes token kinds, half-open byte spans, eligible snippets, and the first scanner error. This is a debugging/compatibility view; use scan for structured tokens.

View Source

pub func lex(source: String) -> String {
	let result = scan(source: source)
	let view = source.utf8()
	let out: [Byte] = []
	append(buf: out, text: "tokens:\n")
	for token in result.tokens {
		append(buf: out, text: "  ")
		append(buf: out, text: kind_name(kind: token.kind))
		append(buf: out, text: " @")
		append_int(buf: out, value: token.span.lower)
		append(buf: out, text: "..")
		append_int(buf: out, value: token.span.upper)
		append_snippet(buf: out, view: view, start: token.span.lower, end: token.span.upper)
		append(buf: out, text: "\n")
	}
	if let .some(message) = result.error {
		append(buf: out, text: "  lexer error: ")
		append(buf: out, text: message)
		append(buf: out, text: "\n")
	}
	String.from_bytes(bytes: out)
}