func lex_tokens(source: String) -> [MetaToken]MARK: The lexing surface (ADR 0043 Stage 5)
Token-level editor queries (highlighting, delimiter matching, identifier validation) read the token stream itself. Comments re-enter here as line_comment tokens, merged by position; a scan failure appends the sentinel token (start -1) after the tokens produced up to the failure.
View Source
pub func lex_tokens(source: String) -> [MetaToken] {
let result = scan(source: source)
let merged: [Token] = []
let ti = 0
let ci = 0
let tokens = result.tokens
let comments = result.comments
loop ti < tokens.count || ci < comments.count {
let take_comment = false
if ci < comments.count {
if ti >= tokens.count {
take_comment = true
} else {
if comments[ci].start < tokens[ti].span.lower { take_comment = true }
}
}
if take_comment {
merged.push(Token(kind: .line_comment, span: comments[ci].start..<comments[ci].end, lexeme: comments[ci].start..<comments[ci].end))
ci = ci + 1
} else {
let tok = tokens[ti]
merged.push(Token(kind: tok.kind, span: tok.span, lexeme: tok.lexeme))
ti = ti + 1
}
}
let (lines, cols) = token_positions(source: source, tokens: merged)
let out: [MetaToken] = []
let i = 0
loop i < merged.count {
let tok = merged[i]
out.push(MetaToken(kind: tok.kind, start: tok.span.lower, end: tok.span.upper, line: lines[i], col: cols[i]))
i = i + 1
}
if let .some(_) = result.error {
out.push(MetaToken(kind: .eof, start: -1, end: -1, line: 0, col: 0))
}
out
}