capture_token_trees

func capture_token_trees(tokens: [Token]) -> TokenTreeResult

Capture a flat token stream into balanced trees. Delimiter failures are structured so a macro caller can attach diagnostics without parsing text.

View Source
pub func capture_token_trees(tokens: [Token]) -> TokenTreeResult {
	let stack: [Frame] = []
	let current: [TokenTree] = []
	let index = 0
	loop index < tokens.count {
		let token = tokens.get(index)
		index = index + 1
		if let .some(delimiter) = opens(kind: token.kind) {
			stack.push(Frame(opener: token, expected: delimiter, parent: current))
			current = empty_trees()
			continue
		}
		if let .some(delimiter) = closes(kind: token.kind) {
			let popped = stack.pop()
			if let .some(frame) = popped {
				if same_delimiter(a: frame.expected, b: delimiter) {
					let children = current
					current = frame.parent
					current.push(TokenTree.group(Group(delimiter: delimiter, open: frame.opener, close: token, children: children)))
					continue
				}
				return TokenTreeResult(trees: empty_trees(), error: Optional.some(TokenTreeError.mismatched_closer(token, frame.opener)))
			}
			return TokenTreeResult(trees: empty_trees(), error: Optional.some(TokenTreeError.unexpected_closer(token)))
		}
		current.push(TokenTree.leaf(token))
	}
	let leftover = stack.pop()
	if let .some(frame) = leftover {
		return TokenTreeResult(trees: empty_trees(), error: Optional.some(TokenTreeError.unclosed(frame.opener)))
	}
	TokenTreeResult(trees: current, error: Optional.none)
}