scan

pub func scan(source: String) -> ScanResult {
	let tokens: [Token] = []
	let comments: [Comment] = []
	let error: String? = Optional.none
	let error_pos = 0
	let view = source.utf8()
	let n = view.count()
	let i = 0
	loop i < n {
		let b = view.at(index: i)._toInt()
		// Newline runs collapse into one token.
		if b == newline_byte {
			let start = i
			loop i < n && view.at(index: i)._toInt() == newline_byte {
				i = i + 1
			}
			tokens.push(token(kind: .newline, start: start, end: i))
			continue
		}
		if b == space_byte || b == tab_byte || b == return_byte {
			i = i + 1
			continue
		}
		// Line comments drop from the token stream; their spans are
		// preserved for the parse dump's comments section.
		if b == slash_byte && i + 1 < n && view.at(index: i + 1)._toInt() == slash_byte {
			let start = i
			loop i < n && view.at(index: i)._toInt() != newline_byte {
				i = i + 1
			}
			comments.push(Comment(start: start, end: i))
			continue
		}
		if is_digit(b: b) {
			let start = i
			loop i < n && (is_digit(b: view.at(index: i)._toInt()) || view.at(index: i)._toInt() == underscore_byte) {
				i = i + 1
			}
			// One dot then a digit continues a Float; two dots are a
			// range operator after an Int. `_` separators ride along.
			let fractional = i + 1 < n && view.at(index: i)._toInt() == dot_byte && is_digit(b: view.at(index: i + 1)._toInt())
			if fractional {
				i = i + 1
				loop i < n && (is_digit(b: view.at(index: i)._toInt()) || view.at(index: i)._toInt() == underscore_byte) {
					i = i + 1
				}
				tokens.push(token(kind: .float_number, start: start, end: i))
			} else {
				tokens.push(token(kind: .int_number, start: start, end: i))
			}
			continue
		}
		if is_alpha(b: b) {
			let start = i
			i = ident_end(view: view, n: n, i: i)
			let word = view.slice(start: start, byte_count: i - start)
			let kind = match word {
				"_" -> TokenKind.underscore,
				"any" -> TokenKind.keyword_any,
				"as" -> TokenKind.keyword_as,
				"func" -> TokenKind.keyword_func,
				"let" -> TokenKind.keyword_let,
				"if" -> TokenKind.keyword_if,
				"else" -> TokenKind.keyword_else,
				"true" -> TokenKind.keyword_true,
				"false" -> TokenKind.keyword_false,
				"loop" -> TokenKind.keyword_loop,
				"enum" -> TokenKind.keyword_enum,
				"case" -> TokenKind.keyword_case,
				"match" -> TokenKind.keyword_match,
				"return" -> TokenKind.keyword_return,
				"struct" -> TokenKind.keyword_struct,
				"extend" -> TokenKind.keyword_extend,
				"break" -> TokenKind.keyword_break,
				"init" -> TokenKind.keyword_init,
				"protocol" -> TokenKind.keyword_protocol,
				"import" -> TokenKind.keyword_import,
				"use" -> TokenKind.keyword_use,
				"pub" -> TokenKind.keyword_pub,
				"public" -> TokenKind.keyword_public,
				"linear" -> TokenKind.keyword_linear,
				"macro" -> TokenKind.keyword_macro,
				"static" -> TokenKind.keyword_static,
				"associated" -> TokenKind.keyword_associated,
				"typealias" -> TokenKind.keyword_typealias,
				"effect" -> TokenKind.keyword_effect,
				"handling" -> TokenKind.keyword_handling,
				"in" -> TokenKind.keyword_in,
				"continue" -> TokenKind.keyword_continue,
				"unreachable" -> TokenKind.keyword_unreachable,
				"mut" -> TokenKind.keyword_mut,
				"consuming" -> TokenKind.keyword_consuming,
				"for" -> TokenKind.keyword_for,
				_ -> TokenKind.identifier
			}
			tokens.push(token(kind: kind, start: start, end: i))
			continue
		}
		// String literal: span includes the quotes; escapes stay raw.
		if b == quote_byte {
			let start = i
			i = i + 1
			let open = true
			let bad = false
			loop i < n && open && bad == false {
				let c = view.at(index: i)._toInt()
				if c == backslash_byte {
					let escape = scan_escape(view: view, n: n, after: i + 1, allow_tick: false, unterminated: "Unterminated string")
					if let .failed(message, failed_at) = escape {
						error = Optional.some(message)
						error_pos = failed_at
						bad = true
					}
					if let .fine(next) = escape {
						i = next
					}
				} else {
					if c == quote_byte {
						i = i + 1
						open = false
					} else {
						i = i + 1
					}
				}
			}
			if bad {
				break
			}
			if open {
				error = Optional.some("Unterminated string")
				error_pos = i
				break
			}
			tokens.push(token(kind: .string_literal, start: start, end: i))
			continue
		}
		// A tick opens an effect name (`'io`), an effect row (`'[…]`),
		// or a character literal (`'a'`). An identifier run not closed
		// by another tick commits to an effect name; otherwise a probe
		// decides without moving the scan.
		if b == tick_byte {
			let start = i
			let effectish = i + 1 < n && is_alnum(b: view.at(index: i + 1)._toInt())
			if effectish {
				let run_end = ident_end(view: view, n: n, i: i + 1)
				let closed = run_end < n && view.at(index: run_end)._toInt() == tick_byte
				if closed == false {
					tokens.push(token_with_lexeme(kind: .effect_name, start: i, end: run_end, lexeme_start: i + 1, lexeme_end: run_end))
					i = run_end
					continue
				}
			}
			let row_opener = false
			if i + 1 < n && view.at(index: i + 1)._toInt() == left_bracket_byte {
				let j = i + 1
				loop j < n && row_opener == false {
					let c = view.at(index: j)._toInt()
					if c == right_bracket_byte {
						row_opener = true
					}
					if c == tick_byte || c == newline_byte {
						j = n
					}
					j = j + 1
				}
			}
			if row_opener {
				// The row's tick stands alone; `[` lexes next round.
				tokens.push(token(kind: .single_quote, start: i, end: i + 1))
				i = i + 1
				continue
			}
			let probe = probe_character(view: view, n: n, after_tick: i + 1)
			if let .failed(message, failed_at) = probe {
				error = Optional.some(message)
				error_pos = failed_at
				break
			}
			if let .literal(end) = probe {
				tokens.push(token(kind: .character_literal, start: start, end: end))
				i = end
				continue
			}
			// Not a literal: a bare tick keeps its own span.
			tokens.push(token(kind: .single_quote, start: i, end: i + 1))
			i = i + 1
			continue
		}
		// Sigil-prefixed forms keep the complete spelling in span and the
		// value after the sigil in lexeme.
		if b == percent_byte && i + 1 < n && (is_digit(b: view.at(index: i + 1)._toInt()) || view.at(index: i + 1)._toInt() == question_byte) {
			let register_start = i + 1
			let j = register_start
			loop j < n && (is_digit(b: view.at(index: j)._toInt()) || view.at(index: j)._toInt() == question_byte) {
				j = j + 1
			}
			tokens.push(token_with_lexeme(kind: .ir_register, start: i, end: j, lexeme_start: register_start, lexeme_end: j))
			i = j
			continue
		}
		// #"..." spells a literal identifier, letting keywords be used
		// as names. Its lexeme excludes the # and the quotes.
		if b == hash_byte && i + 1 < n && view.at(index: i + 1)._toInt() == quote_byte {
			let start = i + 2
			let j = start
			let open = true
			let bad = false
			loop j < n && open {
				let c = view.at(index: j)._toInt()
				if c == newline_byte { break }
				if c == backslash_byte {
					error = Optional.some("Invalid escape: '\\\\'")
					error_pos = j
					bad = true
					break
				}
				if c == quote_byte {
					open = false
				} else {
					j = j + 1
				}
			}
			if bad { break }
			if open {
				error = Optional.some("Unterminated string")
				error_pos = j
				break
			}
			if j == start {
				error = Optional.some("Empty quoted identifier")
				error_pos = j
				break
			}
			tokens.push(token_with_lexeme(kind: .identifier, start: i, end: j + 1, lexeme_start: start, lexeme_end: j))
			i = j + 1
			continue
		}
		if b == at_byte && i + 1 < n && is_alnum(b: view.at(index: i + 1)._toInt()) {
			let end = ident_end(view: view, n: n, i: i + 1)
			tokens.push(token_with_lexeme(kind: .attribute, start: i, end: end, lexeme_start: i + 1, lexeme_end: end))
			i = end
			continue
		}
		if b == dollar_byte && i + 1 < n && is_alnum(b: view.at(index: i + 1)._toInt()) {
			let end = ident_end(view: view, n: n, i: i + 1)
			tokens.push(token_with_lexeme(kind: .bound_var, start: i, end: end, lexeme_start: i + 1, lexeme_end: end))
			i = end
			continue
		}
		if b == dollar_byte {
			tokens.push(token(kind: .dollar, start: i, end: i + 1))
			i = i + 1
			continue
		}
		if b == hash_byte {
			tokens.push(token(kind: .hash, start: i, end: i + 1))
			i = i + 1
			continue
		}
		if b == dot_byte {
			if i + 1 < n && view.at(index: i + 1)._toInt() == dot_byte {
				if i + 2 < n && view.at(index: i + 2)._toInt() == dot_byte {
					tokens.push(token(kind: .dot_dot_dot, start: i, end: i + 3))
					i = i + 3
				} else if i + 2 < n && view.at(index: i + 2)._toInt() == less_byte {
					tokens.push(token(kind: .dot_dot_less, start: i, end: i + 3))
					i = i + 3
				} else {
					tokens.push(token(kind: .dot_dot, start: i, end: i + 2))
					i = i + 2
				}
			} else {
				tokens.push(token(kind: .dot, start: i, end: i + 1))
				i = i + 1
			}
			continue
		}
		// Two-character operators, longest spelling first.
		if i + 1 < n {
			let pair = view.slice(start: i, byte_count: 2)
			let two = match pair {
				"==" -> TokenKind.equals_equals,
				"!=" -> TokenKind.bang_equals,
				"<=" -> TokenKind.less_equals,
				">=" -> TokenKind.greater_equals,
				"+=" -> TokenKind.plus_equals,
				"-=" -> TokenKind.minus_equals,
				"*=" -> TokenKind.star_equals,
				"/=" -> TokenKind.slash_equals,
				"~=" -> TokenKind.tilde_equals,
				"^=" -> TokenKind.caret_equals,
				"&=" -> TokenKind.amp_equals,
				"->" -> TokenKind.arrow,
				"<<" -> TokenKind.less_less,
				">>" -> TokenKind.greater_greater,
				"&&" -> TokenKind.amp_amp,
				"||" -> TokenKind.pipe_pipe,
				"::" -> TokenKind.double_colon,
				_ -> TokenKind.identifier
			}
			if is_operator(kind: two) {
				tokens.push(token(kind: two, start: i, end: i + 2))
				i = i + 2
				continue
			}
		}
		let glyph = view.slice(start: i, byte_count: 1)
		let one = match glyph {
			"=" -> TokenKind.equals,
			"," -> TokenKind.comma,
			"(" -> TokenKind.left_paren,
			")" -> TokenKind.right_paren,
			"[" -> TokenKind.left_bracket,
			"]" -> TokenKind.right_bracket,
			"{" -> TokenKind.left_brace,
			"}" -> TokenKind.right_brace,
			"?" -> TokenKind.question_mark,
			";" -> TokenKind.semicolon,
			"@" -> TokenKind.at_sign,
			"%" -> TokenKind.percent,
			"+" -> TokenKind.plus,
			"-" -> TokenKind.minus,
			"*" -> TokenKind.star,
			"/" -> TokenKind.slash,
			"!" -> TokenKind.bang,
			"<" -> TokenKind.less,
			">" -> TokenKind.greater,
			"&" -> TokenKind.amp,
			"|" -> TokenKind.pipe,
			":" -> TokenKind.colon,
			"~" -> TokenKind.tilde,
			"^" -> TokenKind.caret,
			_ -> TokenKind.identifier
		}
		if is_operator(kind: one) {
			tokens.push(token(kind: one, start: i, end: i + 1))
			i = i + 1
			continue
		}
		// Anything else is a lex error, like the reference lexer, which
		// consumes the character before erroring.
		let unexpected = if b < 128 {
			glyph.to_string()
		} else {
			describe_scalar(view: view, n: n, i: i)
		}
		error = Optional.some("Unexpected character: '" + unexpected + "'")
		error_pos = i + scalar_byte_length(b: b)
		break
	}
	ScanResult(tokens: tokens, comments: comments, error: error, error_pos: error_pos)
}