_utf8_decode

func _utf8_decode(storage: ByteStorage, offset: Int, end: Int) -> Int

Decodes one UTF-8 scalar and packs its value with the consumed byte count. Malformed input yields U+FFFD and consumes one maximal invalid subpart.

View Source
pub func _utf8_decode(storage: ByteStorage, offset: Int, end: Int) -> Int {
	let replacement: Int = 65533
	let b0: Int = storage.get(offset)._toInt()
	// 0xxxxxxx: ASCII.
	if b0 < 128 { return b0 * 8 + 1 }
	// Continuation bytes and the overlong leads C0/C1 cannot start a
	// character.
	if b0 < 194 { return replacement * 8 + 1 }
	// 110xxxxx 10xxxxxx: leads C2..DF.
	if b0 < 224 {
		if end <= offset + 1 { return replacement * 8 + 1 }
		let b1: Int = storage.get(offset + 1)._toInt()
		if _is_continuation(b: b1) == false { return replacement * 8 + 1 }
		let scalar: Int = (b0 - 192) * 64 + (b1 - 128)
		return scalar * 8 + 2
	}
	// 1110xxxx 10xxxxxx 10xxxxxx: leads E0..EF. E0 narrows the first
	// continuation to A0..BF (no overlongs); ED narrows it to 80..9F
	// (no surrogates).
	if b0 < 240 {
		if end <= offset + 1 { return replacement * 8 + 1 }
		let b1: Int = storage.get(offset + 1)._toInt()
		let lo: Int = if b0 == 224 {
			160
		} else {
			128
		}
		let hi: Int = if b0 == 237 {
			159
		} else {
			191
		}
		if b1 < lo { return replacement * 8 + 1 }
		if hi < b1 { return replacement * 8 + 1 }
		if end <= offset + 2 { return replacement * 8 + 2 }
		let b2: Int = storage.get(offset + 2)._toInt()
		if _is_continuation(b: b2) == false { return replacement * 8 + 2 }
		let scalar: Int = (b0 - 224) * 4096 + (b1 - 128) * 64 + (b2 - 128)
		return scalar * 8 + 3
	}
	// 11110xxx + 3 continuations: leads F0..F4. F0 narrows the first
	// continuation to 90..BF (no overlongs); F4 narrows it to 80..8F
	// (nothing above U+10FFFF).
	if b0 < 245 {
		if end <= offset + 1 { return replacement * 8 + 1 }
		let b1: Int = storage.get(offset + 1)._toInt()
		let lo: Int = if b0 == 240 {
			144
		} else {
			128
		}
		let hi: Int = if b0 == 244 {
			143
		} else {
			191
		}
		if b1 < lo { return replacement * 8 + 1 }
		if hi < b1 { return replacement * 8 + 1 }
		if end <= offset + 2 { return replacement * 8 + 2 }
		let b2: Int = storage.get(offset + 2)._toInt()
		if _is_continuation(b: b2) == false { return replacement * 8 + 2 }
		if end <= offset + 3 { return replacement * 8 + 3 }
		let b3: Int = storage.get(offset + 3)._toInt()
		if _is_continuation(b: b3) == false { return replacement * 8 + 3 }
		let scalar: Int = (b0 - 240) * 262144 + (b1 - 128) * 4096 + (b2 - 128) * 64 + (b3 - 128)
		return scalar * 8 + 4
	}
	// F5..FF: never valid leads.
	replacement * 8 + 1
}