func _utf8_decode(storage: ByteStorage, offset: Int, end: Int) -> IntDecodes one UTF-8 scalar and packs its value with the consumed byte count. Malformed input yields U+FFFD and consumes one maximal invalid subpart.
View Source
pub func _utf8_decode(storage: ByteStorage, offset: Int, end: Int) -> Int {
let replacement: Int = 65533
let b0: Int = storage.get(offset)._toInt()
// 0xxxxxxx: ASCII.
if b0 < 128 { return b0 * 8 + 1 }
// Continuation bytes and the overlong leads C0/C1 cannot start a
// character.
if b0 < 194 { return replacement * 8 + 1 }
// 110xxxxx 10xxxxxx: leads C2..DF.
if b0 < 224 {
if end <= offset + 1 { return replacement * 8 + 1 }
let b1: Int = storage.get(offset + 1)._toInt()
if _is_continuation(b: b1) == false { return replacement * 8 + 1 }
let scalar: Int = (b0 - 192) * 64 + (b1 - 128)
return scalar * 8 + 2
}
// 1110xxxx 10xxxxxx 10xxxxxx: leads E0..EF. E0 narrows the first
// continuation to A0..BF (no overlongs); ED narrows it to 80..9F
// (no surrogates).
if b0 < 240 {
if end <= offset + 1 { return replacement * 8 + 1 }
let b1: Int = storage.get(offset + 1)._toInt()
let lo: Int = if b0 == 224 {
160
} else {
128
}
let hi: Int = if b0 == 237 {
159
} else {
191
}
if b1 < lo { return replacement * 8 + 1 }
if hi < b1 { return replacement * 8 + 1 }
if end <= offset + 2 { return replacement * 8 + 2 }
let b2: Int = storage.get(offset + 2)._toInt()
if _is_continuation(b: b2) == false { return replacement * 8 + 2 }
let scalar: Int = (b0 - 224) * 4096 + (b1 - 128) * 64 + (b2 - 128)
return scalar * 8 + 3
}
// 11110xxx + 3 continuations: leads F0..F4. F0 narrows the first
// continuation to 90..BF (no overlongs); F4 narrows it to 80..8F
// (nothing above U+10FFFF).
if b0 < 245 {
if end <= offset + 1 { return replacement * 8 + 1 }
let b1: Int = storage.get(offset + 1)._toInt()
let lo: Int = if b0 == 240 {
144
} else {
128
}
let hi: Int = if b0 == 244 {
143
} else {
191
}
if b1 < lo { return replacement * 8 + 1 }
if hi < b1 { return replacement * 8 + 1 }
if end <= offset + 2 { return replacement * 8 + 2 }
let b2: Int = storage.get(offset + 2)._toInt()
if _is_continuation(b: b2) == false { return replacement * 8 + 2 }
if end <= offset + 3 { return replacement * 8 + 3 }
let b3: Int = storage.get(offset + 3)._toInt()
if _is_continuation(b: b3) == false { return replacement * 8 + 3 }
let scalar: Int = (b0 - 240) * 262144 + (b1 - 128) * 4096 + (b2 - 128) * 64 + (b3 - 128)
return scalar * 8 + 4
}
// F5..FF: never valid leads.
replacement * 8 + 1
}