"""Reproduce UTF-8 chunk boundaries, end-of-input errors, and line framing."""
import codecs
import io
import json
import platform
import sys


def decode_chunks(chunks, encoding="utf-8", errors="strict"):
    decoder = codecs.getincrementaldecoder(encoding)(errors=errors)
    for chunk in chunks:
        yield decoder.decode(chunk, final=False)
    yield decoder.decode(b"", final=True)


def outcome(callback):
    try:
        return {"text": callback()}
    except UnicodeDecodeError as error:
        return {"error": error.reason, "bytes_hex": error.object[error.start:error.end].hex(" ")}


def parse_lines(fragments):
    pending = ""
    for fragment in fragments:
        pending += fragment
        fields = pending.split("\n")
        pending = fields.pop()
        for field in fields:
            yield json.loads(field)
    if pending:
        raise ValueError("expected a final newline")


def run():
    text = "A雨🍊B"
    payload = text.encode("utf-8")
    chunks = [payload[:3], payload[3:6], payload[6:]]
    parts = list(decode_chunks(chunks))
    separate = [chunk.decode("utf-8", errors="replace") for chunk in chunks]
    decoder = codecs.getincrementaldecoder("utf-8")()
    pending = []
    for chunk in chunks:
        decoder.decode(chunk)
        pending.append(decoder.getstate()[0].hex(" "))
    assert parts == ["A", "雨", "🍊B", ""]
    assert separate == ["A�", "��", "��B"]
    assert pending == ["e9 9b", "f0 9f", ""]

    for position in range(len(payload) + 1):
        assert "".join(decode_chunks([payload[:position], payload[position:]])) == text
    for size in range(1, len(payload) + 1):
        chunks_by_size = [payload[start:start + size] for start in range(0, len(payload), size)]
        assert "".join(decode_chunks(chunks_by_size)) == text

    tail = b"OK\xf0\x9f"
    strict = codecs.getincrementaldecoder("utf-8")()
    prefix = strict.decode(tail)
    tail_error = outcome(lambda: strict.decode(b"", final=True))
    bad_middle = outcome(lambda: "".join(decode_chunks([tail, b"X"])))
    tail_replaced = list(decode_chunks([tail], errors="replace"))
    assert prefix == "OK" and tail_error == {"error": "unexpected end of data", "bytes_hex": "f0 9f"}
    assert bad_middle == {"error": "invalid continuation byte", "bytes_hex": "f0 9f"}
    assert tail_replaced == ["OK", "�"]
    assert list(decode_chunks([b""])) == ["", ""]
    nul = bytes.fromhex("00")
    overlong = bytes.fromhex("c0 80")
    invalid_lead = outcome(lambda: "".join(decode_chunks([overlong[:1], overlong[1:]])))
    assert invalid_lead == {"error": "invalid start byte", "bytes_hex": "c0"}
    assert "".join(decode_chunks([nul])) == "\x00"
    literal = "�".encode("utf-8")
    assert literal.hex(" ") == "ef bf bd"
    assert "".join(decode_chunks([literal[:1], literal[1:]])) == "�"
    empty_mid = list(decode_chunks([chunks[0], b"", *chunks[1:]]))
    assert empty_mid == ["A", "", "雨", "🍊B", ""]
    with io.TextIOWrapper(io.BytesIO(payload), encoding="utf-8", errors="strict", newline="") as stream:
        text_reads = list(iter(lambda: stream.read(1), ""))
    assert text_reads == list(text)

    signature = codecs.BOM_UTF8
    bom_chunks = [signature[:1], signature[1:2], signature[2:] + b"A" + signature + b"B"]
    plain_bom = "".join(decode_chunks(bom_chunks))
    stripped_bom = "".join(decode_chunks(bom_chunks, encoding="utf-8-sig"))
    assert plain_bom == "\ufeffA\ufeffB" and stripped_bom == "A\ufeffB"

    records = [{"msg": "雨🍊"}, {"msg": "a\nb"}]
    wire = "".join(json.dumps(item, ensure_ascii=False, separators=(",", ":")) + "\n"
                   for item in records).encode("utf-8")
    line_chunks = [wire[start:start + 2] for start in range(0, len(wire), 2)]
    parsed = list(parse_lines(decode_chunks(line_chunks)))
    assert parsed == records
    try:
        list(parse_lines(decode_chunks([wire[:-1]])))
    except ValueError as error:
        framing_error = str(error)
    else:
        raise AssertionError("a missing record terminator was accepted")
    return {
        "environment": {"os": platform.system(), "os_version": platform.mac_ver()[0] or platform.release(),
                        "machine": platform.machine(), "python": sys.version.split()[0]},
        "text": text,
        "chunks_hex": [chunk.hex(" ") for chunk in chunks],
        "independent_replacement": separate,
        "incremental_parts_including_final_call": parts,
        "buffered_hex_after_each_input_chunk": pending,
        "parts_with_empty_chunk_midstream": empty_mid,
        "text_wrapper_read_one": text_reads,
        "two_part_splits_checked": len(payload) + 1,
        "fixed_chunk_sizes_checked": len(payload),
        "truncated_prefix_before_final": prefix,
        "truncated_strict_at_eof": tail_error,
        "truncated_replace_parts": tail_replaced,
        "invalid_continuation_before_eof": bad_middle,
        "illegal_overlong_nul": invalid_lead,
        "literal_replacement_character_bytes": literal.hex(" "),
        "bom_code_points": {"utf-8": [f"U+{ord(c):04X}" for c in plain_bom],
                            "utf-8-sig": [f"U+{ord(c):04X}" for c in stripped_bom]},
        "record_wire_bytes": len(wire),
        "record_chunk_size": 2,
        "records": parsed,
        "record_error_without_terminator": framing_error,
    }


if __name__ == "__main__":
    print(json.dumps(run(), ensure_ascii=False, indent=2))
