364 lines
17 KiB
Python
364 lines
17 KiB
Python
"""Runtime tests for the reconstructed zlib translation unit.
|
|
|
|
RED STATE: zlib.c / zlib.h do not exist yet, so setUpClass raises "missing zlib
|
|
runtime inputs". Every assertion below is grounded in a specific line of
|
|
iggy-decompiled/raw/durango/zlib.c (cited inline) so a faithful DURANGO
|
|
reconstruction turns these green.
|
|
|
|
zlib is Iggy's DEFLATE / inflate decompressor -- a streaming fork of the
|
|
public-domain stb_image zlib decoder (the recovered file-static helpers
|
|
parse_zlib_header / zbuild_huffman / zhuffman_decode_slow / zreceive /
|
|
compute_huffman_codes / parse_uncompressed_block_header are stb_image inflate
|
|
internals). It exports EXACTLY EIGHT entry points on all four platforms:
|
|
iggy_zlib_decoder, iggy_zlib_decoder_callback, iggy_zlib_decoder_free,
|
|
iggy_zlib_init_defaults, iggy_zlib_is_at_end, iggy_zlib_refill, iggy_zlib_skip,
|
|
iggy_zlib_current_offset.
|
|
|
|
It OWNS NO externally named data global: the sole durango ' R ' symbol nm
|
|
surfaces is the MSVC rodata COMDAT ??_C@..zlib.c -- the embedded __FILE__
|
|
source-path string, a read-only artifact, NOT owned data (cross test's
|
|
EXPECTED_GLOBALS == set()). The real "owned data" is the DEFLATE spec tables
|
|
(length_base / length_extra / dist_base / dist_extra, the 19-entry code-length
|
|
dezigzag) and the fixed-Huffman default_length / default_distance -- all
|
|
FILE-STATIC (nm lowercase d/b/r), recovered byte-for-byte by the cross test.
|
|
|
|
DURANGO 64-bit decoder-struct field map (byte offsets, raw/durango):
|
|
0x00 output read cursor 0x08 output end 0x18 input refill callback fn ptr
|
|
0x20 callback state 0x28 input cur 0x30 input end
|
|
0x38 stbm context 0x48 window size 0x4c window mask
|
|
0x50 self/window pointer (= decoder + 0xdd0)
|
|
0x60 is_at_end (u32) 0x68 consumed-bytes accumulator
|
|
|
|
WHAT IS PINNED (real, deterministic DURANGO behavior):
|
|
- iggy_zlib_init_defaults is a void(void) that only touches the file-static
|
|
fixed-Huffman tables; it is callable and idempotent (raw:451-490).
|
|
- iggy_zlib_decoder validates the 2-byte zlib header (parse_zlib_header,
|
|
raw:922-965): (CMF*256+FLG) % 31 == 0, (FLG & 0x20) == 0, (CMF & 0x0f) == 8.
|
|
A valid header allocates a decoder of window_size + 0xdd0 bytes via
|
|
iggy_stbm_alloc(stbm, size, tag=0, srcfile) (raw:272); an invalid header
|
|
returns NULL with no allocation (raw:270/327).
|
|
- the window size is 1 << ((CMF >> 4) + 8) (raw:951), so the alloc size scales
|
|
with the CMF high nibble.
|
|
- the returned decoder's self/window pointer field[10] (0x50) == decoder +
|
|
0xdd0 (raw:323).
|
|
- iggy_zlib_decoder_free flushes the read callback once then
|
|
iggy_stbm_free(decoder->stbm_ctx, decoder) (raw:436-443).
|
|
- iggy_zlib_decoder_callback wires a user byte-refill callback and tags the
|
|
allocation with the module's own __FILE__ string (raw:367-373).
|
|
- a STORED (uncompressed, BTYPE=00) block round trips through iggy_zlib_refill:
|
|
BFINAL+BTYPE bits, parse_uncompressed_block_header's LEN/NLEN check
|
|
(raw:910-914), then the raw bytes land in the window (raw:635-647).
|
|
- iggy_zlib_current_offset (raw:224) and iggy_zlib_is_at_end (raw:499) read
|
|
field[0x68]-field[0x10]+field[0] and the u32 at 0x60, both zero on a fresh
|
|
decoder; iggy_zlib_skip advances the output cursor (raw:848-864).
|
|
|
|
DEFERRED (flagged structural, NOT exercised here): the Huffman-decode + LZ77
|
|
back-reference core -- the fixed-Huffman (BTYPE=01) and dynamic-Huffman
|
|
(BTYPE=10, compute_huffman_codes) inflate loops, zbuild_huffman table
|
|
construction, zhuffman_decode_slow, and the length/distance extra-bit
|
|
back-reference copy (raw:504-842). Those are reachable only through the deep
|
|
iggy_zlib_refill state machine and are structural, not runtime-pinned here.
|
|
|
|
Full-suite runs use python3.12 (`python3.12 -u -m unittest discover -s tests
|
|
-p 'test_*.py' -v`): the machine python3 is 3.14, which SEGFAULTs at teardown on
|
|
the full suite via an unrelated CPython ctypes-GC bug (exit 139 AFTER "OK" with
|
|
0 fail/err is a teardown artifact, not a failure). Single-module runs are fine
|
|
on either interpreter.
|
|
"""
|
|
|
|
import ctypes
|
|
import shutil
|
|
from pathlib import Path
|
|
import subprocess
|
|
import tempfile
|
|
import unittest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
RECONSTRUCTED = ROOT / "iggy-decompiled/src/reconstructed"
|
|
SOURCE = RECONSTRUCTED / "zlib.c"
|
|
HEADER = RECONSTRUCTED / "zlib.h"
|
|
SUPPORT = ROOT / "tests/support/zlib_test_support.c"
|
|
|
|
# Struct field byte offsets (durango 64-bit; cited per test).
|
|
OFF_OUTPUT_CUR = 0x00 # output read cursor (raw:224/854).
|
|
OFF_OUTPUT_END = 0x08 # output end (raw:857).
|
|
OFF_CALLBACK = 0x18 # input refill callback fn ptr (raw:200/439).
|
|
OFF_CALLBACK_STATE = 0x20 # callback state (raw:368 callback variant).
|
|
OFF_STBM_CTX = 0x38 # stbm context, arg to iggy_stbm_free (raw:441).
|
|
OFF_WINDOW_SIZE = 0x48 # window size (raw:952).
|
|
OFF_WINDOW_MASK = 0x4C # window mask = size - 1 (raw:953).
|
|
OFF_SELF_PTR = 0x50 # self/window pointer = decoder + 0xdd0 (raw:323).
|
|
OFF_IS_AT_END = 0x60 # is_at_end flag u32 (raw:499).
|
|
|
|
DECODER_OVERHEAD = 0xDD0 # decoder bytes ahead of the window (raw:272/323).
|
|
SELF_PTR_DELTA = 0xDD0 # field[10] == decoder + 0x1ba*8 (raw:323).
|
|
|
|
|
|
def _zlib_header(cmf, flg):
|
|
return bytes([cmf, flg])
|
|
|
|
|
|
def _stored_block(payload):
|
|
# DEFLATE stored block: BFINAL=1 BTYPE=00 (byte 0x01), then LEN / ~LEN
|
|
# (little-endian u16, raw:910-911), then the raw bytes (raw:643).
|
|
length = len(payload)
|
|
return (bytes([0x01])
|
|
+ length.to_bytes(2, "little")
|
|
+ (length ^ 0xFFFF).to_bytes(2, "little")
|
|
+ bytes(payload))
|
|
|
|
|
|
class ReconstructedZlibTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
missing = [str(path) for path in (SOURCE, HEADER, SUPPORT)
|
|
if not path.exists()]
|
|
if missing:
|
|
raise AssertionError(
|
|
"missing zlib runtime inputs: " + ", ".join(missing))
|
|
cls.temp = tempfile.TemporaryDirectory()
|
|
cls.base = Path(cls.temp.name) / "zlib.dylib"
|
|
build = subprocess.run([
|
|
"clang", "-dynamiclib", "-std=c11", "-O2", "-Wall", "-Wextra",
|
|
"-Werror", "-Wl,-undefined,dynamic_lookup",
|
|
"-DIGGY_RECONSTRUCT_ZLIB_DURANGO=1",
|
|
"-I", str(RECONSTRUCTED),
|
|
str(SOURCE), str(SUPPORT), "-o", str(cls.base),
|
|
], capture_output=True, text=True)
|
|
if build.returncode:
|
|
raise AssertionError(build.stdout + build.stderr)
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
cls.temp.cleanup()
|
|
|
|
def setUp(self):
|
|
unique = Path(self.temp.name) / f"zlib_{self._testMethodName}.dylib"
|
|
shutil.copyfile(self.base, unique)
|
|
self.lib = ctypes.CDLL(str(unique))
|
|
self._bind(self.lib)
|
|
self.lib.zlib_test_reset()
|
|
self._keep = []
|
|
|
|
def _bind(self, lib):
|
|
void_p = ctypes.c_void_p
|
|
|
|
lib.zlib_test_reset.restype = None
|
|
lib.zlib_test_reset.argtypes = []
|
|
lib.zlib_test_set_input.restype = None
|
|
lib.zlib_test_set_input.argtypes = [void_p, ctypes.c_size_t]
|
|
lib.zlib_test_reader_fn.restype = void_p
|
|
lib.zlib_test_reader_fn.argtypes = []
|
|
for name, restype in (
|
|
("alloc_count", ctypes.c_size_t),
|
|
("free_count", ctypes.c_size_t),
|
|
("alloc_last_ctx", void_p),
|
|
("alloc_last_size", ctypes.c_size_t),
|
|
("alloc_last_tag", ctypes.c_int),
|
|
("alloc_last_srcfile", void_p),
|
|
("free_last_ctx", void_p),
|
|
("free_last_ptr", void_p)):
|
|
fn = getattr(lib, f"zlib_test_{name}")
|
|
fn.restype = restype
|
|
fn.argtypes = []
|
|
|
|
# Module exports under test (durango 64-bit ABI).
|
|
lib.iggy_zlib_decoder.restype = void_p
|
|
lib.iggy_zlib_decoder.argtypes = [
|
|
void_p, void_p, ctypes.c_ssize_t, void_p, ctypes.c_int]
|
|
lib.iggy_zlib_decoder_callback.restype = void_p
|
|
lib.iggy_zlib_decoder_callback.argtypes = [void_p, void_p, void_p]
|
|
lib.iggy_zlib_decoder_free.restype = None
|
|
lib.iggy_zlib_decoder_free.argtypes = [void_p]
|
|
lib.iggy_zlib_init_defaults.restype = None
|
|
lib.iggy_zlib_init_defaults.argtypes = []
|
|
lib.iggy_zlib_refill.restype = None
|
|
lib.iggy_zlib_refill.argtypes = [void_p]
|
|
lib.iggy_zlib_skip.restype = None
|
|
lib.iggy_zlib_skip.argtypes = [void_p, ctypes.c_int]
|
|
lib.iggy_zlib_current_offset.restype = ctypes.c_ssize_t
|
|
lib.iggy_zlib_current_offset.argtypes = [void_p]
|
|
lib.iggy_zlib_is_at_end.restype = ctypes.c_uint32
|
|
lib.iggy_zlib_is_at_end.argtypes = [void_p]
|
|
|
|
# --- helpers ------------------------------------------------------- #
|
|
|
|
def _input(self, data):
|
|
buf = (ctypes.c_uint8 * len(data)).from_buffer_copy(bytes(data))
|
|
self._keep.append(buf)
|
|
return buf
|
|
|
|
def _ctx(self):
|
|
ctx = (ctypes.c_uint8 * 16)()
|
|
self._keep.append(ctx)
|
|
return ctypes.addressof(ctx)
|
|
|
|
def _srcfile(self):
|
|
marker = ctypes.create_string_buffer(b"zlib_runtime_srcfile")
|
|
self._keep.append(marker)
|
|
return ctypes.addressof(marker)
|
|
|
|
def _field_u64(self, decoder, offset):
|
|
return ctypes.c_uint64.from_address(decoder + offset).value
|
|
|
|
def _build(self, stream, ctx=None, srcfile=None):
|
|
buf = self._input(stream)
|
|
ctx = self._ctx() if ctx is None else ctx
|
|
srcfile = self._srcfile() if srcfile is None else srcfile
|
|
decoder = self.lib.iggy_zlib_decoder(
|
|
ctypes.c_void_p(ctx), buf, len(stream),
|
|
ctypes.c_void_p(srcfile), 0)
|
|
return decoder, ctx, srcfile
|
|
|
|
# --- init_defaults ------------------------------------------------- #
|
|
|
|
def test_init_defaults_is_void_no_arg_and_idempotent(self):
|
|
# raw:451 void iggy_zlib_init_defaults(void) fills only the file-static
|
|
# fixed-Huffman code-length tables; calling it twice is a no-op-safe
|
|
# repeat (the tables are overwritten with the same constants).
|
|
self.lib.iggy_zlib_init_defaults()
|
|
self.lib.iggy_zlib_init_defaults()
|
|
|
|
# --- header validation / allocation -------------------------------- #
|
|
|
|
def test_valid_header_allocates_one_decoder(self):
|
|
# raw:949 0x78 0x01 -> (0x7801 % 31 == 0) & !(0x01 & 0x20) &
|
|
# (0x78 & 0x0f == 8): valid. raw:272 exactly one stbm_alloc, non-NULL.
|
|
decoder, ctx, srcfile = self._build(
|
|
_zlib_header(0x78, 0x01) + _stored_block(b""))
|
|
self.assertNotEqual(decoder, 0)
|
|
self.assertEqual(self.lib.zlib_test_alloc_count(), 1)
|
|
self.assertEqual(self.lib.zlib_test_alloc_last_ctx(), ctx)
|
|
self.assertEqual(self.lib.zlib_test_alloc_last_srcfile(), srcfile)
|
|
# raw:272 the allocation is tagged 0.
|
|
self.assertEqual(self.lib.zlib_test_alloc_last_tag(), 0)
|
|
|
|
def test_window_size_formula_scales_alloc(self):
|
|
# raw:951-952 window = 1 << ((CMF >> 4) + 8); alloc = window + 0xdd0.
|
|
# 0x78 -> nibble 7 -> window 0x8000 -> alloc 0x8dd0.
|
|
# 0x08 (nibble 0) -> window 0x100 -> alloc 0xed0. 0x081d is a valid
|
|
# header: 0x081d % 31 == 0, !(0x1d & 0x20), (0x08 & 0x0f == 8).
|
|
for cmf, flg, window in ((0x78, 0x01, 0x8000), (0x08, 0x1D, 0x100)):
|
|
self.lib.zlib_test_reset()
|
|
decoder, _, _ = self._build(
|
|
_zlib_header(cmf, flg) + _stored_block(b""))
|
|
self.assertNotEqual(decoder, 0, hex(cmf))
|
|
self.assertEqual(
|
|
self.lib.zlib_test_alloc_last_size(),
|
|
window + DECODER_OVERHEAD, hex(cmf))
|
|
# size (0x48) and mask (0x4c) are adjacent u32 fields (durango
|
|
# objdump: two movl), so a u64 read spans both -- mask to the field.
|
|
self.assertEqual(
|
|
self._field_u64(decoder, OFF_WINDOW_SIZE) & 0xFFFFFFFF,
|
|
window, hex(cmf))
|
|
self.assertEqual(
|
|
self._field_u64(decoder, OFF_WINDOW_MASK) & 0xFFFFFFFF,
|
|
window - 1, hex(cmf))
|
|
|
|
def test_invalid_headers_return_null_without_allocating(self):
|
|
# raw:949 each of these fails one clause of the header gate.
|
|
cases = {
|
|
"bad_method": _zlib_header(0x77, 0x01), # (0x77 & 0x0f) != 8.
|
|
"preset_dict": _zlib_header(0x78, 0x20), # (FLG & 0x20) != 0.
|
|
"bad_fcheck": _zlib_header(0x78, 0x02), # 0x7802 % 31 != 0.
|
|
}
|
|
for name, header in cases.items():
|
|
self.lib.zlib_test_reset()
|
|
decoder, _, _ = self._build(header + b"\x00\x00\x00\x00")
|
|
# ctypes renders a NULL c_void_p return as None (None == 0 is False).
|
|
self.assertFalse(decoder, name)
|
|
self.assertEqual(self.lib.zlib_test_alloc_count(), 0, name)
|
|
|
|
def test_decoder_self_pointer_and_stbm_context(self):
|
|
# raw:323 puVar5[10] = puVar5 + 0x1ba -> field[0x50] == decoder+0xdd0.
|
|
# raw:268/441 field[0x38] holds the stbm ctx handed to iggy_stbm_free.
|
|
ctx = self._ctx()
|
|
decoder, _, _ = self._build(
|
|
_zlib_header(0x78, 0x01) + _stored_block(b""), ctx=ctx)
|
|
self.assertNotEqual(decoder, 0)
|
|
self.assertEqual(
|
|
self._field_u64(decoder, OFF_SELF_PTR), decoder + SELF_PTR_DELTA)
|
|
self.assertEqual(self._field_u64(decoder, OFF_STBM_CTX), ctx)
|
|
|
|
# --- free ---------------------------------------------------------- #
|
|
|
|
def test_free_calls_stbm_free_with_context_and_decoder(self):
|
|
# raw:441 iggy_stbm_free(decoder->stbm_ctx, decoder) -- exactly one
|
|
# free, with the context captured at build time and the decoder pointer.
|
|
ctx = self._ctx()
|
|
decoder, _, _ = self._build(
|
|
_zlib_header(0x78, 0x01) + _stored_block(b""), ctx=ctx)
|
|
self.assertNotEqual(decoder, 0)
|
|
self.lib.iggy_zlib_decoder_free(ctypes.c_void_p(decoder))
|
|
self.assertEqual(self.lib.zlib_test_free_count(), 1)
|
|
self.assertEqual(self.lib.zlib_test_free_last_ctx(), ctx)
|
|
self.assertEqual(self.lib.zlib_test_free_last_ptr(), decoder)
|
|
|
|
# --- callback (streaming) variant ---------------------------------- #
|
|
|
|
def test_callback_variant_wires_reader_and_internal_srcfile(self):
|
|
# raw:367-373 iggy_zlib_decoder_callback(stbm, read_fn, state) stores
|
|
# read_fn at field[0x18] and state at field[0x20], and tags the
|
|
# allocation with the module's own __FILE__ string (ends "zlib.c").
|
|
ctx = self._ctx()
|
|
reader = self.lib.zlib_test_reader_fn()
|
|
self.lib.zlib_test_set_input(
|
|
self._input(_zlib_header(0x78, 0x01) + _stored_block(b"")),
|
|
len(_zlib_header(0x78, 0x01) + _stored_block(b"")))
|
|
state = 0xABCD
|
|
decoder = self.lib.iggy_zlib_decoder_callback(
|
|
ctypes.c_void_p(ctx), ctypes.c_void_p(reader),
|
|
ctypes.c_void_p(state))
|
|
self.assertNotEqual(decoder, 0)
|
|
self.assertEqual(self.lib.zlib_test_alloc_count(), 1)
|
|
self.assertEqual(self._field_u64(decoder, OFF_CALLBACK), reader)
|
|
self.assertEqual(self._field_u64(decoder, OFF_CALLBACK_STATE), state)
|
|
srcfile = self.lib.zlib_test_alloc_last_srcfile()
|
|
self.assertTrue(srcfile, "callback variant passed a null __FILE__")
|
|
self.assertTrue(ctypes.string_at(srcfile).endswith(b"zlib.c"))
|
|
|
|
# --- stored-block inflate + stream position ------------------------ #
|
|
|
|
def test_fresh_decoder_offset_and_is_at_end_are_zero(self):
|
|
# raw:224/954 current_offset == field[0x68]-field[0x10]+field[0], all
|
|
# zeroed by parse_zlib_header / build; raw:499 is_at_end reads 0x60,
|
|
# also zero before the first refill.
|
|
decoder, _, _ = self._build(
|
|
_zlib_header(0x78, 0x01) + _stored_block(b"ABC"))
|
|
self.assertNotEqual(decoder, 0)
|
|
self.assertEqual(
|
|
self.lib.iggy_zlib_current_offset(ctypes.c_void_p(decoder)), 0)
|
|
self.assertEqual(
|
|
self.lib.iggy_zlib_is_at_end(ctypes.c_void_p(decoder)), 0)
|
|
|
|
def test_stored_block_bytes_land_in_the_window(self):
|
|
# raw:601-655 BTYPE==00 -> parse_uncompressed_block_header (LEN/NLEN)
|
|
# -> the raw payload is copied verbatim into the window (raw:643). One
|
|
# refill of a small final stored block makes [field0, field1) span it.
|
|
payload = b"ABC"
|
|
decoder, _, _ = self._build(
|
|
_zlib_header(0x08, 0x1D) + _stored_block(payload))
|
|
self.assertNotEqual(decoder, 0)
|
|
self.lib.iggy_zlib_refill(ctypes.c_void_p(decoder))
|
|
cur = self._field_u64(decoder, OFF_OUTPUT_CUR)
|
|
end = self._field_u64(decoder, OFF_OUTPUT_END)
|
|
self.assertEqual(end - cur, len(payload))
|
|
self.assertEqual(ctypes.string_at(cur, len(payload)), payload)
|
|
|
|
def test_skip_advances_the_output_cursor(self):
|
|
# raw:848-864 iggy_zlib_skip refills when the output window is empty,
|
|
# then advances field[0] (the output cursor) by n, so current_offset
|
|
# (raw:224) reports n bytes consumed.
|
|
payload = bytes(range(16))
|
|
decoder, _, _ = self._build(
|
|
_zlib_header(0x08, 0x1D) + _stored_block(payload))
|
|
self.assertNotEqual(decoder, 0)
|
|
self.lib.iggy_zlib_skip(ctypes.c_void_p(decoder), 5)
|
|
self.assertEqual(
|
|
self.lib.iggy_zlib_current_offset(ctypes.c_void_p(decoder)), 5)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|