rebuild iggy's utf and ecmascript utility core

This commit is contained in:
coah
2026-07-13 04:19:13 -05:00
parent aa77d9a014
commit d2e23130ba
6 changed files with 1140 additions and 0 deletions

View File

@@ -54,6 +54,22 @@ foreach(platform IN ITEMS durango orbis ps3 vita)
configure_iggy_reconstructed_target(${target} c_std_11)
endforeach()
foreach(platform IN ITEMS durango orbis ps3 vita)
string(TOUPPER ${platform} platform_upper)
set(target iggy_reconstructed_as3vm_util_${platform})
add_library(${target} STATIC as3vm_util.c)
target_compile_definitions(
${target}
PRIVATE IGGY_RECONSTRUCT_AS3VM_UTIL_${platform_upper}
)
configure_iggy_reconstructed_target(${target} c_std_11)
if(CMAKE_C_COMPILER_ID MATCHES "Clang|GNU")
target_compile_options(${target} PRIVATE -fno-builtin)
elseif(MSVC)
target_compile_options(${target} PRIVATE /Oi-)
endif()
endforeach()
add_library(
iggy_reconstructed_as3vm_debugger_interface
STATIC

View File

@@ -34,6 +34,12 @@ surrogates, embedded nulls, and fixed-length results retain their native
behavior. Durango/Vita keep writable tables while Orbis/PS3 keep read-only
tables.
`as3vm_util.c` restores Iggy's UTF-8/UTF-16 conversion and comparison helpers,
ECMAScript number readers, ASCII digit predicate, and UTF-16 formatter. The
four targets preserve native length, replacement, truncation, whitespace,
parser-cursor, variadic, and malformed-input behavior, including the shipping
surrogate-count and source-base quirks relied on by object-level callers.
`as3vm_debugger_interface.c` and `as3vm_op_strings.c` preserve the three
intentional zero-return stubs in the shipping archives. Every platform object
contains the same compiled-out behavior with no debugger metadata, opcode

View File

@@ -0,0 +1,564 @@
#include "as3vm_util.h"
#include <stdarg.h>
#include <stddef.h>
#include <stdint.h>
#if (defined(IGGY_RECONSTRUCT_AS3VM_UTIL_DURANGO) + \
defined(IGGY_RECONSTRUCT_AS3VM_UTIL_ORBIS) + \
defined(IGGY_RECONSTRUCT_AS3VM_UTIL_PS3) + \
defined(IGGY_RECONSTRUCT_AS3VM_UTIL_VITA)) != 1
#error Select exactly one reconstructed as3vm_util platform
#endif
extern size_t strlen(const char *text);
extern double pow(double base, double exponent);
static const uint16_t rr_utf16_simple_printf_str_null[] = {
0x0028, 0x006e, 0x0075, 0x006c, 0x006c, 0x0029, 0x0000};
static int32_t utf8_string_length(const uint8_t *text) {
#if defined(IGGY_RECONSTRUCT_AS3VM_UTIL_DURANGO)
const uint8_t *cursor = text;
while (*cursor != 0) {
cursor++;
}
return (int32_t)(cursor - text);
#else
return (int32_t)strlen((const char *)text);
#endif
}
static int is_ecmascript_whitespace(uint16_t value) {
return (value >= 0x0009U && value <= 0x000dU) || value == 0x0020U ||
value == 0x00a0U || value == 0x1680U || value == 0x180eU ||
(value >= 0x2000U && value <= 0x200aU) || value == 0x2028U ||
value == 0x2029U || value == 0x202fU || value == 0x205fU ||
value == 0x3000U;
}
static const uint16_t *skipwhite(const uint16_t *cursor,
const uint16_t *limit) {
while (cursor < limit && is_ecmascript_whitespace(*cursor)) {
cursor++;
}
return cursor;
}
static int hex_digit(uint16_t value) {
if (value >= 0x0030U && value <= 0x0039U) {
return (int)(value - 0x0030U);
}
if (value >= 0x0041U && value <= 0x0046U) {
return (int)(value - 0x0041U + 10U);
}
if (value >= 0x0061U && value <= 0x0066U) {
return (int)(value - 0x0061U + 10U);
}
return -1;
}
static int conversion_overflow(int32_t truncate, int32_t written) {
return truncate != 0 ? written : -1;
}
int32_t rriswdigit(uint16_t value) {
return value >= 0x0030U && value <= 0x0039U;
}
int32_t rrUTF8toUTF16(uint16_t *destination, int32_t destination_units,
const uint8_t *source, int32_t source_bytes,
int32_t replacement, int32_t truncate) {
int32_t source_limit = source_bytes < 0 ? INT32_MAX : source_bytes;
int32_t source_index = 0;
int32_t output_index = 0;
if (destination == NULL) {
destination_units = INT32_MAX;
}
for (;;) {
uint8_t lead;
uint32_t value;
if (source_bytes < 0) {
if (source[source_index] == 0) {
break;
}
} else if (source_index >= source_bytes) {
break;
}
lead = source[source_index];
if ((lead & 0x80U) == 0) {
source_index++;
value = lead;
} else if ((lead & 0xe0U) == 0xc0U) {
if (source_limit < source_index + 1 ||
(source[source_index + 1] & 0xc0U) != 0x80U) {
goto malformed_structure;
}
value = ((uint32_t)(lead & 0x1fU) << 6U) |
(uint32_t)(source[source_index + 1] & 0x3fU);
source_index += 2;
if (value < 0x80U) {
goto malformed_value;
}
} else if ((lead & 0xf0U) == 0xe0U) {
if (source_limit < source_index + 2 ||
(source[source_index + 1] & 0xc0U) != 0x80U ||
(source[source_index + 2] & 0xc0U) != 0x80U) {
goto malformed_structure;
}
value = ((uint32_t)(lead & 0x0fU) << 12U) |
((uint32_t)(source[source_index + 1] & 0x3fU) << 6U) |
(uint32_t)(source[source_index + 2] & 0x3fU);
source_index += 3;
if (value < 0x800U) {
goto malformed_value;
}
} else if ((lead & 0xf8U) == 0xf0U &&
source_index + 3 <= source_limit &&
(source[source_index + 1] & 0xc0U) == 0x80U &&
(source[source_index + 2] & 0xc0U) == 0x80U &&
(source[source_index + 3] & 0xc0U) == 0x80U) {
uint32_t scalar = ((uint32_t)(lead & 0x07U) << 18U) |
((uint32_t)(source[source_index + 1] & 0x3fU) << 12U) |
((uint32_t)(source[source_index + 2] & 0x3fU) << 6U) |
(uint32_t)(source[source_index + 3] & 0x3fU);
uint32_t surrogate_value = scalar - 0x10000U;
source_index += 4;
if (surrogate_value > 0xfffffU ||
(scalar & 0xfffff800U) == 0x0000d800U) {
goto malformed_value;
}
if (destination_units < output_index + 2) {
return conversion_overflow(truncate, output_index);
}
if (destination != NULL) {
destination[output_index] =
(uint16_t)(0xd800U | (surrogate_value >> 10U));
destination[output_index + 1] =
(uint16_t)(0xdc00U | (surrogate_value & 0x3ffU));
}
output_index += 2;
continue;
} else {
malformed_structure:
source_index++;
if (source_bytes >= 0) {
while (source[source_index] != 0 &&
(source[source_index] & 0xc0U) == 0x80U) {
source_index++;
}
}
malformed_value:
if (replacement < 0) {
return -2;
}
value = (uint16_t)replacement;
}
if (destination_units < output_index + 1) {
return conversion_overflow(truncate, output_index);
}
if (destination != NULL) {
destination[output_index] = (uint16_t)value;
}
output_index++;
}
if (destination_units < output_index + 1) {
if (source_bytes >= 0) {
return output_index;
}
return conversion_overflow(truncate, output_index);
}
if (destination != NULL) {
destination[output_index] = 0;
}
return output_index;
}
int32_t rrUTF16toUTF8(uint8_t *destination, int32_t destination_bytes,
const uint16_t *source, int32_t source_units,
int32_t truncate) {
int32_t source_limit = source_units < 0 ? INT32_MAX : source_units;
int32_t source_index = 0;
int32_t output_index = 0;
if (destination == NULL) {
destination_bytes = INT32_MAX;
}
for (;;) {
uint16_t value;
if (source_units < 0) {
if (source[source_index] == 0) {
break;
}
} else if (source_index >= source_limit) {
break;
}
value = source[source_index];
if (value < 0x80U) {
if (destination_bytes <= output_index) {
return conversion_overflow(truncate, output_index);
}
if (destination != NULL) {
destination[output_index] = (uint8_t)value;
}
output_index++;
source_index++;
} else if (value < 0x800U) {
if (destination_bytes <= output_index + 1) {
return conversion_overflow(truncate, output_index);
}
if (destination != NULL) {
destination[output_index] = (uint8_t)(0xc0U | (value >> 6U));
destination[output_index + 1] =
(uint8_t)(0x80U | (value & 0x3fU));
}
output_index += 2;
source_index++;
} else if ((uint16_t)(source[0] + 0x2800U) < 0x0400U) {
uint16_t low;
uint32_t scalar;
if (source_limit <= source_index + 1) {
return -2;
}
if (destination_bytes <= output_index + 3) {
return conversion_overflow(truncate, output_index);
}
low = source[source_index + 1];
if ((uint16_t)(low + 0x2400U) >= 0x0400U) {
return -2;
}
scalar = (((uint32_t)value - 0xd800U) << 10U) +
((uint32_t)low - 0xdc00U) + 0x10000U;
if (destination != NULL) {
destination[output_index] = (uint8_t)(0xf0U | (scalar >> 18U));
destination[output_index + 1] =
(uint8_t)(0x80U | ((scalar >> 12U) & 0x3fU));
destination[output_index + 2] =
(uint8_t)(0x80U | ((scalar >> 6U) & 0x3fU));
destination[output_index + 3] =
(uint8_t)(0x80U | (scalar & 0x3fU));
}
output_index += 4;
source_index += 2;
} else {
if ((uint16_t)(source[0] + 0x2400U) < 0x0400U) {
return -2;
}
if (destination_bytes <= output_index + 2) {
return conversion_overflow(truncate, output_index);
}
if (destination != NULL) {
destination[output_index] = (uint8_t)(0xe0U | (value >> 12U));
destination[output_index + 1] =
(uint8_t)(0x80U | ((value >> 6U) & 0x3fU));
destination[output_index + 2] =
(uint8_t)(0x80U | (value & 0x3fU));
}
output_index += 3;
source_index++;
}
}
if (source_units >= 0) {
if (destination != NULL && destination_bytes > output_index) {
destination[output_index] = 0;
}
return output_index;
}
if (destination_bytes <= output_index) {
return conversion_overflow(truncate, output_index);
}
if (destination != NULL) {
destination[output_index] = 0;
}
return output_index + 1;
}
int32_t iggyCompareUTF8toUTF16(const uint8_t *utf8, int32_t utf8_bytes,
const uint16_t *utf16, int32_t utf16_units) {
int32_t utf8_length =
utf8_bytes < 0 ? utf8_string_length(utf8) : utf8_bytes;
int32_t utf16_length =
utf16_units < 0 ? rrwcslen(utf16) : utf16_units;
int32_t utf8_limit = utf8_length;
int32_t remaining_units = utf16_length;
int32_t byte_index = 0;
int32_t unit_index = 0;
while (remaining_units != 0) {
uint16_t value = utf16[unit_index++];
uint8_t encoded[4];
int32_t count;
if (value < 0x80U) {
encoded[0] = (uint8_t)value;
count = 1;
} else if (value < 0x800U) {
encoded[0] = (uint8_t)(0xc0U | (value >> 6U));
encoded[1] = (uint8_t)(0x80U | (value & 0x3fU));
count = 2;
} else if ((uint16_t)(value + 0x2800U) < 0x0400U) {
uint16_t low = utf16[unit_index++];
uint32_t scalar = (((uint32_t)value - 0xd800U) << 10U) +
((uint32_t)low - 0xdc00U) + 0x10000U;
encoded[0] = (uint8_t)(0xf0U | (scalar >> 18U));
encoded[1] = (uint8_t)(0x80U | ((scalar >> 12U) & 0x3fU));
encoded[2] = (uint8_t)(0x80U | ((scalar >> 6U) & 0x3fU));
encoded[3] = (uint8_t)(0x80U | (scalar & 0x3fU));
count = 4;
} else {
if ((uint16_t)(value + 0x2400U) < 0x0400U) {
return -1;
}
encoded[0] = (uint8_t)(0xe0U | (value >> 12U));
encoded[1] = (uint8_t)(0x80U | ((value >> 6U) & 0x3fU));
encoded[2] = (uint8_t)(0x80U | (value & 0x3fU));
count = 3;
}
if (utf8_limit - byte_index < count) {
return -1;
}
for (int32_t index = 0; index < count; index++) {
if (utf8[byte_index + index] != encoded[index]) {
return -1;
}
}
byte_index += count;
if (count == 4) {
utf8_limit--;
}
remaining_units--;
}
return byte_index < utf8_limit;
}
int32_t rr_utf16_read_integer_hex_ecmascript(
const uint16_t *text, int32_t length, const uint16_t **end,
int32_t *value, int32_t allow_trailing) {
const uint16_t *limit = text + length;
const uint16_t *cursor = skipwhite(text, limit);
uint32_t accumulator = 0;
if (cursor == limit || cursor[0] != 0x0030U ||
(cursor[1] | 0x0020U) != 0x0078U) {
return 0;
}
cursor += 2;
while (cursor < limit) {
int digit = hex_digit(*cursor);
if (digit < 0) {
break;
}
accumulator = accumulator * 16U + (uint32_t)digit;
cursor++;
}
if (!allow_trailing) {
cursor = skipwhite(cursor, limit);
if (cursor < limit) {
return 0;
}
}
if (end != NULL) {
*end = cursor;
}
*value = (int32_t)accumulator;
return 1;
}
int32_t rr_utf16_read_hex_ecmascript(const uint16_t *text, int32_t length,
const uint16_t **end, double *value,
int32_t allow_trailing) {
const uint16_t *limit = text + length;
const uint16_t *cursor = skipwhite(text, limit);
double accumulator = 0.0;
if (cursor == limit || cursor[0] != 0x0030U ||
(cursor[1] | 0x0020U) != 0x0078U) {
return 0;
}
cursor += 2;
while (cursor < limit) {
int digit = hex_digit(*cursor);
if (digit < 0) {
break;
}
accumulator = accumulator * 16.0 + (double)digit;
cursor++;
}
if (!allow_trailing) {
cursor = skipwhite(cursor, limit);
if (cursor < limit) {
return 0;
}
}
if (end != NULL) {
*end = cursor;
}
*value = accumulator;
return 1;
}
static double exponent_multiplier(int negative) {
#if defined(IGGY_RECONSTRUCT_AS3VM_UTIL_PS3)
(void)negative;
return 1.0;
#else
volatile double exponent = negative ? -0.0 : 0.0;
return pow(10.0, exponent);
#endif
}
int32_t rr_utf16_read_decimal_ecmascript(const uint16_t *text, int32_t length,
const uint16_t **end, double *value,
int32_t allow_trailing) {
const uint16_t *limit = text + length;
const uint16_t *cursor = skipwhite(text, limit);
double sign = 1.0;
double accumulator = 0.0;
if (cursor == limit) {
return 0;
}
if (*cursor == 0x002dU) {
sign = -1.0;
cursor++;
} else if (*cursor == 0x002bU) {
cursor++;
}
if (cursor == limit) {
return 0;
}
if (*cursor >= 0x0030U && *cursor <= 0x0039U) {
while (cursor < limit && *cursor >= 0x0030U && *cursor <= 0x0039U) {
accumulator = accumulator * 10.0 + (double)(*cursor - 0x0030U);
cursor++;
}
} else if (*cursor != 0x002eU) {
return 0;
}
if (cursor < limit && *cursor == 0x002eU) {
double fraction = 0.0;
double denominator = 1.0;
cursor++;
while (cursor < limit && *cursor >= 0x0030U && *cursor <= 0x0039U) {
fraction = fraction * 10.0 + (double)(*cursor - 0x0030U);
denominator *= 10.0;
cursor++;
}
accumulator += fraction / denominator;
}
if (cursor < limit && ((*cursor | 0x0020U) == 0x0065U)) {
const uint16_t *exponent_cursor = cursor + 1;
int exponent_negative = 0;
if (exponent_cursor < limit && *exponent_cursor == 0x002dU) {
exponent_negative = 1;
exponent_cursor++;
} else if (exponent_cursor < limit && *exponent_cursor == 0x002bU) {
exponent_cursor++;
}
if (exponent_cursor == limit || *exponent_cursor < 0x0030U ||
*exponent_cursor > 0x0039U) {
return 0;
}
accumulator *= exponent_multiplier(exponent_negative);
cursor = exponent_cursor;
}
if (!allow_trailing) {
cursor = skipwhite(cursor, limit);
if (cursor < limit) {
return 0;
}
}
if (end != NULL) {
*end = cursor;
}
*value = sign * accumulator;
return 1;
}
static void format_copy(uint16_t **destination, const uint16_t *source,
int32_t limit) {
while (*source != 0 && limit != 0) {
**destination = *source;
(*destination)++;
source++;
limit--;
}
}
static void format_integer(uint16_t **destination, int32_t value) {
uint16_t buffer[12];
uint16_t *cursor = buffer + 11;
uint32_t magnitude =
value < 0 ? 0U - (uint32_t)value : (uint32_t)value;
buffer[11] = 0;
do {
*--cursor = (uint16_t)(0x0030U + magnitude % 10U);
magnitude /= 10U;
} while (magnitude != 0);
if (value < 0) {
*--cursor = 0x002dU;
}
format_copy(destination, cursor, (int32_t)((buffer + 11) - cursor));
}
void rr_utf16_simple_printf(uint16_t *destination, int32_t destination_units,
const uint16_t *format, ...) {
uint16_t *limit;
va_list arguments;
if (destination_units <= 0) {
return;
}
limit = destination + destination_units - 1;
va_start(arguments, format);
while (*format != 0 && destination < limit) {
if (*format != 0x0025U) {
*destination++ = *format++;
continue;
}
format++;
if (*format == 0x0025U) {
*destination++ = 0x0025U;
format++;
} else if (*format == 0x0073U) {
const uint16_t *text = va_arg(arguments, const uint16_t *);
format++;
if (text == NULL) {
text = rr_utf16_simple_printf_str_null;
}
format_copy(&destination, text, -1);
} else if (*format == 0x0053U) {
int32_t length = va_arg(arguments, int32_t);
const uint16_t *text = va_arg(arguments, const uint16_t *);
format++;
if (text == NULL) {
text = rr_utf16_simple_printf_str_null;
length = -1;
}
format_copy(&destination, text, length < 0 ? -1 : length);
} else if (*format == 0x0064U) {
int32_t value = va_arg(arguments, int32_t);
format++;
format_integer(&destination, value);
}
}
va_end(arguments);
*destination = 0;
}

View File

@@ -0,0 +1,38 @@
#ifndef IGGY_RECONSTRUCTED_AS3VM_UTIL_H
#define IGGY_RECONSTRUCTED_AS3VM_UTIL_H
#include <stdint.h>
#if defined(__cplusplus)
extern "C" {
#endif
int32_t rrwcslen(const uint16_t *text);
int32_t rriswdigit(uint16_t value);
int32_t rrUTF8toUTF16(uint16_t *destination, int32_t destination_units,
const uint8_t *source, int32_t source_bytes,
int32_t replacement, int32_t truncate);
int32_t rrUTF16toUTF8(uint8_t *destination, int32_t destination_bytes,
const uint16_t *source, int32_t source_units,
int32_t truncate);
int32_t iggyCompareUTF8toUTF16(const uint8_t *utf8, int32_t utf8_bytes,
const uint16_t *utf16, int32_t utf16_units);
int32_t rr_utf16_read_integer_hex_ecmascript(
const uint16_t *text, int32_t length, const uint16_t **end,
int32_t *value, int32_t allow_trailing);
int32_t rr_utf16_read_hex_ecmascript(const uint16_t *text, int32_t length,
const uint16_t **end, double *value,
int32_t allow_trailing);
int32_t rr_utf16_read_decimal_ecmascript(const uint16_t *text, int32_t length,
const uint16_t **end, double *value,
int32_t allow_trailing);
void rr_utf16_simple_printf(uint16_t *destination, int32_t destination_units,
const uint16_t *format, ...);
#if defined(__cplusplus)
}
#endif
#endif

View File

@@ -4,6 +4,7 @@ iggy_api_functions_global 39 37 durango,orbis,ps3,vita complete
swf_state 2 2 durango,orbis,ps3,vita complete
swf_math 3 3 durango,orbis,ps3,vita complete
iggy_unicode 3 3 durango,orbis,ps3,vita complete
as3vm_util 9 8 durango,orbis,ps3,vita complete
as3vm_debugger_interface 2 2 durango,orbis,ps3,vita complete
as3vm_op_strings 1 1 durango,orbis,ps3,vita complete
iggy_platform_audio 2 2 durango complete
1 module source_functions external_functions platforms status
4 swf_state 2 2 durango,orbis,ps3,vita complete
5 swf_math 3 3 durango,orbis,ps3,vita complete
6 iggy_unicode 3 3 durango,orbis,ps3,vita complete
7 as3vm_util 9 8 durango,orbis,ps3,vita complete
8 as3vm_debugger_interface 2 2 durango,orbis,ps3,vita complete
9 as3vm_op_strings 1 1 durango,orbis,ps3,vita complete
10 iggy_platform_audio 2 2 durango complete

View File

@@ -0,0 +1,515 @@
import ctypes
import math
from pathlib import Path
import subprocess
import tempfile
import unittest
ROOT = Path(__file__).resolve().parents[1]
SOURCE = ROOT / "iggy-decompiled/src/reconstructed/as3vm_util.c"
RECONSTRUCTED = ROOT / "iggy-decompiled/src/reconstructed"
VARIANTS = {
"durango": "IGGY_RECONSTRUCT_AS3VM_UTIL_DURANGO",
"orbis": "IGGY_RECONSTRUCT_AS3VM_UTIL_ORBIS",
"ps3": "IGGY_RECONSTRUCT_AS3VM_UTIL_PS3",
"vita": "IGGY_RECONSTRUCT_AS3VM_UTIL_VITA",
}
HARNESS = r"""
#include <stdint.h>
#include "as3vm_util.h"
int32_t rrwcslen(const uint16_t *text) {
const uint16_t *cursor = text;
while (*cursor != 0) {
cursor++;
}
return (int32_t)(cursor - text);
}
void util_format_literal(uint16_t *dst, int32_t cap, const uint16_t *fmt) {
rr_utf16_simple_printf(dst, cap, fmt);
}
void util_format_d(uint16_t *dst, int32_t cap, const uint16_t *fmt,
int32_t value) {
rr_utf16_simple_printf(dst, cap, fmt, value);
}
void util_format_s(uint16_t *dst, int32_t cap, const uint16_t *fmt,
const uint16_t *value) {
rr_utf16_simple_printf(dst, cap, fmt, value);
}
void util_format_S(uint16_t *dst, int32_t cap, const uint16_t *fmt,
int32_t length, const uint16_t *value) {
rr_utf16_simple_printf(dst, cap, fmt, length, value);
}
void util_format_mix(uint16_t *dst, int32_t cap, const uint16_t *fmt,
int32_t number, const uint16_t *first, int32_t length,
const uint16_t *second) {
rr_utf16_simple_printf(dst, cap, fmt, number, first, length, second);
}
"""
class ReconstructedAs3VmUtilTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.temporary_directory = tempfile.TemporaryDirectory()
temporary_path = Path(cls.temporary_directory.name)
harness_path = temporary_path / "as3vm_util_harness.c"
harness_path.write_text(HARNESS, encoding="utf-8")
cls.libraries = {}
for platform, define in VARIANTS.items():
library_path = temporary_path / f"libas3vm_util_{platform}.dylib"
result = subprocess.run(
[
"clang",
"-dynamiclib",
"-std=c11",
"-O2",
"-Wall",
"-Wextra",
"-Werror",
f"-D{define}=1",
"-I",
str(RECONSTRUCTED),
str(SOURCE),
str(harness_path),
"-o",
str(library_path),
],
capture_output=True,
text=True,
)
if result.returncode != 0:
raise AssertionError(result.stdout + result.stderr)
library = ctypes.CDLL(str(library_path))
cls.configure_library(library)
cls.libraries[platform] = library
@classmethod
def tearDownClass(cls):
cls.temporary_directory.cleanup()
@staticmethod
def configure_library(library):
u8p = ctypes.POINTER(ctypes.c_uint8)
u16p = ctypes.POINTER(ctypes.c_uint16)
u16pp = ctypes.POINTER(u16p)
library.rriswdigit.argtypes = [ctypes.c_uint16]
library.rriswdigit.restype = ctypes.c_int32
library.rrUTF8toUTF16.argtypes = [
u16p,
ctypes.c_int32,
u8p,
ctypes.c_int32,
ctypes.c_int32,
ctypes.c_int32,
]
library.rrUTF8toUTF16.restype = ctypes.c_int32
library.rrUTF16toUTF8.argtypes = [
u8p,
ctypes.c_int32,
u16p,
ctypes.c_int32,
ctypes.c_int32,
]
library.rrUTF16toUTF8.restype = ctypes.c_int32
library.iggyCompareUTF8toUTF16.argtypes = [
u8p,
ctypes.c_int32,
u16p,
ctypes.c_int32,
]
library.iggyCompareUTF8toUTF16.restype = ctypes.c_int32
for name, value_type in (
("rr_utf16_read_integer_hex_ecmascript", ctypes.c_int32),
("rr_utf16_read_hex_ecmascript", ctypes.c_double),
("rr_utf16_read_decimal_ecmascript", ctypes.c_double),
):
function = getattr(library, name)
function.argtypes = [
u16p,
ctypes.c_int32,
u16pp,
ctypes.POINTER(value_type),
ctypes.c_int32,
]
function.restype = ctypes.c_int32
for name in (
"util_format_literal",
"util_format_d",
"util_format_s",
"util_format_S",
"util_format_mix",
):
getattr(library, name).restype = None
@staticmethod
def u8(values, trailing_zero=True):
suffix = [0] if trailing_zero else []
return (ctypes.c_uint8 * (len(values) + len(suffix)))(*(values + suffix))
@staticmethod
def u16(values, trailing_zero=True, padding=0):
suffix = ([0] if trailing_zero else []) + ([0] * padding)
return (ctypes.c_uint16 * (len(values) + len(suffix)))(*(values + suffix))
@staticmethod
def utf16_text(text):
encoded = text.encode("utf-16-le")
values = [int.from_bytes(encoded[index : index + 2], "little") for index in range(0, len(encoded), 2)]
return ReconstructedAs3VmUtilTests.u16(values)
def utf8_to_utf16(self, library, values, source_length, capacity, replacement=0xFFFD, truncate=0):
source = self.u8(values)
output = (ctypes.c_uint16 * max(capacity + 4, 8))(*([0xA55A] * max(capacity + 4, 8)))
result = library.rrUTF8toUTF16(
output, capacity, source, source_length, replacement, truncate
)
return result, list(output)
def utf16_to_utf8(self, library, values, source_length, capacity, truncate=0):
source = self.u16(values, padding=2)
output = (ctypes.c_uint8 * max(capacity + 8, 12))(*([0xA5] * max(capacity + 8, 12)))
result = library.rrUTF16toUTF8(output, capacity, source, source_length, truncate)
return result, bytes(output)
def test_ascii_digit_predicate(self):
for platform, library in self.libraries.items():
actual = [library.rriswdigit(value) for value in (0x2F, 0x30, 0x39, 0x3A, 0x0660)]
self.assertEqual(actual, [0, 1, 1, 0, 0], platform)
def test_utf8_to_utf16_matches_native_length_replacement_and_truncation_rules(self):
fixtures = [
([0x41], 1, [0x0041]),
([0xC2, 0xA2], 2, [0x00A2]),
([0xE2, 0x82, 0xAC], 3, [0x20AC]),
([0xF0, 0x9F, 0x98, 0x80], 4, [0xD83D, 0xDE00]),
([0xED, 0xA0, 0x80], 3, [0xD800]),
([0x41, 0x00, 0x42], 3, [0x0041, 0x0000, 0x0042]),
]
for platform, library in self.libraries.items():
for encoded, source_length, expected in fixtures:
result, output = self.utf8_to_utf16(
library, encoded, source_length, len(expected) + 1
)
self.assertEqual(result, len(expected), (platform, encoded))
self.assertEqual(output[: len(expected) + 1], expected + [0], (platform, encoded))
malformed = self.u8([0xFF, 0x80, 0x80, 0x80])
output = (ctypes.c_uint16 * 8)()
self.assertEqual(library.rrUTF8toUTF16(output, 8, malformed, -1, 0x3F, 0), 4)
self.assertEqual(list(output[:5]), [0x3F, 0x3F, 0x3F, 0x3F, 0])
self.assertEqual(library.rrUTF8toUTF16(output, 8, malformed, 4, 0x3F, 0), 1)
self.assertEqual(list(output[:2]), [0x3F, 0])
self.assertEqual(library.rrUTF8toUTF16(output, 8, malformed, 4, -1, 0), -2)
result, output_values = self.utf8_to_utf16(
library, [0xF0, 0x9F, 0x98, 0x80], 4, 1, truncate=0
)
self.assertEqual(result, -1, platform)
self.assertEqual(output_values[0], 0xA55A, platform)
result, output_values = self.utf8_to_utf16(
library, [0xF0, 0x9F, 0x98, 0x80], 4, 1, truncate=1
)
self.assertEqual(result, 0, platform)
self.assertEqual(output_values[0], 0xA55A, platform)
fixed_result, fixed_output = self.utf8_to_utf16(library, [0x41], 1, 1)
self.assertEqual(fixed_result, 1, platform)
self.assertEqual(fixed_output[:2], [0x41, 0xA55A], platform)
null_result, _ = self.utf8_to_utf16(library, [0x41], -1, 1)
self.assertEqual(null_result, -1, platform)
null_result, _ = self.utf8_to_utf16(library, [0x41], -1, 1, truncate=1)
self.assertEqual(null_result, 1, platform)
query_source = self.u8([0xF0, 0x9F, 0x98, 0x80])
self.assertEqual(
library.rrUTF8toUTF16(None, 0, query_source, 4, 0xFFFD, 0), 2
)
def test_utf16_to_utf8_preserves_native_terminator_and_base_pointer_bug(self):
for platform, library in self.libraries.items():
result, output = self.utf16_to_utf8(library, [0x41], 1, 2)
self.assertEqual((result, output[:2]), (1, b"A\x00"), platform)
result, output = self.utf16_to_utf8(library, [0x41, 0, 0x42], 3, 4)
self.assertEqual((result, output[:4]), (3, b"A\x00B\x00"), platform)
result, output = self.utf16_to_utf8(library, [0x41], -1, 2)
self.assertEqual((result, output[:2]), (2, b"A\x00"), platform)
source = self.u16([0x41])
self.assertEqual(library.rrUTF16toUTF8(None, 0, source, -1, 0), 2)
result, output = self.utf16_to_utf8(library, [0xD83D, 0xDE00], 2, 5)
self.assertEqual((result, output[:5]), (4, bytes.fromhex("f09f988000")), platform)
result, output = self.utf16_to_utf8(library, [0x41, 0xD83D, 0xDE00], 3, 8)
self.assertEqual((result, output[:8]), (7, bytes.fromhex("41eda0bdedb88000")), platform)
result, _ = self.utf16_to_utf8(library, [0xDE00], 1, 8)
self.assertEqual(result, -2, platform)
result, _ = self.utf16_to_utf8(library, [0xD83D], 1, 8)
self.assertEqual(result, -2, platform)
result, _ = self.utf16_to_utf8(
library, [0xD83D, 0x0041], 2, 3, truncate=0
)
self.assertEqual(result, -1, platform)
result, _ = self.utf16_to_utf8(
library, [0xD83D, 0x0041], 2, 3, truncate=1
)
self.assertEqual(result, 0, platform)
result, output = self.utf16_to_utf8(library, [0x20AC], 1, 2, truncate=0)
self.assertEqual(result, -1, platform)
self.assertEqual(output[0], 0xA5, platform)
result, _ = self.utf16_to_utf8(library, [0x20AC], 1, 2, truncate=1)
self.assertEqual(result, 0, platform)
def test_utf8_utf16_comparator_is_equality_oriented(self):
fixtures = [
([0x41], 1, [0x41], 1, 0),
([0x41, 0x42], 2, [0x41], 1, 1),
([0x41], 1, [0x42], 1, -1),
([0x41], 1, [0x41, 0x42], 2, -1),
([0xF0, 0x9F, 0x98, 0x80], 4, [0xD83D, 0xDE00], 2, -1),
([0xF0, 0x9F, 0x98, 0x80], 4, [0xD83D, 0xDE00], 1, 0),
([0xC1, 0x81], 2, [0x41], 1, -1),
([0x41, 0x00, 0x42], 3, [0x41, 0x00, 0x42], 3, 0),
]
for platform, library in self.libraries.items():
for u8_values, u8_len, u16_values, u16_len, expected in fixtures:
actual = library.iggyCompareUTF8toUTF16(
self.u8(u8_values), u8_len, self.u16(u16_values, padding=1), u16_len
)
self.assertEqual(actual, expected, (platform, u8_values, u16_values))
self.assertEqual(
library.iggyCompareUTF8toUTF16(
self.u8([0x41]), -1, self.u16([0x41]), -1
),
0,
platform,
)
def run_parser(self, library, name, values, partial, value_type, sentinel):
source = self.u16(values, padding=2)
source_pointer = ctypes.cast(source, ctypes.POINTER(ctypes.c_uint16))
end = ctypes.cast(
ctypes.byref(source, ctypes.sizeof(ctypes.c_uint16)),
ctypes.POINTER(ctypes.c_uint16),
)
value = value_type(sentinel)
result = getattr(library, name)(
source_pointer, len(values), ctypes.byref(end), ctypes.byref(value), partial
)
end_index = (
ctypes.cast(end, ctypes.c_void_p).value - ctypes.addressof(source)
) // ctypes.sizeof(ctypes.c_uint16)
return result, end_index, value.value
def test_ecmascript_hex_readers_match_whitespace_partial_and_wrap_rules(self):
whitespace = [
*range(0x0009, 0x000E), 0x0020, 0x00A0, 0x1680, 0x180E,
*range(0x2000, 0x200B), 0x2028, 0x2029, 0x202F, 0x205F, 0x3000,
]
for platform, library in self.libraries.items():
for name, value_type in (
("rr_utf16_read_integer_hex_ecmascript", ctypes.c_int32),
("rr_utf16_read_hex_ecmascript", ctypes.c_double),
):
result, end, value = self.run_parser(library, name, [0x30, 0x78], 0, value_type, 77)
self.assertEqual((result, end, value), (1, 2, 0), (platform, name))
result, end, value = self.run_parser(
library, name, [0x20, 0x30, 0x58, 0x31, 0x66, 0x20], 0, value_type, 77
)
self.assertEqual((result, end, value), (1, 6, 31), (platform, name))
result, end, value = self.run_parser(
library, name, [0x30, 0x78, 0x31, 0x30, 0x7A], 1, value_type, 77
)
self.assertEqual((result, end, value), (1, 4, 16), (platform, name))
result, end, value = self.run_parser(
library, name, [0x30, 0x78, 0x31, 0x30, 0x7A], 0, value_type, 77
)
self.assertEqual((result, end, value), (0, 1, 77), (platform, name))
result, end, value = self.run_parser(
library,
"rr_utf16_read_integer_hex_ecmascript",
[0x30, 0x78, *map(ord, "FFFFFFFF")],
0,
ctypes.c_int32,
77,
)
self.assertEqual((result, end, value), (1, 10, -1), platform)
for unit in whitespace:
result, _, value = self.run_parser(
library,
"rr_utf16_read_integer_hex_ecmascript",
[unit, 0x30, 0x78],
0,
ctypes.c_int32,
77,
)
self.assertEqual((result, value), (1, 0), (platform, unit))
result, end, value = self.run_parser(
library,
"rr_utf16_read_integer_hex_ecmascript",
[0xFEFF, 0x30, 0x78],
0,
ctypes.c_int32,
77,
)
self.assertEqual((result, end, value), (0, 1, 77), platform)
def test_ecmascript_decimal_reader_preserves_signed_zero_and_exponent_bug(self):
name = "rr_utf16_read_decimal_ecmascript"
for platform, library in self.libraries.items():
for text, expected in (("123.5", 123.5), (".", 0.0), ("1.", 1.0)):
values = list(map(ord, text))
result, end, value = self.run_parser(
library, name, values, 0, ctypes.c_double, 77.0
)
self.assertEqual((result, end, value), (1, len(values), expected), (platform, text))
for text in ("-0", "-."):
result, end, value = self.run_parser(
library, name, list(map(ord, text)), 0, ctypes.c_double, 77.0
)
self.assertEqual((result, end, value), (1, len(text), -0.0), (platform, text))
self.assertLess(math.copysign(1.0, value), 0.0, (platform, text))
result, end, value = self.run_parser(
library, name, list(map(ord, "1e2")), 1, ctypes.c_double, 77.0
)
self.assertEqual((result, end, value), (1, 2, 1.0), platform)
result, end, value = self.run_parser(
library, name, list(map(ord, "1e2")), 0, ctypes.c_double, 77.0
)
self.assertEqual((result, end, value), (0, 1, 77.0), platform)
result, end, value = self.run_parser(
library, name, list(map(ord, "1e+")), 1, ctypes.c_double, 77.0
)
self.assertEqual((result, end, value), (0, 1, 77.0), platform)
def read_u16_output(self, output):
values = []
for value in output:
if value == 0:
break
values.append(value)
return bytes(sum(([value & 0xFF, value >> 8] for value in values), [])).decode("utf-16-le")
def test_utf16_simple_printf_matches_native_formats_and_literal_truncation(self):
for platform, library in self.libraries.items():
output = (ctypes.c_uint16 * 64)()
library.util_format_literal(output, 64, self.utf16_text("A%%B %q %"))
self.assertEqual(self.read_u16_output(output), "A%B q ", platform)
library.util_format_d(output, 64, self.utf16_text("%d"), -0x80000000)
self.assertEqual(self.read_u16_output(output), "-2147483648", platform)
library.util_format_s(output, 64, self.utf16_text("%s"), None)
self.assertEqual(self.read_u16_output(output), "(null)", platform)
library.util_format_S(output, 64, self.utf16_text("%S"), 3, self.utf16_text("hello"))
self.assertEqual(self.read_u16_output(output), "hel", platform)
output = (ctypes.c_uint16 * 16)(*([0xA55A] * 16))
library.util_format_s(
output, 3, self.utf16_text("%s"), self.utf16_text("hello")
)
self.assertEqual(
list(output[:7]),
[ord("h"), ord("e"), ord("l"), ord("l"), ord("o"), 0, 0xA55A],
platform,
)
library.util_format_mix(
output,
64,
self.utf16_text("%d %s %S"),
-42,
self.utf16_text("xy"),
3,
self.utf16_text("hello"),
)
self.assertEqual(self.read_u16_output(output), "-42 xy hel", platform)
output = (ctypes.c_uint16 * 8)(*([0xA55A] * 8))
library.util_format_literal(output, 4, self.utf16_text("abcdef"))
self.assertEqual(list(output[:5]), [ord("a"), ord("b"), ord("c"), 0, 0xA55A], platform)
output = (ctypes.c_uint16 * 2)(0xA55A, 0xA55A)
library.util_format_literal(output, 0, self.utf16_text("a"))
self.assertEqual(list(output), [0xA55A, 0xA55A], platform)
def test_objects_preserve_exports_dependencies_and_cross_language_header(self):
temporary_path = Path(self.temporary_directory.name)
exports = {
"rriswdigit",
"rrUTF8toUTF16",
"rrUTF16toUTF8",
"iggyCompareUTF8toUTF16",
"rr_utf16_read_integer_hex_ecmascript",
"rr_utf16_read_hex_ecmascript",
"rr_utf16_read_decimal_ecmascript",
"rr_utf16_simple_printf",
}
for platform, define in VARIANTS.items():
object_path = temporary_path / f"as3vm_util_{platform}.o"
result = subprocess.run(
[
"clang", "-c", "-std=c11", "-O2", "-fno-builtin",
"-Wall", "-Wextra", "-Werror", f"-D{define}=1",
"-I", str(RECONSTRUCTED), str(SOURCE), "-o", str(object_path),
],
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
nm_result = subprocess.run(["nm", "-g", str(object_path)], capture_output=True, text=True)
self.assertEqual(nm_result.returncode, 0, nm_result.stderr)
defined = {
line.split()[-1].removeprefix("_")
for line in nm_result.stdout.splitlines()
if " T " in line
}
undefined = {
line.split()[-1].removeprefix("_")
for line in nm_result.stdout.splitlines()
if " U " in line
}
self.assertEqual(defined, exports, platform)
expected_undefined = {"rrwcslen", "strlen"}
if platform != "ps3":
expected_undefined.add("pow")
if platform == "durango":
expected_undefined.remove("strlen")
undefined -= {"__stack_chk_fail", "__stack_chk_guard"}
self.assertEqual(undefined, expected_undefined, platform)
syntax = subprocess.run(
[
"clang", "-target", "i386-unknown-linux-gnu", "-ffreestanding",
"-fsyntax-only", "-std=c11", "-Wall", "-Wextra", "-Werror",
f"-D{define}=1", "-I", str(RECONSTRUCTED), str(SOURCE),
],
capture_output=True,
text=True,
)
self.assertEqual(syntax.returncode, 0, syntax.stdout + syntax.stderr)
cpp_path = temporary_path / "as3vm_util_cpp_linkage.cpp"
cpp_path.write_text(
'#include "as3vm_util.h"\n'
"void use_api(uint16_t *u16, uint8_t *u8) {\n"
" (void)rrUTF8toUTF16(u16, 0, u8, 0, -1, 0);\n"
" (void)rrUTF16toUTF8(u8, 0, u16, 0, 0);\n"
" rr_utf16_simple_printf(u16, 0, u16);\n"
"}\n",
encoding="utf-8",
)
result = subprocess.run(
[
"clang++", "-fsyntax-only", "-std=c++17", "-Wall", "-Wextra",
"-Werror", "-I", str(RECONSTRUCTED), str(cpp_path),
],
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)