thread.c

/*
+----------------------------------------------------------------------+
  | Copyright (c) The PHP Group                                          |
  +----------------------------------------------------------------------+
  | This source file is subject to version 3.01 of the PHP license,      |
  | that is bundled with this package in the file LICENSE, and is        |
  | available through the world-wide-web at the following url:           |
  | https://www.php.net/license/3_01.txt                                 |
  | If you did not receive a copy of the PHP license and are unable to   |
  | obtain it through the world-wide-web, please send a note to          |
  | license@php.net so we can mail you a copy immediately.               |
  +----------------------------------------------------------------------+
  | Author: Edmond                                                       |
  +----------------------------------------------------------------------+
*/

#include "thread.h"
#include "thread_arginfo.h"
#include "coroutine.h"
#include "exceptions.h"
#include "libuv_reactor.h"
#include "php_async.h"
#include "php.h"
#include "php_main.h"
#include "SAPI.h"
#include "zend.h"
#include "zend_API.h"
#include "zend_compile.h"
#include "zend_autoload.h"
#include "zend_hash.h"
#include "zend_exceptions.h"
#include "zend_enum.h"
#include "zend_attributes.h"
#include "zend_vm.h"
#include "zend_interfaces.h"
#include "zend_closures.h"
#include "zend_common.h"
#include "zend_map_ptr.h"
#include "main/SAPI.h"
#include "main/php_streams.h"
#include "zend_constants.h"

///////////////////////////////////////////////////////////
/// 0. Deep copy — arena-based op_array copy
///////////////////////////////////////////////////////////

/**
 * Deep copy context: bump allocator + xlat table for pointer deduplication.
 * All copies go into persistent arena memory so child threads can access
 * them safely. The arena is freed as a whole on snapshot destroy.
 *
 * Using a bump allocator instead of individual pemalloc calls guarantees
 * that opcodes and literals are in the same memory region, keeping
 * RT_CONSTANT int32_t relative offsets within range.
 */
#define THREAD_COPY_ARENA_BLOCK_SIZE (64 * 1024)

typedef struct {
	thread_copy_arena_block_t *current_block;
	HashTable xlat;  /* old_ptr → new_ptr for deduplication */
} thread_copy_ctx_t;

static void thread_copy_ctx_init(thread_copy_ctx_t *ctx)
{
	thread_copy_arena_block_t *block = pemalloc(
		sizeof(thread_copy_arena_block_t) + THREAD_COPY_ARENA_BLOCK_SIZE, 1);
	block->prev = NULL;
	block->size = THREAD_COPY_ARENA_BLOCK_SIZE;
	block->offset = 0;
	ctx->current_block = block;
	zend_hash_init(&ctx->xlat, 128, NULL, NULL, 1);
}

static zend_always_inline void *thread_copy_arena_alloc(thread_copy_ctx_t *ctx, size_t size)
{
	size = ZEND_MM_ALIGNED_SIZE(size);
	thread_copy_arena_block_t *block = ctx->current_block;

	if (EXPECTED(block->offset + size <= block->size)) {
		void *ptr = block->data + block->offset;
		block->offset += size;
		return ptr;
	}

	/* Current block exhausted — allocate a new one */
	size_t new_size = block->size * 2;
	if (UNEXPECTED(new_size < size)) {
		new_size = size;
	}
	thread_copy_arena_block_t *new_block = pemalloc(
		sizeof(thread_copy_arena_block_t) + new_size, 1);
	new_block->prev = block;
	new_block->size = new_size;
	new_block->offset = size;
	ctx->current_block = new_block;
	return new_block->data;
}

static void thread_copy_ctx_destroy(thread_copy_ctx_t *ctx)
{
	zend_hash_destroy(&ctx->xlat);
	/* Arena blocks are NOT freed here — ownership transfers to the snapshot */
}

static void thread_copy_arena_free(thread_copy_arena_block_t *block)
{
	while (block) {
		thread_copy_arena_block_t *prev = block->prev;
		pefree(block, 1);
		block = prev;
	}
}

/* Copy memory into arena, register in xlat (update semantics — overwrites existing) */
static void *thread_persist_copy_xlat(thread_copy_ctx_t *ctx, const void *src, size_t size)
{
	void *dst = thread_copy_arena_alloc(ctx, size);
	memcpy(dst, src, size);
	zend_hash_index_update_ptr(&ctx->xlat, async_ptr_to_index((void *)src), dst);
	return dst;
}

/* Copy memory into arena, register in xlat (add semantics — fails silently on duplicate) */
static void *thread_persist_copy(thread_copy_ctx_t *ctx, const void *src, size_t size)
{
	void *dst = thread_copy_arena_alloc(ctx, size);
	memcpy(dst, src, size);
	zend_hash_index_add_ptr(&ctx->xlat, async_ptr_to_index((void *)src), dst);
	return dst;
}

/* Look up a previously copied pointer */
static zend_always_inline void *thread_xlat_get(const thread_copy_ctx_t *ctx, const void *old_ptr)
{
	return zend_hash_index_find_ptr(&ctx->xlat, async_ptr_to_index((void *)old_ptr));
}


/* Canonical uninitialized bucket for empty hash tables */
static const uint32_t thread_uninitialized_bucket[-HT_MIN_MASK] =
	{HT_INVALID_IDX, HT_INVALID_IDX};

/* Copy a string into persistent memory with interned+permanent flags */
static zend_string *thread_copy_string(thread_copy_ctx_t *ctx, const zend_string *str)
{
	if (!str) {
		return NULL;
	}

	zend_string *new_str = thread_xlat_get(ctx, str);
	if (new_str) {
		return new_str;
	}

	new_str = thread_persist_copy_xlat(ctx, str, _ZSTR_STRUCT_SIZE(ZSTR_LEN(str)));
	zend_string_hash_val(new_str);
	GC_SET_REFCOUNT(new_str, 2);

	const uint32_t flags = GC_STRING
		| (ZSTR_IS_VALID_UTF8(new_str) ? IS_STR_VALID_UTF8 : 0)
		| ((IS_STR_INTERNED | IS_STR_PERMANENT) << GC_FLAGS_SHIFT);
	GC_TYPE_INFO(new_str) = flags;

	return new_str;
}

/* Forward declarations */
static void thread_copy_zval(thread_copy_ctx_t *ctx, zval *z);
static void thread_copy_hash_table(thread_copy_ctx_t *ctx, HashTable *ht);
static zend_ast *thread_copy_ast(thread_copy_ctx_t *ctx, zend_ast *ast);
static void thread_copy_op_array(thread_copy_ctx_t *ctx, zval *zv);
static void thread_copy_op_array_ex(thread_copy_ctx_t *ctx, zend_op_array *op_array);
static void thread_copy_type(thread_copy_ctx_t *ctx, zend_type *type);
static HashTable *thread_copy_attributes(thread_copy_ctx_t *ctx, HashTable *attributes);

/* {{{ thread_copy_hash_table — relocate HashTable internals to pemalloc */
static void thread_copy_hash_table(thread_copy_ctx_t *ctx, HashTable *ht)
{
	HT_FLAGS(ht) |= HASH_FLAG_STATIC_KEYS;
	ht->pDestructor = NULL;
	ht->nInternalPointer = 0;

	if (HT_FLAGS(ht) & HASH_FLAG_UNINITIALIZED) {
		HT_SET_DATA_ADDR(ht, &thread_uninitialized_bucket);
		return;
	}

	if (ht->nNumUsed == 0) {
		ht->nTableMask = HT_MIN_MASK;
		HT_SET_DATA_ADDR(ht, &thread_uninitialized_bucket);
		HT_FLAGS(ht) |= HASH_FLAG_UNINITIALIZED;
		return;
	}

	if (HT_IS_PACKED(ht)) {
		void *data = thread_persist_copy(ctx, HT_GET_DATA_ADDR(ht), HT_PACKED_USED_SIZE(ht));
		HT_SET_DATA_ADDR(ht, data);
	} else {
		void *data = thread_persist_copy(ctx, HT_GET_DATA_ADDR(ht), HT_USED_SIZE(ht));
		HT_SET_DATA_ADDR(ht, data);
	}
}
/* }}} */

/* {{{ thread_copy_ast */
static zend_ast *thread_copy_ast(thread_copy_ctx_t *ctx, zend_ast *ast)
{
	zend_ast *node;

	if (ast->kind == ZEND_AST_ZVAL || ast->kind == ZEND_AST_CONSTANT) {
		zend_ast_zval *copy = thread_persist_copy(ctx, ast, sizeof(zend_ast_zval));
		thread_copy_zval(ctx, &copy->val);
		node = (zend_ast *) copy;
	} else if (zend_ast_is_list(ast)) {
		zend_ast_list *list = zend_ast_get_list(ast);
		zend_ast_list *copy = thread_persist_copy(ctx, ast,
			sizeof(zend_ast_list) - sizeof(zend_ast *) + sizeof(zend_ast *) * list->children);
		for (uint32_t i = 0; i < list->children; i++) {
			if (copy->child[i]) {
				copy->child[i] = thread_copy_ast(ctx, copy->child[i]);
			}
		}
		node = (zend_ast *) copy;
	} else if (ast->kind == ZEND_AST_OP_ARRAY) {
		zend_ast_op_array *copy = thread_persist_copy(ctx, ast, sizeof(zend_ast_op_array));
		zval z;
		ZVAL_PTR(&z, copy->op_array);
		thread_copy_op_array(ctx, &z);
		copy->op_array = Z_PTR(z);
		node = (zend_ast *) copy;
	} else if (ast->kind == ZEND_AST_CALLABLE_CONVERT) {
		zend_ast_fcc *copy = thread_persist_copy(ctx, ast, sizeof(zend_ast_fcc));
		copy->args = thread_copy_ast(ctx, copy->args);
		node = (zend_ast *) copy;
	} else {
		const uint32_t children = zend_ast_get_num_children(ast);
		node = thread_persist_copy(ctx, ast, zend_ast_size(children));
		for (uint32_t i = 0; i < children; i++) {
			if (node->child[i]) {
				node->child[i] = thread_copy_ast(ctx, node->child[i]);
			}
		}
	}

	return node;
}
/* }}} */

/* {{{ thread_copy_zval */
static void thread_copy_zval(thread_copy_ctx_t *ctx, zval *z)
{
	switch (Z_TYPE_P(z)) {
		case IS_STRING:
			Z_STR_P(z) = thread_copy_string(ctx, Z_STR_P(z));
			Z_TYPE_FLAGS_P(z) = 0;
			break;
		case IS_ARRAY: {
			void *new_ptr = thread_xlat_get(ctx, Z_ARR_P(z));
			if (new_ptr) {
				Z_ARR_P(z) = new_ptr;
				Z_TYPE_FLAGS_P(z) = 0;
			} else {
				HashTable *ht = thread_persist_copy_xlat(ctx, Z_ARR_P(z), sizeof(zend_array));
				Z_ARR_P(z) = ht;
				thread_copy_hash_table(ctx, ht);
				if (HT_IS_PACKED(ht)) {
					zval *zv;
					ZEND_HASH_PACKED_FOREACH_VAL(ht, zv) {
						thread_copy_zval(ctx, zv);
					} ZEND_HASH_FOREACH_END();
				} else {
					Bucket *p;
					ZEND_HASH_MAP_FOREACH_BUCKET(ht, p) {
						if (p->key) {
							p->key = thread_copy_string(ctx, p->key);
						}
						thread_copy_zval(ctx, &p->val);
					} ZEND_HASH_FOREACH_END();
				}
				Z_TYPE_FLAGS_P(z) = 0;
				GC_SET_REFCOUNT(Z_COUNTED_P(z), 2);
				GC_ADD_FLAGS(Z_COUNTED_P(z), IS_ARRAY_IMMUTABLE);
			}
			break;
		}
		case IS_CONSTANT_AST: {
			void *new_ptr = thread_xlat_get(ctx, Z_AST_P(z));
			if (new_ptr) {
				Z_AST_P(z) = new_ptr;
				Z_TYPE_FLAGS_P(z) = 0;
			} else {
				zend_ast_ref *old_ref = Z_AST_P(z);
				Z_AST_P(z) = thread_persist_copy_xlat(ctx, Z_AST_P(z), sizeof(zend_ast_ref));

				/* GC_AST reads the tree from just past the header, so the two
				 * allocations have to be neighbours. The arena bumps a pointer
				 * and normally makes them so; it opens a fresh block when a
				 * request does not fit, and a header landing in the last bytes
				 * of a block would leave the root behind in the old one. */
				const zend_ast *root = thread_copy_ast(ctx, GC_AST(old_ref));
				ZEND_ASSERT(root == GC_AST(Z_AST_P(z)));
				Z_TYPE_FLAGS_P(z) = 0;
				GC_SET_REFCOUNT(Z_COUNTED_P(z), 1);
				GC_ADD_FLAGS(Z_COUNTED_P(z), GC_IMMUTABLE);
			}
			break;
		}
		case IS_PTR:
			break;
		default:
			ZEND_ASSERT(Z_TYPE_P(z) < IS_STRING);
			break;
	}
}
/* }}} */

/* {{{ thread_copy_attributes */
static HashTable *thread_copy_attributes(thread_copy_ctx_t *ctx, HashTable *attributes)
{
	HashTable *xlat = thread_xlat_get(ctx, attributes);
	if (xlat) {
		return xlat;
	}

	/* Copy the HashTable struct first, then relocate internals on the copy,
	 * so the original HashTable stays untouched. */
	attributes = thread_persist_copy_xlat(ctx, attributes, sizeof(HashTable));
	thread_copy_hash_table(ctx, attributes);

	zval *v;
	ZEND_HASH_PACKED_FOREACH_VAL(attributes, v) {
		const zend_attribute *attr = Z_PTR_P(v);
		zend_attribute *copy = thread_persist_copy_xlat(ctx, attr, ZEND_ATTRIBUTE_SIZE(attr->argc));

		copy->name = thread_copy_string(ctx, copy->name);
		copy->lcname = thread_copy_string(ctx, copy->lcname);
		if (copy->validation_error) {
			copy->validation_error = thread_copy_string(ctx, copy->validation_error);
		}

		for (uint32_t i = 0; i < copy->argc; i++) {
			if (copy->args[i].name) {
				copy->args[i].name = thread_copy_string(ctx, copy->args[i].name);
			}
			thread_copy_zval(ctx, &copy->args[i].value);
		}

		ZVAL_PTR(v, copy);
	} ZEND_HASH_FOREACH_END();

	GC_SET_REFCOUNT(attributes, 2);
	GC_TYPE_INFO(attributes) = GC_ARRAY | ((IS_ARRAY_IMMUTABLE|GC_NOT_COLLECTABLE) << GC_FLAGS_SHIFT);

	return attributes;
}
/* }}} */

/* {{{ thread_copy_type */
static void thread_copy_type(thread_copy_ctx_t *ctx, zend_type *type)
{
	if (ZEND_TYPE_HAS_LIST(*type)) {
		zend_type_list *list = ZEND_TYPE_LIST(*type);
		list = thread_persist_copy_xlat(ctx, list, ZEND_TYPE_LIST_SIZE(list->num_types));
		ZEND_TYPE_FULL_MASK(*type) &= ~_ZEND_TYPE_ARENA_BIT;
		ZEND_TYPE_SET_PTR(*type, list);
	}

	zend_type *single_type;
	ZEND_TYPE_FOREACH_MUTABLE(*type, single_type) {
		if (ZEND_TYPE_HAS_LIST(*single_type)) {
			thread_copy_type(ctx, single_type);
			continue;
		}
		if (ZEND_TYPE_HAS_NAME(*single_type)) {
			zend_string *type_name = ZEND_TYPE_NAME(*single_type);
			type_name = thread_copy_string(ctx, type_name);
			ZEND_TYPE_SET_PTR(*single_type, type_name);
			/* Skip zend_accel_get_class_name_map_ptr — not safe without OPcache */
		}
	} ZEND_TYPE_FOREACH_END();
}
/* }}} */

/* {{{ thread_copy_op_array_ex — deep copy op_array internals */
static void thread_copy_op_array_ex(thread_copy_ctx_t *ctx, zend_op_array *op_array)
{
	const zval *orig_literals = NULL;

	/* refcount: detach from parent's shared counter without modifying it */
	op_array->refcount = NULL;

	/* Detach per-op_array extension slots. reserved[zend_func_info_rid] holds
	 * opcache's zend_jit_op_array_trace_extension — a zend_shared_alloc'd,
	 * process-wide struct whose ->op_array back-pointer is the SOURCE op_array.
	 * If the copy keeps it, a worker tracing the copy writes its trace result
	 * through that shared extension and JIT-patches the source op_array with
	 * code compiled from the worker's run — corrupting the owning thread.
	 * The copy runs interpreter-only (handlers are re-specialized below), so
	 * it must not carry any extension's per-op_array state. */
	memset(op_array->reserved, 0, sizeof(op_array->reserved));

	if (op_array->function_name) {
		op_array->function_name = thread_copy_string(ctx, op_array->function_name);
	}

	if (op_array->scope) {
		zend_class_entry *scope = thread_xlat_get(ctx, op_array->scope);
		if (scope) {
			op_array->scope = scope;
		}

		if (op_array->prototype) {
			zend_function *ptr = thread_xlat_get(ctx, op_array->prototype);
			if (ptr) {
				op_array->prototype = ptr;
			}
		}

		/* Check if opcodes were already copied (shared method) */
		zend_op *persist_ptr = thread_xlat_get(ctx, op_array->opcodes);
		if (persist_ptr) {
			op_array->opcodes = persist_ptr;
			if (op_array->static_variables) {
				op_array->static_variables = thread_xlat_get(ctx, op_array->static_variables);
				ZEND_ASSERT(op_array->static_variables != NULL);
			}
			if (op_array->literals) {
				op_array->literals = thread_xlat_get(ctx, op_array->literals);
				ZEND_ASSERT(op_array->literals != NULL);
			}
			if (op_array->filename) {
				op_array->filename = thread_xlat_get(ctx, op_array->filename);
				ZEND_ASSERT(op_array->filename != NULL);
			}
			if (op_array->arg_info) {
				zend_arg_info *arg_info = op_array->arg_info;
				if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
					arg_info--;
				}
				arg_info = thread_xlat_get(ctx, arg_info);
				ZEND_ASSERT(arg_info != NULL);
				if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
					arg_info++;
				}
				op_array->arg_info = arg_info;
			}
			if (op_array->live_range) {
				op_array->live_range = thread_xlat_get(ctx, op_array->live_range);
				ZEND_ASSERT(op_array->live_range != NULL);
			}
			if (op_array->doc_comment) {
				op_array->doc_comment = thread_xlat_get(ctx, op_array->doc_comment);
			}
			if (op_array->attributes) {
				op_array->attributes = thread_xlat_get(ctx, op_array->attributes);
				ZEND_ASSERT(op_array->attributes != NULL);
			}
			if (op_array->try_catch_array) {
				op_array->try_catch_array = thread_xlat_get(ctx, op_array->try_catch_array);
				ZEND_ASSERT(op_array->try_catch_array != NULL);
			}
			if (op_array->vars) {
				op_array->vars = thread_xlat_get(ctx, op_array->vars);
				ZEND_ASSERT(op_array->vars != NULL);
			}
			if (op_array->dynamic_func_defs) {
				op_array->dynamic_func_defs = thread_xlat_get(ctx, op_array->dynamic_func_defs);
				ZEND_ASSERT(op_array->dynamic_func_defs != NULL);
			}
			return;
		}
	} else {
		op_array->prototype = NULL;
	}

	/* static_variables — copy the HashTable struct first, then relocate
	 * internals on the copy, so the original HashTable stays untouched. */
	if (op_array->static_variables) {
		Bucket *p;
		op_array->static_variables = thread_persist_copy_xlat(ctx, op_array->static_variables, sizeof(HashTable));
		thread_copy_hash_table(ctx, op_array->static_variables);
		ZEND_HASH_MAP_FOREACH_BUCKET(op_array->static_variables, p) {
			ZEND_ASSERT(p->key != NULL);
			p->key = thread_copy_string(ctx, p->key);
			thread_copy_zval(ctx, &p->val);
		} ZEND_HASH_FOREACH_END();
		GC_SET_REFCOUNT(op_array->static_variables, 2);
		GC_TYPE_INFO(op_array->static_variables) = GC_ARRAY | ((IS_ARRAY_IMMUTABLE|GC_NOT_COLLECTABLE) << GC_FLAGS_SHIFT);
	}

	/* literals */
	if (op_array->literals) {
		orig_literals = op_array->literals;
		zval *p = thread_persist_copy_xlat(ctx, op_array->literals,
			sizeof(zval) * op_array->last_literal);
		const zval *end = p + op_array->last_literal;
		op_array->literals = p;
		while (p < end) {
			thread_copy_zval(ctx, p);
			p++;
		}
	}

	/* opcodes */
	{
		zend_op *new_opcodes = thread_persist_copy_xlat(ctx, op_array->opcodes, sizeof(zend_op) * op_array->last);
		zend_op *opline = new_opcodes;
		const zend_op *end = new_opcodes + op_array->last;

		for (; opline < end; opline++) {
#if ZEND_USE_ABS_CONST_ADDR
			if (opline->op1_type == IS_CONST) {
				opline->op1.zv = (zval*)((char*)opline->op1.zv + ((char*)op_array->literals - (char*)orig_literals));
				if (opline->opcode == ZEND_SEND_VAL
				 || opline->opcode == ZEND_SEND_VAL_EX
				 || opline->opcode == ZEND_QM_ASSIGN) {
					zend_vm_set_opcode_handler_ex(opline, 1 << Z_TYPE_P(opline->op1.zv), 0, 0);
				}
			}
			if (opline->op2_type == IS_CONST) {
				opline->op2.zv = (zval*)((char*)opline->op2.zv + ((char*)op_array->literals - (char*)orig_literals));
			}
#else
			if (opline->op1_type == IS_CONST) {
				opline->op1.constant =
					(char*)(op_array->literals +
						((zval*)((char*)(op_array->opcodes + (opline - new_opcodes)) +
						(int32_t)opline->op1.constant) - orig_literals)) -
					(char*)opline;
				if (opline->opcode == ZEND_SEND_VAL
				 || opline->opcode == ZEND_SEND_VAL_EX
				 || opline->opcode == ZEND_QM_ASSIGN) {
					zend_vm_set_opcode_handler_ex(opline, 0, 0, 0);
				}
			}
			if (opline->op2_type == IS_CONST) {
				opline->op2.constant =
					(char*)(op_array->literals +
						((zval*)((char*)(op_array->opcodes + (opline - new_opcodes)) +
						(int32_t)opline->op2.constant) - orig_literals)) -
					(char*)opline;
			}
#endif
#if ZEND_USE_ABS_JMP_ADDR
			if (op_array->fn_flags & ZEND_ACC_DONE_PASS_TWO) {
				switch (opline->opcode) {
					case ZEND_JMP:
					case ZEND_FAST_CALL:
						opline->op1.jmp_addr = &new_opcodes[opline->op1.jmp_addr - op_array->opcodes];
						break;
					case ZEND_JMPZ:
					case ZEND_JMPNZ:
					case ZEND_JMPZ_EX:
					case ZEND_JMPNZ_EX:
					case ZEND_JMP_SET:
					case ZEND_COALESCE:
					case ZEND_FE_RESET_R:
					case ZEND_FE_RESET_RW:
					case ZEND_ASSERT_CHECK:
					case ZEND_JMP_NULL:
					case ZEND_BIND_INIT_STATIC_OR_JMP:
					case ZEND_JMP_FRAMELESS:
						opline->op2.jmp_addr = &new_opcodes[opline->op2.jmp_addr - op_array->opcodes];
						break;
					case ZEND_CATCH:
						if (!(opline->extended_value & ZEND_LAST_CATCH)) {
							opline->op2.jmp_addr = &new_opcodes[opline->op2.jmp_addr - op_array->opcodes];
						}
						break;
					case ZEND_FE_FETCH_R:
					case ZEND_FE_FETCH_RW:
					case ZEND_SWITCH_LONG:
					case ZEND_SWITCH_STRING:
					case ZEND_MATCH:
						break;
				}
			}
#endif
			if (opline->opcode == ZEND_OP_DATA
					&& (opline-1)->opcode == ZEND_DECLARE_ATTRIBUTED_CONST) {
				zval *literal = RT_CONSTANT(opline, opline->op1);
				HashTable *attributes = Z_PTR_P(literal);
				attributes = thread_copy_attributes(ctx, attributes);
				ZVAL_PTR(literal, attributes);
			}

			/* Reset opcode handler to the standard VM handler. If opcache JIT
			 * has compiled the source op_array, opline->handler was replaced
			 * by a JIT stub with the source op_array's addresses baked in —
			 * executing that stub on our copy (different literals, fresh
			 * run_time_cache, child-thread map_ptr_base) crashes. Re-specialize
			 * from the VM handler table so the copy runs on the interpreter. */
			zend_vm_set_opcode_handler(opline);
		}

		op_array->opcodes = new_opcodes;
	}

	/* filename */
	if (op_array->filename) {
		op_array->filename = thread_copy_string(ctx, op_array->filename);
	}

	/* arg_info */
	if (op_array->arg_info) {
		zend_arg_info *arg_info = op_array->arg_info;
		uint32_t num_args = op_array->num_args;

		if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
			arg_info--;
			num_args++;
		}
		if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
			num_args++;
		}
		arg_info = thread_persist_copy_xlat(ctx, arg_info, sizeof(zend_arg_info) * num_args);
		for (uint32_t i = 0; i < num_args; i++) {
			if (arg_info[i].name) {
				arg_info[i].name = thread_copy_string(ctx, arg_info[i].name);
			}
			/* The doc comment of a parameter is an ordinary string of the
			 * submitting thread, and arg_info holds only a pointer to it. The
			 * worker frees arg_info with its op_array, so it needs its own. */
			if (arg_info[i].doc_comment) {
				arg_info[i].doc_comment = thread_copy_string(ctx, arg_info[i].doc_comment);
			}
			thread_copy_type(ctx, &arg_info[i].type);
		}
		if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
			arg_info++;
		}
		op_array->arg_info = arg_info;
	}

	/* live_range */
	if (op_array->live_range) {
		op_array->live_range = thread_persist_copy_xlat(ctx, op_array->live_range,
			sizeof(zend_live_range) * op_array->last_live_range);
	}

	/* doc_comment — always copy */
	if (op_array->doc_comment) {
		op_array->doc_comment = thread_copy_string(ctx, op_array->doc_comment);
	}

	/* attributes */
	if (op_array->attributes) {
		op_array->attributes = thread_copy_attributes(ctx, op_array->attributes);
	}

	/* try_catch_array */
	if (op_array->try_catch_array) {
		op_array->try_catch_array = thread_persist_copy_xlat(ctx, op_array->try_catch_array,
			sizeof(zend_try_catch_element) * op_array->last_try_catch);
	}

	/* vars */
	if (op_array->vars) {
		op_array->vars = thread_persist_copy_xlat(ctx, op_array->vars,
			sizeof(zend_string*) * op_array->last_var);
		for (int i = 0; i < op_array->last_var; i++) {
			op_array->vars[i] = thread_copy_string(ctx, op_array->vars[i]);
		}
	}

	/* dynamic_func_defs (nested closures/lambdas) */
	if (op_array->num_dynamic_func_defs) {
		op_array->dynamic_func_defs = thread_persist_copy_xlat(ctx, op_array->dynamic_func_defs,
			sizeof(zend_function *) * op_array->num_dynamic_func_defs);
		for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
			zval tmp;
			ZVAL_PTR(&tmp, op_array->dynamic_func_defs[i]);
			thread_copy_op_array(ctx, &tmp);
			op_array->dynamic_func_defs[i] = Z_PTR(tmp);
		}
	}
}
/* }}} */

/* {{{ thread_copy_op_array — copy a standalone user function */
static void thread_copy_op_array(thread_copy_ctx_t *ctx, zval *zv)
{
	zend_op_array *op_array = Z_PTR_P(zv);
	ZEND_ASSERT(op_array->type == ZEND_USER_FUNCTION);

	zend_op_array *old = thread_xlat_get(ctx, op_array);
	if (old) {
		Z_PTR_P(zv) = old;
		return;
	}

	op_array = Z_PTR_P(zv) = thread_persist_copy_xlat(ctx, Z_PTR_P(zv), sizeof(zend_op_array));
	thread_copy_op_array_ex(ctx, op_array);

	op_array->fn_flags |= ZEND_ACC_IMMUTABLE;
	ZEND_MAP_PTR_INIT(op_array->run_time_cache, NULL);
	if (op_array->static_variables) {
		ZEND_MAP_PTR_INIT(op_array->static_variables_ptr, NULL);
	}
}
/* }}} */

///////////////////////////////////////////////////////////
/// 0.5. Zval transfer — copy runtime values between threads
///////////////////////////////////////////////////////////

/**
 * Transfer context: persistent-memory deep copy of runtime zvals.
 * Uses xlat table to preserve object/array identity (same pointer
 * in source → same pointer in destination) and handle cycles.
 */
#define THREAD_TRANSFER_MAX_DEPTH 512

#define THREAD_DEPTH_CHECK(ctx, ret) do { \
	if (UNEXPECTED((ctx)->depth >= THREAD_TRANSFER_MAX_DEPTH)) { \
		(ctx)->error = "Maximum nesting depth exceeded during thread data transfer"; \
		return ret; \
	} \
	(ctx)->depth++; \
} while (0)

#define THREAD_DEPTH_RELEASE(ctx) (ctx)->depth--

static void thread_transfer_ctx_init(thread_transfer_ctx_t *ctx)
{
	zend_hash_init(&ctx->xlat, 32, NULL, NULL, 0);
	ctx->depth = 0;
	ctx->defer_release = NULL;
	ctx->error = NULL;
}

static void thread_transfer_ctx_destroy(thread_transfer_ctx_t *ctx)
{
	if (ctx->defer_release) {
		zend_hash_destroy(ctx->defer_release);
		efree(ctx->defer_release);
		ctx->defer_release = NULL;
	}
	zend_hash_destroy(&ctx->xlat);
}

static void *thread_transfer_xlat_get(const thread_transfer_ctx_t *ctx, const void *ptr)
{
	return zend_hash_index_find_ptr(&ctx->xlat, async_ptr_to_index((void *)ptr));
}

static void thread_transfer_xlat_put(thread_transfer_ctx_t *ctx, const void *old_ptr, void *new_ptr)
{
	zend_hash_index_update_ptr(&ctx->xlat, async_ptr_to_index((void *)old_ptr), new_ptr);
}

/* Forward declarations */
static void thread_transfer_zval_inner(thread_transfer_ctx_t *ctx, zval *dst, const zval *src);
static HashTable *thread_transfer_hash_table(thread_transfer_ctx_t *ctx, const HashTable *src);
static zend_object *thread_transfer_object(thread_transfer_ctx_t *ctx, const zend_object *src);
typedef struct _thread_release_ctx_t thread_release_ctx_t;
static void thread_release_transferred_zval(thread_release_ctx_t *ctx, zval *z);
void async_thread_snapshot_destroy_ex(async_thread_snapshot_t *snapshot,
                                      thread_release_ctx_t *parent_ctx);
/* Refcount-based, for build-error partial releases only; see definition. */
static void thread_release_subgraph_zval(zval *z);

/* Copy a zend_string into persistent memory */
static zend_string *thread_transfer_string(thread_transfer_ctx_t *ctx, const zend_string *str)
{
	zend_string *existing = thread_transfer_xlat_get(ctx, str);
	if (existing) {
		GC_ADDREF(existing);
		return existing;
	}

	zend_string *copy = zend_string_init(ZSTR_VAL(str), ZSTR_LEN(str), 1);
	GC_MAKE_PERSISTENT_LOCAL(copy);
	thread_transfer_xlat_put(ctx, str, copy);

	return copy;
}

/* {{{ thread_transfer_hash_table — deep copy a HashTable into persistent memory */
static HashTable *thread_transfer_hash_table(thread_transfer_ctx_t *ctx, const HashTable *src)
{
	THREAD_DEPTH_CHECK(ctx, NULL);

	HashTable *existing = thread_transfer_xlat_get(ctx, src);
	if (existing) {
		GC_ADDREF(existing);
		THREAD_DEPTH_RELEASE(ctx);
		return existing;
	}

	HashTable *dst = pemalloc(sizeof(HashTable), 1);
	thread_transfer_xlat_put(ctx, src, dst);

	const uint32_t count = zend_hash_num_elements(src);

	zend_hash_init(dst, count, NULL, NULL, 1);
	GC_MAKE_PERSISTENT_LOCAL(dst);

	if (count == 0) {
		THREAD_DEPTH_RELEASE(ctx);
		return dst;
	}

	if (HT_IS_PACKED(src)) {
		zval *val;
		zend_ulong idx;
		ZEND_HASH_PACKED_FOREACH_KEY_VAL((HashTable *)src, idx, val) {
			zval copy;
			thread_transfer_zval_inner(ctx, &copy, val);
			if (UNEXPECTED(ctx->error)) {
				thread_release_subgraph_zval(&copy);
				break;
			}
			zend_hash_index_add(dst, idx, &copy);
		} ZEND_HASH_FOREACH_END();
	} else {
		zend_string *key;
		zend_ulong idx;
		zval *val;
		ZEND_HASH_MAP_FOREACH_KEY_VAL((HashTable *)src, idx, key, val) {
			zval copy;
			thread_transfer_zval_inner(ctx, &copy, val);
			if (UNEXPECTED(ctx->error)) {
				thread_release_subgraph_zval(&copy);
				break;
			}
			if (key) {
				zend_string *pkey = thread_transfer_string(ctx, key);
				zend_hash_add(dst, pkey, &copy);
				zend_string_release(pkey);
			} else {
				zend_hash_index_add(dst, idx, &copy);
			}
		} ZEND_HASH_FOREACH_END();
	}

	THREAD_DEPTH_RELEASE(ctx);
	return dst;
}
/* }}} */

/* Default transfer: pemalloc object of alloc_size (0 = auto), copy properties.
 * Can be passed to transfer_obj handler as default_fn. */
static zend_object *thread_transfer_object_default(
	const zend_object *src, thread_transfer_ctx_t *ctx, size_t alloc_size)
{
	const zend_class_entry *ce = src->ce;
	const uint32_t prop_count = ce->default_properties_count;
	const int offset = src->handlers->offset;
	const size_t obj_size = sizeof(zend_object) + zend_object_properties_size(ce);

	if (alloc_size == 0) {
		/* Auto-detect: offset (wrapper prefix) + zend_object + properties */
		alloc_size = offset + obj_size;
	}

	/* alloc_size is the full wrapper size (including offset bytes before zend_object) */
	char *base = pecalloc(1, alloc_size, 1);

	zend_object *dst = (zend_object *)(base + offset);

	/* Copy zend_object + properties_table */
	memcpy(dst, src, obj_size);
	GC_SET_REFCOUNT(dst, 1);
	GC_MAKE_PERSISTENT_LOCAL(dst);

	/* Store offset for release — extra_flags is unused in transit */
	dst->extra_flags = (uint32_t) offset;

	/* Repurpose fields for transit */
	dst->ce = (zend_class_entry *) thread_transfer_string(ctx, ce->name);
	dst->handlers = (const zend_object_handlers *)(uintptr_t) prop_count;
	dst->properties = NULL;

	/* xlat must be set before property recursion so back-edges
	 * ($x->self = $x, doubly-linked nodes) resolve through xlat. */
	thread_transfer_xlat_put(ctx, src, dst);

	/* Deep-copy each property zval in the copy */
	for (uint32_t i = 0; i < prop_count; i++) {
		zval *prop = &dst->properties_table[i];
		if (Z_TYPE_P(prop) != IS_UNDEF) {
			zval transferred = {0};
			thread_transfer_zval_inner(ctx, &transferred, prop);
			if (UNEXPECTED(ctx->error)) {
				/* Mark remaining properties as UNDEF so release won't
				 * try to free uninitialized memory */
				for (uint32_t j = i; j < prop_count; j++) {
					ZVAL_UNDEF(&dst->properties_table[j]);
				}
				break;
			}
			ZVAL_COPY_VALUE(prop, &transferred);
		}
	}

	return dst;
}

static zend_object *thread_transfer_object(thread_transfer_ctx_t *ctx, const zend_object *src)
{
	THREAD_DEPTH_CHECK(ctx, NULL);

	zend_object *existing = thread_transfer_xlat_get(ctx, src);
	if (existing) {
		GC_ADDREF(existing);
		THREAD_DEPTH_RELEASE(ctx);
		return existing;
	}

	/* Custom transfer handler (e.g. thread-safe shared objects) */
	if (src->handlers->transfer_obj) {
		zend_object *dst = src->handlers->transfer_obj(
			(zend_object *) src, ctx, ZEND_OBJECT_TRANSFER,
			(zend_object_transfer_default_fn) thread_transfer_object_default);
		if (dst) {
			thread_transfer_xlat_put(ctx, src, dst);
		}
		THREAD_DEPTH_RELEASE(ctx);
		return dst;
	}

	/* Reject genuine dynamic properties (stdClass, __set). A materialized
	 * properties table (var_dump, get_object_vars, ...) lists declared props as
	 * IS_INDIRECT slots, so only non-indirect entries are truly dynamic. */
	if (src->properties) {
		zval *prop_val;
		ZEND_HASH_FOREACH_VAL(src->properties, prop_val) {
			if (Z_TYPE_P(prop_val) != IS_INDIRECT) {
				ctx->error = "Cannot transfer object with dynamic properties between threads";
				THREAD_DEPTH_RELEASE(ctx);
				return NULL;
			}
		} ZEND_HASH_FOREACH_END();
	}

	/* xlat_put is performed inside _default before property recursion. */
	zend_object *dst = thread_transfer_object_default(src, ctx, 0);

	THREAD_DEPTH_RELEASE(ctx);
	return dst;
}
/* }}} */

/* {{{ thread_transfer_zval_inner — recursive deep copy of a single zval */
static zend_always_inline void thread_transfer_zval_inner(thread_transfer_ctx_t *ctx, zval *dst, const zval *src)
{
	switch (Z_TYPE_P(src)) {
		case IS_UNDEF:
			ZVAL_UNDEF(dst);
			break;

		case IS_NULL:
			ZVAL_NULL(dst);
			break;

		case IS_FALSE:
			ZVAL_FALSE(dst);
			break;

		case IS_TRUE:
			ZVAL_TRUE(dst);
			break;

		case IS_LONG:
			ZVAL_LONG(dst, Z_LVAL_P(src));
			break;

		case IS_DOUBLE:
			ZVAL_DOUBLE(dst, Z_DVAL_P(src));
			break;

		case IS_STRING: {
			zend_string *copy = thread_transfer_string(ctx, Z_STR_P(src));
			ZVAL_STR(dst, copy);
			break;
		}

		case IS_ARRAY: {
			HashTable *copy = thread_transfer_hash_table(ctx, Z_ARRVAL_P(src));
			if (UNEXPECTED(copy == NULL)) {
				ZVAL_NULL(dst);
				return;
			}
			ZVAL_ARR(dst, copy);
			break;
		}

		case IS_OBJECT: {
			zend_object *copy = thread_transfer_object(ctx, Z_OBJ_P(src));
			if (UNEXPECTED(copy == NULL)) {
				ZVAL_NULL(dst);
				return;
			}
			ZVAL_OBJ(dst, copy);
			break;
		}

		case IS_RESOURCE:
			ctx->error = "Cannot transfer a resource between threads";
			ZVAL_NULL(dst);
			break;

		case IS_REFERENCE:
			ctx->error = "Cannot transfer a reference between threads";
			ZVAL_NULL(dst);
			break;

		case IS_PTR:
			/* Opaque pointer passed through verbatim — not copied, not freed.
			 * The caller guarantees it points at persistent (pemalloc) memory
			 * with a lifetime it manages itself; the channel only carries the
			 * value. Enables C-level cross-thread handoffs (e.g. HTTP/3
			 * cross-worker packet forwarding). Matches thread_copy_zval. */
			ZVAL_PTR(dst, Z_PTR_P(src));
			break;

		default:
			ctx->error = "Cannot transfer zval of unsupported type between threads";
			ZVAL_NULL(dst);
			break;
	}
}
/* }}} */

/**
 * Copy a zval into persistent memory for cross-thread transfer.
 *
 * The result is a deep copy in pemalloc'd memory. Objects and arrays
 * that appear in multiple places share identity (same copy).
 * Cyclic references are handled via xlat table.
 *
 * @param dst  Destination zval (will be overwritten)
 * @param src  Source zval (unchanged)
 */
void async_thread_transfer_zval(zval *dst, const zval *src)
{
	thread_transfer_ctx_t ctx;
	thread_transfer_ctx_init(&ctx);
	thread_transfer_zval_inner(&ctx, dst, src);

	if (UNEXPECTED(ctx.error)) {
		/* Transfer failed — free any partially-allocated persistent memory
		 * before throwing. We must throw AFTER cleanup because
		 * zend_throw_error triggers zend_bailout() when there is no
		 * active execute_data (which is the case during thread transfer). */
		const char *error = ctx.error;
		thread_release_subgraph_zval(dst);
		ZVAL_UNDEF(dst);
		thread_transfer_ctx_destroy(&ctx);
		zend_throw_error(NULL, "%s", error);
		return;
	}

	thread_transfer_ctx_destroy(&ctx);
}

/**
 * Load a persistent zval into the current thread's emalloc heap.
 * Creates a proper refcounted copy that the current thread owns.
 *
 * @param dst  Destination zval (current thread, emalloc'd)
 * @param src  Source zval (persistent memory from thread_transfer_zval)
 */
static void thread_load_zval_inner(thread_transfer_ctx_t *ctx, zval *dst, const zval *src);
static HashTable *thread_load_hash_table(thread_transfer_ctx_t *ctx, const HashTable *src);
static zend_object *thread_load_object(thread_transfer_ctx_t *ctx, const zend_object *src);

static zend_string *thread_load_string(thread_transfer_ctx_t *ctx, const zend_string *str)
{
	zend_string *existing = thread_transfer_xlat_get(ctx, str);
	if (existing) {
		return zend_string_copy(existing);
	}

	zend_string *copy = zend_string_init(ZSTR_VAL(str), ZSTR_LEN(str), 0);
	thread_transfer_xlat_put(ctx, str, copy);

	return copy;
}

static HashTable *thread_load_hash_table(thread_transfer_ctx_t *ctx, const HashTable *src)
{
	THREAD_DEPTH_CHECK(ctx, NULL);

	HashTable *existing = thread_transfer_xlat_get(ctx, src);
	if (existing) {
		GC_ADDREF(existing);
		THREAD_DEPTH_RELEASE(ctx);
		return existing;
	}

	HashTable *dst = emalloc(sizeof(HashTable));
	thread_transfer_xlat_put(ctx, src, dst);

	const uint32_t count = zend_hash_num_elements(src);
	zend_hash_init(dst, count, NULL, ZVAL_PTR_DTOR, 0);

	if (count == 0) {
		return dst;
	}

	zend_string *key;
	zend_ulong idx;
	zval *val;
	ZEND_HASH_FOREACH_KEY_VAL((HashTable *)src, idx, key, val) {
		zval copy;
		thread_load_zval_inner(ctx, &copy, val);
		if (key) {
			zend_string *ekey = thread_load_string(ctx, key);
			zend_hash_add(dst, ekey, &copy);
			zend_string_release(ekey);
		} else {
			zend_hash_index_add(dst, idx, &copy);
		}
	} ZEND_HASH_FOREACH_END();

	THREAD_DEPTH_RELEASE(ctx);
	return dst;
}

/* Default load: resolve class, create emalloc object via create_object, copy properties.
 * alloc_size is ignored for load (object created via zend_objects_new/create_object). */
static zend_object *thread_load_object_default(
	const zend_object *src, thread_transfer_ctx_t *ctx, size_t alloc_size)
{
	(void) alloc_size;

	/* Read transit fields: ce = class_name, handlers = prop_count */
	const zend_string *class_name = (const zend_string *) src->ce;
	const uint32_t src_prop_count = (uint32_t)(uintptr_t) src->handlers;

	/* Resolve class by name via autoload */
	zend_string *lookup_name = zend_string_init(
		ZSTR_VAL(class_name), ZSTR_LEN(class_name), 0);
	zend_class_entry *ce = zend_lookup_class(lookup_name);
	zend_string_release(lookup_name);

	if (UNEXPECTED(ce == NULL)) {
		if (!EG(exception)) {
			zend_throw_error(NULL,
				"Cannot load transferred object: class \"%s\" not found",
				ZSTR_VAL(class_name));
		}
		zend_object *fallback = zend_objects_new(zend_standard_class_def);
		object_properties_init(fallback, zend_standard_class_def);
		return fallback;
	}

	/* Enum cases are singletons — fresh alloc would break `===` / match.
	 * Resolve the canonical case by name (transit prop 0 is the case name). */
	if (ce->ce_flags & ZEND_ACC_ENUM) {
		if (src_prop_count >= 1) {
			const zval *case_name_zv = &src->properties_table[0];
			if (Z_TYPE_P(case_name_zv) == IS_STRING) {
				zend_string *case_name = zend_string_init(
					Z_STRVAL_P(case_name_zv), Z_STRLEN_P(case_name_zv), 0);
				zend_object *canonical = zend_enum_get_case(ce, case_name);
				zend_string_release(case_name);
				if (canonical) {
					GC_ADDREF(canonical);
					thread_transfer_xlat_put(ctx, src, canonical);
					return canonical;
				}
			}
		}
		zend_throw_error(NULL,
			"Cannot resolve transferred enum case for \"%s\"",
			ZSTR_VAL(class_name));
		zend_object *fallback = zend_objects_new(zend_standard_class_def);
		object_properties_init(fallback, zend_standard_class_def);
		return fallback;
	}

	/* Create a normal emalloc'd object (uses create_object if defined) */
	zend_object *dst;
	if (ce->create_object) {
		dst = ce->create_object(ce);
	} else {
		dst = zend_objects_new(ce);
		object_properties_init(dst, ce);
	}

	/* xlat must be set before property recursion (see transfer side). */
	thread_transfer_xlat_put(ctx, src, dst);

	/* Copy declared properties from transit object */
	const uint32_t prop_count = MIN(src_prop_count,
		(uint32_t) ce->default_properties_count);

	for (uint32_t i = 0; i < prop_count; i++) {
		const zval *prop = &src->properties_table[i];
		if (Z_TYPE_P(prop) != IS_UNDEF) {
			zval copy = {0};
			thread_load_zval_inner(ctx, &copy, prop);
			zval_ptr_dtor(&dst->properties_table[i]);
			ZVAL_COPY_VALUE(&dst->properties_table[i], &copy);
		}
	}

	return dst;
}

static zend_object *thread_load_object(thread_transfer_ctx_t *ctx, const zend_object *src)
{
	THREAD_DEPTH_CHECK(ctx, NULL);

	zend_object *existing = thread_transfer_xlat_get(ctx, src);
	if (existing) {
		GC_ADDREF(existing);
		THREAD_DEPTH_RELEASE(ctx);
		return existing;
	}

	/* Read transit class name to resolve ce and check for transfer_obj handler */
	const zend_string *class_name = (const zend_string *) src->ce;
	zend_string *lookup_name = zend_string_init(
		ZSTR_VAL(class_name), ZSTR_LEN(class_name), 0);
	zend_class_entry *ce = zend_lookup_class(lookup_name);
	zend_string_release(lookup_name);

	if (ce && ce->default_object_handlers && ce->default_object_handlers->transfer_obj) {
		zend_object *dst = ce->default_object_handlers->transfer_obj(
			(zend_object *) src, ctx, ZEND_OBJECT_LOAD,
			(zend_object_transfer_default_fn) thread_load_object_default);
		if (dst) {
			thread_transfer_xlat_put(ctx, src, dst);
		}
		THREAD_DEPTH_RELEASE(ctx);
		return dst;
	}

	/* xlat_put is performed inside _default before property recursion. */
	zend_object *dst = thread_load_object_default(src, ctx, 0);

	THREAD_DEPTH_RELEASE(ctx);
	return dst;
}

static zend_always_inline void thread_load_zval_inner(thread_transfer_ctx_t *ctx, zval *dst, const zval *src)
{
	switch (Z_TYPE_P(src)) {
		case IS_UNDEF:
			ZVAL_UNDEF(dst);
			break;

		case IS_NULL:
			ZVAL_NULL(dst);
			break;

		case IS_FALSE:
			ZVAL_FALSE(dst);
			break;

		case IS_TRUE:
			ZVAL_TRUE(dst);
			break;

		case IS_LONG:
			ZVAL_LONG(dst, Z_LVAL_P(src));
			break;

		case IS_DOUBLE:
			ZVAL_DOUBLE(dst, Z_DVAL_P(src));
			break;

		case IS_STRING: {
			zend_string *copy = thread_load_string(ctx, Z_STR_P(src));
			ZVAL_STR(dst, copy);
			break;
		}

		case IS_ARRAY: {
			HashTable *copy = thread_load_hash_table(ctx, Z_ARR_P(src));
			ZVAL_ARR(dst, copy);
			break;
		}

		case IS_OBJECT: {
			zend_object *copy = thread_load_object(ctx, Z_OBJ_P(src));
			ZVAL_OBJ(dst, copy);
			break;
		}

		default:
			ZVAL_NULL(dst);
			break;
	}
}

/**
 * Load a persistent zval into the current thread's emalloc heap.
 *
 * @param dst  Destination zval (emalloc, current thread)
 * @param src  Source zval (pemalloc, from async_thread_transfer_zval)
 */
void async_thread_load_zval(zval *dst, const zval *src)
{
	thread_transfer_ctx_t ctx;
	thread_transfer_ctx_init(&ctx);
	thread_load_zval_inner(&ctx, dst, src);
	thread_transfer_ctx_destroy(&ctx);
}

/* {{{ Release — free persistent zvals created by async_thread_transfer_zval.
 *
 * Graph walk with visited-set keyed by node address. Each pemalloc-block is
 * freed once; revisits (shared edges, self-cycles) are short-circuited.
 * Transit-side refcounts are not consulted — they remain only because the
 * layout is shared with real zend_object/HashTable types. */

struct _thread_release_ctx_t {
	HashTable seen;   /* node address → empty (set semantics) */
};

static zend_always_inline void thread_release_ctx_init(thread_release_ctx_t *ctx)
{
	zend_hash_init(&ctx->seen, 16, NULL, NULL, 0);
}

static zend_always_inline void thread_release_ctx_destroy(thread_release_ctx_t *ctx)
{
	zend_hash_destroy(&ctx->seen);
}

/* Returns true on first visit (records it); false on revisit. */
static zend_always_inline bool thread_release_first_visit(thread_release_ctx_t *ctx, const void *node)
{
	return zend_hash_index_add_empty_element(&ctx->seen, (zend_ulong)(uintptr_t) node) != NULL;
}

static void thread_release_transferred_hash_table(thread_release_ctx_t *ctx, HashTable *ht);
static void thread_release_transferred_object(thread_release_ctx_t *ctx, zend_object *obj);

static void thread_release_transferred_zval(thread_release_ctx_t *ctx, zval *z)
{
	switch (Z_TYPE_P(z)) {
		case IS_STRING:
			zend_string_release(Z_STR_P(z));
			break;

		case IS_ARRAY:
			thread_release_transferred_hash_table(ctx, Z_ARR_P(z));
			break;

		case IS_OBJECT:
			thread_release_transferred_object(ctx, Z_OBJ_P(z));
			break;

		default:
			break;
	}
}

static void thread_release_transferred_hash_table(thread_release_ctx_t *ctx, HashTable *ht)
{
	/* first_visit uses only the pointer value, no deref — safe against UAF
	 * when a shared HT is encountered a second time after being freed. */
	if (!thread_release_first_visit(ctx, ht)) {
		return;
	}

	if (ht->nNumUsed == 0 && ht->nNumOfElements == 0 && ht->nTableSize == 0) {
		return;
	}

	zval *val;
	ZEND_HASH_FOREACH_VAL(ht, val) {
		thread_release_transferred_zval(ctx, val);
	} ZEND_HASH_FOREACH_END();

	/* zend_hash_destroy asserts refcount<=1 on debug builds; transit HTs
	 * carry build-side refcounts >=2 when xlat-shared. Reset before destroy. */
	GC_SET_REFCOUNT(ht, 1);
	zend_hash_destroy(ht);
	pefree(ht, 1);
}

static void thread_release_transferred_object(thread_release_ctx_t *ctx, zend_object *obj)
{
	if (!thread_release_first_visit(ctx, obj)) {
		return;
	}

	const uint32_t prop_count = (uint32_t)(uintptr_t) obj->handlers;
	zend_string * const class_name = (zend_string *) obj->ce;

	/* Closure shell stashes the snapshot pointer in `properties` (which is
	 * always NULL for non-closure shells). LOAD normally consumes it and
	 * resets the field; if LOAD never ran (future rejected without an
	 * awaiter) we still own the snapshot and must free it here. */
	if (obj->properties != NULL && zend_string_equals_literal(class_name, "Closure")) {
		async_thread_snapshot_destroy_ex((async_thread_snapshot_t *) obj->properties, ctx);
		obj->properties = NULL;
	}

	/* Whatever TRANSFER put in the C prefix of this shell is reachable only
	 * through the class: free_obj never runs for a shell. Resolved by name
	 * without autoload — a release runs outside a request. */
	zend_string *lookup_name = zend_string_init(
		ZSTR_VAL(class_name), ZSTR_LEN(class_name), 0);
	zend_class_entry *ce = zend_lookup_class_ex(
		lookup_name, NULL, ZEND_FETCH_CLASS_NO_AUTOLOAD);
	zend_string_release(lookup_name);

	if (ce != NULL && ce->default_object_handlers != NULL
		&& ce->default_object_handlers->transfer_obj != NULL) {
		ce->default_object_handlers->transfer_obj(
			obj, NULL, ZEND_OBJECT_TRANSFER_RELEASE, NULL);
	}

	for (uint32_t i = 0; i < prop_count; i++) {
		thread_release_transferred_zval(ctx, &obj->properties_table[i]);
	}

	zend_string_release(class_name);

	/* Free from base of allocation (offset stored in extra_flags by transfer_default) */
	const uint32_t offset = obj->extra_flags;
	pefree((char *)obj - offset, 1);
}

void async_thread_release_transferred_zval(zval *z)
{
	thread_release_ctx_t ctx;
	thread_release_ctx_init(&ctx);
	thread_release_transferred_zval(&ctx, z);
	thread_release_ctx_destroy(&ctx);
	ZVAL_UNDEF(z);
}

/* One release ctx (one visited set) across all roots — unlike the single-zval call,
 * which builds a fresh set each time — so an object xlat-deduped into more than one
 * root is freed once. Contract and rationale: the typedef in zend_async_API.h. */
void async_thread_release_transferred_zvals(zval **roots, size_t count)
{
	thread_release_ctx_t ctx;
	thread_release_ctx_init(&ctx);
	for (size_t i = 0; i < count; i++) {
		thread_release_transferred_zval(&ctx, roots[i]);
		ZVAL_UNDEF(roots[i]);
	}
	thread_release_ctx_destroy(&ctx);
}

/* Partial-release for build-error paths: refcount-based so xlat-shared
 * sub-pieces still owned by the half-built snapshot survive; the snapshot's
 * full release frees them shortly afterwards. */
static void thread_release_subgraph_ht(HashTable *ht);
static void thread_release_subgraph_obj(zend_object *obj);

static void thread_release_subgraph_zval(zval *z)
{
	switch (Z_TYPE_P(z)) {
		case IS_STRING:
			zend_string_release(Z_STR_P(z));
			break;
		case IS_ARRAY:
			thread_release_subgraph_ht(Z_ARR_P(z));
			break;
		case IS_OBJECT:
			thread_release_subgraph_obj(Z_OBJ_P(z));
			break;
		default:
			break;
	}
}

static void thread_release_subgraph_ht(HashTable *ht)
{
	if (ht->nNumUsed == 0 && ht->nNumOfElements == 0 && ht->nTableSize == 0) {
		return;
	}
	if (GC_DELREF(ht) > 0) {
		return;
	}
	zval *val;
	ZEND_HASH_FOREACH_VAL(ht, val) {
		thread_release_subgraph_zval(val);
	} ZEND_HASH_FOREACH_END();
	zend_hash_destroy(ht);
	pefree(ht, 1);
}

static void thread_release_subgraph_obj(zend_object *obj)
{
	if (GC_DELREF(obj) > 0) {
		return;
	}
	const uint32_t prop_count = (uint32_t)(uintptr_t) obj->handlers;
	zend_string * const class_name = (zend_string *) obj->ce;

	/* Refcounted walk — no visited set to share. */
	if (obj->properties != NULL && zend_string_equals_literal(class_name, "Closure")) {
		async_thread_snapshot_destroy((async_thread_snapshot_t *) obj->properties);
		obj->properties = NULL;
	}
	for (uint32_t i = 0; i < prop_count; i++) {
		thread_release_subgraph_zval(&obj->properties_table[i]);
	}
	zend_string_release(class_name);
	const uint32_t offset = obj->extra_flags;
	pefree((char *)obj - offset, 1);
}

/* }}} */

/* {{{ Recursion helpers for transfer_obj handlers living in Zend core.
 * Registered into the zend_async_thread_*_fn function pointers via
 * zend_async_thread_pool_register so handlers (e.g. for WeakReference and
 * WeakMap in Zend/zend_weakrefs.c) can recursively deep-copy child zvals
 * within an existing ctx, preserving identity and handling cycles through
 * the shared xlat table. */
void async_thread_transfer_zval_ctx(
	zend_async_thread_transfer_ctx_t *ctx, zval *dst, const zval *src)
{
	thread_transfer_zval_inner(ctx, dst, src);
}

void async_thread_load_zval_ctx(
	zend_async_thread_transfer_ctx_t *ctx, zval *dst, const zval *src)
{
	thread_load_zval_inner(ctx, dst, src);
}

void async_thread_xlat_put_ctx(
	zend_async_thread_transfer_ctx_t *ctx, const void *src, void *dst)
{
	thread_transfer_xlat_put(ctx, src, dst);
}

void async_thread_defer_release_ctx(
	zend_async_thread_transfer_ctx_t *ctx, zval *z)
{
	if (ctx->defer_release == NULL) {
		ctx->defer_release = emalloc(sizeof(HashTable));
		zend_hash_init(ctx->defer_release, 8, NULL, ZVAL_PTR_DTOR, 0);
	}
	zend_hash_next_index_insert(ctx->defer_release, z);
	ZVAL_UNDEF(z);
}
/* }}} */

///////////////////////////////////////////////////////////
/// 1. Snapshot — transfer callable between threads
///////////////////////////////////////////////////////////

/* A function value written into a constant expression belongs to the thread
 * that compiled it: a first-class callable names a function resolved against
 * that thread's function table and caches its zend_function beside the name,
 * and a closure is an op_array the tree points at rather than owns. Neither
 * can be given to the worker, so a closure carrying one is refused on
 * transfer, while the caller still has a stack to catch it.
 *
 * The three below answer with the word for the error message, or NULL when the
 * tree holds neither form. */
static const char *async_thread_ast_unshareable_function(const zend_ast *ast)
{
	if (ast == NULL) {
		return NULL;
	}

	if (ast->kind == ZEND_AST_CALLABLE_CONVERT) {
		return "first-class callable";
	}

	if (ast->kind == ZEND_AST_OP_ARRAY) {
		return "closure";
	}

	if (ast->kind == ZEND_AST_ZVAL || ast->kind == ZEND_AST_CONSTANT) {
		return NULL;
	}

	if (zend_ast_is_list((zend_ast *) ast)) {
		const zend_ast_list *list = zend_ast_get_list((zend_ast *) ast);

		for (uint32_t i = 0; i < list->children; i++) {
			const char *found = async_thread_ast_unshareable_function(list->child[i]);

			if (found != NULL) {
				return found;
			}
		}

		return NULL;
	}

	const uint32_t children = zend_ast_get_num_children((zend_ast *) ast);

	for (uint32_t i = 0; i < children; i++) {
		const char *found = async_thread_ast_unshareable_function(ast->child[i]);

		if (found != NULL) {
			return found;
		}
	}

	return NULL;
}

/* A constant expression is stored either as the tree itself or nested inside an
 * array value. */
static const char *async_thread_zval_unshareable_function(const zval *value)
{
	if (Z_TYPE_P(value) == IS_CONSTANT_AST) {
		return async_thread_ast_unshareable_function(Z_ASTVAL_P(value));
	}

	if (Z_TYPE_P(value) == IS_ARRAY) {
		const zval *item;

		ZEND_HASH_FOREACH_VAL(Z_ARRVAL_P(value), item) {
			const char *found = async_thread_zval_unshareable_function(item);

			if (found != NULL) {
				return found;
			}
		} ZEND_HASH_FOREACH_END();
	}

	return NULL;
}

/* Three tables of an op_array can hold a constant expression: a parameter
 * default lands in the literals, the initializer of a static variable in the
 * static variables, an attribute argument in the attributes. */
static const char *async_thread_op_array_unshareable_function(const zend_op_array *op_array)
{
	for (uint32_t i = 0; i < op_array->last_literal; i++) {
		const char *found = async_thread_zval_unshareable_function(&op_array->literals[i]);

		if (found != NULL) {
			return found;
		}
	}

	if (op_array->static_variables != NULL) {
		const zval *value;

		ZEND_HASH_FOREACH_VAL(op_array->static_variables, value) {
			const char *found = async_thread_zval_unshareable_function(value);

			if (found != NULL) {
				return found;
			}
		} ZEND_HASH_FOREACH_END();
	}

	if (op_array->attributes != NULL) {
		const zval *slot;

		ZEND_HASH_PACKED_FOREACH_VAL(op_array->attributes, slot) {
			const zend_attribute *attr = Z_PTR_P(slot);

			for (uint32_t i = 0; i < attr->argc; i++) {
				const char *found = async_thread_zval_unshareable_function(&attr->args[i].value);

				if (found != NULL) {
					return found;
				}
			}
		} ZEND_HASH_FOREACH_END();
	}

	return NULL;
}

/* Reject opcodes whose snapshot replay would assert in the worker:
 * class/function declarations need compile-time class_table state we don't
 * reproduce. Recurses into dynamic_func_defs. Throws on first illegal
 * opcode; sets ASYNC_FN_FLAG_THREAD_TRANSFER_OK on success so subsequent
 * transfers (ThreadPool resubmits, channel resends) skip the scan. */
static bool async_thread_check_op_array(zend_op_array *op_array)
{
	if (op_array->type != ZEND_USER_FUNCTION
		|| (op_array->fn_flags2 & ASYNC_FN_FLAG_THREAD_TRANSFER_OK)) {
		return true;
	}

	const zend_op *it = op_array->opcodes;
	const zend_op *const end = it + op_array->last;
	for (; it < end; it++) {
		const char *kind;
		switch (it->opcode) {
			case ZEND_DECLARE_CLASS:
			case ZEND_DECLARE_CLASS_DELAYED:
				kind = "class";
				break;
			case ZEND_DECLARE_ANON_CLASS:
				kind = "new class";
				break;
			case ZEND_DECLARE_FUNCTION:
				kind = "function";
				break;
			default:
				continue;
		}
		zend_throw_error(NULL,
			"Cannot transfer closure to another thread: illegal %s declaration at %s:%u",
			kind,
			op_array->filename ? ZSTR_VAL(op_array->filename) : "(closure)",
			it->lineno);
		return false;
	}

	const char *unshareable = async_thread_op_array_unshareable_function(op_array);

	if (UNEXPECTED(unshareable != NULL)) {
		zend_throw_error(NULL,
			"Cannot transfer closure to another thread: %s in a constant expression at %s",
			unshareable,
			op_array->filename ? ZSTR_VAL(op_array->filename) : "(closure)");
		return false;
	}

	for (uint32_t i = 0; i < op_array->num_dynamic_func_defs; i++) {
		if (!async_thread_check_op_array(op_array->dynamic_func_defs[i])) {
			return false;
		}
	}

	/* OPcache puts immutable op_arrays into read-only shared memory; mutating
	 * fn_flags2 there SEGVs. Re-scanning them is cheap (linear opcode walk,
	 * no allocations), so we just skip the cache write. */
	if (!(op_array->fn_flags & ZEND_ACC_IMMUTABLE)) {
		op_array->fn_flags2 |= ASYNC_FN_FLAG_THREAD_TRANSFER_OK;
	}
	return true;
}

/**
 * Deep-copy a callable into persistent memory.
 * Copies op_array via thread_copy_op_array and transfers
 * captured variables via async_thread_transfer_zval.
 */
static void thread_copy_callable(
	thread_copy_ctx_t *ctx, const zend_fcall_t *fcall, async_thread_closure_copy_t *dst,
	bool strip_scope, thread_transfer_ctx_t *parent_transfer)
{
	zend_op_array *src_op = &fcall->fci_cache.function_handler->op_array;

	/* Validate before allocating anything in the arena. On failure
	 * EG(exception) is set and async_thread_snapshot_create() will tear
	 * down the half-built snapshot. */
	if (!async_thread_check_op_array(src_op)) {
		return;
	}

	/* CE pointers are thread-local: carry scope/called_scope by name and
	 * re-resolve them in the target thread. Anonymous classes have no
	 * resolvable name there. Bootloaders are copied with strip_scope: they
	 * bootstrap a thread where their own declaring class may not exist yet. */
	const zend_class_entry *scope = strip_scope ? NULL : src_op->scope;
	const zend_class_entry *called_scope = strip_scope ? NULL : fcall->fci_cache.called_scope;

	if ((scope != NULL && (scope->ce_flags & ZEND_ACC_ANON_CLASS) != 0)
		|| (called_scope != NULL && (called_scope->ce_flags & ZEND_ACC_ANON_CLASS) != 0)) {
		zend_throw_error(NULL, "Cannot transfer a closure scoped to an anonymous class between threads");
		return;
	}

	/* Deep copy the op_array into arena */
	zval tmp;
	ZVAL_PTR(&tmp, (void *) src_op);
	thread_copy_op_array(ctx, &tmp);
	dst->func = Z_PTR(tmp);

	ZVAL_UNDEF(&dst->bound_this);
	dst->bound_vars = NULL;
	dst->scope_name = NULL;
	dst->called_scope_name = NULL;

	if (scope != NULL) {
		dst->scope_name = zend_string_init(ZSTR_VAL(scope->name), ZSTR_LEN(scope->name), 1);
		GC_MAKE_PERSISTENT_LOCAL(dst->scope_name);
	}

	if (called_scope != NULL) {
		dst->called_scope_name = zend_string_init(ZSTR_VAL(called_scope->name), ZSTR_LEN(called_scope->name), 1);
		GC_MAKE_PERSISTENT_LOCAL(dst->called_scope_name);
	}

	HashTable *static_vars = ZEND_MAP_PTR_GET(src_op->static_variables_ptr);
	if (!static_vars) {
		static_vars = src_op->static_variables;
	}
	const bool has_vars = (static_vars && zend_hash_num_elements(static_vars) > 0);
	zend_object *bound_obj = fcall->fci_cache.object;

	if (!has_vars && bound_obj == NULL) {
		return;
	}

	/* One transfer_ctx for both bound vars and $this — xlat preserves identity
	 * when the same object is captured via use() and as $this.
	 *
	 * Join the caller's ctx when there is one. A cycle can run through a closure
	 * — $o->handler = function () use ($o) — and a private xlat would not know
	 * $o was already copied: it would copy it again, reach the closure again,
	 * and recurse until the stack ran out. The load and release walks share
	 * their ctx for the same reason. */
	thread_transfer_ctx_t  own_ctx;
	thread_transfer_ctx_t *transfer = parent_transfer;

	if (transfer == NULL) {
		thread_transfer_ctx_init(&own_ctx);
		transfer = &own_ctx;
	}

	/* bound_vars use pemalloc (not arena) — released individually in
	 * thread_release_closure_copy. */
	if (has_vars) {
		dst->bound_vars = pemalloc(sizeof(HashTable), 1);
		zend_hash_init(dst->bound_vars, zend_hash_num_elements(static_vars), NULL, NULL, 1);
		GC_MAKE_PERSISTENT_LOCAL(dst->bound_vars);

		zend_string *key;
		zval *val;
		ZEND_HASH_FOREACH_STR_KEY_VAL(static_vars, key, val) {
			zval transferred;
			thread_transfer_zval_inner(transfer, &transferred, val);
			if (UNEXPECTED(transfer->error)) {
				thread_release_subgraph_zval(&transferred);
				break;
			}
			/* Private persistent copy: an interned key aliases the parent's
			 * interned table and dangles after the parent shuts down. */
			zend_string *pkey = zend_string_init(ZSTR_VAL(key), ZSTR_LEN(key), 1);
			GC_MAKE_PERSISTENT_LOCAL(pkey);
			zend_hash_add(dst->bound_vars, pkey, &transferred);
			zend_string_release(pkey);
		} ZEND_HASH_FOREACH_END();
	}

	if (!transfer->error && bound_obj != NULL) {
		zval src_this;
		ZVAL_OBJ(&src_this, bound_obj);
		thread_transfer_zval_inner(transfer, &dst->bound_this, &src_this);
		if (UNEXPECTED(transfer->error)) {
			thread_release_subgraph_zval(&dst->bound_this);
			ZVAL_UNDEF(&dst->bound_this);
		}
	}

	if (UNEXPECTED(transfer->error)) {
		if (dst->bound_vars) {
			zval *v;
			ZEND_HASH_FOREACH_VAL(dst->bound_vars, v) {
				thread_release_subgraph_zval(v);
			} ZEND_HASH_FOREACH_END();
			zend_hash_destroy(dst->bound_vars);
			pefree(dst->bound_vars, 1);
			dst->bound_vars = NULL;
		}
		zend_throw_error(NULL, "%s", transfer->error);
	}

	/* A borrowed ctx is the caller's to tear down; its error is already there. */
	if (transfer == &own_ctx) {
		thread_transfer_ctx_destroy(&own_ctx);
	}
}

/**
 * Free closure-copy bound vars/this. ctx is shared across calls so nodes
 * referenced from multiple roots are released exactly once.
 */
static void thread_release_closure_copy(thread_release_ctx_t *ctx, async_thread_closure_copy_t *copy)
{
	/* Release name strings (no-op while interned; the base ref once materialized). */
	if (copy->func != NULL) {
		if (copy->func->function_name != NULL) {
			zend_string_release(copy->func->function_name);
		}
		if (copy->func->filename != NULL) {
			zend_string_release(copy->func->filename);
		}
	}

	if (copy->scope_name != NULL) {
		zend_string_release(copy->scope_name);
		copy->scope_name = NULL;
	}
	if (copy->called_scope_name != NULL) {
		zend_string_release(copy->called_scope_name);
		copy->called_scope_name = NULL;
	}

	if (copy->bound_vars) {
		zval *val;
		ZEND_HASH_FOREACH_VAL(copy->bound_vars, val) {
			thread_release_transferred_zval(ctx, val);
		} ZEND_HASH_FOREACH_END();
		/* zend_hash_destroy releases string keys itself */
		zend_hash_destroy(copy->bound_vars);
		pefree(copy->bound_vars, 1);
	}
	if (!Z_ISUNDEF(copy->bound_this)) {
		thread_release_transferred_zval(ctx, &copy->bound_this);
		ZVAL_UNDEF(&copy->bound_this);
	}
}

/**
 * Create a snapshot: deep-copy closures into arena memory.
 * entry_is_bootloader: the entry slot holds a bootloader (pool case) and is
 * copied unscoped, like the regular bootloader slot.
 */
async_thread_snapshot_t *async_thread_snapshot_create_ex(
	const zend_fcall_t *entry, const zend_fcall_t *bootloader, bool entry_is_bootloader,
	thread_transfer_ctx_t *parent_transfer)
{
	async_thread_snapshot_t *snapshot = pecalloc(1, sizeof(async_thread_snapshot_t), 1);

	thread_copy_ctx_t ctx;
	thread_copy_ctx_init(&ctx);

	thread_copy_callable(&ctx, entry, &snapshot->entry, entry_is_bootloader, parent_transfer);

	if (bootloader != NULL && !EG(exception)) {
		thread_copy_callable(&ctx, bootloader, &snapshot->bootloader, true, parent_transfer);
	}

	/* Store arena block list in snapshot — needed even for the failure path
	 * so that destroy can release any op_array data already copied. */
	snapshot->arena_blocks = ctx.current_block;
	thread_copy_ctx_destroy(&ctx);

	/* Bound-var transfer or op_array copy may have thrown (e.g. a captured
	 * object's transfer_obj handler refused). In that case the snapshot is
	 * partially initialized — destroy it and propagate the exception. */
	if (UNEXPECTED(EG(exception))) {
		async_thread_snapshot_destroy(snapshot);
		return NULL;
	}

	return snapshot;
}

/* Top-level: no transfer in progress, so the closure gets its own ctx. */
async_thread_snapshot_t *async_thread_snapshot_create(
	const zend_fcall_t *entry, const zend_fcall_t *bootloader, bool entry_is_bootloader)
{
	return async_thread_snapshot_create_ex(entry, bootloader, entry_is_bootloader, NULL);
}

/* Heap-copy the op_array's interned (arena-backed) name strings so holders that
 * outlive the arena (closure, PG(last_error_file)) are freed by refcount. */
static void thread_materialize_op_array_names(zend_op_array *op_array)
{
	if (op_array->function_name != NULL && ZSTR_IS_INTERNED(op_array->function_name)) {
		op_array->function_name = zend_string_init(
			ZSTR_VAL(op_array->function_name), ZSTR_LEN(op_array->function_name), 0);
	}

	if (op_array->filename != NULL && ZSTR_IS_INTERNED(op_array->filename)) {
		op_array->filename = zend_string_init(
			ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename), 0);
	}
}

void async_thread_snapshot_materialize_entry(async_thread_snapshot_t *snapshot)
{
	if (snapshot != NULL && snapshot->entry.func != NULL) {
		thread_materialize_op_array_names(snapshot->entry.func);
	}
}

/**
 * Free snapshot resources.
 */
void async_thread_snapshot_destroy_ex(async_thread_snapshot_t *snapshot,
                                      thread_release_ctx_t *parent_ctx)
{
	/* Join the caller's visited set — see thread_copy_callable. */
	thread_release_ctx_t  own_ctx;
	thread_release_ctx_t *rel = parent_ctx;

	if (rel == NULL) {
		thread_release_ctx_init(&own_ctx);
		rel = &own_ctx;
	}

	thread_release_closure_copy(rel, &snapshot->entry);

	if (snapshot->bootloader.func != NULL) {
		thread_release_closure_copy(rel, &snapshot->bootloader);
	}

	if (rel == &own_ctx) {
		thread_release_ctx_destroy(&own_ctx);
	}

	/* Free all arena blocks at once */
	thread_copy_arena_free(snapshot->arena_blocks);
	snapshot->arena_blocks = NULL;

	pefree(snapshot, 1);
}

void async_thread_snapshot_destroy(async_thread_snapshot_t *snapshot)
{
	async_thread_snapshot_destroy_ex(snapshot, NULL);
}

/**
 * Create a RemoteException wrapping a loaded remote exception object.
 */
/**
 * Create a ThreadTransferException with message and optional previous exception.
 * Consumes the previous exception reference (OBJ_RELEASE after setting).
 */
static zend_object *thread_create_transfer_exception(
	const char *message, zend_object *previous)
{
	zval exception_zv;
	object_init_ex(&exception_zv, async_ce_thread_transfer_exception);

	zval msg_zv;
	ZVAL_STRING(&msg_zv, message);
	zend_update_property_ex(async_ce_thread_transfer_exception,
		Z_OBJ(exception_zv), ZSTR_KNOWN(ZEND_STR_MESSAGE), &msg_zv);
	zval_ptr_dtor(&msg_zv);

	if (previous) {
		zend_exception_set_previous(Z_OBJ(exception_zv), previous);
	}

	return Z_OBJ(exception_zv);
}

static zend_object *thread_wrap_remote_exception(
	zend_object *remote_obj, const char *remote_class_name)
{
	zval wrapper_zval;
	object_init_ex(&wrapper_zval, async_ce_remote_exception);
	zend_object *wrapper = Z_OBJ(wrapper_zval);

	/* Set message from remote exception */
	zval rv;
	zval *remote_message = zend_read_property_ex(
		remote_obj->ce, remote_obj, ZSTR_KNOWN(ZEND_STR_MESSAGE), 1, &rv);
	if (remote_message && Z_TYPE_P(remote_message) == IS_STRING) {
		zend_update_property_ex(async_ce_remote_exception, wrapper,
			ZSTR_KNOWN(ZEND_STR_MESSAGE), remote_message);
	}

	/* Set code from remote exception */
	zval *remote_code = zend_read_property_ex(
		remote_obj->ce, remote_obj, ZSTR_KNOWN(ZEND_STR_CODE), 1, &rv);
	if (remote_code && Z_TYPE_P(remote_code) == IS_LONG) {
		zval code_zv;
		ZVAL_LONG(&code_zv, Z_LVAL_P(remote_code));
		zend_update_property_ex(async_ce_remote_exception, wrapper,
			ZSTR_KNOWN(ZEND_STR_CODE), &code_zv);
	}

	/* Store the original remote exception */
	zval remote_exception_zval;
	ZVAL_OBJ_COPY(&remote_exception_zval, remote_obj);
	zend_update_property(async_ce_remote_exception, wrapper,
		"remoteException", sizeof("remoteException") - 1, &remote_exception_zval);
	zval_ptr_dtor(&remote_exception_zval);

	/* Store the remote class name */
	zend_update_property_string(async_ce_remote_exception, wrapper,
		"remoteClass", sizeof("remoteClass") - 1, remote_class_name);

	return wrapper;
}

/**
 * Load thread result/exception from persistent memory into parent's emalloc.
 * Called from reactor's notify callback in the parent thread.
 */
void async_thread_load_result(zend_async_thread_event_t *event)
{
	ZEND_ASYNC_EVENT_CLR_EXCEPTION_HANDLED(&event->base);

	/* Handle bailout from child thread (fatal error, OOM, exit()) */
	if (UNEXPECTED(event->context && event->context->bailout_error_message != NULL)) {
		event->exception = thread_create_transfer_exception(
			event->context->bailout_error_message, NULL);
		pefree(event->context->bailout_error_message, 1);
		event->context->bailout_error_message = NULL;
		ZEND_THREAD_SET_RESULT_LOADED(event);
		return;
	}

	/* Load exception and wrap in RemoteException */
	if (event->exception != NULL) {
		/* remote_class_name lives in persistent memory (transit format: ce = class_name) */
		const char *remote_class_name = ZSTR_VAL((zend_string *) event->exception->ce);

		zval persistent_exception, loaded_exception;
		ZVAL_OBJ(&persistent_exception, event->exception);
		async_thread_load_zval(&loaded_exception, &persistent_exception);

		if (UNEXPECTED(EG(exception))) {
			zend_object *loading_exception = EG(exception);
			GC_ADDREF(loading_exception);
			zend_clear_exception();

			zend_string *message = zend_strpprintf(0,
				"Failed to load remote exception of class \"%s\"", remote_class_name);
			event->exception = thread_create_transfer_exception(
				ZSTR_VAL(message), loading_exception);
			zend_string_release(message);

			zval_ptr_dtor(&loaded_exception);
			async_thread_release_transferred_zval(&persistent_exception);
		} else {
			event->exception = thread_wrap_remote_exception(
				Z_OBJ(loaded_exception), remote_class_name);
			zval_ptr_dtor(&loaded_exception);
			async_thread_release_transferred_zval(&persistent_exception);
		}
	}

	/* Load result */
	if (!Z_ISUNDEF(event->result)) {
		zval loaded_result;
		async_thread_load_zval(&loaded_result, &event->result);

		if (UNEXPECTED(EG(exception))) {
			zend_object *loading_exception = EG(exception);
			GC_ADDREF(loading_exception);
			zend_clear_exception();

			async_thread_release_transferred_zval(&event->result);
			zval_ptr_dtor(&loaded_result);
			event->exception = thread_create_transfer_exception(
				"Failed to load thread result into parent thread", loading_exception);
			ZEND_THREAD_SET_RESULT_LOADED(event);
			return;
		}

		async_thread_release_transferred_zval(&event->result);
		ZVAL_COPY_VALUE(&event->result, &loaded_result);
		ZEND_ASYNC_EVENT_SET_ZVAL_RESULT(&event->base);
	}

	ZEND_THREAD_SET_RESULT_LOADED(event);
}

///////////////////////////////////////////////////////////
/// 2. Thread lifecycle — PHP request in child thread
///////////////////////////////////////////////////////////

/**
 * Initialize TSRM for the child thread.
 * Must be called BEFORE any zend_try block (EG(bailout) is not available yet).
 * After this call, EG()/SG()/PG() macros work correctly.
 */
void async_thread_tsrm_init(void)
{
#ifdef ZTS
	ts_resource(0);
	TSRMLS_CACHE_UPDATE();

	/* Update TSRMLS cache in the SAPI module (e.g. php.exe on Windows).
	 * Without this, SG()/EG() calls from SAPI callbacks would use
	 * an uninitialized per-module thread-local cache. */
	if (sapi_module.thread_init) {
		sapi_module.thread_init();
	}
#endif
}

/**
 * Per-thread STDIN/STDOUT/STDERR registration.
 *
 * Mirrors cli_register_file_handles() (sapi/cli/php_cli.c) but runs once per
 * child thread instead of once per process. CLI registers these constants
 * only in main()'s thread; in ZTS, EG(zend_constants) and EG(regular_list)
 * are per-thread, so child threads see neither the constants nor the
 * underlying php_stream resources unless we recreate them here.
 *
 * Each thread gets its OWN php_stream wrapping the shared fd 0/1/2:
 * concurrent fwrite() from different threads goes through independent
 * buffers; the resulting write(2) calls hit a shared fd but are atomic for
 * short messages (<= PIPE_BUF) per POSIX. Sharing a single php_stream
 * across threads would be a data race — its buffer/position/refcount are
 * not guarded.
 *
 * PHP_STREAM_FLAG_NO_RSCR_DTOR_CLOSE keeps request shutdown from closing
 * fd 0/1/2 — other threads/the parent still need them.
 */
#ifdef ZTS
static void async_thread_register_file_handles(void)
{
	php_stream *s_in  = php_stream_open_wrapper_ex("php://stdin",  "rb", 0, NULL, NULL);
	php_stream *s_out = php_stream_open_wrapper_ex("php://stdout", "wb", 0, NULL, NULL);
	php_stream *s_err = php_stream_open_wrapper_ex("php://stderr", "wb", 0, NULL, NULL);

	if (s_in == NULL || s_out == NULL || s_err == NULL) {
		if (s_in)  php_stream_close(s_in);
		if (s_out) php_stream_close(s_out);
		if (s_err) php_stream_close(s_err);
		return;
	}

	s_in->flags  |= PHP_STREAM_FLAG_NO_RSCR_DTOR_CLOSE;
	s_out->flags |= PHP_STREAM_FLAG_NO_RSCR_DTOR_CLOSE;
	s_err->flags |= PHP_STREAM_FLAG_NO_RSCR_DTOR_CLOSE;

	zend_constant ic, oc, ec;

	php_stream_to_zval(s_in,  &ic.value);
	php_stream_to_zval(s_out, &oc.value);
	php_stream_to_zval(s_err, &ec.value);

	Z_CONSTANT_FLAGS(ic.value) = 0;
	ic.name = zend_string_init_interned("STDIN", sizeof("STDIN") - 1, 0);
	zend_register_constant(&ic);

	Z_CONSTANT_FLAGS(oc.value) = 0;
	oc.name = zend_string_init_interned("STDOUT", sizeof("STDOUT") - 1, 0);
	zend_register_constant(&oc);

	Z_CONSTANT_FLAGS(ec.value) = 0;
	ec.name = zend_string_init_interned("STDERR", sizeof("STDERR") - 1, 0);
	zend_register_constant(&ec);
}
#endif

int async_thread_request_startup(const async_thread_snapshot_t *snapshot)
{
#ifdef ZTS
	PG(expose_php) = 0;
	PG(auto_globals_jit) = 1;

	if (UNEXPECTED(php_request_startup() == FAILURE)) {
		ts_free_thread();
		return FAILURE;
	}

	async_thread_register_file_handles();

	/* Suppress HTTP-specific behavior */
	PG(during_request_startup) = 0;
	SG(sapi_started) = 0;
	SG(headers_sent) = 1;
	SG(request_info).no_headers = 1;

	/* Set script filename from entry closure for error reporting */
	if (snapshot != NULL && snapshot->entry.func && snapshot->entry.func->filename) {
		SG(request_info).path_translated = estrndup(
			ZSTR_VAL(snapshot->entry.func->filename),
			ZSTR_LEN(snapshot->entry.func->filename));
	}

	/* $_SERVER and $_ENV are initialized lazily (auto_globals_jit=1).
	 * Do NOT force zend_is_auto_global() here — it triggers interned string
	 * creation via a global (non-TLS) function pointer that may race with
	 * the main thread's request shutdown. */

	return SUCCESS;
#else
	return FAILURE;
#endif
}

void async_thread_request_shutdown(void)
{
#ifdef ZTS
	/* Free path_translated before shutdown destroys emalloc heap */
	if (SG(request_info).path_translated) {
		efree(SG(request_info).path_translated);
		SG(request_info).path_translated = NULL;
	}
	php_request_shutdown(NULL);
#endif
}

/* Closure struct layout — defined in zend_closures.c, not exported */
typedef struct {
	zend_object       std;
	zend_function     func;
	zval              this_ptr;
	zend_class_entry *called_scope;
	zif_handler       orig_internal_handler;
} async_zend_closure_t;

/* Deep-copy a zend_type into emalloc memory. Mirrors thread_copy_type
 * (persistent arena variant) but uses emalloc + zend_string_init so the
 * resulting type tree is self-contained in the worker's request heap.
 * Without this, arg_info[i].type points at zend_string slots inside the
 * source persistent arena; once that arena is destroyed in the parent
 * thread, the worker's zend_lookup_class_ex reads garbage as zend_string
 * length and triggers a multi-exabyte _emalloc → memory_limit bailout. */
static void op_array_emalloc_copy_type(zend_type *type)
{
	if (ZEND_TYPE_HAS_LIST(*type)) {
		const zend_type_list *old_list = ZEND_TYPE_LIST(*type);
		size_t size = ZEND_TYPE_LIST_SIZE(old_list->num_types);
		zend_type_list *new_list = emalloc(size);
		memcpy(new_list, old_list, size);
		ZEND_TYPE_FULL_MASK(*type) &= ~_ZEND_TYPE_ARENA_BIT;
		ZEND_TYPE_SET_LIST(*type, new_list);
		zend_type *single;
		ZEND_TYPE_LIST_FOREACH_MUTABLE(new_list, single) {
			op_array_emalloc_copy_type(single);
		} ZEND_TYPE_LIST_FOREACH_END();
	} else if (ZEND_TYPE_HAS_NAME(*type)) {
		zend_string *old_name = ZEND_TYPE_NAME(*type);
		zend_string *fresh = zend_string_init(ZSTR_VAL(old_name), ZSTR_LEN(old_name), 0);
		ZEND_TYPE_SET_PTR(*type, fresh);
	}
}

/* Everything below rebuilds, in this thread's heap, the parts of a materialized
 * op_array that would otherwise keep pointing into the snapshot arena. The
 * arena is freed whole when its owner is done with it, and every string it
 * holds is marked interned, so a plain copy shares a pointer that nobody keeps
 * alive and nobody may free. */
static void op_array_emalloc_copy_value(zval *dst, const zval *src);

static zend_string *op_array_emalloc_copy_string(const zend_string *src)
{
	zend_string *copy = zend_string_init(ZSTR_VAL(src), ZSTR_LEN(src), 0);

	/* Precomputed: INIT_FCALL_SPEC_CONST and its kin look a literal up with
	 * zend_hash_find_known_hash, which trusts key->h and never recomputes it.
	 * A fresh string carries h == 0 and would send every such lookup to bucket
	 * 0, onto whatever pointer lives there. */
	zend_string_hash_val(copy);

	return copy;
}

/* Rebuilds a literal array or the static variables of a nested definition.
 *
 * The rebuild keeps the bucket order, which carries meaning for the second of
 * those: BIND_LEXICAL addresses a captured variable by a byte offset into
 * arData, computed when the closure was compiled. It cannot keep a hole — the
 * iteration skips an UNDEF slot — so the caller that needs the offsets asserts
 * there is none; see the static_variables branch of op_array_to_emalloc.
 *
 * A literal array may hold one: [0 => 'a', 2 => 'b'] compiles to a packed hash
 * whose slot 1 is UNDEF. Rebuilding it by key gives the same array back, since
 * the keys are what the VM reads, so the hole costs nothing here. */
static HashTable *op_array_emalloc_copy_array(const HashTable *src)
{
	HashTable *dst = zend_new_array(zend_hash_num_elements(src));

	zend_string *key;
	zend_ulong idx;
	zval *val;

	ZEND_HASH_FOREACH_KEY_VAL((HashTable *) src, idx, key, val) {
		zval copy;
		op_array_emalloc_copy_value(&copy, val);

		if (key != NULL) {
			zend_string *local = op_array_emalloc_copy_string(key);
			zend_hash_add_new(dst, local, &copy);
			zend_string_release(local);
		} else {
			zend_hash_index_add_new(dst, idx, &copy);
		}
	} ZEND_HASH_FOREACH_END();

	return dst;
}

/* A constant expression is kept as an unevaluated tree until something asks for
 * it. zend_ast_copy gives the tree its own block but carries leaf values over
 * as they are, so the leaves are rebuilt here.
 *
 * Two leaf kinds are passed over: a closure and a first-class callable written
 * inside a constant expression name a function of the compiling thread, and
 * async_thread_check_op_array refuses the transfer before a worker sees one. */
static void op_array_emalloc_localize_ast(zend_ast *ast)
{
	if (ast == NULL) {
		return;
	}

	if (ast->kind == ZEND_AST_ZVAL || ast->kind == ZEND_AST_CONSTANT) {
		zval *leaf = &((zend_ast_zval *) ast)->val;
		zval local;

		op_array_emalloc_copy_value(&local, leaf);
		ZVAL_COPY_VALUE(leaf, &local);

		return;
	}

	if (ast->kind == ZEND_AST_OP_ARRAY || ast->kind == ZEND_AST_CALLABLE_CONVERT) {
		return;
	}

	if (zend_ast_is_list(ast)) {
		const zend_ast_list *list = zend_ast_get_list(ast);

		for (uint32_t i = 0; i < list->children; i++) {
			op_array_emalloc_localize_ast(list->child[i]);
		}

		return;
	}

	const uint32_t children = zend_ast_get_num_children(ast);

	for (uint32_t i = 0; i < children; i++) {
		op_array_emalloc_localize_ast(ast->child[i]);
	}
}

/* The single localizer. What it writes into `dst` belongs to the caller and is
 * freed with the op_array it goes into. */
static void op_array_emalloc_copy_value(zval *dst, const zval *src)
{
	switch (Z_TYPE_P(src)) {
		case IS_STRING:
			ZVAL_NEW_STR(dst, op_array_emalloc_copy_string(Z_STR_P(src)));
			break;
		case IS_ARRAY:
			ZVAL_ARR(dst, op_array_emalloc_copy_array(Z_ARR_P(src)));
			break;
		case IS_CONSTANT_AST: {
			zend_ast_ref *tree = zend_ast_copy(Z_ASTVAL_P(src));
			op_array_emalloc_localize_ast(GC_AST(tree));
			ZVAL_AST(dst, tree);
			break;
		}
		default:
			ZVAL_COPY(dst, src);
			break;
	}
}

/* The attribute table of a materialized op_array. The snapshot's one is marked
 * immutable and carries no destructor, so it belongs to the arena; the rebuilt
 * one belongs to this op_array and goes with it. */
static HashTable *op_array_emalloc_copy_attributes(const HashTable *src)
{
	HashTable *dst = NULL;
	zval *v;

	ZEND_HASH_PACKED_FOREACH_VAL((HashTable *) src, v) {
		const zend_attribute *attr = Z_PTR_P(v);
		zend_string *name = zend_string_init(ZSTR_VAL(attr->name), ZSTR_LEN(attr->name), 0);

		zend_attribute *copy = zend_add_attribute(&dst, name, attr->argc,
			attr->flags & ~ZEND_ATTRIBUTE_PERSISTENT, attr->offset, attr->lineno);

		zend_string_release(name);

		/* zend_add_attribute leaves this NULL. A target error deferred by
		 * DelayedTargetValidation is raised from this string when the attribute
		 * is instantiated, and losing it turns the error into silence. */
		if (attr->validation_error != NULL) {
			copy->validation_error = op_array_emalloc_copy_string(attr->validation_error);
		}

		for (uint32_t i = 0; i < attr->argc; i++) {
			if (attr->args[i].name != NULL) {
				copy->args[i].name = op_array_emalloc_copy_string(attr->args[i].name);
			}

			op_array_emalloc_copy_value(&copy->args[i].value, &attr->args[i].value);
		}
	} ZEND_HASH_FOREACH_END();

	return dst;
}

/**
 * Copy op_array internals from persistent/arena memory into emalloc.
 * After this call, the op_array is fully self-contained in emalloc
 * and the persistent source (snapshot arena) can be freed.
 */
static void op_array_to_emalloc(zend_op_array *op_array)
{
	/* refcount — own copy */
	uint32_t *rc = emalloc(sizeof(uint32_t));
	*rc = 1;
	op_array->refcount = rc;

	/* function_name — already addref'd by zend_create_closure, but points to
	 * persistent string. Create emalloc copy. */
	if (op_array->function_name) {
		zend_string *old = op_array->function_name;
		op_array->function_name = zend_string_init(ZSTR_VAL(old), ZSTR_LEN(old), 0);
		zend_string_release(old); /* release the addref from zend_create_closure */
	}

	/* filename */
	if (op_array->filename) {
		op_array->filename = zend_string_init(
			ZSTR_VAL(op_array->filename), ZSTR_LEN(op_array->filename), 0);
	}

	/* opcodes + literals
	 *
	 * The layout is the one pass_two() leaves behind, and it differs by
	 * platform, because destroy_op_array() frees the two by that same rule:
	 * one block on 64-bit, where the literals live inside the opcode block and
	 * only the opcode block is freed, and two blocks under
	 * ZEND_USE_ABS_CONST_ADDR, where op_array->literals is freed on its own
	 * and must therefore be an allocation of its own. */
	const zval *orig_literals = op_array->literals;
	const zend_op *orig_opcodes = op_array->opcodes;

	if (op_array->opcodes) {
		const size_t literals_size = sizeof(zval) * op_array->last_literal;

		zend_op *new_opcodes;
		zval *new_literals = NULL;

#if ZEND_USE_ABS_CONST_ADDR
		new_opcodes = (zend_op *) emalloc(sizeof(zend_op) * op_array->last);

		if (op_array->last_literal) {
			new_literals = (zval *) emalloc(literals_size);
		}
#else
		/* RT_CONSTANT stores the literal as an int32_t offset from the opline.
		 * emalloc may place independent allocations several GB apart, so
		 * separate blocks would overflow the offset and send the VM into
		 * unrelated memory. Opcodes first, 16-byte aligned, literals
		 * immediately after; destroy_op_array() reads that layout from the
		 * ZEND_ACC_DONE_PASS_TWO flag, which we leave set. */
		const size_t opcodes_size =
			ZEND_MM_ALIGNED_SIZE_EX(sizeof(zend_op) * op_array->last, 16);

		new_opcodes = (zend_op *) emalloc(opcodes_size + literals_size);

		if (op_array->last_literal) {
			new_literals = (zval *) ((char *) new_opcodes + opcodes_size);
		}
#endif

		memcpy(new_opcodes, orig_opcodes, sizeof(zend_op) * op_array->last);

		for (uint32_t i = 0; i < op_array->last; i++) {
			if (new_opcodes[i].opcode != ZEND_DECLARE_LAMBDA_FUNCTION) {
				continue;
			}

			/* The cache slot of this opcode memoizes the Closure object and
			 * pins it in EG(lambda_cache), which is drained when the request
			 * ends. A worker's request outlives every task it runs, so the
			 * pinned object holds the task's nested body at refcount 1 and
			 * destroy_op_array leaves it behind. The memo is worthless here
			 * anyway: a task gets a copy of the op_array with a cache of its
			 * own, so nothing carries over to the next one. This is the
			 * decision zend_compile_func_decl already makes for the top level
			 * of a script, and for the same reason. */
			new_opcodes[i].extended_value = (uint32_t) -1;
		}

		if (op_array->last_literal) {
			for (uint32_t i = 0; i < op_array->last_literal; i++) {
				/* Deep-copy refcounted literals into the worker's heap.
				 * ZVAL_COPY alone only bumps the refcount and leaves the
				 * payload (zend_string / zend_array) in the source
				 * persistent arena. async_thread_snapshot_destroy() runs
				 * right after op_array_to_emalloc and frees that arena;
				 * any opcode that later reads such a literal (e.g. the
				 * function-name string for INIT_FCALL_SPEC_CONST) reads
				 * freed memory and segfaults under load. Strings and
				 * arrays are the only refcounted literal types Zend emits
				 * for user op_arrays. */
				/* The IS_STR_INTERNED flag says nothing here: every arena
				 * string carries it so the source side skips refcounting, and
				 * a literal that looks interned dies with the arena. */
				op_array_emalloc_copy_value(&new_literals[i], &orig_literals[i]);

				/* u2 is not part of the value: for a literal it holds the
				 * runtime cache slot the VM reads through Z_CACHE_SLOT_P, so
				 * losing it sends that read to an arbitrary offset. */
				new_literals[i].u2 = orig_literals[i].u2;
			}
		}
		op_array->literals = new_literals;

#if ZEND_USE_ABS_CONST_ADDR
		for (uint32_t i = 0; i < op_array->last; i++) {
			zend_op *opline = &new_opcodes[i];
			if (opline->op1_type == IS_CONST) {
				opline->op1.zv = (zval *)((char *)opline->op1.zv +
					((char *)new_literals - (char *)orig_literals));
			}
			if (opline->op2_type == IS_CONST) {
				opline->op2.zv = (zval *)((char *)opline->op2.zv +
					((char *)new_literals - (char *)orig_literals));
			}
		}
#else
		for (uint32_t i = 0; i < op_array->last; i++) {
			zend_op *opline = &new_opcodes[i];
			const zend_op *old_opline = &orig_opcodes[i];
			if (opline->op1_type == IS_CONST) {
				zval *old_literal = (zval *)((char *) old_opline +
					(int32_t) opline->op1.constant);
				zval *new_literal = new_literals + (old_literal - orig_literals);
				opline->op1.constant = (uint32_t)((char *) new_literal - (char *) opline);
			}
			if (opline->op2_type == IS_CONST) {
				zval *old_literal = (zval *)((char *) old_opline +
					(int32_t) opline->op2.constant);
				zval *new_literal = new_literals + (old_literal - orig_literals);
				opline->op2.constant = (uint32_t)((char *) new_literal - (char *) opline);
			}
		}
#endif

#if ZEND_USE_ABS_JMP_ADDR
		for (uint32_t i = 0; i < op_array->last; i++) {
			zend_op *opline = &new_opcodes[i];
			switch (opline->opcode) {
				case ZEND_JMP:
				case ZEND_FAST_CALL:
					opline->op1.jmp_addr = &new_opcodes[opline->op1.jmp_addr - orig_opcodes];
					break;
				case ZEND_JMPZ:
				case ZEND_JMPNZ:
				case ZEND_JMPZ_EX:
				case ZEND_JMPNZ_EX:
				case ZEND_JMP_SET:
				case ZEND_COALESCE:
				case ZEND_FE_RESET_R:
				case ZEND_FE_RESET_RW:
				case ZEND_ASSERT_CHECK:
				case ZEND_JMP_NULL:
				case ZEND_BIND_INIT_STATIC_OR_JMP:
				case ZEND_JMP_FRAMELESS:
					opline->op2.jmp_addr = &new_opcodes[opline->op2.jmp_addr - orig_opcodes];
					break;
				case ZEND_CATCH:
					if (!(opline->extended_value & ZEND_LAST_CATCH)) {
						opline->op2.jmp_addr = &new_opcodes[opline->op2.jmp_addr - orig_opcodes];
					}
					break;
				default:
					break;
			}
		}
#endif
		op_array->opcodes = new_opcodes;
	}

	/* arg_info */
	if (op_array->arg_info) {
		zend_arg_info *src = op_array->arg_info;
		uint32_t num_args = op_array->num_args;
		if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
			src--;
			num_args++;
		}
		if (op_array->fn_flags & ZEND_ACC_VARIADIC) {
			num_args++;
		}
		zend_arg_info *new_info = safe_emalloc(num_args, sizeof(zend_arg_info), 0);
		memcpy(new_info, src, sizeof(zend_arg_info) * num_args);
		for (uint32_t i = 0; i < num_args; i++) {
			if (new_info[i].name) {
				new_info[i].name = zend_string_init(
					ZSTR_VAL(new_info[i].name), ZSTR_LEN(new_info[i].name), 0);
			}
			if (new_info[i].doc_comment) {
				new_info[i].doc_comment = zend_string_init(
					ZSTR_VAL(new_info[i].doc_comment), ZSTR_LEN(new_info[i].doc_comment), 0);
			}
			op_array_emalloc_copy_type(&new_info[i].type);
		}
		if (op_array->fn_flags & ZEND_ACC_HAS_RETURN_TYPE) {
			new_info++;
		}
		op_array->arg_info = new_info;
	}

	/* vars */
	if (op_array->vars) {
		zend_string **new_vars = safe_emalloc(op_array->last_var, sizeof(zend_string *), 0);
		for (int i = 0; i < op_array->last_var; i++) {
			new_vars[i] = zend_string_init(
				ZSTR_VAL(op_array->vars[i]), ZSTR_LEN(op_array->vars[i]), 0);
		}
		op_array->vars = new_vars;
	}

	/* attributes */
	if (op_array->attributes) {
		op_array->attributes = op_array_emalloc_copy_attributes(op_array->attributes);
	}

	/* live_range */
	if (op_array->live_range) {
		zend_live_range *new_lr = safe_emalloc(op_array->last_live_range, sizeof(zend_live_range), 0);
		memcpy(new_lr, op_array->live_range, sizeof(zend_live_range) * op_array->last_live_range);
		op_array->live_range = new_lr;
	}

	/* try_catch_array */
	if (op_array->try_catch_array) {
		zend_try_catch_element *new_tc = safe_emalloc(op_array->last_try_catch, sizeof(zend_try_catch_element), 0);
		memcpy(new_tc, op_array->try_catch_array, sizeof(zend_try_catch_element) * op_array->last_try_catch);
		op_array->try_catch_array = new_tc;
	}

	/* doc_comment */
	if (op_array->doc_comment) {
		op_array->doc_comment = zend_string_init(
			ZSTR_VAL(op_array->doc_comment), ZSTR_LEN(op_array->doc_comment), 0);
	}

	/* Nested defs carry the snapshot's persistent immutable static_variables
	 * (refcount 2); dup into a worker-local array destroy_op_array can free.
	 *
	 * BIND_LEXICAL reaches a captured variable by a byte offset into arData,
	 * so a rebuild that dropped a slot would shift every offset past it. The
	 * compiler seeds every slot and leaves no hole, which is what this states. */
	if (op_array->static_variables) {
		ZEND_ASSERT(zend_hash_num_elements(op_array->static_variables)
				== op_array->static_variables->nNumUsed);
		op_array->static_variables = op_array_emalloc_copy_array(op_array->static_variables);
	}

	/* dynamic_func_defs — outer closures with nested function/closure
	 * definitions point into the snapshot's arena via dynamic_func_defs[i].
	 * Without an emalloc-side copy, async_thread_snapshot_destroy frees those
	 * nested op_arrays and the next destroy_op_array() (e.g. when the holder
	 * closure is freed at request shutdown) walks dangling pointers and
	 * triggers heap corruption. Recursively rebuild each nested op_array on
	 * the worker's heap so the loaded closure is fully self-contained. */
	if (op_array->num_dynamic_func_defs) {
		/* Lay out the pointer table and the op_array structs in a single
		 * allocation so destroy_op_array's `efree(dynamic_func_defs)`
		 * releases everything. destroy_op_array calls destroy_op_array on
		 * each dynamic_func_defs[i] (frees its internals) but never frees
		 * the function struct itself — we cannot rely on it doing so. */
		const uint32_t n = op_array->num_dynamic_func_defs;
		const size_t ptrs_size = n * sizeof(zend_op_array *);
		zend_op_array **new_defs = (zend_op_array **) emalloc(ptrs_size + n * sizeof(zend_op_array));
		zend_op_array *bodies = (zend_op_array *)((char *) new_defs + ptrs_size);
		for (uint32_t i = 0; i < n; i++) {
			zend_op_array *src = op_array->dynamic_func_defs[i];
			memcpy(&bodies[i], src, sizeof(zend_op_array));
			bodies[i].fn_flags &= ~ZEND_ACC_IMMUTABLE;
			op_array_to_emalloc(&bodies[i]);
			new_defs[i] = &bodies[i];
		}
		op_array->dynamic_func_defs = new_defs;
	}

	/* Keep ZEND_ACC_DONE_PASS_TWO set: destroy_op_array() uses it to know
	 * that literals are embedded in the opcodes allocation and skips the
	 * separate efree(literals) call. */
}

/* Resolve a transferred scope class name in the current thread. The stored
 * name is persistent (foreign) — lookup and autoload need a local copy. */
static zend_class_entry *thread_resolve_scope_class(const zend_string *name)
{
	zend_string *local = zend_string_init(ZSTR_VAL(name), ZSTR_LEN(name), 0);
	zend_class_entry *ce = zend_lookup_class(local);
	zend_string_release(local);

	if (UNEXPECTED(ce == NULL)) {
		zend_throw_error(NULL,
			"Cannot restore closure scope: class \"%s\" not found in the target thread",
			ZSTR_VAL(name));
	}

	return ce;
}

void async_thread_create_closure_ex(
	const async_thread_closure_copy_t *copy, zval *closure_zv,
	thread_transfer_ctx_t *parent_ctx)
{
	ZEND_ASSERT(copy->func != NULL);

	/* Re-resolve the closure's scope here before building anything. A missing
	 * class is a hard error — silently dropping the scope would break
	 * self::/static:: and member visibility at first use. */
	zend_class_entry *scope = NULL;
	zend_class_entry *called_scope = NULL;

	if (copy->scope_name != NULL) {
		scope = thread_resolve_scope_class(copy->scope_name);

		if (UNEXPECTED(scope == NULL)) {
			ZVAL_UNDEF(closure_zv);
			return;
		}
	}

	if (copy->called_scope_name != NULL) {
		if (scope != NULL && zend_string_equals(copy->called_scope_name, copy->scope_name)) {
			called_scope = scope;
		} else {
			called_scope = thread_resolve_scope_class(copy->called_scope_name);

			if (UNEXPECTED(called_scope == NULL)) {
				ZVAL_UNDEF(closure_zv);
				return;
			}
		}
	}

	zend_function func;
	memcpy(&func, copy->func, sizeof(zend_op_array));
	func.op_array.fn_flags &= ~ZEND_ACC_IMMUTABLE;

	/* Set refcount to NULL: zend_create_closure skips increment,
	 * and destroy_op_array returns early without freeing pemalloc'd data. */
	func.op_array.refcount = NULL;

	/* One load_ctx for both bound vars and $this, joining the caller's when a
	 * load is already in progress — see thread_copy_callable. */
	thread_transfer_ctx_t  own_load_ctx;
	thread_transfer_ctx_t *load = parent_ctx;

	if (load == NULL) {
		thread_transfer_ctx_init(&own_load_ctx);
		load = &own_load_ctx;
	}

	HashTable *loaded_vars = NULL;
	if (copy->bound_vars) {
		loaded_vars = zend_new_array(
			zend_hash_num_elements(copy->bound_vars));

		zend_string *key;
		zval *val;
		ZEND_HASH_FOREACH_STR_KEY_VAL(copy->bound_vars, key, val) {
			zval loaded;
			thread_load_zval_inner(load, &loaded, val);
			zend_string *local_key = zend_string_init(ZSTR_VAL(key), ZSTR_LEN(key), 0);
			zend_hash_add(loaded_vars, local_key, &loaded);
			zend_string_release(local_key);
		} ZEND_HASH_FOREACH_END();

		ZEND_MAP_PTR_INIT(func.op_array.static_variables_ptr, loaded_vars);
	} else {
		ZEND_MAP_PTR_INIT(func.op_array.static_variables_ptr, NULL);
	}

	zval loaded_this;
	ZVAL_UNDEF(&loaded_this);
	if (!Z_ISUNDEF(copy->bound_this)) {
		thread_load_zval_inner(load, &loaded_this, &copy->bound_this);
	}

	if (load == &own_load_ctx) {
		thread_transfer_ctx_destroy(&own_load_ctx);
	}

	/* Detach from persistent static_variables — we already loaded them
	 * into loaded_vars / static_variables_ptr above */
	func.op_array.static_variables = NULL;

	/* Let zend_create_closure allocate runtime cache itself */
	ZEND_MAP_PTR_INIT(func.op_array.run_time_cache, NULL);
	func.op_array.fn_flags &= ~ZEND_ACC_HEAP_RT_CACHE;

	zval *this_arg = NULL;
	if (Z_TYPE(loaded_this) == IS_OBJECT) {
		this_arg = &loaded_this;
	}

	/* Kill the source thread's stale CE pointer before handing to the engine. */
	func.op_array.scope = scope;

	zend_create_closure(closure_zv, &func, scope, called_scope != NULL ? called_scope : scope, this_arg);

	/* zend_create_closure took its own ref on $this; release our load ref. */
	if (Z_TYPE(loaded_this) == IS_OBJECT) {
		zval_ptr_dtor(&loaded_this);
	}

	/* zend_create_closure duplicates static_variables via zend_array_dup,
	 * so we must free our intermediate copy */
	if (loaded_vars) {
		zend_array_destroy(loaded_vars);
	}

	/* Self-contained, always. A nested def shared with the snapshot races on
	 * run_time_cache (#176), and anything still pointing into the arena
	 * outlives it: the arena goes with the task, what the body left in the
	 * thread's state does not (#251). */
	async_zend_closure_t *closure = (async_zend_closure_t *) Z_OBJ_P(closure_zv);
	op_array_to_emalloc(&closure->func.op_array);
}

/* Top-level: no load in progress, so the closure gets its own ctx. */
void async_thread_create_closure(
	const async_thread_closure_copy_t *copy, zval *closure_zv)
{
	async_thread_create_closure_ex(copy, closure_zv, NULL);
}

/**
 * Call a deep-copied closure in the child thread.
 *
 * Creates a Closure from the copy, executes it, and captures any exception
 * immediately (before dtors that could trigger bailout). The event is
 * intentionally not touched here — the caller hands the transferred values
 * to the parent event under a lock once the whole run is finished.
 *
 * @param copy           Deep-copied closure from snapshot
 * @param out_exception  Output: transferred (persistent) exception on failure
 * @param retval         Output: return value (UNDEF on exception or void return)
 * @return true on success, false if exception was captured
 */
static bool thread_call_closure(
	const async_thread_closure_copy_t *copy,
	zend_object **out_exception,
	zval *retval)
{
	zval closure_zv;
	async_thread_create_closure(copy, &closure_zv);

	ZVAL_UNDEF(retval);

	if (EXPECTED(EG(exception) == NULL)) {
		const zend_function *func = zend_get_closure_method_def(Z_OBJ(closure_zv));
		async_zend_closure_t *closure = (async_zend_closure_t *) Z_OBJ(closure_zv);

		/* Execute closure directly via VM, bypassing zend_call_function.
		 * zend_call_function would trigger zend_throw_exception_internal
		 * when current_execute_data is NULL (no PHP caller above us),
		 * converting any uncaught exception into a fatal bailout.
		 * With zend_execute_ex we get the exception cleanly in EG(exception). */
		uint32_t call_info = ZEND_CALL_TOP_FUNCTION;
		void *object_or_scope = NULL;
		if (Z_TYPE(closure->this_ptr) == IS_OBJECT) {
			call_info |= ZEND_CALL_HAS_THIS;
			object_or_scope = Z_OBJ(closure->this_ptr);
		} else if (closure->called_scope != NULL) {
			/* Scoped static call: the frame carries called_scope, no $this. */
			object_or_scope = closure->called_scope;
		}
		zend_execute_data *frame = zend_vm_stack_push_call_frame(
			call_info, (zend_function *) func, 0, object_or_scope);
		zend_init_func_execute_data(frame, (zend_op_array *) &func->op_array, retval);
		zend_execute_ex(frame);
	}

	/* EG(exception) is set when closure creation failed (e.g. scope class
	 * missing in this thread) or the closure threw — no bailout occurred. */
	const bool has_exception = EG(exception) != NULL;
	if (UNEXPECTED(has_exception)) {
		zval exception_zval, transferred_exception;
		ZVAL_OBJ_COPY(&exception_zval, EG(exception));
		zend_clear_exception();
		async_thread_transfer_zval(&transferred_exception, &exception_zval);
		*out_exception = Z_OBJ(transferred_exception);
		zval_ptr_dtor(&exception_zval);
		zval_ptr_dtor(retval);
		ZVAL_UNDEF(retval);
	}

	zval_ptr_dtor(&closure_zv);
	return !has_exception;
}

/**
 * Capture bailout error message into the event.
 * Safe to call after zend_catch — uses only pemalloc and PG().
 *
 * @param fallback  Pre-allocated fallback message (ownership transferred
 *                  if PG(last_error_message) is NULL). Caller must set
 *                  its pointer to NULL after this call to avoid double-free.
 */
static zend_always_inline void thread_capture_bailout(zend_async_thread_context_t *context, char **fallback)
{
	const zend_string *last_error = PG(last_error_message);

	if (last_error != NULL) {
		context->bailout_error_message = pestrdup(ZSTR_VAL(last_error), 1);
	} else {
		context->bailout_error_message = *fallback;
		*fallback = NULL;
	}
}

void async_thread_run(void *arg)
{
	zend_async_thread_context_t *context = (zend_async_thread_context_t *) arg;
	zend_async_thread_event_t *event =
		(zend_async_thread_event_t *) zend_atomic_ptr_load(&context->event); /* may be NULL */
	const async_thread_snapshot_t *snapshot = context->snapshot;
	bool request_started = false;

	/* Result/exception are accumulated locally in persistent memory and only
	 * handed to the parent event at the very end, under context->event_mutex.
	 * Declared up here so they are in scope on every `goto notify` path. */
	zval thread_result;
	ZVAL_UNDEF(&thread_result);
	zend_object *thread_exception = NULL;
#ifdef ZTS
	bool tsrm_initialized = false;
#endif
	char *fallback_message = pestrdup("Unknown fatal error in child thread", 1);
	/* Capture registry key now — context may be released later in cleanup
	 * and we still need the key to self-remove on every exit path. */
	const zend_async_thread_handle_t my_key = context->key;

	/* Record OS thread ID */
#ifdef _WIN32
	zend_atomic_int64_store(&context->thread_id, (int64_t) GetCurrentThreadId());
#else
	zend_atomic_int64_store(&context->thread_id, (int64_t) pthread_self());
#endif

	if (UNEXPECTED(snapshot == NULL && context->internal_entry == NULL)) {
		goto notify;
	}

	/* 1a. Initialize TSRM — must happen before any zend_try
	 * because zend_try uses EG(bailout) which requires TSRM. */
#if ZEND_RC_DEBUG
	/* Permanently disable zend_rc_debug for this process.
	 *
	 * zend_rc_debug is a DEBUG-only global flag (added by Dmitry Stogov
	 * in 2017, commit 49ea143bbd in php/php-src) that asserts on every
	 * GC_ADDREF/GC_DELREF: any GC_PERSISTENT refcounted must also have
	 * GC_PERSISTENT_LOCAL. Intent — catch cross-thread refcount races
	 * on shared persistent data.
	 *
	 * In practice, most PHP core and extension code allocates persistent
	 * data in MINIT/GINIT without setting GC_PERSISTENT_LOCAL — it was
	 * safe in the historical ZTS scenario (Apache worker MPM: TSRM init
	 * once on server start, one thread per request, no shared mutables),
	 * so nobody ever bothered marking. Only ~24 files in the entire
	 * php-src tree use the flag at all.
	 *
	 * When user code spawns new PHP threads (this extension, or e.g.
	 * ext/parallel), each child runs ts_resource() which copies main-thread
	 * globals — including zend_constants. copy_zend_constant calls
	 * GC_ADDREF on the constant's persistent `attributes` HashTable, which
	 * has GC_PERSISTENT but not GC_PERSISTENT_LOCAL. Assert fires, worker
	 * aborts. The same is true for many other persistent allocations made
	 * by core extensions during their per-thread GINIT.
	 *
	 * We cannot fix every extension's GINIT — too many, including
	 * third-party. The flag is DEBUG-only; production builds (release-ZTS)
	 * never read it, so this loses no production safety. We do mark our
	 * own pemalloc'd data with GC_PERSISTENT_LOCAL where appropriate.
	 *
	 * ext/parallel makes the same pragmatic choice (it never installs any
	 * rc_debug guard around its ts_resource() in scheduler.c).
	 *
	 * Idempotent: flag may already be false from another worker. */
	zend_rc_debug = false;
#endif
	async_thread_tsrm_init();
#ifdef ZTS
	tsrm_initialized = true;
#endif

	/* 1b. Start PHP request (can bailout).
	 * zend_first_try initializes EG(bailout) for this thread. */
	zend_first_try {
		if (UNEXPECTED(async_thread_request_startup(snapshot) == FAILURE)) {
			goto notify;
		}
		request_started = true;
	} zend_catch {
		thread_capture_bailout(context, &fallback_message);
		goto notify;
	} zend_end_try();

	zval retval;
	ZVAL_UNDEF(&retval);

	/* 2. Execute entry point */
	zend_first_try {
		if (context->internal_entry != NULL) {
			/* C-level entry: copy handler/ctx, free entry struct, call handler */
			void (*handler)(zend_async_thread_event_t *, void *) = context->internal_entry->handler;
			void *ctx = context->internal_entry->ctx;
			pefree(context->internal_entry, 1);
			context->internal_entry = NULL;

			handler(event, ctx);
			goto cleanup;
		}

		/* Bootloader (optional) */
		if (snapshot->bootloader.func != NULL) {
			if (!thread_call_closure(&snapshot->bootloader, &thread_exception, &retval)) {
				goto cleanup;
			}
		}

		/* Entry closure */
		if (thread_call_closure(&snapshot->entry, &thread_exception, &retval)) {
			if (!Z_ISUNDEF(retval)) {
				async_thread_transfer_zval(&thread_result, &retval);

				/* Transfer itself may fail (unsupported types, depth limit) */
				if (UNEXPECTED(EG(exception))) {
					zval exception_zval, transferred_exception;
					ZVAL_OBJ_COPY(&exception_zval, EG(exception));
					zend_clear_exception();
					async_thread_transfer_zval(&transferred_exception, &exception_zval);
					thread_exception = Z_OBJ(transferred_exception);
					zval_ptr_dtor(&exception_zval);
					ZVAL_UNDEF(&thread_result);
				}
			}
		}

cleanup:
		zval_ptr_dtor(&retval);
	} zend_catch {
		thread_capture_bailout(context, &fallback_message);
	} zend_end_try();

	/* 3. Shut down PHP request — must happen BEFORE snapshot destroy
	 * because request shutdown may destroy closures (e.g. autoloader
	 * callbacks) whose op_arrays reference arena-allocated strings. */
	if (EXPECTED(request_started)) {
		zend_first_try {
			async_thread_request_shutdown();
		} zend_catch {
			/* Bailout during shutdown — not much we can do,
			 * but don't overwrite a prior bailout message. */
			if (context->bailout_error_message == NULL) {
				thread_capture_bailout(context, &fallback_message);
			}
		} zend_end_try();
	}

	/* 4. Free snapshot arena — safe now that all PHP objects referencing
	 * arena-allocated op_array data have been destroyed by request shutdown. */
	if (context->snapshot != NULL) {
		async_thread_snapshot_destroy(context->snapshot);
		context->snapshot = NULL;
	}

notify:
	/* Hand the result/exception to the parent event under context->event_mutex.
	 * The parent's dispose path stores NULL into context->event and drains
	 * this mutex before freeing the event, so a non-NULL load here means the
	 * event stays alive for the whole locked section. A NULL load means the
	 * parent already gave up — the transferred values are ours to release.
	 *
	 * Done before ts_free_thread: the handoff and the persistent-memory
	 * release below touch only pemalloc'd data and uv_async_send, none of
	 * which need this thread's TSRM storage. */
	ZEND_ASYNC_THREAD_CONTEXT_EVENT_MUTEX_LOCK(context);
	{
		zend_async_thread_event_t *parent_event =
			(zend_async_thread_event_t *) zend_atomic_ptr_load(&context->event);
		if (parent_event != NULL) {
			if (!Z_ISUNDEF(thread_result)) {
				parent_event->result = thread_result;
				ZVAL_UNDEF(&thread_result);
			}
			if (thread_exception != NULL) {
				parent_event->exception = thread_exception;
				thread_exception = NULL;
			}
			parent_event->notify_parent(parent_event);
		}
	}
	ZEND_ASYNC_THREAD_CONTEXT_EVENT_MUTEX_UNLOCK(context);

	/* Parent already detached — release whatever was not handed off. */
	if (!Z_ISUNDEF(thread_result)) {
		async_thread_release_transferred_zval(&thread_result);
	}
	if (thread_exception != NULL) {
		zval exc_pz;
		ZVAL_OBJ(&exc_pz, thread_exception);
		async_thread_release_transferred_zval(&exc_pz);
		thread_exception = NULL;
	}

	/* Mirror of main.c clear_last_error() (static, no PHPAPI exposed).
	 * php_request_shutdown() normally clears these via php_free_request_globals,
	 * but if request shutdown bailed out (caught by zend_first_try above) or
	 * a worker-internal error fired between shutdown and ts_free_thread, the
	 * strings survive. core_globals_dtor asserts !last_error_message on debug
	 * builds; on release it would silently leak. Unconditional + idempotent. */
#ifdef ZTS
	if (tsrm_initialized) {
		if (PG(last_error_message)) {
			zend_string_release(PG(last_error_message));
			PG(last_error_message) = NULL;
		}
		if (PG(last_error_file)) {
			zend_string_release(PG(last_error_file));
			PG(last_error_file) = NULL;
		}
	}
#endif

	/* Explicit OpenSSL 3 per-thread state cleanup.
	 *
	 * OpenSSL 3 registers a pthread_key destructor on first use to clean up
	 * per-thread RCU reader state. That destructor only frees state allocated
	 * against the *default* libctx — state allocated against a custom libctx
	 * (OSSL_LIB_CTX_new) is invisible to it, even after OSSL_LIB_CTX_free.
	 *
	 * PHP's openssl extension hits exactly this combination: MINIT initialises
	 * the default libctx (via OPENSSL_init_ssl), then per-thread GINIT creates
	 * its own custom libctx in php_openssl_backend_init_libctx and calls
	 * CONF_modules_load_file_ex on it — which allocates ~240 B per worker
	 * thread that the destructor cannot reach.
	 *
	 * Verified standalone: pthread_create + pthread_join, main-thread call to
	 * CONF_modules_load(NULL,...) + worker creating custom libctx → 240 B
	 * leak. Same code without the main-thread call: no leak. Same code with
	 * an explicit OPENSSL_thread_stop() in the worker: no leak.
	 *
	 * Weak external so we don't take a hard libcrypto dependency. If openssl
	 * is not loaded the symbol is NULL and the call is skipped. */
#if defined(__GNUC__) || defined(__clang__)
	{
		extern void OPENSSL_thread_stop(void) __attribute__((weak));
		if (OPENSSL_thread_stop != NULL) {
			OPENSSL_thread_stop();
		}
	}
#endif

	/* Free TSRM storage after all zend_end_try blocks.
	 * Must be separate because zend_end_try accesses EG(bailout).
	 * Skipped on early-exit paths where TSRM was never initialized. */
#ifdef ZTS
	if (tsrm_initialized) {
# if ZEND_RC_DEBUG
		/* See rationale at the startup site. ts_free_thread() runs the same
		 * GINIT-mirror destructors for every loaded module, so the same
		 * unmarked-persistent assert fires here without this. */
		zend_rc_debug = false;
# endif
		ts_free_thread();
	}
#endif

	if (fallback_message != NULL) {
		pefree(fallback_message, 1);
	}

	/* Self-remove from the reactor's child thread registry.
	 * Must happen after ts_free_thread so the main thread, once it wakes
	 * in libuv_reactor_quiesce, can safely proceed into php_module_shutdown
	 * without racing this child against ts_free_id(). The key is captured
	 * at entry and stable, so this fires on every exit path. */
	async_libuv_thread_registry_remove(my_key);

	/* Release our context ref last — the handoff above needs it alive, and
	 * whoever drops the last ref also frees context->event_mutex. */
	ZEND_ASYNC_THREAD_CONTEXT_RELEASE(context);
	context = NULL;
}

///////////////////////////////////////////////////////////
/// 3. Thread PHP object — Async\Thread class
///////////////////////////////////////////////////////////

#define METHOD(name) PHP_METHOD(Async_Thread, name)
#define THIS_THREAD() Z_ASYNC_THREAD_P(ZEND_THIS)

zend_class_entry *async_ce_thread = NULL;
zend_class_entry *async_ce_remote_exception = NULL;
zend_class_entry *async_ce_thread_transfer_exception = NULL;

static zend_object_handlers thread_object_handlers;

/* ---- RemoteException methods ---- */

PHP_METHOD(Async_RemoteException, getRemoteException)
{
	ZEND_PARSE_PARAMETERS_NONE();
	zval *prop = zend_read_property(async_ce_remote_exception, Z_OBJ_P(ZEND_THIS),
		"remoteException", sizeof("remoteException") - 1, 0, NULL);
	RETURN_ZVAL(prop, 1, 0);
}

PHP_METHOD(Async_RemoteException, getRemoteClass)
{
	ZEND_PARSE_PARAMETERS_NONE();
	zval *prop = zend_read_property(async_ce_remote_exception, Z_OBJ_P(ZEND_THIS),
		"remoteClass", sizeof("remoteClass") - 1, 0, NULL);
	RETURN_ZVAL(prop, 1, 0);
}

/* ---- Object Lifecycle ---- */

static zend_object *thread_object_create(zend_class_entry *class_entry)
{
	async_thread_object_t *thread = zend_object_alloc(sizeof(async_thread_object_t), class_entry);

	ZEND_ASYNC_EVENT_REF_SET(thread, offsetof(async_thread_object_t, std), NULL);

	thread->thread_event = NULL;
	thread->parent_scope = NULL;
	thread->finally_handlers = NULL;

	zend_object_std_init(&thread->std, class_entry);
	object_properties_init(&thread->std, class_entry);

	return &thread->std;
}

/* Release the extra Thread ref bumped before dispatching finally handlers.
 * Invoked by finally_handlers_iterator_dtor() once all handlers finish. */
static void thread_finally_handlers_dtor(finally_handlers_context_t *context)
{
	if (context->target != NULL) {
		async_thread_object_t *thread = (async_thread_object_t *) context->target;
		context->target = NULL;
		OBJ_RELEASE(&thread->std);
	}
}

static void thread_object_dtor(zend_object *object)
{
	async_thread_object_t *thread = async_thread_object_from_obj(object);

	if (thread->thread_event != NULL) {
		zend_async_event_t *event = &thread->thread_event->base;

		/* Call finally handlers if the thread completed. The parent scope
		 * was captured at spawn time and is kept alive by a refcount held
		 * on thread->parent_scope (released in thread_object_free), so
		 * async_call_finally_handlers() always has a valid parent to spawn
		 * the handler-dispatch coroutine under. If the async subsystem is
		 * already off (late zend_call_destructors path), it is not safe to
		 * spawn anything — drop the handlers instead. */
		if (thread->finally_handlers != NULL
			&& zend_hash_num_elements(thread->finally_handlers) > 0
			&& ZEND_ASYNC_EVENT_IS_CLOSED(event)) {

			if (thread->parent_scope == NULL || ZEND_ASYNC_IS_OFF) {
				zend_array_destroy(thread->finally_handlers);
				thread->finally_handlers = NULL;
			} else {
				finally_handlers_context_t *finally_context = ecalloc(1, sizeof(finally_handlers_context_t));
				finally_context->target = thread;
				finally_context->scope = thread->parent_scope;
				finally_context->dtor = thread_finally_handlers_dtor;
				finally_context->params_count = 1;
				ZVAL_OBJ(&finally_context->params[0], &thread->std);

				HashTable *handlers = thread->finally_handlers;
				thread->finally_handlers = NULL;

				if (async_call_finally_handlers(handlers, finally_context, 0)) {
					GC_ADDREF(&thread->std);
				} else {
					efree(finally_context);
					zend_array_destroy(handlers);
				}
			}
		}

		/* Dispose the underlying event (Thread object is the sole owner) */
		if (event->dispose != NULL) {
			event->dispose(event);
		}
		thread->thread_event = NULL;
	}

	if (thread->finally_handlers) {
		zend_array_destroy(thread->finally_handlers);
		thread->finally_handlers = NULL;
	}
}

static void thread_object_free(zend_object *object)
{
	async_thread_object_t *thread = async_thread_object_from_obj(object);

	/* free_obj must dispose the event too: php_error_cb() marks every object
	 * as destructed before a fatal bails out, so dtor_obj does not run at all.
	 * An event left undisposed keeps its notify handle open, which fails
	 * uv_loop_close() at reactor shutdown and leaks the thread context. */
	if (thread->thread_event != NULL) {
		zend_async_event_t *event = &thread->thread_event->base;
		thread->thread_event = NULL;

		if (event->dispose != NULL) {
			event->dispose(event);
		}
	}

	if (thread->finally_handlers != NULL) {
		zend_array_destroy(thread->finally_handlers);
		thread->finally_handlers = NULL;
	}

	/* Release the scope ref captured at spawn time. This runs after the
	 * dtor has finished (and after any delayed finally-handlers kept the
	 * Thread alive via GC_ADDREF), so the scope outlives every handler
	 * that might reference it. */
	if (thread->parent_scope != NULL) {
		ZEND_ASYNC_EVENT_RELEASE(&thread->parent_scope->event);
		thread->parent_scope = NULL;
	}

	zend_object_std_dtor(object);
}

static HashTable *thread_object_gc(zend_object *object, zval **table, int *num)
{
	async_thread_object_t *thread = async_thread_object_from_obj(object);

	zend_get_gc_buffer *buf = zend_get_gc_buffer_create();

	if (thread->thread_event != NULL) {
		if (!Z_ISUNDEF(thread->thread_event->result)) {
			zend_get_gc_buffer_add_zval(buf, &thread->thread_event->result);
		}
		if (thread->thread_event->exception != NULL) {
			zend_get_gc_buffer_add_obj(buf, thread->thread_event->exception);
		}
	}

	if (thread->finally_handlers) {
		zval *val;
		ZEND_HASH_FOREACH_VAL(thread->finally_handlers, val)
		{
			zend_get_gc_buffer_add_zval(buf, val);
		}
		ZEND_HASH_FOREACH_END();
	}

	zend_get_gc_buffer_use(buf, table, num);

	return NULL;
}

/* ---- PHP Methods ---- */

METHOD(__construct)
{
	zend_throw_error(NULL, "Cannot directly construct Async\\Thread");
}

METHOD(isRunning)
{
	ZEND_PARSE_PARAMETERS_NONE();

	const async_thread_object_t *thread = THIS_THREAD();

	if (UNEXPECTED(thread->thread_event == NULL)) {
		RETURN_FALSE;
	}

	const zend_async_event_t *event = &thread->thread_event->base;

	RETURN_BOOL(event->loop_ref_count > 0
		&& !ZEND_ASYNC_EVENT_IS_CLOSED(event));
}

METHOD(isCompleted)
{
	ZEND_PARSE_PARAMETERS_NONE();

	const async_thread_object_t *thread = THIS_THREAD();

	if (UNEXPECTED(thread->thread_event == NULL)) {
		RETURN_TRUE;
	}

	RETURN_BOOL(ZEND_ASYNC_EVENT_IS_CLOSED(&thread->thread_event->base));
}

METHOD(isCancelled)
{
	ZEND_PARSE_PARAMETERS_NONE();

	const async_thread_object_t *thread = THIS_THREAD();

	if (UNEXPECTED(thread->thread_event == NULL)) {
		RETURN_FALSE;
	}

	/* TODO: thread cancellation not yet implemented */
	RETURN_FALSE;
}

METHOD(getResult)
{
	ZEND_PARSE_PARAMETERS_NONE();

	const async_thread_object_t *thread = THIS_THREAD();

	if (UNEXPECTED(thread->thread_event == NULL)
		|| !ZEND_ASYNC_EVENT_IS_CLOSED(&thread->thread_event->base)) {
		RETURN_NULL();
	}

	if (!Z_ISUNDEF(thread->thread_event->result)) {
		RETURN_COPY(&thread->thread_event->result);
	}

	RETURN_NULL();
}

METHOD(getException)
{
	ZEND_PARSE_PARAMETERS_NONE();

	const async_thread_object_t *thread = THIS_THREAD();

	if (UNEXPECTED(thread->thread_event == NULL)
		|| !ZEND_ASYNC_EVENT_IS_CLOSED(&thread->thread_event->base)) {
		RETURN_NULL();
	}

	if (thread->thread_event->exception != NULL) {
		RETURN_OBJ_COPY(thread->thread_event->exception);
	}

	RETURN_NULL();
}

METHOD(cancel)
{
	zval *cancellation = NULL;

	ZEND_PARSE_PARAMETERS_START(0, 1)
		Z_PARAM_OPTIONAL
		Z_PARAM_OBJECT_OF_CLASS_OR_NULL(cancellation, async_ce_completable)
	ZEND_PARSE_PARAMETERS_END();

	async_thread_object_t *thread = THIS_THREAD();

	if (UNEXPECTED(thread->thread_event == NULL)) {
		return;
	}

	zend_async_event_t *event = &thread->thread_event->base;

	if (ZEND_ASYNC_EVENT_IS_CLOSED(event)) {
		return;
	}

	/* TODO: implement thread cancellation mechanism */
	async_throw_error("Thread cancellation is not yet implemented");
}

METHOD(finally)
{
	zval *callable;

	ZEND_PARSE_PARAMETERS_START(1, 1)
		Z_PARAM_ZVAL(callable)
	ZEND_PARSE_PARAMETERS_END();

	if (UNEXPECTED(!zend_is_callable(callable, 0, NULL))) {
		async_throw_error("Argument #1 ($callback) must be callable");
		RETURN_THROWS();
	}

	async_thread_object_t *thread = THIS_THREAD();

	/* If the thread is already completed, call the callback immediately */
	if (thread->thread_event != NULL
		&& ZEND_ASYNC_EVENT_IS_CLOSED(&thread->thread_event->base)) {
		zval rv;
		ZVAL_UNDEF(&rv);

		zval param;
		ZVAL_OBJ(&param, &thread->std);

		zend_fcall_info fci;
		zend_fcall_info_cache fcc;
		if (zend_fcall_info_init(callable, 0, &fci, &fcc, NULL, NULL) == SUCCESS) {
			fci.retval = &rv;
			fci.param_count = 1;
			fci.params = &param;
			zend_call_function(&fci, &fcc);
		}

		zval_ptr_dtor(&rv);
		return;
	}

	if (thread->finally_handlers == NULL) {
		thread->finally_handlers = zend_new_array(0);
	}

	if (UNEXPECTED(zend_hash_next_index_insert(thread->finally_handlers, callable) == NULL)) {
		async_throw_error("Failed to add finally handler to thread");
		RETURN_THROWS();
	}

	Z_TRY_ADDREF_P(callable);
}

/* API wrappers for opaque void* signatures */
/* API-compatible wrappers with opaque void* signatures */
void *async_thread_snapshot_create_api(
	const zend_fcall_t *entry, const zend_fcall_t *bootloader)
{
	return async_thread_snapshot_create(entry, bootloader, false);
}

void async_thread_snapshot_destroy_api(void *snapshot)
{
	async_thread_snapshot_destroy((async_thread_snapshot_t *) snapshot);
}

///////////////////////////////////////////////////////////
/// Closure transfer_obj handler
///////////////////////////////////////////////////////////

/**
 * Persistent wrapper for a transferred closure.
 * Layout: zend_object header only — snapshot pointer stashed in `properties`
 * (always NULL for non-closure shells, so its non-NULL value uniquely
 * identifies a closure shell for thread_release_transferred_object).
 */
static zend_object *closure_transfer_obj(
	zend_object *object, zend_async_thread_transfer_ctx_t *ctx,
	zend_object_transfer_kind_t kind, zend_object_transfer_default_fn default_fn)
{
	if (kind == ZEND_OBJECT_TRANSFER_RELEASE) {
		/* The snapshot is freed by thread_release_transferred_object, which
		 * recognises a closure shell by its non-NULL `properties`. */
		return NULL;
	}

	if (kind == ZEND_OBJECT_TRANSFER) {
		/* Source thread → persistent: deep-copy closure via snapshot */
		const zend_function *func = zend_get_closure_method_def(object);

		zend_fcall_t fcall;
		memset(&fcall, 0, sizeof(fcall));
		fcall.fci_cache.function_handler = (zend_function *) func;
		/* zend_closure has std as first member, so the cast is valid. */
		fcall.fci_cache.called_scope = ((async_zend_closure_t *) object)->called_scope;

		/* Preserve the closure's bound $this. thread_copy_callable reads the
		 * bound instance from fci_cache.object; without this the snapshot is
		 * built unbound and the destination thread recreates an object-less
		 * closure — a method/object-bound closure then dereferences a NULL
		 * $this on first use. */
		zval closure_zv;
		ZVAL_OBJ(&closure_zv, object);
		zval *bound_this = zend_get_closure_this_ptr(&closure_zv);
		if (bound_this != NULL && Z_TYPE_P(bound_this) == IS_OBJECT) {
			fcall.fci_cache.object = Z_OBJ_P(bound_this);
		}

		/* Share our ctx: the captured vars may point back at an object this
		 * transfer already copied — see thread_copy_callable. */
		async_thread_snapshot_t *snapshot =
			async_thread_snapshot_create_ex(&fcall, NULL, false, ctx);
		if (snapshot == NULL) {
			return NULL;
		}

		/* Minimal persistent shell: just a zend_object header. The snapshot
		 * pointer lives in `properties` (NULL for any other shell, so its
		 * non-NULL value uniquely identifies a closure shell). */
		zend_object *dst = pecalloc(1, sizeof(zend_object), 1);
		GC_SET_REFCOUNT(dst, 1);
		GC_MAKE_PERSISTENT_LOCAL(dst);
		dst->extra_flags = 0; /* offset = 0 */

		/* Store class name for LOAD phase lookup */
		dst->ce = (zend_class_entry *) thread_transfer_string(ctx, object->ce->name);
		dst->handlers = (const zend_object_handlers *)(uintptr_t) 0; /* prop_count = 0 */
		dst->properties = (HashTable *) snapshot; /* repurposed as snapshot ptr */

		return dst;
	} else {
		/* Destination thread → emalloc: recreate closure from snapshot */
		async_thread_snapshot_t *snapshot = (async_thread_snapshot_t *) object->properties;

		/* Share our ctx — mirror of the store side. */
		zval closure_zv;
		async_thread_create_closure_ex(&snapshot->entry, &closure_zv, ctx);

		if (UNEXPECTED(EG(exception) != NULL)) {
			/* Scope class missing here or a bound value failed to load.
			 * Follow the default-loader convention: throw + stdClass fallback. */
			zval_ptr_dtor(&closure_zv);
			async_thread_snapshot_destroy(snapshot);
			object->properties = NULL;

			zend_object *fallback = zend_objects_new(zend_standard_class_def);
			object_properties_init(fallback, zend_standard_class_def);
			return fallback;
		}

		/* Not freed here: with a cycle the snapshot's captured vars include an
		 * object still being loaded above us. The shell's release walk frees it
		 * afterwards, and its visited set makes that happen exactly once. */

		return Z_OBJ(closure_zv);
	}
}

void async_register_thread_ce(void)
{
	async_ce_remote_exception = register_class_Async_RemoteException(async_ce_async_exception);
	async_ce_thread_transfer_exception = register_class_Async_ThreadTransferException(async_ce_async_exception);

	async_ce_thread = register_class_Async_Thread(async_ce_completable);
	async_ce_thread->create_object = thread_object_create;
	async_ce_thread->default_object_handlers = &thread_object_handlers;

	thread_object_handlers = std_object_handlers;
	thread_object_handlers.offset = offsetof(async_thread_object_t, std);
	thread_object_handlers.clone_obj = NULL;
	thread_object_handlers.dtor_obj = thread_object_dtor;
	thread_object_handlers.free_obj = thread_object_free;
	thread_object_handlers.get_gc = thread_object_gc;

	/* Register transfer_obj for Closure — enables cross-thread closure transfer */
	((zend_object_handlers *) zend_ce_closure->default_object_handlers)->transfer_obj = closure_transfer_obj;
}