diff --git a/TODO b/TODO index 663dfca..7238f71 100644 --- a/TODO +++ b/TODO @@ -11,7 +11,7 @@ - irreducable loops detection/support - range inference and PI node -- SCCP edge cases +- SCCP edge cases (remove MERGE with PHIs) - Folding after SCCP (see combo4.ir) - Extend SCCP to remove dead LOADs diff --git a/ir.h b/ir.h index 2696797..2aca460 100644 --- a/ir.h +++ b/ir.h @@ -264,7 +264,6 @@ typedef enum _ir_type { \ /* tracing JIT helpers */ \ _(EXITCALL, x2, src, def, ___) /* save all CPU registers */ \ - _(EXITGROUP, c1X2, src, num, num) /* code to push exit number */ \ _(SNAPSHOT, xN, src, def, def) /* CALL(src, args...) */ \ @@ -613,6 +612,8 @@ void ir_consistency_check(void); /* Code patching (implementation in ir_patch.c) */ int ir_patch(const void *code, size_t size, uint32_t jmp_table_size, const void *from_addr, const void *to_addr); +/* Deoptimization helpers */ +const void *ir_emit_exitgroup(uint32_t first_exit_point, uint32_t exit_points_per_group, const void *exit_addr, void *code_buffer, size_t code_buffer_size, size_t *size_ptr); /* IR Memmory Allocation */ #ifndef ir_mem_malloc diff --git a/ir_aarch64.dasc b/ir_aarch64.dasc index 90c879b..57dd8fe 100644 --- a/ir_aarch64.dasc +++ b/ir_aarch64.dasc @@ -4435,28 +4435,6 @@ static void ir_emit_exitcall(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } -static void ir_emit_exitgroup(ir_ctx *ctx, ir_ref def, ir_insn *insn) -{ - ir_backend_data *data = ctx->data; - dasm_State **Dst = &data->dasm_state; - uint32_t i; - uint32_t first_exit_point = insn->op2; - uint32_t exit_points_per_group = insn->op3; - - | bl >2 - |1: - for (i = 1; i < exit_points_per_group; i++) { - | bl >2 - } - |2: - | adr Rx(IR_REG_INT_TMP), <1 - | sub Rx(IR_REG_INT_TMP), lr, Rx(IR_REG_INT_TMP) - | lsr Rx(IR_REG_INT_TMP), Rx(IR_REG_INT_TMP), #2 - if (first_exit_point) { - | add Rx(IR_REG_INT_TMP), Rx(IR_REG_INT_TMP), #first_exit_point - } -} - static int ir_emit_dessa_move(ir_ctx *ctx, uint8_t type, ir_ref from, ir_ref to) { ir_backend_data *data = ctx->data; @@ -5319,9 +5297,6 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_EXITCALL: ir_emit_exitcall(ctx, i, insn); break; - case IR_EXITGROUP: - ir_emit_exitgroup(ctx, i, insn); - break; case IR_GUARD: case IR_GUARD_NOT: ir_emit_guard(ctx, i, insn); @@ -5481,3 +5456,81 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) return entry; } + +const void *ir_emit_exitgroup(uint32_t first_exit_point, uint32_t exit_points_per_group, const void *exit_addr, void *code_buffer, size_t code_buffer_size, size_t *size_ptr) +{ + void *entry; + size_t size; + uint32_t i; + dasm_State **Dst, *dasm_state; + int ret; + + /* IR_ASSERT(aarch64_may_use_b(ctx, exit_addr)) */ + IR_ASSERT(code_buffer); + if ((char*)exit_addr >= (char*)code_buffer && (char*)exit_addr < (char*)code_buffer + code_buffer_size) { + IR_ASSERT(code_buffer_size < B_IMM); + } else if ((char*)exit_addr >= (char*)code_buffer + code_buffer_size) { + IR_ASSERT(((char*)exit_addr - (char*)code_buffer) < B_IMM); + } else if ((char*)exit_addr < (char*)code_buffer) { + IR_ASSERT(((((char*)(code_buffer)) + code_buffer_size) - (char*)exit_addr) < B_IMM); + } else { + IR_ASSERT(0); + } + + Dst = &dasm_state; + dasm_state = NULL; + dasm_init(&dasm_state, DASM_MAXSECTION); + dasm_setupglobal(&dasm_state, dasm_labels, ir_lb_MAX); + dasm_setup(&dasm_state, dasm_actions); + + | bl >2 + |1: + for (i = 1; i < exit_points_per_group; i++) { + | bl >2 + } + |2: + | adr Rx(IR_REG_INT_TMP), <1 + | sub Rx(IR_REG_INT_TMP), lr, Rx(IR_REG_INT_TMP) + | lsr Rx(IR_REG_INT_TMP), Rx(IR_REG_INT_TMP), #2 + if (first_exit_point) { + | add Rx(IR_REG_INT_TMP), Rx(IR_REG_INT_TMP), #first_exit_point + } + | b &exit_addr + + ret = dasm_link(&dasm_state, &size); + if (ret != DASM_S_OK) { + IR_ASSERT(0); + return NULL; + } + + if (code_buffer != NULL) { + if (IR_ALIGNED_SIZE(size, 16) > code_buffer_size) { + return NULL; + } + entry = code_buffer; + IR_ASSERT((uintptr_t)entry % 16 == 0); + } else { + entry = ir_mem_mmap(size); + ir_mem_unprotect(entry, size); + } + + ret = dasm_encode(&dasm_state, entry); + if (ret != DASM_S_OK) { + IR_ASSERT(0); + if (code_buffer == NULL) { + ir_mem_unmap(entry, size); + } + return NULL; + } + + dasm_free(&dasm_state); + + ir_mem_flush(entry, size); + + if (code_buffer == NULL) { + ir_mem_protect(entry, size); + } + + *size_ptr = size; + return entry; +} diff --git a/ir_x86.dasc b/ir_x86.dasc index 4f53117..134ce05 100644 --- a/ir_x86.dasc +++ b/ir_x86.dasc @@ -6893,23 +6893,6 @@ static void ir_emit_exitcall(ir_ctx *ctx, ir_ref def, ir_insn *insn) } } -static void ir_emit_exitgroup(ir_ctx *ctx, ir_ref def, ir_insn *insn) -{ - ir_backend_data *data = ctx->data; - dasm_State **Dst = &data->dasm_state; - uint32_t i; - uint32_t first_exit_point = insn->op2; - uint32_t exit_points_per_group = insn->op3; - - for (i = 0; i < exit_points_per_group - 1; i++) { - | push byte i - | .byte 0xeb, (4*(exit_points_per_group-i)-6) // jmp >1 - } - | push byte i - |// 1: - | add aword [r4], first_exit_point -} - static int ir_emit_dessa_move(ir_ctx *ctx, uint8_t type, ir_ref from, ir_ref to) { ir_backend_data *data = ctx->data; @@ -8162,9 +8145,6 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) case IR_EXITCALL: ir_emit_exitcall(ctx, i, insn); break; - case IR_EXITGROUP: - ir_emit_exitgroup(ctx, i, insn); - break; case IR_GUARD: case IR_GUARD_NOT: ir_emit_guard(ctx, i, insn); @@ -8318,3 +8298,68 @@ void *ir_emit_code(ir_ctx *ctx, size_t *size_ptr) return entry; } + +const void *ir_emit_exitgroup(uint32_t first_exit_point, uint32_t exit_points_per_group, const void *exit_addr, void *code_buffer, size_t code_buffer_size, size_t *size_ptr) +{ + void *entry; + size_t size; + uint32_t i; + dasm_State **Dst, *dasm_state; + int ret; + + IR_ASSERT(code_buffer); + IR_ASSERT(IR_IS_SIGNED_32BIT((char*)exit_addr - (char*)code_buffer)); + IR_ASSERT(IR_IS_SIGNED_32BIT((char*)exit_addr - ((char*)code_buffer + code_buffer_size))); + + Dst = &dasm_state; + dasm_state = NULL; + dasm_init(&dasm_state, DASM_MAXSECTION); + dasm_setupglobal(&dasm_state, dasm_labels, ir_lb_MAX); + dasm_setup(&dasm_state, dasm_actions); + + for (i = 0; i < exit_points_per_group - 1; i++) { + | push byte i + | .byte 0xeb, (4*(exit_points_per_group-i)-6) // jmp >1 + } + | push byte i + |// 1: + | add aword [r4], first_exit_point + | jmp aword &exit_addr + + ret = dasm_link(&dasm_state, &size); + if (ret != DASM_S_OK) { + IR_ASSERT(0); + return NULL; + } + + if (code_buffer != NULL) { + if (IR_ALIGNED_SIZE(size, 16) > code_buffer_size) { + return NULL; + } + entry = code_buffer; + IR_ASSERT((uintptr_t)entry % 16 == 0); + } else { + entry = ir_mem_mmap(size); + ir_mem_unprotect(entry, size); + } + + ret = dasm_encode(&dasm_state, entry); + if (ret != DASM_S_OK) { + IR_ASSERT(0); + if (code_buffer == NULL) { + ir_mem_unmap(entry, size); + } + return NULL; + } + + dasm_free(&dasm_state); + + ir_mem_flush(entry, size); + + if (code_buffer == NULL) { + ir_mem_protect(entry, size); + } + + *size_ptr = size; + return entry; +}