diff --git a/ir.c b/ir.c index 8ff31d9..d6ec1c6 100644 --- a/ir.c +++ b/ir.c @@ -318,6 +318,8 @@ void ir_init(ir_ctx *ctx, uint32_t flags, ir_ref consts_limit, ir_ref insns_limi ctx->fixed_regset = 0; ctx->fixed_save_regset = 0; ctx->live_intervals = NULL; + ctx->arena = NULL; + ctx->unused_ranges = NULL; ctx->regs = NULL; ctx->prev_ref = NULL; ctx->data = NULL; @@ -379,7 +381,10 @@ void ir_free(ir_ctx *ctx) ir_mem_free(ctx->vregs); } if (ctx->live_intervals) { - ir_free_live_intervals(ctx->live_intervals, ctx->vregs_count); + ir_mem_free(ctx->live_intervals); + } + if (ctx->arena) { + ir_arena_free(ctx->arena); } if (ctx->regs) { ir_mem_free(ctx->regs); diff --git a/ir.h b/ir.h index 1421ee1..8c5d735 100644 --- a/ir.h +++ b/ir.h @@ -504,6 +504,7 @@ void ir_strtab_free(ir_strtab *strtab); typedef struct _ir_ctx ir_ctx; typedef struct _ir_use_list ir_use_list; typedef struct _ir_block ir_block; +typedef struct _ir_arena ir_arena; typedef struct _ir_live_interval ir_live_interval; typedef struct _ir_live_range ir_live_range; typedef int8_t ir_regs[4]; @@ -538,6 +539,7 @@ struct _ir_ctx { int32_t fixed_call_stack_size; /* fixed preallocated stack for parameter passing (default 0) */ uint64_t fixed_save_regset; /* registers that always saved/restored in prologue/epilugue */ ir_live_interval **live_intervals; + ir_arena *arena; ir_live_range *unused_ranges; ir_regs *regs; ir_ref *prev_ref; diff --git a/ir_aarch64.dasc b/ir_aarch64.dasc index 6542cfd..2638a23 100644 --- a/ir_aarch64.dasc +++ b/ir_aarch64.dasc @@ -4525,6 +4525,11 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) /* vregs + tmp + fixed + SRATCH + ALL */ ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_NUM + 2, sizeof(ir_live_interval*)); + + if (!ctx->arena) { + ctx->arena = ir_arena_create(16 * 1024); + } + for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) { IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE)); for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) { @@ -4566,7 +4571,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) } } if (!ctx->live_intervals[ctx->vregs[i]]) { - ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval)); + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); + memset(ival, 0, sizeof(ir_live_interval)); ctx->live_intervals[ctx->vregs[i]] = ival; ival->type = insn->type; ival->reg = IR_REG_NONE; @@ -4591,7 +4597,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) if (use_insn->op == IR_VLOAD) { if (ctx->vregs[use] && !ctx->live_intervals[ctx->vregs[use]]) { - ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval)); + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); + memset(ival, 0, sizeof(ir_live_interval)); ctx->live_intervals[ctx->vregs[use]] = ival; ival->type = insn->type; ival->reg = IR_REG_NONE; @@ -4603,7 +4610,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) if (!IR_IS_CONST_REF(use_insn->op3) && ctx->vregs[use_insn->op3] && !ctx->live_intervals[ctx->vregs[use_insn->op3]]) { - ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval)); + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); + memset(ival, 0, sizeof(ir_live_interval)); ctx->live_intervals[ctx->vregs[use_insn->op3]] = ival; ival->type = insn->type; ival->reg = IR_REG_NONE; diff --git a/ir_private.h b/ir_private.h index 374a46f..f4d92d8 100644 --- a/ir_private.h +++ b/ir_private.h @@ -221,6 +221,79 @@ IR_ALWAYS_INLINE int ir_nlzl(uint64_t num) /*** Helper data types ***/ +/* Arena */ +struct _ir_arena { + char *ptr; + char *end; + ir_arena *prev; +}; + +IR_ALWAYS_INLINE ir_arena* ir_arena_create(size_t size) +{ + ir_arena *arena; + + IR_ASSERT(size >= IR_ALIGNED_SIZE(sizeof(ir_arena), 8)); + arena = (ir_arena*)ir_mem_malloc(size); + arena->ptr = (char*) arena + IR_ALIGNED_SIZE(sizeof(ir_arena), 8); + arena->end = (char*) arena + size; + arena->prev = NULL; + return arena; +} + +IR_ALWAYS_INLINE void ir_arena_free(ir_arena *arena) +{ + do { + ir_arena *prev = arena->prev; + ir_mem_free(arena); + arena = prev; + } while (arena); +} + +IR_ALWAYS_INLINE void* ir_arena_alloc(ir_arena **arena_ptr, size_t size) +{ + ir_arena *arena = *arena_ptr; + char *ptr = arena->ptr; + + size = IR_ALIGNED_SIZE(size, 8); + + if (EXPECTED(size <= (size_t)(arena->end - ptr))) { + arena->ptr = ptr + size; + } else { + size_t arena_size = + UNEXPECTED((size + IR_ALIGNED_SIZE(sizeof(ir_arena), 8)) > (size_t)(arena->end - (char*) arena)) ? + (size + IR_ALIGNED_SIZE(sizeof(ir_arena), 8)) : + (size_t)(arena->end - (char*) arena); + ir_arena *new_arena = (ir_arena*)ir_mem_malloc(arena_size); + + ptr = (char*) new_arena + IR_ALIGNED_SIZE(sizeof(ir_arena), 8); + new_arena->ptr = (char*) new_arena + IR_ALIGNED_SIZE(sizeof(ir_arena), 8) + size; + new_arena->end = (char*) new_arena + arena_size; + new_arena->prev = arena; + *arena_ptr = new_arena; + } + + return (void*) ptr; +} + +IR_ALWAYS_INLINE void* ir_arena_checkpoint(ir_arena *arena) +{ + return arena->ptr; +} + +IR_ALWAYS_INLINE void ir_release(ir_arena **arena_ptr, void *checkpoint) +{ + ir_arena *arena = *arena_ptr; + + while (UNEXPECTED((char*)checkpoint > arena->end) || + UNEXPECTED((char*)checkpoint <= (char*)arena)) { + ir_arena *prev = arena->prev; + ir_mem_free(arena); + *arena_ptr = arena = prev; + } + IR_ASSERT((char*)checkpoint > (char*)arena && (char*)checkpoint <= arena->end); + arena->ptr = (char*)checkpoint; +} + /* Bitsets */ #if defined(IR_TARGET_X86) # define IR_BITSET_BITS 32 @@ -967,8 +1040,6 @@ struct _ir_live_interval { typedef int (*emit_copy_t)(ir_ctx *ctx, uint8_t type, ir_ref from, ir_ref to); int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy); -void ir_free_live_ranges(ir_live_range *live_range); -void ir_free_live_intervals(ir_live_interval **live_intervals, int count); #if defined(IR_REGSET_64BIT) diff --git a/ir_ra.c b/ir_ra.c index eb0ba1b..34dfb9c 100644 --- a/ir_ra.c +++ b/ir_ra.c @@ -123,7 +123,7 @@ static void ir_add_local_var(ir_ctx *ctx, int v, uint8_t type) IR_ASSERT(!ival); - ival = ir_mem_malloc(sizeof(ir_live_interval)); + ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); IR_ASSERT(type != IR_VOID); ival->type = type; ival->reg = IR_REG_NONE; @@ -148,7 +148,7 @@ static void ir_add_live_range(ir_ctx *ctx, int v, uint8_t type, ir_live_pos star ir_live_range *p, *q, *next, *prev; if (!ival) { - ival = ir_mem_malloc(sizeof(ir_live_interval)); + ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); IR_ASSERT(type != IR_VOID); ival->type = type; ival->reg = IR_REG_NONE; @@ -203,7 +203,7 @@ static void ir_add_live_range(ir_ctx *ctx, int v, uint8_t type, ir_live_pos star q = ctx->unused_ranges; ctx->unused_ranges = q->next; } else { - q = ir_mem_malloc(sizeof(ir_live_range)); + q = ir_arena_alloc(&ctx->arena, sizeof(ir_live_range)); } if (prev) { prev->next = q; @@ -227,7 +227,7 @@ static void ir_add_fixed_live_range(ir_ctx *ctx, ir_reg reg, ir_live_pos start, int v = ctx->vregs_count + 1 + reg; ir_live_interval *ival = ctx->live_intervals[v]; if (!ival) { - ival = ir_mem_malloc(sizeof(ir_live_interval)); + ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); ival->type = IR_VOID; ival->reg = reg; ival->flags = IR_LIVE_INTERVAL_FIXED; @@ -249,7 +249,7 @@ static void ir_add_fixed_live_range(ir_ctx *ctx, ir_reg reg, ir_live_pos start, static void ir_add_tmp(ir_ctx *ctx, ir_ref ref, ir_ref tmp_ref, int32_t tmp_op_num, ir_tmp_reg tmp_reg) { - ir_live_interval *ival = ir_mem_malloc(sizeof(ir_live_interval)); + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); ival->type = tmp_reg.type; ival->reg = IR_REG_NONE; @@ -336,11 +336,11 @@ static void ir_add_use_pos(ir_ctx *ctx, int v, ir_use_pos *use_pos) } } -static void ir_add_use(ir_ctx *ctx, int v, int op_num, ir_live_pos pos, ir_reg hint, uint8_t use_flags, ir_ref hint_ref) +IR_ALWAYS_INLINE void ir_add_use(ir_ctx *ctx, int v, int op_num, ir_live_pos pos, ir_reg hint, uint8_t use_flags, ir_ref hint_ref) { ir_use_pos *use_pos; - use_pos = ir_mem_malloc(sizeof(ir_use_pos)); + use_pos = ir_arena_alloc(&ctx->arena, sizeof(ir_use_pos)); use_pos->op_num = op_num; use_pos->hint = hint; use_pos->flags = use_flags; @@ -355,7 +355,7 @@ static void ir_add_phi_use(ir_ctx *ctx, int v, int op_num, ir_live_pos pos, ir_r ir_use_pos *use_pos; IR_ASSERT(phi_ref > 0); - use_pos = ir_mem_malloc(sizeof(ir_use_pos)); + use_pos = ir_arena_alloc(&ctx->arena, sizeof(ir_use_pos)); use_pos->op_num = op_num; use_pos->hint = IR_REG_NONE; use_pos->flags = IR_PHI_USE | IR_USE_SHOULD_BE_IN_REG; // TODO: ??? @@ -561,12 +561,15 @@ int ir_compute_live_ranges(ir_ctx *ctx) /* vregs + tmp + fixed + SRATCH + ALL */ ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_NUM + 2, sizeof(ir_live_interval*)); - ctx->unused_ranges = NULL; #ifdef IR_DEBUG visited = ir_bitset_malloc(ctx->cfg_blocks_count + 1); #endif + if (!ctx->arena) { + ctx->arena = ir_arena_create(16 * 1024); + } + /* for each basic block in reverse order */ for (b = ctx->cfg_blocks_count; b > 0; b--) { bb = &ctx->cfg_blocks[b]; @@ -884,11 +887,6 @@ int ir_compute_live_ranges(ir_ctx *ctx) } } - if (ctx->unused_ranges) { - ir_free_live_ranges(ctx->unused_ranges); - ctx->unused_ranges = NULL; - } - if (loops) { ir_mem_free(loops); ir_bitqueue_free(&queue); @@ -902,46 +900,6 @@ int ir_compute_live_ranges(ir_ctx *ctx) return 1; } -void ir_free_live_ranges(ir_live_range *live_range) -{ - ir_live_range *p; - - while (live_range) { - p = live_range; - live_range = live_range->next; - ir_mem_free(p); - } -} - -void ir_free_live_intervals(ir_live_interval **live_intervals, int count) -{ - int i; - ir_live_interval *ival, *next; - ir_use_pos *use_pos; - - /* vregs + tmp + fixed + SRATCH + ALL */ - count += IR_REG_NUM + 2; - for (i = 0; i <= count; i++) { - ival = live_intervals[i]; - while (ival) { - if (ival->range.next) { - ir_free_live_ranges(ival->range.next); - } - use_pos = ival->use_pos; - while (use_pos) { - ir_use_pos *p = use_pos; - use_pos = p->next; - ir_mem_free(p); - } - next = ival->next; - ir_mem_free(ival); - ival = next; - } - } - ir_mem_free(live_intervals); -} - - /* Live Ranges coalescing */ static ir_live_pos ir_ivals_overlap(ir_live_range *lrg1, ir_live_range *lrg2) @@ -1035,7 +993,8 @@ static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2) ctx->live_intervals[r1]->flags |= IR_LIVE_INTERVAL_COALESCED; ctx->live_intervals[r2] = NULL; - ir_mem_free(ival); + // TODO: remember to reuse ??? + //ir_mem_free(ival); } static bool ir_try_coalesce(ir_ctx *ctx, ir_ref from, ir_ref to) @@ -1262,7 +1221,6 @@ int ir_coalesce(ir_ctx *ctx) ir_worklist blocks; bool compact = 0; - ctx->unused_ranges = NULL; /* Collect a list of blocks which are predecossors to block with phi finctions */ ir_worklist_init(&blocks, ctx->cfg_blocks_count + 1); for (b = 1, bb = &ctx->cfg_blocks[1]; b <= ctx->cfg_blocks_count; b++, bb++) { @@ -1324,10 +1282,6 @@ int ir_coalesce(ir_ctx *ctx) } } } - if (ctx->unused_ranges) { - ir_free_live_ranges(ctx->unused_ranges); - ctx->unused_ranges = NULL; - } ir_worklist_free(&blocks); if (ctx->rules) { @@ -1760,7 +1714,7 @@ static ir_live_interval *ir_split_interval_at(ir_ctx *ctx, ir_live_interval *iva } } - child = ir_mem_malloc(sizeof(ir_live_interval)); + child = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); child->type = ival->type; child->reg = IR_REG_NONE; child->flags = 0; @@ -1779,7 +1733,9 @@ static ir_live_interval *ir_split_interval_at(ir_ctx *ctx, ir_live_interval *iva if (pos == p->start) { prev->next = NULL; ival->end = prev->end; - ir_mem_free(p); + /* Cache to reuse */ + p->next = ctx->unused_ranges; + ctx->unused_ranges = p; } else { p->end = ival->end = pos; p->next = NULL; diff --git a/ir_x86.dasc b/ir_x86.dasc index 3e4bfea..ccbb45c 100644 --- a/ir_x86.dasc +++ b/ir_x86.dasc @@ -7570,6 +7570,11 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) /* vregs + tmp + fixed + SRATCH + ALL */ ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_NUM + 2, sizeof(ir_live_interval*)); + + if (!ctx->arena) { + ctx->arena = ir_arena_create(16 * 1024); + } + for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) { IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE)); for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) { @@ -7612,7 +7617,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) } } if (!ctx->live_intervals[ctx->vregs[i]]) { - ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval)); + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); + memset(ival, 0, sizeof(ir_live_interval)); ctx->live_intervals[ctx->vregs[i]] = ival; ival->type = insn->type; ival->reg = IR_REG_NONE; @@ -7637,7 +7643,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) if (use_insn->op == IR_VLOAD) { if (ctx->vregs[use] && !ctx->live_intervals[ctx->vregs[use]]) { - ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval)); + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); + memset(ival, 0, sizeof(ir_live_interval)); ctx->live_intervals[ctx->vregs[use]] = ival; ival->type = insn->type; ival->reg = IR_REG_NONE; @@ -7649,7 +7656,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx) if (!IR_IS_CONST_REF(use_insn->op3) && ctx->vregs[use_insn->op3] && !ctx->live_intervals[ctx->vregs[use_insn->op3]]) { - ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval)); + ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval)); + memset(ival, 0, sizeof(ir_live_interval)); ctx->live_intervals[ctx->vregs[use_insn->op3]] = ival; ival->type = insn->type; ival->reg = IR_REG_NONE;