Use arena to allocate live_intervals and nested data structures

ir.c

@@ -318,6 +318,8 @@ void ir_init(ir_ctx *ctx, uint32_t flags, ir_ref consts_limit, ir_ref insns_limi
 	ctx->fixed_regset = 0;
 	ctx->fixed_save_regset = 0;
 	ctx->live_intervals = NULL;
+	ctx->arena = NULL;
+	ctx->unused_ranges = NULL;
 	ctx->regs = NULL;
 	ctx->prev_ref = NULL;
 	ctx->data = NULL;
@@ -379,7 +381,10 @@ void ir_free(ir_ctx *ctx)
 		ir_mem_free(ctx->vregs);
 	}
 	if (ctx->live_intervals) {
-		ir_free_live_intervals(ctx->live_intervals, ctx->vregs_count);
+		ir_mem_free(ctx->live_intervals);
+	}
+	if (ctx->arena) {
+		ir_arena_free(ctx->arena);
 	}
 	if (ctx->regs) {
 		ir_mem_free(ctx->regs);

ir.h

@@ -504,6 +504,7 @@ void ir_strtab_free(ir_strtab *strtab);
 typedef struct _ir_ctx           ir_ctx;
 typedef struct _ir_use_list      ir_use_list;
 typedef struct _ir_block         ir_block;
+typedef struct _ir_arena         ir_arena;
 typedef struct _ir_live_interval ir_live_interval;
 typedef struct _ir_live_range    ir_live_range;
 typedef int8_t ir_regs[4];
@@ -538,6 +539,7 @@ struct _ir_ctx {
 	int32_t            fixed_call_stack_size;   /* fixed preallocated stack for parameter passing (default 0) */
 	uint64_t           fixed_save_regset;       /* registers that always saved/restored in prologue/epilugue */
 	ir_live_interval **live_intervals;
+	ir_arena          *arena;
 	ir_live_range     *unused_ranges;
 	ir_regs           *regs;
 	ir_ref            *prev_ref;

ir_aarch64.dasc

@@ -4525,6 +4525,11 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx)
 
 	/* vregs + tmp + fixed + SRATCH + ALL */
 	ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_NUM + 2, sizeof(ir_live_interval*));
+
+    if (!ctx->arena) {
+		ctx->arena = ir_arena_create(16 * 1024);
+	}
+
 	for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) {
 		IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE));
 		for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) {
@@ -4566,7 +4571,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx)
 							}
 						}
 						if (!ctx->live_intervals[ctx->vregs[i]]) {
-							ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval));
+							ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
+							memset(ival, 0, sizeof(ir_live_interval));
 							ctx->live_intervals[ctx->vregs[i]] = ival;
 							ival->type = insn->type;
 							ival->reg = IR_REG_NONE;
@@ -4591,7 +4597,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx)
 									if (use_insn->op == IR_VLOAD) {
 										if (ctx->vregs[use]
 										 && !ctx->live_intervals[ctx->vregs[use]]) {
-											ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval));
+											ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
+											memset(ival, 0, sizeof(ir_live_interval));
 											ctx->live_intervals[ctx->vregs[use]] = ival;
 											ival->type = insn->type;
 											ival->reg = IR_REG_NONE;
@@ -4603,7 +4610,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx)
 										if (!IR_IS_CONST_REF(use_insn->op3)
 										 && ctx->vregs[use_insn->op3]
 										 && !ctx->live_intervals[ctx->vregs[use_insn->op3]]) {
-											ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval));
+											ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
+											memset(ival, 0, sizeof(ir_live_interval));
 											ctx->live_intervals[ctx->vregs[use_insn->op3]] = ival;
 											ival->type = insn->type;
 											ival->reg = IR_REG_NONE;

ir_private.h

@@ -221,6 +221,79 @@ IR_ALWAYS_INLINE int ir_nlzl(uint64_t num)
 
 /*** Helper data types ***/
 
+/* Arena */
+struct _ir_arena {
+	char     *ptr;
+	char     *end;
+	ir_arena *prev;
+};
+
+IR_ALWAYS_INLINE ir_arena* ir_arena_create(size_t size)
+{
+	ir_arena *arena;
+
+	IR_ASSERT(size >= IR_ALIGNED_SIZE(sizeof(ir_arena), 8));
+	arena = (ir_arena*)ir_mem_malloc(size);
+	arena->ptr = (char*) arena + IR_ALIGNED_SIZE(sizeof(ir_arena), 8);
+	arena->end = (char*) arena + size;
+	arena->prev = NULL;
+	return arena;
+}
+
+IR_ALWAYS_INLINE void ir_arena_free(ir_arena *arena)
+{
+	do {
+		ir_arena *prev = arena->prev;
+		ir_mem_free(arena);
+		arena = prev;
+	} while (arena);
+}
+
+IR_ALWAYS_INLINE void* ir_arena_alloc(ir_arena **arena_ptr, size_t size)
+{
+	ir_arena *arena = *arena_ptr;
+	char *ptr = arena->ptr;
+
+	size = IR_ALIGNED_SIZE(size, 8);
+
+	if (EXPECTED(size <= (size_t)(arena->end - ptr))) {
+		arena->ptr = ptr + size;
+	} else {
+		size_t arena_size =
+			UNEXPECTED((size + IR_ALIGNED_SIZE(sizeof(ir_arena), 8)) > (size_t)(arena->end - (char*) arena)) ?
+				(size + IR_ALIGNED_SIZE(sizeof(ir_arena), 8)) :
+				(size_t)(arena->end - (char*) arena);
+		ir_arena *new_arena = (ir_arena*)ir_mem_malloc(arena_size);
+
+		ptr = (char*) new_arena + IR_ALIGNED_SIZE(sizeof(ir_arena), 8);
+		new_arena->ptr = (char*) new_arena + IR_ALIGNED_SIZE(sizeof(ir_arena), 8) + size;
+		new_arena->end = (char*) new_arena + arena_size;
+		new_arena->prev = arena;
+		*arena_ptr = new_arena;
+	}
+
+	return (void*) ptr;
+}
+
+IR_ALWAYS_INLINE void* ir_arena_checkpoint(ir_arena *arena)
+{
+	return arena->ptr;
+}
+
+IR_ALWAYS_INLINE void ir_release(ir_arena **arena_ptr, void *checkpoint)
+{
+	ir_arena *arena = *arena_ptr;
+
+	while (UNEXPECTED((char*)checkpoint > arena->end) ||
+	       UNEXPECTED((char*)checkpoint <= (char*)arena)) {
+		ir_arena *prev = arena->prev;
+		ir_mem_free(arena);
+		*arena_ptr = arena = prev;
+	}
+	IR_ASSERT((char*)checkpoint > (char*)arena && (char*)checkpoint <= arena->end);
+	arena->ptr = (char*)checkpoint;
+}
+
 /* Bitsets */
 #if defined(IR_TARGET_X86)
 # define IR_BITSET_BITS 32
@@ -967,8 +1040,6 @@ struct _ir_live_interval {
 typedef int (*emit_copy_t)(ir_ctx *ctx, uint8_t type, ir_ref from, ir_ref to);
 
 int ir_gen_dessa_moves(ir_ctx *ctx, uint32_t b, emit_copy_t emit_copy);
-void ir_free_live_ranges(ir_live_range *live_range);
-void ir_free_live_intervals(ir_live_interval **live_intervals, int count);
 
 #if defined(IR_REGSET_64BIT)
 

ir_ra.c

@@ -123,7 +123,7 @@ static void ir_add_local_var(ir_ctx *ctx, int v, uint8_t type)
 
 	IR_ASSERT(!ival);
 
-	ival = ir_mem_malloc(sizeof(ir_live_interval));
+	ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
 	IR_ASSERT(type != IR_VOID);
 	ival->type = type;
 	ival->reg = IR_REG_NONE;
@@ -148,7 +148,7 @@ static void ir_add_live_range(ir_ctx *ctx, int v, uint8_t type, ir_live_pos star
 	ir_live_range *p, *q, *next, *prev;
 
 	if (!ival) {
-		ival = ir_mem_malloc(sizeof(ir_live_interval));
+		ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
 		IR_ASSERT(type != IR_VOID);
 		ival->type = type;
 		ival->reg = IR_REG_NONE;
@@ -203,7 +203,7 @@ static void ir_add_live_range(ir_ctx *ctx, int v, uint8_t type, ir_live_pos star
 		q = ctx->unused_ranges;
 		ctx->unused_ranges = q->next;
 	} else {
-		q = ir_mem_malloc(sizeof(ir_live_range));
+		q = ir_arena_alloc(&ctx->arena, sizeof(ir_live_range));
 	}
 	if (prev) {
 		prev->next = q;
@@ -227,7 +227,7 @@ static void ir_add_fixed_live_range(ir_ctx *ctx, ir_reg reg, ir_live_pos start,
 	int v = ctx->vregs_count + 1 + reg;
 	ir_live_interval *ival = ctx->live_intervals[v];
 	if (!ival) {
-		ival = ir_mem_malloc(sizeof(ir_live_interval));
+		ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
 		ival->type = IR_VOID;
 		ival->reg = reg;
 		ival->flags = IR_LIVE_INTERVAL_FIXED;
@@ -249,7 +249,7 @@ static void ir_add_fixed_live_range(ir_ctx *ctx, ir_reg reg, ir_live_pos start,
 
 static void ir_add_tmp(ir_ctx *ctx, ir_ref ref, ir_ref tmp_ref, int32_t tmp_op_num, ir_tmp_reg tmp_reg)
 {
-	ir_live_interval *ival = ir_mem_malloc(sizeof(ir_live_interval));
+	ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
 
 	ival->type = tmp_reg.type;
 	ival->reg = IR_REG_NONE;
@@ -336,11 +336,11 @@ static void ir_add_use_pos(ir_ctx *ctx, int v, ir_use_pos *use_pos)
 	}
 }
 
-static void ir_add_use(ir_ctx *ctx, int v, int op_num, ir_live_pos pos, ir_reg hint, uint8_t use_flags, ir_ref hint_ref)
+IR_ALWAYS_INLINE void ir_add_use(ir_ctx *ctx, int v, int op_num, ir_live_pos pos, ir_reg hint, uint8_t use_flags, ir_ref hint_ref)
 {
 	ir_use_pos *use_pos;
 
-	use_pos = ir_mem_malloc(sizeof(ir_use_pos));
+	use_pos = ir_arena_alloc(&ctx->arena, sizeof(ir_use_pos));
 	use_pos->op_num = op_num;
 	use_pos->hint = hint;
 	use_pos->flags = use_flags;
@@ -355,7 +355,7 @@ static void ir_add_phi_use(ir_ctx *ctx, int v, int op_num, ir_live_pos pos, ir_r
 	ir_use_pos *use_pos;
 
 	IR_ASSERT(phi_ref > 0);
-	use_pos = ir_mem_malloc(sizeof(ir_use_pos));
+	use_pos = ir_arena_alloc(&ctx->arena, sizeof(ir_use_pos));
 	use_pos->op_num = op_num;
 	use_pos->hint = IR_REG_NONE;
 	use_pos->flags = IR_PHI_USE | IR_USE_SHOULD_BE_IN_REG; // TODO: ???
@@ -561,12 +561,15 @@ int ir_compute_live_ranges(ir_ctx *ctx)
 
 	/* vregs + tmp + fixed + SRATCH + ALL */
 	ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_NUM + 2, sizeof(ir_live_interval*));
-	ctx->unused_ranges = NULL;
 
 #ifdef IR_DEBUG
 	visited = ir_bitset_malloc(ctx->cfg_blocks_count + 1);
 #endif
 
+    if (!ctx->arena) {
+		ctx->arena = ir_arena_create(16 * 1024);
+	}
+
 	/* for each basic block in reverse order */
 	for (b = ctx->cfg_blocks_count; b > 0; b--) {
 		bb = &ctx->cfg_blocks[b];
@@ -884,11 +887,6 @@ int ir_compute_live_ranges(ir_ctx *ctx)
 		}
 	}
 
-	if (ctx->unused_ranges) {
-		ir_free_live_ranges(ctx->unused_ranges);
-		ctx->unused_ranges = NULL;
-	}
-
 	if (loops) {
 		ir_mem_free(loops);
 		ir_bitqueue_free(&queue);
@@ -902,46 +900,6 @@ int ir_compute_live_ranges(ir_ctx *ctx)
 	return 1;
 }
 
-void ir_free_live_ranges(ir_live_range *live_range)
-{
-	ir_live_range *p;
-
-	while (live_range) {
-		p = live_range;
-		live_range = live_range->next;
-		ir_mem_free(p);
-	}
-}
-
-void ir_free_live_intervals(ir_live_interval **live_intervals, int count)
-{
-	int i;
-	ir_live_interval *ival, *next;
-	ir_use_pos *use_pos;
-
-	/* vregs + tmp + fixed + SRATCH + ALL */
-	count += IR_REG_NUM + 2;
-	for (i = 0; i <= count; i++) {
-		ival = live_intervals[i];
-		while (ival) {
-			if (ival->range.next) {
-				ir_free_live_ranges(ival->range.next);
-			}
-			use_pos = ival->use_pos;
-			while (use_pos) {
-				ir_use_pos *p = use_pos;
-				use_pos = p->next;
-				ir_mem_free(p);
-			}
-			next = ival->next;
-			ir_mem_free(ival);
-			ival = next;
-		}
-	}
-	ir_mem_free(live_intervals);
-}
-
-
 /* Live Ranges coalescing */
 
 static ir_live_pos ir_ivals_overlap(ir_live_range *lrg1, ir_live_range *lrg2)
@@ -1035,7 +993,8 @@ static void ir_vregs_join(ir_ctx *ctx, uint32_t r1, uint32_t r2)
 	ctx->live_intervals[r1]->flags |= IR_LIVE_INTERVAL_COALESCED;
 	ctx->live_intervals[r2] = NULL;
 
-	ir_mem_free(ival);
+	// TODO: remember to reuse ???
+	//ir_mem_free(ival);
 }
 
 static bool ir_try_coalesce(ir_ctx *ctx, ir_ref from, ir_ref to)
@@ -1262,7 +1221,6 @@ int ir_coalesce(ir_ctx *ctx)
 	ir_worklist blocks;
 	bool compact = 0;
 
-	ctx->unused_ranges = NULL;
 	/* Collect a list of blocks which are predecossors to block with phi finctions */
 	ir_worklist_init(&blocks, ctx->cfg_blocks_count + 1);
 	for (b = 1, bb = &ctx->cfg_blocks[1]; b <= ctx->cfg_blocks_count; b++, bb++) {
@@ -1324,10 +1282,6 @@ int ir_coalesce(ir_ctx *ctx)
 			}
 		}
 	}
-	if (ctx->unused_ranges) {
-		ir_free_live_ranges(ctx->unused_ranges);
-		ctx->unused_ranges = NULL;
-	}
 	ir_worklist_free(&blocks);
 
 	if (ctx->rules) {
@@ -1760,7 +1714,7 @@ static ir_live_interval *ir_split_interval_at(ir_ctx *ctx, ir_live_interval *iva
 		}
 	}
 
-	child = ir_mem_malloc(sizeof(ir_live_interval));
+	child = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
 	child->type = ival->type;
 	child->reg = IR_REG_NONE;
 	child->flags = 0;
@@ -1779,7 +1733,9 @@ static ir_live_interval *ir_split_interval_at(ir_ctx *ctx, ir_live_interval *iva
 	if (pos == p->start) {
 		prev->next = NULL;
 		ival->end = prev->end;
-		ir_mem_free(p);
+		/* Cache to reuse */
+		p->next = ctx->unused_ranges;
+		ctx->unused_ranges = p;
 	} else {
 		p->end = ival->end = pos;
 		p->next = NULL;

ir_x86.dasc

@@ -7570,6 +7570,11 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx)
 
 	/* vregs + tmp + fixed + SRATCH + ALL */
 	ctx->live_intervals = ir_mem_calloc(ctx->vregs_count + 1 + IR_REG_NUM + 2, sizeof(ir_live_interval*));
+
+    if (!ctx->arena) {
+		ctx->arena = ir_arena_create(16 * 1024);
+	}
+
 	for (b = 1, bb = ctx->cfg_blocks + b; b <= ctx->cfg_blocks_count; b++, bb++) {
 		IR_ASSERT(!(bb->flags & IR_BB_UNREACHABLE));
 		for (i = bb->start, insn = ctx->ir_base + i, rule = ctx->rules + i; i <= bb->end;) {
@@ -7612,7 +7617,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx)
 							}
 						}
 						if (!ctx->live_intervals[ctx->vregs[i]]) {
-							ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval));
+							ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
+							memset(ival, 0, sizeof(ir_live_interval));
 							ctx->live_intervals[ctx->vregs[i]] = ival;
 							ival->type = insn->type;
 							ival->reg = IR_REG_NONE;
@@ -7637,7 +7643,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx)
 									if (use_insn->op == IR_VLOAD) {
 										if (ctx->vregs[use]
 										 && !ctx->live_intervals[ctx->vregs[use]]) {
-											ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval));
+											ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
+											memset(ival, 0, sizeof(ir_live_interval));
 											ctx->live_intervals[ctx->vregs[use]] = ival;
 											ival->type = insn->type;
 											ival->reg = IR_REG_NONE;
@@ -7649,7 +7656,8 @@ static void ir_allocate_unique_spill_slots(ir_ctx *ctx)
 										if (!IR_IS_CONST_REF(use_insn->op3)
 										 && ctx->vregs[use_insn->op3]
 										 && !ctx->live_intervals[ctx->vregs[use_insn->op3]]) {
-											ir_live_interval *ival = ir_mem_calloc(1, sizeof(ir_live_interval));
+											ir_live_interval *ival = ir_arena_alloc(&ctx->arena, sizeof(ir_live_interval));
+											memset(ival, 0, sizeof(ir_live_interval));
 											ctx->live_intervals[ctx->vregs[use_insn->op3]] = ival;
 											ival->type = insn->type;
 											ival->reg = IR_REG_NONE;