From 3ebb25d3e744989261b9b501926326106e376a73 Mon Sep 17 00:00:00 2001 From: Yen-Fu Chen Date: Thu, 28 Dec 2023 12:35:08 +0800 Subject: [PATCH 1/2] Remove predicted block Because we already have block chaining, there is no need for predicted block. --- src/emulate.c | 21 ++++----------------- src/riscv_private.h | 1 - 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/src/emulate.c b/src/emulate.c index c79feca9..0b5fb70c 100644 --- a/src/emulate.c +++ b/src/emulate.c @@ -301,7 +301,6 @@ static block_t *block_alloc(riscv_t *rv) block_t *block = mpool_alloc(rv->block_mp); assert(block); block->n_insn = 0; - block->predict = NULL; #if RV32_HAS(JIT) block->translatable = true; block->hot = false; @@ -990,13 +989,6 @@ static block_t *block_find_or_translate(riscv_t *rv) mpool_free(rv->block_mp, delete_target); } #endif - /* update the block prediction. - * When translating a new block, the block predictor may benefit, - * but updating it after finding a particular block may penalize - * significantly. - */ - if (prev) - prev->predict = next; } return next; @@ -1016,15 +1008,10 @@ void rv_step(riscv_t *rv, int32_t cycles) /* loop until hitting the cycle target */ while (rv->csr_cycle < cycles_target && !rv->halt) { block_t *block; - /* try to predict the next block */ - if (prev && prev->predict && prev->predict->pc_start == rv->PC) { - block = prev->predict; - } else { - /* lookup the next block in block map or translate a new block, - * and move onto the next block. - */ - block = block_find_or_translate(rv); - } + /* lookup the next block in block map or translate a new block, + * and move onto the next block. + */ + block = block_find_or_translate(rv); /* by now, a block should be available */ assert(block); diff --git a/src/riscv_private.h b/src/riscv_private.h index 558c9253..ed602030 100644 --- a/src/riscv_private.h +++ b/src/riscv_private.h @@ -59,7 +59,6 @@ enum { typedef struct block { uint32_t n_insn; /**< number of instructions encompased */ uint32_t pc_start, pc_end; /**< address range of the basic block */ - struct block *predict; /**< block prediction */ rv_insn_t *ir_head, *ir_tail; /**< the first and last ir for this block */ bool backward; From e266b69ee5d60f28e14b8eadd94d4192d8861ee2 Mon Sep 17 00:00:00 2001 From: Yen-Fu Chen Date: Sat, 20 Jan 2024 23:58:18 +0800 Subject: [PATCH 2/2] Enforce doubly-linked chained block Previously, the chained block was a linear structure where previous block only pointed to the next block. Now, we have introduced additional information, allowing the next block to also reference the previous block. This modification enhance the deletion of replaced block. Close: #329 --- src/cache.c | 62 --------------------------- src/emulate.c | 100 +++++++++++++++++++++++++++++++++++--------- src/jit.c | 12 +++--- src/riscv.c | 4 ++ src/riscv_private.h | 10 +++++ src/rv32_template.c | 37 ---------------- src/utils.h | 62 +++++++++++++++++++++++++++ 7 files changed, 162 insertions(+), 125 deletions(-) diff --git a/src/cache.c b/src/cache.c index 92a1d33a..f70f5d8a 100644 --- a/src/cache.c +++ b/src/cache.c @@ -27,10 +27,6 @@ static struct mpool *cache_mp; /* hash function for the cache */ HASH_FUNC_IMPL(cache_hash, cache_size_bits, cache_size) -struct list_head { - struct list_head *prev, *next; -}; - struct hlist_head { struct hlist_node *first; }; @@ -58,64 +54,6 @@ typedef struct cache { uint32_t capacity; } cache_t; -static inline void INIT_LIST_HEAD(struct list_head *head) -{ - head->next = head; - head->prev = head; -} - -static inline int list_empty(const struct list_head *head) -{ - return (head->next == head); -} - -static inline void list_add(struct list_head *node, struct list_head *head) -{ - struct list_head *next = head->next; - - next->prev = node; - node->next = next; - node->prev = head; - head->next = node; -} - -static inline void list_del(struct list_head *node) -{ - struct list_head *next = node->next; - struct list_head *prev = node->prev; - - next->prev = prev; - prev->next = next; -} - -static inline void list_del_init(struct list_head *node) -{ - list_del(node); - INIT_LIST_HEAD(node); -} - -#define list_entry(node, type, member) container_of(node, type, member) - -#define list_first_entry(head, type, member) \ - list_entry((head)->next, type, member) - -#define list_last_entry(head, type, member) \ - list_entry((head)->prev, type, member) - -#ifdef __HAVE_TYPEOF -#define list_for_each_entry_safe(entry, safe, head, member) \ - for (entry = list_entry((head)->next, __typeof__(*entry), member), \ - safe = list_entry(entry->member.next, __typeof__(*entry), member); \ - &entry->member != (head); entry = safe, \ - safe = list_entry(safe->member.next, __typeof__(*entry), member)) -#else -#define list_for_each_entry_safe(entry, safe, head, member, type) \ - for (entry = list_entry((head)->next, type, member), \ - safe = list_entry(entry->member.next, type, member); \ - &entry->member != (head); \ - entry = safe, safe = list_entry(safe->member.next, type, member)) -#endif - #define INIT_HLIST_HEAD(ptr) ((ptr)->first = NULL) static inline void INIT_HLIST_NODE(struct hlist_node *h) diff --git a/src/emulate.c b/src/emulate.c index 0b5fb70c..d33a2a0a 100644 --- a/src/emulate.c +++ b/src/emulate.c @@ -305,6 +305,7 @@ static block_t *block_alloc(riscv_t *rv) block->translatable = true; block->hot = false; block->backward = false; + INIT_LIST_HEAD(&block->list); #endif return block; } @@ -978,6 +979,49 @@ static block_t *block_find_or_translate(riscv_t *rv) /* insert the block into block cache */ block_t *delete_target = cache_put(rv->block_cache, rv->PC, &(*next)); if (delete_target) { + if (prev == delete_target) + prev = NULL; + chain_entry_t *entry, *safe; + /* correctly remove deleted block from its chained block */ + rv_insn_t *taken = delete_target->ir_tail->branch_taken, + *untaken = delete_target->ir_tail->branch_untaken; + if (taken && taken->pc != delete_target->pc_start) { + block_t *target = cache_get(rv->block_cache, taken->pc); + bool flag = false; + list_for_each_entry_safe (entry, safe, &target->list, list) { + if (entry->block == delete_target) { + list_del_init(&entry->list); + mpool_free(rv->chain_entry_mp, entry); + flag = true; + } + } + assert(flag); + } + if (untaken && untaken->pc != delete_target->pc_start) { + block_t *target = cache_get(rv->block_cache, untaken->pc); + assert(target); + bool flag = false; + list_for_each_entry_safe (entry, safe, &target->list, list) { + if (entry->block == delete_target) { + list_del_init(&entry->list); + mpool_free(rv->chain_entry_mp, entry); + flag = true; + } + } + assert(flag); + } + /* correctly remove deleted block from the block chained to it */ + list_for_each_entry_safe (entry, safe, &delete_target->list, list) { + if (entry->block == delete_target) + continue; + rv_insn_t *target = entry->block->ir_tail; + if (target->branch_taken == delete_target->ir_head) + target->branch_taken = NULL; + else if (target->branch_untaken == delete_target->ir_head) + target->branch_untaken = NULL; + mpool_free(rv->chain_entry_mp, entry); + } + /* free deleted block */ uint32_t idx; rv_insn_t *ir, *next; for (idx = 0, ir = delete_target->ir_head; @@ -1007,12 +1051,18 @@ void rv_step(riscv_t *rv, int32_t cycles) /* loop until hitting the cycle target */ while (rv->csr_cycle < cycles_target && !rv->halt) { - block_t *block; + if (prev && prev->pc_start != last_pc) { + /* update previous block */ +#if !RV32_HAS(JIT) + prev = block_find(&rv->block_map, last_pc); +#else + prev = cache_get(rv->block_cache, last_pc); +#endif + } /* lookup the next block in block map or translate a new block, * and move onto the next block. */ - block = block_find_or_translate(rv); - + block_t *block = block_find_or_translate(rv); /* by now, a block should be available */ assert(block); @@ -1021,28 +1071,38 @@ void rv_step(riscv_t *rv, int32_t cycles) * assigned to either the branch_taken or branch_untaken pointer of * the previous block. */ + if (prev) { - /* update previous block */ - if (prev->pc_start != last_pc) -#if !RV32_HAS(JIT) - prev = block_find(&rv->block_map, last_pc); -#else - prev = cache_get(rv->block_cache, last_pc); + rv_insn_t *last_ir = prev->ir_tail; + /* chain block */ + if (!insn_is_unconditional_branch(last_ir->opcode)) { + if (is_branch_taken && !last_ir->branch_taken) { + last_ir->branch_taken = block->ir_head; +#if RV32_HAS(JIT) + chain_entry_t *new_entry = mpool_alloc(rv->chain_entry_mp); + new_entry->block = prev; + list_add(&new_entry->list, &block->list); #endif - if (prev) { - rv_insn_t *last_ir = prev->ir_tail; - /* chain block */ - if (!insn_is_unconditional_branch(last_ir->opcode)) { - if (is_branch_taken) - last_ir->branch_taken = block->ir_head; - else if (!is_branch_taken) - last_ir->branch_untaken = block->ir_head; - } else if (IF_insn(last_ir, jal) + } else if (!is_branch_taken && !last_ir->branch_untaken) { + last_ir->branch_untaken = block->ir_head; +#if RV32_HAS(JIT) + chain_entry_t *new_entry = mpool_alloc(rv->chain_entry_mp); + new_entry->block = prev; + list_add(&new_entry->list, &block->list); +#endif + } + } else if (IF_insn(last_ir, jal) #if RV32_HAS(EXT_C) - || IF_insn(last_ir, cj) || IF_insn(last_ir, cjal) + || IF_insn(last_ir, cj) || IF_insn(last_ir, cjal) #endif - ) { + ) { + if (!last_ir->branch_taken) { last_ir->branch_taken = block->ir_head; +#if RV32_HAS(JIT) + chain_entry_t *new_entry = mpool_alloc(rv->chain_entry_mp); + new_entry->block = prev; + list_add(&new_entry->list, &block->list); +#endif } } } diff --git a/src/jit.c b/src/jit.c index bc0b06e9..ba7af608 100644 --- a/src/jit.c +++ b/src/jit.c @@ -1496,14 +1496,14 @@ static void translate_chained_block(struct jit_state *state, offset_map_insert(state, block->pc_start); translate(state, rv, block); rv_insn_t *ir = block->ir_tail; - if (ir->branch_untaken && !set_has(set, ir->pc + 4)) { - block_t *block1 = cache_get(rv->block_cache, ir->pc + 4); - if (block1 && block1->translatable) + if (ir->branch_untaken && !set_has(set, ir->branch_untaken->pc)) { + block_t *block1 = cache_get(rv->block_cache, ir->branch_untaken->pc); + if (block1->translatable) translate_chained_block(state, rv, block1, set); } - if (ir->branch_taken && !set_has(set, ir->pc + ir->imm)) { - block_t *block1 = cache_get(rv->block_cache, ir->pc + ir->imm); - if (block1 && block1->translatable) + if (ir->branch_taken && !set_has(set, ir->branch_taken->pc)) { + block_t *block1 = cache_get(rv->block_cache, ir->branch_taken->pc); + if (block1->translatable) translate_chained_block(state, rv, block1, set); } } diff --git a/src/riscv.c b/src/riscv.c index e2b95218..91261e5e 100644 --- a/src/riscv.c +++ b/src/riscv.c @@ -134,6 +134,9 @@ riscv_t *rv_create(const riscv_io_t *io, /* initialize the block map */ block_map_init(&rv->block_map, BLOCK_MAP_CAPACITY_BITS); #else + rv->chain_entry_mp = + mpool_create(sizeof(chain_entry_t) << BLOCK_IR_MAP_CAPACITY_BITS, + sizeof(chain_entry_t)); rv->jit_state = jit_state_init(CODE_CACHE_SIZE); rv->block_cache = cache_create(BLOCK_MAP_CAPACITY_BITS); assert(rv->block_cache); @@ -165,6 +168,7 @@ void rv_delete(riscv_t *rv) #if !RV32_HAS(JIT) block_map_destroy(rv); #else + mpool_destroy(rv->chain_entry_mp); jit_state_exit(rv->jit_state); cache_free(rv->block_cache); #endif diff --git a/src/riscv_private.h b/src/riscv_private.h index ed602030..14babc25 100644 --- a/src/riscv_private.h +++ b/src/riscv_private.h @@ -12,6 +12,7 @@ #endif #include "decode.h" #include "riscv.h" +#include "utils.h" #if RV32_HAS(JIT) #include "cache.h" #endif @@ -67,9 +68,17 @@ typedef struct block { uint32_t offset; bool translatable; /**< Determine the block has RV32AF insturctions or not */ + struct list_head list; #endif } block_t; +#if RV32_HAS(JIT) +typedef struct { + block_t *block; + struct list_head list; +} chain_entry_t; +#endif + typedef struct { uint32_t block_capacity; /**< max number of entries in the block map */ uint32_t size; /**< number of entries currently in the map */ @@ -116,6 +125,7 @@ struct riscv_internal { block_map_t block_map; /**< basic block map */ #else struct cache *block_cache; + struct mpool *chain_entry_mp; #endif struct mpool *block_mp, *block_ir_mp; /* print exit code on syscall_exit */ diff --git a/src/rv32_template.c b/src/rv32_template.c index 8f7c7137..db70aa1c 100644 --- a/src/rv32_template.c +++ b/src/rv32_template.c @@ -148,11 +148,6 @@ RVOP( struct rv_insn *taken = ir->branch_taken; if (taken) { #if RV32_HAS(JIT) - block_t *block = cache_get(rv->block_cache, PC); - if (!block) { - ir->branch_taken = NULL; - goto end_insn; - } if (cache_hot(rv->block_cache, PC)) goto end_insn; #endif @@ -1863,11 +1858,6 @@ RVOP( struct rv_insn *taken = ir->branch_taken; if (taken) { #if RV32_HAS(JIT) - block_t *block = cache_get(rv->block_cache, PC); - if (!block) { - ir->branch_taken = NULL; - goto end_insn; - } if (cache_hot(rv->block_cache, PC)) goto end_insn; #endif @@ -2028,11 +2018,6 @@ RVOP( struct rv_insn *taken = ir->branch_taken; if (taken) { #if RV32_HAS(JIT) - block_t *block = cache_get(rv->block_cache, PC); - if (!block) { - ir->branch_taken = NULL; - goto end_insn; - } if (cache_hot(rv->block_cache, PC)) goto end_insn; #endif @@ -2065,12 +2050,6 @@ RVOP( if (!untaken) goto nextop; #if RV32_HAS(JIT) - block_t *block = cache_get(rv->block_cache, PC + 2); - if (!block) { - ir->branch_untaken = NULL; - goto nextop; - } - untaken = ir->branch_untaken = block->ir_head; if (cache_hot(rv->block_cache, PC + 2)) goto nextop; #endif @@ -2083,11 +2062,6 @@ RVOP( struct rv_insn *taken = ir->branch_taken; if (taken) { #if RV32_HAS(JIT) - block_t *block = cache_get(rv->block_cache, PC); - if (!block) { - ir->branch_taken = NULL; - goto end_insn; - } if (cache_hot(rv->block_cache, PC)) goto end_insn; #endif @@ -2129,12 +2103,6 @@ RVOP( if (!untaken) goto nextop; #if RV32_HAS(JIT) - block_t *block = cache_get(rv->block_cache, PC + 2); - if (!block) { - ir->branch_untaken = NULL; - goto nextop; - } - untaken = ir->branch_untaken = block->ir_head; if (cache_hot(rv->block_cache, PC + 2)) goto nextop; #endif @@ -2147,11 +2115,6 @@ RVOP( struct rv_insn *taken = ir->branch_taken; if (taken) { #if RV32_HAS(JIT) - block_t *block = cache_get(rv->block_cache, PC); - if (!block) { - ir->branch_taken = NULL; - goto end_insn; - } if (cache_hot(rv->block_cache, PC)) goto end_insn; #endif diff --git a/src/utils.h b/src/utils.h index eb567cd3..c8a533d4 100644 --- a/src/utils.h +++ b/src/utils.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -56,3 +57,64 @@ static inline uintptr_t align_up(uintptr_t sz, size_t alignment) return ((sz + mask) & ~mask); return (((sz + mask) / alignment) * alignment); } + +struct list_head { + struct list_head *prev, *next; +}; + +static inline void INIT_LIST_HEAD(struct list_head *head) +{ + head->next = head->prev = head; +} + +static inline bool list_empty(const struct list_head *head) +{ + return (head->next == head); +} + +static inline void list_add(struct list_head *node, struct list_head *head) +{ + struct list_head *next = head->next; + + next->prev = node; + node->next = next; + node->prev = head; + head->next = node; +} + +static inline void list_del(struct list_head *node) +{ + struct list_head *next = node->next; + struct list_head *prev = node->prev; + + next->prev = prev; + prev->next = next; +} + +static inline void list_del_init(struct list_head *node) +{ + list_del(node); + INIT_LIST_HEAD(node); +} + +#define list_entry(node, type, member) container_of(node, type, member) + +#define list_first_entry(head, type, member) \ + list_entry((head)->next, type, member) + +#define list_last_entry(head, type, member) \ + list_entry((head)->prev, type, member) + +#ifdef __HAVE_TYPEOF +#define list_for_each_entry_safe(entry, safe, head, member) \ + for (entry = list_entry((head)->next, __typeof__(*entry), member), \ + safe = list_entry(entry->member.next, __typeof__(*entry), member); \ + &entry->member != (head); entry = safe, \ + safe = list_entry(safe->member.next, __typeof__(*entry), member)) +#else +#define list_for_each_entry_safe(entry, safe, head, member, type) \ + for (entry = list_entry((head)->next, type, member), \ + safe = list_entry(entry->member.next, type, member); \ + &entry->member != (head); \ + entry = safe, safe = list_entry(safe->member.next, type, member)) +#endif