diff options
author | Andy Wingo <wingo@pobox.com> | 2019-04-26 12:41:47 +0200 |
---|---|---|
committer | Andy Wingo <wingo@pobox.com> | 2019-04-26 12:41:47 +0200 |
commit | 5b45f04cc8410d0308cdef02d9ed0be383281610 (patch) | |
tree | 550c40fd6a71681764c11a82f91a97d57fadb1b9 | |
parent | 4e5329676bd24c4747cb1d0ca53b1dc3ec9d5641 (diff) | |
parent | 04d89a7ce27ba60bc2735e2f563d88c28fff2208 (diff) | |
download | guile-5b45f04cc8410d0308cdef02d9ed0be383281610.tar.gz |
Merge from upstream Lightening
-rw-r--r-- | libguile/lightening/lightening.h | 4 | ||||
-rw-r--r-- | libguile/lightening/lightening/lightening.c | 555 | ||||
-rw-r--r-- | libguile/lightening/lightening/x86-cpu.c | 116 | ||||
-rw-r--r-- | libguile/lightening/lightening/x86.c | 552 |
4 files changed, 610 insertions, 617 deletions
diff --git a/libguile/lightening/lightening.h b/libguile/lightening/lightening.h index d5a96fa8b..437ba1872 100644 --- a/libguile/lightening/lightening.h +++ b/libguile/lightening/lightening.h @@ -78,7 +78,6 @@ typedef struct jit_reloc { uint8_t kind; uint8_t inst_start_offset; - uint16_t flags; uint32_t offset; } jit_reloc_t; @@ -226,6 +225,9 @@ JIT_API void jit_patch_there(jit_state_t*, jit_reloc_t, jit_pointer_t); JIT_API void jit_move_operands (jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src, size_t argc); +JIT_API size_t jit_align_stack (jit_state_t *_jit, size_t expand); +JIT_API void jit_shrink_stack (jit_state_t *_jit, size_t diff); + /* Note that all functions that take jit_operand_t args[] use the args as scratch space while shuffling values into position. */ JIT_API void jit_calli(jit_state_t *, jit_pointer_t f, diff --git a/libguile/lightening/lightening/lightening.c b/libguile/lightening/lightening/lightening.c index 5a98eb43c..faba86bdb 100644 --- a/libguile/lightening/lightening/lightening.c +++ b/libguile/lightening/lightening/lightening.c @@ -119,15 +119,11 @@ struct jit_state uint8_t temp_gpr_saved; uint8_t temp_fpr_saved; uint8_t overflow; + int frame_size; // Used to know when to align stack. void* (*alloc)(size_t); void (*free)(void*); }; -enum jit_reloc_flags -{ - JIT_RELOC_CAN_SHORTEN = 1<<0 -}; - #define ASSERT(x) do { if (!(x)) abort(); } while (0) #if defined(__GNUC__) # define UNLIKELY(exprn) __builtin_expect(exprn, 0) @@ -138,7 +134,8 @@ enum jit_reloc_flags static jit_bool_t jit_get_cpu(void); static jit_bool_t jit_init(jit_state_t *); static void jit_flush(void *fptr, void *tptr); -static void jit_try_shorten(jit_state_t *_jit, jit_reloc_t reloc); +static void jit_try_shorten(jit_state_t *_jit, jit_reloc_t reloc, + jit_pointer_t addr); jit_bool_t init_jit(void) @@ -189,6 +186,7 @@ jit_begin(jit_state_t *_jit, uint8_t* buf, size_t length) _jit->pc.uc = _jit->start = buf; _jit->limit = buf + length; _jit->overflow = 0; + _jit->frame_size = 0; } jit_bool_t @@ -204,6 +202,7 @@ jit_reset(jit_state_t *_jit) ASSERT (_jit->start); _jit->pc.uc = _jit->start = _jit->limit = NULL; _jit->overflow = 0; + _jit->frame_size = 0; } void* @@ -225,6 +224,7 @@ jit_end(jit_state_t *_jit, size_t *length) _jit->pc.uc = _jit->start = _jit->limit = NULL; _jit->overflow = 0; + _jit->frame_size = 0; return code; } @@ -279,13 +279,12 @@ static inline void emit_u64(jit_state_t *_jit, uint64_t u64) { static inline jit_reloc_t jit_reloc (jit_state_t *_jit, enum jit_reloc_kind kind, - uint8_t inst_start_offset, uint16_t flags) + uint8_t inst_start_offset) { jit_reloc_t ret; ret.kind = kind; ret.inst_start_offset = inst_start_offset; - ret.flags = 0; ret.offset = _jit->pc.uc - _jit->start; switch (kind) @@ -327,6 +326,7 @@ jit_patch_there(jit_state_t* _jit, jit_reloc_t reloc, jit_pointer_t addr) if (_jit->overflow) return; union jit_pc loc; + uint8_t *end; loc.uc = _jit->start + reloc.offset; ptrdiff_t diff; @@ -337,38 +337,37 @@ jit_patch_there(jit_state_t* _jit, jit_reloc_t reloc, jit_pointer_t addr) *loc.ui = (uintptr_t)addr; else *loc.ul = (uintptr_t)addr; - if (loc.uc + sizeof(diff) == _jit->pc.uc && - (reloc.flags & JIT_RELOC_CAN_SHORTEN)) - jit_try_shorten (_jit, reloc); + end = loc.uc + sizeof(diff); break; case JIT_RELOC_REL8: diff = ((uint8_t*)addr) - (loc.uc + 1); ASSERT (INT8_MIN <= diff && diff <= INT8_MAX); *loc.uc = diff; + end = loc.uc + 1; break; case JIT_RELOC_REL16: diff = ((uint8_t*)addr) - (loc.uc + 2); ASSERT (INT16_MIN <= diff && diff <= INT16_MAX); *loc.us = diff; - if ((loc.uc + 1) == _jit->pc.uc && (reloc.flags & JIT_RELOC_CAN_SHORTEN)) - jit_try_shorten (_jit, reloc); + end = loc.uc + 2; break; case JIT_RELOC_REL32: diff = ((uint8_t*)addr) - (loc.uc + 4); ASSERT (INT32_MIN <= diff && diff <= INT32_MAX); *loc.ui = diff; - if ((loc.ui + 1) == _jit->pc.ui && (reloc.flags & JIT_RELOC_CAN_SHORTEN)) - jit_try_shorten (_jit, reloc); + end = loc.uc + 4; break; case JIT_RELOC_REL64: diff = ((uint8_t*)addr) - (loc.uc + 8); *loc.ul = diff; - if ((loc.ul + 1) == _jit->pc.ul && (reloc.flags & JIT_RELOC_CAN_SHORTEN)) - jit_try_shorten (_jit, reloc); + end = loc.uc + 8; break; default: abort (); } + + if (end == _jit->pc.uc) + jit_try_shorten (_jit, reloc, addr); } #if defined(__i386__) || defined(__x86_64__) @@ -467,3 +466,525 @@ jit_patch_there(jit_state_t* _jit, jit_reloc_t reloc, jit_pointer_t addr) #define IMPL_INSTRUCTION(kind, stem) JIT_IMPL_##kind(stem) FOR_EACH_INSTRUCTION(IMPL_INSTRUCTION) #undef IMPL_INSTRUCTION + +static void +abi_imm_to_gpr(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t dst, + intptr_t imm) +{ + switch (abi) { + case JIT_OPERAND_ABI_UINT8: + ASSERT(0 <= imm); + ASSERT(imm <= UINT8_MAX); + break; + case JIT_OPERAND_ABI_INT8: + ASSERT(INT8_MIN <= imm); + ASSERT(imm <= INT8_MAX); + break; + case JIT_OPERAND_ABI_UINT16: + ASSERT(0 <= imm); + ASSERT(imm <= UINT16_MAX); + break; + case JIT_OPERAND_ABI_INT16: + ASSERT(INT16_MIN <= imm); + ASSERT(imm <= INT16_MAX); + break; + case JIT_OPERAND_ABI_UINT32: + ASSERT(0 <= imm); + ASSERT(imm <= UINT32_MAX); + break; + case JIT_OPERAND_ABI_INT32: + ASSERT(INT32_MIN <= imm); + ASSERT(imm <= INT32_MAX); + break; +#if __WORDSIZE > 32 + case JIT_OPERAND_ABI_UINT64: + case JIT_OPERAND_ABI_INT64: + break; +#endif + case JIT_OPERAND_ABI_POINTER: + break; + default: + abort(); + } + jit_movi (_jit, dst, imm); +} + +static void +abi_gpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, + jit_gpr_t base, ptrdiff_t offset, jit_gpr_t src) +{ + switch (abi) { + case JIT_OPERAND_ABI_UINT8: + case JIT_OPERAND_ABI_INT8: + jit_stxi_c(_jit, offset, base, src); + break; + case JIT_OPERAND_ABI_UINT16: + case JIT_OPERAND_ABI_INT16: + jit_stxi_s(_jit, offset, base, src); + break; + case JIT_OPERAND_ABI_UINT32: + case JIT_OPERAND_ABI_INT32: +#if __WORDSIZE == 32 + case JIT_OPERAND_ABI_POINTER: +#endif + jit_stxi_i(_jit, offset, base, src); + break; +#if __WORDSIZE == 64 + case JIT_OPERAND_ABI_UINT64: + case JIT_OPERAND_ABI_INT64: + case JIT_OPERAND_ABI_POINTER: + jit_stxi_l(_jit, offset, base, src); + break; +#endif + default: + abort(); + } +} + +static void +abi_fpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, + jit_gpr_t base, ptrdiff_t offset, jit_fpr_t src) +{ + switch (abi) { + case JIT_OPERAND_ABI_FLOAT: + jit_stxi_f(_jit, offset, base, src); + break; + case JIT_OPERAND_ABI_DOUBLE: + jit_stxi_d(_jit, offset, base, src); + break; + default: + abort(); + } +} + +static void +abi_mem_to_gpr(jit_state_t *_jit, enum jit_operand_abi abi, + jit_gpr_t dst, jit_gpr_t base, ptrdiff_t offset) +{ + switch (abi) { + case JIT_OPERAND_ABI_UINT8: + jit_ldxi_uc(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_INT8: + jit_ldxi_c(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_UINT16: + jit_ldxi_us(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_INT16: + jit_ldxi_s(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_UINT32: + jit_ldxi_ui(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_INT32: + jit_ldxi_i(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_UINT64: + jit_ldxi_l(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_INT64: + jit_ldxi_l(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_POINTER: + jit_ldxi_l(_jit, dst, base, offset); + break; + default: + abort(); + } +} + +static void +abi_mem_to_fpr(jit_state_t *_jit, enum jit_operand_abi abi, + jit_fpr_t dst, jit_gpr_t base, ptrdiff_t offset) +{ + switch (abi) { + case JIT_OPERAND_ABI_FLOAT: + jit_ldxi_f(_jit, dst, base, offset); + break; + case JIT_OPERAND_ABI_DOUBLE: + jit_ldxi_d(_jit, dst, base, offset); + break; + default: + abort(); + } +} + +static void +abi_imm_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t base, + ptrdiff_t offset, intmax_t imm) +{ + ASSERT(!is_fpr_arg(abi)); + + jit_gpr_t tmp = get_temp_gpr(_jit); + abi_imm_to_gpr(_jit, abi, tmp, imm); + abi_gpr_to_mem(_jit, abi, base, offset, tmp); + unget_temp_gpr(_jit); +} + +static void +abi_mem_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t base, + ptrdiff_t offset, jit_gpr_t src_base, ptrdiff_t src_offset) +{ + if (is_gpr_arg (abi)) { + jit_gpr_t tmp = get_temp_gpr(_jit); + abi_mem_to_gpr(_jit, abi, tmp, src_base, src_offset); + abi_gpr_to_mem(_jit, abi, base, offset, tmp); + unget_temp_gpr(_jit); + } else { + jit_fpr_t tmp = get_temp_xpr(_jit); + abi_mem_to_fpr(_jit, abi, tmp, src_base, src_offset); + abi_fpr_to_mem(_jit, abi, base, offset, tmp); + unget_temp_xpr(_jit); + } +} + +#define MOVE_KIND(a, b) ((((int) a) << 4) | ((int) b)) + +#define MOVE_KIND_ENUM(a, b) \ + MOVE_##a##_TO_##b = MOVE_KIND(JIT_OPERAND_KIND_##a, JIT_OPERAND_KIND_##b) +enum move_kind { + MOVE_KIND_ENUM(IMM, GPR), + MOVE_KIND_ENUM(GPR, GPR), + MOVE_KIND_ENUM(MEM, GPR), + MOVE_KIND_ENUM(FPR, FPR), + MOVE_KIND_ENUM(MEM, FPR), + MOVE_KIND_ENUM(IMM, MEM), + MOVE_KIND_ENUM(GPR, MEM), + MOVE_KIND_ENUM(FPR, MEM), + MOVE_KIND_ENUM(MEM, MEM) +}; +#undef MOVE_KIND_ENUM + +static void +move_operand(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src) +{ + switch (MOVE_KIND (src.kind, dst.kind)) { + case MOVE_IMM_TO_GPR: + return abi_imm_to_gpr(_jit, src.abi, dst.loc.gpr.gpr, src.loc.imm); + + case MOVE_GPR_TO_GPR: + return jit_movr(_jit, dst.loc.gpr.gpr, src.loc.gpr.gpr); + + case MOVE_MEM_TO_GPR: + return abi_mem_to_gpr(_jit, src.abi, dst.loc.gpr.gpr, src.loc.mem.base, + src.loc.mem.offset); + + case MOVE_FPR_TO_FPR: + return jit_movr_d(_jit, dst.loc.fpr, src.loc.fpr); + + case MOVE_MEM_TO_FPR: + return abi_mem_to_fpr(_jit, src.abi, dst.loc.fpr, src.loc.mem.base, + src.loc.mem.offset); + + case MOVE_IMM_TO_MEM: + return abi_imm_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, + src.loc.imm); + + case MOVE_GPR_TO_MEM: + return abi_gpr_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, + src.loc.gpr.gpr); + + case MOVE_FPR_TO_MEM: + return abi_fpr_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, + src.loc.fpr); + + case MOVE_MEM_TO_MEM: + return abi_mem_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, + src.loc.mem.base, src.loc.mem.offset); + + default: + abort(); + } +} + +// A direct transliteration of "Tilting at windmills with Coq: formal +// verification of a compilation algorithm for parallel moves" by +// Laurence Rideau, Bernard Paul Serpette, and Xavier Leroy: +// https://xavierleroy.org/publi/parallel-move.pdf + +enum move_status { TO_MOVE, BEING_MOVED, MOVED }; + +static inline int +already_in_place(jit_operand_t src, jit_operand_t dst) +{ + switch (MOVE_KIND(src.kind, dst.kind)) { + case MOVE_GPR_TO_GPR: + return jit_same_gprs (src.loc.gpr.gpr, dst.loc.gpr.gpr); + case MOVE_FPR_TO_FPR: + return jit_same_fprs (src.loc.fpr, dst.loc.fpr); + case MOVE_MEM_TO_MEM: + return jit_same_gprs (src.loc.mem.base, dst.loc.mem.base) && + src.loc.mem.offset == dst.loc.mem.offset; + default: + return 0; + } +} + +static inline int +write_would_clobber(jit_operand_t src, jit_operand_t dst) +{ + if (already_in_place (src, dst)) + return 1; + + if (MOVE_KIND(src.kind, dst.kind) == MOVE_MEM_TO_GPR) + return jit_same_gprs(src.loc.mem.base, dst.loc.gpr.gpr); + + return 0; +} + +static inline ptrdiff_t +operand_addend(jit_operand_t op) +{ + switch (op.kind) { + case JIT_OPERAND_KIND_GPR: + return op.loc.gpr.addend; + case JIT_OPERAND_KIND_MEM: + return op.loc.mem.addend; + default: + abort(); + } +} + +static void +move_one(jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src, + size_t argc, enum move_status *status, size_t i) +{ + int tmp_gpr = 0, tmp_fpr = 0; + + if (already_in_place(src[i], dst[i])) + return; + + status[i] = BEING_MOVED; + for (size_t j = 0; j < argc; j++) { + if (write_would_clobber(src[j], dst[i])) { + switch (status[j]) { + case TO_MOVE: + move_one(_jit, dst, src, argc, status, j); + break; + case BEING_MOVED: { + jit_operand_t tmp; + if (is_fpr_arg (src[j].kind)) { + tmp_fpr = 1; + tmp = jit_operand_fpr(src[j].abi, get_temp_xpr(_jit)); + } else { + tmp_gpr = 1; + /* Preserve addend, if any, from source operand, to be applied + at the end. */ + tmp = jit_operand_gpr_with_addend(src[j].abi, get_temp_gpr(_jit), + operand_addend(src[j])); + } + move_operand (_jit, tmp, src[j]); + src[j] = tmp; + break; + } + case MOVED: + break; + default: + abort (); + } + } + } + + move_operand (_jit, dst[i], src[i]); + status[i] = MOVED; + if (tmp_gpr) + unget_temp_gpr(_jit); + else if (tmp_fpr) + unget_temp_xpr(_jit); +} + +static void +apply_addend(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src) +{ + switch (MOVE_KIND(src.kind, dst.kind)) { + case MOVE_GPR_TO_GPR: + case MOVE_MEM_TO_GPR: + if (operand_addend(src)) + jit_addi(_jit, dst.loc.gpr.gpr, dst.loc.gpr.gpr, operand_addend(src)); + break; + case MOVE_GPR_TO_MEM: + case MOVE_MEM_TO_MEM: + if (operand_addend(src)) { + jit_gpr_t tmp = get_temp_gpr(_jit); + abi_mem_to_gpr(_jit, dst.abi, tmp, dst.loc.mem.base, dst.loc.mem.offset); + jit_addi(_jit, tmp, tmp, operand_addend(src)); + abi_gpr_to_mem(_jit, dst.abi, dst.loc.mem.base, dst.loc.mem.offset, tmp); + } + break; + default: + break; + } +} + +/* Preconditions: No dest operand is IMM. No dest operand aliases + another dest operand. No dest MEM operand uses a base register which + is used as a dest GPR. No dst operand has an addend. The registers + returned by get_temp_gpr and get_temp_fpr do not appear in source or + dest args. */ +void +jit_move_operands(jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src, + size_t argc) +{ + // Check preconditions, except the condition about tmp registers. + { + uint64_t src_gprs = 0; + uint64_t dst_gprs = 0; + uint64_t dst_fprs = 0; + uint64_t dst_mem_base_gprs = 0; + for (size_t i = 0; i < argc; i++) { + switch (src[i].kind) { + case JIT_OPERAND_KIND_GPR: + src_gprs |= 1ULL << jit_gpr_regno(src[i].loc.gpr.gpr); + break; + case JIT_OPERAND_KIND_FPR: + case JIT_OPERAND_KIND_IMM: + case JIT_OPERAND_KIND_MEM: + break; + default: + abort(); + } + switch (dst[i].kind) { + case JIT_OPERAND_KIND_GPR: { + ASSERT(dst[i].loc.gpr.addend == 0); + uint64_t bit = 1ULL << jit_gpr_regno(dst[i].loc.gpr.gpr); + ASSERT((dst_gprs & bit) == 0); + dst_gprs |= bit; + break; + } + case JIT_OPERAND_KIND_FPR: { + uint64_t bit = 1ULL << jit_fpr_regno(dst[i].loc.fpr); + ASSERT((dst_fprs & bit) == 0); + dst_fprs |= bit; + break; + } + case JIT_OPERAND_KIND_MEM: { + ASSERT(dst[i].loc.mem.addend == 0); + uint64_t bit = 1ULL << jit_gpr_regno(dst[i].loc.mem.base); + dst_mem_base_gprs |= bit; + break; + } + case JIT_OPERAND_KIND_IMM: + default: + abort(); + break; + } + } + ASSERT(((src_gprs | dst_gprs) & dst_mem_base_gprs) == 0); + } + + enum move_status status[argc]; + for (size_t i = 0; i < argc; i++) + status[i] = TO_MOVE; + for (size_t i = 0; i < argc; i++) + if (status[i] == TO_MOVE) + move_one(_jit, dst, src, argc, status, i); + + // Apply addends at the end. We could do it earlier in some cases but + // at least at the end we know that an in-place increment of one + // operand won't alias another. + for (size_t i = 0; i < argc; i++) + apply_addend(_jit, dst[i], src[i]); +} + +size_t +jit_align_stack(jit_state_t *_jit, size_t expand) +{ + size_t new_size = _jit->frame_size + expand; + // Align stack to double-word boundaries. This isn't really a + // principle but it does work for Aarch32, AArch64 and x86-64. + size_t alignment = __WORDSIZE * 2; + size_t aligned_size = (new_size + alignment - 1) & ~(alignment - 1); + size_t diff = aligned_size - _jit->frame_size; + if (diff) + jit_subi (_jit, JIT_SP, JIT_SP, diff); + _jit->frame_size = aligned_size; + return diff; +} + +void +jit_shrink_stack(jit_state_t *_jit, size_t diff) +{ + if (diff) + jit_addi (_jit, JIT_SP, JIT_SP, diff); + _jit->frame_size -= diff; +} + +// Precondition: stack is already aligned. +static size_t +prepare_call_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) +{ + jit_operand_t dst[argc]; + struct abi_arg_iterator iter; + + // Compute shuffle destinations and space for spilled arguments. + reset_abi_arg_iterator(&iter, argc, args); + for (size_t i = 0; i < argc; i++) + next_abi_arg(&iter, &dst[i]); + + // Reserve space for spilled arguments and ensure stack alignment. + size_t stack_size = jit_align_stack(_jit, iter.stack_size); + + // Fix up SP-relative operands. + for (size_t i = 0; i < argc; i++) { + switch(args[i].kind) { + case JIT_OPERAND_KIND_GPR: + if (jit_same_gprs (args[i].loc.mem.base, JIT_SP)) + args[i].loc.gpr.addend += stack_size; + break; + case JIT_OPERAND_KIND_MEM: + if (jit_same_gprs (args[i].loc.mem.base, JIT_SP)) + args[i].loc.mem.offset += stack_size; + break; + default: + break; + } + } + + jit_move_operands(_jit, dst, args, argc); + + return stack_size; +} + +void +jit_calli(jit_state_t *_jit, jit_pointer_t f, size_t argc, jit_operand_t args[]) +{ + size_t stack_bytes = prepare_call_args(_jit, argc, args); + + calli(_jit, (jit_word_t)f); + + jit_shrink_stack(_jit, stack_bytes); +} + +void +jit_callr(jit_state_t *_jit, jit_gpr_t f, size_t argc, jit_operand_t args[]) +{ + size_t stack_bytes = prepare_call_args(_jit, argc, args); + + callr(_jit, jit_gpr_regno(f)); + + jit_shrink_stack(_jit, stack_bytes); +} + +void +jit_locate_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) +{ + struct abi_arg_iterator iter; + + reset_abi_arg_iterator(&iter, argc, args); + for (size_t i = 0; i < argc; i++) + next_abi_arg(&iter, &args[i]); +} + +/* Precondition: args are distinct locations of type GPR or FPR. All + addends of arg operands are zero. No GPR arg is SP. */ +void +jit_load_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) +{ + jit_operand_t src[argc]; + + memcpy(src, args, sizeof(src[0]) * argc); + + jit_locate_args(_jit, argc, src); + jit_move_operands(_jit, args, src, argc); +} diff --git a/libguile/lightening/lightening/x86-cpu.c b/libguile/lightening/lightening/x86-cpu.c index 018a58b9a..aa9d68a68 100644 --- a/libguile/lightening/lightening/x86-cpu.c +++ b/libguile/lightening/lightening/x86-cpu.c @@ -254,6 +254,7 @@ rx(jit_state_t *_jit, int32_t rd, int32_t md, static void pushr(jit_state_t *_jit, int32_t r0) { + _jit->frame_size += __WORDSIZE; rex(_jit, 0, WIDE, 0, 0, r0); ic(_jit, 0x50 | r7(r0)); } @@ -261,6 +262,7 @@ pushr(jit_state_t *_jit, int32_t r0) static void popr(jit_state_t *_jit, int32_t r0) { + _jit->frame_size -= __WORDSIZE; rex(_jit, 0, WIDE, 0, 0, r0); ic(_jit, 0x58 | r7(r0)); } @@ -401,7 +403,7 @@ mov_addr(jit_state_t *_jit, int32_t r0) rex(_jit, 0, WIDE, _NOREG, _NOREG, r0); ic(_jit, 0xb8 | r7(r0)); ptrdiff_t inst_start = _jit->pc.uc - pc_start; - return jit_reloc(_jit, JIT_RELOC_ABSOLUTE, inst_start, 0); + return jit_reloc(_jit, JIT_RELOC_ABSOLUTE, inst_start); } static void @@ -658,15 +660,6 @@ testi(jit_state_t *_jit, int32_t r0, jit_word_t i0) } static void -cc(jit_state_t *_jit, int32_t code, int32_t r0) -{ - rex(_jit, 0, 0, _NOREG, _NOREG, r0); - ic(_jit, 0x0f); - ic(_jit, 0x90 | code); - mrm(_jit, 0x03, 0x00, r7(r0)); -} - -static void negr(jit_state_t *_jit, int32_t r0, int32_t r1) { if (r0 == r1) { @@ -1464,69 +1457,6 @@ xori(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0) } static void -cr(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1, int32_t r2) -{ - if (reg8_p(r0)) { - jit_bool_t same = r0 == r1 || r0 == r2; - if (!same) - ixorr(_jit, r0, r0); - icmpr(_jit, r1, r2); - if (same) - imovi(_jit, r0, 0); - cc(_jit, code, r0); - } else { - jit_gpr_t reg = get_temp_gpr(_jit); - ixorr(_jit, jit_gpr_regno(reg), jit_gpr_regno(reg)); - icmpr(_jit, r1, r2); - cc(_jit, code, jit_gpr_regno(reg)); - movr(_jit, r0, jit_gpr_regno(reg)); - unget_temp_gpr(_jit); - } -} - -static void -ci(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1, jit_word_t i0) -{ - if (reg8_p(r0)) { - jit_bool_t same = r0 == r1; - if (!same) - ixorr(_jit, r0, r0); - icmpi(_jit, r1, i0); - if (same) - imovi(_jit, r0, 0); - cc(_jit, code, r0); - } else { - jit_gpr_t reg = get_temp_gpr(_jit); - ixorr(_jit, jit_gpr_regno(reg), jit_gpr_regno(reg)); - icmpi(_jit, r1, i0); - cc(_jit, code, jit_gpr_regno(reg)); - movr(_jit, r0, jit_gpr_regno(reg)); - unget_temp_gpr(_jit); - } -} - -static void -ci0(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1) -{ - if (reg8_p(r0)) { - jit_bool_t same = r0 == r1; - if (!same) - ixorr(_jit, r0, r0); - testr(_jit, r1, r1); - if (same) - imovi(_jit, r0, 0); - cc(_jit, code, r0); - } else { - jit_gpr_t reg = get_temp_gpr(_jit); - ixorr(_jit, jit_gpr_regno(reg), jit_gpr_regno(reg)); - testr(_jit, r1, r1); - cc(_jit, code, jit_gpr_regno(reg)); - movr(_jit, r0, jit_gpr_regno(reg)); - unget_temp_gpr(_jit); - } -} - -static void extr_c(jit_state_t *_jit, int32_t r0, int32_t r1) { if (reg8_p(r1)) { @@ -2253,7 +2183,7 @@ static jit_reloc_t jccs(jit_state_t *_jit, int32_t code) { ic(_jit, 0x70 | code); - return jit_reloc(_jit, JIT_RELOC_REL8, 1, 0); + return jit_reloc(_jit, JIT_RELOC_REL8, 1); } static jit_reloc_t @@ -2261,7 +2191,26 @@ jcc(jit_state_t *_jit, int32_t code) { ic(_jit, 0x0f); ic(_jit, 0x80 | code); - return jit_reloc(_jit, JIT_RELOC_REL32, 2, 0); + return jit_reloc(_jit, JIT_RELOC_REL32, 2); +} + +static void +jcci(jit_state_t *_jit, int32_t code, jit_word_t i0) +{ + ptrdiff_t rel8 = i0 - (_jit->pc.w + 1 + 1); + ptrdiff_t rel32 = i0 - (_jit->pc.w + 2 + 4); + if (INT8_MIN <= rel8 && rel8 <= INT8_MAX) + { + ic(_jit, 0x70 | code); + ic(_jit, rel8); + } + else + { + ASSERT(INT32_MIN <= rel32 && rel32 <= INT32_MAX); + ic(_jit, 0x0f); + ic(_jit, 0x80 | code); + ii(_jit, rel32); + } } #define DEFINE_JUMPS(cc, CC, code) \ @@ -2664,8 +2613,14 @@ jmpr(jit_state_t *_jit, int32_t r0) static void jmpi(jit_state_t *_jit, jit_word_t i0) { + ptrdiff_t rel8 = i0 - (_jit->pc.w + 1 + 1); ptrdiff_t rel32 = i0 - (_jit->pc.w + 1 + 4); - if (INT32_MIN <= rel32 && rel32 <= INT32_MAX) + if (INT8_MIN <= rel8 && rel8 <= INT8_MAX) + { + ic(_jit, 0xeb); + ic(_jit, rel8); + } + else if (INT32_MIN <= rel32 && rel32 <= INT32_MAX) { ic(_jit, 0xe9); ii(_jit, rel32); @@ -2683,14 +2638,7 @@ static jit_reloc_t jmp(jit_state_t *_jit) { ic(_jit, 0xe9); - return jit_reloc(_jit, JIT_RELOC_REL32, 1, 0); -} - -static jit_reloc_t -jmpsi(jit_state_t *_jit) -{ - ic(_jit, 0xeb); - return jit_reloc(_jit, JIT_RELOC_REL8, 1, 0); + return jit_reloc(_jit, JIT_RELOC_REL32, 1); } static void diff --git a/libguile/lightening/lightening/x86.c b/libguile/lightening/lightening/x86.c index 144816dfd..4031bb1ca 100644 --- a/libguile/lightening/lightening/x86.c +++ b/libguile/lightening/lightening/x86.c @@ -214,12 +214,6 @@ jit_init(jit_state_t *_jit) return jit_cpu.sse2; } -void -jit_epilog(jit_state_t *_jit) -{ - /* TODO: Restore registers. */ -} - static jit_bool_t is_fpr_arg(enum jit_operand_abi arg) { @@ -249,426 +243,6 @@ is_gpr_arg(enum jit_operand_abi arg) return !is_fpr_arg(arg); } -static void -abi_imm_to_gpr(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t dst, - intptr_t imm) -{ - switch (abi) { - case JIT_OPERAND_ABI_UINT8: - ASSERT(0 <= imm); - ASSERT(imm <= UINT8_MAX); - break; - case JIT_OPERAND_ABI_INT8: - ASSERT(INT8_MIN <= imm); - ASSERT(imm <= INT8_MAX); - break; - case JIT_OPERAND_ABI_UINT16: - ASSERT(0 <= imm); - ASSERT(imm <= UINT16_MAX); - break; - case JIT_OPERAND_ABI_INT16: - ASSERT(INT16_MIN <= imm); - ASSERT(imm <= INT16_MAX); - break; - case JIT_OPERAND_ABI_UINT32: - ASSERT(0 <= imm); - ASSERT(imm <= UINT32_MAX); - break; - case JIT_OPERAND_ABI_INT32: - ASSERT(INT32_MIN <= imm); - ASSERT(imm <= INT32_MAX); - break; -#if __WORDSIZE > 32 - case JIT_OPERAND_ABI_UINT64: - case JIT_OPERAND_ABI_INT64: - break; -#endif - case JIT_OPERAND_ABI_POINTER: - break; - default: - abort(); - } - jit_movi (_jit, dst, imm); -} - -static void -abi_gpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, - jit_gpr_t base, ptrdiff_t offset, jit_gpr_t src) -{ - switch (abi) { - case JIT_OPERAND_ABI_UINT8: - case JIT_OPERAND_ABI_INT8: - jit_stxi_c(_jit, offset, base, src); - break; - case JIT_OPERAND_ABI_UINT16: - case JIT_OPERAND_ABI_INT16: - jit_stxi_s(_jit, offset, base, src); - break; - case JIT_OPERAND_ABI_UINT32: - case JIT_OPERAND_ABI_INT32: -#if __WORDSIZE == 32 - case JIT_OPERAND_ABI_POINTER: -#endif - jit_stxi_i(_jit, offset, base, src); - break; -#if __WORDSIZE == 64 - case JIT_OPERAND_ABI_UINT64: - case JIT_OPERAND_ABI_INT64: - case JIT_OPERAND_ABI_POINTER: - jit_stxi_l(_jit, offset, base, src); - break; -#endif - default: - abort(); - } -} - -static void -abi_fpr_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, - jit_gpr_t base, ptrdiff_t offset, jit_fpr_t src) -{ - switch (abi) { - case JIT_OPERAND_ABI_FLOAT: - jit_stxi_f(_jit, offset, base, src); - break; - case JIT_OPERAND_ABI_DOUBLE: - jit_stxi_d(_jit, offset, base, src); - break; - default: - abort(); - } -} - -static void -abi_mem_to_gpr(jit_state_t *_jit, enum jit_operand_abi abi, - jit_gpr_t dst, jit_gpr_t base, ptrdiff_t offset) -{ - switch (abi) { - case JIT_OPERAND_ABI_UINT8: - jit_ldxi_uc(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_INT8: - jit_ldxi_c(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_UINT16: - jit_ldxi_us(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_INT16: - jit_ldxi_s(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_UINT32: - jit_ldxi_ui(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_INT32: - jit_ldxi_i(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_UINT64: - jit_ldxi_l(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_INT64: - jit_ldxi_l(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_POINTER: - jit_ldxi_l(_jit, dst, base, offset); - break; - default: - abort(); - } -} - -static void -abi_mem_to_fpr(jit_state_t *_jit, enum jit_operand_abi abi, - jit_fpr_t dst, jit_gpr_t base, ptrdiff_t offset) -{ - switch (abi) { - case JIT_OPERAND_ABI_FLOAT: - jit_ldxi_f(_jit, dst, base, offset); - break; - case JIT_OPERAND_ABI_DOUBLE: - jit_ldxi_d(_jit, dst, base, offset); - break; - default: - abort(); - } -} - -static void -abi_imm_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t base, - ptrdiff_t offset, intmax_t imm) -{ - ASSERT(!is_fpr_arg(abi)); - - jit_gpr_t tmp = get_temp_gpr(_jit); - abi_imm_to_gpr(_jit, abi, tmp, imm); - abi_gpr_to_mem(_jit, abi, base, offset, tmp); - unget_temp_gpr(_jit); -} - -static void -abi_mem_to_mem(jit_state_t *_jit, enum jit_operand_abi abi, jit_gpr_t base, - ptrdiff_t offset, jit_gpr_t src_base, ptrdiff_t src_offset) -{ - if (is_gpr_arg (abi)) { - jit_gpr_t tmp = get_temp_gpr(_jit); - abi_mem_to_gpr(_jit, abi, tmp, src_base, src_offset); - abi_gpr_to_mem(_jit, abi, base, offset, tmp); - unget_temp_gpr(_jit); - } else { - jit_fpr_t tmp = get_temp_xpr(_jit); - abi_mem_to_fpr(_jit, abi, tmp, src_base, src_offset); - abi_fpr_to_mem(_jit, abi, base, offset, tmp); - unget_temp_xpr(_jit); - } -} - -#define MOVE_KIND(a, b) ((((int) a) << 4) | ((int) b)) - -#define MOVE_KIND_ENUM(a, b) \ - MOVE_##a##_TO_##b = MOVE_KIND(JIT_OPERAND_KIND_##a, JIT_OPERAND_KIND_##b) -enum move_kind { - MOVE_KIND_ENUM(IMM, GPR), - MOVE_KIND_ENUM(GPR, GPR), - MOVE_KIND_ENUM(MEM, GPR), - MOVE_KIND_ENUM(FPR, FPR), - MOVE_KIND_ENUM(MEM, FPR), - MOVE_KIND_ENUM(IMM, MEM), - MOVE_KIND_ENUM(GPR, MEM), - MOVE_KIND_ENUM(FPR, MEM), - MOVE_KIND_ENUM(MEM, MEM) -}; -#undef MOVE_KIND_ENUM - -static void -move_operand(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src) -{ - switch (MOVE_KIND (src.kind, dst.kind)) { - case MOVE_IMM_TO_GPR: - return abi_imm_to_gpr(_jit, src.abi, dst.loc.gpr.gpr, src.loc.imm); - - case MOVE_GPR_TO_GPR: - return jit_movr(_jit, dst.loc.gpr.gpr, src.loc.gpr.gpr); - - case MOVE_MEM_TO_GPR: - return abi_mem_to_gpr(_jit, src.abi, dst.loc.gpr.gpr, src.loc.mem.base, - src.loc.mem.offset); - - case MOVE_FPR_TO_FPR: - return jit_movr_d(_jit, dst.loc.fpr, src.loc.fpr); - - case MOVE_MEM_TO_FPR: - return abi_mem_to_fpr(_jit, src.abi, dst.loc.fpr, src.loc.mem.base, - src.loc.mem.offset); - - case MOVE_IMM_TO_MEM: - return abi_imm_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, - src.loc.imm); - - case MOVE_GPR_TO_MEM: - return abi_gpr_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, - src.loc.gpr.gpr); - - case MOVE_FPR_TO_MEM: - return abi_fpr_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, - src.loc.fpr); - - case MOVE_MEM_TO_MEM: - return abi_mem_to_mem(_jit, src.abi, dst.loc.mem.base, dst.loc.mem.offset, - src.loc.mem.base, src.loc.mem.offset); - - default: - abort(); - } -} - -// A direct transliteration of "Tilting at windmills with Coq: formal -// verification of a compilation algorithm for parallel moves" by -// Laurence Rideau, Bernard Paul Serpette, and Xavier Leroy: -// https://xavierleroy.org/publi/parallel-move.pdf - -enum move_status { TO_MOVE, BEING_MOVED, MOVED }; - -static inline int -already_in_place(jit_operand_t src, jit_operand_t dst) -{ - switch (MOVE_KIND(src.kind, dst.kind)) { - case MOVE_GPR_TO_GPR: - return jit_same_gprs (src.loc.gpr.gpr, dst.loc.gpr.gpr); - case MOVE_FPR_TO_FPR: - return jit_same_fprs (src.loc.fpr, dst.loc.fpr); - case MOVE_MEM_TO_MEM: - return jit_same_gprs (src.loc.mem.base, dst.loc.mem.base) && - src.loc.mem.offset == dst.loc.mem.offset; - default: - return 0; - } -} - -static inline int -write_would_clobber(jit_operand_t src, jit_operand_t dst) -{ - if (already_in_place (src, dst)) - return 1; - - if (MOVE_KIND(src.kind, dst.kind) == MOVE_MEM_TO_GPR) - return jit_same_gprs(src.loc.mem.base, dst.loc.gpr.gpr); - - return 0; -} - -static inline ptrdiff_t -operand_addend(jit_operand_t op) -{ - switch (op.kind) { - case JIT_OPERAND_KIND_GPR: - return op.loc.gpr.addend; - case JIT_OPERAND_KIND_MEM: - return op.loc.mem.addend; - default: - abort(); - } -} - -static void -move_one(jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src, - size_t argc, enum move_status *status, size_t i) -{ - int tmp_gpr = 0, tmp_fpr = 0; - - if (already_in_place(src[i], dst[i])) - return; - - status[i] = BEING_MOVED; - for (size_t j = 0; j < argc; j++) { - if (write_would_clobber(src[j], dst[i])) { - switch (status[j]) { - case TO_MOVE: - move_one(_jit, dst, src, argc, status, j); - break; - case BEING_MOVED: { - jit_operand_t tmp; - if (is_fpr_arg (src[j].kind)) { - tmp_fpr = 1; - tmp = jit_operand_fpr(src[j].abi, get_temp_xpr(_jit)); - } else { - tmp_gpr = 1; - /* Preserve addend, if any, from source operand, to be applied - at the end. */ - tmp = jit_operand_gpr_with_addend(src[j].abi, get_temp_gpr(_jit), - operand_addend(src[j])); - } - move_operand (_jit, tmp, src[j]); - src[j] = tmp; - break; - } - case MOVED: - break; - default: - abort (); - } - } - } - - move_operand (_jit, dst[i], src[i]); - status[i] = MOVED; - if (tmp_gpr) - unget_temp_gpr(_jit); - else if (tmp_fpr) - unget_temp_xpr(_jit); -} - -static void -apply_addend(jit_state_t *_jit, jit_operand_t dst, jit_operand_t src) -{ - switch (MOVE_KIND(src.kind, dst.kind)) { - case MOVE_GPR_TO_GPR: - case MOVE_MEM_TO_GPR: - if (operand_addend(src)) - jit_addi(_jit, dst.loc.gpr.gpr, dst.loc.gpr.gpr, operand_addend(src)); - break; - case MOVE_GPR_TO_MEM: - case MOVE_MEM_TO_MEM: - if (operand_addend(src)) { - jit_gpr_t tmp = get_temp_gpr(_jit); - abi_mem_to_gpr(_jit, dst.abi, tmp, dst.loc.mem.base, dst.loc.mem.offset); - jit_addi(_jit, tmp, tmp, operand_addend(src)); - abi_gpr_to_mem(_jit, dst.abi, dst.loc.mem.base, dst.loc.mem.offset, tmp); - } - break; - default: - break; - } -} - -/* Preconditions: No dest operand is IMM. No dest operand aliases - another dest operand. No dest MEM operand uses a base register which - is used as a dest GPR. No dst operand has an addend. The registers - returned by get_temp_gpr and get_temp_fpr do not appear in source or - dest args. */ -void -jit_move_operands(jit_state_t *_jit, jit_operand_t *dst, jit_operand_t *src, - size_t argc) -{ - // Check preconditions, except the condition about tmp registers. - { - uint64_t src_gprs = 0; - uint64_t dst_gprs = 0; - uint64_t dst_fprs = 0; - uint64_t dst_mem_base_gprs = 0; - for (size_t i = 0; i < argc; i++) { - switch (src[i].kind) { - case JIT_OPERAND_KIND_GPR: - src_gprs |= 1ULL << jit_gpr_regno(src[i].loc.gpr.gpr); - break; - case JIT_OPERAND_KIND_FPR: - case JIT_OPERAND_KIND_IMM: - case JIT_OPERAND_KIND_MEM: - break; - default: - abort(); - } - switch (dst[i].kind) { - case JIT_OPERAND_KIND_GPR: { - ASSERT(dst[i].loc.gpr.addend == 0); - uint64_t bit = 1ULL << jit_gpr_regno(dst[i].loc.gpr.gpr); - ASSERT((dst_gprs & bit) == 0); - dst_gprs |= bit; - break; - } - case JIT_OPERAND_KIND_FPR: { - uint64_t bit = 1ULL << jit_fpr_regno(dst[i].loc.fpr); - ASSERT((dst_fprs & bit) == 0); - dst_fprs |= bit; - break; - } - case JIT_OPERAND_KIND_MEM: { - ASSERT(dst[i].loc.mem.addend == 0); - uint64_t bit = 1ULL << jit_gpr_regno(dst[i].loc.mem.base); - dst_mem_base_gprs |= bit; - break; - } - case JIT_OPERAND_KIND_IMM: - default: - abort(); - break; - } - } - ASSERT(((src_gprs | dst_gprs) & dst_mem_base_gprs) == 0); - } - - enum move_status status[argc]; - for (size_t i = 0; i < argc; i++) - status[i] = TO_MOVE; - for (size_t i = 0; i < argc; i++) - if (status[i] == TO_MOVE) - move_one(_jit, dst, src, argc, status, i); - - // Apply addends at the end. We could do it earlier in some cases but - // at least at the end we know that an in-place increment of one - // operand won't alias another. - for (size_t i = 0; i < argc; i++) - apply_addend(_jit, dst[i], src[i]); -} - static const jit_gpr_t abi_gpr_args[] = { #if __X32 /* No GPRs in args. */ @@ -771,100 +345,48 @@ next_abi_arg(struct abi_arg_iterator *iter, jit_operand_t *arg) iter->arg_idx++; } -// Precondition: stack is already aligned. -static size_t -prepare_call_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) -{ - jit_operand_t dst[argc]; - struct abi_arg_iterator iter; - - // Compute shuffle destinations and space for spilled arguments. - reset_abi_arg_iterator(&iter, argc, args); - for (size_t i = 0; i < argc; i++) - next_abi_arg(&iter, &dst[i]); - - size_t stack_size = iter.stack_size; - - // Reserve space for spilled arguments, and fix up SP-relative - // operands. - if (stack_size) - { - // Align stack to 16-byte boundaries on 64-bit targets. - if (__WORDSIZE == 64) - stack_size = (stack_size + 15) & ~15; - jit_subi(_jit, JIT_SP, JIT_SP, stack_size); - for (size_t i = 0; i < argc; i++) { - switch(args[i].kind) { - case JIT_OPERAND_KIND_GPR: - if (jit_same_gprs (args[i].loc.mem.base, JIT_SP)) - args[i].loc.gpr.addend += stack_size; - break; - case JIT_OPERAND_KIND_MEM: - if (jit_same_gprs (args[i].loc.mem.base, JIT_SP)) - args[i].loc.mem.offset += stack_size; - break; - default: - break; - } - } - } - - jit_move_operands(_jit, dst, args, argc); - - return stack_size; -} - -void -jit_calli(jit_state_t *_jit, jit_pointer_t f, size_t argc, jit_operand_t args[]) -{ - size_t spill_size = prepare_call_args(_jit, argc, args); - - calli(_jit, (jit_word_t)f); - - if (spill_size) - jit_addi(_jit, JIT_SP, JIT_SP, spill_size); -} - -void -jit_callr(jit_state_t *_jit, jit_gpr_t f, size_t argc, jit_operand_t args[]) -{ - size_t spill_size = prepare_call_args(_jit, argc, args); - - callr(_jit, jit_gpr_regno(f)); - - if (spill_size) - jit_addi(_jit, JIT_SP, JIT_SP, spill_size); -} - -void -jit_locate_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) -{ - struct abi_arg_iterator iter; - - reset_abi_arg_iterator(&iter, argc, args); - for (size_t i = 0; i < argc; i++) - next_abi_arg(&iter, &args[i]); -} - -/* Precondition: args are distinct locations of type GPR or FPR. All - addends of arg operands are zero. No GPR arg is SP. */ -void -jit_load_args(jit_state_t *_jit, size_t argc, jit_operand_t args[]) -{ - jit_operand_t src[argc]; - - memcpy(src, args, sizeof(src[0]) * argc); - - jit_locate_args(_jit, argc, src); - jit_move_operands(_jit, args, src, argc); -} - void jit_flush(void *fptr, void *tptr) { } static void -jit_try_shorten(jit_state_t *_jit, jit_reloc_t reloc) +jit_try_shorten(jit_state_t *_jit, jit_reloc_t reloc, jit_pointer_t addr) { + uint8_t *loc = _jit->start + reloc.offset; + uint8_t *start = loc - reloc.inst_start_offset; + jit_imm_t i0 = (jit_imm_t)addr; + + switch (reloc.kind) + { + case JIT_RELOC_ABSOLUTE: { + _jit->pc.uc = start; + ASSERT((loc[-1] & ~7) == 0xb8); // MOVI + int32_t r0 = loc[-1] & 7; + if (start != loc - 1) { + ASSERT(start == loc - 2); + r0 |= (loc[-2] & 1) << 3; + } + return movi(_jit, r0, i0); + } + case JIT_RELOC_REL8: + ASSERT((loc[-1] & ~0xf) == 0x70 || loc[-1] == 0xeb); // JCCSI or JMPSI + /* Nothing useful to do. */ + return; + case JIT_RELOC_REL16: + /* We don't emit these. */ + abort (); + case JIT_RELOC_REL32: + _jit->pc.uc = start; + if (start[0] == 0xe9) { // JMP + return jmpi(_jit, i0); + } + ASSERT(start[0] == 0x0f); // JCC + return jcci(_jit, start[1] & ~0x80, i0); + case JIT_RELOC_REL64: + /* We don't emit these. */ + abort (); + default: + abort (); + } } |