diff options
author | Andy Wingo <wingo@pobox.com> | 2018-08-19 17:39:16 +0200 |
---|---|---|
committer | Andy Wingo <wingo@pobox.com> | 2018-08-19 17:39:16 +0200 |
commit | 698bff8748492c10d9935d7837776c323dc4f383 (patch) | |
tree | 65c9ed3972d324240b6fbc0d68df26323535cea0 | |
parent | 110310b3e71cefd9e625ae8c775358a58f8822ac (diff) | |
download | guile-698bff8748492c10d9935d7837776c323dc4f383.tar.gz |
First implementation of a template JIT
* libguile/jit.c: Implement most of a JIT. Untested and still needs to
be wired up.
-rw-r--r-- | libguile/jit.c | 2400 |
1 files changed, 2316 insertions, 84 deletions
diff --git a/libguile/jit.c b/libguile/jit.c index afd3f056b..7f0bac578 100644 --- a/libguile/jit.c +++ b/libguile/jit.c @@ -26,18 +26,739 @@ #include <lightning.h> #endif +#include "frames.h" +#include "gsubr.h" +#include "instructions.h" +#include "intrinsics.h" #include "threads.h" +#include "vm-builtins.h" #include "vm-operations.h" #include "jit.h" + + + typedef struct { + jit_state_t *jit; scm_thread *thread; const uint32_t *start; uint32_t *ip; const uint32_t *end; + int32_t frame_size; + uint8_t hooks_enabled; } scm_jit_state; +/* Lightning routines take an implicit parameter, _jit. All functions + that call lightning API should have a parameter "scm_jit_state *j"; + this definition makes lightning load its state from that + parameter. */ +#define _jit (j->jit) + +static const uint32_t program_word_offset_free_variable = 2; + +static const uint32_t frame_offset_mra = 0 * sizeof(union scm_vm_stack_element); +static const uint32_t frame_offset_vra = 1 * sizeof(union scm_vm_stack_element); +static const uint32_t frame_offset_prev = 2 * sizeof(union scm_vm_stack_element); +static const uint32_t frame_overhead_slots = 3; + +#define DEFINE_THREAD_OFFSET(f) \ + static const uint32_t thread_offset_##f = \ + offsetof (struct scm_thread, f) + +DEFINE_THREAD_OFFSET (handle); +DEFINE_THREAD_OFFSET (pending_asyncs); +DEFINE_THREAD_OFFSET (block_asyncs); + +#define DEFINE_THREAD_VP_OFFSET(f) \ + static const uint32_t thread_offset_##f = \ + offsetof (struct scm_thread, vm) + offsetof (struct scm_vm, f) + +DEFINE_THREAD_VP_OFFSET (fp); +DEFINE_THREAD_VP_OFFSET (sp); +DEFINE_THREAD_VP_OFFSET (ip); +DEFINE_THREAD_VP_OFFSET (compare_result); +DEFINE_THREAD_VP_OFFSET (sp_min_since_gc); +DEFINE_THREAD_VP_OFFSET (stack_limit); +DEFINE_THREAD_VP_OFFSET (trace_level); + +static const jit_gpr_t THREAD = JIT_V0; +static const jit_gpr_t SP = JIT_V1; + +static const jit_gpr_t T0 = JIT_R0; +static const jit_gpr_t T1 = JIT_R1; +static const jit_gpr_t T2 = JIT_R2; +static const jit_gpr_t T3 = JIT_V2; + +/* Sometimes you want to call out the fact that T3 is preserved across + calls. In that case, use T3_PRESERVED. */ +static const jit_gpr_t T3_PRESERVED = JIT_V2; + +#ifdef WORDS_BIGENDIAN +#define BIGENDIAN 1 +#else +#define BIGENDIAN 0 +#endif + +#if BIGENDIAN +static const uint32_t uint32_offset_low_byte = 3; +#else +static const uint32_t uint32_offset_low_byte = 0; +#endif + +#if SCM_SIZEOF_UINTPTR_T == 4 +static const uint32_t log2_sizeof_uintptr_t = 2; +#elif SCM_SIZEOF_UINTPTR_T == 8 +static const uint32_t log2_sizeof_uintptr_t = 3; +#else +#error unhandled uintptr_t size +#endif + +static void +emit_reload_sp (scm_jit_state *j) +{ + jit_ldxi (SP, THREAD, thread_offset_sp); +} + +static void +emit_store_sp (scm_jit_state *j) +{ + jit_stxi (thread_offset_sp, THREAD, SP); +} + +static void +emit_load_fp (scm_jit_state *j, jit_gpr_t dst) +{ + jit_ldxi (dst, THREAD, thread_offset_fp); +} + +static void +emit_store_fp (scm_jit_state *j, jit_gpr_t fp) +{ + jit_stxi (thread_offset_fp, THREAD, fp); +} + +static void +emit_subtract_stack_slots (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t src, + uint32_t n) +{ + jit_subi (dst, src, n * sizeof (union scm_vm_stack_element)); +} + +static void +emit_load_mra (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t fp) +{ + if (frame_offset_mra != 0) + abort (); + jit_ldr (dst, fp); +} + +static jit_node_t * +emit_store_mra (scm_jit_state *j, jit_gpr_t fp, jit_gpr_t t) +{ + jit_node_t *addr = jit_movi (t, 0); /* patched later */ + if (frame_offset_mra != 0) + abort (); + jit_str (fp, t); + return addr; +} + +static void +emit_load_vra (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t fp) +{ + jit_ldxi (dst, fp, frame_offset_vra); +} + +static void +emit_store_vra (scm_jit_state *j, jit_gpr_t fp, jit_gpr_t t, const uint32_t *vra) +{ + jit_movi (t, (intptr_t) vra); + jit_stxi (frame_offset_vra, fp, t); +} + +static void +emit_load_prev_frame_size (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t fp) +{ + jit_ldxi (dst, fp, frame_offset_prev); +} + +static void +emit_store_prev_frame_size (scm_jit_state *j, jit_gpr_t fp, jit_gpr_t t, + uint32_t n) +{ + jit_movi (t, n); + jit_stxi (frame_offset_prev, fp, t); +} + +static void +emit_store_ip (scm_jit_state *j, jit_gpr_t ip) +{ + jit_stxi (thread_offset_ip, THREAD, ip); +} + +static void +emit_store_current_ip (scm_jit_state *j, jit_gpr_t t) +{ + jit_movi (t, (intptr_t) j->ip); + emit_store_ip (j, t); +} + +static void +emit_load_compare_result (scm_jit_state *j, jit_gpr_t dst) +{ + jit_ldxi_uc (dst, THREAD, thread_offset_compare_result); +} + +static void +emit_store_compare_result (scm_jit_state *j, jit_gpr_t src) +{ + jit_stxi_c (thread_offset_compare_result, THREAD, src); +} + +static void +emit_reset_frame (scm_jit_state *j, jit_gpr_t fp, uint32_t nlocals) +{ + emit_subtract_stack_slots (j, SP, fp, nlocals); + emit_store_sp (j); +} + +static void +emit_call (scm_jit_state *j, void *f) +{ + jit_prepare (); + jit_finishi (f); +} + +static void +emit_call_r (scm_jit_state *j, void *f, jit_gpr_t a) +{ + jit_prepare (); + jit_pushargr (a); + jit_finishi (f); +} + +static void +emit_call_i (scm_jit_state *j, void *f, intptr_t a) +{ + jit_prepare (); + jit_pushargi (a); + jit_finishi (f); +} + +static void +emit_call_r_r (scm_jit_state *j, void *f, jit_gpr_t a, jit_gpr_t b) +{ + jit_prepare (); + jit_pushargr (a); + jit_pushargr (b); + jit_finishi (f); +} + +static void +emit_call_r_i (scm_jit_state *j, void *f, jit_gpr_t a, intptr_t b) +{ + jit_prepare (); + jit_pushargr (a); + jit_pushargi ((intptr_t) b); + jit_finishi (f); +} + +static void +emit_call_r_r_r (scm_jit_state *j, void *f, jit_gpr_t a, jit_gpr_t b, + jit_gpr_t c) +{ + jit_prepare (); + jit_pushargr (a); + jit_pushargr (b); + jit_pushargr (c); + jit_finishi (f); +} + +static void +emit_alloc_frame_for_sp (scm_jit_state *j, jit_gpr_t fp, jit_gpr_t t) +{ + jit_node_t *k, *fast, *watermark; + + jit_ldxr (t, THREAD, thread_offset_sp_min_since_gc); + fast = jit_bger (SP, t); + jit_ldxr (t, THREAD, thread_offset_stack_limit); + watermark = jit_bger (SP, t); + + /* Slow case: call out to expand stack. */ + emit_store_current_ip (j, t); + emit_call_r_r (j, scm_vm_intrinsics.expand_stack, THREAD, SP); + emit_reload_sp (j); + k = jit_jmpi (); + + /* Past sp_min_since_gc, but within stack_limit: update watermark and + fall through. */ + jit_patch (watermark); + jit_stxr (thread_offset_sp_min_since_gc, THREAD, SP); + jit_patch (fast); + /* Fast case: Just update sp. */ + emit_store_sp (j); + jit_patch (k); +} + +static void +emit_alloc_frame (scm_jit_state *j, jit_gpr_t fp, jit_gpr_t t, uint32_t nlocals) +{ + emit_subtract_stack_slots (j, SP, fp, nlocals); + emit_alloc_frame_for_sp (j, fp, t); +} + +static void +emit_get_callee_vcode (scm_jit_state *j, jit_gpr_t dst) +{ + emit_call_r (j, scm_vm_intrinsics.get_callee_vcode, THREAD); + jit_retval (dst); +} + +static void +emit_get_vcode_low_byte (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t addr) +{ + if (uint32_offset_low_byte == 0) + jit_ldr_uc (dst, addr); + else + jit_ldxi_uc (dst, addr, uint32_offset_low_byte); +} + +static void +emit_get_ip_relative_addr (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t ip, + uint32_t offset) +{ + uint32_t byte_offset = offset * sizeof (uint32_t); + jit_ldxi (dst, ip, byte_offset); + jit_lshi (dst, dst, 2); /* Multiply by sizeof (uint32_t) */ + jit_addr (dst, dst, ip); +} + +static jit_node_t* +emit_push_frame (scm_jit_state *j, uint32_t proc_slot, uint32_t nlocals, + const uint32_t *vra) +{ + jit_gpr_t fp = T0, old_fp = T1; + jit_node_t *continuation; + + emit_load_fp (j, old_fp); + emit_subtract_stack_slots (j, fp, old_fp, proc_slot); + continuation = emit_store_mra (j, fp, T1); + emit_store_vra (j, fp, T1, vra); + emit_store_prev_frame_size (j, fp, T1, proc_slot - frame_overhead_slots); + emit_store_fp (j, fp); + emit_reset_frame (j, fp, nlocals); + + return continuation; +} + +static void +emit_indirect_tail_call (scm_jit_state *j) +{ + jit_node_t *not_instrumented, *no_mcode; + + emit_get_callee_vcode (j, T0); + + emit_get_vcode_low_byte (j, T1, T0); + not_instrumented = jit_bnei (T1, scm_op_instrument_entry); + emit_get_ip_relative_addr (j, T1, T0, 1); + jit_ldr (T1, T1); + no_mcode = jit_beqi (T1, 0); + jit_jmpr (T1); + + jit_patch (not_instrumented); + jit_patch (no_mcode); + + emit_store_ip (j, T0); + jit_ret (); +} + +static void +emit_direct_tail_call (scm_jit_state *j, const uint32_t *vcode) +{ + if ((vcode[0] & 0xff) != scm_op_instrument_entry) + { + jit_movi (T0, (intptr_t) vcode); + emit_store_ip (j, T0); + jit_ret (); + } + else + { + struct scm_jit_function_data *data; + data = (struct scm_jit_function_data *) (vcode + (int32_t)(vcode[1])); + + if (data->mcode) + { + jit_patch_abs (jit_jmpi (), data->mcode); + } + else + { + jit_node_t *no_mcode; + + jit_ldi (T0, &data->mcode); + no_mcode = jit_beqi (T0, 0); + jit_jmpr (T0); + jit_patch (no_mcode); + jit_movi (T0, (intptr_t) vcode); + emit_store_ip (j, T0); + jit_ret (); + } + } +} + +static void +emit_fp_ref_scm (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t fp, uint32_t slot) +{ + jit_ldxi (dst, fp, -8 * (slot + 1)); +} + +static void +emit_fp_set_scm (scm_jit_state *j, jit_gpr_t fp, uint32_t slot, jit_gpr_t val) +{ + jit_stxi (-8 * (slot + 1), fp, val); +} + +static void +emit_sp_ref_scm (scm_jit_state *j, jit_gpr_t dst, uint32_t slot) +{ + if (slot == 0) + jit_ldr (dst, SP); + else + jit_ldxi (dst, SP, 8 * slot); +} + +static void +emit_sp_set_scm (scm_jit_state *j, uint32_t slot, jit_gpr_t val) +{ + if (slot == 0) + jit_str (SP, val); + else + jit_stxi (8 * slot, SP, val); +} + +static void +emit_mov (scm_jit_state *j, uint32_t dst, uint32_t src, jit_gpr_t t) +{ + /* FIXME: The compiler currently emits "push" for SCM, F64, U64, + and S64 variables. However SCM values are the usual case, and + on a 32-bit machine it might be cheaper to move a SCM than to + move a 64-bit number. */ + if (sizeof (void*) < sizeof (union scm_vm_stack_element)) + { + uintptr_t src_offset = src * sizeof (union scm_vm_stack_element); + uintptr_t dst_offset = dst * sizeof (union scm_vm_stack_element); + + jit_ldxi (t, SP, src_offset + sizeof (void*)); + jit_stxi (dst_offset + sizeof (void*), SP, t); + if (src_offset == 0) + jit_ldr (t, SP); + else + jit_ldxi (t, SP, src_offset); + if (dst_offset == 0) + jit_str (SP, t); + else + jit_stxi (dst_offset, SP, t); + } + else + { + emit_sp_ref_scm (j, t, src); + emit_sp_set_scm (j, dst, t); + } +} + +static void +emit_run_hook (scm_jit_state *j, jit_gpr_t t, scm_t_thread_intrinsic f) +{ + jit_node_t *k; + jit_ldxi_ui (T0, THREAD, thread_offset_trace_level); + k = jit_beqi (T0, 0); + emit_store_current_ip (j, T0); + emit_call_r (j, f, THREAD); + emit_reload_sp (j); + jit_patch (k); +} + +static jit_node_t* +emit_branch_if_frame_locals_count_less_than (scm_jit_state *j, jit_gpr_t fp, + jit_gpr_t t, uint32_t nlocals) +{ + jit_subr (t, fp, SP); + return jit_blti (t, nlocals * sizeof (union scm_vm_stack_element)); +} + +static jit_node_t* +emit_branch_if_frame_locals_count_eq (scm_jit_state *j, jit_gpr_t fp, + jit_gpr_t t, uint32_t nlocals) +{ + jit_subr (t, fp, SP); + return jit_beqi (t, nlocals * sizeof (union scm_vm_stack_element)); +} + +static jit_node_t* +emit_branch_if_frame_locals_count_greater_than (scm_jit_state *j, jit_gpr_t fp, + jit_gpr_t t, uint32_t nlocals) +{ + jit_subr (t, fp, SP); + return jit_bgti (t, nlocals * sizeof (union scm_vm_stack_element)); +} + +static void +emit_load_fp_slot (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t fp, uint32_t slot) +{ + jit_subi (dst, fp, (slot + 1) * sizeof (union scm_vm_stack_element)); +} + +static jit_node_t * +emit_branch_if_immediate (scm_jit_state *j, jit_gpr_t r) +{ + return jit_bmsi (r, 6); +} + +static void +emit_load_heap_object_word (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t r, + uint32_t word) +{ + if (word == 0) + jit_ldr (dst, r); + else + jit_ldxi (dst, r, word * sizeof(SCM)); +} + +static void +emit_load_heap_object_tc (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t r, + scm_t_bits mask) +{ + emit_load_heap_object_word (j, dst, r, 0); + jit_andi (dst, dst, mask); +} + +static jit_node_t * +emit_branch_if_heap_object_has_tc (scm_jit_state *j, jit_gpr_t r, jit_gpr_t t, + scm_t_bits mask, scm_t_bits tc) +{ + emit_load_heap_object_tc (j, t, r, mask); + return jit_beqi (t, tc); +} + +static jit_node_t * +emit_branch_if_heap_object_not_tc (scm_jit_state *j, jit_gpr_t r, jit_gpr_t t, + scm_t_bits mask, scm_t_bits tc) +{ + emit_load_heap_object_tc (j, t, r, mask); + return jit_bnei (t, tc); +} + +static jit_node_t * +emit_branch_if_heap_object_not_tc7 (scm_jit_state *j, jit_gpr_t r, jit_gpr_t t, + scm_t_bits tc7) +{ + return emit_branch_if_heap_object_not_tc (j, r, t, 0x7f, tc7); +} + +static void +emit_entry_trampoline (scm_jit_state *j) +{ + jit_node_t *thread, *ip; + jit_prolog (); + thread = jit_arg (); + ip = jit_arg (); + /* Ensure that callee-saved registers are used and thus saved by + lightning in the prolog. */ + jit_xorr (JIT_V0, JIT_V0, JIT_V0); + jit_xorr (JIT_V1, JIT_V1, JIT_V1); + jit_xorr (JIT_V2, JIT_V2, JIT_V2); + /* Load our reserved registers: THREAD and SP. */ + jit_getarg (THREAD, thread); + emit_reload_sp (j); + /* Call the mcode! */ + jit_getarg (JIT_R0, ip); + jit_callr (JIT_R0); + /* When mcode returns, interpreter should continue with vp->ip. */ + jit_ret (); +} + +static void +emit_free_variable_ref (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t prog, + size_t n) +{ + emit_load_heap_object_word (j, dst, prog, + n + program_word_offset_free_variable); +} + +/* Use when you know that the u64 value will be within the size_t range, + for example when it's ensured by the compiler. */ +static void +emit_sp_ref_sz (scm_jit_state *j, jit_gpr_t dst, uint32_t src) +{ + if (BIGENDIAN && sizeof (size_t) == 4) + jit_ldxi (dst, SP, src * 8 + 4); + else + jit_ldxi (dst, SP, src * 8); +} + +static void +emit_sp_set_sz (scm_jit_state *j, uint32_t dst, jit_gpr_t src) +{ + size_t offset = dst * 8; + + if (sizeof (size_t) == 4) + { + size_t lo, hi; + if (BIGENDIAN) + lo = offset + 4, hi = offset; + else + lo = offset, hi = offset + 4; + + jit_stxi (lo, SP, src); + /* Set high word to 0. Clobber src. */ + jit_xorr (src, src, src); + jit_stxi (hi, SP, src); + } + else + jit_stxi (offset, SP, src); +} + +#if SIZEOF_UINTPTR_T >= 8 +static void +emit_sp_ref_u64 (scm_jit_state *j, jit_gpr_t dst, uint32_t src) +{ + size_t offset = src * 8; + + if (offset == 0) + jit_ldr (dst, SP); + else + jit_ldxi (dst, SP, offset); +} + +static void +emit_sp_set_u64 (scm_jit_state *j, uint32_t dst, jit_gpr_t src) +{ + size_t offset = dst * 8; + + if (dst == 0) + jit_str (SP, src); + else + jit_stxi (offset, SP, src); +} + +static void +emit_sp_ref_s64 (scm_jit_state *j, jit_gpr_t dst, uint32_t src) +{ + emit_sp_ref_u64 (j, dst, src); +} + +static void +emit_sp_set_s64 (scm_jit_state *j, uint32_t dst, jit_gpr_t src) +{ + emit_sp_set_u64 (j, dst, src); +} + +static void +emit_sp_ref_ptr (scm_jit_state *j, jit_gpr_t dst, uint32_t src) +{ + emit_sp_ref_u64 (j, dst, src); +} +#else +static void +emit_sp_ref_u64 (scm_jit_state *j, jit_gpr_t dst_lo, jit_gpr_t dst_hi, + uint32_t src) +{ + size_t offset = src * 8; + jit_gpr_t first, second; + +#if BIGENDIAN + first = dst_hi, second = dst_lo; +#else + first = dst_lo, second = dst_hi; +#endif + + if (offset == 0) + jit_ldr (first, SP); + else + jit_ldxi (first, SP, offset); + jit_ldxi (second, SP, offset + 4); +} + +static void +emit_sp_set_u64 (scm_jit_state *j, uint32_t dst, jit_gpr_t lo, jit_gpr_t hi) +{ + size_t offset = dst * 8; + jit_gpr_t first, second; + +#if BIGENDIAN + first = hi, second = lo; +#else + first = lo, second = hi; +#endif + + if (offset == 0) + jit_str (SP, first); + else + jit_stxi (offset, SP, first); + jit_stxi (offset + 4, SP, second); +} + +static void +emit_sp_ref_s64 (scm_jit_state *j, jit_gpr_t dst_lo, jit_gpr_t dst_hi, + uint32_t src) +{ + emit_sp_ref_u64 (j, dst_lo, dst_hi, src); +} + +static void +emit_sp_set_s64 (scm_jit_state *j, uint32_t dst, jit_gpr_t lo, jit_gpr_t hi) +{ + emit_sp_set_u64 (j, dst, lo, hi); +} + +static void +emit_sp_ref_u64_lower_half (scm_jit_state *j, jit_gpr_t dst, uint32_t src) +{ + size_t offset = src * 8; + + if (offset == 0) + emit_ldr (dst, SP); + else + emit_ldxi (dst, SP, offset); +} + +static void +emit_sp_ref_ptr (scm_jit_state *j, jit_gpr_t dst, uint32_t src) +{ + emit_sp_ref_u64_lower_half (j, dst, src); +} +#endif + +static void +emit_sp_ref_f64 (scm_jit_state *j, jit_gpr_t dst, uint32_t src) +{ + size_t offset = src * 8; + + if (offset == 0) + jit_ldr_d (dst, SP); + else + jit_ldxi_d (dst, SP, offset); +} + +static void +emit_sp_set_f64 (scm_jit_state *j, uint32_t dst, jit_gpr_t src) +{ + size_t offset = dst * 8; + + if (offset == 0) + jit_str_d (SP, src); + else + jit_stxi_d (offset, SP, src); +} + +static void +add_inter_instruction_patch (scm_jit_state *j, jit_node_t *label, + const uint32_t *target) +{ + abort (); +} + + + static void bad_instruction (scm_jit_state *j) { @@ -51,773 +772,2284 @@ compile_halt (scm_jit_state *j) } static void -compile_call (scm_jit_state *j, uint32_t a, uint32_t b) +compile_call (scm_jit_state *j, uint32_t proc, uint32_t nlocals) { + /* 2 = size of call inst */ + jit_node_t *mcont = emit_push_frame (j, proc, nlocals, j->ip + 2); + + emit_indirect_tail_call (j); + + jit_patch (mcont); + + j->frame_size = -1; } static void -compile_call_label (scm_jit_state *j, uint32_t a, uint32_t b, const uint32_t *vcode) +compile_call_label (scm_jit_state *j, uint32_t proc, uint32_t nlocals, const uint32_t *vcode) { + /* 2 = size of call-label inst */ + jit_node_t *mcont = emit_push_frame (j, proc, nlocals, j->ip + 3); + + emit_direct_tail_call (j, vcode); + + jit_patch (mcont); + + j->frame_size = -1; } static void compile_tail_call (scm_jit_state *j) { + emit_indirect_tail_call (j); + + j->frame_size = -1; } static void compile_tail_call_label (scm_jit_state *j, const uint32_t *vcode) { + emit_direct_tail_call (j, vcode); + + j->frame_size = -1; } static void compile_instrument_entry (scm_jit_state *j, void *data) { + if (j->hooks_enabled) + emit_run_hook (j, T0, scm_vm_intrinsics.invoke_apply_hook); } static void compile_instrument_loop (scm_jit_state *j, void *data) { + /* Nothing to do. */ } static void -compile_receive (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t b) +compile_receive (scm_jit_state *j, uint16_t dst, uint16_t proc, uint32_t nlocals) { + jit_gpr_t fp = T0, t = T1; + jit_node_t *k; + + emit_load_fp (j, fp); + k = emit_branch_if_frame_locals_count_greater_than (j, fp, t, proc); + emit_store_current_ip (j, T0); + emit_call (j, scm_vm_intrinsics.error_no_values); + jit_patch (k); + emit_fp_ref_scm (j, t, fp, proc); + emit_fp_set_scm (j, fp, dst, t); + emit_reset_frame (j, fp, nlocals); + + j->frame_size = nlocals; } static void -compile_receive_values (scm_jit_state *j, uint32_t a, uint8_t b, uint32_t c) +compile_receive_values (scm_jit_state *j, uint32_t proc, uint8_t allow_extra, + uint32_t nvalues) { + jit_gpr_t fp = T0, t = T1; + + emit_load_fp (j, fp); + if (allow_extra) + { + jit_node_t *k; + k = emit_branch_if_frame_locals_count_greater_than (j, fp, t, + proc + nvalues - 1); + emit_store_current_ip (j, T0); + emit_call (j, scm_vm_intrinsics.error_not_enough_values); + jit_patch (k); + } + else + { + jit_node_t *k; + k = emit_branch_if_frame_locals_count_eq (j, fp, t, proc + nvalues); + emit_store_current_ip (j, T0); + emit_call_i (j, scm_vm_intrinsics.error_wrong_number_of_values, nvalues); + jit_patch (k); + + j->frame_size = proc + nvalues; + } } static void compile_shuffle_down (scm_jit_state *j, uint16_t from, uint16_t to) { + jit_gpr_t fp = T0, walk = T0, t = T1; + size_t offset = (from - to) * sizeof (union scm_vm_stack_element); + jit_node_t *done, *head, *back; + + emit_load_fp (j, fp); + emit_load_fp_slot (j, walk, fp, from); + done = jit_bltr (walk, SP); + head = jit_label (); + jit_ldr (t, walk); + jit_stxi (offset, walk, t); + jit_subi (walk, walk, sizeof (union scm_vm_stack_element)); + back = jit_bltr (walk, SP); + jit_patch_at (back, head); + jit_patch (done); + jit_addi (SP, SP, offset); + emit_store_sp (j); + + if (j->frame_size >= 0) + j->frame_size -= (from - to); } static void compile_return_values (scm_jit_state *j) { + jit_gpr_t old_fp = T0, offset = T1, new_fp = T1, ra = T1; + jit_node_t *interp; + if (j->hooks_enabled) + emit_run_hook (j, T0, scm_vm_intrinsics.invoke_return_hook); + + emit_load_fp (j, old_fp); + emit_load_prev_frame_size (j, offset, old_fp); + jit_addi (offset, offset, frame_overhead_slots); + jit_lshi (offset, offset, 3); /* Multiply by sizeof (scm_vm_stack_element) */ + jit_addr (new_fp, old_fp, offset); + emit_store_fp (j, new_fp); + + emit_load_mra (j, ra, old_fp); + interp = jit_beqi (ra, 0); + jit_jmpr (ra); + + jit_patch (interp); + emit_load_vra (j, ra, old_fp); + emit_store_ip (j, ra); + jit_ret (); } static void compile_subr_call (scm_jit_state *j, uint32_t idx) { + jit_gpr_t fp = T0, t = T1, ret = T2; + void *subr; + uint32_t i; + jit_node_t *immediate, *not_values, *k; + + if (j->frame_size < 0) + abort (); + + subr = scm_subr_function_by_index (idx); + emit_store_current_ip (j, t); + emit_load_fp (j, fp); + jit_prepare (); + for (i = 0; i < j->frame_size; i++) + { + emit_fp_ref_scm (j, t, fp, i); + jit_pushargr (t); + } + jit_finishi (subr); + jit_retval (ret); + + emit_load_fp (j, fp); + + immediate = emit_branch_if_immediate (j, ret); + not_values = emit_branch_if_heap_object_not_tc7 (j, ret, t, scm_tc7_values); + emit_call_r_r (j, scm_vm_intrinsics.unpack_values_object, THREAD, ret); + emit_reload_sp (j); + k = jit_jmpi (); + + jit_patch (immediate); + jit_patch (not_values); + emit_subtract_stack_slots (j, SP, fp, 1); + emit_store_sp (j); + jit_str (SP, ret); + jit_patch (k); } static void -compile_foreign_call (scm_jit_state *j, uint16_t a, uint16_t b) +compile_foreign_call (scm_jit_state *j, uint16_t cif_idx, uint16_t ptr_idx) { + emit_store_current_ip (j, T0); + emit_load_fp (j, T0); + emit_fp_ref_scm (j, T0, T0, 0); + emit_free_variable_ref (j, T1, T0, cif_idx); + emit_free_variable_ref (j, T2, T0, ptr_idx); + + /* FIXME: Inline the foreign call. */ + emit_call_r_r_r (j, scm_vm_intrinsics.foreign_call, THREAD, T1, T2); + emit_reload_sp (j); + + j->frame_size = 2; /* Return value and errno. */ } static void -compile_continuation_call (scm_jit_state *j, uint32_t a) +compile_continuation_call (scm_jit_state *j, uint32_t contregs_idx) { + emit_store_current_ip (j, T0); + emit_load_fp (j, T0); + emit_fp_ref_scm (j, T0, T0, 0); + emit_free_variable_ref (j, T0, T0, contregs_idx); + emit_call_r_r (j, scm_vm_intrinsics.reinstate_continuation_x, THREAD, T0); + /* Does not fall through. */ + + j->frame_size = -1; } static void -compile_compose_continuation (scm_jit_state *j, uint32_t a) +compile_compose_continuation (scm_jit_state *j, uint32_t cont_idx) { + jit_node_t *interp; + + emit_store_current_ip (j, T0); + emit_load_fp (j, T0); + emit_fp_ref_scm (j, T0, T0, 0); + emit_free_variable_ref (j, T0, T0, cont_idx); + emit_call_r_r (j, scm_vm_intrinsics.compose_continuation, THREAD, T0); + jit_retval (T0); + emit_reload_sp (j); + interp = jit_bnei (T0, 0); + jit_jmpr (T0); + + jit_patch (interp); + jit_ret (); + + j->frame_size = -1; } static void compile_capture_continuation (scm_jit_state *j, uint32_t dst) { + emit_store_current_ip (j, T0); + emit_call_r (j, scm_vm_intrinsics.capture_continuation, THREAD); + jit_retval (T0); + emit_sp_set_scm (j, dst, T0); } static void compile_abort (scm_jit_state *j) { + jit_node_t *k, *interp; + + jit_movi (T0, (intptr_t) (j->ip + 1)); + emit_store_ip (j, T0); + k = jit_movi (T0, 0); + emit_call_r_r (j, scm_vm_intrinsics.abort_to_prompt, THREAD, T0); + jit_retval (T3_PRESERVED); + + if (j->hooks_enabled) + emit_run_hook (j, T0, scm_vm_intrinsics.invoke_abort_hook); + + interp = jit_bnei (T3_PRESERVED, 0); + emit_reload_sp (j); + jit_jmpr (T3_PRESERVED); + + jit_patch (interp); + jit_ret (); + + jit_patch (k); + + j->frame_size = -1; } static void -compile_builtin_ref (scm_jit_state *j, uint16_t dst, uint16_t a) +compile_builtin_ref (scm_jit_state *j, uint16_t dst, uint16_t idx) { + SCM builtin = scm_vm_builtin_ref (idx); + + jit_movi (T0, SCM_UNPACK (builtin)); + emit_sp_set_scm (j, dst, T0); } static void -compile_throw (scm_jit_state *j, uint16_t a, uint16_t b) +compile_throw (scm_jit_state *j, uint16_t key, uint16_t args) { + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, key); + emit_sp_ref_scm (j, T1, args); + emit_call_r_r (j, scm_vm_intrinsics.throw_, T0, T1); + /* throw_ does not return. */ } static void -compile_throw_value (scm_jit_state *j, uint32_t a, const void *data) +compile_throw_value (scm_jit_state *j, uint32_t val, + const void *key_subr_and_message) { + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, val); + emit_call_r_i (j, scm_vm_intrinsics.throw_with_value, T0, + (intptr_t) key_subr_and_message); + /* throw_with_value does not return. */ } static void -compile_throw_value_and_data (scm_jit_state *j, uint32_t a, const void *data) +compile_throw_value_and_data (scm_jit_state *j, uint32_t val, + const void *key_subr_and_message) { + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, val); + emit_call_r_i (j, scm_vm_intrinsics.throw_with_value_and_data, T0, + (intptr_t) key_subr_and_message); + /* throw_with_value_and_data does not return. */ } static void -compile_assert_nargs_ee (scm_jit_state *j, uint32_t a) +compile_assert_nargs_ee (scm_jit_state *j, uint32_t nlocals) { + jit_node_t *k; + jit_gpr_t fp = T0, t = T1; + + emit_load_fp (j, fp); + k = emit_branch_if_frame_locals_count_eq (j, fp, t, nlocals); + emit_store_current_ip (j, t); + emit_call_r (j, scm_vm_intrinsics.error_wrong_num_args, THREAD); + jit_patch (k); + + j->frame_size = nlocals; } static void -compile_assert_nargs_ge (scm_jit_state *j, uint32_t a) +compile_assert_nargs_ge (scm_jit_state *j, uint32_t nlocals) { + if (nlocals > 0) + { + jit_gpr_t fp = T0, t = T1; + jit_node_t *k; + emit_load_fp (j, fp); + k = emit_branch_if_frame_locals_count_greater_than (j, fp, t, nlocals-1); + emit_store_current_ip (j, t); + emit_call_r (j, scm_vm_intrinsics.error_wrong_num_args, THREAD); + jit_patch (k); + } } static void -compile_assert_nargs_le (scm_jit_state *j, uint32_t a) +compile_assert_nargs_le (scm_jit_state *j, uint32_t nlocals) { + jit_node_t *k; + jit_gpr_t fp = T0, t = T1; + + emit_load_fp (j, fp); + k = emit_branch_if_frame_locals_count_less_than (j, fp, t, nlocals + 1); + emit_store_current_ip (j, t); + emit_call_r (j, scm_vm_intrinsics.error_wrong_num_args, THREAD); + jit_patch (k); } static void -compile_alloc_frame (scm_jit_state *j, uint32_t a) +compile_alloc_frame (scm_jit_state *j, uint32_t nlocals) { + jit_gpr_t fp = T0, t = T1; + emit_load_fp (j, fp); + if (j->frame_size < 0) + jit_movr (T3_PRESERVED, SP); + emit_alloc_frame (j, fp, t, nlocals); + + if (j->frame_size >= 0) + { + int32_t slots = nlocals - j->frame_size; + + if (slots > 0) + { + jit_movi (T0, SCM_UNPACK (SCM_UNDEFINED)); + while (slots-- > 0) + emit_sp_set_scm (j, slots, T0); + } + } + else + { + jit_node_t *head, *k, *back; + jit_movi (T0, SCM_UNPACK (SCM_UNDEFINED)); + k = jit_bler (T3_PRESERVED, SP); + head = jit_label (); + jit_str (T3_PRESERVED, T0); + jit_subi (T3_PRESERVED, T3_PRESERVED, sizeof (union scm_vm_stack_element)); + back = jit_bner (T3_PRESERVED, SP); + jit_patch_at (back, head); + jit_patch (k); + } + + j->frame_size = nlocals; } static void -compile_reset_frame (scm_jit_state *j, uint32_t a) +compile_reset_frame (scm_jit_state *j, uint32_t nlocals) { + jit_gpr_t fp = T0; + emit_load_fp (j, fp); + emit_reset_frame (j, fp, nlocals); + + j->frame_size = nlocals; } static void -compile_push (scm_jit_state *j, uint32_t a) +compile_push (scm_jit_state *j, uint32_t src) { + jit_gpr_t fp = T0, t = T1; + emit_load_fp (j, fp); + jit_subi (SP, SP, sizeof (union scm_vm_stack_element)); + emit_alloc_frame_for_sp (j, fp, t); + emit_mov (j, 0, src + 1, t); + + if (j->frame_size >= 0) + j->frame_size++; } static void compile_pop (scm_jit_state *j, uint32_t dst) { + emit_mov (j, dst + 1, 0, T0); + jit_addi (SP, SP, sizeof (union scm_vm_stack_element)); + emit_store_sp (j); + + if (j->frame_size >= 0) + j->frame_size--; } static void -compile_drop (scm_jit_state *j, uint32_t a) +compile_drop (scm_jit_state *j, uint32_t nvalues) { + jit_addi (SP, SP, nvalues * sizeof (union scm_vm_stack_element)); + emit_store_sp (j); + + if (j->frame_size >= 0) + j->frame_size -= nvalues; } static void -compile_assert_nargs_ee_locals (scm_jit_state *j, uint16_t a, uint16_t b) +compile_assert_nargs_ee_locals (scm_jit_state *j, uint16_t expected, + uint16_t nlocals) { + compile_assert_nargs_ee (j, expected); + compile_alloc_frame (j, expected + nlocals); } static void compile_expand_apply_argument (scm_jit_state *j) { + emit_store_current_ip (j, T0); + emit_call_r (j, scm_vm_intrinsics.expand_apply_argument, THREAD); + emit_reload_sp (j); + + j->frame_size = -1; } static void -compile_bind_kwargs (scm_jit_state *j, uint32_t a, uint8_t b, uint32_t c, uint32_t d, const void *data) +compile_bind_kwargs (scm_jit_state *j, uint32_t nreq, uint8_t flags, + uint32_t nreq_and_opt, uint32_t ntotal, const void *kw) { + uint8_t allow_other_keys = flags & 0x1, has_rest = flags & 0x2; + jit_gpr_t t = T0, npositional = T1, fp = T1; + + emit_store_current_ip (j, t); + + jit_prepare (); + jit_pushargr (THREAD); + jit_pushargi (nreq); + jit_pushargi (nreq_and_opt - nreq); + jit_finishi (scm_vm_intrinsics.compute_kwargs_npositional); + jit_retval_i (npositional); + + jit_prepare (); + jit_pushargr (THREAD); + jit_pushargr (npositional); + jit_pushargi (ntotal); + jit_pushargi ((intptr_t) kw); + jit_pushargi (!has_rest); + jit_pushargi (allow_other_keys); + jit_finishi (scm_vm_intrinsics.bind_kwargs); + + emit_reload_sp (j); + + if (has_rest) + { + emit_call_r_i (j, scm_vm_intrinsics.cons_rest, THREAD, ntotal); + jit_retval (t); + emit_load_fp (j, fp); + emit_fp_set_scm (j, fp, nreq_and_opt, t); + } + else + emit_load_fp (j, fp); + + emit_reset_frame (j, T1, ntotal); + j->frame_size = ntotal; } static void compile_bind_rest (scm_jit_state *j, uint32_t dst) { + jit_node_t *k, *cons; + jit_gpr_t fp = T0, t = T1; + + emit_load_fp (j, fp); + cons = emit_branch_if_frame_locals_count_greater_than (j, fp, t, dst); + + compile_alloc_frame (j, dst + 1); + jit_movi (t, SCM_UNPACK (SCM_EOL)); + emit_sp_set_scm (j, 0, t); + k = jit_jmpi (); + + jit_patch (cons); + emit_store_current_ip (j, t); + emit_call_r_i (j, scm_vm_intrinsics.cons_rest, THREAD, dst); + jit_retval (t); + emit_sp_set_scm (j, 0, t); + compile_reset_frame (j, dst + 1); + + jit_patch (k); } static void -compile_allocate_words (scm_jit_state *j, uint16_t dst, uint16_t a) +compile_allocate_words (scm_jit_state *j, uint16_t dst, uint16_t nwords) { + jit_gpr_t t = T0; + + emit_store_current_ip (j, t); + emit_sp_ref_sz (j, t, nwords); + emit_call_r_r (j, scm_vm_intrinsics.allocate_words, THREAD, t); + jit_retval (t); + emit_sp_set_scm (j, dst, t); } static void -compile_allocate_words_immediate (scm_jit_state *j, uint16_t dst, uint16_t a) +compile_allocate_words_immediate (scm_jit_state *j, uint16_t dst, uint16_t nwords) { + jit_gpr_t t = T0; + + emit_store_current_ip (j, t); + jit_movi (t, nwords); + emit_call_r_r (j, scm_vm_intrinsics.allocate_words, THREAD, t); + jit_retval (t); + emit_sp_set_scm (j, dst, t); } static void -compile_scm_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_scm_ref (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_sz (j, T1, idx); + jit_lshi (T1, T1, log2_sizeof_uintptr_t); + jit_ldxr (T0, T0, T1); + emit_sp_set_scm (j, dst, T0); } static void -compile_scm_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_scm_set (scm_jit_state *j, uint8_t obj, uint8_t idx, uint8_t val) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_sz (j, T1, idx); + emit_sp_ref_scm (j, T2, val); + jit_lshi (T1, T1, log2_sizeof_uintptr_t); + jit_stxr (T0, T1, T2); } static void -compile_scm_ref_tag (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_scm_ref_tag (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t tag) { + emit_sp_ref_scm (j, T0, obj); + jit_ldr (T0, T0); + jit_subi (T0, T0, tag); + emit_sp_set_scm (j, dst, T0); } static void -compile_scm_set_tag (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_scm_set_tag (scm_jit_state *j, uint8_t obj, uint8_t tag, uint8_t val) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_scm (j, T1, val); + jit_addi (T1, T1, tag); + jit_str (T0, T1); } static void -compile_scm_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_scm_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx) { + emit_sp_ref_scm (j, T0, obj); + jit_ldxi (T0, T0, idx * sizeof (SCM)); + emit_sp_set_scm (j, dst, T0); } static void -compile_scm_set_immediate (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_scm_set_immediate (scm_jit_state *j, uint8_t obj, uint8_t idx, uint8_t val) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_scm (j, T1, val); + jit_stxi (idx * sizeof (SCM), T0, T1); } static void -compile_word_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_word_ref (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_sz (j, T1, idx); + jit_lshi (T1, T1, log2_sizeof_uintptr_t); + jit_ldxr (T0, T0, T1); + emit_sp_set_sz (j, dst, T0); } static void -compile_word_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_word_set (scm_jit_state *j, uint8_t obj, uint8_t idx, uint8_t val) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_sz (j, T1, idx); + emit_sp_ref_sz (j, T2, val); + jit_lshi (T1, T1, log2_sizeof_uintptr_t); + jit_stxr (T0, T1, T2); } static void -compile_word_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_word_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx) { + emit_sp_ref_scm (j, T0, obj); + jit_ldxi (T0, T0, idx * sizeof (SCM)); + emit_sp_set_sz (j, dst, T0); } static void -compile_word_set_immediate (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_word_set_immediate (scm_jit_state *j, uint8_t obj, uint8_t idx, uint8_t val) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_sz (j, T1, val); + jit_stxi (idx * sizeof (SCM), T0, T1); } static void -compile_pointer_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_pointer_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx) { + emit_sp_ref_scm (j, T0, obj); + jit_ldxi (T0, T0, idx * sizeof (SCM)); + emit_sp_set_scm (j, dst, T0); } static void -compile_pointer_set_immediate (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_pointer_set_immediate (scm_jit_state *j, uint8_t obj, uint8_t idx, uint8_t val) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_scm (j, T1, val); + jit_stxi (idx * sizeof (SCM), T0, T1); } static void -compile_tail_pointer_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_tail_pointer_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx) { + emit_sp_ref_scm (j, T0, obj); + jit_addi (T0, T0, idx * sizeof (SCM)); + emit_sp_set_scm (j, dst, T0); } static void -compile_mov (scm_jit_state *j, uint16_t dst, uint16_t a) +compile_mov (scm_jit_state *j, uint16_t dst, uint16_t src) { + emit_mov (j, dst, src, T0); } static void -compile_long_mov (scm_jit_state *j, uint32_t dst, uint32_t a) +compile_long_mov (scm_jit_state *j, uint32_t dst, uint32_t src) { + emit_mov (j, dst, src, T0); } static void -compile_long_fmov (scm_jit_state *j, uint32_t dst, uint32_t a) +compile_long_fmov (scm_jit_state *j, uint32_t dst, uint32_t src) { + jit_gpr_t fp = T0, t = T1; + emit_load_fp (j, fp); + emit_fp_ref_scm (j, t, fp, src); + emit_fp_set_scm (j, fp, dst, t); } static void -compile_call_scm_from_scm_scm (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b, uint32_t c) +compile_call_scm_from_scm_scm (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + emit_sp_ref_scm (j, T1, b); + emit_call_r_r (j, intrinsic, T0, T1); + jit_retval (T0); + emit_reload_sp (j); + emit_sp_set_scm (j, dst, T0); } static void -compile_call_scm_from_scm_uimm (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b, uint32_t c) +compile_call_scm_from_scm_uimm (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + jit_prepare (); + jit_pushargr (T0); + jit_pushargi (b); + jit_finishi (intrinsic); + jit_retval (T0); + emit_reload_sp (j); + emit_sp_set_scm (j, dst, T0); } static void -compile_call_scm_sz_u32 (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c, uint32_t d) +compile_call_scm_sz_u32 (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + emit_sp_ref_sz (j, T1, b); + emit_sp_ref_sz (j, T2, c); + jit_prepare (); + jit_pushargr (T0); + jit_pushargr (T1); + jit_pushargr (T2); + jit_finishi (intrinsic); + emit_reload_sp (j); } static void -compile_call_scm_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t b) +compile_call_scm_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + jit_prepare (); + jit_pushargr (T0); + jit_finishi (intrinsic); + jit_retval (T0); + emit_reload_sp (j); + emit_sp_set_scm (j, dst, T0); } static void -compile_call_f64_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t b) +compile_call_f64_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + jit_prepare (); + jit_pushargr (T0); + jit_finishi (intrinsic); + jit_retval (JIT_F0); + emit_reload_sp (j); + emit_sp_set_f64 (j, dst, JIT_F0); } static void -compile_call_u64_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t b) +compile_call_u64_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); +#if INDIRECT_INT64_INTRINSICS + jit_prepare (); + jit_addi (T1, SP, dst * sizeof (union scm_vm_stack_element)); + jit_pushargr (T1); + jit_pushargr (T0); + jit_finishi (intrinsic); +#else + jit_prepare (); + jit_pushargr (T0); + jit_finishi (intrinsic); + jit_retval (T0); + emit_sp_set_u64 (j, dst, T0); +#endif } static void compile_make_short_immediate (scm_jit_state *j, uint8_t dst, SCM a) { + jit_movi (T0, SCM_UNPACK (a)); + emit_sp_set_scm (j, dst, T0); } static void compile_make_long_immediate (scm_jit_state *j, uint32_t dst, SCM a) { + jit_movi (T0, SCM_UNPACK (a)); + emit_sp_set_scm (j, dst, T0); } static void compile_make_long_long_immediate (scm_jit_state *j, uint32_t dst, SCM a) { + jit_movi (T0, SCM_UNPACK (a)); + emit_sp_set_scm (j, dst, T0); } static void compile_make_non_immediate (scm_jit_state *j, uint32_t dst, const void *data) { + jit_movi (T0, (uintptr_t)data); + emit_sp_set_scm (j, dst, T0); } static void compile_static_ref (scm_jit_state *j, uint32_t dst, void *loc) { + jit_ldi (T0, loc); + emit_sp_set_scm (j, dst, T0); } static void -compile_static_set (scm_jit_state *j, uint32_t a, void *loc) +compile_static_set (scm_jit_state *j, uint32_t obj, void *loc) { + emit_sp_ref_scm (j, T0, obj); + jit_sti (loc, T0); } static void compile_static_patch (scm_jit_state *j, void *dst, const void *src) { + jit_movi (T0, (uintptr_t) src); + jit_sti (dst, T0); } static void -compile_prompt (scm_jit_state *j, uint32_t a, uint8_t b, uint32_t c, const uint32_t *vcode) +compile_prompt (scm_jit_state *j, uint32_t tag, uint8_t escape_only_p, + uint32_t proc_slot, const uint32_t *vcode) { + jit_node_t *mra; + emit_store_current_ip (j, T0); + jit_prepare (); + jit_pushargr (THREAD); + jit_pushargi (escape_only_p); + emit_sp_ref_scm (j, T0, tag); + jit_pushargr (T0); + emit_load_fp (j, T1); + jit_subi (T1, T1, proc_slot * sizeof (union scm_vm_stack_element)); + jit_pushargr (T1); + jit_pushargi ((uintptr_t) vcode); + mra = jit_movi (T2, 0); + jit_finishi (scm_vm_intrinsics.push_prompt); + add_inter_instruction_patch (j, mra, vcode); } static void compile_load_label (scm_jit_state *j, uint32_t dst, const uint32_t *vcode) { + jit_movi (T0, (uintptr_t) vcode); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_set_u64 (j, dst, T0); +#else + jit_movi (T1, 0); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void -compile_call_s64_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t b) +compile_call_s64_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t idx) { + compile_call_u64_from_scm (j, dst, a, idx); } static void -compile_call_scm_from_u64 (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t b) +compile_call_scm_from_u64 (scm_jit_state *j, uint16_t dst, uint16_t src, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + jit_prepare (); +#if INDIRECT_INT64_INTRINSICS + jit_addi (T0, SP, src * sizeof (union scm_vm_stack_element)); +#else + emit_sp_ref_u64 (j, T0, src); + jit_pushargr (T0); +#endif + jit_finishi (intrinsic); + jit_retval (T0); + emit_sp_set_scm (j, dst, T0); } static void compile_call_scm_from_s64 (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t b) { + compile_call_scm_from_u64 (j, dst, a, b); } static void -compile_tag_char (scm_jit_state *j, uint16_t dst, uint16_t a) +compile_tag_char (scm_jit_state *j, uint16_t dst, uint16_t src) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, src); +#else + emit_sp_ref_u64_lower_half (j, T0, src); +#endif + jit_lshi (T0, T0, 8); + jit_addi (T0, T0, scm_tc8_char); + emit_sp_set_scm (j, dst, T0); } static void -compile_untag_char (scm_jit_state *j, uint16_t dst, uint16_t a) +compile_untag_char (scm_jit_state *j, uint16_t dst, uint16_t src) { + emit_sp_ref_scm (j, T0, src); + jit_rshi (T0, T0, 8); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_set_u64 (j, dst, T0); +#else + jit_movi (T1, 0); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void -compile_atomic_ref_scm_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_atomic_ref_scm_immediate (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t offset) { + emit_sp_ref_scm (j, T0, obj); + jit_addi (T0, T0, offset * sizeof (SCM)); + emit_call_r (j, scm_vm_intrinsics.atomic_ref_scm, T0); + jit_retval (T0); + emit_sp_set_scm (j, dst, T0); } static void -compile_atomic_set_scm_immediate (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_atomic_set_scm_immediate (scm_jit_state *j, uint8_t obj, uint8_t offset, uint8_t val) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_scm (j, T1, val); + jit_addi (T0, T0, offset * sizeof (SCM)); + emit_call_r_r (j, scm_vm_intrinsics.atomic_set_scm, T0, T1); } static void -compile_atomic_scm_swap_immediate (scm_jit_state *j, uint32_t dst, uint32_t a, uint8_t b, uint32_t c) +compile_atomic_scm_swap_immediate (scm_jit_state *j, uint32_t dst, uint32_t obj, uint8_t offset, uint32_t val) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_scm (j, T1, val); + jit_addi (T0, T0, offset * sizeof (SCM)); + emit_call_r_r (j, scm_vm_intrinsics.atomic_swap_scm, T0, T1); + jit_retval (T0); + emit_sp_set_scm (j, dst, T0); } static void -compile_atomic_scm_compare_and_swap_immediate (scm_jit_state *j, uint32_t dst, uint32_t a, uint8_t b, uint32_t c, uint32_t d) +compile_atomic_scm_compare_and_swap_immediate (scm_jit_state *j, uint32_t dst, + uint32_t obj, uint8_t offset, + uint32_t expected, uint32_t desired) { + emit_sp_ref_scm (j, T0, obj); + emit_sp_ref_scm (j, T1, expected); + emit_sp_ref_scm (j, T2, desired); + jit_addi (T0, T0, offset * sizeof (SCM)); + emit_call_r_r_r (j, scm_vm_intrinsics.atomic_swap_scm, T0, T1, T3); + jit_retval (T0); + emit_sp_set_scm (j, dst, T0); } static void -compile_call_thread_scm_scm (scm_jit_state *j, uint16_t a, uint16_t b, uint32_t c) +compile_call_thread_scm_scm (scm_jit_state *j, uint16_t a, uint16_t b, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + emit_sp_ref_scm (j, T1, b); + emit_call_r_r_r (j, intrinsic, THREAD, T0, T1); + emit_reload_sp (j); } static void -compile_call_thread (scm_jit_state *j, uint32_t a) +compile_call_thread (scm_jit_state *j, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_call_r (j, intrinsic, THREAD); + emit_reload_sp (j); } static void -compile_call_scm_from_thread_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t b) +compile_call_scm_from_thread_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + emit_call_r_r (j, intrinsic, THREAD, T0); + jit_retval (T0); + emit_reload_sp (j); + emit_sp_set_scm (j, dst, T0); } static void -compile_call_thread_scm (scm_jit_state *j, uint32_t a, uint32_t b) +compile_call_thread_scm (scm_jit_state *j, uint32_t a, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + emit_call_r_r (j, intrinsic, THREAD, T0); + emit_reload_sp (j); } static void -compile_call_scm_from_scm_u64 (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b, uint32_t c) +compile_call_scm_from_scm_u64 (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + jit_prepare (); + jit_pushargr (T0); +#if INDIRECT_INT64_INTRINSICS + jit_addi (T1, SP, b * sizeof (union scm_vm_stack_element)); +#else + emit_sp_ref_u64 (j, T1, b); + jit_pushargr (T1); +#endif + jit_finishi (intrinsic); + jit_retval (T0); + emit_reload_sp (j); + emit_sp_set_scm (j, dst, T0); } static void -compile_call_scm_from_thread (scm_jit_state *j, uint32_t dst, uint32_t a) +compile_call_scm_from_thread (scm_jit_state *j, uint32_t dst, uint32_t idx) { + void *intrinsic = ((void **) &scm_vm_intrinsics)[idx]; + + emit_store_current_ip (j, T0); + emit_call_r (j, intrinsic, THREAD); + jit_retval (T0); + emit_reload_sp (j); + emit_sp_set_scm (j, dst, T0); } static void compile_fadd (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { + emit_sp_ref_f64 (j, JIT_F0, a); + emit_sp_ref_f64 (j, JIT_F1, b); + jit_addr_d (JIT_F0, JIT_F0, JIT_F1); + emit_sp_set_f64 (j, dst, JIT_F0); } static void compile_fsub (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { + emit_sp_ref_f64 (j, JIT_F0, a); + emit_sp_ref_f64 (j, JIT_F1, b); + jit_subr_d (JIT_F0, JIT_F0, JIT_F1); + emit_sp_set_f64 (j, dst, JIT_F0); } static void compile_fmul (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { + emit_sp_ref_f64 (j, JIT_F0, a); + emit_sp_ref_f64 (j, JIT_F1, b); + jit_mulr_d (JIT_F0, JIT_F0, JIT_F1); + emit_sp_set_f64 (j, dst, JIT_F0); } static void compile_fdiv (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { + emit_sp_ref_f64 (j, JIT_F0, a); + emit_sp_ref_f64 (j, JIT_F1, b); + jit_divr_d (JIT_F0, JIT_F0, JIT_F1); + emit_sp_set_f64 (j, dst, JIT_F0); } static void compile_uadd (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + jit_addr (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, VT3, b); + jit_addcr (T0, T0, T2); + jit_addxr (T1, T1, T3); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_usub (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + jit_subr (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, VT3, b); + jit_subcr (T0, T0, T2); + jit_subxr (T1, T1, T3); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_umul (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + jit_mulr (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + /* FIXME: This is untested! */ + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, VT3, b); + jit_mulr (T1, T1, T2); /* High A times low B */ + jit_mulr (VT3, VT3, T0); /* High B times low A */ + jit_addr (T1, T1, VT3); /* Add high results, throw away overflow */ + jit_qmulr_u (T0, T2, T0, T2); /* Low A times low B */ + jit_addr (T1, T1, T2); /* Add high result of low product */ + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_uadd_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + jit_addi (T0, T0, a); + emit_sp_set_u64 (j, dst, T0); +#else + emit_sp_ref_u64 (j, T0, T1, a); + jit_addci (T0, T0, a); + jit_addxi (T1, T1, 0); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_usub_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + jit_subi (T0, T0, a); + emit_sp_set_u64 (j, dst, T0); +#else + emit_sp_ref_u64 (j, T0, T1, a); + jit_subci (T0, T0, a); + jit_subxi (T1, T1, 0); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_umul_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + jit_muli (T0, T0, b); + emit_sp_set_u64 (j, dst, T0); +#else + /* FIXME: This is untested! */ + emit_sp_ref_u64 (j, T0, T1, a); + jit_muli (T1, T1, b); /* High A times low B */ + /* High B times low A is 0. */ + jit_movi (T2, b); + jit_qmulr_u (T0, T2, T0, T2); /* Low A times low B */ + jit_addr (T1, T1, T2); /* Add high result of low product */ + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_load_f64 (scm_jit_state *j, uint32_t dst, double a) { + jit_movi_d (JIT_F0, a); + emit_sp_set_f64 (j, dst, JIT_F0); } static void compile_load_u64 (scm_jit_state *j, uint32_t dst, uint64_t a) { +#if SIZEOF_UINTPTR_T >= 8 + jit_movi (T0, a); + emit_sp_set_u64 (j, dst, T0); +#else + jit_movi (T0, a & 0xffffffff); + jit_movi (T1, a >> 32); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_load_s64 (scm_jit_state *j, uint32_t dst, int64_t a) { + compile_load_u64 (j, dst, a); } static void compile_current_thread (scm_jit_state *j, uint32_t dst) { + jit_ldxr (T0, THREAD, thread_offset_handle); + emit_sp_set_scm (j, dst, T0); } static void compile_ulogand (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + jit_andr (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, T3, b); + jit_andr (T0, T0, T2); + jit_andr (T1, T1, T3); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_ulogior (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + jit_orr (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, T3, b); + jit_orr (T0, T0, T2); + jit_orr (T1, T1, T3); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_ulogsub (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + jit_comr (T1, T1); + jit_andr (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, T3, b); + jit_comr (T2, T2); + jit_comr (T3, T3); + jit_andr (T0, T0, T2); + jit_andr (T1, T1, T3); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_ursh (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + jit_andi (T1, T1, 63); + jit_rshr_u (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + /* FIXME: Not tested. */ + jit_node_t *zero, *both, *done; + + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, T3, b); + jit_andi (T2, T2, 63); + zero = jit_beqi (T2, 0); + both = jit_blti (T2, 32); + + /* 32 <= s < 64: hi = 0, lo = hi >> (s-32) */ + jit_subi (T2, 32); + jit_rshr_u (T0, T1, T2); + jit_movi (T1, 0); + done = jit_jmpi (); + + jit_patch (both); + /* 0 < s < 32: hi = hi >> s, lo = lo >> s + hi << (32-s) */ + jit_negr (T3, T2); + jit_addi (T3, T3, 32); + jit_lshr (T3, T1, T3); + jit_rshr_u (T1, T1, T2); + jit_rshr_u (T0, T0, T2); + jit_addr (T0, T0, T3); + + jit_patch (done); + jit_patch (zero); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_ulsh (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + jit_andi (T1, T1, 63); + jit_lshr (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + /* FIXME: Not tested. */ + jit_node_t *zero, *both, *done; + + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, T3, b); + jit_andi (T2, T2, 63); + zero = jit_beqi (T2, 0); + both = jit_blti (T2, 32); + + /* 32 <= s < 64: hi = lo << (s-32), lo = 0 */ + jit_subi (T2, 32); + jit_lshr (T1, T0, T2); + jit_movi (T0, 0); + done = jit_jmpi (); + + jit_patch (both); + /* 0 < s < 32: hi = hi << s + lo >> (32-s), lo = lo << s */ + jit_negr (T3, T2); + jit_addi (T3, T3, 32); + jit_rshr_u (T3, T0, T3); + jit_lshr (T1, T1, T2); + jit_lshr (T0, T0, T2); + jit_addr (T1, T1, T3); + + jit_patch (done); + jit_patch (zero); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_ursh_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { + b &= 63; + +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + jit_rshi_u (T0, T0, b); + emit_sp_set_u64 (j, dst, T0); +#else + /* FIXME: Not tested. */ + emit_sp_ref_u64 (j, T0, T1, a); + if (b == 0) + { + /* Nothing to do. */ + } + else if (b >= 32) + { + /* hi = 0, lo = hi >> (s-32) */ + jit_rshi_u (T0, T1, b - 32); + jit_movi (T1, 0); + } + else + { + /* 0 < s < 32: hi = hi >> s, lo = lo >> s + hi << (32-s) */ + jit_lshi (T2, T1, 32 - b); + jit_rshi_u (T1, T1, b); + jit_rshi_u (T0, T0, b); + jit_addr (T0, T0, T2); + } + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_ulsh_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { + b &= 63; + +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + jit_lshi (T0, T0, b); + emit_sp_set_u64 (j, dst, T0); +#else + /* FIXME: Not tested. */ + emit_sp_ref_u64 (j, T0, T1, a); + if (b == 0) + { + /* Nothing to do. */ + } + else if (b >= 32) + { + /* hi = lo << (s-32), lo = 0 */ + jit_lshr (T1, T0, b - 32); + jit_movi (T0, 0); + } + else + { + /* hi = hi << s + lo >> (32-s), lo = lo << s */ + jit_rshi_u (T2, T0, 32 - b); + jit_lshi (T1, T1, b); + jit_lshi (T0, T0, b); + jit_addr (T1, T1, T2); + } + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_ulogxor (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + jit_xorr (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, T3, b); + jit_xorr (T0, T0, T2); + jit_xorr (T1, T1, T3); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void compile_handle_interrupts (scm_jit_state *j) { + jit_node_t *again, *mra, *none_pending, *blocked; + + again = jit_label (); + jit_addi (T0, THREAD, thread_offset_pending_asyncs); + emit_call_r (j, scm_vm_intrinsics.atomic_ref_scm, T0); + jit_retval (T0); + none_pending = jit_beqi (T0, SCM_UNPACK (SCM_EOL)); + jit_ldxi_i (T0, THREAD, thread_offset_block_asyncs); + blocked = jit_beqi (T0, 0); + + emit_store_current_ip (j, T0); + mra = jit_movi (T0, 0); + jit_patch_at (mra, again); + emit_call_r_r (j, scm_vm_intrinsics.push_interrupt_frame, THREAD, T0); + emit_reload_sp (j); + emit_direct_tail_call (j, scm_vm_intrinsics.handle_interrupt_code); + + jit_patch (none_pending); + jit_patch (blocked); } static void compile_return_from_interrupt (scm_jit_state *j) { + jit_gpr_t old_fp = T0, offset = T1, new_fp = T1, ra = T1; + jit_node_t *interp; + + if (j->hooks_enabled) + emit_run_hook (j, T0, scm_vm_intrinsics.invoke_return_hook); + + emit_load_fp (j, old_fp); + emit_load_prev_frame_size (j, offset, old_fp); + jit_addi (offset, offset, frame_overhead_slots); + jit_lshi (offset, offset, 3); /* Multiply by sizeof (scm_vm_stack_element) */ + jit_addr (new_fp, old_fp, offset); + emit_store_fp (j, new_fp); + + emit_load_mra (j, ra, old_fp); + interp = jit_beqi (ra, 0); + jit_addi (SP, old_fp, frame_overhead_slots * sizeof (union scm_vm_stack_element)); + emit_store_sp (j); + jit_jmpr (ra); + + jit_patch (interp); + emit_load_vra (j, ra, old_fp); + emit_store_ip (j, ra); + jit_addi (SP, old_fp, frame_overhead_slots * sizeof (union scm_vm_stack_element)); + emit_store_sp (j); + jit_ret (); } static void compile_u64_numerically_equal (scm_jit_state *j, uint16_t a, uint16_t b) { +#if SIZEOF_UINTPTR_T >= 8 + jit_node_t *k; + jit_movi (T2, SCM_F_COMPARE_NONE); + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + k = jit_bner (T0, T1); + jit_movi (T2, SCM_F_COMPARE_EQUAL); + jit_patch (k); +#else + jit_node_t *k1, *k2, *k3; + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, T3, b); + k1 = jit_bner (T0, T2); + k2 = jit_bner (T1, T3); + jit_movi (T2, SCM_F_COMPARE_EQUAL); + k3 = jit_jmpi (); + jit_patch (k1); + jit_patch (k2); + jit_movi (T2, SCM_F_COMPARE_NONE); +#endif + emit_store_compare_result (j, T2); } static void compile_u64_less (scm_jit_state *j, uint16_t a, uint16_t b) { +#if SIZEOF_UINTPTR_T >= 8 + jit_node_t *k; + jit_movi (T2, SCM_F_COMPARE_NONE); + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + k = jit_bger_u (T0, T1); + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + jit_patch (k); +#else + jit_node_t *k1, *k2, *less, *k3; + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, T3, b); + less = jit_bltr_u (T1, T3); + k1 = jit_bner (T1, T3); + k2 = jit_bger_u (T0, T2); + jit_patch (less); + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + k3 = jit_jmpi (); + jit_patch (k1); + jit_patch (k2); + jit_movi (T2, SCM_F_COMPARE_NONE); + jit_patch (k3); +#endif + emit_store_compare_result (j, T2); } static void compile_s64_numerically_equal (scm_jit_state *j, uint16_t a, uint16_t b) { + compile_u64_numerically_equal (j, a, b); } static void compile_s64_less (scm_jit_state *j, uint16_t a, uint16_t b) { +#if SIZEOF_UINTPTR_T >= 8 + jit_node_t *k; + jit_movi (T2, SCM_F_COMPARE_NONE); + emit_sp_ref_u64 (j, T0, a); + emit_sp_ref_u64 (j, T1, b); + k = jit_bger (T0, T1); + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + jit_patch (k); +#else + jit_node_t *k1, *k2, *less, *k3; + emit_sp_ref_u64 (j, T0, T1, a); + emit_sp_ref_u64 (j, T2, T3, b); + less = jit_bltr (T1, T3); + k1 = jit_bner (T1, T3); + k2 = jit_bger (T0, T2); + jit_patch (less); + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + k3 = jit_jmpi (); + jit_patch (k1); + jit_patch (k2); + jit_movi (T2, SCM_F_COMPARE_NONE); + jit_patch (k3); +#endif + emit_store_compare_result (j, T2); } static void compile_f64_numerically_equal (scm_jit_state *j, uint16_t a, uint16_t b) { + jit_node_t *k; + jit_movi (T2, SCM_F_COMPARE_EQUAL); + emit_sp_ref_f64 (j, JIT_F0, a); + emit_sp_ref_f64 (j, JIT_F1, b); + k = jit_beqr_d (JIT_F0, JIT_F1); + jit_movi (T2, SCM_F_COMPARE_NONE); + jit_patch (k); + emit_store_compare_result (j, T2); } static void compile_f64_less (scm_jit_state *j, uint16_t a, uint16_t b) { + jit_node_t *less, *ge, *k1, *k2; + emit_sp_ref_f64 (j, JIT_F0, a); + emit_sp_ref_f64 (j, JIT_F1, b); + less = jit_bltr_d (JIT_F0, JIT_F1); + ge = jit_bger_d (JIT_F0, JIT_F1); + jit_movi (T2, SCM_F_COMPARE_INVALID); + k1 = jit_jmpi (); + jit_patch (ge); + jit_movi (T2, SCM_F_COMPARE_NONE); + k2 = jit_jmpi (); + jit_patch (less); + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + jit_patch (k1); + jit_patch (k2); + emit_store_compare_result (j, T2); } static void compile_numerically_equal (scm_jit_state *j, uint16_t a, uint16_t b) { + jit_node_t *k; + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + emit_sp_ref_scm (j, T1, b); + emit_call_r_r (j, scm_vm_intrinsics.numerically_equal_p, T0, T1); + jit_retval (T0); + emit_reload_sp (j); + jit_movi (T2, SCM_F_COMPARE_NONE); + k = jit_bnei (T0, 0); + jit_movi (T2, SCM_F_COMPARE_EQUAL); + jit_patch (k); + emit_store_compare_result (j, T2); } static void compile_less (scm_jit_state *j, uint16_t a, uint16_t b) { + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + emit_sp_ref_scm (j, T1, b); + emit_call_r_r (j, scm_vm_intrinsics.less_p, T0, T1); + jit_retval (T2); + emit_reload_sp (j); + emit_store_compare_result (j, T2); } static void -compile_check_arguments (scm_jit_state *j, uint32_t a) +compile_check_arguments (scm_jit_state *j, uint32_t expected) { + jit_node_t *eq, *k; + jit_gpr_t fp = T0, t = T1, res = T2; + + emit_load_fp (j, fp); + jit_movi (res, SCM_F_COMPARE_EQUAL); + eq = emit_branch_if_frame_locals_count_eq (j, fp, t, expected); + if (expected > 0) + { + jit_node_t *k2, *ge; + ge = emit_branch_if_frame_locals_count_greater_than (j, fp, t, expected-1); + jit_movi (res, SCM_F_COMPARE_LESS_THAN); + k2 = jit_jmpi (); + jit_patch (ge); + jit_movi (res, SCM_F_COMPARE_NONE); + jit_patch (k2); + } + else + jit_movi (res, SCM_F_COMPARE_NONE); + jit_patch (eq); + jit_patch (k); + emit_store_compare_result (j, T2); } static void -compile_check_positional_arguments (scm_jit_state *j, uint32_t a, uint32_t b) +compile_check_positional_arguments (scm_jit_state *j, uint32_t nreq, uint32_t expected) { + jit_node_t *k, *head, *lt, *eq, *done1, *done2; + jit_gpr_t walk = T0, npos = T1, obj = T2, t = T3, res = T0; + + emit_load_fp (j, walk); + if (nreq == 0) abort (); + emit_subtract_stack_slots (j, walk, walk, nreq-1); + jit_movi (npos, nreq - 1); + + head = jit_label (); + jit_addi (npos, npos, 1); + emit_subtract_stack_slots (j, walk, walk, 1); + k = jit_beqr (walk, SP); + jit_ldr (obj, walk); + jit_patch_at (emit_branch_if_immediate (j, obj), head); + jit_patch_at (emit_branch_if_heap_object_not_tc7 (j, obj, t, scm_tc7_keyword), + head); + jit_patch (k); + + lt = jit_blti (npos, expected); + eq = jit_beqi (npos, expected); + jit_movi (res, SCM_F_COMPARE_NONE); + done1 = jit_jmpi (); + jit_patch (lt); + jit_movi (res, SCM_F_COMPARE_LESS_THAN); + done2 = jit_jmpi (); + jit_patch (eq); + jit_movi (res, SCM_F_COMPARE_EQUAL); + jit_patch (done1); + jit_patch (done2); + jit_stxi (thread_offset_compare_result, THREAD, res); } static void -compile_immediate_tag_equals (scm_jit_state *j, uint32_t a, uint16_t b, uint16_t c) +compile_immediate_tag_equals (scm_jit_state *j, uint32_t a, uint16_t mask, + uint16_t expected) { + jit_node_t *k; + emit_sp_ref_scm (j, T0, a); + jit_andi (T0, T0, mask); + jit_movi (T2, SCM_F_COMPARE_EQUAL); + k = jit_beqi (T0, expected); + jit_movi (T2, SCM_F_COMPARE_NONE); + jit_patch (k); + emit_store_compare_result (j, T2); } static void -compile_heap_tag_equals (scm_jit_state *j, uint32_t a, uint16_t b, uint16_t c) +compile_heap_tag_equals (scm_jit_state *j, uint32_t obj, + uint16_t mask, uint16_t expected) { + jit_node_t *k; + emit_sp_ref_scm (j, T0, obj); + jit_movi (T2, SCM_F_COMPARE_EQUAL); + k = emit_branch_if_heap_object_has_tc (j, T0, T0, mask, expected); + jit_movi (T2, SCM_F_COMPARE_NONE); + jit_patch (k); + emit_store_compare_result (j, T2); } static void compile_eq (scm_jit_state *j, uint16_t a, uint16_t b) { + jit_node_t *k; + emit_sp_ref_scm (j, T0, a); + emit_sp_ref_scm (j, T1, b); + jit_movi (T2, SCM_F_COMPARE_EQUAL); + k = jit_beqi (T0, T1); + jit_movi (T2, SCM_F_COMPARE_NONE); + jit_patch (k); + emit_store_compare_result (j, T2); } static void compile_j (scm_jit_state *j, const uint32_t *vcode) { + jit_node_t *jmp; + jmp = jit_jmpi (); + add_inter_instruction_patch (j, jmp, vcode); } static void compile_jl (scm_jit_state *j, const uint32_t *vcode) { + jit_node_t *jmp; + emit_load_compare_result (j, T0); + jmp = jit_beqi (T0, SCM_F_COMPARE_LESS_THAN); + add_inter_instruction_patch (j, jmp, vcode); } static void compile_je (scm_jit_state *j, const uint32_t *vcode) { + jit_node_t *jmp; + emit_load_compare_result (j, T0); + jmp = jit_beqi (T0, SCM_F_COMPARE_EQUAL); + add_inter_instruction_patch (j, jmp, vcode); } static void compile_jnl (scm_jit_state *j, const uint32_t *vcode) { + jit_node_t *jmp; + emit_load_compare_result (j, T0); + jmp = jit_bnei (T0, SCM_F_COMPARE_LESS_THAN); + add_inter_instruction_patch (j, jmp, vcode); } static void compile_jne (scm_jit_state *j, const uint32_t *vcode) { + jit_node_t *jmp; + emit_load_compare_result (j, T0); + jmp = jit_bnei (T0, SCM_F_COMPARE_EQUAL); + add_inter_instruction_patch (j, jmp, vcode); } static void compile_jge (scm_jit_state *j, const uint32_t *vcode) { + jit_node_t *jmp; + emit_load_compare_result (j, T0); + jmp = jit_beqi (T0, SCM_F_COMPARE_NONE); + add_inter_instruction_patch (j, jmp, vcode); } static void compile_jnge (scm_jit_state *j, const uint32_t *vcode) { + jit_node_t *jmp; + emit_load_compare_result (j, T0); + jmp = jit_bnei (T0, SCM_F_COMPARE_NONE); + add_inter_instruction_patch (j, jmp, vcode); } static void compile_heap_numbers_equal (scm_jit_state *j, uint16_t a, uint16_t b) { + jit_node_t *k; + emit_store_current_ip (j, T0); + emit_sp_ref_scm (j, T0, a); + emit_sp_ref_scm (j, T1, b); + emit_call_r_r (j, scm_vm_intrinsics.heap_numbers_equal_p, T0, T1); + jit_retval (T0); + emit_reload_sp (j); + jit_movi (T2, SCM_F_COMPARE_NONE); + k = jit_bnei (T0, 0); + jit_movi (T2, SCM_F_COMPARE_EQUAL); + jit_patch (k); + emit_store_compare_result (j, T2); } static void compile_untag_fixnum (scm_jit_state *j, uint16_t dst, uint16_t a) { + emit_sp_ref_scm (j, T0, a); + jit_rshi (T0, T0, 2); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_set_s64 (j, dst, T0); +#else + /* FIXME: Untested! */ + jit_movi (T1, T0); + jit_rshi (T1, T1, 31); + emit_sp_set_s64 (j, dst, T0, T1); +#endif } static void compile_tag_fixnum (scm_jit_state *j, uint16_t dst, uint16_t a) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_s64 (j, T0, a); +#else + emit_sp_ref_s32 (j, T0, a); +#endif + jit_lshi (T0, T0, 2); + jit_addi (T0, T0, scm_tc2_int); + emit_sp_set_scm (j, dst, T0); } static void compile_srsh (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_s64 (j, T0, a); + emit_sp_ref_s64 (j, T1, b); + jit_andi (T1, T1, 63); + jit_rshr (T0, T0, T1); + emit_sp_set_s64 (j, dst, T0); +#else + /* FIXME: Not tested. */ + jit_node_t *zero, *both, *done; + + emit_sp_ref_s64 (j, T0, T1, a); + emit_sp_ref_s64 (j, T2, T3, b); + jit_andi (T2, T2, 63); + zero = jit_beqi (T2, 0); + both = jit_blti (T2, 32); + + /* 32 <= s < 64: hi = hi >> 31, lo = hi >> (s-32) */ + jit_subi (T2, 32); + jit_rshr (T0, T1, T2); + jit_rshi (T1, T1, 31); + done = jit_jmpi (); + + jit_patch (both); + /* 0 < s < 32: hi = hi >> s, lo = lo >> s + hi << (32-s) */ + jit_negr (T3, T2); + jit_addi (T3, T3, 32); + jit_lshr (T3, T1, T3); + jit_rshr (T1, T1, T2); + jit_rshr_u (T0, T0, T2); + jit_addr (T0, T0, T3); + + jit_patch (done); + jit_patch (zero); + emit_sp_set_s64 (j, dst, T0, T1); +#endif } static void compile_srsh_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) { + b &= 63; + +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_s64 (j, T0, a); + jit_rshi (T0, T0, b); + emit_sp_set_s64 (j, dst, T0); +#else + /* FIXME: Not tested. */ + emit_sp_ref_s64 (j, T0, T1, a); + if (b == 0) + { + /* Nothing to do. */ + } + else if (b >= 32) + { + /* hi = sign-ext, lo = hi >> (s-32) */ + jit_rshi (T0, T1, b - 32); + jit_rshi (T1, T1, 31); + } + else + { + /* 0 < s < 32: hi = hi >> s, lo = lo >> s + hi << (32-s) */ + jit_lshi (T2, T1, 32 - b); + jit_rshi (T1, T1, b); + jit_rshi_u (T0, T0, b); + jit_addr (T0, T0, T2); + } + emit_sp_set_s64 (j, dst, T0, T1); +#endif } static void compile_s64_imm_numerically_equal (scm_jit_state *j, uint16_t a, int16_t b) { +#if SIZEOF_UINTPTR_T >= 8 + jit_node_t *k; + jit_movi (T2, SCM_F_COMPARE_NONE); + emit_sp_ref_u64 (j, T0, a); + k = jit_bnei (T0, b); + jit_movi (T2, SCM_F_COMPARE_EQUAL); + jit_patch (k); +#else + jit_node_t *k1, *k2; + jit_movi (T2, SCM_F_COMPARE_NONE); + emit_sp_ref_u64 (j, T0, T1, a); + k1 = jit_bnei (T0, b); + k2 = jit_bnei (T1, b < 0 ? -1 : 0); + jit_movi (T2, SCM_F_COMPARE_EQUAL); + jit_patch (k1); + jit_patch (k2); +#endif + emit_store_compare_result (j, T2); } static void compile_u64_imm_less (scm_jit_state *j, uint16_t a, uint16_t b) { +#if SIZEOF_UINTPTR_T >= 8 + jit_node_t *k; + jit_movi (T2, SCM_F_COMPARE_NONE); + emit_sp_ref_u64 (j, T0, a); + k = jit_bgei_u (T0, b); + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + jit_patch (k); +#else + jit_node_t *k1, *k2; + jit_movi (T2, SCM_F_COMPARE_NONE); + emit_sp_ref_u64 (j, T0, T1, a); + k1 = jit_bgei_u (T0, b); + k2 = jit_bnei (T1, 0); + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + jit_patch (k1); + jit_patch (k2); +#endif + emit_store_compare_result (j, T2); } static void compile_imm_u64_less (scm_jit_state *j, uint16_t a, uint16_t b) { +#if SIZEOF_UINTPTR_T >= 8 + jit_node_t *k; + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + emit_sp_ref_u64 (j, T0, a); + k = jit_bgti_u (T0, b); + jit_movi (T2, SCM_F_COMPARE_NONE); + jit_patch (k); +#else + jit_node_t *k1, *k2; + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + emit_sp_ref_u64 (j, T0, T1, a); + k1 = jit_bnei (T1, 0); + k2 = jit_bgti_u (T0, b); + jit_movi (T2, SCM_F_COMPARE_NONE); + jit_patch (k1); + jit_patch (k2); +#endif + emit_store_compare_result (j, T2); } static void compile_s64_imm_less (scm_jit_state *j, uint16_t a, int16_t b) { +#if SIZEOF_UINTPTR_T >= 8 + jit_node_t *k; + jit_movi (T2, SCM_F_COMPARE_NONE); + emit_sp_ref_s64 (j, T0, a); + k = jit_bgei (T0, b); + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + jit_patch (k); +#else + jit_node_t *k1, *k2, *k3; + int32_t sign = b < 0 ? -1 : 0; + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + emit_sp_ref_s64 (j, T0, T1, a); + k1 = jit_blti (T1, sign); + k2 = jit_bnei (T1, sign); + k3 = jit_blti (T0, b); + jit_patch (k2); + jit_movi (T2, SCM_F_COMPARE_NONE); + jit_patch (k1); + jit_patch (k3); +#endif + emit_store_compare_result (j, T2); } static void compile_imm_s64_less (scm_jit_state *j, uint16_t a, int16_t b) { +#if SIZEOF_UINTPTR_T >= 8 + jit_node_t *k; + jit_movi (T2, SCM_F_COMPARE_NONE); + emit_sp_ref_s64 (j, T0, a); + k = jit_blei (T0, b); + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + jit_patch (k); +#else + jit_node_t *k1, *k2, *k3; + int32_t sign = b < 0 ? -1 : 0; + jit_movi (T2, SCM_F_COMPARE_NONE); + emit_sp_ref_s64 (j, T0, T1, a); + k1 = jit_blti (T1, sign); + k2 = jit_bnei (T1, sign); + k3 = jit_blei (T0, b); + jit_patch (k2); + jit_movi (T2, SCM_F_COMPARE_LESS_THAN); + jit_patch (k1); + jit_patch (k3); +#endif + emit_store_compare_result (j, T2); } static void -compile_u8_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_u8_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); + jit_ldxr_uc (T0, T0, T1); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_set_u64 (j, dst, T0); +#else + jit_movi (T1, 0); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void -compile_u16_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_u16_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); + jit_ldxr_us (T0, T0, T1); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_set_u64 (j, dst, T0); +#else + jit_movi (T1, 0); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void -compile_u32_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_u32_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); +#if SIZEOF_UINTPTR_T >= 8 + jit_ldxr_ui (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + jit_ldxr (T0, T0, T1); + jit_movi (T1, 0); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void -compile_u64_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_u64_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); +#if SIZEOF_UINTPTR_T >= 8 + jit_ldxr (T0, T0, T1); + emit_sp_set_u64 (j, dst, T0); +#else + jit_addr (T0, T0, T1); + if (BIGENDIAN) + { + jit_ldxi (T1, T0, 4); + jit_ldr (T0, T0); + } + else + { + jit_ldr (T1, T0); + jit_ldxi (T0, T0, 4); + } + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void -compile_u8_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_u8_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T2, v); +#else + emit_sp_ref_u64_lower_half (j, T2, v); +#endif + jit_stxr_c (T0, T1, T2); } static void -compile_u16_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_u16_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T2, v); +#else + emit_sp_ref_u64_lower_half (j, T2, v); +#endif + jit_stxr_s (T0, T1, T2); } static void -compile_u32_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_u32_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T2, v); + jit_stxr_i (T0, T1, T2); +#else + emit_sp_ref_u64_lower_half (j, T2, v); + jit_stxr (T0, T1, T2); +#endif } static void -compile_u64_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_u64_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_ref_u64 (j, T2, v); + jit_stxr (T0, T1, T2); +#else + jit_addr (T0, T0, T1); + emit_sp_ref_u64 (j, T1, T2, v); + if (BIGENDIAN) + { + jit_str (T0, T2); + jit_stxi (4, T0, T1); + } + else + { + jit_str (T0, T1); + jit_stxi (4, T0, T2); + } +#endif } static void -compile_s8_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_s8_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); + jit_ldxr_c (T0, T0, T1); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_set_s64 (j, dst, T0); +#else + jit_rshi (T1, T0, 7); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void -compile_s16_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_s16_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); + jit_ldxr_s (T0, T0, T1); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_set_s64 (j, dst, T0); +#else + jit_rshi (T1, T0, 15); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void -compile_s32_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_s32_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); + jit_ldxr_i (T0, T0, T1); +#if SIZEOF_UINTPTR_T >= 8 + emit_sp_set_s64 (j, dst, T0); +#else + jit_rshi (T1, T0, 31); + emit_sp_set_u64 (j, dst, T0, T1); +#endif } static void -compile_s64_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_s64_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx) { + compile_u64_ref (j, dst, ptr, idx); } static void -compile_s8_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_s8_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v) { + compile_u8_set (j, ptr, idx, v); } static void -compile_s16_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_s16_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v) { + compile_u16_set (j, ptr, idx, v); } static void -compile_s32_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_s32_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v) { + compile_u32_set (j, ptr, idx, v); } static void -compile_s64_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_s64_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v) { + compile_u64_set (j, ptr, idx, v); } static void -compile_f32_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_f32_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); + jit_ldxr_f (JIT_F0, T0, T1); + jit_extr_f_d (JIT_F0, JIT_F0); + emit_sp_set_f64 (j, dst, JIT_F0); } static void -compile_f64_ref (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b) +compile_f64_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); + jit_ldxr_d (JIT_F0, T0, T1); + emit_sp_set_f64 (j, dst, JIT_F0); } static void -compile_f32_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_f32_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); + emit_sp_ref_f64 (j, JIT_F0, v); + jit_extr_d_f (JIT_F0, JIT_F0); + jit_stxr_d (T0, T1, JIT_F0); } static void -compile_f64_set (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c) +compile_f64_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v) { + emit_sp_ref_ptr (j, T0, ptr); + emit_sp_ref_sz (j, T1, idx); + emit_sp_ref_f64 (j, JIT_F0, v); + jit_stxr_d (T0, T1, JIT_F0); } |