summaryrefslogtreecommitdiff
path: root/libguile/jit.c
diff options
context:
space:
mode:
Diffstat (limited to 'libguile/jit.c')
-rw-r--r--libguile/jit.c4835
1 files changed, 4835 insertions, 0 deletions
diff --git a/libguile/jit.c b/libguile/jit.c
new file mode 100644
index 000000000..6c7399a75
--- /dev/null
+++ b/libguile/jit.c
@@ -0,0 +1,4835 @@
+/* Copyright 2018-2019
+ Free Software Foundation, Inc.
+
+ This file is part of Guile.
+
+ Guile is free software: you can redistribute it and/or modify it
+ under the terms of the GNU Lesser General Public License as published
+ by the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ Guile is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+ License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with Guile. If not, see
+ <https://www.gnu.org/licenses/>. */
+
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+/* All of this whole file is within an ENABLE_JIT flag. */
+#if ENABLE_JIT
+
+#include <stdio.h>
+#include <sys/mman.h>
+
+#include <lightening.h>
+
+#include "frames.h"
+#include "gsubr.h"
+#include "instructions.h"
+#include "intrinsics.h"
+#include "simpos.h" /* scm_getenv_int */
+#include "threads.h"
+#include "vm-builtins.h"
+#include "vm-operations.h"
+
+#include "jit.h"
+
+
+
+
+/* Guile's just-in-time (JIT) compiler is a simple "template JIT". It
+ produces machine code corresponding to each VM instruction,
+ substituting in the arguments from the bytecode. The generated code
+ performs the same operations on the Guile program state the VM
+ interpreter would: the same stack reads and writes, the same calls,
+ the same control flow: the same thing. It's a very simple JIT.
+
+ This JIT uses GNU Lightning, a library for generating assembly code.
+ It has backends for every architecture you can think of. Lightning
+ exposes a minimum of 3 "volatile" or "scratch" registers, those that
+ may be overwritten by called functions, and 3 "non-volatile" or
+ "preserved" registers, those whose values will persist over calls.
+ Guile's JIT uses two preserved registers for itself, to store the
+ current thread and the current stack pointer. The other four
+ registers are available for the JIT. However as Guile's JIT is
+ really simple and doesn't do register allocation, no other register
+ is live between bytecodes; the other four registers are just scratch
+ space.
+
+ Machine code emitted by the JIT (mcode) should only ever be entered
+ from the interpreter (the VM). To enter bytecode, the interpreter
+ calls an "entry trampoline" that saves the needed non-volatile
+ registers, reserves some stack space, loads the thread and stack
+ pointer into the reserved registers, then jumps into the mcode. The
+ mcode then does its thing.
+
+ When mcode needs to call out to another function, e.g. via the "call"
+ instruction, it makes a new frame in just the same way the VM would,
+ with the difference that it also sets the machine return address
+ (mRA) in the stack frame, in addition to the virtual (bytecode)
+ return address (vRA). If the callee has mcode, then the caller jumps
+ to the callee's mcode. It's a jump, not a call, as the stack is
+ maintained on the side: it's not the stack used by the e.g. x86
+ "call" instruction.
+
+ When mcode calls a function that doesn't have vcode, or returns to a
+ continuation that doesn't have vcode, the mcode simply returns to the
+ VM interpreter, allowing the interpreter to pick up from there. The
+ return actually happens via an exit trampoline, which restores the
+ saved register values.
+
+ Every function in Guile's VM begins with an "instrument-entry"
+ instruction. The instruction links to a statically allocated "struct
+ scm_jit_function_data" corresponding to that function. When the
+ interpreter sees instrument-entry, first it checks that if the
+ function has mcode, by looking in the scm_jit_function_data. If it
+ has mcode, the interpreter enters mcode directly, as described above.
+
+ If a function doesn't have mcode, "instrument-entry" will increment a
+ counter in the scm_jit_function_data. If the counter exceeds a
+ threshold, the interpreter will ask the JIT compiler to produce
+ mcode. If the JIT compiler was able to do so (always possible except
+ in case of resource exhaustion), then it sets the mcode pointer in
+ the scm_jit_function_data, and returns the mcode pointer to the
+ interpreter. At that point the interpreter will enter mcode.
+
+ If the counter value does not exceed the threshold, then the VM
+ will interpret the function instead of running compiled code.
+
+ Additionally, Guile puts an "instrument-loop" instruction into the
+ body of each loop iteration. It works similarly, except that the
+ returned mcode pointer starts in the middle of the function, at the
+ point that corresponds to the program point of the "instrument-loop"
+ instruction. The idea is that some functions have long-running loops
+ in them, and it would be a shame to have to wait until the next time
+ they're called to enter mcode. Being able to "tier up" from inside a
+ loop reduces overall program latency.
+
+ Think of the JIT as microarchitecture. The interpreter specifies the
+ architecture of the VM, in terms of the stack, stack and frame
+ pointers, and a virtual instruction pointer. Sometimes this
+ architectural state is manipulated by the interpreter. Sometimes
+ it's compiled down to native code. But the existence of native code
+ is a detail that's fully encapsulated; systems-oriented Guile Scheme
+ can walk stacks, throw errors, reinstate partial continuations, and
+ so on without being aware of the existence of the JIT. */
+
+
+
+
+static const uint32_t default_jit_threshold = 1000;
+
+/* Threshold for when to JIT-compile a function. Set from the
+ GUILE_JIT_THRESHOLD environment variable. */
+uint32_t scm_jit_counter_threshold = -1;
+
+/* If positive, stop JIT compilation after the Nth compilation. Useful
+ for hunting down bugs. */
+static int jit_stop_after = -1;
+
+/* If nonzero, pause when stopping JIT compilation after the Nth
+ compilation. For debugging. */
+static int jit_pause_when_stopping = 0;
+
+/* Log level for JIT events. 0 means off. */
+static int jit_log_level = 0;
+
+/* Entry trampoline: saves registers, initializes THREAD and SP
+ registers, and jumps into mcode. */
+static void (*enter_mcode) (scm_thread *thread, const uint8_t *mcode);
+
+/* Exit trampoline: restores registers and returns to interpreter. */
+static void *exit_mcode;
+
+/* Handle interrupts trampoline: the slow path of the handle-interrupts
+ instruction, compiled as a stub on the side to reduce code size. */
+static void *handle_interrupts_trampoline;
+
+/* Thread-local buffer into which to write code. */
+struct code_arena
+{
+ uint8_t *base;
+ size_t used;
+ size_t size;
+ struct code_arena *prev;
+};
+
+/* Branches between instructions. */
+struct pending_reloc
+{
+ jit_reloc_t reloc;
+ ptrdiff_t target_vcode_offset;
+};
+
+/* State of the JIT compiler for the current thread. */
+struct scm_jit_state {
+ jit_state_t *jit;
+ scm_thread *thread;
+ const uint32_t *start;
+ uint32_t *ip;
+ uint32_t *next_ip;
+ const uint32_t *end;
+ uint32_t *entry;
+ uint8_t *op_attrs;
+ struct pending_reloc *relocs;
+ size_t reloc_idx;
+ size_t reloc_count;
+ void **labels;
+ int32_t frame_size;
+ uint32_t register_state;
+ jit_gpr_t sp_cache_gpr;
+ jit_fpr_t sp_cache_fpr;
+ uint32_t sp_cache_gpr_idx;
+ uint32_t sp_cache_fpr_idx;
+ struct code_arena *code_arena;
+};
+
+typedef struct scm_jit_state scm_jit_state;
+
+static const uint32_t program_word_offset_free_variable = 2;
+
+static const uint32_t frame_offset_mra = 0 * sizeof(union scm_vm_stack_element);
+static const uint32_t frame_offset_vra = 1 * sizeof(union scm_vm_stack_element);
+static const uint32_t frame_offset_prev = 2 * sizeof(union scm_vm_stack_element);
+static const uint32_t frame_overhead_slots = 3;
+
+#define DEFINE_THREAD_OFFSET(f) \
+ static const uint32_t thread_offset_##f = \
+ offsetof (struct scm_thread, f)
+
+DEFINE_THREAD_OFFSET (handle);
+DEFINE_THREAD_OFFSET (pending_asyncs);
+DEFINE_THREAD_OFFSET (block_asyncs);
+
+#define DEFINE_THREAD_VP_OFFSET(f) \
+ static const uint32_t thread_offset_##f = \
+ offsetof (struct scm_thread, vm) + offsetof (struct scm_vm, f)
+
+DEFINE_THREAD_VP_OFFSET (fp);
+DEFINE_THREAD_VP_OFFSET (sp);
+DEFINE_THREAD_VP_OFFSET (ip);
+DEFINE_THREAD_VP_OFFSET (sp_min_since_gc);
+DEFINE_THREAD_VP_OFFSET (stack_limit);
+
+/* The current scm_thread*. Preserved across callouts. */
+static const jit_gpr_t THREAD = JIT_V0;
+
+/* The current stack pointer. Clobbered across callouts. Can be
+ reloaded from the thread. Note that any callout that might
+ recursively enter the VM may move the stack pointer. */
+static const jit_gpr_t SP = JIT_R0;
+
+/* During calls and returns -- the parts of the code that manipulate the
+ frame pointer -- the current frame pointer is stored in FP.
+ Otherwise this is a temp register. It can always be reloaded from
+ THREAD. Like SP, it can move. */
+static const jit_gpr_t FP = JIT_R1;
+
+/* Scratch registers. */
+static const jit_gpr_t T0 = JIT_V1;
+static const jit_gpr_t T1 = JIT_V2;
+static const jit_gpr_t T2 = JIT_R2;
+SCM_UNUSED static const jit_gpr_t T3_OR_FP = JIT_R1;
+SCM_UNUSED static const jit_gpr_t T4_OR_SP = JIT_R0;
+
+/* Sometimes you want to call out the fact that T0 and T1 are preserved
+ across calls. In that case, use these. */
+static const jit_gpr_t T0_PRESERVED = JIT_V1;
+static const jit_gpr_t T1_PRESERVED = JIT_V2;
+
+static const uint32_t SP_IN_REGISTER = 0x1;
+static const uint32_t FP_IN_REGISTER = 0x2;
+static const uint32_t SP_CACHE_GPR = 0x4;
+static const uint32_t SP_CACHE_FPR = 0x8;
+
+static const uint8_t OP_ATTR_BLOCK = 0x1;
+static const uint8_t OP_ATTR_ENTRY = 0x2;
+
+#ifdef WORDS_BIGENDIAN
+#define BIGENDIAN 1
+#else
+#define BIGENDIAN 0
+#endif
+
+#if BIGENDIAN
+static const uint32_t uint32_offset_low_byte = 3;
+#else
+static const uint32_t uint32_offset_low_byte = 0;
+#endif
+
+#if SCM_SIZEOF_UINTPTR_T == 4
+static const uint32_t log2_sizeof_uintptr_t = 2;
+#elif SCM_SIZEOF_UINTPTR_T == 8
+static const uint32_t log2_sizeof_uintptr_t = 3;
+#else
+#error unhandled uintptr_t size
+#endif
+
+#define LENGTH_NOP 0
+#define LENGTH_OP1(a) 1
+#define LENGTH_OP2(a,b) 2
+#define LENGTH_OP3(a,b,c) 3
+#define LENGTH_OP4(a,b,c,d) 4
+#define LENGTH_DOP1(a) 1
+#define LENGTH_DOP2(a,b) 2
+#define LENGTH_DOP3(a,b,c) 3
+#define LENGTH_DOP4(a,b,c,d) 4
+
+static const uint8_t op_lengths[256] = {
+#define OP_LENGTH(code, cname, name, arity) LENGTH_##arity,
+FOR_EACH_VM_OPERATION(OP_LENGTH)
+#undef OP_LENGTH
+};
+
+static void die (int line, const char *msg) SCM_NORETURN;
+static void
+die (int line, const char *msg)
+{
+ fprintf (stderr, "jit.c:%d: fatal: %s\n", line, msg);
+ abort ();
+}
+
+#define DIE(msg) die(__LINE__, msg)
+
+#define ASSERT(x) \
+ do { if (SCM_UNLIKELY (!(x))) DIE ("assertion failed"); } while (0)
+
+#define UNREACHABLE() \
+ DIE ("unreachable")
+
+#define _LOG(level, ...) \
+ do { \
+ if (SCM_UNLIKELY (jit_log_level >= level)) \
+ fprintf (stderr, "jit: " __VA_ARGS__); \
+ } while (0)
+
+enum {
+ LOG_LEVEL_NONE,
+ LOG_LEVEL_INFO,
+ LOG_LEVEL_DEBUG,
+ LOG_LEVEL_LOG
+};
+
+#define INFO(...) _LOG(LOG_LEVEL_INFO, __VA_ARGS__)
+#define DEBUG(...) _LOG(LOG_LEVEL_DEBUG, __VA_ARGS__)
+#define LOG(...) _LOG(LOG_LEVEL_LOG, __VA_ARGS__)
+
+static void
+reset_register_state (scm_jit_state *j, uint32_t state)
+{
+ j->register_state = state;
+}
+
+static void
+clear_register_state (scm_jit_state *j, uint32_t state)
+{
+ j->register_state &= ~state;
+}
+
+static void
+clear_scratch_register_state (scm_jit_state *j)
+{
+ reset_register_state (j, 0);
+}
+
+static void
+set_register_state (scm_jit_state *j, uint32_t state)
+{
+ j->register_state |= state;
+}
+
+static uint32_t
+has_register_state (scm_jit_state *j, uint32_t state)
+{
+ return (j->register_state & state) == state;
+}
+
+#define ASSERT_HAS_REGISTER_STATE(state) ASSERT (has_register_state (j, state))
+
+static void
+record_gpr_clobber (scm_jit_state *j, jit_gpr_t r)
+{
+ if (jit_same_gprs (j->sp_cache_gpr, r))
+ clear_register_state (j, SP_CACHE_GPR);
+
+ if (jit_same_gprs (r, SP))
+ clear_register_state (j, SP_IN_REGISTER);
+ else if (jit_same_gprs (r, FP))
+ clear_register_state (j, FP_IN_REGISTER);
+}
+
+static void
+record_fpr_clobber (scm_jit_state *j, jit_fpr_t r)
+{
+ if (jit_same_fprs (j->sp_cache_fpr, r))
+ clear_register_state (j, SP_CACHE_FPR);
+}
+
+static void
+set_sp_cache_gpr (scm_jit_state *j, uint32_t idx, jit_gpr_t r)
+{
+ set_register_state (j, SP_CACHE_GPR);
+ j->sp_cache_gpr_idx = idx;
+ if (j->sp_cache_fpr_idx == idx)
+ clear_register_state (j, SP_CACHE_FPR);
+}
+
+static void
+set_sp_cache_fpr (scm_jit_state *j, uint32_t idx, jit_fpr_t r)
+{
+ set_register_state (j, SP_CACHE_FPR);
+ j->sp_cache_fpr_idx = idx;
+ if (j->sp_cache_gpr_idx == idx)
+ clear_register_state (j, SP_CACHE_GPR);
+}
+
+/* Q: When should I use emit_retval instead of jit_retval? When to use
+ emit_movi, emit_ldxi?
+
+ A: Generally you should use the emit_ variants instead of the jit_
+ variants. Guile's JIT compiler has a primitive form of local
+ (intrablock) register allocation that records recent stores. A
+ subsequent load might be able to replace a register read instead of a
+ memory load. This simple allocator works for straight-line code, and
+ it works as long as register writes are recorded. The JIT itself
+ will clear the register allocator state at control-flow joins, but
+ control flow within an instruction needs to be careful.
+
+ It's OK to use the jit_emit, jit_retval etc primitives if you
+ manually make corresponding changes to the register_state, perhaps by
+ inserting record_gpr_clobber calls. If the register is later
+ clobbered by e.g. emit_sp_set_scm, sometimes those can be omitted
+ though. Also, if your instruction includes a call, that code will
+ invalidate any cached register-stack-index associations, so if
+ there's a call, maybe you can avoid calling emit_*.
+
+ Note of course that an association between registers and
+ stack-indexed locals is also invalidated if the stack frame expands
+ via alloc-frame or push, or shrinks via reset-frame, pop, drop,
+ etc. */
+static void
+emit_retval (scm_jit_state *j, jit_gpr_t r)
+{
+ jit_retval (j->jit, r);
+ record_gpr_clobber (j, r);
+}
+
+static void
+emit_retval_d (scm_jit_state *j, jit_fpr_t r)
+{
+ jit_retval_d (j->jit, r);
+ record_fpr_clobber (j, r);
+}
+
+static void
+emit_movi (scm_jit_state *j, jit_gpr_t r, jit_word_t i)
+{
+ jit_movi (j->jit, r, i);
+ record_gpr_clobber (j, r);
+}
+
+static jit_reloc_t
+emit_mov_addr (scm_jit_state *j, jit_gpr_t r)
+{
+ record_gpr_clobber (j, r);
+ return jit_mov_addr (j->jit, r);
+}
+
+static void
+emit_ldxi (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t src, jit_word_t offset)
+{
+ if (offset == 0)
+ jit_ldr (j->jit, dst, src);
+ else
+ jit_ldxi (j->jit, dst, src, offset);
+ record_gpr_clobber (j, dst);
+}
+
+#define DEFINE_CLOBBER_RECORDING_EMITTER_R(stem, typ) \
+static void \
+emit_##stem (scm_jit_state *j, jit_##typ##_t dst, jit_##typ##_t a) \
+{ \
+ jit_##stem (j->jit, dst, a); \
+ record_##typ##_clobber (j, dst); \
+}
+
+#define DEFINE_CLOBBER_RECORDING_EMITTER_P(stem, typ) \
+static void \
+emit_##stem (scm_jit_state *j, jit_##typ##_t dst, jit_pointer_t a) \
+{ \
+ jit_##stem (j->jit, dst, a); \
+ record_##typ##_clobber (j, dst); \
+}
+
+#define DEFINE_CLOBBER_RECORDING_EMITTER_R_I(stem, typ) \
+static void \
+emit_##stem (scm_jit_state *j, jit_##typ##_t dst, \
+ jit_##typ##_t a, jit_word_t b) \
+{ \
+ jit_##stem (j->jit, dst, a, b); \
+ record_##typ##_clobber (j, dst); \
+}
+
+#define DEFINE_CLOBBER_RECORDING_EMITTER_R_R(stem, typ) \
+static void \
+emit_##stem (scm_jit_state *j, jit_##typ##_t dst, \
+ jit_##typ##_t a, jit_##typ##_t b) \
+{ \
+ jit_##stem (j->jit, dst, a, b); \
+ record_##typ##_clobber (j, dst); \
+}
+
+#define DEFINE_CLOBBER_RECORDING_EMITTER_R_R_2(stem, typ) \
+static void \
+emit_##stem (scm_jit_state *j, \
+ jit_##typ##_t dst1, jit_##typ##_t dst2, \
+ jit_##typ##_t a, jit_##typ##_t b) \
+{ \
+ jit_##stem (j->jit, dst1, dst2, a, b); \
+ record_##typ##_clobber (j, dst1); \
+ record_##typ##_clobber (j, dst2); \
+}
+
+DEFINE_CLOBBER_RECORDING_EMITTER_R(ldr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_P(ldi, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R(movr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R(comr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(ldxr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(addi, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(addr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(addr_d, fpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(subi, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(subr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(subr_d, fpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(muli, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(mulr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(mulr_d, fpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(divr_d, fpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(andi, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(andr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(orr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(xorr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(rshi, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(rshi_u, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(rshr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(rshr_u, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(lshi, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(lshr, gpr)
+
+#if SIZEOF_UINTPTR_T < 8
+DEFINE_CLOBBER_RECORDING_EMITTER_R(negr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(addci, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(addcr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(addxi, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(addxr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(subci, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(subcr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_I(subxi, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R(subxr, gpr)
+DEFINE_CLOBBER_RECORDING_EMITTER_R_R_2(qmulr_u, gpr)
+#endif
+
+static void
+emit_reload_sp (scm_jit_state *j)
+{
+ emit_ldxi (j, SP, THREAD, thread_offset_sp);
+ set_register_state (j, SP_IN_REGISTER);
+}
+
+static void
+emit_store_sp (scm_jit_state *j)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+ jit_stxi (j->jit, thread_offset_sp, THREAD, SP);
+}
+
+static void
+emit_reload_fp (scm_jit_state *j)
+{
+ emit_ldxi (j, FP, THREAD, thread_offset_fp);
+ set_register_state (j, FP_IN_REGISTER);
+}
+
+static void
+emit_store_fp (scm_jit_state *j)
+{
+ ASSERT_HAS_REGISTER_STATE (FP_IN_REGISTER);
+ jit_stxi (j->jit, thread_offset_fp, THREAD, FP);
+}
+
+static uint32_t
+save_reloadable_register_state (scm_jit_state *j)
+{
+ return j->register_state & (SP_IN_REGISTER | FP_IN_REGISTER);
+}
+
+static void
+restore_reloadable_register_state (scm_jit_state *j, uint32_t state)
+{
+ if ((state & SP_IN_REGISTER) && !has_register_state (j, SP_IN_REGISTER))
+ emit_reload_sp (j);
+ if ((state & FP_IN_REGISTER) && !has_register_state (j, FP_IN_REGISTER))
+ emit_reload_fp (j);
+}
+
+static void
+emit_subtract_stack_slots (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t src,
+ uint32_t n)
+{
+ emit_subi (j, dst, src, n * sizeof (union scm_vm_stack_element));
+}
+
+static void
+emit_load_mra (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t fp)
+{
+ emit_ldxi (j, dst, fp, frame_offset_mra);
+}
+
+static jit_reloc_t
+emit_store_mra (scm_jit_state *j, jit_gpr_t fp, jit_gpr_t t)
+{
+ jit_reloc_t reloc = emit_mov_addr (j, t);
+ ASSERT (frame_offset_mra == 0);
+ jit_str (j->jit, fp, t);
+ return reloc;
+}
+
+static void
+emit_load_vra (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t fp)
+{
+ emit_ldxi (j, dst, fp, frame_offset_vra);
+}
+
+static void
+emit_store_vra (scm_jit_state *j, jit_gpr_t fp, jit_gpr_t t, const uint32_t *vra)
+{
+ emit_movi (j, t, (intptr_t) vra);
+ jit_stxi (j->jit, frame_offset_vra, fp, t);
+}
+
+static void
+emit_load_prev_fp_offset (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t fp)
+{
+ emit_ldxi (j, dst, fp, frame_offset_prev);
+}
+
+static void
+emit_store_prev_fp_offset (scm_jit_state *j, jit_gpr_t fp, jit_gpr_t t,
+ uint32_t n)
+{
+ emit_movi (j, t, n);
+ jit_stxi (j->jit, frame_offset_prev, fp, t);
+}
+
+static void
+emit_store_ip (scm_jit_state *j, jit_gpr_t ip)
+{
+ jit_stxi (j->jit, thread_offset_ip, THREAD, ip);
+}
+
+static void
+emit_store_current_ip (scm_jit_state *j, jit_gpr_t t)
+{
+ emit_movi (j, t, (intptr_t) j->ip);
+ emit_store_ip (j, t);
+}
+
+static void
+emit_pop_fp (scm_jit_state *j, jit_gpr_t old_fp)
+{
+ emit_ldxi (j, old_fp, THREAD, thread_offset_fp);
+ emit_load_prev_fp_offset (j, FP, old_fp);
+ emit_lshi (j, FP, FP, 3); /* Multiply by sizeof (scm_vm_stack_element) */
+ emit_addr (j, FP, old_fp, FP);
+ set_register_state (j, FP_IN_REGISTER);
+ emit_store_fp (j);
+}
+
+static void
+emit_reset_frame (scm_jit_state *j, uint32_t nlocals)
+{
+ ASSERT_HAS_REGISTER_STATE (FP_IN_REGISTER);
+ emit_subtract_stack_slots (j, SP, FP, nlocals);
+ set_register_state (j, SP_IN_REGISTER);
+ emit_store_sp (j);
+ clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
+}
+
+static jit_operand_t
+thread_operand (void)
+{
+ return jit_operand_gpr (JIT_OPERAND_ABI_POINTER, THREAD);
+}
+
+static void
+emit_call_0 (scm_jit_state *j, void *f)
+{
+ jit_calli_0 (j->jit, f);
+ clear_scratch_register_state (j);
+}
+
+static void
+emit_call_1 (scm_jit_state *j, void *f, jit_operand_t a)
+{
+ jit_calli_1 (j->jit, f, a);
+ clear_scratch_register_state (j);
+}
+
+static void
+emit_call_2 (scm_jit_state *j, void *f, jit_operand_t a, jit_operand_t b)
+{
+ jit_calli_2 (j->jit, f, a, b);
+ clear_scratch_register_state (j);
+}
+
+static void
+emit_call_3 (scm_jit_state *j, void *f, jit_operand_t a, jit_operand_t b,
+ jit_operand_t c)
+{
+ jit_calli_3 (j->jit, f, a, b, c);
+ clear_scratch_register_state (j);
+}
+
+static void
+emit_alloc_frame_for_sp (scm_jit_state *j, jit_gpr_t t)
+{
+ jit_reloc_t k, fast, watermark;
+ uint32_t saved_state = save_reloadable_register_state (j);
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ emit_ldxi (j, t, THREAD, thread_offset_sp_min_since_gc);
+ fast = jit_bger (j->jit, SP, t);
+ emit_ldxi (j, t, THREAD, thread_offset_stack_limit);
+ watermark = jit_bger (j->jit, SP, t);
+
+ /* Slow case: call out to expand stack. */
+ emit_store_current_ip (j, t);
+ emit_call_2 (j, scm_vm_intrinsics.expand_stack, thread_operand (),
+ jit_operand_gpr (JIT_OPERAND_ABI_POINTER, SP));
+ restore_reloadable_register_state (j, saved_state);
+ k = jit_jmp (j->jit);
+
+ /* Past sp_min_since_gc, but within stack_limit: update watermark and
+ fall through. */
+ jit_patch_here (j->jit, watermark);
+ jit_stxi (j->jit, thread_offset_sp_min_since_gc, THREAD, SP);
+ jit_patch_here (j->jit, fast);
+ /* Fast case: Just update sp. */
+ emit_store_sp (j);
+ jit_patch_here (j->jit, k);
+
+ clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
+}
+
+static void
+emit_alloc_frame (scm_jit_state *j, jit_gpr_t t, uint32_t nlocals)
+{
+ ASSERT_HAS_REGISTER_STATE (FP_IN_REGISTER);
+ emit_subtract_stack_slots (j, SP, FP, nlocals);
+ set_register_state (j, SP_IN_REGISTER);
+ emit_alloc_frame_for_sp (j, t);
+}
+
+static void
+emit_get_callee_vcode (scm_jit_state *j, jit_gpr_t dst)
+{
+ emit_call_1 (j, scm_vm_intrinsics.get_callee_vcode, thread_operand ());
+ emit_retval (j, dst);
+ emit_reload_sp (j);
+ emit_reload_fp (j);
+}
+
+static void
+emit_get_vcode_low_byte (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t addr)
+{
+ if (uint32_offset_low_byte == 0)
+ jit_ldr_uc (j->jit, dst, addr);
+ else
+ jit_ldxi_uc (j->jit, dst, addr, uint32_offset_low_byte);
+ record_gpr_clobber (j, dst);
+}
+
+static void
+emit_get_ip_relative_addr (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t ip,
+ uint32_t offset)
+{
+ uint32_t byte_offset = offset * sizeof (uint32_t);
+ jit_ldxi_i (j->jit, dst, ip, byte_offset);
+ record_gpr_clobber (j, dst);
+ emit_lshi (j, dst, dst, 2); /* Multiply by sizeof (uint32_t) */
+ emit_addr (j, dst, dst, ip);
+}
+
+static void
+emit_exit (scm_jit_state *j)
+{
+ jit_jmpi (j->jit, exit_mcode);
+}
+
+static jit_reloc_t
+emit_push_frame (scm_jit_state *j, uint32_t proc_slot, uint32_t nlocals,
+ const uint32_t *vra)
+{
+ jit_gpr_t t = T0;
+ jit_reloc_t continuation;
+
+ emit_reload_fp (j);
+ emit_subtract_stack_slots (j, FP, FP, proc_slot);
+ set_register_state (j, FP_IN_REGISTER);
+ continuation = emit_store_mra (j, FP, t);
+ emit_store_vra (j, FP, t, vra);
+ emit_store_prev_fp_offset (j, FP, t, proc_slot);
+ emit_store_fp (j);
+ emit_reset_frame (j, nlocals);
+
+ return continuation;
+}
+
+static void
+emit_indirect_tail_call (scm_jit_state *j)
+{
+ jit_reloc_t not_instrumented, no_mcode;
+
+ emit_get_callee_vcode (j, T0);
+
+ /* FIXME: If all functions start with instrument-entry, no need for
+ this check. */
+ emit_get_vcode_low_byte (j, T1, T0);
+ not_instrumented = jit_bnei (j->jit, T1, scm_op_instrument_entry);
+
+ emit_get_ip_relative_addr (j, T1, T0, 1);
+ emit_ldxi (j, T1, T1, 0);
+ no_mcode = jit_beqi (j->jit, T1, 0);
+ ASSERT_HAS_REGISTER_STATE (FP_IN_REGISTER | SP_IN_REGISTER);
+ jit_jmpr (j->jit, T1);
+
+ jit_patch_here (j->jit, not_instrumented);
+ jit_patch_here (j->jit, no_mcode);
+
+ emit_store_ip (j, T0);
+ emit_exit (j);
+}
+
+static void
+emit_direct_tail_call (scm_jit_state *j, const uint32_t *vcode)
+{
+ ASSERT_HAS_REGISTER_STATE (FP_IN_REGISTER | SP_IN_REGISTER);
+
+ if (vcode == j->start)
+ {
+ jit_jmpi (j->jit, j->labels[0]);
+ }
+ else if ((vcode[0] & 0xff) != scm_op_instrument_entry)
+ {
+ emit_movi (j, T0, (intptr_t) vcode);
+ emit_store_ip (j, T0);
+ emit_exit (j);
+ }
+ else
+ {
+ struct scm_jit_function_data *data;
+ data = (struct scm_jit_function_data *) (vcode + (int32_t)(vcode[1]));
+
+ if (data->mcode)
+ {
+ /* FIXME: Jump indirectly, to allow mcode to be changed
+ (e.g. to add/remove breakpoints or hooks). */
+ jit_jmpi (j->jit, data->mcode);
+ }
+ else
+ {
+ jit_reloc_t no_mcode;
+
+ /* No need to track clobbers. */
+ jit_ldi (j->jit, T0, &data->mcode);
+ no_mcode = jit_beqi (j->jit, T0, 0);
+ jit_jmpr (j->jit, T0);
+ jit_patch_here (j->jit, no_mcode);
+ jit_movi (j->jit, T0, (intptr_t) vcode);
+ emit_store_ip (j, T0);
+ emit_exit (j);
+ }
+ }
+}
+
+static jit_operand_t
+fp_scm_operand (scm_jit_state *j, uint32_t slot)
+{
+ ASSERT_HAS_REGISTER_STATE (FP_IN_REGISTER);
+
+ return jit_operand_mem (JIT_OPERAND_ABI_POINTER, FP,
+ -8 * ((ptrdiff_t) slot + 1));
+}
+
+static void
+emit_fp_ref_scm (scm_jit_state *j, jit_gpr_t dst, uint32_t slot)
+{
+ ASSERT_HAS_REGISTER_STATE (FP_IN_REGISTER);
+
+ emit_ldxi (j, dst, FP, -8 * ((ptrdiff_t) slot + 1));
+}
+
+static void
+emit_fp_set_scm (scm_jit_state *j, uint32_t slot, jit_gpr_t val)
+{
+ ASSERT_HAS_REGISTER_STATE (FP_IN_REGISTER);
+
+ jit_stxi (j->jit, -8 * ((ptrdiff_t) slot + 1), FP, val);
+ clear_register_state (j, SP_CACHE_GPR);
+}
+
+static jit_operand_t
+sp_slot_operand (scm_jit_state *j, uint32_t slot)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ return jit_operand_addi (jit_operand_gpr (JIT_OPERAND_ABI_POINTER, SP),
+ 8 * slot);
+}
+
+static jit_operand_t
+sp_scm_operand (scm_jit_state *j, uint32_t slot)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ return jit_operand_mem (JIT_OPERAND_ABI_POINTER, SP, 8 * slot);
+}
+
+static void
+emit_sp_ref_scm (scm_jit_state *j, jit_gpr_t dst, uint32_t slot)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ emit_ldxi (j, dst, SP, 8 * slot);
+}
+
+static void
+emit_sp_set_scm (scm_jit_state *j, uint32_t slot, jit_gpr_t val)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ if (slot == 0)
+ jit_str (j->jit, SP, val);
+ else
+ jit_stxi (j->jit, 8 * slot, SP, val);
+
+ set_sp_cache_gpr (j, slot, val);
+}
+
+/* Use when you know that the u64 value will be within the size_t range,
+ for example when it's ensured by the compiler. */
+static jit_operand_t
+sp_sz_operand (scm_jit_state *j, uint32_t src)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ enum jit_operand_abi abi =
+ sizeof (size_t) == 4 ? JIT_OPERAND_ABI_UINT32 : JIT_OPERAND_ABI_UINT64;
+
+ if (BIGENDIAN && sizeof (size_t) == 4)
+ return jit_operand_mem (abi, SP, src * 8 + 4);
+ else
+ return jit_operand_mem (abi, SP, src * 8);
+}
+
+static void
+emit_sp_ref_sz (scm_jit_state *j, jit_gpr_t dst, uint32_t src)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ if (BIGENDIAN && sizeof (size_t) == 4)
+ emit_ldxi (j, dst, SP, src * 8 + 4);
+ else
+ emit_ldxi (j, dst, SP, src * 8);
+}
+
+static void
+emit_sp_set_sz (scm_jit_state *j, uint32_t dst, jit_gpr_t src)
+{
+ size_t offset = dst * 8;
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ if (sizeof (size_t) == 4)
+ {
+ size_t lo, hi;
+ if (BIGENDIAN)
+ lo = offset + 4, hi = offset;
+ else
+ lo = offset, hi = offset + 4;
+
+ jit_stxi (j->jit, lo, SP, src);
+ /* Set high word to 0. Clobber src. */
+ emit_xorr (j, src, src, src);
+ jit_stxi (j->jit, hi, SP, src);
+ }
+ else
+ {
+ jit_stxi (j->jit, offset, SP, src);
+ set_sp_cache_gpr (j, dst, src);
+ }
+}
+
+static jit_operand_t
+sp_u64_operand (scm_jit_state *j, uint32_t slot)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ return jit_operand_mem (JIT_OPERAND_ABI_UINT64, SP, 8 * slot);
+}
+
+#if SIZEOF_UINTPTR_T >= 8
+static void
+emit_sp_ref_u64 (scm_jit_state *j, jit_gpr_t dst, uint32_t src)
+{
+ size_t offset = src * 8;
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ emit_ldxi (j, dst, SP, offset);
+}
+
+static void
+emit_sp_set_u64 (scm_jit_state *j, uint32_t dst, jit_gpr_t src)
+{
+ size_t offset = dst * 8;
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ if (dst == 0)
+ jit_str (j->jit, SP, src);
+ else
+ jit_stxi (j->jit, offset, SP, src);
+
+ set_sp_cache_gpr (j, dst, src);
+}
+
+static void
+emit_sp_ref_s64 (scm_jit_state *j, jit_gpr_t dst, uint32_t src)
+{
+ emit_sp_ref_u64 (j, dst, src);
+}
+
+static void
+emit_sp_set_s64 (scm_jit_state *j, uint32_t dst, jit_gpr_t src)
+{
+ emit_sp_set_u64 (j, dst, src);
+}
+
+static void
+emit_sp_ref_ptr (scm_jit_state *j, jit_gpr_t dst, uint32_t src)
+{
+ emit_sp_ref_u64 (j, dst, src);
+}
+
+#else /* SCM_SIZEOF_UINTPTR_T >= 8 */
+
+static jit_operand_t
+sp_s32_operand (scm_jit_state *j, uint32_t src)
+{
+ return sp_sz_operand (j, src);
+}
+
+static void
+emit_sp_ref_s32 (scm_jit_state *j, jit_gpr_t dst, uint32_t src)
+{
+ emit_sp_ref_sz (j, dst, src);
+}
+
+static void
+emit_sp_ref_u64 (scm_jit_state *j, jit_gpr_t dst_lo, jit_gpr_t dst_hi,
+ uint32_t src)
+{
+ size_t offset = src * 8;
+ jit_gpr_t first, second;
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+#if BIGENDIAN
+ first = dst_hi, second = dst_lo;
+#else
+ first = dst_lo, second = dst_hi;
+#endif
+
+ emit_ldxi (j, first, SP, offset);
+ emit_ldxi (j, second, SP, offset + 4);
+}
+
+static void
+emit_sp_set_u64 (scm_jit_state *j, uint32_t dst, jit_gpr_t lo, jit_gpr_t hi)
+{
+ size_t offset = dst * 8;
+ jit_gpr_t first, second;
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+#if BIGENDIAN
+ first = hi, second = lo;
+#else
+ first = lo, second = hi;
+#endif
+
+ if (offset == 0)
+ jit_str (j->jit, SP, first);
+ else
+ jit_stxi (j->jit, offset, SP, first);
+ jit_stxi (j->jit, offset + 4, SP, second);
+
+ clear_register_state (j, SP_CACHE_GPR);
+}
+
+static void
+emit_sp_ref_s64 (scm_jit_state *j, jit_gpr_t dst_lo, jit_gpr_t dst_hi,
+ uint32_t src)
+{
+ emit_sp_ref_u64 (j, dst_lo, dst_hi, src);
+}
+
+static void
+emit_sp_set_s64 (scm_jit_state *j, uint32_t dst, jit_gpr_t lo, jit_gpr_t hi)
+{
+ emit_sp_set_u64 (j, dst, lo, hi);
+}
+
+static void
+emit_sp_ref_u64_lower_half (scm_jit_state *j, jit_gpr_t dst, uint32_t src)
+{
+ size_t offset = src * 8;
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ emit_ldxi (j, dst, SP, offset);
+}
+
+static void
+emit_sp_ref_ptr (scm_jit_state *j, jit_gpr_t dst, uint32_t src)
+{
+ emit_sp_ref_u64_lower_half (j, dst, src);
+}
+#endif /* SCM_SIZEOF_UINTPTR_T >= 8 */
+
+static void
+emit_sp_ref_f64 (scm_jit_state *j, jit_fpr_t dst, uint32_t src)
+{
+ size_t offset = src * 8;
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ if (offset == 0)
+ jit_ldr_d (j->jit, dst, SP);
+ else
+ jit_ldxi_d (j->jit, dst, SP, offset);
+
+ record_fpr_clobber (j, dst);
+}
+
+static void
+emit_sp_set_f64 (scm_jit_state *j, uint32_t dst, jit_fpr_t src)
+{
+ size_t offset = dst * 8;
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+
+ if (offset == 0)
+ jit_str_d (j->jit, SP, src);
+ else
+ jit_stxi_d (j->jit, offset, SP, src);
+
+ set_sp_cache_fpr (j, dst, src);
+}
+
+static void
+emit_mov (scm_jit_state *j, uint32_t dst, uint32_t src, jit_gpr_t t)
+{
+ emit_sp_ref_scm (j, t, src);
+ emit_sp_set_scm (j, dst, t);
+
+ /* FIXME: The compiler currently emits "push", "mov", etc for SCM,
+ F64, U64, and S64 variables. However SCM values are the usual
+ case, and on a 32-bit machine it might be cheaper to move a SCM
+ than to move a 64-bit number. */
+ if (sizeof (void*) < sizeof (union scm_vm_stack_element))
+ {
+ /* Copy the high word as well. */
+ uintptr_t src_offset = src * sizeof (union scm_vm_stack_element);
+ uintptr_t dst_offset = dst * sizeof (union scm_vm_stack_element);
+
+ jit_ldxi (j->jit, t, SP, src_offset + sizeof (void*));
+ jit_stxi (j->jit, dst_offset + sizeof (void*), SP, t);
+
+ clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
+ }
+ else
+ /* In any case since we move the register using GPRs, it won't be in
+ a cached FPR. */
+ clear_register_state (j, SP_CACHE_FPR);
+}
+
+static jit_reloc_t
+emit_branch_if_frame_locals_count_less_than (scm_jit_state *j, jit_gpr_t t,
+ uint32_t nlocals)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER | FP_IN_REGISTER);
+
+ emit_subr (j, t, FP, SP);
+ return jit_blti (j->jit, t, nlocals * sizeof (union scm_vm_stack_element));
+}
+
+static jit_reloc_t
+emit_branch_if_frame_locals_count_eq (scm_jit_state *j, jit_gpr_t t,
+ uint32_t nlocals)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER | FP_IN_REGISTER);
+
+ emit_subr (j, t, FP, SP);
+ return jit_beqi (j->jit, t, nlocals * sizeof (union scm_vm_stack_element));
+}
+
+static jit_reloc_t
+emit_branch_if_frame_locals_count_not_eq (scm_jit_state *j, jit_gpr_t t,
+ uint32_t nlocals)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER | FP_IN_REGISTER);
+
+ emit_subr (j, t, FP, SP);
+ return jit_bnei (j->jit, t, nlocals * sizeof (union scm_vm_stack_element));
+}
+
+static jit_reloc_t
+emit_branch_if_frame_locals_count_greater_than (scm_jit_state *j, jit_gpr_t t,
+ uint32_t nlocals)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER | FP_IN_REGISTER);
+
+ emit_subr (j, t, FP, SP);
+ return jit_bgti (j->jit, t, nlocals * sizeof (union scm_vm_stack_element));
+}
+
+static void
+emit_load_fp_slot (scm_jit_state *j, jit_gpr_t dst, uint32_t slot)
+{
+ ASSERT_HAS_REGISTER_STATE (FP_IN_REGISTER);
+
+ emit_subi (j, dst, FP, (slot + 1) * sizeof (union scm_vm_stack_element));
+}
+
+static jit_reloc_t
+emit_branch_if_immediate (scm_jit_state *j, jit_gpr_t r)
+{
+ return jit_bmsi (j->jit, r, 6);
+}
+
+static void
+emit_load_heap_object_word (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t r,
+ uint32_t word)
+{
+ emit_ldxi (j, dst, r, word * sizeof(SCM));
+}
+
+static void
+emit_load_heap_object_tc (scm_jit_state *j, jit_gpr_t dst, jit_gpr_t r,
+ scm_t_bits mask)
+{
+ emit_load_heap_object_word (j, dst, r, 0);
+ emit_andi (j, dst, dst, mask);
+}
+
+static jit_reloc_t
+emit_branch_if_heap_object_has_tc (scm_jit_state *j, jit_gpr_t r, jit_gpr_t t,
+ scm_t_bits mask, scm_t_bits tc)
+{
+ emit_load_heap_object_tc (j, t, r, mask);
+ return jit_beqi (j->jit, t, tc);
+}
+
+static jit_reloc_t
+emit_branch_if_heap_object_not_tc (scm_jit_state *j, jit_gpr_t r, jit_gpr_t t,
+ scm_t_bits mask, scm_t_bits tc)
+{
+ emit_load_heap_object_tc (j, t, r, mask);
+ return jit_bnei (j->jit, t, tc);
+}
+
+static jit_reloc_t
+emit_branch_if_heap_object_not_tc7 (scm_jit_state *j, jit_gpr_t r, jit_gpr_t t,
+ scm_t_bits tc7)
+{
+ return emit_branch_if_heap_object_not_tc (j, r, t, 0x7f, tc7);
+}
+
+static void
+emit_entry_trampoline (scm_jit_state *j)
+{
+ size_t align = jit_enter_jit_abi(j->jit, 3, 0, 0);
+
+ /* Load our reserved registers: THREAD and SP. Also load IP for the
+ mcode jump. */
+ jit_load_args_2 (j->jit, thread_operand (),
+ jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T0));
+ emit_reload_sp (j);
+
+ /* Load FP, set during call sequences. */
+ emit_reload_fp (j);
+
+ /* Jump to the mcode! */
+ jit_jmpr (j->jit, T0);
+
+ /* Initialize global exit_mcode to point here. */
+ exit_mcode = jit_address (j->jit);
+
+ jit_leave_jit_abi(j->jit, 3, 0, align);
+
+ /* When mcode finishes, interpreter will continue with vp->ip. */
+ jit_ret (j->jit);
+}
+
+static void
+emit_handle_interrupts_trampoline (scm_jit_state *j)
+{
+ /* Precondition: IP synced, MRA in T0. */
+ emit_call_2 (j, scm_vm_intrinsics.push_interrupt_frame,
+ thread_operand (),
+ jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T0));
+ emit_reload_sp (j);
+ emit_reload_fp (j);
+ emit_direct_tail_call (j, scm_vm_intrinsics.handle_interrupt_code);
+}
+
+/* To limit the number of mmap calls and re-emission of JIT code, use
+ 256 kB code arenas. Unused pages won't be resident. Assume pages
+ are power-of-two-sized and this size is a multiple of the page size
+ on all architectures. */
+static const size_t default_code_arena_size = 0x40000;
+
+static struct code_arena *
+allocate_code_arena (size_t size, struct code_arena *prev)
+{
+ struct code_arena *ret = malloc (sizeof (struct code_arena));
+
+ if (!ret) return NULL;
+
+ memset (ret, 0, sizeof (*ret));
+ ret->used = 0;
+ ret->size = size;
+ ret->prev = prev;
+ ret->base = mmap (NULL, ret->size,
+ PROT_EXEC | PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+
+ if (ret->base == MAP_FAILED)
+ {
+ perror ("allocating JIT code buffer failed");
+ free (ret);
+ return NULL;
+ }
+
+ INFO ("allocated code arena, %p-%p\n", ret->base, ret->base + ret->size);
+
+ return ret;
+}
+
+static void *
+emit_code (scm_jit_state *j, void (*emit) (scm_jit_state *))
+{
+ if (!j->code_arena)
+ j->code_arena = allocate_code_arena (default_code_arena_size, NULL);
+
+ if (!j->code_arena)
+ /* Resource exhaustion; turn off JIT. */
+ return NULL;
+
+ while (1)
+ {
+ struct code_arena *arena = j->code_arena;
+
+ jit_begin(j->jit, arena->base + arena->used, arena->size - arena->used);
+
+ uint8_t *ret = jit_address (j->jit);
+
+ emit (j);
+
+ if (!jit_has_overflow (j->jit))
+ {
+ size_t size;
+ jit_end (j->jit, &size);
+ ASSERT (size <= (arena->size - arena->used));
+ DEBUG ("mcode: %p,+%zu\n", ret, size);
+ arena->used += size;
+ /* Align next JIT to 16-byte boundaries to optimize initial
+ icache fetch. */
+ arena->used = (arena->used + 15) & ~15;
+ /* Assertion should not be invalidated as arena size is a
+ multiple of 16. */
+ ASSERT (arena->used <= arena->size);
+ return ret;
+ }
+ else
+ {
+ jit_reset (j->jit);
+ if (arena->used == 0)
+ {
+ /* Code too big to fit into empty arena; allocate a larger
+ one. */
+ INFO ("code didn't fit in empty arena of size %zu\n", arena->size);
+ arena = allocate_code_arena (arena->size * 2, arena->prev);
+ if (!arena)
+ return NULL;
+ munmap (j->code_arena->base, j->code_arena->size);
+ free (j->code_arena);
+ j->code_arena = arena;
+ }
+ else
+ {
+ /* Arena full; allocate another. */
+ /* FIXME: If partial code that we wrote crosses a page
+ boundary, we could tell the OS to forget about the tail
+ pages. */
+ INFO ("code didn't fit in arena tail %zu\n",
+ arena->size - arena->used);
+ arena = allocate_code_arena (arena->size, arena);
+ if (!arena)
+ return NULL;
+ j->code_arena = arena;
+ }
+ }
+ }
+}
+
+static jit_operand_t
+free_variable_operand (scm_jit_state *j, jit_gpr_t src, size_t n)
+{
+ ptrdiff_t offset = (n + program_word_offset_free_variable) * sizeof(SCM);
+ return jit_operand_mem (JIT_OPERAND_ABI_POINTER, src, offset);
+}
+
+static void
+add_inter_instruction_patch (scm_jit_state *j, jit_reloc_t reloc,
+ const uint32_t *target)
+{
+ ASSERT (j->start <= target && target < j->end);
+ ptrdiff_t offset = target - j->start;
+
+ if (j->labels[offset])
+ {
+ jit_patch_there (j->jit, reloc, j->labels[offset]);
+ return;
+ }
+
+ if (j->reloc_idx >= j->reloc_count)
+ {
+ size_t count = j->reloc_count * 2;
+ if (!count) count = 10;
+ size_t size = sizeof(*j->relocs) * count;
+ ASSERT(size / sizeof(*j->relocs) == count);
+ struct pending_reloc *relocs = realloc (j->relocs, size);
+ if (relocs)
+ {
+ j->reloc_count = count;
+ j->relocs = relocs;
+ }
+ }
+
+ ASSERT (j->reloc_idx < j->reloc_count);
+ j->relocs[j->reloc_idx].reloc = reloc;
+ j->relocs[j->reloc_idx].target_vcode_offset = offset;
+ j->reloc_idx++;
+}
+
+
+
+static void
+bad_instruction (scm_jit_state *j)
+{
+ ASSERT (0);
+}
+
+static void
+compile_halt (scm_jit_state *j)
+{
+ bad_instruction (j);
+}
+
+static void
+compile_call (scm_jit_state *j, uint32_t proc, uint32_t nlocals)
+{
+ /* 2 = size of call inst */
+ jit_reloc_t mcont = emit_push_frame (j, proc, nlocals, j->ip + 2);
+
+ emit_indirect_tail_call (j);
+
+ jit_patch_here (j->jit, mcont);
+
+ reset_register_state (j, FP_IN_REGISTER | SP_IN_REGISTER);
+ j->frame_size = -1;
+}
+
+static void
+compile_call_label (scm_jit_state *j, uint32_t proc, uint32_t nlocals, const uint32_t *vcode)
+{
+ /* 2 = size of call-label inst */
+ jit_reloc_t mcont = emit_push_frame (j, proc, nlocals, j->ip + 3);
+
+ emit_direct_tail_call (j, vcode);
+
+ jit_patch_here (j->jit, mcont);
+
+ reset_register_state (j, FP_IN_REGISTER | SP_IN_REGISTER);
+ j->frame_size = -1;
+}
+
+static void
+compile_tail_call (scm_jit_state *j)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+ restore_reloadable_register_state (j, FP_IN_REGISTER);
+
+ emit_indirect_tail_call (j);
+
+ j->frame_size = -1;
+}
+
+static void
+compile_tail_call_label (scm_jit_state *j, const uint32_t *vcode)
+{
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER);
+ restore_reloadable_register_state (j, FP_IN_REGISTER);
+
+ emit_direct_tail_call (j, vcode);
+
+ j->frame_size = -1;
+}
+
+static void
+compile_instrument_entry (scm_jit_state *j, void *data)
+{
+}
+
+static void
+compile_instrument_loop (scm_jit_state *j, void *data)
+{
+ /* Nothing to do. */
+}
+
+static void
+compile_receive (scm_jit_state *j, uint16_t dst, uint16_t proc, uint32_t nlocals)
+{
+ jit_gpr_t t = T0;
+ jit_reloc_t k;
+ uint32_t saved_state = j->register_state;
+
+ k = emit_branch_if_frame_locals_count_greater_than (j, t, proc);
+ emit_store_current_ip (j, T0);
+ emit_call_0 (j, scm_vm_intrinsics.error_no_values);
+ j->register_state = saved_state;
+ jit_patch_here (j->jit, k);
+ emit_fp_ref_scm (j, t, proc);
+ emit_fp_set_scm (j, dst, t);
+ emit_reset_frame (j, nlocals);
+
+ j->frame_size = nlocals;
+}
+
+static void
+compile_receive_values (scm_jit_state *j, uint32_t proc, uint8_t allow_extra,
+ uint32_t nvalues)
+{
+ jit_gpr_t t = T0;
+ uint32_t saved_state = j->register_state;
+
+ if (allow_extra)
+ {
+ jit_reloc_t k;
+ k = emit_branch_if_frame_locals_count_greater_than (j, t, proc+nvalues-1);
+ emit_store_current_ip (j, T0);
+ emit_call_0 (j, scm_vm_intrinsics.error_not_enough_values);
+ j->register_state = saved_state;
+ jit_patch_here (j->jit, k);
+ }
+ else
+ {
+ jit_reloc_t k;
+ k = emit_branch_if_frame_locals_count_eq (j, t, proc + nvalues);
+ emit_store_current_ip (j, T0);
+ emit_call_1 (j, scm_vm_intrinsics.error_wrong_number_of_values,
+ jit_operand_imm (JIT_OPERAND_ABI_UINT32, nvalues));
+ j->register_state = saved_state;
+ jit_patch_here (j->jit, k);
+
+ j->frame_size = proc + nvalues;
+ }
+
+ clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
+}
+
+static void
+compile_shuffle_down (scm_jit_state *j, uint16_t from, uint16_t to)
+{
+ jit_gpr_t walk = T0, t = T1;
+ size_t offset = (from - to) * sizeof (union scm_vm_stack_element);
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER | FP_IN_REGISTER);
+
+ emit_load_fp_slot (j, walk, from);
+ jit_reloc_t done = jit_bltr (j->jit, walk, SP);
+ void *head = jit_address (j->jit);
+ jit_ldr (j->jit, t, walk);
+ jit_stxi (j->jit, offset, walk, t);
+ jit_subi (j->jit, walk, walk, sizeof (union scm_vm_stack_element));
+ jit_patch_there (j->jit, jit_bger (j->jit, walk, SP), head);
+ jit_patch_here (j->jit, done);
+ jit_addi (j->jit, SP, SP, offset);
+ emit_store_sp (j);
+
+ clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
+
+ if (j->frame_size >= 0)
+ j->frame_size -= (from - to);
+}
+
+static void
+compile_return_values (scm_jit_state *j)
+{
+ jit_gpr_t old_fp = T0, ra = T1;
+ jit_reloc_t interp;
+
+ emit_pop_fp (j, old_fp);
+
+ emit_load_mra (j, ra, old_fp);
+ interp = jit_beqi (j->jit, ra, 0);
+ jit_jmpr (j->jit, ra);
+
+ jit_patch_here (j->jit, interp);
+ emit_load_vra (j, ra, old_fp);
+ emit_store_ip (j, ra);
+ emit_exit (j);
+
+ j->frame_size = -1;
+}
+
+static void
+compile_subr_call (scm_jit_state *j, uint32_t idx)
+{
+ jit_gpr_t t = T0, ret = T1;
+ void *subr;
+ jit_reloc_t immediate, not_values, k;
+ jit_operand_t args[10];
+
+ ASSERT (j->frame_size > 0);
+ size_t argc = j->frame_size - 1;
+ ASSERT (argc <= 10);
+
+ subr = scm_subr_function_by_index (idx);
+ emit_store_current_ip (j, t);
+ for (size_t i = 2; i <= j->frame_size; i++)
+ args[i - 2] = sp_scm_operand (j, (j->frame_size - i));
+ jit_calli (j->jit, subr, argc, args);
+ clear_scratch_register_state (j);
+ jit_retval (j->jit, ret);
+
+ immediate = emit_branch_if_immediate (j, ret);
+ not_values = emit_branch_if_heap_object_not_tc7 (j, ret, t, scm_tc7_values);
+ emit_call_2 (j, scm_vm_intrinsics.unpack_values_object, thread_operand (),
+ jit_operand_gpr (JIT_OPERAND_ABI_POINTER, ret));
+ emit_reload_fp (j);
+ emit_reload_sp (j);
+ k = jit_jmp (j->jit);
+
+ jit_patch_here (j->jit, immediate);
+ jit_patch_here (j->jit, not_values);
+ emit_reload_fp (j);
+ emit_subtract_stack_slots (j, SP, FP, 1);
+ set_register_state (j, SP_IN_REGISTER);
+ emit_store_sp (j);
+ jit_str (j->jit, SP, ret);
+ jit_patch_here (j->jit, k);
+
+ clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
+
+ j->frame_size = -1;
+}
+
+static void
+compile_foreign_call (scm_jit_state *j, uint16_t cif_idx, uint16_t ptr_idx)
+{
+ uint32_t saved_state;
+
+ ASSERT (j->frame_size >= 0);
+
+ emit_store_current_ip (j, T0);
+ emit_sp_ref_scm (j, T0, j->frame_size - 1);
+
+ /* FIXME: Inline the foreign call. */
+ saved_state = save_reloadable_register_state (j);
+ emit_call_3 (j, scm_vm_intrinsics.foreign_call, thread_operand (),
+ free_variable_operand (j, T0, cif_idx),
+ free_variable_operand (j, T0, ptr_idx));
+ restore_reloadable_register_state (j, saved_state);
+
+ j->frame_size = 2; /* Return value and errno. */
+}
+
+static void
+compile_continuation_call (scm_jit_state *j, uint32_t contregs_idx)
+{
+ emit_reload_fp (j);
+ emit_store_current_ip (j, T0);
+ emit_fp_ref_scm (j, T0, 0);
+ emit_call_2 (j, scm_vm_intrinsics.reinstate_continuation_x,
+ thread_operand (), free_variable_operand (j, T0, contregs_idx));
+ /* Does not fall through. */
+
+ j->frame_size = -1;
+}
+
+static void
+compile_compose_continuation (scm_jit_state *j, uint32_t cont_idx)
+{
+ jit_reloc_t interp;
+
+ ASSERT_HAS_REGISTER_STATE (SP_IN_REGISTER | FP_IN_REGISTER);
+
+ emit_store_current_ip (j, T0);
+ emit_fp_ref_scm (j, T0, 0);
+ emit_call_2 (j, scm_vm_intrinsics.compose_continuation,
+ thread_operand (), free_variable_operand (j, T0, cont_idx));
+ jit_retval (j->jit, T0);
+ interp = jit_beqi (j->jit, T0, 0);
+ emit_reload_sp (j);
+ emit_reload_fp (j);
+ jit_jmpr (j->jit, T0);
+
+ jit_patch_here (j->jit, interp);
+ emit_exit (j);
+
+ j->frame_size = -1;
+}
+
+static void
+compile_capture_continuation (scm_jit_state *j, uint32_t dst)
+{
+ emit_store_current_ip (j, T0);
+ emit_call_1 (j, scm_vm_intrinsics.capture_continuation, thread_operand ());
+ jit_retval (j->jit, T0);
+ emit_reload_sp (j);
+ emit_reload_fp (j);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_abort (scm_jit_state *j)
+{
+ jit_reloc_t k, interp;
+
+ jit_movi (j->jit, T0, (intptr_t) (j->ip + 1));
+ emit_store_ip (j, T0);
+ k = jit_mov_addr (j->jit, T0);
+ emit_call_2 (j, scm_vm_intrinsics.abort_to_prompt, thread_operand (),
+ jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T0));
+ jit_retval (j->jit, T1_PRESERVED);
+
+ interp = jit_beqi (j->jit, T1_PRESERVED, 0);
+ emit_reload_sp (j);
+ emit_reload_fp (j);
+ jit_jmpr (j->jit, T1_PRESERVED);
+
+ jit_patch_here (j->jit, interp);
+ emit_exit (j);
+
+ jit_patch_here (j->jit, k);
+
+ j->frame_size = -1;
+}
+
+static void
+compile_builtin_ref (scm_jit_state *j, uint16_t dst, uint16_t idx)
+{
+ SCM builtin = scm_vm_builtin_ref (idx);
+
+ emit_movi (j, T0, SCM_UNPACK (builtin));
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_throw (scm_jit_state *j, uint16_t key, uint16_t args)
+{
+ emit_store_current_ip (j, T0);
+ emit_call_2 (j, scm_vm_intrinsics.throw_, sp_scm_operand (j, key),
+ sp_scm_operand (j, args));
+ /* throw_ does not return. */
+}
+
+static void
+compile_throw_value (scm_jit_state *j, uint32_t val,
+ const void *key_subr_and_message)
+{
+ emit_store_current_ip (j, T0);
+ emit_call_2 (j, scm_vm_intrinsics.throw_with_value, sp_scm_operand (j, val),
+ jit_operand_imm (JIT_OPERAND_ABI_POINTER,
+ (intptr_t) key_subr_and_message));
+ /* throw_with_value does not return. */
+}
+
+static void
+compile_throw_value_and_data (scm_jit_state *j, uint32_t val,
+ const void *key_subr_and_message)
+{
+ emit_store_current_ip (j, T0);
+ emit_call_2 (j, scm_vm_intrinsics.throw_with_value_and_data,
+ sp_scm_operand (j, val),
+ jit_operand_imm (JIT_OPERAND_ABI_POINTER,
+ (intptr_t) key_subr_and_message));
+ /* throw_with_value_and_data does not return. */
+}
+
+static void
+compile_assert_nargs_ee (scm_jit_state *j, uint32_t nlocals)
+{
+ jit_reloc_t k;
+ jit_gpr_t t = T0;
+ uint32_t saved_state = j->register_state;
+
+ k = emit_branch_if_frame_locals_count_eq (j, t, nlocals);
+ emit_store_current_ip (j, t);
+ emit_call_1 (j, scm_vm_intrinsics.error_wrong_num_args,
+ thread_operand ());
+ jit_patch_here (j->jit, k);
+
+ j->register_state = saved_state;
+ j->frame_size = nlocals;
+}
+
+static void
+compile_assert_nargs_ge (scm_jit_state *j, uint32_t nlocals)
+{
+ if (nlocals > 0)
+ {
+ jit_gpr_t t = T0;
+ jit_reloc_t k;
+ uint32_t saved_state = j->register_state;
+
+ k = emit_branch_if_frame_locals_count_greater_than (j, t, nlocals-1);
+ emit_store_current_ip (j, t);
+ emit_call_1 (j, scm_vm_intrinsics.error_wrong_num_args,
+ thread_operand ());
+ jit_patch_here (j->jit, k);
+ j->register_state = saved_state;
+ }
+}
+
+static void
+compile_assert_nargs_le (scm_jit_state *j, uint32_t nlocals)
+{
+ jit_reloc_t k;
+ jit_gpr_t t = T0;
+ uint32_t saved_state = j->register_state;
+
+ k = emit_branch_if_frame_locals_count_less_than (j, t, nlocals + 1);
+ emit_store_current_ip (j, t);
+ emit_call_1 (j, scm_vm_intrinsics.error_wrong_num_args,
+ thread_operand ());
+ jit_patch_here (j->jit, k);
+
+ j->register_state = saved_state;
+}
+
+static void
+compile_alloc_frame (scm_jit_state *j, uint32_t nlocals)
+{
+ jit_gpr_t t = T0, saved_frame_size = T1_PRESERVED;
+
+ if (j->frame_size < 0)
+ jit_subr (j->jit, saved_frame_size, FP, SP);
+
+ /* This will clear the regalloc, so no need to track clobbers. */
+ emit_alloc_frame (j, t, nlocals);
+
+ if (j->frame_size >= 0)
+ {
+ int32_t slots = nlocals - j->frame_size;
+
+ if (slots > 0)
+ {
+ jit_movi (j->jit, t, SCM_UNPACK (SCM_UNDEFINED));
+ while (slots-- > 0)
+ emit_sp_set_scm (j, slots, t);
+ }
+ }
+ else
+ {
+ jit_gpr_t walk = saved_frame_size;
+
+ jit_subr (j->jit, walk, FP, saved_frame_size);
+ jit_reloc_t k = jit_bler (j->jit, walk, SP);
+ jit_movi (j->jit, t, SCM_UNPACK (SCM_UNDEFINED));
+ void *head = jit_address (j->jit);
+ jit_subi (j->jit, walk, walk, sizeof (union scm_vm_stack_element));
+ jit_str (j->jit, walk, t);
+ jit_patch_there (j->jit, jit_bner (j->jit, walk, SP), head);
+ jit_patch_here (j->jit, k);
+ }
+
+ j->frame_size = nlocals;
+}
+
+static void
+compile_reset_frame (scm_jit_state *j, uint32_t nlocals)
+{
+ restore_reloadable_register_state (j, FP_IN_REGISTER);
+ emit_reset_frame (j, nlocals);
+
+ j->frame_size = nlocals;
+}
+
+static void
+compile_push (scm_jit_state *j, uint32_t src)
+{
+ jit_gpr_t t = T0;
+ jit_subi (j->jit, SP, SP, sizeof (union scm_vm_stack_element));
+ emit_alloc_frame_for_sp (j, t);
+ emit_mov (j, 0, src + 1, t);
+
+ clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
+
+ if (j->frame_size >= 0)
+ j->frame_size++;
+}
+
+static void
+compile_pop (scm_jit_state *j, uint32_t dst)
+{
+ emit_mov (j, dst + 1, 0, T0);
+ jit_addi (j->jit, SP, SP, sizeof (union scm_vm_stack_element));
+ emit_store_sp (j);
+
+ clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
+
+ if (j->frame_size >= 0)
+ j->frame_size--;
+}
+
+static void
+compile_drop (scm_jit_state *j, uint32_t nvalues)
+{
+ jit_addi (j->jit, SP, SP, nvalues * sizeof (union scm_vm_stack_element));
+ emit_store_sp (j);
+
+ clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
+
+ if (j->frame_size >= 0)
+ j->frame_size -= nvalues;
+}
+
+static void
+compile_assert_nargs_ee_locals (scm_jit_state *j, uint16_t expected,
+ uint16_t nlocals)
+{
+ compile_assert_nargs_ee (j, expected);
+ if (nlocals)
+ compile_alloc_frame (j, expected + nlocals);
+}
+
+static void
+compile_expand_apply_argument (scm_jit_state *j)
+{
+ emit_store_current_ip (j, T0);
+ emit_call_1 (j, scm_vm_intrinsics.expand_apply_argument, thread_operand ());
+ emit_reload_sp (j);
+ emit_reload_fp (j);
+
+ j->frame_size = -1;
+}
+
+static void
+compile_bind_kwargs (scm_jit_state *j, uint32_t nreq, uint8_t flags,
+ uint32_t nreq_and_opt, uint32_t ntotal, const void *kw)
+{
+ uint8_t allow_other_keys = flags & 0x1, has_rest = flags & 0x2;
+ jit_gpr_t t = T0, npositional = T1;
+
+ emit_store_current_ip (j, t);
+
+ emit_call_3 (j, scm_vm_intrinsics.compute_kwargs_npositional,
+ thread_operand (),
+ jit_operand_imm (JIT_OPERAND_ABI_UINT32, nreq),
+ jit_operand_imm (JIT_OPERAND_ABI_UINT32, nreq_and_opt - nreq));
+ jit_retval_i (j->jit, npositional);
+
+ jit_operand_t args[] =
+ { jit_operand_gpr (JIT_OPERAND_ABI_POINTER, THREAD),
+ jit_operand_gpr (JIT_OPERAND_ABI_UINT32, npositional),
+ jit_operand_imm (JIT_OPERAND_ABI_UINT32, ntotal),
+ jit_operand_imm (JIT_OPERAND_ABI_POINTER, (intptr_t)kw),
+ jit_operand_imm (JIT_OPERAND_ABI_UINT8, !has_rest),
+ jit_operand_imm (JIT_OPERAND_ABI_UINT8, allow_other_keys) };
+ jit_calli (j->jit, scm_vm_intrinsics.bind_kwargs, 6, args);
+ clear_scratch_register_state (j);
+
+ if (has_rest)
+ {
+ emit_call_2 (j, scm_vm_intrinsics.cons_rest, thread_operand (),
+ jit_operand_imm (JIT_OPERAND_ABI_UINT32, ntotal));
+ jit_retval (j->jit, t);
+ emit_reload_fp (j);
+ emit_fp_set_scm (j, nreq_and_opt, t);
+ }
+ else
+ emit_reload_fp (j);
+
+ emit_reset_frame (j, ntotal);
+ j->frame_size = ntotal;
+}
+
+static void
+compile_bind_rest (scm_jit_state *j, uint32_t dst)
+{
+ jit_reloc_t k, cons;
+ jit_gpr_t t = T1;
+
+ cons = emit_branch_if_frame_locals_count_greater_than (j, t, dst);
+
+ compile_alloc_frame (j, dst + 1);
+ emit_movi (j, t, SCM_UNPACK (SCM_EOL));
+ emit_sp_set_scm (j, 0, t);
+ k = jit_jmp (j->jit);
+
+ jit_patch_here (j->jit, cons);
+ emit_store_current_ip (j, t);
+ emit_call_2 (j, scm_vm_intrinsics.cons_rest, thread_operand (),
+ jit_operand_imm (JIT_OPERAND_ABI_UINT32, dst));
+ emit_retval (j, t);
+ compile_reset_frame (j, dst + 1);
+ emit_sp_set_scm (j, 0, t);
+
+ jit_patch_here (j->jit, k);
+}
+
+static void
+compile_allocate_words (scm_jit_state *j, uint16_t dst, uint16_t nwords)
+{
+ jit_gpr_t t = T0;
+
+ emit_store_current_ip (j, t);
+ emit_call_2 (j, scm_vm_intrinsics.allocate_words, thread_operand (),
+ sp_sz_operand (j, nwords));
+ emit_retval (j, t);
+ record_gpr_clobber (j, t);
+ emit_reload_sp (j);
+ emit_sp_set_scm (j, dst, t);
+}
+
+static void
+compile_allocate_words_immediate (scm_jit_state *j, uint16_t dst, uint16_t nwords)
+{
+ jit_gpr_t t = T0;
+
+ emit_store_current_ip (j, t);
+ emit_call_2 (j, scm_vm_intrinsics.allocate_words, thread_operand (),
+ jit_operand_imm (JIT_OPERAND_ABI_WORD, nwords));
+ emit_retval (j, t);
+ emit_reload_sp (j);
+ emit_sp_set_scm (j, dst, t);
+}
+
+static void
+compile_scm_ref (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_sp_ref_sz (j, T1, idx);
+ emit_lshi (j, T1, T1, log2_sizeof_uintptr_t);
+ emit_ldxr (j, T0, T0, T1);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_scm_set (scm_jit_state *j, uint8_t obj, uint8_t idx, uint8_t val)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_sp_ref_sz (j, T1, idx);
+ emit_sp_ref_scm (j, T2, val);
+ emit_lshi (j, T1, T1, log2_sizeof_uintptr_t);
+ jit_stxr (j->jit, T0, T1, T2);
+}
+
+static void
+compile_scm_ref_tag (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t tag)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_ldr (j, T0, T0);
+ emit_subi (j, T0, T0, tag);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_scm_set_tag (scm_jit_state *j, uint8_t obj, uint8_t tag, uint8_t val)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_sp_ref_scm (j, T1, val);
+ emit_addi (j, T1, T1, tag);
+ jit_str (j->jit, T0, T1);
+}
+
+static void
+compile_scm_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_ldxi (j, T0, T0, idx * sizeof (SCM));
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_scm_set_immediate (scm_jit_state *j, uint8_t obj, uint8_t idx, uint8_t val)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_sp_ref_scm (j, T1, val);
+ jit_stxi (j->jit, idx * sizeof (SCM), T0, T1);
+}
+
+static void
+compile_word_ref (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_sp_ref_sz (j, T1, idx);
+ emit_lshi (j, T1, T1, log2_sizeof_uintptr_t);
+ emit_ldxr (j, T0, T0, T1);
+ emit_sp_set_sz (j, dst, T0);
+}
+
+static void
+compile_word_set (scm_jit_state *j, uint8_t obj, uint8_t idx, uint8_t val)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_sp_ref_sz (j, T1, idx);
+ emit_sp_ref_sz (j, T2, val);
+ emit_lshi (j, T1, T1, log2_sizeof_uintptr_t);
+ jit_stxr (j->jit, T0, T1, T2);
+}
+
+static void
+compile_word_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_ldxi (j, T0, T0, idx * sizeof (SCM));
+ emit_sp_set_sz (j, dst, T0);
+}
+
+static void
+compile_word_set_immediate (scm_jit_state *j, uint8_t obj, uint8_t idx, uint8_t val)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_sp_ref_sz (j, T1, val);
+ jit_stxi (j->jit, idx * sizeof (SCM), T0, T1);
+}
+
+static void
+compile_pointer_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_ldxi (j, T0, T0, idx * sizeof (SCM));
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_pointer_set_immediate (scm_jit_state *j, uint8_t obj, uint8_t idx, uint8_t val)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_sp_ref_scm (j, T1, val);
+ jit_stxi (j->jit, idx * sizeof (SCM), T0, T1);
+}
+
+static void
+compile_tail_pointer_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t idx)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ emit_addi (j, T0, T0, idx * sizeof (SCM));
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_mov (scm_jit_state *j, uint16_t dst, uint16_t src)
+{
+ emit_mov (j, dst, src, T0);
+}
+
+static void
+compile_long_mov (scm_jit_state *j, uint32_t dst, uint32_t src)
+{
+ emit_mov (j, dst, src, T0);
+}
+
+static void
+compile_long_fmov (scm_jit_state *j, uint32_t dst, uint32_t src)
+{
+ jit_gpr_t t = T0;
+ restore_reloadable_register_state (j, FP_IN_REGISTER);
+ emit_fp_ref_scm (j, t, src);
+ emit_fp_set_scm (j, dst, t);
+}
+
+static void
+compile_call_scm_from_scm_scm (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+ int has_fast = 0;
+ jit_reloc_t fast;
+
+ jit_operand_t op_a = sp_scm_operand (j, a);
+ jit_operand_t op_b = sp_scm_operand (j, b);
+
+ switch ((enum scm_vm_intrinsic) idx)
+ {
+ case SCM_VM_INTRINSIC_ADD:
+ {
+ emit_sp_ref_scm (j, T0, a);
+ emit_sp_ref_scm (j, T1, b);
+ op_a = jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T0);
+ op_b = jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T1);
+ jit_reloc_t a_not_inum = jit_bmci (j->jit, T0, scm_tc2_int);
+ jit_reloc_t b_not_inum = jit_bmci (j->jit, T1, scm_tc2_int);
+ jit_subi (j->jit, T0, T0, scm_tc2_int);
+ fast = jit_bxaddr (j->jit, T0, T1);
+ has_fast = 1;
+ /* Restore previous value before slow path. */
+ jit_subr (j->jit, T0, T0, T1);
+ jit_addi (j->jit, T0, T0, scm_tc2_int);
+ jit_patch_here (j->jit, a_not_inum);
+ jit_patch_here (j->jit, b_not_inum);
+ break;
+ }
+ case SCM_VM_INTRINSIC_SUB:
+ {
+ emit_sp_ref_scm (j, T0, a);
+ emit_sp_ref_scm (j, T1, b);
+ op_a = jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T0);
+ op_b = jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T1);
+ jit_reloc_t a_not_inum = jit_bmci (j->jit, T0, scm_tc2_int);
+ jit_reloc_t b_not_inum = jit_bmci (j->jit, T1, scm_tc2_int);
+ jit_subi (j->jit, T1, T1, scm_tc2_int);
+ fast = jit_bxsubr (j->jit, T0, T1);
+ has_fast = 1;
+ /* Restore previous values before slow path. */
+ jit_addr (j->jit, T0, T0, T1);
+ jit_addi (j->jit, T1, T1, scm_tc2_int);
+ jit_patch_here (j->jit, a_not_inum);
+ jit_patch_here (j->jit, b_not_inum);
+ break;
+ }
+ default:
+ break;
+ }
+
+ emit_store_current_ip (j, T2);
+ emit_call_2 (j, intrinsic, op_a, op_b);
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+
+ if (has_fast)
+ jit_patch_here (j->jit, fast);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_call_scm_from_scm_uimm (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+ int has_fast = 0;
+ jit_reloc_t fast;
+
+ jit_operand_t op_a = sp_scm_operand (j, a);
+ jit_operand_t op_b = jit_operand_imm (JIT_OPERAND_ABI_UINT8, b);
+
+ switch ((enum scm_vm_intrinsic) idx)
+ {
+ case SCM_VM_INTRINSIC_ADD_IMMEDIATE:
+ {
+ emit_sp_ref_scm (j, T0, a);
+ op_a = jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T0);
+ scm_t_bits addend = b << 2;
+ jit_reloc_t not_inum = jit_bmci (j->jit, T0, 2);
+ fast = jit_bxaddi (j->jit, T0, addend);
+ has_fast = 1;
+ /* Restore previous value before slow path. */
+ jit_subi (j->jit, T0, T0, addend);
+ jit_patch_here (j->jit, not_inum);
+ break;
+ }
+ case SCM_VM_INTRINSIC_SUB_IMMEDIATE:
+ {
+ emit_sp_ref_scm (j, T0, a);
+ op_a = jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T0);
+ scm_t_bits subtrahend = b << 2;
+ jit_reloc_t not_inum = jit_bmci (j->jit, T0, 2);
+ fast = jit_bxsubi (j->jit, T0, subtrahend);
+ has_fast = 1;
+ /* Restore previous value before slow path. */
+ jit_addi (j->jit, T0, T0, subtrahend);
+ jit_patch_here (j->jit, not_inum);
+ break;
+ }
+ default:
+ break;
+ }
+
+ emit_store_current_ip (j, T1);
+ emit_call_2 (j, intrinsic, op_a, op_b);
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+
+ if (has_fast)
+ jit_patch_here (j->jit, fast);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_call_scm_sz_u32 (scm_jit_state *j, uint8_t a, uint8_t b, uint8_t c, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+ emit_call_3 (j, intrinsic, sp_scm_operand (j, a), sp_sz_operand (j, b),
+ sp_sz_operand (j, c));
+ emit_reload_sp (j);
+}
+
+static void
+compile_call_scm_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+ emit_call_1 (j, intrinsic, sp_scm_operand (j, a));
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_call_f64_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+ emit_call_1 (j, intrinsic, sp_scm_operand (j, a));
+ emit_retval_d (j, JIT_F0);
+ emit_reload_sp (j);
+ emit_sp_set_f64 (j, dst, JIT_F0);
+}
+
+static void
+compile_call_u64_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+#if INDIRECT_INT64_INTRINSICS
+ emit_call_2 (j, intrinsic, sp_slot_operand (j, dst), sp_scm_operand (j, a));
+ emit_reload_sp (j);
+#else
+ emit_call_1 (j, intrinsic, sp_scm_operand (j, a));
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+ emit_sp_set_u64 (j, dst, T0);
+#endif
+}
+
+static void
+compile_make_short_immediate (scm_jit_state *j, uint8_t dst, SCM a)
+{
+ emit_movi (j, T0, SCM_UNPACK (a));
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_make_long_immediate (scm_jit_state *j, uint32_t dst, SCM a)
+{
+ emit_movi (j, T0, SCM_UNPACK (a));
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_make_long_long_immediate (scm_jit_state *j, uint32_t dst, SCM a)
+{
+ emit_movi (j, T0, SCM_UNPACK (a));
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_make_non_immediate (scm_jit_state *j, uint32_t dst, const void *data)
+{
+ emit_movi (j, T0, (uintptr_t)data);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_static_ref (scm_jit_state *j, uint32_t dst, void *loc)
+{
+ emit_ldi (j, T0, loc);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_static_set (scm_jit_state *j, uint32_t obj, void *loc)
+{
+ emit_sp_ref_scm (j, T0, obj);
+ jit_sti (j->jit, loc, T0);
+}
+
+static void
+compile_static_patch (scm_jit_state *j, void *dst, const void *src)
+{
+ emit_movi (j, T0, (uintptr_t) src);
+ jit_sti (j->jit, dst, T0);
+}
+
+static void
+compile_prompt (scm_jit_state *j, uint32_t tag, uint8_t escape_only_p,
+ uint32_t proc_slot, const uint32_t *vcode)
+{
+ emit_store_current_ip (j, T0);
+
+ emit_reload_fp (j);
+ jit_subi (j->jit, FP, FP, proc_slot * sizeof (union scm_vm_stack_element));
+ jit_reloc_t mra = emit_mov_addr (j, T2);
+
+ jit_operand_t args[] =
+ { thread_operand (),
+ jit_operand_imm (JIT_OPERAND_ABI_UINT8, escape_only_p),
+ sp_scm_operand (j, tag),
+ jit_operand_gpr (JIT_OPERAND_ABI_POINTER, FP),
+ jit_operand_imm (JIT_OPERAND_ABI_POINTER, (uintptr_t)vcode),
+ jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T2) };
+ jit_calli (j->jit, scm_vm_intrinsics.push_prompt, 6, args);
+ clear_scratch_register_state (j);
+ emit_reload_sp (j);
+ emit_reload_fp (j);
+ add_inter_instruction_patch (j, mra, vcode);
+}
+
+static void
+compile_load_label (scm_jit_state *j, uint32_t dst, const uint32_t *vcode)
+{
+ emit_movi (j, T0, (uintptr_t) vcode);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_movi (j, T1, 0);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_call_s64_from_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t idx)
+{
+ compile_call_u64_from_scm (j, dst, a, idx);
+}
+
+static void
+compile_call_scm_from_u64 (scm_jit_state *j, uint16_t dst, uint16_t src, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+#if INDIRECT_INT64_INTRINSICS
+ emit_call_1 (j, intrinsic, sp_slot_operand (j, src));
+#else
+ emit_call_1 (j, intrinsic, sp_u64_operand (j, src));
+#endif
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_call_scm_from_s64 (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t b)
+{
+ compile_call_scm_from_u64 (j, dst, a, b);
+}
+
+static void
+compile_tag_char (scm_jit_state *j, uint16_t dst, uint16_t src)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, src);
+#else
+ emit_sp_ref_u64_lower_half (j, T0, src);
+#endif
+ emit_lshi (j, T0, T0, 8);
+ emit_addi (j, T0, T0, scm_tc8_char);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_untag_char (scm_jit_state *j, uint16_t dst, uint16_t src)
+{
+ emit_sp_ref_scm (j, T0, src);
+ emit_rshi (j, T0, T0, 8);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_movi (j, T1, 0);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_atomic_scm_ref_immediate (scm_jit_state *j, uint8_t dst, uint8_t obj, uint8_t offset)
+{
+#if defined(__i386__) || defined(__x86_64__)
+ /* Disassembly of atomic_ref_scm is just a mov. */
+ emit_sp_ref_scm (j, T0, obj);
+ emit_ldxi (j, T0, T0, offset * sizeof (SCM));
+#else
+ emit_movr (j, T1_PRESERVED, SP);
+ emit_call_1 (j, scm_vm_intrinsics.atomic_ref_scm,
+ jit_operand_addi (sp_scm_operand (j, obj),
+ offset * sizeof (SCM)));
+ emit_retval (j, T0);
+ emit_movr (j, SP, T1_PRESERVED);
+ set_register_state (j, SP_IN_REGISTER);
+#endif
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_atomic_scm_set_immediate (scm_jit_state *j, uint8_t obj, uint8_t offset, uint8_t val)
+{
+ emit_movr (j, T0_PRESERVED, SP);
+ emit_call_2 (j, scm_vm_intrinsics.atomic_set_scm,
+ jit_operand_addi (sp_scm_operand (j, obj),
+ offset * sizeof (SCM)),
+ sp_scm_operand (j, val));
+ emit_movr (j, SP, T0_PRESERVED);
+ set_register_state (j, SP_IN_REGISTER);
+}
+
+static void
+compile_atomic_scm_swap_immediate (scm_jit_state *j, uint32_t dst, uint32_t obj, uint8_t offset, uint32_t val)
+{
+ emit_movr (j, T0_PRESERVED, SP);
+ emit_call_2 (j, scm_vm_intrinsics.atomic_swap_scm,
+ jit_operand_addi (sp_scm_operand (j, obj),
+ offset * sizeof (SCM)),
+ sp_scm_operand (j, val));
+ emit_retval (j, T1);
+ emit_movr (j, SP, T0_PRESERVED);
+ set_register_state (j, SP_IN_REGISTER);
+ emit_sp_set_scm (j, dst, T1);
+}
+
+static void
+compile_atomic_scm_compare_and_swap_immediate (scm_jit_state *j, uint32_t dst,
+ uint32_t obj, uint8_t offset,
+ uint32_t expected, uint32_t desired)
+{
+ emit_call_3 (j, scm_vm_intrinsics.atomic_compare_and_swap_scm,
+ jit_operand_addi (sp_scm_operand (j, obj),
+ offset * sizeof (SCM)),
+ sp_scm_operand (j, expected), sp_scm_operand (j, desired));
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_call_thread_scm_scm (scm_jit_state *j, uint16_t a, uint16_t b, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+ emit_call_3 (j, intrinsic, thread_operand (), sp_scm_operand (j, a),
+ sp_scm_operand (j, b));
+ emit_reload_sp (j);
+}
+
+static void
+compile_call_thread (scm_jit_state *j, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+ emit_call_1 (j, intrinsic, thread_operand ());
+ emit_reload_sp (j);
+}
+
+static void
+compile_call_scm_from_thread_scm (scm_jit_state *j, uint16_t dst, uint16_t a, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+ emit_call_2 (j, intrinsic, thread_operand (), sp_scm_operand (j, a));
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_call_thread_scm (scm_jit_state *j, uint32_t a, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+ emit_call_2 (j, intrinsic, thread_operand (), sp_scm_operand (j, a));
+ emit_reload_sp (j);
+}
+
+static void
+compile_call_scm_from_scm_u64 (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+#if INDIRECT_INT64_INTRINSICS
+ emit_call_2 (j, intrinsic, sp_scm_operand (j, a), sp_slot_operand (j, b));
+#else
+ emit_call_2 (j, intrinsic, sp_scm_operand (j, a), sp_u64_operand (j, b));
+#endif
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_call_scm_from_thread (scm_jit_state *j, uint32_t dst, uint32_t idx)
+{
+ void *intrinsic = ((void **) &scm_vm_intrinsics)[idx];
+
+ emit_store_current_ip (j, T0);
+ emit_call_1 (j, intrinsic, thread_operand ());
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_fadd (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+ emit_sp_ref_f64 (j, JIT_F0, a);
+ emit_sp_ref_f64 (j, JIT_F1, b);
+ emit_addr_d (j, JIT_F0, JIT_F0, JIT_F1);
+ emit_sp_set_f64 (j, dst, JIT_F0);
+}
+
+static void
+compile_fsub (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+ emit_sp_ref_f64 (j, JIT_F0, a);
+ emit_sp_ref_f64 (j, JIT_F1, b);
+ emit_subr_d (j, JIT_F0, JIT_F0, JIT_F1);
+ emit_sp_set_f64 (j, dst, JIT_F0);
+}
+
+static void
+compile_fmul (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+ emit_sp_ref_f64 (j, JIT_F0, a);
+ emit_sp_ref_f64 (j, JIT_F1, b);
+ emit_mulr_d (j, JIT_F0, JIT_F0, JIT_F1);
+ emit_sp_set_f64 (j, dst, JIT_F0);
+}
+
+static void
+compile_fdiv (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+ emit_sp_ref_f64 (j, JIT_F0, a);
+ emit_sp_ref_f64 (j, JIT_F1, b);
+ emit_divr_d (j, JIT_F0, JIT_F0, JIT_F1);
+ emit_sp_set_f64 (j, dst, JIT_F0);
+}
+
+static void
+compile_uadd (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ emit_addr (j, T0, T0, T1);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ emit_addcr (j, T0, T0, T2);
+ emit_addxr (j, T1, T1, T3_OR_FP);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_usub (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ emit_subr (j, T0, T0, T1);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ emit_subcr (j, T0, T0, T2);
+ emit_subxr (j, T1, T1, T3_OR_FP);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_umul (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ emit_mulr (j, T0, T0, T1);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ /* FIXME: This is untested! */
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ emit_mulr (j, T1, T1, T2); /* High A times low B */
+ emit_mulr (j, T3_OR_FP, T3_OR_FP, T0); /* High B times low A */
+ emit_addr (j, T1, T1, T3_OR_FP); /* Add high results, throw away overflow */
+ emit_qmulr_u (j, T0, T2, T0, T2); /* Low A times low B */
+ emit_addr (j, T1, T1, T2); /* Add high result of low product */
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_uadd_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_addi (j, T0, T0, b);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_addci (j, T0, T0, b);
+ emit_addxi (j, T1, T1, 0);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_usub_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_subi (j, T0, T0, b);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_subci (j, T0, T0, b);
+ emit_subxi (j, T1, T1, 0);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_umul_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_muli (j, T0, T0, b);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ /* FIXME: This is untested! */
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_muli (j, T1, T1, b); /* High A times low B */
+ /* High B times low A is 0. */
+ emit_movi (j, T2, b);
+ emit_qmulr_u (j, T0, T2, T0, T2); /* Low A times low B */
+ emit_addr (j, T1, T1, T2); /* Add high result of low product */
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_load_f64 (scm_jit_state *j, uint32_t dst, double a)
+{
+ jit_movi_d (j->jit, JIT_F0, a);
+ record_fpr_clobber (j, JIT_F0);
+ emit_sp_set_f64 (j, dst, JIT_F0);
+}
+
+static void
+compile_load_u64 (scm_jit_state *j, uint32_t dst, uint64_t a)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_movi (j, T0, a);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_movi (j, T0, a & 0xffffffff);
+ emit_movi (j, T1, a >> 32);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_load_s64 (scm_jit_state *j, uint32_t dst, int64_t a)
+{
+ compile_load_u64 (j, dst, a);
+}
+
+static void
+compile_current_thread (scm_jit_state *j, uint32_t dst)
+{
+ emit_ldxi (j, T0, THREAD, thread_offset_handle);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_ulogand (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ emit_andr (j, T0, T0, T1);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ emit_andr (j, T0, T0, T2);
+ emit_andr (j, T1, T1, T3_OR_FP);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_ulogior (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ emit_orr (j, T0, T0, T1);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ emit_orr (j, T0, T0, T2);
+ emit_orr (j, T1, T1, T3_OR_FP);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_ulogsub (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ emit_comr (j, T1, T1);
+ emit_andr (j, T0, T0, T1);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ emit_comr (j, T2, T2);
+ emit_comr (j, T3_OR_FP, T3_OR_FP);
+ emit_andr (j, T0, T0, T2);
+ emit_andr (j, T1, T1, T3_OR_FP);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_ursh (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ emit_andi (j, T1, T1, 63);
+ emit_rshr_u (j, T0, T0, T1);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ /* FIXME: Not tested. */
+ jit_reloc_t zero, both, done;
+
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ emit_andi (j, T2, T2, 63);
+ zero = jit_beqi (j->jit, T2, 0);
+ both = jit_blti (j->jit, T2, 32);
+
+ /* 32 <= s < 64: hi = 0, lo = hi >> (s-32) */
+ emit_subi (j, T2, T2, 32);
+ emit_rshr_u (j, T0, T1, T2);
+ emit_movi (j, T1, 0);
+ done = jit_jmp (j->jit);
+
+ jit_patch_here (j->jit, both);
+ /* 0 < s < 32: hi = hi >> s, lo = lo >> s + hi << (32-s) */
+ emit_negr (j, T3_OR_FP, T2);
+ emit_addi (j, T3_OR_FP, T3_OR_FP, 32);
+ emit_lshr (j, T3_OR_FP, T1, T3_OR_FP);
+ emit_rshr_u (j, T1, T1, T2);
+ emit_rshr_u (j, T0, T0, T2);
+ emit_addr (j, T0, T0, T3_OR_FP);
+
+ jit_patch_here (j->jit, done);
+ jit_patch_here (j->jit, zero);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_ulsh (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ emit_andi (j, T1, T1, 63);
+ emit_lshr (j, T0, T0, T1);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ /* FIXME: Not tested. */
+ jit_reloc_t zero, both, done;
+
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ emit_andi (j, T2, T2, 63);
+ zero = jit_beqi (j->jit, T2, 0);
+ both = jit_blti (j->jit, T2, 32);
+
+ /* 32 <= s < 64: hi = lo << (s-32), lo = 0 */
+ emit_subi (j, T2, T2, 32);
+ emit_lshr (j, T1, T0, T2);
+ emit_movi (j, T0, 0);
+ done = jit_jmp (j->jit);
+
+ jit_patch_here (j->jit, both);
+ /* 0 < s < 32: hi = hi << s + lo >> (32-s), lo = lo << s */
+ emit_negr (j, T3_OR_FP, T2);
+ emit_addi (j, T3_OR_FP, T3_OR_FP, 32);
+ emit_rshr_u (j, T3_OR_FP, T0, T3_OR_FP);
+ emit_lshr (j, T1, T1, T2);
+ emit_lshr (j, T0, T0, T2);
+ emit_addr (j, T1, T1, T3_OR_FP);
+
+ jit_patch_here (j->jit, done);
+ jit_patch_here (j->jit, zero);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_ursh_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+ b &= 63;
+
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_rshi_u (j, T0, T0, b);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ /* FIXME: Not tested. */
+ emit_sp_ref_u64 (j, T0, T1, a);
+ if (b == 0)
+ {
+ /* Nothing to do. */
+ }
+ else if (b < 32)
+ {
+ /* 0 < s < 32: hi = hi >> s, lo = lo >> s + hi << (32-s) */
+ emit_lshi (j, T2, T1, 32 - b);
+ emit_rshi_u (j, T1, T1, b);
+ emit_rshi_u (j, T0, T0, b);
+ emit_addr (j, T0, T0, T2);
+ }
+ else if (b == 32)
+ {
+ /* hi = 0, lo = hi */
+ emit_movr (j, T0, T1);
+ emit_movi (j, T1, 0);
+ }
+ else /* b > 32 */
+ {
+ /* hi = 0, lo = hi >> (s-32) */
+ emit_rshi_u (j, T0, T1, b - 32);
+ emit_movi (j, T1, 0);
+ }
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_ulsh_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+ b &= 63;
+
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_lshi (j, T0, T0, b);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ /* FIXME: Not tested. */
+ emit_sp_ref_u64 (j, T0, T1, a);
+ if (b == 0)
+ {
+ /* Nothing to do. */
+ }
+ else if (b < 32)
+ {
+ /* hi = hi << s + lo >> (32-s), lo = lo << s */
+ emit_rshi_u (j, T2, T0, 32 - b);
+ emit_lshi (j, T1, T1, b);
+ emit_lshi (j, T0, T0, b);
+ emit_addr (j, T1, T1, T2);
+ }
+ else if (b == 32)
+ {
+ /* hi = lo, lo = 0 */
+ emit_movr (j, T1, T0);
+ emit_movi (j, T0, 0);
+ }
+ else /* b > 32 */
+ {
+ /* hi = lo << (s-32), lo = 0 */
+ emit_lshi (j, T1, T0, b - 32);
+ emit_movi (j, T0, 0);
+ }
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_ulogxor (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ emit_xorr (j, T0, T0, T1);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ emit_xorr (j, T0, T0, T2);
+ emit_xorr (j, T1, T1, T3_OR_FP);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_handle_interrupts (scm_jit_state *j)
+{
+ uint32_t saved_state = save_reloadable_register_state (j);
+
+ /* This instruction invalidates SP_CACHE_GPR / SP_CACHE_FPR. */
+
+ void *again = jit_address (j->jit);
+
+#if defined(__i386__) || defined(__x86_64__)
+ /* Disassembly of atomic_ref_scm is just a mov. */
+ jit_ldxi (j->jit, T0, THREAD, thread_offset_pending_asyncs);
+#else
+ emit_call_1 (j, scm_vm_intrinsics.atomic_ref_scm,
+ jit_operand_addi (thread_operand (),
+ thread_offset_pending_asyncs));
+ emit_retval (j, T0);
+ restore_reloadable_register_state (j, saved_state);
+#endif
+ jit_reloc_t none_pending = jit_beqi (j->jit, T0, SCM_UNPACK (SCM_EOL));
+ jit_ldxi_i (j->jit, T0, THREAD, thread_offset_block_asyncs);
+ jit_reloc_t blocked = jit_bnei (j->jit, T0, 0);
+
+ emit_store_current_ip (j, T0);
+ emit_movi (j, T0, (uintptr_t)again);
+ jit_jmpi (j->jit, handle_interrupts_trampoline);
+
+ jit_patch_here (j->jit, none_pending);
+ jit_patch_here (j->jit, blocked);
+ j->register_state = saved_state;
+}
+
+static void
+compile_return_from_interrupt (scm_jit_state *j)
+{
+ jit_gpr_t old_fp = T0, ra = T1;
+ jit_reloc_t interp;
+
+ emit_pop_fp (j, old_fp);
+
+ emit_load_mra (j, ra, old_fp);
+ interp = jit_beqi (j->jit, ra, 0);
+ jit_addi (j->jit, SP, old_fp, frame_overhead_slots * sizeof (union scm_vm_stack_element));
+ set_register_state (j, SP_IN_REGISTER);
+ emit_store_sp (j);
+ jit_jmpr (j->jit, ra);
+
+ jit_patch_here (j->jit, interp);
+ emit_load_vra (j, ra, old_fp);
+ emit_store_ip (j, ra);
+ jit_addi (j->jit, SP, old_fp, frame_overhead_slots * sizeof (union scm_vm_stack_element));
+ set_register_state (j, SP_IN_REGISTER);
+ emit_store_sp (j);
+ emit_exit (j);
+
+ clear_register_state (j, SP_CACHE_GPR | SP_CACHE_FPR);
+}
+
+static enum scm_opcode
+fuse_conditional_branch (scm_jit_state *j, uint32_t **target)
+{
+ uint8_t next = j->next_ip[0] & 0xff;
+
+ switch (next)
+ {
+ case scm_op_jl:
+ case scm_op_je:
+ case scm_op_jnl:
+ case scm_op_jne:
+ case scm_op_jge:
+ case scm_op_jnge:
+ *target = j->next_ip + (((int32_t) j->next_ip[0]) >> 8);
+ j->next_ip += op_lengths[next];
+ return next;
+ default:
+ ASSERT (0);
+ }
+}
+
+static void
+compile_u64_numerically_equal (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+ uint32_t *target;
+#if SIZEOF_UINTPTR_T >= 8
+ jit_reloc_t k;
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_je:
+ k = jit_beqr (j->jit, T0, T1);
+ break;
+ case scm_op_jne:
+ k = jit_bner (j->jit, T0, T1);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+#else
+ jit_reloc_t k1, k2;
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_je:
+ k1 = jit_bner (j->jit, T0, T2);
+ k2 = jit_beqr (j->jit, T1, T3_OR_FP);
+ jit_patch_here (j->jit, k1);
+ add_inter_instruction_patch (j, k2, target);
+ break;
+ case scm_op_jne:
+ k1 = jit_bner (j->jit, T0, T2);
+ k2 = jit_bner (j->jit, T1, T3_OR_FP);
+ add_inter_instruction_patch (j, k1, target);
+ add_inter_instruction_patch (j, k2, target);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+#endif
+}
+
+static void
+compile_u64_less (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+ uint32_t *target;
+#if SIZEOF_UINTPTR_T >= 8
+ jit_reloc_t k;
+ emit_sp_ref_u64 (j, T0, a);
+ emit_sp_ref_u64 (j, T1, b);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k = jit_bltr_u (j->jit, T0, T1);
+ break;
+ case scm_op_jnl:
+ k = jit_bger_u (j->jit, T0, T1);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+#else
+ jit_reloc_t k1, k2, k3;
+ emit_sp_ref_u64 (j, T0, T1, a);
+ emit_sp_ref_u64 (j, T2, T3_OR_FP, b);
+ k1 = jit_bltr_u (j->jit, T1, T3_OR_FP);
+ k2 = jit_bner (j->jit, T1, T3_OR_FP);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k3 = jit_bltr_u (j->jit, T0, T2);
+ jit_patch_here (j->jit, k2);
+ add_inter_instruction_patch (j, k1, target);
+ add_inter_instruction_patch (j, k3, target);
+ break;
+ case scm_op_jnl:
+ k3 = jit_bger_u (j->jit, T0, T2);
+ jit_patch_here (j->jit, k1);
+ add_inter_instruction_patch (j, k2, target);
+ add_inter_instruction_patch (j, k3, target);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+#endif
+}
+
+static void
+compile_s64_less (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+ uint32_t *target;
+#if SIZEOF_UINTPTR_T >= 8
+ jit_reloc_t k;
+ emit_sp_ref_s64 (j, T0, a);
+ emit_sp_ref_s64 (j, T1, b);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k = jit_bltr (j->jit, T0, T1);
+ break;
+ case scm_op_jnl:
+ k = jit_bger (j->jit, T0, T1);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+#else
+ jit_reloc_t k1, k2, k3;
+ emit_sp_ref_s64 (j, T0, T1, a);
+ emit_sp_ref_s64 (j, T2, T3_OR_FP, b);
+ k1 = jit_bltr (j->jit, T1, T3_OR_FP);
+ k2 = jit_bner (j->jit, T1, T3_OR_FP);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k3 = jit_bltr (j->jit, T0, T2);
+ jit_patch_here (j->jit, k2);
+ add_inter_instruction_patch (j, k1, target);
+ add_inter_instruction_patch (j, k3, target);
+ break;
+ case scm_op_jnl:
+ k3 = jit_bger (j->jit, T0, T2);
+ jit_patch_here (j->jit, k1);
+ add_inter_instruction_patch (j, k2, target);
+ add_inter_instruction_patch (j, k3, target);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+#endif
+}
+
+static void
+compile_f64_numerically_equal (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_sp_ref_f64 (j, JIT_F0, a);
+ emit_sp_ref_f64 (j, JIT_F1, b);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_je:
+ k = jit_beqr_d (j->jit, JIT_F0, JIT_F1);
+ break;
+ case scm_op_jne:
+ k = jit_bner_d (j->jit, JIT_F0, JIT_F1);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+}
+
+static void
+compile_f64_less (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_sp_ref_f64 (j, JIT_F0, a);
+ emit_sp_ref_f64 (j, JIT_F1, b);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k = jit_bltr_d (j->jit, JIT_F0, JIT_F1);
+ break;
+ case scm_op_jnl:
+ k = jit_bunger_d (j->jit, JIT_F0, JIT_F1);
+ break;
+ case scm_op_jge:
+ k = jit_bger_d (j->jit, JIT_F0, JIT_F1);
+ break;
+ case scm_op_jnge:
+ k = jit_bunltr_d (j->jit, JIT_F0, JIT_F1);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+}
+
+static void
+compile_numerically_equal (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_store_current_ip (j, T0);
+ emit_call_2 (j, scm_vm_intrinsics.numerically_equal_p,
+ sp_scm_operand (j, a), sp_scm_operand (j, b));
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_je:
+ k = jit_bnei (j->jit, T0, 0);
+ break;
+ case scm_op_jne:
+ k = jit_beqi (j->jit, T0, 0);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+}
+
+static void
+compile_less (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+ jit_reloc_t fast, k2, k3;
+ jit_reloc_t k1;
+ uint32_t *target;
+ enum scm_opcode op = fuse_conditional_branch (j, &target);
+
+ emit_store_current_ip (j, T0);
+ emit_sp_ref_scm (j, T0, a);
+ emit_sp_ref_scm (j, T1, b);
+
+ emit_andr (j, T2, T0, T1);
+ fast = jit_bmsi (j->jit, T2, scm_tc2_int);
+
+ emit_call_2 (j, scm_vm_intrinsics.less_p,
+ jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T0),
+ jit_operand_gpr (JIT_OPERAND_ABI_POINTER, T1));
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+ switch (op)
+ {
+ case scm_op_jl:
+ k1 = jit_beqi (j->jit, T0, SCM_F_COMPARE_LESS_THAN);
+ break;
+ case scm_op_jnl:
+ k1 = jit_bnei (j->jit, T0, SCM_F_COMPARE_LESS_THAN);
+ break;
+ case scm_op_jge:
+ k1 = jit_beqi (j->jit, T0, SCM_F_COMPARE_NONE);
+ break;
+ case scm_op_jnge:
+ k1 = jit_bnei (j->jit, T0, SCM_F_COMPARE_NONE);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ k2 = jit_jmp (j->jit);
+
+ jit_patch_here (j->jit, fast);
+ switch (op)
+ {
+ case scm_op_jl:
+ case scm_op_jnge:
+ k3 = jit_bltr (j->jit, T0, T1);
+ break;
+ case scm_op_jnl:
+ case scm_op_jge:
+ k3 = jit_bger (j->jit, T0, T1);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+
+ jit_patch_here (j->jit, k2);
+
+ add_inter_instruction_patch (j, k1, target);
+ add_inter_instruction_patch (j, k3, target);
+}
+
+static void
+compile_check_arguments (scm_jit_state *j, uint32_t expected)
+{
+ jit_reloc_t k;
+ uint32_t *target;
+ jit_gpr_t t = T0;
+
+ emit_reload_fp (j);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jne:
+ k = emit_branch_if_frame_locals_count_not_eq (j, t, expected);
+ break;
+ case scm_op_jl:
+ k = emit_branch_if_frame_locals_count_less_than (j, t, expected);
+ break;
+ case scm_op_jge:
+ /* The arguments<=? instruction sets NONE to indicate
+ greater-than, whereas for <, NONE usually indicates
+ greater-than-or-equal, hence the name jge. So we need to fuse
+ to greater-than, not greater-than-or-equal. Perhaps we just
+ need to rename jge to br-if-none. */
+ k = emit_branch_if_frame_locals_count_greater_than (j, t, expected);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+}
+
+static void
+compile_check_positional_arguments (scm_jit_state *j, uint32_t nreq, uint32_t expected)
+{
+ uint32_t *target;
+ jit_reloc_t lt, gt;
+ jit_gpr_t walk = T0, min = T1, obj = T2;
+
+ ASSERT_HAS_REGISTER_STATE (FP_IN_REGISTER | SP_IN_REGISTER);
+
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jge:
+ /* Like arguments<=?, this instruction sets NONE to indicate
+ greater-than, whereas for <, NONE usually indicates
+ greater-than-or-equal, hence the name jge. So we need to fuse
+ to greater-than, not greater-than-or-equal. Perhaps we just
+ need to rename jge to br-if-none. */
+ /* Break to target if npos > expected. */
+ break;
+ default:
+ UNREACHABLE ();
+ }
+
+ emit_subtract_stack_slots (j, min, FP, expected);
+ emit_subtract_stack_slots (j, walk, FP, nreq);
+
+ void *head = jit_address (j->jit);
+ /* npos > expected if walk < min. */
+ gt = jit_bltr (j->jit, walk, min);
+ emit_subtract_stack_slots (j, walk, walk, 1);
+ lt = jit_bltr (j->jit, walk, SP);
+ emit_ldr (j, obj, walk);
+ jit_patch_there
+ (j->jit,
+ emit_branch_if_immediate (j, obj),
+ head);
+ jit_patch_there
+ (j->jit,
+ emit_branch_if_heap_object_not_tc7 (j, obj, obj, scm_tc7_keyword),
+ head);
+ jit_patch_here (j->jit, lt);
+ add_inter_instruction_patch (j, gt, target);
+}
+
+static void
+compile_immediate_tag_equals (scm_jit_state *j, uint32_t a, uint16_t mask,
+ uint16_t expected)
+{
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_sp_ref_scm (j, T0, a);
+ emit_andi (j, T0, T0, mask);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_je:
+ k = jit_beqi (j->jit, T0, expected);
+ break;
+ case scm_op_jne:
+ k = jit_bnei (j->jit, T0, expected);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+}
+
+static void
+compile_heap_tag_equals (scm_jit_state *j, uint32_t obj,
+ uint16_t mask, uint16_t expected)
+{
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_sp_ref_scm (j, T0, obj);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_je:
+ k = emit_branch_if_heap_object_has_tc (j, T0, T0, mask, expected);
+ break;
+ case scm_op_jne:
+ k = emit_branch_if_heap_object_not_tc (j, T0, T0, mask, expected);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+}
+
+static void
+compile_eq (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_sp_ref_scm (j, T0, a);
+ emit_sp_ref_scm (j, T1, b);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_je:
+ k = jit_beqr (j->jit, T0, T1);
+ break;
+ case scm_op_jne:
+ k = jit_bner (j->jit, T0, T1);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+}
+
+static void
+compile_j (scm_jit_state *j, const uint32_t *vcode)
+{
+ jit_reloc_t jmp;
+ jmp = jit_jmp (j->jit);
+ add_inter_instruction_patch (j, jmp, vcode);
+}
+
+static void
+compile_jl (scm_jit_state *j, const uint32_t *vcode)
+{
+ UNREACHABLE (); /* All tests should fuse their following branches. */
+}
+
+static void
+compile_je (scm_jit_state *j, const uint32_t *vcode)
+{
+ UNREACHABLE (); /* All tests should fuse their following branches. */
+}
+
+static void
+compile_jnl (scm_jit_state *j, const uint32_t *vcode)
+{
+ UNREACHABLE (); /* All tests should fuse their following branches. */
+}
+
+static void
+compile_jne (scm_jit_state *j, const uint32_t *vcode)
+{
+ UNREACHABLE (); /* All tests should fuse their following branches. */
+}
+
+static void
+compile_jge (scm_jit_state *j, const uint32_t *vcode)
+{
+ UNREACHABLE (); /* All tests should fuse their following branches. */
+}
+
+static void
+compile_jnge (scm_jit_state *j, const uint32_t *vcode)
+{
+ UNREACHABLE (); /* All tests should fuse their following branches. */
+}
+
+static void
+compile_heap_numbers_equal (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_store_current_ip (j, T0);
+ emit_call_2 (j, scm_vm_intrinsics.heap_numbers_equal_p, sp_scm_operand (j, a),
+ sp_scm_operand (j, b));
+ emit_retval (j, T0);
+ emit_reload_sp (j);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_je:
+ k = jit_bnei (j->jit, T0, 0);
+ break;
+ case scm_op_jne:
+ k = jit_beqi (j->jit, T0, 0);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+}
+
+static void
+compile_untag_fixnum (scm_jit_state *j, uint16_t dst, uint16_t a)
+{
+ emit_sp_ref_scm (j, T0, a);
+ emit_rshi (j, T0, T0, 2);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_set_s64 (j, dst, T0);
+#else
+ /* FIXME: Untested! */
+ emit_rshi (j, T1, T0, 31);
+ emit_sp_set_s64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_tag_fixnum (scm_jit_state *j, uint16_t dst, uint16_t a)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_s64 (j, T0, a);
+#else
+ emit_sp_ref_s32 (j, T0, a);
+#endif
+ emit_lshi (j, T0, T0, 2);
+ emit_addi (j, T0, T0, scm_tc2_int);
+ emit_sp_set_scm (j, dst, T0);
+}
+
+static void
+compile_srsh (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_s64 (j, T0, a);
+ emit_sp_ref_s64 (j, T1, b);
+ emit_andi (j, T1, T1, 63);
+ emit_rshr (j, T0, T0, T1);
+ emit_sp_set_s64 (j, dst, T0);
+#else
+ /* FIXME: Not tested. */
+ jit_reloc_t zero, both, done;
+
+ emit_sp_ref_s64 (j, T0, T1, a);
+ emit_sp_ref_s64 (j, T2, T3_OR_FP, b);
+ emit_andi (j, T2, T2, 63);
+ zero = jit_beqi (j->jit, T2, 0);
+ both = jit_blti (j->jit, T2, 32);
+
+ /* 32 <= s < 64: hi = hi >> 31, lo = hi >> (s-32) */
+ emit_subi (j, T2, T2, 32);
+ emit_rshr (j, T0, T1, T2);
+ emit_rshi (j, T1, T1, 31);
+ done = jit_jmp (j->jit);
+
+ jit_patch_here (j->jit, both);
+ /* 0 < s < 32: hi = hi >> s, lo = lo >> s + hi << (32-s) */
+ emit_negr (j, T3_OR_FP, T2);
+ emit_addi (j, T3_OR_FP, T3_OR_FP, 32);
+ emit_lshr (j, T3_OR_FP, T1, T3_OR_FP);
+ emit_rshr (j, T1, T1, T2);
+ emit_rshr_u (j, T0, T0, T2);
+ emit_addr (j, T0, T0, T3_OR_FP);
+
+ jit_patch_here (j->jit, done);
+ jit_patch_here (j->jit, zero);
+ emit_sp_set_s64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_srsh_immediate (scm_jit_state *j, uint8_t dst, uint8_t a, uint8_t b)
+{
+ b &= 63;
+
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_s64 (j, T0, a);
+ emit_rshi (j, T0, T0, b);
+ emit_sp_set_s64 (j, dst, T0);
+#else
+ /* FIXME: Not tested. */
+ emit_sp_ref_s64 (j, T0, T1, a);
+ if (b == 0)
+ {
+ /* Nothing to do. */
+ }
+ else if (b < 32)
+ {
+ /* 0 < s < 32: hi = hi >> s, lo = lo >> s + hi << (32-s) */
+ emit_lshi (j, T2, T1, 32 - b);
+ emit_rshi (j, T1, T1, b);
+ emit_rshi_u (j, T0, T0, b);
+ emit_addr (j, T0, T0, T2);
+ }
+ else if (b == 32)
+ {
+ /* hi = sign-ext, lo = hi */
+ emit_movr (j, T0, T1);
+ emit_rshi (j, T1, T1, 31);
+ }
+ else /* b > 32 */
+ {
+ /* hi = sign-ext, lo = hi >> (s-32) */
+ emit_rshi (j, T0, T1, b - 32);
+ emit_rshi (j, T1, T1, 31);
+ }
+ emit_sp_set_s64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_s64_imm_numerically_equal (scm_jit_state *j, uint16_t a, int16_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_sp_ref_s64 (j, T0, a);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_je:
+ k = jit_beqi (j->jit, T0, b);
+ break;
+ case scm_op_jne:
+ k = jit_bnei (j->jit, T0, b);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+#else
+ jit_reloc_t k1, k2;
+ uint32_t *target;
+
+ emit_sp_ref_s64 (j, T0, T1, a);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_je:
+ k1 = jit_bnei (j->jit, T0, b);
+ k2 = jit_beqi (j->jit, T1, b < 0 ? -1 : 0);
+ jit_patch_here (j->jit, k1);
+ add_inter_instruction_patch (j, k2, target);
+ break;
+ case scm_op_jne:
+ k1 = jit_bnei (j->jit, T0, b);
+ k2 = jit_bnei (j->jit, T1, b < 0 ? -1 : 0);
+ add_inter_instruction_patch (j, k1, target);
+ add_inter_instruction_patch (j, k2, target);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+#endif
+}
+
+static void
+compile_u64_imm_less (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_sp_ref_u64 (j, T0, a);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k = jit_blti_u (j->jit, T0, b);
+ break;
+ case scm_op_jnl:
+ k = jit_bgei_u (j->jit, T0, b);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+#else
+ jit_reloc_t k1, k2;
+ uint32_t *target;
+
+ emit_sp_ref_u64 (j, T0, T1, a);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k1 = jit_bnei (j->jit, T1, 0);
+ k2 = jit_blti_u (j->jit, T0, b);
+ jit_patch_here (j->jit, k1);
+ add_inter_instruction_patch (j, k2, target);
+ break;
+ case scm_op_jnl:
+ k1 = jit_bnei (j->jit, T1, 0);
+ k2 = jit_bgei_u (j->jit, T0, b);
+ add_inter_instruction_patch (j, k1, target);
+ add_inter_instruction_patch (j, k2, target);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+#endif
+}
+
+static void
+compile_imm_u64_less (scm_jit_state *j, uint16_t a, uint16_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_sp_ref_u64 (j, T0, a);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k = jit_bgti_u (j->jit, T0, b);
+ break;
+ case scm_op_jnl:
+ k = jit_blei_u (j->jit, T0, b);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+#else
+ jit_reloc_t k1, k2;
+ uint32_t *target;
+
+ emit_sp_ref_u64 (j, T0, T1, a);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k1 = jit_bnei (j->jit, T1, 0);
+ k2 = jit_bgti_u (j->jit, T0, b);
+ add_inter_instruction_patch (j, k1, target);
+ add_inter_instruction_patch (j, k2, target);
+ break;
+ case scm_op_jnl:
+ k1 = jit_bnei (j->jit, T1, 0);
+ k2 = jit_blei_u (j->jit, T0, b);
+ jit_patch_here (j->jit, k1);
+ add_inter_instruction_patch (j, k2, target);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+#endif
+}
+
+static void
+compile_s64_imm_less (scm_jit_state *j, uint16_t a, int16_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_sp_ref_s64 (j, T0, a);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k = jit_blti (j->jit, T0, b);
+ break;
+ case scm_op_jnl:
+ k = jit_bgei (j->jit, T0, b);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+#else
+ jit_reloc_t k1, k2, k3;
+ int32_t sign = b < 0 ? -1 : 0;
+ uint32_t *target;
+
+ emit_sp_ref_s64 (j, T0, T1, a);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k1 = jit_blti (j->jit, T1, sign);
+ k2 = jit_bnei (j->jit, T1, sign);
+ k3 = jit_blti (j->jit, T0, b);
+ add_inter_instruction_patch (j, k1, target);
+ jit_patch_here (j->jit, k2);
+ add_inter_instruction_patch (j, k3, target);
+ break;
+ case scm_op_jnl:
+ k1 = jit_blti (j->jit, T1, sign);
+ k2 = jit_bnei (j->jit, T1, sign);
+ k3 = jit_bgei (j->jit, T0, b);
+ jit_patch_here (j->jit, k1);
+ add_inter_instruction_patch (j, k2, target);
+ add_inter_instruction_patch (j, k3, target);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+#endif
+}
+
+static void
+compile_imm_s64_less (scm_jit_state *j, uint16_t a, int16_t b)
+{
+#if SIZEOF_UINTPTR_T >= 8
+ jit_reloc_t k;
+ uint32_t *target;
+
+ emit_sp_ref_s64 (j, T0, a);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k = jit_bgti (j->jit, T0, b);
+ break;
+ case scm_op_jnl:
+ k = jit_blei (j->jit, T0, b);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+ add_inter_instruction_patch (j, k, target);
+#else
+ jit_reloc_t k1, k2, k3;
+ int32_t sign = b < 0 ? -1 : 0;
+ uint32_t *target;
+
+ emit_sp_ref_s64 (j, T0, T1, a);
+ switch (fuse_conditional_branch (j, &target))
+ {
+ case scm_op_jl:
+ k1 = jit_blti (j->jit, T1, sign);
+ k2 = jit_bnei (j->jit, T1, sign);
+ k3 = jit_bgti (j->jit, T0, b);
+ jit_patch_here (j->jit, k1);
+ add_inter_instruction_patch (j, k2, target);
+ add_inter_instruction_patch (j, k3, target);
+ break;
+ case scm_op_jnl:
+ k1 = jit_blti (j->jit, T1, sign);
+ k2 = jit_bnei (j->jit, T1, sign);
+ k3 = jit_blei (j->jit, T0, b);
+ add_inter_instruction_patch (j, k1, target);
+ jit_patch_here (j->jit, k2);
+ add_inter_instruction_patch (j, k3, target);
+ break;
+ default:
+ UNREACHABLE ();
+ }
+#endif
+}
+
+static void
+compile_u8_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+ jit_ldxr_uc (j->jit, T0, T0, T1);
+ record_gpr_clobber (j, T0);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_movi (j, T1, 0);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_u16_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+ jit_ldxr_us (j->jit, T0, T0, T1);
+ record_gpr_clobber (j, T0);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_movi (j, T1, 0);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_u32_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+#if SIZEOF_UINTPTR_T >= 8
+ jit_ldxr_ui (j->jit, T0, T0, T1);
+ record_gpr_clobber (j, T0);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_ldxr (j, T0, T0, T1);
+ emit_movi (j, T1, 0);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_u64_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_ldxr (j, T0, T0, T1);
+ emit_sp_set_u64 (j, dst, T0);
+#else
+ emit_addr (j, T0, T0, T1);
+ if (BIGENDIAN)
+ {
+ emit_ldr (j, T1, T0);
+ emit_ldxi (j, T0, T0, 4);
+ }
+ else
+ {
+ emit_ldxi (j, T1, T0, 4);
+ emit_ldr (j, T0, T0);
+ }
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_u8_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T2, v);
+#else
+ emit_sp_ref_u64_lower_half (j, T2, v);
+#endif
+ jit_stxr_c (j->jit, T0, T1, T2);
+}
+
+static void
+compile_u16_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T2, v);
+#else
+ emit_sp_ref_u64_lower_half (j, T2, v);
+#endif
+ jit_stxr_s (j->jit, T0, T1, T2);
+}
+
+static void
+compile_u32_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T2, v);
+ jit_stxr_i (j->jit, T0, T1, T2);
+#else
+ emit_sp_ref_u64_lower_half (j, T2, v);
+ jit_stxr (j->jit, T0, T1, T2);
+#endif
+}
+
+static void
+compile_u64_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_ref_u64 (j, T2, v);
+ jit_stxr (j->jit, T0, T1, T2);
+#else
+ jit_addr (j->jit, T0, T0, T1);
+ emit_sp_ref_u64 (j, T1, T2, v);
+ if (BIGENDIAN)
+ {
+ jit_str (j->jit, T0, T2);
+ jit_stxi (j->jit, 4, T0, T1);
+ }
+ else
+ {
+ jit_str (j->jit, T0, T1);
+ jit_stxi (j->jit, 4, T0, T2);
+ }
+#endif
+}
+
+static void
+compile_s8_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+ jit_ldxr_c (j->jit, T0, T0, T1);
+ record_gpr_clobber (j, T0);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_set_s64 (j, dst, T0);
+#else
+ emit_rshi (j, T1, T0, 7);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_s16_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+ jit_ldxr_s (j->jit, T0, T0, T1);
+ record_gpr_clobber (j, T0);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_set_s64 (j, dst, T0);
+#else
+ emit_rshi (j, T1, T0, 15);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_s32_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+ jit_ldxr_i (j->jit, T0, T0, T1);
+ record_gpr_clobber (j, T0);
+#if SIZEOF_UINTPTR_T >= 8
+ emit_sp_set_s64 (j, dst, T0);
+#else
+ emit_rshi (j, T1, T0, 31);
+ emit_sp_set_u64 (j, dst, T0, T1);
+#endif
+}
+
+static void
+compile_s64_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx)
+{
+ compile_u64_ref (j, dst, ptr, idx);
+}
+
+static void
+compile_s8_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v)
+{
+ compile_u8_set (j, ptr, idx, v);
+}
+
+static void
+compile_s16_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v)
+{
+ compile_u16_set (j, ptr, idx, v);
+}
+
+static void
+compile_s32_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v)
+{
+ compile_u32_set (j, ptr, idx, v);
+}
+
+static void
+compile_s64_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v)
+{
+ compile_u64_set (j, ptr, idx, v);
+}
+
+static void
+compile_f32_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+ jit_ldxr_f (j->jit, JIT_F0, T0, T1);
+ record_fpr_clobber (j, JIT_F0);
+ jit_extr_f_d (j->jit, JIT_F0, JIT_F0);
+ emit_sp_set_f64 (j, dst, JIT_F0);
+}
+
+static void
+compile_f64_ref (scm_jit_state *j, uint8_t dst, uint8_t ptr, uint8_t idx)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+ jit_ldxr_d (j->jit, JIT_F0, T0, T1);
+ record_fpr_clobber (j, JIT_F0);
+ emit_sp_set_f64 (j, dst, JIT_F0);
+}
+
+static void
+compile_f32_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+ emit_sp_ref_f64 (j, JIT_F0, v);
+ jit_extr_d_f (j->jit, JIT_F0, JIT_F0);
+ record_fpr_clobber (j, JIT_F0);
+ jit_stxr_f (j->jit, T0, T1, JIT_F0);
+}
+
+static void
+compile_f64_set (scm_jit_state *j, uint8_t ptr, uint8_t idx, uint8_t v)
+{
+ emit_sp_ref_ptr (j, T0, ptr);
+ emit_sp_ref_sz (j, T1, idx);
+ emit_sp_ref_f64 (j, JIT_F0, v);
+ jit_stxr_d (j->jit, T0, T1, JIT_F0);
+}
+
+
+#define UNPACK_8_8_8(op,a,b,c) \
+ do \
+ { \
+ a = (op >> 8) & 0xff; \
+ b = (op >> 16) & 0xff; \
+ c = op >> 24; \
+ } \
+ while (0)
+
+#define UNPACK_8_16(op,a,b) \
+ do \
+ { \
+ a = (op >> 8) & 0xff; \
+ b = op >> 16; \
+ } \
+ while (0)
+
+#define UNPACK_12_12(op,a,b) \
+ do \
+ { \
+ a = (op >> 8) & 0xfff; \
+ b = op >> 20; \
+ } \
+ while (0)
+
+#define UNPACK_24(op,a) \
+ do \
+ { \
+ a = op >> 8; \
+ } \
+ while (0)
+
+#define UNPACK_8_24(op,a,b) \
+ do \
+ { \
+ a = op & 0xff; \
+ b = op >> 8; \
+ } \
+ while (0)
+
+#define UNPACK_16_16(op,a,b) \
+ do \
+ { \
+ a = op & 0xffff; \
+ b = op >> 16; \
+ } \
+ while (0)
+
+#define COMPILE_OP1(t0) \
+ COMPILE_##t0
+#define COMPILE_OP2(t0, t1) \
+ COMPILE_##t0##__##t1
+#define COMPILE_OP3(t0, t1, t2) \
+ COMPILE_##t0##__##t1##__##t2
+#define COMPILE_OP4(t0, t1, t2, t3) \
+ COMPILE_##t0##__##t1##__##t2##__##t3
+#define COMPILE_OP5(t0, t1, t2, t3, t4) \
+ COMPILE_##t0##__##t1##__##t2##__##t3##__##t4
+
+#define COMPILE_DOP1(t0) COMPILE_OP1(t0)
+#define COMPILE_DOP2(t0, t1) COMPILE_OP2(t0, t1)
+#define COMPILE_DOP3(t0, t1, t2) COMPILE_OP3(t0, t1, t2)
+#define COMPILE_DOP4(t0, t1, t2, t3) COMPILE_OP4(t0, t1, t2, t3)
+#define COMPILE_DOP5(t0, t1, t2, t3, t4) COMPILE_OP5(t0, t1, t2, t3, t4)
+
+#define COMPILE_NOP(j, comp) \
+ { \
+ bad_instruction (j); \
+ }
+
+#define COMPILE_X32(j, comp) \
+ { \
+ comp (j); \
+ }
+
+#define COMPILE_X8_C24(j, comp) \
+ { \
+ uint32_t a; \
+ UNPACK_24 (j->ip[0], a); \
+ comp (j, a); \
+ }
+#define COMPILE_X8_F24(j, comp) \
+ COMPILE_X8_C24 (j, comp)
+#define COMPILE_X8_S24(j, comp) \
+ COMPILE_X8_C24 (j, comp)
+
+#define COMPILE_X8_L24(j, comp) \
+ { \
+ int32_t a = j->ip[0]; \
+ a >>= 8; /* Sign extension. */ \
+ comp (j, j->ip + a); \
+ }
+#define COMPILE_X8_C12_C12(j, comp) \
+ { \
+ uint16_t a, b; \
+ UNPACK_12_12 (j->ip[0], a, b); \
+ comp (j, a, b); \
+ }
+#define COMPILE_X8_S12_C12(j, comp) \
+ COMPILE_X8_C12_C12 (j, comp)
+#define COMPILE_X8_S12_S12(j, comp) \
+ COMPILE_X8_C12_C12 (j, comp)
+#define COMPILE_X8_F12_F12(j, comp) \
+ COMPILE_X8_C12_C12 (j, comp)
+
+#define COMPILE_X8_S12_Z12(j, comp) \
+ { \
+ uint16_t a = (j->ip[0] >> 8) & 0xfff; \
+ int16_t b = ((int32_t) j->ip[0]) >> 20; /* Sign extension. */ \
+ comp (j, a, b); \
+ }
+
+#define COMPILE_X8_S8_C8_S8(j, comp) \
+ { \
+ uint8_t a, b, c; \
+ UNPACK_8_8_8 (j->ip[0], a, b, c); \
+ comp (j, a, b, c); \
+ }
+#define COMPILE_X8_S8_S8_C8(j, comp) \
+ COMPILE_X8_S8_C8_S8 (j, comp)
+#define COMPILE_X8_S8_S8_S8(j, comp) \
+ COMPILE_X8_S8_C8_S8 (j, comp)
+
+#define COMPILE_X8_S8_I16(j, comp) \
+ { \
+ uint8_t a; \
+ scm_t_bits b; \
+ UNPACK_8_16 (j->ip[0], a, b); \
+ comp (j, a, SCM_PACK (b)); \
+ }
+
+#define COMPILE_X32__C32(j, comp) \
+ { \
+ comp (j, j->ip[1]); \
+ }
+
+#define COMPILE_X32__L32(j, comp) \
+ { \
+ int32_t a = j->ip[1]; \
+ comp (j, j->ip + a); \
+ }
+#define COMPILE_X32__N32(j, comp) \
+ COMPILE_X32__L32 (j, comp)
+
+#define COMPILE_X8_C24__L32(j, comp) \
+ { \
+ uint32_t a; \
+ int32_t b; \
+ UNPACK_24 (j->ip[0], a); \
+ b = j->ip[1]; \
+ comp (j, a, j->ip + b); \
+ }
+#define COMPILE_X8_S24__L32(j, comp) \
+ COMPILE_X8_C24__L32 (j, comp)
+#define COMPILE_X8_S24__LO32(j, comp) \
+ COMPILE_X8_C24__L32 (j, comp)
+#define COMPILE_X8_S24__N32(j, comp) \
+ COMPILE_X8_C24__L32 (j, comp)
+#define COMPILE_X8_S24__R32(j, comp) \
+ COMPILE_X8_C24__L32 (j, comp)
+
+#define COMPILE_X8_C24__X8_C24(j, comp) \
+ { \
+ uint32_t a, b; \
+ UNPACK_24 (j->ip[0], a); \
+ UNPACK_24 (j->ip[1], b); \
+ comp (j, a, b); \
+ }
+#define COMPILE_X8_F24__X8_C24(j, comp) \
+ COMPILE_X8_C24__X8_C24(j, comp)
+#define COMPILE_X8_F24__X8_F24(j, comp) \
+ COMPILE_X8_C24__X8_C24(j, comp)
+#define COMPILE_X8_S24__X8_S24(j, comp) \
+ COMPILE_X8_C24__X8_C24(j, comp)
+
+#define COMPILE_X8_F12_F12__X8_C24(j, comp) \
+ { \
+ uint16_t a, b; \
+ uint32_t c; \
+ UNPACK_12_12 (j->ip[0], a, b); \
+ UNPACK_24 (j->ip[1], c); \
+ comp (j, a, b, c); \
+ }
+
+#define COMPILE_X8_F24__B1_X7_C24(j, comp) \
+ { \
+ uint32_t a, c; \
+ uint8_t b; \
+ UNPACK_24 (j->ip[0], a); \
+ b = j->ip[1] & 0x1; \
+ UNPACK_24 (j->ip[1], c); \
+ comp (j, a, b, c); \
+ }
+
+#define COMPILE_X8_S12_S12__C32(j, comp) \
+ { \
+ uint16_t a, b; \
+ uint32_t c; \
+ UNPACK_12_12 (j->ip[0], a, b); \
+ c = j->ip[1]; \
+ comp (j, a, b, c); \
+ }
+
+#define COMPILE_X8_S24__C16_C16(j, comp) \
+ { \
+ uint32_t a; \
+ uint16_t b, c; \
+ UNPACK_24 (j->ip[0], a); \
+ UNPACK_16_16 (j->ip[1], b, c); \
+ comp (j, a, b, c); \
+ }
+
+#define COMPILE_X8_S24__C32(j, comp) \
+ { \
+ uint32_t a, b; \
+ UNPACK_24 (j->ip[0], a); \
+ b = j->ip[1]; \
+ comp (j, a, b); \
+ }
+
+#define COMPILE_X8_S24__I32(j, comp) \
+ { \
+ uint32_t a; \
+ scm_t_bits b; \
+ UNPACK_24 (j->ip[0], a); \
+ b = j->ip[1]; \
+ comp (j, a, SCM_PACK (b)); \
+ }
+
+#define COMPILE_X8_S8_S8_C8__C32(j, comp) \
+ { \
+ uint8_t a, b, c; \
+ uint32_t d; \
+ UNPACK_8_8_8 (j->ip[0], a, b, c); \
+ d = j->ip[1]; \
+ comp (j, a, b, c, d); \
+ }
+#define COMPILE_X8_S8_S8_S8__C32(j, comp) \
+ COMPILE_X8_S8_S8_C8__C32(j, comp)
+
+#define COMPILE_X32__LO32__L32(j, comp) \
+ { \
+ int32_t a = j->ip[1], b = j->ip[2]; \
+ comp (j, j->ip + a, j->ip + b); \
+ }
+
+#define COMPILE_X8_F24__X8_C24__L32(j, comp) \
+ { \
+ uint32_t a, b; \
+ int32_t c; \
+ UNPACK_24 (j->ip[0], a); \
+ UNPACK_24 (j->ip[1], b); \
+ c = j->ip[2]; \
+ comp (j, a, b, j->ip + c); \
+ }
+
+#define COMPILE_X8_S24__A32__B32(j, comp) \
+ { \
+ uint32_t a; \
+ uint64_t b; \
+ UNPACK_24 (j->ip[0], a); \
+ b = (((uint64_t) j->ip[1]) << 32) | ((uint64_t) j->ip[2]); \
+ ASSERT (b <= (uint64_t) UINTPTR_MAX); \
+ comp (j, a, SCM_PACK ((uintptr_t) b)); \
+ }
+
+#define COMPILE_X8_S24__AF32__BF32(j, comp) \
+ { \
+ uint32_t a; \
+ union { uint64_t u; double d; } b; \
+ UNPACK_24 (j->ip[0], a); \
+ b.u = (((uint64_t) j->ip[1]) << 32) | ((uint64_t) j->ip[2]); \
+ comp (j, a, b.d); \
+ }
+
+#define COMPILE_X8_S24__AS32__BS32(j, comp) \
+ { \
+ uint32_t a; \
+ uint64_t b; \
+ UNPACK_24 (j->ip[0], a); \
+ b = (((uint64_t) j->ip[1]) << 32) | ((uint64_t) j->ip[2]); \
+ comp (j, a, (int64_t) b); \
+ }
+
+#define COMPILE_X8_S24__AU32__BU32(j, comp) \
+ { \
+ uint32_t a; \
+ uint64_t b; \
+ UNPACK_24 (j->ip[0], a); \
+ b = (((uint64_t) j->ip[1]) << 32) | ((uint64_t) j->ip[2]); \
+ comp (j, a, b); \
+ }
+
+#define COMPILE_X8_S24__B1_X7_F24__X8_L24(j, comp) \
+ { \
+ uint32_t a, c; \
+ uint8_t b; \
+ int32_t d; \
+ UNPACK_24 (j->ip[0], a); \
+ b = j->ip[1] & 0x1; \
+ UNPACK_24 (j->ip[1], c); \
+ d = j->ip[2]; d >>= 8; /* Sign extension. */ \
+ comp (j, a, b, c, j->ip + d); \
+ }
+
+#define COMPILE_X8_S24__X8_S24__C8_S24(j, comp) \
+ { \
+ uint32_t a, b, d; \
+ uint8_t c; \
+ UNPACK_24 (j->ip[0], a); \
+ UNPACK_24 (j->ip[1], b); \
+ UNPACK_8_24 (j->ip[2], c, d); \
+ comp (j, a, b, c, d); \
+ }
+
+#define COMPILE_X8_C24__C8_C24__X8_C24__N32(j, comp) \
+ { \
+ uint32_t a, c, d; \
+ uint8_t b; \
+ int32_t e; \
+ UNPACK_24 (j->ip[0], a); \
+ UNPACK_8_24 (j->ip[1], b, c); \
+ UNPACK_24 (j->ip[2], d); \
+ e = j->ip[3]; \
+ comp (j, a, b, c, d, j->ip + e); \
+ }
+
+#define COMPILE_X8_S24__X8_S24__C8_S24__X8_S24(j, comp) \
+ { \
+ uint32_t a, b, d, e; \
+ uint8_t c; \
+ UNPACK_24 (j->ip[0], a); \
+ UNPACK_24 (j->ip[1], b); \
+ UNPACK_8_24 (j->ip[2], c, d); \
+ UNPACK_24 (j->ip[3], e); \
+ comp (j, a, b, c, d, e); \
+ }
+
+static uint8_t first_seen[256];
+
+static void
+compile1 (scm_jit_state *j)
+{
+ uint8_t opcode = j->ip[0] & 0xff;
+
+ if (!first_seen[opcode])
+ {
+ const char *n;
+ switch (opcode)
+ {
+#define NAME(code, cname, name, arity) case code: n = name; break;
+ FOR_EACH_VM_OPERATION(NAME)
+#undef NAME
+ default:
+ UNREACHABLE ();
+ }
+ first_seen[opcode] = 1;
+ DEBUG ("Instruction first seen at vcode %p: %s\n", j->ip, n);
+ }
+
+ j->next_ip = j->ip + op_lengths[opcode];
+
+ switch (opcode)
+ {
+#define COMPILE1(code, cname, name, arity) \
+ case code: COMPILE_##arity(j, compile_##cname); break;
+ FOR_EACH_VM_OPERATION(COMPILE1)
+#undef COMPILE1
+ default:
+ UNREACHABLE ();
+ }
+
+ j->ip = j->next_ip;
+}
+
+static void
+analyze (scm_jit_state *j)
+{
+ memset (j->op_attrs, 0, j->end - j->start);
+
+ j->op_attrs[0] = OP_ATTR_BLOCK | OP_ATTR_ENTRY;
+
+ for (j->ip = (uint32_t *) j->start; j->ip < j->end; j->ip = j->next_ip)
+ {
+ uint8_t opcode = j->ip[0] & 0xff;
+ uint8_t attrs = 0;
+ uint32_t *target;
+
+ j->next_ip = j->ip + op_lengths[opcode];
+
+ switch (opcode)
+ {
+ case scm_op_check_arguments:
+ case scm_op_check_positional_arguments:
+ attrs |= OP_ATTR_ENTRY;
+ /* Fall through. */
+ case scm_op_u64_numerically_equal:
+ case scm_op_u64_less:
+ case scm_op_s64_less:
+ case scm_op_f64_numerically_equal:
+ case scm_op_f64_less:
+ case scm_op_numerically_equal:
+ case scm_op_less:
+ case scm_op_immediate_tag_equals:
+ case scm_op_heap_tag_equals:
+ case scm_op_eq:
+ case scm_op_heap_numbers_equal:
+ case scm_op_s64_imm_numerically_equal:
+ case scm_op_u64_imm_less:
+ case scm_op_imm_u64_less:
+ case scm_op_s64_imm_less:
+ case scm_op_imm_s64_less:
+ attrs |= OP_ATTR_BLOCK;
+ fuse_conditional_branch (j, &target);
+ j->op_attrs[target - j->start] |= attrs;
+ break;
+
+ case scm_op_j:
+ target = j->ip + (((int32_t)j->ip[0]) >> 8);
+ j->op_attrs[target - j->start] |= OP_ATTR_BLOCK;
+ break;
+
+ case scm_op_call:
+ case scm_op_call_label:
+ attrs = OP_ATTR_BLOCK;
+ target = j->next_ip;
+ j->op_attrs[target - j->start] |= OP_ATTR_BLOCK | OP_ATTR_ENTRY;
+ break;
+
+ case scm_op_prompt:
+ target = j->ip + (((int32_t) j->ip[2]) >> 8);
+ j->op_attrs[target - j->start] |= OP_ATTR_BLOCK | OP_ATTR_ENTRY;
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ /* Even in loops, the entry should be a jump target. */
+ ASSERT (j->op_attrs[j->entry - j->start] & OP_ATTR_BLOCK);
+}
+
+static void
+compile (scm_jit_state *j)
+{
+ j->ip = (uint32_t *) j->start;
+ set_register_state (j, SP_IN_REGISTER | FP_IN_REGISTER);
+ j->frame_size = -1;
+
+ for (ptrdiff_t offset = 0; j->ip + offset < j->end; offset++)
+ j->labels[offset] = NULL;
+
+ j->reloc_idx = 0;
+
+ while (j->ip < j->end)
+ {
+ ptrdiff_t offset = j->ip - j->start;
+ uint8_t attrs = j->op_attrs[offset];
+ j->labels[offset] = jit_address (j->jit);
+ if (attrs & OP_ATTR_BLOCK)
+ {
+ uint32_t state = SP_IN_REGISTER;
+ if (attrs & OP_ATTR_ENTRY)
+ state |= FP_IN_REGISTER;
+ j->register_state = state;
+ }
+ compile1 (j);
+
+ if (jit_has_overflow (j->jit))
+ return;
+ }
+
+ for (size_t i = 0; i < j->reloc_idx; i++)
+ {
+ void *target = j->labels[j->relocs[i].target_vcode_offset];
+ ASSERT(target);
+ jit_patch_there (j->jit, j->relocs[i].reloc, target);
+ }
+}
+
+static scm_i_pthread_once_t initialize_jit_once = SCM_I_PTHREAD_ONCE_INIT;
+
+static void*
+jit_alloc_fn (size_t size)
+{
+ return scm_gc_malloc (size, "jit state");
+}
+
+static void
+jit_free_fn (void *unused)
+{
+}
+
+static scm_jit_state *
+initialize_thread_jit_state (scm_thread *thread)
+{
+ scm_jit_state *j;
+
+ ASSERT (!thread->jit_state);
+
+ j = scm_gc_malloc (sizeof (*j), "jit state");
+ memset (j, 0, sizeof (*j));
+ thread->jit_state = j;
+ j->jit = jit_new_state (jit_alloc_fn, jit_free_fn);
+
+ return j;
+}
+
+static void
+initialize_jit (void)
+{
+ scm_jit_state *j;
+
+ if (!init_jit ())
+ {
+ scm_jit_counter_threshold = -1;
+ fprintf (stderr, "JIT failed to initialize\n");
+ fprintf (stderr, "disabling automatic JIT compilation\n");
+ return;
+ }
+
+ /* Init the thread's jit state so we can emit the entry
+ trampoline and the handle-interrupts trampoline. */
+ j = initialize_thread_jit_state (SCM_I_CURRENT_THREAD);
+
+ enter_mcode = emit_code (j, emit_entry_trampoline);
+ ASSERT (enter_mcode);
+
+ handle_interrupts_trampoline =
+ emit_code (j, emit_handle_interrupts_trampoline);
+ ASSERT (handle_interrupts_trampoline);
+}
+
+static uint8_t *
+compute_mcode (scm_thread *thread, uint32_t *entry_ip,
+ struct scm_jit_function_data *data)
+{
+ scm_jit_state *j = thread->jit_state;
+ uint8_t *entry_mcode;
+
+ if (!j)
+ {
+ scm_i_pthread_once (&initialize_jit_once, initialize_jit);
+ if (scm_jit_counter_threshold == -1)
+ {
+ /* initialization failed! */
+ return NULL;
+ }
+
+ j = thread->jit_state;
+ /* It's possible that initialize_jit_once inits this thread's jit
+ state. */
+ if (!j)
+ j = initialize_thread_jit_state (thread);
+ }
+
+ j->thread = thread;
+ j->start = (const uint32_t *) (((char *)data) + data->start);
+ j->end = (const uint32_t *) (((char *)data) + data->end);
+ j->entry = entry_ip;
+
+ ASSERT (j->start < j->end);
+ ASSERT (j->start <= j->entry);
+ ASSERT (j->entry < j->end);
+
+ j->op_attrs = calloc ((j->end - j->start), sizeof (*j->op_attrs));
+ ASSERT (j->op_attrs);
+ j->labels = calloc ((j->end - j->start), sizeof (*j->labels));
+ ASSERT (j->labels);
+
+ j->frame_size = -1;
+
+ INFO ("vcode: start=%p,+%zu entry=+%zu\n", j->start, j->end - j->start,
+ j->entry - j->start);
+
+ analyze (j);
+
+ data->mcode = emit_code (j, compile);
+ if (data->mcode)
+ entry_mcode = j->labels[j->entry - j->start];
+ else
+ entry_mcode = NULL;
+
+ free (j->op_attrs);
+ j->op_attrs = NULL;
+ free (j->labels);
+ j->labels = NULL;
+ free (j->relocs);
+ j->relocs = NULL;
+ j->reloc_idx = 0;
+ j->reloc_count = 0;
+
+ j->start = j->end = j->ip = j->entry = NULL;
+ j->frame_size = -1;
+
+ return entry_mcode;
+}
+
+const uint8_t *
+scm_jit_compute_mcode (scm_thread *thread, struct scm_jit_function_data *data)
+{
+ const uint32_t *vcode_start = (const uint32_t *) (((char *)data) + data->start);
+
+ if (data->mcode)
+ {
+ if (vcode_start == thread->vm.ip)
+ return data->mcode;
+
+ /* FIXME: The function has mcode, compiled via some other
+ activation (possibly in another thread), but right now we're
+ currently in an interpreted loop (not at the beginning of the
+ function). We should re-compute the offset into the mcode.
+ For now though, just punt. */
+ return NULL;
+ }
+ else
+ {
+ uint8_t *mcode = compute_mcode (thread, thread->vm.ip, data);
+
+ if (!mcode)
+ {
+ scm_jit_counter_threshold = -1;
+ fprintf (stderr, "JIT failed due to resource exhaustion\n");
+ fprintf (stderr, "disabling automatic JIT compilation\n");
+ }
+ else if (--jit_stop_after == 0)
+ {
+ scm_jit_counter_threshold = -1;
+ fprintf (stderr, "stopping automatic JIT compilation, as requested\n");
+ if (jit_pause_when_stopping)
+ {
+ fprintf (stderr, "sleeping for 30s; to debug:\n");
+ fprintf (stderr, " gdb -p %d\n\n", getpid ());
+ sleep (30);
+ }
+ }
+
+ return mcode;
+ }
+}
+
+void
+scm_jit_enter_mcode (scm_thread *thread, const uint8_t *mcode)
+{
+ LOG ("entering mcode: %p\n", mcode);
+ enter_mcode (thread, mcode);
+ LOG ("exited mcode\n");
+}
+
+void
+scm_jit_state_free (scm_jit_state *j)
+{
+ /* Nothing to do; we leave j->jit NULL between compilations. */
+}
+
+void
+scm_init_jit (void)
+{
+ scm_jit_counter_threshold = scm_getenv_int ("GUILE_JIT_THRESHOLD",
+ default_jit_threshold);
+ jit_stop_after = scm_getenv_int ("GUILE_JIT_STOP_AFTER", -1);
+ jit_pause_when_stopping = scm_getenv_int ("GUILE_JIT_PAUSE_WHEN_STOPPING", 0);
+ jit_log_level = scm_getenv_int ("GUILE_JIT_LOG", 0);
+}
+
+#endif /* ENABLE_JIT */