summaryrefslogtreecommitdiff
path: root/libguile/lightning/lightening/x86-cpu.c
diff options
context:
space:
mode:
Diffstat (limited to 'libguile/lightning/lightening/x86-cpu.c')
-rw-r--r--libguile/lightning/lightening/x86-cpu.c2762
1 files changed, 2762 insertions, 0 deletions
diff --git a/libguile/lightning/lightening/x86-cpu.c b/libguile/lightning/lightening/x86-cpu.c
new file mode 100644
index 000000000..1c1da2b4f
--- /dev/null
+++ b/libguile/lightning/lightening/x86-cpu.c
@@ -0,0 +1,2762 @@
+/*
+ * Copyright (C) 2012-2019 Free Software Foundation, Inc.
+ *
+ * This file is part of GNU lightning.
+ *
+ * GNU lightning is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; either version 3, or (at your option)
+ * any later version.
+ *
+ * GNU lightning is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+ * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+ * License for more details.
+ *
+ * Authors:
+ * Paulo Cesar Pereira de Andrade
+ */
+
+/* avoid using it due to partial stalls */
+#define USE_INC_DEC 0
+
+#if __X32 || __X64_32
+# define WIDE 0
+# define IF_WIDE(wide, narrow) narrow
+#else
+# define WIDE 1
+# define IF_WIDE(wide, narrow) wide
+#endif
+
+#define _RAX_REGNO 0
+#define _RCX_REGNO 1
+#define _RDX_REGNO 2
+#define _RBX_REGNO 3
+#define _RSP_REGNO 4
+#define _RBP_REGNO 5
+#define _RSI_REGNO 6
+#define _RDI_REGNO 7
+#define _R8_REGNO 8
+#define _R9_REGNO 9
+#define _R10_REGNO 10
+#define _R11_REGNO 11
+#define _R12_REGNO 12
+#define _R13_REGNO 13
+#define _R14_REGNO 14
+#define _R15_REGNO 15
+#define r7(reg) ((reg) & 7)
+#define r8(reg) ((reg) & 15)
+#if __X32 || __CYGWIN__ || __X64_32
+# define reg8_p(rn) ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
+#else
+# define reg8_p(rn) 1
+#endif
+
+#define can_sign_extend_int_p(im) \
+ IF_WIDE((((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
+ ((im) < 0 && (long long)(im) > -0x80000000LL)), \
+ 1)
+#define can_zero_extend_int_p(im) \
+ IF_WIDE(((im) >= 0 && (im) < 0x80000000LL), \
+ 1)
+#define fits_uint32_p(im) \
+ IF_WIDE((((im) & 0xffffffff00000000LL) == 0), \
+ 1)
+
+#define _SCL1 0x00
+#define _SCL2 0x01
+#define _SCL4 0x02
+#define _SCL8 0x03
+
+#define X86_ADD 0
+#define X86_OR 1 << 3
+#define X86_ADC 2 << 3
+#define X86_SBB 3 << 3
+#define X86_AND 4 << 3
+#define X86_SUB 5 << 3
+#define X86_XOR 6 << 3
+#define X86_CMP 7 << 3
+#define X86_ROL 0
+#define X86_ROR 1
+#define X86_RCL 2
+#define X86_RCR 3
+#define X86_SHL 4
+#define X86_SHR 5
+#define X86_SAR 7
+#define X86_NOT 2
+#define X86_NEG 3
+#define X86_MUL 4
+#define X86_IMUL 5
+#define X86_DIV 6
+#define X86_IDIV 7
+
+#define FOR_EACH_CC(M) \
+ M(o, O, 0x0) \
+ M(no, NO, 0x1) \
+ M(nae, NAE, 0x2) \
+ M(b, B, 0x2) \
+ M(c, C, 0x2) \
+ M(ae, AE, 0x3) \
+ M(nb, NB, 0x3) \
+ M(nc, NC, 0x3) \
+ M(e, E, 0x4) \
+ M(z, Z, 0x4) \
+ M(ne, NE, 0x5) \
+ M(nz, NZ, 0x5) \
+ M(be, BE, 0x6) \
+ M(na, NA, 0x6) \
+ M(a, A, 0x7) \
+ M(nbe, NBE, 0x7) \
+ M(s, S, 0x8) \
+ M(ns, NS, 0x9) \
+ M(p, P, 0xa) \
+ M(pe, PE, 0xa) \
+ M(np, NP, 0xb) \
+ M(po, PO, 0xb) \
+ M(l, L, 0xc) \
+ M(nge, NGE, 0xc) \
+ M(ge, GE, 0xd) \
+ M(nl_, NL, 0xd) \
+ M(le, LE, 0xe) \
+ M(ng, NG, 0xe) \
+ M(g, G, 0xf) \
+ M(nle, NLE, 0xf) \
+ /* EOL */
+
+enum x86_cc
+{
+#define DEFINE_ENUM(cc, CC, code) X86_CC_##CC = code,
+ FOR_EACH_CC(DEFINE_ENUM)
+#undef DEFINE_ENUM
+};
+
+static inline void
+mrm(jit_state_t *_jit, uint8_t md, uint8_t r, uint8_t m)
+{
+ emit_u8(_jit, (md<<6) | (r<<3) | m);
+}
+
+static inline void
+sib(jit_state_t *_jit, uint8_t sc, uint8_t i, uint8_t b)
+{
+ emit_u8(_jit, (sc<<6) | (i<<3) | b);
+}
+
+static inline void
+ic(jit_state_t *_jit, uint8_t c)
+{
+ emit_u8(_jit, c);
+}
+
+static inline void
+is(jit_state_t *_jit, uint16_t s)
+{
+ emit_u16(_jit, s);
+}
+
+static inline void
+ii(jit_state_t *_jit, uint32_t i)
+{
+ emit_u32(_jit, i);
+}
+
+static inline void
+il(jit_state_t *_jit, unsigned long l)
+{
+#if __X64 && !__X64_32
+ emit_u64(_jit, l);
+#else
+ ii(_jit, l);
+#endif
+}
+
+static void
+rex(jit_state_t *_jit, int32_t l, int32_t w,
+ int32_t r, int32_t x, int32_t b)
+{
+#if __X64
+ int32_t v = 0x40 | (w << 3);
+
+ if (r != _NOREG)
+ v |= (r & 8) >> 1;
+ if (x != _NOREG)
+ v |= (x & 8) >> 2;
+ if (b != _NOREG)
+ v |= (b & 8) >> 3;
+ if (l || v != 0x40)
+ ic(_jit, v);
+#endif
+}
+
+static void
+rx(jit_state_t *_jit, int32_t rd, int32_t md,
+ int32_t rb, int32_t ri, int32_t ms)
+{
+ if (ri == _NOREG) {
+ if (rb == _NOREG) {
+#if __X32
+ mrm(_jit, 0x00, r7(rd), 0x05);
+#else
+ mrm(_jit, 0x00, r7(rd), 0x04);
+ sib(_jit, _SCL1, 0x04, 0x05);
+#endif
+ ii(_jit, md);
+ } else if (r7(rb) == _RSP_REGNO) {
+ if (md == 0) {
+ mrm(_jit, 0x00, r7(rd), 0x04);
+ sib(_jit, ms, 0x04, 0x04);
+ }
+ else if ((int8_t)md == md) {
+ mrm(_jit, 0x01, r7(rd), 0x04);
+ sib(_jit, ms, 0x04, 0x04);
+ ic(_jit, md);
+ } else {
+ mrm(_jit, 0x02, r7(rd), 0x04);
+ sib(_jit, ms, 0x04, 0x04);
+ ii(_jit, md);
+ }
+ } else {
+ if (md == 0 && r7(rb) != _RBP_REGNO)
+ mrm(_jit, 0x00, r7(rd), r7(rb));
+ else if ((int8_t)md == md) {
+ mrm(_jit, 0x01, r7(rd), r7(rb));
+ ic(_jit, md);
+ } else {
+ mrm(_jit, 0x02, r7(rd), r7(rb));
+ ii(_jit, md);
+ }
+ }
+ }
+ else if (rb == _NOREG) {
+ mrm(_jit, 0x00, r7(rd), 0x04);
+ sib(_jit, ms, r7(ri), 0x05);
+ ii(_jit, md);
+ }
+ else if (r8(ri) != _RSP_REGNO) {
+ if (md == 0 && r7(rb) != _RBP_REGNO) {
+ mrm(_jit, 0x00, r7(rd), 0x04);
+ sib(_jit, ms, r7(ri), r7(rb));
+ } else if ((int8_t)md == md) {
+ mrm(_jit, 0x01, r7(rd), 0x04);
+ sib(_jit, ms, r7(ri), r7(rb));
+ ic(_jit, md);
+ } else {
+ mrm(_jit, 0x02, r7(rd), 0x04);
+ sib(_jit, ms, r7(ri), r7(rb));
+ ic(_jit, md);
+ }
+ } else {
+ fprintf(stderr, "illegal index register");
+ abort();
+ }
+}
+
+static void
+pushr(jit_state_t *_jit, int32_t r0)
+{
+ rex(_jit, 0, WIDE, 0, 0, r0);
+ ic(_jit, 0x50 | r7(r0));
+}
+
+static void
+popr(jit_state_t *_jit, int32_t r0)
+{
+ rex(_jit, 0, WIDE, 0, 0, r0);
+ ic(_jit, 0x58 | r7(r0));
+}
+
+static jit_gpr_t
+get_temp_gpr(jit_state_t *_jit)
+{
+ ASSERT(!_jit->temp_gpr_saved);
+ _jit->temp_gpr_saved = 1;
+#if __X32
+ pushr(_jit, _RBP_REGNO);
+ return JIT_GPR(_RBP);
+#else
+ return JIT_GPR(_R8);
+#endif
+}
+
+static void
+unget_temp_gpr(jit_state_t *_jit)
+{
+ ASSERT(_jit->temp_gpr_saved);
+ _jit->temp_gpr_saved = 0;
+#if __X32
+ popr(_jit, _RBP_REGNO);
+#endif
+}
+
+static void
+nop(jit_state_t *_jit, int32_t count)
+{
+ switch (count) {
+ case 0:
+ break;
+ case 1: /* NOP */
+ ic(_jit, 0x90);
+ break;
+ case 2: /* 66 NOP */
+ ic(_jit, 0x66); ic(_jit, 0x90);
+ break;
+ case 3: /* NOP DWORD ptr [EAX] */
+ ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x00);
+ break;
+ case 4: /* NOP DWORD ptr [EAX + 00H] */
+ ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x40); ic(_jit, 0x00);
+ break;
+ case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
+ ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x44); ic(_jit, 0x00);
+ ic(_jit, 0x00);
+ break;
+ case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
+ ic(_jit, 0x66); ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x44);
+ ic(_jit, 0x00); ic(_jit, 0x00);
+ break;
+ case 7: /* NOP DWORD ptr [EAX + 00000000H] */
+ ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x80); ii(_jit, 0x0000);
+ break;
+ case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
+ ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x84); ic(_jit, 0x00);
+ ii(_jit, 0x0000);
+ break;
+ case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
+ ic(_jit, 0x66); ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x84);
+ ic(_jit, 0x00); ii(_jit, 0x0000);
+ break;
+ default:
+ abort();
+ }
+}
+
+static void
+movr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ if (r0 != r1) {
+ rex(_jit, 0, 1, r1, _NOREG, r0);
+ ic(_jit, 0x89);
+ ic(_jit, 0xc0 | (r1 << 3) | r7(r0));
+ }
+}
+
+static void
+movcr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xbe);
+ mrm(_jit, 0x03, r7(r0), r7(r1));
+}
+
+static void
+movcr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb6);
+ mrm(_jit, 0x03, r7(r0), r7(r1));
+}
+
+static void
+movsr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xbf);
+ mrm(_jit, 0x03, r7(r0), r7(r1));
+}
+
+static void
+movsr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb7);
+ mrm(_jit, 0x03, r7(r0), r7(r1));
+}
+
+#if __X64
+static void
+movir(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, 1, r0, _NOREG, r1);
+ ic(_jit, 0x63);
+ mrm(_jit, 0x03, r7(r0), r7(r1));
+}
+
+static void
+movir_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, 0, r1, _NOREG, r0);
+ ic(_jit, 0x89);
+ ic(_jit, 0xc0 | (r1 << 3) | r7(r0));
+}
+#endif
+
+static jit_reloc_t
+mov_addr(jit_state_t *_jit, int32_t r0)
+{
+ uint8_t *pc_start = _jit->pc.uc;
+ rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
+ ic(_jit, 0xb8 | r7(r0));
+ ptrdiff_t inst_start = _jit->pc.uc - pc_start;
+ return jit_reloc(_jit, JIT_RELOC_ABSOLUTE, inst_start, 0);
+}
+
+static void
+imovi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+#if __X64
+# if !__X64_32
+ if (fits_uint32_p(i0)) {
+# endif
+ rex(_jit, 0, 0, _NOREG, _NOREG, r0);
+ ic(_jit, 0xb8 | r7(r0));
+ ii(_jit, i0);
+# if !__X64_32
+ } else {
+ rex(_jit, 0, 1, _NOREG, _NOREG, r0);
+ ic(_jit, 0xb8 | r7(r0));
+ il(_jit, i0);
+ }
+# endif
+#else
+ ic(_jit, 0xb8 | r7(r0));
+ ii(_jit, i0);
+#endif
+}
+
+static void
+alur(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r1, _NOREG, r0);
+ ic(_jit, code | 0x01);
+ mrm(_jit, 0x03, r7(r1), r7(r0));
+}
+
+static inline void
+icmpr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return alur(_jit, X86_CMP, r0, r1);
+}
+static inline void
+iaddr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return alur(_jit, X86_ADD, r0, r1);
+}
+static inline void
+iaddxr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return alur(_jit, X86_ADC, r0, r1);
+}
+static inline void
+isubr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return alur(_jit, X86_SUB, r0, r1);
+}
+static inline void
+isubxr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return alur(_jit, X86_SBB, r0, r1);
+}
+static inline void
+iandr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return alur(_jit, X86_AND, r0, r1);
+}
+static inline void
+iorr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return alur(_jit, X86_OR, r0, r1);
+}
+static inline void
+ixorr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return alur(_jit, X86_XOR, r0, r1);
+}
+
+static void
+movi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ if (i0)
+ imovi(_jit, r0, i0);
+ else
+ ixorr(_jit, r0, r0);
+}
+
+static void
+alui(jit_state_t *_jit, int32_t code, int32_t r0, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
+ if ((int8_t)i0 == i0) {
+ ic(_jit, 0x83);
+ ic(_jit, 0xc0 | code | r7(r0));
+ ic(_jit, i0);
+ } else {
+ if (r0 == _RAX_REGNO) {
+ ic(_jit, code | 0x05);
+ } else {
+ ic(_jit, 0x81);
+ ic(_jit, 0xc0 | code | r7(r0));
+ }
+ ii(_jit, i0);
+ }
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ alur(_jit, code, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static inline void
+icmpi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ return alui(_jit, X86_CMP, r0, i0);
+}
+static inline void
+iaddi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ return alui(_jit, X86_ADD, r0, i0);
+}
+static inline void
+iaddxi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ return alui(_jit, X86_ADC, r0, i0);
+}
+static inline void
+isubi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ return alui(_jit, X86_SUB, r0, i0);
+}
+static inline void
+isubxi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ return alui(_jit, X86_SBB, r0, i0);
+}
+static inline void
+iandi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ return alui(_jit, X86_AND, r0, i0);
+}
+static inline void
+iori(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ return alui(_jit, X86_OR, r0, i0);
+}
+static inline void
+ixori(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ return alui(_jit, X86_XOR, r0, i0);
+}
+
+static void
+unr(jit_state_t *_jit, int32_t code, int32_t r0)
+{
+ rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
+ ic(_jit, 0xf7);
+ mrm(_jit, 0x03, code, r7(r0));
+}
+
+static inline void
+umulr(jit_state_t *_jit, int32_t r0)
+{
+ return unr(_jit, X86_IMUL, r0);
+}
+static inline void
+umulr_u(jit_state_t *_jit, int32_t r0)
+{
+ return unr(_jit, X86_MUL, r0);
+}
+static inline void
+idivr(jit_state_t *_jit, int32_t r0)
+{
+ return unr(_jit, X86_IDIV, r0);
+}
+static inline void
+idivr_u(jit_state_t *_jit, int32_t r0)
+{
+ return unr(_jit, X86_DIV, r0);
+}
+static inline void
+inegr(jit_state_t *_jit, int32_t r0)
+{
+ return unr(_jit, X86_NEG, r0);
+}
+static inline void
+icomr(jit_state_t *_jit, int32_t r0)
+{
+ return unr(_jit, X86_NOT, r0);
+}
+
+#if USE_INC_DEC
+static void
+incr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ movr(_jit, r0, r1);
+# if __X64
+ rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
+ ic(_jit, 0xff);
+ ic(_jit, 0xc0 | r7(r0));
+# else
+ ic(_jit, 0x40 | r7(r0));
+# endif
+}
+
+static void
+decr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ movr(_jit, r0, r1);
+# if __X64
+ rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
+ ic(_jit, 0xff);
+ ic(_jit, 0xc8 | r7(r0));
+# else
+ ic(_jit, 0x48 | r7(r0));
+# endif
+}
+#endif
+
+static void
+lea(jit_state_t *_jit, int32_t md, int32_t rb,
+ int32_t ri, int32_t ms, int32_t rd)
+{
+ rex(_jit, 0, WIDE, rd, ri, rb);
+ ic(_jit, 0x8d);
+ rx(_jit, rd, md, rb, ri, ms);
+}
+
+static void
+xchgr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r1, _NOREG, r0);
+ ic(_jit, 0x87);
+ mrm(_jit, 0x03, r7(r1), r7(r0));
+}
+
+static void
+testr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r1, _NOREG, r0);
+ ic(_jit, 0x85);
+ mrm(_jit, 0x03, r7(r1), r7(r0));
+}
+
+static void
+testi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
+ if (r0 == _RAX_REGNO) {
+ ic(_jit, 0xa9);
+ } else {
+ ic(_jit, 0xf7);
+ mrm(_jit, 0x03, 0x00, r7(r0));
+ }
+ ii(_jit, i0);
+}
+
+static void
+cc(jit_state_t *_jit, int32_t code, int32_t r0)
+{
+ rex(_jit, 0, 0, _NOREG, _NOREG, r0);
+ ic(_jit, 0x0f);
+ ic(_jit, 0x90 | code);
+ mrm(_jit, 0x03, 0x00, r7(r0));
+}
+
+static void
+negr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ if (r0 == r1) {
+ inegr(_jit, r0);
+ } else {
+ ixorr(_jit, r0, r0);
+ isubr(_jit, r0, r1);
+ }
+}
+
+static void
+addr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r0 == r1)
+ iaddr(_jit, r0, r2);
+ else if (r0 == r2)
+ iaddr(_jit, r0, r1);
+ else
+ lea(_jit, 0, r1, r2, _SCL1, r0);
+}
+
+static void
+addi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (i0 == 0)
+ movr(_jit, r0, r1);
+#if USE_INC_DEC
+ else if (i0 == 1)
+ incr(_jit, r0, r1);
+ else if (i0 == -1)
+ decr(_jit, r0, r1);
+#endif
+ else if (can_sign_extend_int_p(i0)) {
+ if (r0 == r1)
+ iaddi(_jit, r0, i0);
+ else
+ lea(_jit, i0, r1, _NOREG, _SCL1, r0);
+ }
+ else if (r0 != r1) {
+ movi(_jit, r0, i0);
+ iaddr(_jit, r0, r1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ iaddr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+addcr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r0 == r2) {
+ iaddr(_jit, r0, r1);
+ } else {
+ movr(_jit, r0, r1);
+ iaddr(_jit, r0, r2);
+ }
+}
+
+static void
+addci(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ movr(_jit, r0, r1);
+ iaddi(_jit, r0, i0);
+ }
+ else if (r0 == r1) {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ iaddr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ } else {
+ movi(_jit, r0, i0);
+ iaddr(_jit, r0, r1);
+ }
+}
+
+static void
+addxr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r0 == r2) {
+ iaddxr(_jit, r0, r1);
+ } else {
+ movr(_jit, r0, r1);
+ iaddxr(_jit, r0, r2);
+ }
+}
+
+static void
+addxi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ movr(_jit, r0, r1);
+ iaddxi(_jit, r0, i0);
+ }
+ else if (r0 == r1) {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ iaddxr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ } else {
+ movi(_jit, r0, i0);
+ iaddxr(_jit, r0, r1);
+ }
+}
+
+static void
+subr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r1 == r2)
+ ixorr(_jit, r0, r0);
+ else if (r0 == r2) {
+ isubr(_jit, r0, r1);
+ inegr(_jit, r0);
+ } else {
+ movr(_jit, r0, r1);
+ isubr(_jit, r0, r2);
+ }
+}
+
+static void
+subi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (i0 == 0)
+ movr(_jit, r0, r1);
+#if USE_INC_DEC
+ else if (i0 == 1)
+ decr(_jit, r0, r1);
+ else if (i0 == -1)
+ incr(_jit, r0, r1);
+#endif
+ else if (can_sign_extend_int_p(i0)) {
+ if (r0 == r1)
+ isubi(_jit, r0, i0);
+ else
+ lea(_jit, -i0, r1, _NOREG, _SCL1, r0);
+ }
+ else if (r0 != r1) {
+ movi(_jit, r0, -i0);
+ iaddr(_jit, r0, r1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ isubr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+subcr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r0 == r2 && r0 != r1) {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movr(_jit, rn(reg), r0);
+ movr(_jit, r0, r1);
+ isubr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ } else {
+ movr(_jit, r0, r1);
+ isubr(_jit, r0, r2);
+ }
+}
+
+static void
+subci(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ movr(_jit, r0, r1);
+ if (can_sign_extend_int_p(i0)) {
+ isubi(_jit, r0, i0);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ isubr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+subxr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r0 == r2 && r0 != r1) {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movr(_jit, rn(reg), r0);
+ movr(_jit, r0, r1);
+ isubxr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ } else {
+ movr(_jit, r0, r1);
+ isubxr(_jit, r0, r2);
+ }
+}
+
+static void
+subxi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ movr(_jit, r0, r1);
+ if (can_sign_extend_int_p(i0)) {
+ isubxi(_jit, r0, i0);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ imovi(_jit, rn(reg), i0);
+ isubxr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+irotshr(jit_state_t *_jit, int32_t code, int32_t r0)
+{
+ rex(_jit, 0, WIDE, _RCX_REGNO, _NOREG, r0);
+ ic(_jit, 0xd3);
+ mrm(_jit, 0x03, code, r7(r0));
+}
+
+static void
+rotshr(jit_state_t *_jit, int32_t code,
+ int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r0 == _RCX_REGNO) {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movr(_jit, rn(reg), r1);
+ if (r2 != _RCX_REGNO)
+ movr(_jit, _RCX_REGNO, r2);
+ irotshr(_jit, code, rn(reg));
+ movr(_jit, _RCX_REGNO, rn(reg));
+ unget_temp_gpr(_jit);
+ } else if (r2 != _RCX_REGNO) {
+ /* Already know that R0 isn't RCX. */
+ pushr(_jit, _RCX_REGNO);
+ if (r1 == _RCX_REGNO) {
+ if (r0 == r2)
+ xchgr(_jit, r0, _RCX_REGNO);
+ else {
+ movr(_jit, r0, r1);
+ movr(_jit, _RCX_REGNO, r2);
+ }
+ } else {
+ movr(_jit, _RCX_REGNO, r2);
+ movr(_jit, r0, r1);
+ }
+ irotshr(_jit, code, r0);
+ popr(_jit, _RCX_REGNO);
+ } else {
+ movr(_jit, r0, r1);
+ irotshr(_jit, code, r0);
+ }
+}
+
+static void
+irotshi(jit_state_t *_jit, int32_t code, int32_t r0, jit_word_t i0)
+{
+ rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
+ if (i0 == 1) {
+ ic(_jit, 0xd1);
+ mrm(_jit, 0x03, code, r7(r0));
+ } else {
+ ic(_jit, 0xc1);
+ mrm(_jit, 0x03, code, r7(r0));
+ ic(_jit, i0);
+ }
+}
+
+static void
+rotshi(jit_state_t *_jit, int32_t code,
+ int32_t r0, int32_t r1, jit_word_t i0)
+{
+ movr(_jit, r0, r1);
+ if (i0)
+ irotshi(_jit, code, r0, i0);
+}
+
+static void
+lshi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (i0 == 0)
+ movr(_jit, r0, r1);
+ else if (i0 <= 3)
+ lea(_jit, 0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
+ else
+ rotshi(_jit, X86_SHL, r0, r1, i0);
+}
+
+static void
+lshr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ return rotshr(_jit, X86_SHL, r0, r1, r2);
+}
+
+static void
+rshr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ return rotshr(_jit, X86_SAR, r0, r1, r2);
+}
+
+static void
+rshi(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t i0)
+{
+ return rotshi(_jit, X86_SAR, r0, r1, i0);
+}
+
+static void
+rshr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ return rotshr(_jit, X86_SHR, r0, r1, r2);
+}
+
+static void
+rshi_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t i0)
+{
+ return rotshi(_jit, X86_SHR, r0, r1, i0);
+}
+
+static void
+imulr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xaf);
+ mrm(_jit, 0x03, r7(r0), r7(r1));
+}
+
+static void
+imuli(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ if ((int8_t)i0 == i0) {
+ ic(_jit, 0x6b);
+ mrm(_jit, 0x03, r7(r0), r7(r1));
+ ic(_jit, i0);
+ } else {
+ ic(_jit, 0x69);
+ mrm(_jit, 0x03, r7(r0), r7(r1));
+ ii(_jit, i0);
+ }
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ imulr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+mulr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r0 == r1)
+ imulr(_jit, r0, r2);
+ else if (r0 == r2) {
+ imulr(_jit, r0, r1);
+ } else {
+ movr(_jit, r0, r1);
+ imulr(_jit, r0, r2);
+ }
+}
+
+static int
+ffsw(jit_word_t i)
+{
+ if (sizeof(int) == sizeof(i))
+ return ffs(i);
+ int bit = ffs((int)i);
+ if (bit == 0) {
+ bit = ffs((int)((unsigned long)i >> 32));
+ if (bit)
+ bit += 32;
+ }
+ return bit;
+}
+
+static void
+muli(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ switch (i0) {
+ case 0:
+ ixorr(_jit, r0, r0);
+ break;
+ case 1:
+ movr(_jit, r0, r1);
+ break;
+ case -1:
+ negr(_jit, r0, r1);
+ break;
+ case 2:
+ lea(_jit, 0, _NOREG, r1, _SCL2, r0);
+ break;
+ case 4:
+ lea(_jit, 0, _NOREG, r1, _SCL4, r0);
+ break;
+ case 8:
+ lea(_jit, 0, _NOREG, r1, _SCL8, r0);
+ break;
+ default:
+ if (i0 > 0 && !(i0 & (i0 - 1)))
+ lshi(_jit, r0, r1, ffsw(i0) - 1);
+ else if (can_sign_extend_int_p(i0))
+ imuli(_jit, r0, r1, i0);
+ else if (r0 != r1) {
+ movi(_jit, r0, i0);
+ imulr(_jit, r0, r1);
+ }
+ else
+ imuli(_jit, r0, r0, i0);
+ break;
+ }
+}
+
+static void
+iqmulr(jit_state_t *_jit, int32_t r0, int32_t r1,
+ int32_t r2, int32_t r3, jit_bool_t sign)
+{
+ if (r0 != _RAX_REGNO && r1 != _RAX_REGNO)
+ pushr(_jit, _RAX_REGNO);
+ if (r0 != _RDX_REGNO && r1 != _RDX_REGNO)
+ pushr(_jit, _RDX_REGNO);
+
+ int32_t mul;
+ if (r3 == _RAX_REGNO) {
+ mul = r2;
+ } else {
+ mul = r3;
+ movr(_jit, _RAX_REGNO, r2);
+ }
+ if (sign)
+ umulr(_jit, mul);
+ else
+ umulr_u(_jit, mul);
+
+ if (r0 == _RDX_REGNO && r1 == _RAX_REGNO) {
+ xchgr(_jit, _RAX_REGNO, _RDX_REGNO);
+ } else {
+ if (r0 != _RDX_REGNO)
+ movr(_jit, r0, _RAX_REGNO);
+ movr(_jit, r1, _RDX_REGNO);
+ if (r0 == _RDX_REGNO)
+ movr(_jit, r0, _RAX_REGNO);
+ }
+
+ if (r0 != _RDX_REGNO && r1 != _RDX_REGNO)
+ popr(_jit, _RDX_REGNO);
+ if (r0 != _RAX_REGNO && r1 != _RAX_REGNO)
+ popr(_jit, _RAX_REGNO);
+}
+
+static void
+qmulr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, int32_t r3)
+{
+ return iqmulr(_jit, r0, r1, r2, r3, 1);
+}
+
+static void
+qmulr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, int32_t r3)
+{
+ return iqmulr(_jit, r0, r1, r2, r3, 0);
+}
+
+static void
+iqmuli(jit_state_t *_jit, int32_t r0, int32_t r1,
+ int32_t r2, jit_word_t i0, jit_bool_t sign)
+{
+ if (i0 == 0) {
+ ixorr(_jit, r0, r0);
+ ixorr(_jit, r1, r1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ if (sign)
+ qmulr(_jit, r0, r1, r2, rn(reg));
+ else
+ qmulr_u(_jit, r0, r1, r2, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+qmuli(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, jit_word_t i0)
+{
+ return iqmuli(_jit, r0, r1, r2, i0, 1);
+}
+
+static void
+qmuli_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, jit_word_t i0)
+{
+ return iqmuli(_jit, r0, r1, r2, i0, 0);
+}
+
+static void
+sign_extend_rdx_rax(jit_state_t *_jit)
+{
+ rex(_jit, 0, WIDE, 0, 0, 0);
+ ic(_jit, 0x99);
+}
+
+static void
+divremr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2,
+ jit_bool_t sign, jit_bool_t divide)
+{
+ if (r0 != _RAX_REGNO)
+ pushr(_jit, _RAX_REGNO);
+ if (r0 != _RDX_REGNO)
+ pushr(_jit, _RDX_REGNO);
+
+ int tmp_divisor = 0;
+ if (r2 == _RAX_REGNO || r2 == _RDX_REGNO) {
+ jit_gpr_t tmp = get_temp_gpr(_jit);
+ movr(_jit, rn(tmp), r2);
+ r2 = rn(tmp);
+ tmp_divisor = 1;
+ }
+
+ movr(_jit, _RAX_REGNO, r1);
+
+ if (sign) {
+ sign_extend_rdx_rax(_jit);
+ idivr(_jit, r2);
+ } else {
+ ixorr(_jit, _RDX_REGNO, _RDX_REGNO);
+ idivr_u(_jit, r2);
+ }
+
+ if (divide)
+ movr(_jit, r0, _RAX_REGNO);
+ else
+ movr(_jit, r0, _RDX_REGNO);
+
+ if (tmp_divisor)
+ unget_temp_gpr(_jit);
+
+ if (r0 != _RDX_REGNO)
+ popr(_jit, _RDX_REGNO);
+ if (r0 != _RAX_REGNO)
+ popr(_jit, _RAX_REGNO);
+}
+
+static void
+divremi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0,
+ jit_bool_t sign, jit_bool_t divide)
+{
+ jit_gpr_t tmp = get_temp_gpr(_jit);
+ movi(_jit, rn(tmp), i0);
+
+ divremr(_jit, r0, r1, rn(tmp), sign, divide);
+}
+
+static void
+divr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ return divremr(_jit, r0, r1, r2, 1, 1);
+}
+
+static void
+divi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ return divremi(_jit, r0, r1, i0, 1, 1);
+}
+
+static void
+divr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ return divremr(_jit, r0, r1, r2, 0, 1);
+}
+
+static void
+divi_u(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ return divremi(_jit, r0, r1, i0, 0, 1);
+}
+
+
+static void
+remr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ return divremr(_jit, r0, r1, r2, 1, 0);
+}
+
+static void
+remi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ return divremi(_jit, r0, r1, i0, 1, 0);
+}
+
+static void
+remr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ return divremr(_jit, r0, r1, r2, 0, 0);
+}
+
+static void
+remi_u(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ return divremi(_jit, r0, r1, i0, 0, 0);
+}
+
+static void
+iqdivr(jit_state_t *_jit, int32_t r0, int32_t r1,
+ int32_t r2, int32_t r3, jit_bool_t sign)
+{
+ if (r0 != _RAX_REGNO && r1 != _RAX_REGNO)
+ pushr(_jit, _RAX_REGNO);
+ if (r0 != _RDX_REGNO && r1 != _RDX_REGNO)
+ pushr(_jit, _RDX_REGNO);
+
+ int tmp_divisor = 0;
+ if (r3 == _RAX_REGNO || r3 == _RDX_REGNO) {
+ jit_gpr_t tmp = get_temp_gpr(_jit);
+ movr(_jit, rn(tmp), r3);
+ r3 = rn(tmp);
+ tmp_divisor = 1;
+ }
+
+ movr(_jit, _RAX_REGNO, r2);
+
+ if (sign) {
+ sign_extend_rdx_rax(_jit);
+ idivr(_jit, r3);
+ } else {
+ ixorr(_jit, _RDX_REGNO, _RDX_REGNO);
+ idivr_u(_jit, r3);
+ }
+
+ if (r0 == _RDX_REGNO && r1 == _RAX_REGNO) {
+ xchgr(_jit, _RAX_REGNO, _RDX_REGNO);
+ } else {
+ if (r0 != _RDX_REGNO)
+ movr(_jit, r0, _RAX_REGNO);
+ movr(_jit, r1, _RDX_REGNO);
+ if (r0 == _RDX_REGNO)
+ movr(_jit, r0, _RAX_REGNO);
+ }
+
+ if (tmp_divisor)
+ unget_temp_gpr(_jit);
+
+ if (r0 != _RDX_REGNO && r1 != _RDX_REGNO)
+ popr(_jit, _RDX_REGNO);
+ if (r0 != _RAX_REGNO && r1 != _RAX_REGNO)
+ popr(_jit, _RAX_REGNO);
+}
+
+static void
+qdivr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, int32_t r3)
+{
+ return iqdivr(_jit, r0, r1, r2, r3, 1);
+}
+
+static void
+qdivr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, int32_t r3)
+{
+ return iqdivr(_jit, r0, r1, r2, r3, 0);
+}
+
+static void
+iqdivi(jit_state_t *_jit, int32_t r0, int32_t r1,
+ int32_t r2, jit_word_t i0, jit_bool_t sign)
+{
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ if (sign)
+ qdivr(_jit, r0, r1, r2, rn(reg));
+ else
+ qdivr_u(_jit, r0, r1, r2, rn(reg));
+ unget_temp_gpr(_jit);
+}
+
+static void
+qdivi(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, jit_word_t i0)
+{
+ return iqdivi(_jit, r0, r1, r2, i0, 1);
+}
+
+static void
+qdivi_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, jit_word_t i0)
+{
+ return iqdivi(_jit, r0, r1, r2, i0, 0);
+}
+
+static void
+comr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ movr(_jit, r0, r1);
+ icomr(_jit, r0);
+}
+
+static void
+andr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r1 == r2)
+ movr(_jit, r0, r1);
+ else if (r0 == r1)
+ iandr(_jit, r0, r2);
+ else if (r0 == r2) {
+ iandr(_jit, r0, r1);
+ } else {
+ movr(_jit, r0, r1);
+ iandr(_jit, r0, r2);
+ }
+}
+
+static void
+andi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+
+ if (i0 == 0)
+ ixorr(_jit, r0, r0);
+ else if (i0 == -1)
+ movr(_jit, r0, r1);
+ else if (r0 == r1) {
+ if (can_sign_extend_int_p(i0)) {
+ iandi(_jit, r0, i0);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ iandr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+ } else {
+ movi(_jit, r0, i0);
+ iandr(_jit, r0, r1);
+ }
+}
+
+static void
+orr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r1 == r2)
+ movr(_jit, r0, r1);
+ else if (r0 == r1)
+ iorr(_jit, r0, r2);
+ else if (r0 == r2) {
+ iorr(_jit, r0, r1);
+ } else {
+ movr(_jit, r0, r1);
+ iorr(_jit, r0, r2);
+ }
+}
+
+static void
+ori(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (i0 == 0)
+ movr(_jit, r0, r1);
+ else if (i0 == -1)
+ movi(_jit, r0, -1);
+ else if (can_sign_extend_int_p(i0)) {
+ movr(_jit, r0, r1);
+ iori(_jit, r0, i0);
+ }
+ else if (r0 != r1) {
+ movi(_jit, r0, i0);
+ iorr(_jit, r0, r1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ iorr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+xorr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (r1 == r2)
+ ixorr(_jit, r0, r0);
+ else if (r0 == r1)
+ ixorr(_jit, r0, r2);
+ else if (r0 == r2) {
+ ixorr(_jit, r0, r1);
+ } else {
+ movr(_jit, r0, r1);
+ ixorr(_jit, r0, r2);
+ }
+}
+
+static void
+xori(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (i0 == 0)
+ movr(_jit, r0, r1);
+ else if (i0 == -1)
+ comr(_jit, r0, r1);
+ else if (can_sign_extend_int_p(i0)) {
+ movr(_jit, r0, r1);
+ ixori(_jit, r0, i0);
+ }
+ else if (r0 != r1) {
+ movi(_jit, r0, i0);
+ ixorr(_jit, r0, r1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ixorr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+cr(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1, int32_t r2)
+{
+ if (reg8_p(r0)) {
+ jit_bool_t same = r0 == r1 || r0 == r2;
+ if (!same)
+ ixorr(_jit, r0, r0);
+ icmpr(_jit, r1, r2);
+ if (same)
+ imovi(_jit, r0, 0);
+ cc(_jit, code, r0);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ ixorr(_jit, rn(reg), rn(reg));
+ icmpr(_jit, r1, r2);
+ cc(_jit, code, rn(reg));
+ movr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+ci(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (reg8_p(r0)) {
+ jit_bool_t same = r0 == r1;
+ if (!same)
+ ixorr(_jit, r0, r0);
+ icmpi(_jit, r1, i0);
+ if (same)
+ imovi(_jit, r0, 0);
+ cc(_jit, code, r0);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ ixorr(_jit, rn(reg), rn(reg));
+ icmpi(_jit, r1, i0);
+ cc(_jit, code, rn(reg));
+ movr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+ci0(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1)
+{
+ if (reg8_p(r0)) {
+ jit_bool_t same = r0 == r1;
+ if (!same)
+ ixorr(_jit, r0, r0);
+ testr(_jit, r1, r1);
+ if (same)
+ imovi(_jit, r0, 0);
+ cc(_jit, code, r0);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ ixorr(_jit, rn(reg), rn(reg));
+ testr(_jit, r1, r1);
+ cc(_jit, code, rn(reg));
+ movr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+extr_c(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ if (reg8_p(r1)) {
+ movcr(_jit, r0, r1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movr(_jit, rn(reg), r1);
+ movcr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+extr_uc(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ if (reg8_p(r1)) {
+ movcr_u(_jit, r0, r1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movr(_jit, rn(reg), r1);
+ movcr_u(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+extr_s(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return movsr(_jit, r0, r1);
+}
+
+static void
+extr_us(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return movsr_u(_jit, r0, r1);
+}
+
+#if __X64 && !__X64_32
+static void
+extr_i(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return movir(_jit, r0, r1);
+}
+static void
+extr_ui(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return movir_u(_jit, r0, r1);
+}
+#endif
+
+static void
+bswapr_us(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ extr_us(_jit, r0, r1);
+ ic(_jit, 0x66);
+ rex(_jit, 0, 0, _NOREG, _NOREG, r0);
+ ic(_jit, 0xc1);
+ mrm(_jit, 0x03, X86_ROR, r7(r0));
+ ic(_jit, 8);
+}
+
+static void
+bswapr_ui(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ movr(_jit, r0, r1);
+ rex(_jit, 0, 0, _NOREG, _NOREG, r0);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xc8 | r7(r0));
+}
+
+#if __X64 && !__X64_32
+static void
+bswapr_ul(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ movr(_jit, r0, r1);
+ rex(_jit, 0, 1, _NOREG, _NOREG, r0);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xc8 | r7(r0));
+}
+#endif
+
+static void
+ldr_c(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xbe);
+ rx(_jit, r0, 0, r1, _NOREG, _SCL1);
+}
+
+static void
+ldi_c(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, WIDE, r0, _NOREG, _NOREG);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xbe);
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldr_c(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+ldr_uc(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb6);
+ rx(_jit, r0, 0, r1, _NOREG, _SCL1);
+}
+
+static void
+ldi_uc(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, WIDE, r0, _NOREG, _NOREG);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb6);
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldr_uc(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+ldr_s(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xbf);
+ rx(_jit, r0, 0, r1, _NOREG, _SCL1);
+}
+
+static void
+ldi_s(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, WIDE, r0, _NOREG, _NOREG);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xbf);
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldr_s(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+ldr_us(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb7);
+ rx(_jit, r0, 0, r1, _NOREG, _SCL1);
+}
+
+static void
+ldi_us(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, WIDE, r0, _NOREG, _NOREG);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb7);
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldr_us(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+#if __X32 || !__X64_32
+static void
+ldr_i(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+#if __X64
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x63);
+#else
+ ic(_jit, 0x8b);
+#endif
+ rx(_jit, r0, 0, r1, _NOREG, _SCL1);
+}
+
+static void
+ldi_i(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+#if __X64
+ rex(_jit, 0, WIDE, r0, _NOREG, _NOREG);
+ ic(_jit, 0x63);
+#else
+ ic(_jit, 0x8b);
+#endif
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldr_i(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+#endif
+
+#if __X64
+static void
+ldr_ui(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, 0, r0, _NOREG, r1);
+ ic(_jit, 0x63);
+ rx(_jit, r0, 0, r1, _NOREG, _SCL1);
+}
+
+static void
+ldi_ui(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, 0, r0, _NOREG, _NOREG);
+ ic(_jit, 0x63);
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldr_ui(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+# if !__X64_32
+static void
+ldr_l(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, 1, r0, _NOREG, r1);
+ ic(_jit, 0x8b);
+ rx(_jit, r0, 0, r1, _NOREG, _SCL1);
+}
+
+static void
+ldi_l(jit_state_t *_jit, int32_t r0, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, 1, r0, _NOREG, _NOREG);
+ ic(_jit, 0x8b);
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldr_l(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+# endif
+#endif
+
+static void
+ldxr_c(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+#if __X64_32
+ addr(_jit, r0, r1, r2);
+ ldr_c(r0, r0);
+#else
+ rex(_jit, 0, WIDE, r0, r1, r2);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xbe);
+ rx(_jit, r0, 0, r2, r1, _SCL1);
+#endif
+}
+
+static void
+ldxi_c(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xbe);
+ rx(_jit, r0, i0, r1, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldxr_c(_jit, r0, r1, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+ldxr_uc(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+#if __X64_32
+ addr(_jit, r0, r1, r2);
+ ldr_uc(_jit, r0, r0);
+#else
+ rex(_jit, 0, WIDE, r0, r1, r2);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb6);
+ rx(_jit, r0, 0, r2, r1, _SCL1);
+#endif
+}
+
+static void
+ldxi_uc(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb6);
+ rx(_jit, r0, i0, r1, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldxr_uc(_jit, r0, r1, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+ldxr_s(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+#if __X64_32
+ addr(_jit, r0, r1, r2);
+ ldr_s(_jit, r0, r0);
+#else
+ rex(_jit, 0, WIDE, r0, r1, r2);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xbf);
+ rx(_jit, r0, 0, r2, r1, _SCL1);
+#endif
+}
+
+static void
+ldxi_s(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xbf);
+ rx(_jit, r0, i0, r1, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldxr_s(_jit, r0, r1, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+ldxr_us(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+#if __X64_32
+ addr(_jit, r0, r1, r2);
+ ldr_us(_jit, r0, r0);
+#else
+ rex(_jit, 0, WIDE, r0, r1, r2);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb7);
+ rx(_jit, r0, 0, r2, r1, _SCL1);
+#endif
+}
+
+static void
+ldxi_us(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x0f);
+ ic(_jit, 0xb7);
+ rx(_jit, r0, i0, r1, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldxr_us(_jit, r0, r1, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+#if __X64 || !__X64_32
+static void
+ldxr_i(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+#if __X64
+ rex(_jit, 0, WIDE, r0, r1, r2);
+ ic(_jit, 0x63);
+#else
+ ic(_jit, 0x8b);
+#endif
+ rx(_jit, r0, 0, r2, r1, _SCL1);
+}
+
+static void
+ldxi_i(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+#if __X64
+ rex(_jit, 0, WIDE, r0, _NOREG, r1);
+ ic(_jit, 0x63);
+#else
+ ic(_jit, 0x8b);
+#endif
+ rx(_jit, r0, i0, r1, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldxr_i(_jit, r0, r1, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+#endif
+
+#if __X64
+static void
+ldxr_ui(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+#if __X64_32
+ addr(_jit, r0, r1, r2);
+ /* to avoid confusion with macro renames */
+ _ldr_ui(_jit, r0, r0);
+#else
+ rex(_jit, 0, 0, r0, r1, r2);
+ ic(_jit, 0x8b);
+ rx(_jit, r0, 0, r2, r1, _SCL1);
+#endif
+}
+
+static void
+ldxi_ui(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, 0, r0, _NOREG, r1);
+ ic(_jit, 0x8b);
+ rx(_jit, r0, i0, r1, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldxr_ui(_jit, r0, r1, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+# if !__X64_32
+static void
+ldxr_l(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ rex(_jit, 0, 1, r0, r1, r2);
+ ic(_jit, 0x8b);
+ rx(_jit, r0, 0, r2, r1, _SCL1);
+}
+
+static void
+ldxi_l(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, 1, r0, _NOREG, r1);
+ ic(_jit, 0x8b);
+ rx(_jit, r0, i0, r1, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ ldxr_l(_jit, r0, r1, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+# endif
+#endif
+
+static void
+str_c(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ if (reg8_p(r1)) {
+ rex(_jit, 0, 0, r1, _NOREG, r0);
+ ic(_jit, 0x88);
+ rx(_jit, r1, 0, r0, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movr(_jit, rn(reg), r1);
+ rex(_jit, 0, 0, rn(reg), _NOREG, r0);
+ ic(_jit, 0x88);
+ rx(_jit, rn(reg), 0, r0, _NOREG, _SCL1);
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+sti_c(jit_state_t *_jit, jit_word_t i0, int32_t r0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ if (reg8_p(r0)) {
+ rex(_jit, 0, 0, r0, _NOREG, _NOREG);
+ ic(_jit, 0x88);
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movr(_jit, rn(reg), r0);
+ rex(_jit, 0, 0, rn(reg), _NOREG, _NOREG);
+ ic(_jit, 0x88);
+ rx(_jit, rn(reg), i0, _NOREG, _NOREG, _SCL1);
+ unget_temp_gpr(_jit);
+ }
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ str_c(_jit, rn(reg), r0);
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+str_s(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ ic(_jit, 0x66);
+ rex(_jit, 0, 0, r1, _NOREG, r0);
+ ic(_jit, 0x89);
+ rx(_jit, r1, 0, r0, _NOREG, _SCL1);
+}
+
+static void
+sti_s(jit_state_t *_jit, jit_word_t i0, int32_t r0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ ic(_jit, 0x66);
+ rex(_jit, 0, 0, r0, _NOREG, _NOREG);
+ ic(_jit, 0x89);
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ str_s(_jit, rn(reg), r0);
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+str_i(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, 0, r1, _NOREG, r0);
+ ic(_jit, 0x89);
+ rx(_jit, r1, 0, r0, _NOREG, _SCL1);
+}
+
+static void
+sti_i(jit_state_t *_jit, jit_word_t i0, int32_t r0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, 0, r0, _NOREG, _NOREG);
+ ic(_jit, 0x89);
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ str_i(_jit, rn(reg), r0);
+ unget_temp_gpr(_jit);
+ }
+}
+
+#if __X64 && !__X64_32
+static void
+str_l(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ rex(_jit, 0, 1, r1, _NOREG, r0);
+ ic(_jit, 0x89);
+ rx(_jit, r1, 0, r0, _NOREG, _SCL1);
+}
+
+static void
+sti_l(jit_state_t *_jit, jit_word_t i0, int32_t r0)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, 1, r0, _NOREG, _NOREG);
+ ic(_jit, 0x89);
+ rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ str_l(_jit, rn(reg), r0);
+ unget_temp_gpr(_jit);
+ }
+}
+#endif
+
+static void
+stxr_c(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+#if __X64_32
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ addr(_jit, rn(reg), r0, r1);
+ str_c(_jit, rn(reg), r2);
+ unget_temp_gpr(_jit);
+#else
+ if (reg8_p(r2)) {
+ rex(_jit, 0, 0, r2, r1, r0);
+ ic(_jit, 0x88);
+ rx(_jit, r2, 0, r0, r1, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movr(_jit, rn(reg), r2);
+ rex(_jit, 0, 0, rn(reg), r1, r0);
+ ic(_jit, 0x88);
+ rx(_jit, rn(reg), 0, r0, r1, _SCL1);
+ unget_temp_gpr(_jit);
+ }
+#endif
+}
+
+static void
+stxi_c(jit_state_t *_jit, jit_word_t i0, int32_t r0, int32_t r1)
+{
+ if (can_sign_extend_int_p(i0)) {
+ if (reg8_p(r1)) {
+ rex(_jit, 0, 0, r1, _NOREG, r0);
+ ic(_jit, 0x88);
+ rx(_jit, r1, i0, r0, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movr(_jit, rn(reg), r1);
+ rex(_jit, 0, 0, rn(reg), _NOREG, r0);
+ ic(_jit, 0x88);
+ rx(_jit, rn(reg), i0, r0, _NOREG, _SCL1);
+ unget_temp_gpr(_jit);
+ }
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ stxr_c(_jit, rn(reg), r0, r1);
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+stxr_s(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+#if __X64_32
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ addr(_jit, rn(reg), r0, r1);
+ str_s(_jit, rn(reg), r2);
+ unget_temp_gpr(_jit);
+#else
+ ic(_jit, 0x66);
+ rex(_jit, 0, 0, r2, r1, r0);
+ ic(_jit, 0x89);
+ rx(_jit, r2, 0, r0, r1, _SCL1);
+#endif
+}
+
+static void
+stxi_s(jit_state_t *_jit, jit_word_t i0, int32_t r0, int32_t r1)
+{
+ if (can_sign_extend_int_p(i0)) {
+ ic(_jit, 0x66);
+ rex(_jit, 0, 0, r1, _NOREG, r0);
+ ic(_jit, 0x89);
+ rx(_jit, r1, i0, r0, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ stxr_s(_jit, rn(reg), r0, r1);
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+stxr_i(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+#if __X64_32
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ addr(_jit, rn(reg), r0, r1);
+ str_i(rn(reg), r2);
+ unget_temp_gpr(_jit);
+#else
+ rex(_jit, 0, 0, r2, r1, r0);
+ ic(_jit, 0x89);
+ rx(_jit, r2, 0, r0, r1, _SCL1);
+#endif
+}
+
+static void
+stxi_i(jit_state_t *_jit, jit_word_t i0, int32_t r0, int32_t r1)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, 0, r1, _NOREG, r0);
+ ic(_jit, 0x89);
+ rx(_jit, r1, i0, r0, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ stxr_i(_jit, rn(reg), r0, r1);
+ unget_temp_gpr(_jit);
+ }
+}
+
+#if __X64 && !__X64_32
+static void
+stxr_l(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
+{
+ rex(_jit, 0, 1, r2, r1, r0);
+ ic(_jit, 0x89);
+ rx(_jit, r2, 0, r0, r1, _SCL1);
+}
+
+static void
+stxi_l(jit_state_t *_jit, jit_word_t i0, int32_t r0, int32_t r1)
+{
+ if (can_sign_extend_int_p(i0)) {
+ rex(_jit, 0, 1, r1, _NOREG, r0);
+ ic(_jit, 0x89);
+ rx(_jit, r1, i0, r0, _NOREG, _SCL1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i0);
+ stxr_l(_jit, rn(reg), r0, r1);
+ unget_temp_gpr(_jit);
+ }
+}
+#endif
+
+static jit_reloc_t
+jccs(jit_state_t *_jit, int32_t code)
+{
+ ic(_jit, 0x70 | code);
+ return jit_reloc(_jit, JIT_RELOC_REL8, 1, 0);
+}
+
+static jit_reloc_t
+jcc(jit_state_t *_jit, int32_t code)
+{
+ ic(_jit, 0x0f);
+ ic(_jit, 0x80 | code);
+ return jit_reloc(_jit, JIT_RELOC_REL32, 2, 0);
+}
+
+#define DEFINE_JUMPS(cc, CC, code) \
+ static inline jit_reloc_t j##cc(jit_state_t *_jit) \
+ { \
+ return jcc(_jit, X86_CC_##CC); \
+ } \
+ static inline jit_reloc_t j##cc##s(jit_state_t *_jit) \
+ { \
+ return jccs(_jit, X86_CC_##CC); \
+ }
+FOR_EACH_CC(DEFINE_JUMPS)
+#undef DEFINE_JUMPS
+
+static jit_reloc_t
+jcr(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1)
+{
+ alur(_jit, X86_CMP, r0, r1);
+ return jcc(_jit, code);
+}
+
+static jit_reloc_t
+jci(jit_state_t *_jit, int32_t code, int32_t r0, jit_word_t i0)
+{
+ alui(_jit, X86_CMP, r0, i0);
+ return jcc(_jit, code);
+}
+
+static jit_reloc_t
+jci0(jit_state_t *_jit, int32_t code, int32_t r0)
+{
+ testr(_jit, r0, r0);
+ return jcc(_jit, code);
+}
+
+static jit_reloc_t
+bltr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return jcr(_jit, X86_CC_L, r0, r1);
+}
+
+static jit_reloc_t
+blti(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (i1) return jci (_jit, X86_CC_L, r0, i1);
+ else return jci0(_jit, X86_CC_S, r0);
+}
+
+static jit_reloc_t
+bltr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return jcr(_jit, X86_CC_B, r0, r1);
+}
+
+static jit_reloc_t
+blti_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (i1) return jci (_jit, X86_CC_B, r0, i1);
+ else return jci0(_jit, X86_CC_B, r0);
+}
+
+static jit_reloc_t
+bler(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return jcr (_jit, X86_CC_LE, r0, r1);
+}
+
+static jit_reloc_t
+blei(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (i1) return jci (_jit, X86_CC_LE, r0, i1);
+ else return jci0(_jit, X86_CC_LE, r0);
+}
+
+static jit_reloc_t
+bler_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return jcr (_jit, X86_CC_BE, r0, r1);
+}
+
+static jit_reloc_t
+blei_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (i1) return jci (_jit, X86_CC_BE, r0, i1);
+ else return jci0(_jit, X86_CC_BE, r0);
+}
+
+static jit_reloc_t
+beqr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return jcr (_jit, X86_CC_E, r0, r1);
+}
+
+static jit_reloc_t
+beqi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (i1) return jci (_jit, X86_CC_E, r0, i1);
+ else return jci0(_jit, X86_CC_E, r0);
+}
+
+static jit_reloc_t
+bger(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return jcr (_jit, X86_CC_GE, r0, r1);
+}
+
+static jit_reloc_t
+bgei(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (i1) return jci (_jit, X86_CC_GE, r0, i1);
+ else return jci0(_jit, X86_CC_NS, r0);
+}
+
+static jit_reloc_t
+bger_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return jcr (_jit, X86_CC_AE, r0, r1);
+}
+
+static jit_reloc_t
+bgei_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ return jci (_jit, X86_CC_AE, r0, i1);
+}
+
+static jit_reloc_t
+bgtr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return jcr(_jit, X86_CC_G, r0, r1);
+}
+
+static jit_reloc_t
+bgti(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ return jci(_jit, X86_CC_G, r0, i1);
+}
+
+static jit_reloc_t
+bgtr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return jcr(_jit, X86_CC_A, r0, r1);
+}
+
+static jit_reloc_t
+bgti_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (i1) return jci (_jit, X86_CC_A, r0, i1);
+ else return jci0(_jit, X86_CC_NE, r0);
+}
+
+static jit_reloc_t
+bner(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ return jcr(_jit, X86_CC_NE, r0, r1);
+}
+
+static jit_reloc_t
+bnei(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (i1) return jci (_jit, X86_CC_NE, r0, i1);
+ else return jci0(_jit, X86_CC_NE, r0);
+}
+
+static jit_reloc_t
+bmsr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ testr(_jit, r0, r1);
+ return jnz(_jit);
+}
+
+static jit_reloc_t
+bmsi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (can_zero_extend_int_p(i1)) {
+ testi(_jit, r0, i1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i1);
+ testr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+ return jnz(_jit);
+}
+
+static jit_reloc_t
+bmcr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ testr(_jit, r0, r1);
+ return jz(_jit);
+}
+
+static jit_reloc_t
+bmci(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (can_zero_extend_int_p(i1)) {
+ testi(_jit, r0, i1);
+ } else {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i1);
+ testr(_jit, r0, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+ return jz(_jit);
+}
+
+static jit_reloc_t
+boaddr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ iaddr(_jit, r0, r1);
+ return jo(_jit);
+}
+
+static jit_reloc_t
+boaddi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (can_sign_extend_int_p(i1)) {
+ iaddi(_jit, r0, i1);
+ return jo(_jit);
+ }
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i1);
+ unget_temp_gpr(_jit);
+ return boaddr(_jit, r0, rn(reg));
+}
+
+static jit_reloc_t
+boaddr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ iaddr(_jit, r0, r1);
+ return jc(_jit);
+}
+
+static jit_reloc_t
+boaddi_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (can_sign_extend_int_p(i1)) {
+ iaddi(_jit, r0, i1);
+ return jc(_jit);
+ }
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i1);
+ unget_temp_gpr(_jit);
+ return boaddr_u(_jit, r0, rn(reg));
+}
+
+static jit_reloc_t
+bxaddr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ iaddr(_jit, r0, r1);
+ return jno(_jit);
+}
+
+static jit_reloc_t
+bxaddi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (can_sign_extend_int_p(i1)) {
+ iaddi(_jit, r0, i1);
+ return jno(_jit);
+ }
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i1);
+ unget_temp_gpr(_jit);
+ return bxaddr(_jit, r0, rn(reg));
+}
+
+static jit_reloc_t
+bxaddr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ iaddr(_jit, r0, r1);
+ return jnc(_jit);
+}
+
+static jit_reloc_t
+bxaddi_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (can_sign_extend_int_p(i1)) {
+ iaddi(_jit, r0, i1);
+ return jnc(_jit);
+ }
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i1);
+ unget_temp_gpr(_jit);
+ return bxaddr_u(_jit, r0, rn(reg));
+}
+
+static jit_reloc_t
+bosubr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ isubr(_jit, r0, r1);
+ return jo(_jit);
+}
+
+static jit_reloc_t
+bosubi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (can_sign_extend_int_p(i1)) {
+ isubi(_jit, r0, i1);
+ return jo(_jit);
+ }
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i1);
+ unget_temp_gpr(_jit);
+ return bosubr(_jit, r0, rn(reg));
+}
+
+static jit_reloc_t
+bosubr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ isubr(_jit, r0, r1);
+ return jc(_jit);
+}
+
+static jit_reloc_t
+bosubi_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (can_sign_extend_int_p(i1)) {
+ isubi(_jit, r0, i1);
+ return jc(_jit);
+ }
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i1);
+ unget_temp_gpr(_jit);
+ return bosubr_u(_jit, r0, rn(reg));
+}
+
+static jit_reloc_t
+bxsubr(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ isubr(_jit, r0, r1);
+ return jno(_jit);
+}
+
+static jit_reloc_t
+bxsubi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (can_sign_extend_int_p(i1)) {
+ isubi(_jit, r0, i1);
+ return jno(_jit);
+ }
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i1);
+ unget_temp_gpr(_jit);
+ return bxsubr(_jit, r0, rn(reg));
+}
+
+static jit_reloc_t
+bxsubr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
+{
+ isubr(_jit, r0, r1);
+ return jnc(_jit);
+}
+
+static jit_reloc_t
+bxsubi_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
+{
+ if (can_sign_extend_int_p(i1)) {
+ isubi(_jit, r0, i1);
+ return jnc(_jit);
+ }
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ movi(_jit, rn(reg), i1);
+ unget_temp_gpr(_jit);
+ return bxsubr_u(_jit, r0, rn(reg));
+}
+
+static void
+callr(jit_state_t *_jit, int32_t r0)
+{
+ rex(_jit, 0, 0, _NOREG, _NOREG, r0);
+ ic(_jit, 0xff);
+ mrm(_jit, 0x03, 0x02, r7(r0));
+}
+
+static void
+calli(jit_state_t *_jit, jit_word_t i0)
+{
+ ptrdiff_t rel32 = i0 - (_jit->pc.w + 4);
+ if (INT32_MIN <= rel32 && rel32 <= INT32_MAX)
+ {
+ ic(_jit, 0xe8);
+ ii(_jit, rel32);
+ }
+ else
+ {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ jit_patch_there(_jit, mov_addr(_jit, rn(reg)), (void*)i0);
+ callr(_jit, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static void
+jmpr(jit_state_t *_jit, int32_t r0)
+{
+ rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
+ ic(_jit, 0xff);
+ mrm(_jit, 0x03, 0x04, r7(r0));
+}
+
+static void
+jmpi(jit_state_t *_jit, jit_word_t i0)
+{
+ ptrdiff_t rel32 = i0 - (_jit->pc.w + 4);
+ if (INT32_MIN <= rel32 && rel32 <= INT32_MAX)
+ {
+ ic(_jit, 0xe9);
+ ii(_jit, rel32);
+ }
+ else
+ {
+ jit_gpr_t reg = get_temp_gpr(_jit);
+ jit_patch_there(_jit, mov_addr(_jit, rn(reg)), (void*)i0);
+ jmpr(_jit, rn(reg));
+ unget_temp_gpr(_jit);
+ }
+}
+
+static jit_reloc_t
+jmp(jit_state_t *_jit)
+{
+ ic(_jit, 0xe9);
+ return jit_reloc(_jit, JIT_RELOC_REL32, 1, 0);
+}
+
+static jit_reloc_t
+jmpsi(jit_state_t *_jit)
+{
+ ic(_jit, 0xeb);
+ return jit_reloc(_jit, JIT_RELOC_REL8, 1, 0);
+}
+
+static void
+ret(jit_state_t *_jit)
+{
+ ic(_jit, 0xc3);
+}
+
+static void
+retr(jit_state_t *_jit, int32_t r0)
+{
+ movr(_jit, _RAX_REGNO, r0);
+ ret(_jit);
+}
+
+static void
+reti(jit_state_t *_jit, jit_word_t i0)
+{
+ movi(_jit, _RAX_REGNO, i0);
+ ret(_jit);
+}
+
+static void
+retval_c(jit_state_t *_jit, int32_t r0)
+{
+ extr_c(_jit, r0, rn(JIT_RET));
+}
+
+static void
+retval_uc(jit_state_t *_jit, int32_t r0)
+{
+ extr_uc(_jit, r0, rn(JIT_RET));
+}
+
+static void
+retval_s(jit_state_t *_jit, int32_t r0)
+{
+ extr_s(_jit, r0, rn(JIT_RET));
+}
+
+static void
+retval_us(jit_state_t *_jit, int32_t r0)
+{
+ extr_us(_jit, r0, rn(JIT_RET));
+}
+
+static void
+retval_i(jit_state_t *_jit, int32_t r0)
+{
+#if __X32 || __X64_32
+ movr(_jit, r0, rn(JIT_RET));
+#else
+ extr_i(_jit, r0, rn(JIT_RET));
+#endif
+}
+
+#if __X64 && !__X64_32
+static void
+retval_ui(jit_state_t *_jit, int32_t r0)
+{
+ extr_ui(_jit, r0, rn(JIT_RET));
+}
+
+static void
+retval_l(jit_state_t *_jit, int32_t r0)
+{
+ movr(_jit, r0, rn(JIT_RET));
+}
+#endif