summaryrefslogtreecommitdiff
path: root/libguile/lightning/lightening/x86-cpu.c
diff options
context:
space:
mode:
authorAndy Wingo <wingo@pobox.com>2019-04-03 14:07:42 +0200
committerAndy Wingo <wingo@pobox.com>2019-04-03 14:08:06 +0200
commite9be4ea73d3073c5634b6ae28a65732f33f26568 (patch)
tree9c7fae1956a4abe4bddd7af6817e2529f6a3381d /libguile/lightning/lightening/x86-cpu.c
parent2e4fb1e665562bfd84a0d35c965ce0c29a0abc9d (diff)
downloadguile-e9be4ea73d3073c5634b6ae28a65732f33f26568.tar.gz
Adapt to lightning -> lightening name change
* libguile/jit.c: Include lightening.h. * libguile/Makefile.am: Adapt.
Diffstat (limited to 'libguile/lightning/lightening/x86-cpu.c')
-rw-r--r--libguile/lightning/lightening/x86-cpu.c2762
1 files changed, 0 insertions, 2762 deletions
diff --git a/libguile/lightning/lightening/x86-cpu.c b/libguile/lightning/lightening/x86-cpu.c
deleted file mode 100644
index 1c1da2b4f..000000000
--- a/libguile/lightning/lightening/x86-cpu.c
+++ /dev/null
@@ -1,2762 +0,0 @@
-/*
- * Copyright (C) 2012-2019 Free Software Foundation, Inc.
- *
- * This file is part of GNU lightning.
- *
- * GNU lightning is free software; you can redistribute it and/or modify it
- * under the terms of the GNU Lesser General Public License as published
- * by the Free Software Foundation; either version 3, or (at your option)
- * any later version.
- *
- * GNU lightning is distributed in the hope that it will be useful, but
- * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
- * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
- * License for more details.
- *
- * Authors:
- * Paulo Cesar Pereira de Andrade
- */
-
-/* avoid using it due to partial stalls */
-#define USE_INC_DEC 0
-
-#if __X32 || __X64_32
-# define WIDE 0
-# define IF_WIDE(wide, narrow) narrow
-#else
-# define WIDE 1
-# define IF_WIDE(wide, narrow) wide
-#endif
-
-#define _RAX_REGNO 0
-#define _RCX_REGNO 1
-#define _RDX_REGNO 2
-#define _RBX_REGNO 3
-#define _RSP_REGNO 4
-#define _RBP_REGNO 5
-#define _RSI_REGNO 6
-#define _RDI_REGNO 7
-#define _R8_REGNO 8
-#define _R9_REGNO 9
-#define _R10_REGNO 10
-#define _R11_REGNO 11
-#define _R12_REGNO 12
-#define _R13_REGNO 13
-#define _R14_REGNO 14
-#define _R15_REGNO 15
-#define r7(reg) ((reg) & 7)
-#define r8(reg) ((reg) & 15)
-#if __X32 || __CYGWIN__ || __X64_32
-# define reg8_p(rn) ((rn) >= _RAX_REGNO && (rn) <= _RBX_REGNO)
-#else
-# define reg8_p(rn) 1
-#endif
-
-#define can_sign_extend_int_p(im) \
- IF_WIDE((((im) >= 0 && (long long)(im) <= 0x7fffffffLL) || \
- ((im) < 0 && (long long)(im) > -0x80000000LL)), \
- 1)
-#define can_zero_extend_int_p(im) \
- IF_WIDE(((im) >= 0 && (im) < 0x80000000LL), \
- 1)
-#define fits_uint32_p(im) \
- IF_WIDE((((im) & 0xffffffff00000000LL) == 0), \
- 1)
-
-#define _SCL1 0x00
-#define _SCL2 0x01
-#define _SCL4 0x02
-#define _SCL8 0x03
-
-#define X86_ADD 0
-#define X86_OR 1 << 3
-#define X86_ADC 2 << 3
-#define X86_SBB 3 << 3
-#define X86_AND 4 << 3
-#define X86_SUB 5 << 3
-#define X86_XOR 6 << 3
-#define X86_CMP 7 << 3
-#define X86_ROL 0
-#define X86_ROR 1
-#define X86_RCL 2
-#define X86_RCR 3
-#define X86_SHL 4
-#define X86_SHR 5
-#define X86_SAR 7
-#define X86_NOT 2
-#define X86_NEG 3
-#define X86_MUL 4
-#define X86_IMUL 5
-#define X86_DIV 6
-#define X86_IDIV 7
-
-#define FOR_EACH_CC(M) \
- M(o, O, 0x0) \
- M(no, NO, 0x1) \
- M(nae, NAE, 0x2) \
- M(b, B, 0x2) \
- M(c, C, 0x2) \
- M(ae, AE, 0x3) \
- M(nb, NB, 0x3) \
- M(nc, NC, 0x3) \
- M(e, E, 0x4) \
- M(z, Z, 0x4) \
- M(ne, NE, 0x5) \
- M(nz, NZ, 0x5) \
- M(be, BE, 0x6) \
- M(na, NA, 0x6) \
- M(a, A, 0x7) \
- M(nbe, NBE, 0x7) \
- M(s, S, 0x8) \
- M(ns, NS, 0x9) \
- M(p, P, 0xa) \
- M(pe, PE, 0xa) \
- M(np, NP, 0xb) \
- M(po, PO, 0xb) \
- M(l, L, 0xc) \
- M(nge, NGE, 0xc) \
- M(ge, GE, 0xd) \
- M(nl_, NL, 0xd) \
- M(le, LE, 0xe) \
- M(ng, NG, 0xe) \
- M(g, G, 0xf) \
- M(nle, NLE, 0xf) \
- /* EOL */
-
-enum x86_cc
-{
-#define DEFINE_ENUM(cc, CC, code) X86_CC_##CC = code,
- FOR_EACH_CC(DEFINE_ENUM)
-#undef DEFINE_ENUM
-};
-
-static inline void
-mrm(jit_state_t *_jit, uint8_t md, uint8_t r, uint8_t m)
-{
- emit_u8(_jit, (md<<6) | (r<<3) | m);
-}
-
-static inline void
-sib(jit_state_t *_jit, uint8_t sc, uint8_t i, uint8_t b)
-{
- emit_u8(_jit, (sc<<6) | (i<<3) | b);
-}
-
-static inline void
-ic(jit_state_t *_jit, uint8_t c)
-{
- emit_u8(_jit, c);
-}
-
-static inline void
-is(jit_state_t *_jit, uint16_t s)
-{
- emit_u16(_jit, s);
-}
-
-static inline void
-ii(jit_state_t *_jit, uint32_t i)
-{
- emit_u32(_jit, i);
-}
-
-static inline void
-il(jit_state_t *_jit, unsigned long l)
-{
-#if __X64 && !__X64_32
- emit_u64(_jit, l);
-#else
- ii(_jit, l);
-#endif
-}
-
-static void
-rex(jit_state_t *_jit, int32_t l, int32_t w,
- int32_t r, int32_t x, int32_t b)
-{
-#if __X64
- int32_t v = 0x40 | (w << 3);
-
- if (r != _NOREG)
- v |= (r & 8) >> 1;
- if (x != _NOREG)
- v |= (x & 8) >> 2;
- if (b != _NOREG)
- v |= (b & 8) >> 3;
- if (l || v != 0x40)
- ic(_jit, v);
-#endif
-}
-
-static void
-rx(jit_state_t *_jit, int32_t rd, int32_t md,
- int32_t rb, int32_t ri, int32_t ms)
-{
- if (ri == _NOREG) {
- if (rb == _NOREG) {
-#if __X32
- mrm(_jit, 0x00, r7(rd), 0x05);
-#else
- mrm(_jit, 0x00, r7(rd), 0x04);
- sib(_jit, _SCL1, 0x04, 0x05);
-#endif
- ii(_jit, md);
- } else if (r7(rb) == _RSP_REGNO) {
- if (md == 0) {
- mrm(_jit, 0x00, r7(rd), 0x04);
- sib(_jit, ms, 0x04, 0x04);
- }
- else if ((int8_t)md == md) {
- mrm(_jit, 0x01, r7(rd), 0x04);
- sib(_jit, ms, 0x04, 0x04);
- ic(_jit, md);
- } else {
- mrm(_jit, 0x02, r7(rd), 0x04);
- sib(_jit, ms, 0x04, 0x04);
- ii(_jit, md);
- }
- } else {
- if (md == 0 && r7(rb) != _RBP_REGNO)
- mrm(_jit, 0x00, r7(rd), r7(rb));
- else if ((int8_t)md == md) {
- mrm(_jit, 0x01, r7(rd), r7(rb));
- ic(_jit, md);
- } else {
- mrm(_jit, 0x02, r7(rd), r7(rb));
- ii(_jit, md);
- }
- }
- }
- else if (rb == _NOREG) {
- mrm(_jit, 0x00, r7(rd), 0x04);
- sib(_jit, ms, r7(ri), 0x05);
- ii(_jit, md);
- }
- else if (r8(ri) != _RSP_REGNO) {
- if (md == 0 && r7(rb) != _RBP_REGNO) {
- mrm(_jit, 0x00, r7(rd), 0x04);
- sib(_jit, ms, r7(ri), r7(rb));
- } else if ((int8_t)md == md) {
- mrm(_jit, 0x01, r7(rd), 0x04);
- sib(_jit, ms, r7(ri), r7(rb));
- ic(_jit, md);
- } else {
- mrm(_jit, 0x02, r7(rd), 0x04);
- sib(_jit, ms, r7(ri), r7(rb));
- ic(_jit, md);
- }
- } else {
- fprintf(stderr, "illegal index register");
- abort();
- }
-}
-
-static void
-pushr(jit_state_t *_jit, int32_t r0)
-{
- rex(_jit, 0, WIDE, 0, 0, r0);
- ic(_jit, 0x50 | r7(r0));
-}
-
-static void
-popr(jit_state_t *_jit, int32_t r0)
-{
- rex(_jit, 0, WIDE, 0, 0, r0);
- ic(_jit, 0x58 | r7(r0));
-}
-
-static jit_gpr_t
-get_temp_gpr(jit_state_t *_jit)
-{
- ASSERT(!_jit->temp_gpr_saved);
- _jit->temp_gpr_saved = 1;
-#if __X32
- pushr(_jit, _RBP_REGNO);
- return JIT_GPR(_RBP);
-#else
- return JIT_GPR(_R8);
-#endif
-}
-
-static void
-unget_temp_gpr(jit_state_t *_jit)
-{
- ASSERT(_jit->temp_gpr_saved);
- _jit->temp_gpr_saved = 0;
-#if __X32
- popr(_jit, _RBP_REGNO);
-#endif
-}
-
-static void
-nop(jit_state_t *_jit, int32_t count)
-{
- switch (count) {
- case 0:
- break;
- case 1: /* NOP */
- ic(_jit, 0x90);
- break;
- case 2: /* 66 NOP */
- ic(_jit, 0x66); ic(_jit, 0x90);
- break;
- case 3: /* NOP DWORD ptr [EAX] */
- ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x00);
- break;
- case 4: /* NOP DWORD ptr [EAX + 00H] */
- ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x40); ic(_jit, 0x00);
- break;
- case 5: /* NOP DWORD ptr [EAX + EAX*1 + 00H] */
- ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x44); ic(_jit, 0x00);
- ic(_jit, 0x00);
- break;
- case 6: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00H] */
- ic(_jit, 0x66); ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x44);
- ic(_jit, 0x00); ic(_jit, 0x00);
- break;
- case 7: /* NOP DWORD ptr [EAX + 00000000H] */
- ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x80); ii(_jit, 0x0000);
- break;
- case 8: /* NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
- ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x84); ic(_jit, 0x00);
- ii(_jit, 0x0000);
- break;
- case 9: /* 66 NOP DWORD ptr [EAX + EAX*1 + 00000000H] */
- ic(_jit, 0x66); ic(_jit, 0x0f); ic(_jit, 0x1f); ic(_jit, 0x84);
- ic(_jit, 0x00); ii(_jit, 0x0000);
- break;
- default:
- abort();
- }
-}
-
-static void
-movr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- if (r0 != r1) {
- rex(_jit, 0, 1, r1, _NOREG, r0);
- ic(_jit, 0x89);
- ic(_jit, 0xc0 | (r1 << 3) | r7(r0));
- }
-}
-
-static void
-movcr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xbe);
- mrm(_jit, 0x03, r7(r0), r7(r1));
-}
-
-static void
-movcr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xb6);
- mrm(_jit, 0x03, r7(r0), r7(r1));
-}
-
-static void
-movsr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xbf);
- mrm(_jit, 0x03, r7(r0), r7(r1));
-}
-
-static void
-movsr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xb7);
- mrm(_jit, 0x03, r7(r0), r7(r1));
-}
-
-#if __X64
-static void
-movir(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, 1, r0, _NOREG, r1);
- ic(_jit, 0x63);
- mrm(_jit, 0x03, r7(r0), r7(r1));
-}
-
-static void
-movir_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, 0, r1, _NOREG, r0);
- ic(_jit, 0x89);
- ic(_jit, 0xc0 | (r1 << 3) | r7(r0));
-}
-#endif
-
-static jit_reloc_t
-mov_addr(jit_state_t *_jit, int32_t r0)
-{
- uint8_t *pc_start = _jit->pc.uc;
- rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
- ic(_jit, 0xb8 | r7(r0));
- ptrdiff_t inst_start = _jit->pc.uc - pc_start;
- return jit_reloc(_jit, JIT_RELOC_ABSOLUTE, inst_start, 0);
-}
-
-static void
-imovi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
-#if __X64
-# if !__X64_32
- if (fits_uint32_p(i0)) {
-# endif
- rex(_jit, 0, 0, _NOREG, _NOREG, r0);
- ic(_jit, 0xb8 | r7(r0));
- ii(_jit, i0);
-# if !__X64_32
- } else {
- rex(_jit, 0, 1, _NOREG, _NOREG, r0);
- ic(_jit, 0xb8 | r7(r0));
- il(_jit, i0);
- }
-# endif
-#else
- ic(_jit, 0xb8 | r7(r0));
- ii(_jit, i0);
-#endif
-}
-
-static void
-alur(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r1, _NOREG, r0);
- ic(_jit, code | 0x01);
- mrm(_jit, 0x03, r7(r1), r7(r0));
-}
-
-static inline void
-icmpr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return alur(_jit, X86_CMP, r0, r1);
-}
-static inline void
-iaddr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return alur(_jit, X86_ADD, r0, r1);
-}
-static inline void
-iaddxr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return alur(_jit, X86_ADC, r0, r1);
-}
-static inline void
-isubr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return alur(_jit, X86_SUB, r0, r1);
-}
-static inline void
-isubxr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return alur(_jit, X86_SBB, r0, r1);
-}
-static inline void
-iandr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return alur(_jit, X86_AND, r0, r1);
-}
-static inline void
-iorr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return alur(_jit, X86_OR, r0, r1);
-}
-static inline void
-ixorr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return alur(_jit, X86_XOR, r0, r1);
-}
-
-static void
-movi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- if (i0)
- imovi(_jit, r0, i0);
- else
- ixorr(_jit, r0, r0);
-}
-
-static void
-alui(jit_state_t *_jit, int32_t code, int32_t r0, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
- if ((int8_t)i0 == i0) {
- ic(_jit, 0x83);
- ic(_jit, 0xc0 | code | r7(r0));
- ic(_jit, i0);
- } else {
- if (r0 == _RAX_REGNO) {
- ic(_jit, code | 0x05);
- } else {
- ic(_jit, 0x81);
- ic(_jit, 0xc0 | code | r7(r0));
- }
- ii(_jit, i0);
- }
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- alur(_jit, code, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static inline void
-icmpi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- return alui(_jit, X86_CMP, r0, i0);
-}
-static inline void
-iaddi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- return alui(_jit, X86_ADD, r0, i0);
-}
-static inline void
-iaddxi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- return alui(_jit, X86_ADC, r0, i0);
-}
-static inline void
-isubi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- return alui(_jit, X86_SUB, r0, i0);
-}
-static inline void
-isubxi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- return alui(_jit, X86_SBB, r0, i0);
-}
-static inline void
-iandi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- return alui(_jit, X86_AND, r0, i0);
-}
-static inline void
-iori(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- return alui(_jit, X86_OR, r0, i0);
-}
-static inline void
-ixori(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- return alui(_jit, X86_XOR, r0, i0);
-}
-
-static void
-unr(jit_state_t *_jit, int32_t code, int32_t r0)
-{
- rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
- ic(_jit, 0xf7);
- mrm(_jit, 0x03, code, r7(r0));
-}
-
-static inline void
-umulr(jit_state_t *_jit, int32_t r0)
-{
- return unr(_jit, X86_IMUL, r0);
-}
-static inline void
-umulr_u(jit_state_t *_jit, int32_t r0)
-{
- return unr(_jit, X86_MUL, r0);
-}
-static inline void
-idivr(jit_state_t *_jit, int32_t r0)
-{
- return unr(_jit, X86_IDIV, r0);
-}
-static inline void
-idivr_u(jit_state_t *_jit, int32_t r0)
-{
- return unr(_jit, X86_DIV, r0);
-}
-static inline void
-inegr(jit_state_t *_jit, int32_t r0)
-{
- return unr(_jit, X86_NEG, r0);
-}
-static inline void
-icomr(jit_state_t *_jit, int32_t r0)
-{
- return unr(_jit, X86_NOT, r0);
-}
-
-#if USE_INC_DEC
-static void
-incr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- movr(_jit, r0, r1);
-# if __X64
- rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
- ic(_jit, 0xff);
- ic(_jit, 0xc0 | r7(r0));
-# else
- ic(_jit, 0x40 | r7(r0));
-# endif
-}
-
-static void
-decr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- movr(_jit, r0, r1);
-# if __X64
- rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
- ic(_jit, 0xff);
- ic(_jit, 0xc8 | r7(r0));
-# else
- ic(_jit, 0x48 | r7(r0));
-# endif
-}
-#endif
-
-static void
-lea(jit_state_t *_jit, int32_t md, int32_t rb,
- int32_t ri, int32_t ms, int32_t rd)
-{
- rex(_jit, 0, WIDE, rd, ri, rb);
- ic(_jit, 0x8d);
- rx(_jit, rd, md, rb, ri, ms);
-}
-
-static void
-xchgr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r1, _NOREG, r0);
- ic(_jit, 0x87);
- mrm(_jit, 0x03, r7(r1), r7(r0));
-}
-
-static void
-testr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r1, _NOREG, r0);
- ic(_jit, 0x85);
- mrm(_jit, 0x03, r7(r1), r7(r0));
-}
-
-static void
-testi(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
- if (r0 == _RAX_REGNO) {
- ic(_jit, 0xa9);
- } else {
- ic(_jit, 0xf7);
- mrm(_jit, 0x03, 0x00, r7(r0));
- }
- ii(_jit, i0);
-}
-
-static void
-cc(jit_state_t *_jit, int32_t code, int32_t r0)
-{
- rex(_jit, 0, 0, _NOREG, _NOREG, r0);
- ic(_jit, 0x0f);
- ic(_jit, 0x90 | code);
- mrm(_jit, 0x03, 0x00, r7(r0));
-}
-
-static void
-negr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- if (r0 == r1) {
- inegr(_jit, r0);
- } else {
- ixorr(_jit, r0, r0);
- isubr(_jit, r0, r1);
- }
-}
-
-static void
-addr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- if (r0 == r1)
- iaddr(_jit, r0, r2);
- else if (r0 == r2)
- iaddr(_jit, r0, r1);
- else
- lea(_jit, 0, r1, r2, _SCL1, r0);
-}
-
-static void
-addi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (i0 == 0)
- movr(_jit, r0, r1);
-#if USE_INC_DEC
- else if (i0 == 1)
- incr(_jit, r0, r1);
- else if (i0 == -1)
- decr(_jit, r0, r1);
-#endif
- else if (can_sign_extend_int_p(i0)) {
- if (r0 == r1)
- iaddi(_jit, r0, i0);
- else
- lea(_jit, i0, r1, _NOREG, _SCL1, r0);
- }
- else if (r0 != r1) {
- movi(_jit, r0, i0);
- iaddr(_jit, r0, r1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- iaddr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-addcr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- if (r0 == r2) {
- iaddr(_jit, r0, r1);
- } else {
- movr(_jit, r0, r1);
- iaddr(_jit, r0, r2);
- }
-}
-
-static void
-addci(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- movr(_jit, r0, r1);
- iaddi(_jit, r0, i0);
- }
- else if (r0 == r1) {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- iaddr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- } else {
- movi(_jit, r0, i0);
- iaddr(_jit, r0, r1);
- }
-}
-
-static void
-addxr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- if (r0 == r2) {
- iaddxr(_jit, r0, r1);
- } else {
- movr(_jit, r0, r1);
- iaddxr(_jit, r0, r2);
- }
-}
-
-static void
-addxi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- movr(_jit, r0, r1);
- iaddxi(_jit, r0, i0);
- }
- else if (r0 == r1) {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- iaddxr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- } else {
- movi(_jit, r0, i0);
- iaddxr(_jit, r0, r1);
- }
-}
-
-static void
-subr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- if (r1 == r2)
- ixorr(_jit, r0, r0);
- else if (r0 == r2) {
- isubr(_jit, r0, r1);
- inegr(_jit, r0);
- } else {
- movr(_jit, r0, r1);
- isubr(_jit, r0, r2);
- }
-}
-
-static void
-subi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (i0 == 0)
- movr(_jit, r0, r1);
-#if USE_INC_DEC
- else if (i0 == 1)
- decr(_jit, r0, r1);
- else if (i0 == -1)
- incr(_jit, r0, r1);
-#endif
- else if (can_sign_extend_int_p(i0)) {
- if (r0 == r1)
- isubi(_jit, r0, i0);
- else
- lea(_jit, -i0, r1, _NOREG, _SCL1, r0);
- }
- else if (r0 != r1) {
- movi(_jit, r0, -i0);
- iaddr(_jit, r0, r1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- isubr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-subcr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- if (r0 == r2 && r0 != r1) {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movr(_jit, rn(reg), r0);
- movr(_jit, r0, r1);
- isubr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- } else {
- movr(_jit, r0, r1);
- isubr(_jit, r0, r2);
- }
-}
-
-static void
-subci(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- movr(_jit, r0, r1);
- if (can_sign_extend_int_p(i0)) {
- isubi(_jit, r0, i0);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- isubr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-subxr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- if (r0 == r2 && r0 != r1) {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movr(_jit, rn(reg), r0);
- movr(_jit, r0, r1);
- isubxr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- } else {
- movr(_jit, r0, r1);
- isubxr(_jit, r0, r2);
- }
-}
-
-static void
-subxi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- movr(_jit, r0, r1);
- if (can_sign_extend_int_p(i0)) {
- isubxi(_jit, r0, i0);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- imovi(_jit, rn(reg), i0);
- isubxr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-irotshr(jit_state_t *_jit, int32_t code, int32_t r0)
-{
- rex(_jit, 0, WIDE, _RCX_REGNO, _NOREG, r0);
- ic(_jit, 0xd3);
- mrm(_jit, 0x03, code, r7(r0));
-}
-
-static void
-rotshr(jit_state_t *_jit, int32_t code,
- int32_t r0, int32_t r1, int32_t r2)
-{
- if (r0 == _RCX_REGNO) {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movr(_jit, rn(reg), r1);
- if (r2 != _RCX_REGNO)
- movr(_jit, _RCX_REGNO, r2);
- irotshr(_jit, code, rn(reg));
- movr(_jit, _RCX_REGNO, rn(reg));
- unget_temp_gpr(_jit);
- } else if (r2 != _RCX_REGNO) {
- /* Already know that R0 isn't RCX. */
- pushr(_jit, _RCX_REGNO);
- if (r1 == _RCX_REGNO) {
- if (r0 == r2)
- xchgr(_jit, r0, _RCX_REGNO);
- else {
- movr(_jit, r0, r1);
- movr(_jit, _RCX_REGNO, r2);
- }
- } else {
- movr(_jit, _RCX_REGNO, r2);
- movr(_jit, r0, r1);
- }
- irotshr(_jit, code, r0);
- popr(_jit, _RCX_REGNO);
- } else {
- movr(_jit, r0, r1);
- irotshr(_jit, code, r0);
- }
-}
-
-static void
-irotshi(jit_state_t *_jit, int32_t code, int32_t r0, jit_word_t i0)
-{
- rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
- if (i0 == 1) {
- ic(_jit, 0xd1);
- mrm(_jit, 0x03, code, r7(r0));
- } else {
- ic(_jit, 0xc1);
- mrm(_jit, 0x03, code, r7(r0));
- ic(_jit, i0);
- }
-}
-
-static void
-rotshi(jit_state_t *_jit, int32_t code,
- int32_t r0, int32_t r1, jit_word_t i0)
-{
- movr(_jit, r0, r1);
- if (i0)
- irotshi(_jit, code, r0, i0);
-}
-
-static void
-lshi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (i0 == 0)
- movr(_jit, r0, r1);
- else if (i0 <= 3)
- lea(_jit, 0, _NOREG, r1, i0 == 1 ? _SCL2 : i0 == 2 ? _SCL4 : _SCL8, r0);
- else
- rotshi(_jit, X86_SHL, r0, r1, i0);
-}
-
-static void
-lshr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- return rotshr(_jit, X86_SHL, r0, r1, r2);
-}
-
-static void
-rshr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- return rotshr(_jit, X86_SAR, r0, r1, r2);
-}
-
-static void
-rshi(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t i0)
-{
- return rotshi(_jit, X86_SAR, r0, r1, i0);
-}
-
-static void
-rshr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- return rotshr(_jit, X86_SHR, r0, r1, r2);
-}
-
-static void
-rshi_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t i0)
-{
- return rotshi(_jit, X86_SHR, r0, r1, i0);
-}
-
-static void
-imulr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xaf);
- mrm(_jit, 0x03, r7(r0), r7(r1));
-}
-
-static void
-imuli(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- if ((int8_t)i0 == i0) {
- ic(_jit, 0x6b);
- mrm(_jit, 0x03, r7(r0), r7(r1));
- ic(_jit, i0);
- } else {
- ic(_jit, 0x69);
- mrm(_jit, 0x03, r7(r0), r7(r1));
- ii(_jit, i0);
- }
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- imulr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-mulr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- if (r0 == r1)
- imulr(_jit, r0, r2);
- else if (r0 == r2) {
- imulr(_jit, r0, r1);
- } else {
- movr(_jit, r0, r1);
- imulr(_jit, r0, r2);
- }
-}
-
-static int
-ffsw(jit_word_t i)
-{
- if (sizeof(int) == sizeof(i))
- return ffs(i);
- int bit = ffs((int)i);
- if (bit == 0) {
- bit = ffs((int)((unsigned long)i >> 32));
- if (bit)
- bit += 32;
- }
- return bit;
-}
-
-static void
-muli(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- switch (i0) {
- case 0:
- ixorr(_jit, r0, r0);
- break;
- case 1:
- movr(_jit, r0, r1);
- break;
- case -1:
- negr(_jit, r0, r1);
- break;
- case 2:
- lea(_jit, 0, _NOREG, r1, _SCL2, r0);
- break;
- case 4:
- lea(_jit, 0, _NOREG, r1, _SCL4, r0);
- break;
- case 8:
- lea(_jit, 0, _NOREG, r1, _SCL8, r0);
- break;
- default:
- if (i0 > 0 && !(i0 & (i0 - 1)))
- lshi(_jit, r0, r1, ffsw(i0) - 1);
- else if (can_sign_extend_int_p(i0))
- imuli(_jit, r0, r1, i0);
- else if (r0 != r1) {
- movi(_jit, r0, i0);
- imulr(_jit, r0, r1);
- }
- else
- imuli(_jit, r0, r0, i0);
- break;
- }
-}
-
-static void
-iqmulr(jit_state_t *_jit, int32_t r0, int32_t r1,
- int32_t r2, int32_t r3, jit_bool_t sign)
-{
- if (r0 != _RAX_REGNO && r1 != _RAX_REGNO)
- pushr(_jit, _RAX_REGNO);
- if (r0 != _RDX_REGNO && r1 != _RDX_REGNO)
- pushr(_jit, _RDX_REGNO);
-
- int32_t mul;
- if (r3 == _RAX_REGNO) {
- mul = r2;
- } else {
- mul = r3;
- movr(_jit, _RAX_REGNO, r2);
- }
- if (sign)
- umulr(_jit, mul);
- else
- umulr_u(_jit, mul);
-
- if (r0 == _RDX_REGNO && r1 == _RAX_REGNO) {
- xchgr(_jit, _RAX_REGNO, _RDX_REGNO);
- } else {
- if (r0 != _RDX_REGNO)
- movr(_jit, r0, _RAX_REGNO);
- movr(_jit, r1, _RDX_REGNO);
- if (r0 == _RDX_REGNO)
- movr(_jit, r0, _RAX_REGNO);
- }
-
- if (r0 != _RDX_REGNO && r1 != _RDX_REGNO)
- popr(_jit, _RDX_REGNO);
- if (r0 != _RAX_REGNO && r1 != _RAX_REGNO)
- popr(_jit, _RAX_REGNO);
-}
-
-static void
-qmulr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, int32_t r3)
-{
- return iqmulr(_jit, r0, r1, r2, r3, 1);
-}
-
-static void
-qmulr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, int32_t r3)
-{
- return iqmulr(_jit, r0, r1, r2, r3, 0);
-}
-
-static void
-iqmuli(jit_state_t *_jit, int32_t r0, int32_t r1,
- int32_t r2, jit_word_t i0, jit_bool_t sign)
-{
- if (i0 == 0) {
- ixorr(_jit, r0, r0);
- ixorr(_jit, r1, r1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- if (sign)
- qmulr(_jit, r0, r1, r2, rn(reg));
- else
- qmulr_u(_jit, r0, r1, r2, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-qmuli(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, jit_word_t i0)
-{
- return iqmuli(_jit, r0, r1, r2, i0, 1);
-}
-
-static void
-qmuli_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, jit_word_t i0)
-{
- return iqmuli(_jit, r0, r1, r2, i0, 0);
-}
-
-static void
-sign_extend_rdx_rax(jit_state_t *_jit)
-{
- rex(_jit, 0, WIDE, 0, 0, 0);
- ic(_jit, 0x99);
-}
-
-static void
-divremr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2,
- jit_bool_t sign, jit_bool_t divide)
-{
- if (r0 != _RAX_REGNO)
- pushr(_jit, _RAX_REGNO);
- if (r0 != _RDX_REGNO)
- pushr(_jit, _RDX_REGNO);
-
- int tmp_divisor = 0;
- if (r2 == _RAX_REGNO || r2 == _RDX_REGNO) {
- jit_gpr_t tmp = get_temp_gpr(_jit);
- movr(_jit, rn(tmp), r2);
- r2 = rn(tmp);
- tmp_divisor = 1;
- }
-
- movr(_jit, _RAX_REGNO, r1);
-
- if (sign) {
- sign_extend_rdx_rax(_jit);
- idivr(_jit, r2);
- } else {
- ixorr(_jit, _RDX_REGNO, _RDX_REGNO);
- idivr_u(_jit, r2);
- }
-
- if (divide)
- movr(_jit, r0, _RAX_REGNO);
- else
- movr(_jit, r0, _RDX_REGNO);
-
- if (tmp_divisor)
- unget_temp_gpr(_jit);
-
- if (r0 != _RDX_REGNO)
- popr(_jit, _RDX_REGNO);
- if (r0 != _RAX_REGNO)
- popr(_jit, _RAX_REGNO);
-}
-
-static void
-divremi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0,
- jit_bool_t sign, jit_bool_t divide)
-{
- jit_gpr_t tmp = get_temp_gpr(_jit);
- movi(_jit, rn(tmp), i0);
-
- divremr(_jit, r0, r1, rn(tmp), sign, divide);
-}
-
-static void
-divr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- return divremr(_jit, r0, r1, r2, 1, 1);
-}
-
-static void
-divi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- return divremi(_jit, r0, r1, i0, 1, 1);
-}
-
-static void
-divr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- return divremr(_jit, r0, r1, r2, 0, 1);
-}
-
-static void
-divi_u(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- return divremi(_jit, r0, r1, i0, 0, 1);
-}
-
-
-static void
-remr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- return divremr(_jit, r0, r1, r2, 1, 0);
-}
-
-static void
-remi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- return divremi(_jit, r0, r1, i0, 1, 0);
-}
-
-static void
-remr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- return divremr(_jit, r0, r1, r2, 0, 0);
-}
-
-static void
-remi_u(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- return divremi(_jit, r0, r1, i0, 0, 0);
-}
-
-static void
-iqdivr(jit_state_t *_jit, int32_t r0, int32_t r1,
- int32_t r2, int32_t r3, jit_bool_t sign)
-{
- if (r0 != _RAX_REGNO && r1 != _RAX_REGNO)
- pushr(_jit, _RAX_REGNO);
- if (r0 != _RDX_REGNO && r1 != _RDX_REGNO)
- pushr(_jit, _RDX_REGNO);
-
- int tmp_divisor = 0;
- if (r3 == _RAX_REGNO || r3 == _RDX_REGNO) {
- jit_gpr_t tmp = get_temp_gpr(_jit);
- movr(_jit, rn(tmp), r3);
- r3 = rn(tmp);
- tmp_divisor = 1;
- }
-
- movr(_jit, _RAX_REGNO, r2);
-
- if (sign) {
- sign_extend_rdx_rax(_jit);
- idivr(_jit, r3);
- } else {
- ixorr(_jit, _RDX_REGNO, _RDX_REGNO);
- idivr_u(_jit, r3);
- }
-
- if (r0 == _RDX_REGNO && r1 == _RAX_REGNO) {
- xchgr(_jit, _RAX_REGNO, _RDX_REGNO);
- } else {
- if (r0 != _RDX_REGNO)
- movr(_jit, r0, _RAX_REGNO);
- movr(_jit, r1, _RDX_REGNO);
- if (r0 == _RDX_REGNO)
- movr(_jit, r0, _RAX_REGNO);
- }
-
- if (tmp_divisor)
- unget_temp_gpr(_jit);
-
- if (r0 != _RDX_REGNO && r1 != _RDX_REGNO)
- popr(_jit, _RDX_REGNO);
- if (r0 != _RAX_REGNO && r1 != _RAX_REGNO)
- popr(_jit, _RAX_REGNO);
-}
-
-static void
-qdivr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, int32_t r3)
-{
- return iqdivr(_jit, r0, r1, r2, r3, 1);
-}
-
-static void
-qdivr_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, int32_t r3)
-{
- return iqdivr(_jit, r0, r1, r2, r3, 0);
-}
-
-static void
-iqdivi(jit_state_t *_jit, int32_t r0, int32_t r1,
- int32_t r2, jit_word_t i0, jit_bool_t sign)
-{
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- if (sign)
- qdivr(_jit, r0, r1, r2, rn(reg));
- else
- qdivr_u(_jit, r0, r1, r2, rn(reg));
- unget_temp_gpr(_jit);
-}
-
-static void
-qdivi(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, jit_word_t i0)
-{
- return iqdivi(_jit, r0, r1, r2, i0, 1);
-}
-
-static void
-qdivi_u(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2, jit_word_t i0)
-{
- return iqdivi(_jit, r0, r1, r2, i0, 0);
-}
-
-static void
-comr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- movr(_jit, r0, r1);
- icomr(_jit, r0);
-}
-
-static void
-andr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- if (r1 == r2)
- movr(_jit, r0, r1);
- else if (r0 == r1)
- iandr(_jit, r0, r2);
- else if (r0 == r2) {
- iandr(_jit, r0, r1);
- } else {
- movr(_jit, r0, r1);
- iandr(_jit, r0, r2);
- }
-}
-
-static void
-andi(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
-
- if (i0 == 0)
- ixorr(_jit, r0, r0);
- else if (i0 == -1)
- movr(_jit, r0, r1);
- else if (r0 == r1) {
- if (can_sign_extend_int_p(i0)) {
- iandi(_jit, r0, i0);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- iandr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
- } else {
- movi(_jit, r0, i0);
- iandr(_jit, r0, r1);
- }
-}
-
-static void
-orr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- if (r1 == r2)
- movr(_jit, r0, r1);
- else if (r0 == r1)
- iorr(_jit, r0, r2);
- else if (r0 == r2) {
- iorr(_jit, r0, r1);
- } else {
- movr(_jit, r0, r1);
- iorr(_jit, r0, r2);
- }
-}
-
-static void
-ori(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (i0 == 0)
- movr(_jit, r0, r1);
- else if (i0 == -1)
- movi(_jit, r0, -1);
- else if (can_sign_extend_int_p(i0)) {
- movr(_jit, r0, r1);
- iori(_jit, r0, i0);
- }
- else if (r0 != r1) {
- movi(_jit, r0, i0);
- iorr(_jit, r0, r1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- iorr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-xorr(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- if (r1 == r2)
- ixorr(_jit, r0, r0);
- else if (r0 == r1)
- ixorr(_jit, r0, r2);
- else if (r0 == r2) {
- ixorr(_jit, r0, r1);
- } else {
- movr(_jit, r0, r1);
- ixorr(_jit, r0, r2);
- }
-}
-
-static void
-xori(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (i0 == 0)
- movr(_jit, r0, r1);
- else if (i0 == -1)
- comr(_jit, r0, r1);
- else if (can_sign_extend_int_p(i0)) {
- movr(_jit, r0, r1);
- ixori(_jit, r0, i0);
- }
- else if (r0 != r1) {
- movi(_jit, r0, i0);
- ixorr(_jit, r0, r1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ixorr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-cr(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1, int32_t r2)
-{
- if (reg8_p(r0)) {
- jit_bool_t same = r0 == r1 || r0 == r2;
- if (!same)
- ixorr(_jit, r0, r0);
- icmpr(_jit, r1, r2);
- if (same)
- imovi(_jit, r0, 0);
- cc(_jit, code, r0);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- ixorr(_jit, rn(reg), rn(reg));
- icmpr(_jit, r1, r2);
- cc(_jit, code, rn(reg));
- movr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-ci(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (reg8_p(r0)) {
- jit_bool_t same = r0 == r1;
- if (!same)
- ixorr(_jit, r0, r0);
- icmpi(_jit, r1, i0);
- if (same)
- imovi(_jit, r0, 0);
- cc(_jit, code, r0);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- ixorr(_jit, rn(reg), rn(reg));
- icmpi(_jit, r1, i0);
- cc(_jit, code, rn(reg));
- movr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-ci0(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1)
-{
- if (reg8_p(r0)) {
- jit_bool_t same = r0 == r1;
- if (!same)
- ixorr(_jit, r0, r0);
- testr(_jit, r1, r1);
- if (same)
- imovi(_jit, r0, 0);
- cc(_jit, code, r0);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- ixorr(_jit, rn(reg), rn(reg));
- testr(_jit, r1, r1);
- cc(_jit, code, rn(reg));
- movr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-extr_c(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- if (reg8_p(r1)) {
- movcr(_jit, r0, r1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movr(_jit, rn(reg), r1);
- movcr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-extr_uc(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- if (reg8_p(r1)) {
- movcr_u(_jit, r0, r1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movr(_jit, rn(reg), r1);
- movcr_u(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-extr_s(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return movsr(_jit, r0, r1);
-}
-
-static void
-extr_us(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return movsr_u(_jit, r0, r1);
-}
-
-#if __X64 && !__X64_32
-static void
-extr_i(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return movir(_jit, r0, r1);
-}
-static void
-extr_ui(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return movir_u(_jit, r0, r1);
-}
-#endif
-
-static void
-bswapr_us(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- extr_us(_jit, r0, r1);
- ic(_jit, 0x66);
- rex(_jit, 0, 0, _NOREG, _NOREG, r0);
- ic(_jit, 0xc1);
- mrm(_jit, 0x03, X86_ROR, r7(r0));
- ic(_jit, 8);
-}
-
-static void
-bswapr_ui(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- movr(_jit, r0, r1);
- rex(_jit, 0, 0, _NOREG, _NOREG, r0);
- ic(_jit, 0x0f);
- ic(_jit, 0xc8 | r7(r0));
-}
-
-#if __X64 && !__X64_32
-static void
-bswapr_ul(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- movr(_jit, r0, r1);
- rex(_jit, 0, 1, _NOREG, _NOREG, r0);
- ic(_jit, 0x0f);
- ic(_jit, 0xc8 | r7(r0));
-}
-#endif
-
-static void
-ldr_c(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xbe);
- rx(_jit, r0, 0, r1, _NOREG, _SCL1);
-}
-
-static void
-ldi_c(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, WIDE, r0, _NOREG, _NOREG);
- ic(_jit, 0x0f);
- ic(_jit, 0xbe);
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldr_c(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-ldr_uc(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xb6);
- rx(_jit, r0, 0, r1, _NOREG, _SCL1);
-}
-
-static void
-ldi_uc(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, WIDE, r0, _NOREG, _NOREG);
- ic(_jit, 0x0f);
- ic(_jit, 0xb6);
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldr_uc(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-ldr_s(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xbf);
- rx(_jit, r0, 0, r1, _NOREG, _SCL1);
-}
-
-static void
-ldi_s(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, WIDE, r0, _NOREG, _NOREG);
- ic(_jit, 0x0f);
- ic(_jit, 0xbf);
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldr_s(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-ldr_us(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xb7);
- rx(_jit, r0, 0, r1, _NOREG, _SCL1);
-}
-
-static void
-ldi_us(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, WIDE, r0, _NOREG, _NOREG);
- ic(_jit, 0x0f);
- ic(_jit, 0xb7);
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldr_us(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-#if __X32 || !__X64_32
-static void
-ldr_i(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
-#if __X64
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x63);
-#else
- ic(_jit, 0x8b);
-#endif
- rx(_jit, r0, 0, r1, _NOREG, _SCL1);
-}
-
-static void
-ldi_i(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
-#if __X64
- rex(_jit, 0, WIDE, r0, _NOREG, _NOREG);
- ic(_jit, 0x63);
-#else
- ic(_jit, 0x8b);
-#endif
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldr_i(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-#endif
-
-#if __X64
-static void
-ldr_ui(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, 0, r0, _NOREG, r1);
- ic(_jit, 0x63);
- rx(_jit, r0, 0, r1, _NOREG, _SCL1);
-}
-
-static void
-ldi_ui(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, 0, r0, _NOREG, _NOREG);
- ic(_jit, 0x63);
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldr_ui(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-# if !__X64_32
-static void
-ldr_l(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, 1, r0, _NOREG, r1);
- ic(_jit, 0x8b);
- rx(_jit, r0, 0, r1, _NOREG, _SCL1);
-}
-
-static void
-ldi_l(jit_state_t *_jit, int32_t r0, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, 1, r0, _NOREG, _NOREG);
- ic(_jit, 0x8b);
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldr_l(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-# endif
-#endif
-
-static void
-ldxr_c(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
-#if __X64_32
- addr(_jit, r0, r1, r2);
- ldr_c(r0, r0);
-#else
- rex(_jit, 0, WIDE, r0, r1, r2);
- ic(_jit, 0x0f);
- ic(_jit, 0xbe);
- rx(_jit, r0, 0, r2, r1, _SCL1);
-#endif
-}
-
-static void
-ldxi_c(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xbe);
- rx(_jit, r0, i0, r1, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldxr_c(_jit, r0, r1, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-ldxr_uc(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
-#if __X64_32
- addr(_jit, r0, r1, r2);
- ldr_uc(_jit, r0, r0);
-#else
- rex(_jit, 0, WIDE, r0, r1, r2);
- ic(_jit, 0x0f);
- ic(_jit, 0xb6);
- rx(_jit, r0, 0, r2, r1, _SCL1);
-#endif
-}
-
-static void
-ldxi_uc(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xb6);
- rx(_jit, r0, i0, r1, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldxr_uc(_jit, r0, r1, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-ldxr_s(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
-#if __X64_32
- addr(_jit, r0, r1, r2);
- ldr_s(_jit, r0, r0);
-#else
- rex(_jit, 0, WIDE, r0, r1, r2);
- ic(_jit, 0x0f);
- ic(_jit, 0xbf);
- rx(_jit, r0, 0, r2, r1, _SCL1);
-#endif
-}
-
-static void
-ldxi_s(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xbf);
- rx(_jit, r0, i0, r1, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldxr_s(_jit, r0, r1, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-ldxr_us(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
-#if __X64_32
- addr(_jit, r0, r1, r2);
- ldr_us(_jit, r0, r0);
-#else
- rex(_jit, 0, WIDE, r0, r1, r2);
- ic(_jit, 0x0f);
- ic(_jit, 0xb7);
- rx(_jit, r0, 0, r2, r1, _SCL1);
-#endif
-}
-
-static void
-ldxi_us(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x0f);
- ic(_jit, 0xb7);
- rx(_jit, r0, i0, r1, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldxr_us(_jit, r0, r1, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-#if __X64 || !__X64_32
-static void
-ldxr_i(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
-#if __X64
- rex(_jit, 0, WIDE, r0, r1, r2);
- ic(_jit, 0x63);
-#else
- ic(_jit, 0x8b);
-#endif
- rx(_jit, r0, 0, r2, r1, _SCL1);
-}
-
-static void
-ldxi_i(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
-#if __X64
- rex(_jit, 0, WIDE, r0, _NOREG, r1);
- ic(_jit, 0x63);
-#else
- ic(_jit, 0x8b);
-#endif
- rx(_jit, r0, i0, r1, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldxr_i(_jit, r0, r1, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-#endif
-
-#if __X64
-static void
-ldxr_ui(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
-#if __X64_32
- addr(_jit, r0, r1, r2);
- /* to avoid confusion with macro renames */
- _ldr_ui(_jit, r0, r0);
-#else
- rex(_jit, 0, 0, r0, r1, r2);
- ic(_jit, 0x8b);
- rx(_jit, r0, 0, r2, r1, _SCL1);
-#endif
-}
-
-static void
-ldxi_ui(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, 0, r0, _NOREG, r1);
- ic(_jit, 0x8b);
- rx(_jit, r0, i0, r1, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldxr_ui(_jit, r0, r1, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-# if !__X64_32
-static void
-ldxr_l(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- rex(_jit, 0, 1, r0, r1, r2);
- ic(_jit, 0x8b);
- rx(_jit, r0, 0, r2, r1, _SCL1);
-}
-
-static void
-ldxi_l(jit_state_t *_jit, int32_t r0, int32_t r1, jit_word_t i0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, 1, r0, _NOREG, r1);
- ic(_jit, 0x8b);
- rx(_jit, r0, i0, r1, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- ldxr_l(_jit, r0, r1, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-# endif
-#endif
-
-static void
-str_c(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- if (reg8_p(r1)) {
- rex(_jit, 0, 0, r1, _NOREG, r0);
- ic(_jit, 0x88);
- rx(_jit, r1, 0, r0, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movr(_jit, rn(reg), r1);
- rex(_jit, 0, 0, rn(reg), _NOREG, r0);
- ic(_jit, 0x88);
- rx(_jit, rn(reg), 0, r0, _NOREG, _SCL1);
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-sti_c(jit_state_t *_jit, jit_word_t i0, int32_t r0)
-{
- if (can_sign_extend_int_p(i0)) {
- if (reg8_p(r0)) {
- rex(_jit, 0, 0, r0, _NOREG, _NOREG);
- ic(_jit, 0x88);
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movr(_jit, rn(reg), r0);
- rex(_jit, 0, 0, rn(reg), _NOREG, _NOREG);
- ic(_jit, 0x88);
- rx(_jit, rn(reg), i0, _NOREG, _NOREG, _SCL1);
- unget_temp_gpr(_jit);
- }
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- str_c(_jit, rn(reg), r0);
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-str_s(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- ic(_jit, 0x66);
- rex(_jit, 0, 0, r1, _NOREG, r0);
- ic(_jit, 0x89);
- rx(_jit, r1, 0, r0, _NOREG, _SCL1);
-}
-
-static void
-sti_s(jit_state_t *_jit, jit_word_t i0, int32_t r0)
-{
- if (can_sign_extend_int_p(i0)) {
- ic(_jit, 0x66);
- rex(_jit, 0, 0, r0, _NOREG, _NOREG);
- ic(_jit, 0x89);
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- str_s(_jit, rn(reg), r0);
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-str_i(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, 0, r1, _NOREG, r0);
- ic(_jit, 0x89);
- rx(_jit, r1, 0, r0, _NOREG, _SCL1);
-}
-
-static void
-sti_i(jit_state_t *_jit, jit_word_t i0, int32_t r0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, 0, r0, _NOREG, _NOREG);
- ic(_jit, 0x89);
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- str_i(_jit, rn(reg), r0);
- unget_temp_gpr(_jit);
- }
-}
-
-#if __X64 && !__X64_32
-static void
-str_l(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- rex(_jit, 0, 1, r1, _NOREG, r0);
- ic(_jit, 0x89);
- rx(_jit, r1, 0, r0, _NOREG, _SCL1);
-}
-
-static void
-sti_l(jit_state_t *_jit, jit_word_t i0, int32_t r0)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, 1, r0, _NOREG, _NOREG);
- ic(_jit, 0x89);
- rx(_jit, r0, i0, _NOREG, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- str_l(_jit, rn(reg), r0);
- unget_temp_gpr(_jit);
- }
-}
-#endif
-
-static void
-stxr_c(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
-#if __X64_32
- jit_gpr_t reg = get_temp_gpr(_jit);
- addr(_jit, rn(reg), r0, r1);
- str_c(_jit, rn(reg), r2);
- unget_temp_gpr(_jit);
-#else
- if (reg8_p(r2)) {
- rex(_jit, 0, 0, r2, r1, r0);
- ic(_jit, 0x88);
- rx(_jit, r2, 0, r0, r1, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movr(_jit, rn(reg), r2);
- rex(_jit, 0, 0, rn(reg), r1, r0);
- ic(_jit, 0x88);
- rx(_jit, rn(reg), 0, r0, r1, _SCL1);
- unget_temp_gpr(_jit);
- }
-#endif
-}
-
-static void
-stxi_c(jit_state_t *_jit, jit_word_t i0, int32_t r0, int32_t r1)
-{
- if (can_sign_extend_int_p(i0)) {
- if (reg8_p(r1)) {
- rex(_jit, 0, 0, r1, _NOREG, r0);
- ic(_jit, 0x88);
- rx(_jit, r1, i0, r0, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movr(_jit, rn(reg), r1);
- rex(_jit, 0, 0, rn(reg), _NOREG, r0);
- ic(_jit, 0x88);
- rx(_jit, rn(reg), i0, r0, _NOREG, _SCL1);
- unget_temp_gpr(_jit);
- }
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- stxr_c(_jit, rn(reg), r0, r1);
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-stxr_s(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
-#if __X64_32
- jit_gpr_t reg = get_temp_gpr(_jit);
- addr(_jit, rn(reg), r0, r1);
- str_s(_jit, rn(reg), r2);
- unget_temp_gpr(_jit);
-#else
- ic(_jit, 0x66);
- rex(_jit, 0, 0, r2, r1, r0);
- ic(_jit, 0x89);
- rx(_jit, r2, 0, r0, r1, _SCL1);
-#endif
-}
-
-static void
-stxi_s(jit_state_t *_jit, jit_word_t i0, int32_t r0, int32_t r1)
-{
- if (can_sign_extend_int_p(i0)) {
- ic(_jit, 0x66);
- rex(_jit, 0, 0, r1, _NOREG, r0);
- ic(_jit, 0x89);
- rx(_jit, r1, i0, r0, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- stxr_s(_jit, rn(reg), r0, r1);
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-stxr_i(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
-#if __X64_32
- jit_gpr_t reg = get_temp_gpr(_jit);
- addr(_jit, rn(reg), r0, r1);
- str_i(rn(reg), r2);
- unget_temp_gpr(_jit);
-#else
- rex(_jit, 0, 0, r2, r1, r0);
- ic(_jit, 0x89);
- rx(_jit, r2, 0, r0, r1, _SCL1);
-#endif
-}
-
-static void
-stxi_i(jit_state_t *_jit, jit_word_t i0, int32_t r0, int32_t r1)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, 0, r1, _NOREG, r0);
- ic(_jit, 0x89);
- rx(_jit, r1, i0, r0, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- stxr_i(_jit, rn(reg), r0, r1);
- unget_temp_gpr(_jit);
- }
-}
-
-#if __X64 && !__X64_32
-static void
-stxr_l(jit_state_t *_jit, int32_t r0, int32_t r1, int32_t r2)
-{
- rex(_jit, 0, 1, r2, r1, r0);
- ic(_jit, 0x89);
- rx(_jit, r2, 0, r0, r1, _SCL1);
-}
-
-static void
-stxi_l(jit_state_t *_jit, jit_word_t i0, int32_t r0, int32_t r1)
-{
- if (can_sign_extend_int_p(i0)) {
- rex(_jit, 0, 1, r1, _NOREG, r0);
- ic(_jit, 0x89);
- rx(_jit, r1, i0, r0, _NOREG, _SCL1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i0);
- stxr_l(_jit, rn(reg), r0, r1);
- unget_temp_gpr(_jit);
- }
-}
-#endif
-
-static jit_reloc_t
-jccs(jit_state_t *_jit, int32_t code)
-{
- ic(_jit, 0x70 | code);
- return jit_reloc(_jit, JIT_RELOC_REL8, 1, 0);
-}
-
-static jit_reloc_t
-jcc(jit_state_t *_jit, int32_t code)
-{
- ic(_jit, 0x0f);
- ic(_jit, 0x80 | code);
- return jit_reloc(_jit, JIT_RELOC_REL32, 2, 0);
-}
-
-#define DEFINE_JUMPS(cc, CC, code) \
- static inline jit_reloc_t j##cc(jit_state_t *_jit) \
- { \
- return jcc(_jit, X86_CC_##CC); \
- } \
- static inline jit_reloc_t j##cc##s(jit_state_t *_jit) \
- { \
- return jccs(_jit, X86_CC_##CC); \
- }
-FOR_EACH_CC(DEFINE_JUMPS)
-#undef DEFINE_JUMPS
-
-static jit_reloc_t
-jcr(jit_state_t *_jit, int32_t code, int32_t r0, int32_t r1)
-{
- alur(_jit, X86_CMP, r0, r1);
- return jcc(_jit, code);
-}
-
-static jit_reloc_t
-jci(jit_state_t *_jit, int32_t code, int32_t r0, jit_word_t i0)
-{
- alui(_jit, X86_CMP, r0, i0);
- return jcc(_jit, code);
-}
-
-static jit_reloc_t
-jci0(jit_state_t *_jit, int32_t code, int32_t r0)
-{
- testr(_jit, r0, r0);
- return jcc(_jit, code);
-}
-
-static jit_reloc_t
-bltr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return jcr(_jit, X86_CC_L, r0, r1);
-}
-
-static jit_reloc_t
-blti(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (i1) return jci (_jit, X86_CC_L, r0, i1);
- else return jci0(_jit, X86_CC_S, r0);
-}
-
-static jit_reloc_t
-bltr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return jcr(_jit, X86_CC_B, r0, r1);
-}
-
-static jit_reloc_t
-blti_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (i1) return jci (_jit, X86_CC_B, r0, i1);
- else return jci0(_jit, X86_CC_B, r0);
-}
-
-static jit_reloc_t
-bler(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return jcr (_jit, X86_CC_LE, r0, r1);
-}
-
-static jit_reloc_t
-blei(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (i1) return jci (_jit, X86_CC_LE, r0, i1);
- else return jci0(_jit, X86_CC_LE, r0);
-}
-
-static jit_reloc_t
-bler_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return jcr (_jit, X86_CC_BE, r0, r1);
-}
-
-static jit_reloc_t
-blei_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (i1) return jci (_jit, X86_CC_BE, r0, i1);
- else return jci0(_jit, X86_CC_BE, r0);
-}
-
-static jit_reloc_t
-beqr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return jcr (_jit, X86_CC_E, r0, r1);
-}
-
-static jit_reloc_t
-beqi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (i1) return jci (_jit, X86_CC_E, r0, i1);
- else return jci0(_jit, X86_CC_E, r0);
-}
-
-static jit_reloc_t
-bger(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return jcr (_jit, X86_CC_GE, r0, r1);
-}
-
-static jit_reloc_t
-bgei(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (i1) return jci (_jit, X86_CC_GE, r0, i1);
- else return jci0(_jit, X86_CC_NS, r0);
-}
-
-static jit_reloc_t
-bger_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return jcr (_jit, X86_CC_AE, r0, r1);
-}
-
-static jit_reloc_t
-bgei_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- return jci (_jit, X86_CC_AE, r0, i1);
-}
-
-static jit_reloc_t
-bgtr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return jcr(_jit, X86_CC_G, r0, r1);
-}
-
-static jit_reloc_t
-bgti(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- return jci(_jit, X86_CC_G, r0, i1);
-}
-
-static jit_reloc_t
-bgtr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return jcr(_jit, X86_CC_A, r0, r1);
-}
-
-static jit_reloc_t
-bgti_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (i1) return jci (_jit, X86_CC_A, r0, i1);
- else return jci0(_jit, X86_CC_NE, r0);
-}
-
-static jit_reloc_t
-bner(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- return jcr(_jit, X86_CC_NE, r0, r1);
-}
-
-static jit_reloc_t
-bnei(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (i1) return jci (_jit, X86_CC_NE, r0, i1);
- else return jci0(_jit, X86_CC_NE, r0);
-}
-
-static jit_reloc_t
-bmsr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- testr(_jit, r0, r1);
- return jnz(_jit);
-}
-
-static jit_reloc_t
-bmsi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (can_zero_extend_int_p(i1)) {
- testi(_jit, r0, i1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i1);
- testr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
- return jnz(_jit);
-}
-
-static jit_reloc_t
-bmcr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- testr(_jit, r0, r1);
- return jz(_jit);
-}
-
-static jit_reloc_t
-bmci(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (can_zero_extend_int_p(i1)) {
- testi(_jit, r0, i1);
- } else {
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i1);
- testr(_jit, r0, rn(reg));
- unget_temp_gpr(_jit);
- }
- return jz(_jit);
-}
-
-static jit_reloc_t
-boaddr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- iaddr(_jit, r0, r1);
- return jo(_jit);
-}
-
-static jit_reloc_t
-boaddi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (can_sign_extend_int_p(i1)) {
- iaddi(_jit, r0, i1);
- return jo(_jit);
- }
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i1);
- unget_temp_gpr(_jit);
- return boaddr(_jit, r0, rn(reg));
-}
-
-static jit_reloc_t
-boaddr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- iaddr(_jit, r0, r1);
- return jc(_jit);
-}
-
-static jit_reloc_t
-boaddi_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (can_sign_extend_int_p(i1)) {
- iaddi(_jit, r0, i1);
- return jc(_jit);
- }
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i1);
- unget_temp_gpr(_jit);
- return boaddr_u(_jit, r0, rn(reg));
-}
-
-static jit_reloc_t
-bxaddr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- iaddr(_jit, r0, r1);
- return jno(_jit);
-}
-
-static jit_reloc_t
-bxaddi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (can_sign_extend_int_p(i1)) {
- iaddi(_jit, r0, i1);
- return jno(_jit);
- }
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i1);
- unget_temp_gpr(_jit);
- return bxaddr(_jit, r0, rn(reg));
-}
-
-static jit_reloc_t
-bxaddr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- iaddr(_jit, r0, r1);
- return jnc(_jit);
-}
-
-static jit_reloc_t
-bxaddi_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (can_sign_extend_int_p(i1)) {
- iaddi(_jit, r0, i1);
- return jnc(_jit);
- }
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i1);
- unget_temp_gpr(_jit);
- return bxaddr_u(_jit, r0, rn(reg));
-}
-
-static jit_reloc_t
-bosubr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- isubr(_jit, r0, r1);
- return jo(_jit);
-}
-
-static jit_reloc_t
-bosubi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (can_sign_extend_int_p(i1)) {
- isubi(_jit, r0, i1);
- return jo(_jit);
- }
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i1);
- unget_temp_gpr(_jit);
- return bosubr(_jit, r0, rn(reg));
-}
-
-static jit_reloc_t
-bosubr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- isubr(_jit, r0, r1);
- return jc(_jit);
-}
-
-static jit_reloc_t
-bosubi_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (can_sign_extend_int_p(i1)) {
- isubi(_jit, r0, i1);
- return jc(_jit);
- }
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i1);
- unget_temp_gpr(_jit);
- return bosubr_u(_jit, r0, rn(reg));
-}
-
-static jit_reloc_t
-bxsubr(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- isubr(_jit, r0, r1);
- return jno(_jit);
-}
-
-static jit_reloc_t
-bxsubi(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (can_sign_extend_int_p(i1)) {
- isubi(_jit, r0, i1);
- return jno(_jit);
- }
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i1);
- unget_temp_gpr(_jit);
- return bxsubr(_jit, r0, rn(reg));
-}
-
-static jit_reloc_t
-bxsubr_u(jit_state_t *_jit, int32_t r0, int32_t r1)
-{
- isubr(_jit, r0, r1);
- return jnc(_jit);
-}
-
-static jit_reloc_t
-bxsubi_u(jit_state_t *_jit, int32_t r0, jit_word_t i1)
-{
- if (can_sign_extend_int_p(i1)) {
- isubi(_jit, r0, i1);
- return jnc(_jit);
- }
- jit_gpr_t reg = get_temp_gpr(_jit);
- movi(_jit, rn(reg), i1);
- unget_temp_gpr(_jit);
- return bxsubr_u(_jit, r0, rn(reg));
-}
-
-static void
-callr(jit_state_t *_jit, int32_t r0)
-{
- rex(_jit, 0, 0, _NOREG, _NOREG, r0);
- ic(_jit, 0xff);
- mrm(_jit, 0x03, 0x02, r7(r0));
-}
-
-static void
-calli(jit_state_t *_jit, jit_word_t i0)
-{
- ptrdiff_t rel32 = i0 - (_jit->pc.w + 4);
- if (INT32_MIN <= rel32 && rel32 <= INT32_MAX)
- {
- ic(_jit, 0xe8);
- ii(_jit, rel32);
- }
- else
- {
- jit_gpr_t reg = get_temp_gpr(_jit);
- jit_patch_there(_jit, mov_addr(_jit, rn(reg)), (void*)i0);
- callr(_jit, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static void
-jmpr(jit_state_t *_jit, int32_t r0)
-{
- rex(_jit, 0, WIDE, _NOREG, _NOREG, r0);
- ic(_jit, 0xff);
- mrm(_jit, 0x03, 0x04, r7(r0));
-}
-
-static void
-jmpi(jit_state_t *_jit, jit_word_t i0)
-{
- ptrdiff_t rel32 = i0 - (_jit->pc.w + 4);
- if (INT32_MIN <= rel32 && rel32 <= INT32_MAX)
- {
- ic(_jit, 0xe9);
- ii(_jit, rel32);
- }
- else
- {
- jit_gpr_t reg = get_temp_gpr(_jit);
- jit_patch_there(_jit, mov_addr(_jit, rn(reg)), (void*)i0);
- jmpr(_jit, rn(reg));
- unget_temp_gpr(_jit);
- }
-}
-
-static jit_reloc_t
-jmp(jit_state_t *_jit)
-{
- ic(_jit, 0xe9);
- return jit_reloc(_jit, JIT_RELOC_REL32, 1, 0);
-}
-
-static jit_reloc_t
-jmpsi(jit_state_t *_jit)
-{
- ic(_jit, 0xeb);
- return jit_reloc(_jit, JIT_RELOC_REL8, 1, 0);
-}
-
-static void
-ret(jit_state_t *_jit)
-{
- ic(_jit, 0xc3);
-}
-
-static void
-retr(jit_state_t *_jit, int32_t r0)
-{
- movr(_jit, _RAX_REGNO, r0);
- ret(_jit);
-}
-
-static void
-reti(jit_state_t *_jit, jit_word_t i0)
-{
- movi(_jit, _RAX_REGNO, i0);
- ret(_jit);
-}
-
-static void
-retval_c(jit_state_t *_jit, int32_t r0)
-{
- extr_c(_jit, r0, rn(JIT_RET));
-}
-
-static void
-retval_uc(jit_state_t *_jit, int32_t r0)
-{
- extr_uc(_jit, r0, rn(JIT_RET));
-}
-
-static void
-retval_s(jit_state_t *_jit, int32_t r0)
-{
- extr_s(_jit, r0, rn(JIT_RET));
-}
-
-static void
-retval_us(jit_state_t *_jit, int32_t r0)
-{
- extr_us(_jit, r0, rn(JIT_RET));
-}
-
-static void
-retval_i(jit_state_t *_jit, int32_t r0)
-{
-#if __X32 || __X64_32
- movr(_jit, r0, rn(JIT_RET));
-#else
- extr_i(_jit, r0, rn(JIT_RET));
-#endif
-}
-
-#if __X64 && !__X64_32
-static void
-retval_ui(jit_state_t *_jit, int32_t r0)
-{
- extr_ui(_jit, r0, rn(JIT_RET));
-}
-
-static void
-retval_l(jit_state_t *_jit, int32_t r0)
-{
- movr(_jit, r0, rn(JIT_RET));
-}
-#endif