7990392006-04-27Tor Edvardsson /* * Machine code generator for AMD64. */ #include "operators.h" #include "constants.h" #include "object.h" #include "builtin_functions.h" /* This is defined on windows */ #ifdef REG_NONE #undef REG_NONE #endif /* Register encodings */ enum amd64_reg {REG_RAX = 0, REG_RBX = 3, REG_RCX = 1, REG_RDX = 2,
d1fa802011-05-09Henrik Grubbström (Grubba)  REG_RSP = 4, REG_RBP = 5, REG_RSI = 6, REG_RDI = 7, REG_R8 = 8, REG_R9 = 9, REG_R10 = 10, REG_R11 = 11, REG_R12 = 12, REG_R13 = 13, REG_R14 = 14, REG_R15 = 15, REG_NONE = 4};
7990392006-04-27Tor Edvardsson 
302d102011-05-20Henrik Grubbström (Grubba) /* We reserve register r12 and above (as well as RSP, RBP and RBX). */
d1fa802011-05-09Henrik Grubbström (Grubba) #define REG_BITMASK ((1 << REG_MAX) - 1)
302d102011-05-20Henrik Grubbström (Grubba) #define REG_RESERVED (REG_RSP|REG_RBP|REG_RBX)
639a932011-05-15Henrik Grubbström (Grubba) #define REG_MAX REG_R12 #define PIKE_MARK_SP_REG REG_R12 #define PIKE_SP_REG REG_R13 #define PIKE_FP_REG REG_R14 #define Pike_interpreter_reg REG_R15
d1fa802011-05-09Henrik Grubbström (Grubba)  #ifdef __NT__ /* From http://software.intel.com/en-us/articles/introduction-to-x64-assembly/ * * Note: Space for the arguments needs to be allocated on the stack as well. */ #define ARG1_REG REG_RCX #define ARG2_REG REG_RDX #define ARG3_REG REG_R8 #define ARG4_REG REG_R9 #else /* From SysV ABI for AMD64 draft 0.99.5. */ #define ARG1_REG REG_RDI #define ARG2_REG REG_RSI #define ARG3_REG REG_RDX #define ARG4_REG REG_RCX #define ARG5_REG REG_R8 #define ARG6_REG REG_R9 #endif
7990392006-04-27Tor Edvardsson 
719c3a2012-06-12Per Hedbor 
15e8a02012-06-13Henrik Grubbström (Grubba) #define MAX_LABEL_USES 6
719c3a2012-06-12Per Hedbor struct label { int n_label_uses;
15e8a02012-06-13Henrik Grubbström (Grubba)  ptrdiff_t addr;
719c3a2012-06-12Per Hedbor  ptrdiff_t offset[MAX_LABEL_USES]; }; static void label( struct label *l ) { int i;
15e8a02012-06-13Henrik Grubbström (Grubba)  if (l->addr >= 0) Pike_fatal("Label reused.\n");
719c3a2012-06-12Per Hedbor  for( i=0; i<l->n_label_uses; i++ ) { int dist = PIKE_PC - (l->offset[i] + 1); if( dist > 0x7f || dist < -0x80 ) Pike_fatal("Branch too far\n"); Pike_compiler->new_program->program[l->offset[i]] = dist; /* fprintf( stderr, "assigning label @%x[%02x >%02x< %02x] -> %d\n", */ /* l->offset[i], */ /* Pike_compiler->new_program->program[l->offset[i]-1], */ /* Pike_compiler->new_program->program[l->offset[i]], */ /* Pike_compiler->new_program->program[l->offset[i]+1], */ /* dist ); */ } l->n_label_uses = 0;
15e8a02012-06-13Henrik Grubbström (Grubba)  l->addr = PIKE_PC;
719c3a2012-06-12Per Hedbor }
99c1e92012-06-20Per Hedbor static void ib( char x ) { add_to_program( x ); } static void iw( short x ) { add_to_program( x>>8 ); add_to_program( x ); } static void id( int x ) { add_to_program( (x)&0xff ); add_to_program( (x>>8)&0xff ); add_to_program( (x>>16)&0xff ); add_to_program( (x>>24)&0xff ); } /* x86 opcodes */ #define opcode(X) ib(X)
719c3a2012-06-12Per Hedbor static void modrm( int mod, int r, int m ) {
99c1e92012-06-20Per Hedbor  ib( ((mod<<6) | ((r&0x7)<<3) | (m&0x7)) );
719c3a2012-06-12Per Hedbor } static void sib( int scale, int index, enum amd64_reg base ) {
99c1e92012-06-20Per Hedbor  ib( (scale<<6) | ((index&0x7)<<3) | (base&0x7) ); } static void modrm_sib( int mod, int r, int m ) { modrm( mod, r, m ); if( (m&7) == REG_RSP) sib(0, REG_RSP, REG_RSP );
719c3a2012-06-12Per Hedbor } static void rex( int w, enum amd64_reg r, int x, enum amd64_reg b ) { unsigned char res = 1<<6; /* bit 7, 5-4 == 0 */ if( w ) res |= 1<<3; if( r > 0x7 ) res |= 1<<2; if( x ) res |= 1<<1; if( b > 0x7 ) res |= 1<<0; if( res != (1<<6) ) add_to_program( res ); }
99c1e92012-06-20Per Hedbor static void offset_modrm_sib( int offset, int r, int m )
719c3a2012-06-12Per Hedbor {
99c1e92012-06-20Per Hedbor  /* OFFSET */ if( offset < -128 || offset > 127 ) { modrm_sib( 2, r, m ); id( offset ); } else if( offset || (m&7) == REG_RSP || (m&7) == REG_RBP ) { modrm_sib( 1, r, m ); ib( offset ); } else { modrm_sib( 0, r, m ); }
719c3a2012-06-12Per Hedbor } static void ret() { opcode(0xc3); } static void push(enum amd64_reg reg ) { if (reg & 0x08) add_to_program(0x41); add_to_program(0x50 + (reg & 0x07)); } static void pop(enum amd64_reg reg ) { if (reg & 0x08) add_to_program(0x41); add_to_program(0x58 + (reg & 0x07)); } static void mov_reg_reg(enum amd64_reg from_reg, enum amd64_reg to_reg ) { rex( 1, from_reg, 0, to_reg ); opcode( 0x89 ); modrm( 3, from_reg, to_reg ); }
7990392006-04-27Tor Edvardsson #define PUSH_INT(X) ins_int((INT32)(X), (void (*)(char))add_to_program)
6785ae2012-06-15Per Hedbor static void low_mov_mem_reg(enum amd64_reg from_reg, ptrdiff_t offset, enum amd64_reg to_reg)
719c3a2012-06-12Per Hedbor { opcode( 0x8b );
99c1e92012-06-20Per Hedbor  offset_modrm_sib(offset, to_reg, from_reg );
719c3a2012-06-12Per Hedbor } static void mov_mem_reg( enum amd64_reg from_reg, ptrdiff_t offset, enum amd64_reg to_reg ) { rex( 1, to_reg, 0, from_reg ); low_mov_mem_reg( from_reg, offset, to_reg ); }
99c1e92012-06-20Per Hedbor static void xor_reg_reg( enum amd64_reg reg1, enum amd64_reg reg2 )
719c3a2012-06-12Per Hedbor {
99c1e92012-06-20Per Hedbor  rex(1,reg1,0,reg2); opcode( 0x31 ); modrm(3,reg1,reg2);
719c3a2012-06-12Per Hedbor } static void and_reg_imm( enum amd64_reg reg, int imm32 ) { rex( 1, 0, 0, reg ); if( imm32 < -0x80 || imm32 > 0x7f ) { if( reg == REG_RAX ) {
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x25 ); /* AND rax,imm32 */
719c3a2012-06-12Per Hedbor  id( imm32 ); } else {
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x81 ); /* AND REG,imm32 */
719c3a2012-06-12Per Hedbor  modrm( 3,4, reg); id( imm32 ); } } else {
1c1a242012-06-13Henrik Grubbström (Grubba)  add_to_program(0x83); /* AND REG,imm8 */
719c3a2012-06-12Per Hedbor  modrm( 3, 4, reg ); ib( imm32 ); } }
99c1e92012-06-20Per Hedbor static void mov_mem32_reg( enum amd64_reg from_reg, ptrdiff_t offset, enum amd64_reg to_reg ) { rex( 0, to_reg, 0, from_reg ); low_mov_mem_reg( from_reg, offset, to_reg ); }
719c3a2012-06-12Per Hedbor static void mov_mem16_reg( enum amd64_reg from_reg, ptrdiff_t offset, enum amd64_reg to_reg ) {
99c1e92012-06-20Per Hedbor  /* Actually doing a 16-bit read seems slower.. */
719c3a2012-06-12Per Hedbor  mov_mem32_reg( from_reg, offset, to_reg );
99c1e92012-06-20Per Hedbor  and_reg_imm(to_reg, 0xffff);
719c3a2012-06-12Per Hedbor } static void add_reg_imm( enum amd64_reg src, int imm32); static void shl_reg_imm( enum amd64_reg from_reg, int shift ) { rex( 1, from_reg, 0, 0 ); if( shift == 1 ) {
b233f42012-06-17Henrik Grubbström (Grubba)  opcode( 0xd1 ); /* SAL */ modrm( 3, 4, from_reg );
719c3a2012-06-12Per Hedbor  } else { opcode( 0xc1 );
b233f42012-06-17Henrik Grubbström (Grubba)  modrm( 3, 4, from_reg );
719c3a2012-06-12Per Hedbor  ib( shift ); } } static void clear_reg( enum amd64_reg reg ) { xor_reg_reg( reg, reg ); }
6785ae2012-06-15Per Hedbor static void neg_reg( enum amd64_reg reg ) { rex(1,0,0,reg); opcode(0xf7); modrm(3,3,reg); }
719c3a2012-06-12Per Hedbor static void mov_imm_reg( long imm, enum amd64_reg reg ) { if( (imm > 0x7fffffffLL) || (imm < -0x80000000LL) ) { rex(1,0,0,reg); opcode(0xb8 | (reg&0x7)); /* mov imm64 -> reg64 */ id( (imm & 0xffffffffLL) ); id( ((imm >> 32)&0xffffffffLL) ); } else { rex(1,0,0,reg); opcode( 0xc7 ); /* mov imm32 -> reg/m 64, SE*/ modrm( 3,0,reg ); id( (int)imm ); } }
99c1e92012-06-20Per Hedbor static void low_mov_reg_mem(enum amd64_reg from_reg, enum amd64_reg to_reg, ptrdiff_t offset ) { opcode( 0x89 ); offset_modrm_sib( offset, from_reg, to_reg ); }
719c3a2012-06-12Per Hedbor static void mov_reg_mem( enum amd64_reg from_reg, enum amd64_reg to_reg, ptrdiff_t offset ) { rex(1, from_reg, 0, to_reg );
99c1e92012-06-20Per Hedbor  low_mov_reg_mem( from_reg, to_reg, offset ); }
719c3a2012-06-12Per Hedbor 
99c1e92012-06-20Per Hedbor static void mov_reg_mem32( enum amd64_reg from_reg, enum amd64_reg to_reg, ptrdiff_t offset ) { rex(0, from_reg, 0, to_reg ); low_mov_reg_mem( from_reg, to_reg, offset ); } static void mov_reg_mem16( enum amd64_reg from_reg, enum amd64_reg to_reg, ptrdiff_t offset ) { opcode( 0x66 ); rex(0, from_reg, 0, to_reg ); low_mov_reg_mem( from_reg, to_reg, offset );
719c3a2012-06-12Per Hedbor }
99c1e92012-06-20Per Hedbor 
719c3a2012-06-12Per Hedbor static void mov_imm_mem( long imm, enum amd64_reg to_reg, ptrdiff_t offset ) { if( imm >= -0x80000000LL && imm <= 0x7fffffffLL ) { rex( 1, 0, 0, to_reg ); opcode( 0xc7 ); /* mov imm32 -> r/m64 (sign extend)*/
99c1e92012-06-20Per Hedbor  offset_modrm_sib( offset, 0, to_reg );
719c3a2012-06-12Per Hedbor  id( imm ); } else { if( to_reg == REG_RAX ) Pike_fatal( "Clobbered TMP REG_RAX reg\n"); mov_imm_reg( imm, REG_RAX ); mov_reg_mem( REG_RAX, to_reg, offset ); } } static void mov_imm_mem32( int imm, enum amd64_reg to_reg, ptrdiff_t offset ) { rex( 0, 0, 0, to_reg ); opcode( 0xc7 ); /* mov imm32 -> r/m32 (sign extend)*/ /* This does not work for rg&7 == 4 or 5. */
99c1e92012-06-20Per Hedbor  offset_modrm_sib( offset, 0, to_reg );
719c3a2012-06-12Per Hedbor  id( imm ); }
99c1e92012-06-20Per Hedbor static void sub_reg_imm( enum amd64_reg reg, int imm32 );
719c3a2012-06-12Per Hedbor static void add_reg_imm( enum amd64_reg reg, int imm32 ) { if( !imm32 ) return;
99c1e92012-06-20Per Hedbor  if( imm32 < 0 ) { /* This makes the disassembly easier to read. */ sub_reg_imm( reg, -imm32 ); return; }
719c3a2012-06-12Per Hedbor  rex( 1, 0, 0, reg ); if( imm32 < -0x80 || imm32 > 0x7f ) { if( reg == REG_RAX ) {
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x05 ); /* ADD rax,imm32 */
719c3a2012-06-12Per Hedbor  id( imm32 ); } else {
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x81 ); /* ADD REG,imm32 */
719c3a2012-06-12Per Hedbor  modrm( 3, 0, reg); id( imm32 ); } } else {
1c1a242012-06-13Henrik Grubbström (Grubba)  add_to_program(0x83); /* ADD REG,imm8 */
719c3a2012-06-12Per Hedbor  modrm( 3, 0, reg ); ib( imm32 ); } }
99c1e92012-06-20Per Hedbor  static void low_add_mem_imm(int w, enum amd64_reg reg, int offset, int imm32 )
719c3a2012-06-12Per Hedbor { int r2 = imm32 == -1 ? 1 : 0; int large = 0; if( !imm32 ) return;
99c1e92012-06-20Per Hedbor  rex( w, 0, 0, reg );
719c3a2012-06-12Per Hedbor  if( r2 ) imm32 = -imm32; if( imm32 == 1 )
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0xff ); /* INCL(DECL) r/m32 */
719c3a2012-06-12Per Hedbor  else if( imm32 >= -128 && imm32 < 128 )
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x83 ); /* ADD imm8,r/m32 */
719c3a2012-06-12Per Hedbor  else {
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x81 ); /* ADD imm32,r/m32 */
719c3a2012-06-12Per Hedbor  large = 1; }
99c1e92012-06-20Per Hedbor  offset_modrm_sib( offset, r2, reg );
719c3a2012-06-12Per Hedbor  if( imm32 != 1 ) { if( large ) id( imm32 ); else ib( imm32 ); } }
99c1e92012-06-20Per Hedbor static void add_mem32_imm( enum amd64_reg reg, int offset, int imm32 )
719c3a2012-06-12Per Hedbor {
99c1e92012-06-20Per Hedbor  low_add_mem_imm( 0, reg, offset, imm32 );
6785ae2012-06-15Per Hedbor }
99c1e92012-06-20Per Hedbor static void add_mem_imm( enum amd64_reg reg, int offset, int imm32 )
6785ae2012-06-15Per Hedbor {
99c1e92012-06-20Per Hedbor  low_add_mem_imm( 1, reg, offset, imm32 );
6785ae2012-06-15Per Hedbor } static void add_mem8_imm( enum amd64_reg reg, int offset, int imm32 ) { int r2 = imm32 == -1 ? 1 : 0; if( !imm32 ) return; rex( 0, 0, 0, reg ); if( imm32 == 1 || imm32 == -1 ) opcode( 0xfe ); /* INCL r/m8 */ else if( imm32 >= -128 && imm32 < 128 ) opcode( 0x80 ); /* ADD imm8,r/m32 */ else Pike_fatal("Not sensible");
719c3a2012-06-12Per Hedbor 
99c1e92012-06-20Per Hedbor  offset_modrm_sib( offset, r2, reg );
6785ae2012-06-15Per Hedbor  if( imm32 != 1 && !r2 ) {
719c3a2012-06-12Per Hedbor  ib( imm32 );
6785ae2012-06-15Per Hedbor  }
719c3a2012-06-12Per Hedbor } static void sub_reg_imm( enum amd64_reg reg, int imm32 ) { if( !imm32 ) return;
99c1e92012-06-20Per Hedbor  if( imm32 < 0 ) /* This makes the disassembly easier to read. */ return add_reg_imm( reg, -imm32 );
719c3a2012-06-12Per Hedbor  rex( 1, 0, 0, reg ); if( imm32 < -0x80 || imm32 > 0x7f ) { if( reg == REG_RAX ) {
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x2d ); /* SUB rax,imm32 */
719c3a2012-06-12Per Hedbor  id( imm32 ); } else {
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x81 ); /* SUB REG,imm32 */
719c3a2012-06-12Per Hedbor  modrm( 3, 5, reg); id( imm32 ); } } else {
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode(0x83); /* SUB REG,imm8 */
719c3a2012-06-12Per Hedbor  modrm( 3, 5, reg ); ib( imm32 ); } } static void test_reg_reg( enum amd64_reg reg1, enum amd64_reg reg2 ) { rex(1,reg1,0,reg2); opcode(0x85); modrm(3, reg1, reg2 ); } static void test_reg( enum amd64_reg reg1 ) { test_reg_reg( reg1, reg1 ); } static void cmp_reg_imm( enum amd64_reg reg, int imm32 ) { rex(1, 0, 0, reg); if( imm32 > 0x7f || imm32 < -0x80 ) { if( reg == REG_RAX ) { opcode( 0x3d ); id( imm32 ); } else { opcode( 0x81 ); modrm(3,7,reg); id( imm32 ); } } else { opcode( 0x83 ); modrm( 3,7,reg); ib( imm32 ); } } static void cmp_reg_reg( enum amd64_reg reg1, enum amd64_reg reg2 ) { rex(1, reg1, 0, reg2); opcode( 0x39 ); modrm( 3, reg1, reg2 ); } static int jmp_rel_imm32( int rel ) { int res; opcode( 0xe9 ); res = PIKE_PC; id( rel ); return res; } static void jmp_rel_imm( int rel ) { if(rel >= -0x80 && rel <= 0x7f ) { opcode( 0xeb ); ib( rel ); return; } jmp_rel_imm32( rel ); }
6785ae2012-06-15Per Hedbor static void call_rel_imm32( int rel ) { rel -= 5; // counts from the next instruction opcode( 0xe8 ); id( rel ); return; }
719c3a2012-06-12Per Hedbor static void jmp_reg( enum amd64_reg reg ) { rex(0,reg,0,0); opcode( 0xff ); modrm( 3, 4, reg ); } static void call_reg( enum amd64_reg reg ) { rex(0,reg,0,0); opcode( 0xff ); modrm( 3, 2, reg ); } static void call_imm( void *ptr ) { size_t addr = (size_t)ptr; if( (addr & ~0x7fffffffLL) && !(addr & ~0x3fffffff8LL) ) { mov_imm_reg( addr>>3, REG_RAX); shl_reg_imm( REG_RAX, 3 ); } else { mov_imm_reg(addr, REG_RAX ); } call_reg( REG_RAX ); }
6785ae2012-06-15Per Hedbor /* reg += reg2 */ static void add_reg_reg( enum amd64_reg reg, enum amd64_reg reg2 ) { rex(1,reg2,0,reg); opcode( 0x1 ); modrm( 3, reg2, reg ); } /* reg -= reg2 */ static void sub_reg_reg( enum amd64_reg reg, enum amd64_reg reg2 ) { rex(1,reg2,0,reg); opcode( 0x29 ); modrm( 3, reg2, reg ); } /* dst += *(src+off) */ static void add_reg_mem( enum amd64_reg dst, enum amd64_reg src, long off ) { rex(1,dst,0,src); opcode( 0x3 );
99c1e92012-06-20Per Hedbor  offset_modrm_sib( off, dst, src );
6785ae2012-06-15Per Hedbor } /* dst -= *(src+off) */ static void sub_reg_mem( enum amd64_reg dst, enum amd64_reg src, long off ) { rex(1,dst,0,src); opcode( 0x2b );
99c1e92012-06-20Per Hedbor  offset_modrm_sib( off, dst, src );
6785ae2012-06-15Per Hedbor }
719c3a2012-06-12Per Hedbor  /* dst = src + imm32 (LEA, does not set flags!) */ static void add_reg_imm_reg( enum amd64_reg src, long imm32, enum amd64_reg dst ) { if( imm32 > 0x7fffffffLL || imm32 <-0x80000000LL) Pike_fatal("LEA [reg+imm] > 32bit Not supported\n"); if( src == dst ) { if( !imm32 ) return; add_reg_imm( src, imm32 ); } else { if( !imm32 ) { mov_reg_reg( src, dst ); return; } rex(1,dst,0,src); opcode( 0x8d ); /* LEA r64,m */
99c1e92012-06-20Per Hedbor  offset_modrm_sib( imm32, dst, src );
719c3a2012-06-12Per Hedbor  } } /* load code adress + imm to reg, always 32bit offset */ static void mov_rip_imm_reg( int imm, enum amd64_reg reg ) {
1c1a242012-06-13Henrik Grubbström (Grubba)  imm -= 7; /* The size of this instruction. */
719c3a2012-06-12Per Hedbor  rex( 1, reg, 0, 0 );
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x8d ); /* LEA */
719c3a2012-06-12Per Hedbor  modrm( 0, reg, 5 ); id( imm ); } static void add_imm_mem( int imm32, enum amd64_reg reg, int offset ) { int r2 = (imm32 == -1) ? 1 : 0; int large = 0; /* OPCODE */ rex( 1, 0, 0, reg ); if( imm32 == 1 || imm32 == -1 )
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0xff ); /* INCL(decl) r/m32 */
719c3a2012-06-12Per Hedbor  else if( -128 <= imm32 && 128 > imm32 )
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x83 ); /* ADD imm8,r/m32 */
719c3a2012-06-12Per Hedbor  else {
1c1a242012-06-13Henrik Grubbström (Grubba)  opcode( 0x81 ); /* ADD imm32,r/m32 */
719c3a2012-06-12Per Hedbor  large = 1; }
99c1e92012-06-20Per Hedbor  offset_modrm_sib( offset, r2, reg );
719c3a2012-06-12Per Hedbor  /* VALUE */ if( imm32 != 1 && !r2 ) { if( large ) id( imm32 ); else ib( imm32 ); } } static void jump_rel8( struct label *res, unsigned char op ) { opcode( op );
7990392006-04-27Tor Edvardsson 
15e8a02012-06-13Henrik Grubbström (Grubba)  if (res->addr >= 0) { ib(res->addr - (PIKE_PC+1)); return; }
719c3a2012-06-12Per Hedbor  if( res->n_label_uses >= MAX_LABEL_USES ) Pike_fatal( "Label used too many times\n" ); res->offset[res->n_label_uses] = PIKE_PC; res->n_label_uses++; ib(0); } static int jnz_imm_rel32( int rel ) { int res; opcode( 0xf ); opcode( 0x85 ); res = PIKE_PC; id( rel ); return res; } static int jz_imm_rel32( int rel ) { int res; opcode( 0xf ); opcode( 0x84 ); res = PIKE_PC; id( rel ); return res; } #define jne(X) jnz(X) #define je(X) jz(X) static void jmp( struct label *l ) { return jump_rel8( l, 0xeb ); } static void jg( struct label *l ) { return jump_rel8( l, 0x7f ); }
15e8a02012-06-13Henrik Grubbström (Grubba) static void jge( struct label *l ) { return jump_rel8( l, 0x7d ); }
b505a72012-06-13Per Hedbor static void jl( struct label *l ) { return jump_rel8( l, 0x7c ); }
15e8a02012-06-13Henrik Grubbström (Grubba) static void jle( struct label *l ) { return jump_rel8( l, 0x7e ); }
719c3a2012-06-12Per Hedbor static void jo( struct label *l ) { return jump_rel8( l, 0x70 ); }
6785ae2012-06-15Per Hedbor static void jno( struct label *l ) { return jump_rel8( l, 0x71 ); }
91080b2012-06-15Henrik Grubbström (Grubba) static void jc( struct label *l ) { return jump_rel8( l, 0x72 ); } static void jnc( struct label *l ) { return jump_rel8( l, 0x73 ); }
6785ae2012-06-15Per Hedbor static void jz( struct label *l ) { return jump_rel8( l, 0x74 ); } static void jnz( struct label *l ) { return jump_rel8( l, 0x75 ); }
719c3a2012-06-12Per Hedbor 
6785ae2012-06-15Per Hedbor #define LABELS() struct label label_A, label_B, label_C, label_D;label_A.addr = -1;label_A.n_label_uses = 0;label_B.addr = -1;label_B.n_label_uses = 0;label_C.addr = -1;label_C.n_label_uses = 0;label_D.addr = -1;label_D.n_label_uses = 0;
719c3a2012-06-12Per Hedbor #define LABEL_A label(&label_A) #define LABEL_B label(&label_B) #define LABEL_C label(&label_C)
6785ae2012-06-15Per Hedbor #define LABEL_D label(&label_D)
d1fa802011-05-09Henrik Grubbström (Grubba)  /* Machine code entry prologue. * * On entry: * RDI: Pike_interpreter (ARG1_REG) * * During interpreting:
639a932011-05-15Henrik Grubbström (Grubba)  * R15: Pike_interpreter
d1fa802011-05-09Henrik Grubbström (Grubba)  */ void amd64_ins_entry(void) { /* Push all registers that the ABI requires to be preserved. */
719c3a2012-06-12Per Hedbor  push(REG_RBP); mov_reg_reg(REG_RSP, REG_RBP); push(REG_R15); push(REG_R14); push(REG_R13); push(REG_R12); push(REG_RBX); sub_reg_imm(REG_RSP, 8); /* Align on 16 bytes. */
d1fa802011-05-09Henrik Grubbström (Grubba) 
719c3a2012-06-12Per Hedbor  mov_reg_reg(ARG1_REG, Pike_interpreter_reg);
639a932011-05-15Henrik Grubbström (Grubba)  amd64_flush_code_generator_state();
d1fa802011-05-09Henrik Grubbström (Grubba) }
54a26b2011-05-11Henrik Grubbström (Grubba) static enum amd64_reg sp_reg = 0, fp_reg = 0, mark_sp_reg = 0;
719c3a2012-06-12Per Hedbor static int dirty_regs = 0, ret_for_func = 0;
d1fa802011-05-09Henrik Grubbström (Grubba) ptrdiff_t amd64_prev_stored_pc = -1; /* PROG_PC at the last point Pike_fp->pc was updated. */
6785ae2012-06-15Per Hedbor static int branch_check_threads_update_etc = -1;
d1fa802011-05-09Henrik Grubbström (Grubba) 
54a26b2011-05-11Henrik Grubbström (Grubba) void amd64_flush_code_generator_state(void) { sp_reg = 0; fp_reg = 0; mark_sp_reg = 0; dirty_regs = 0;
719c3a2012-06-12Per Hedbor  ret_for_func = 0;
54a26b2011-05-11Henrik Grubbström (Grubba)  amd64_prev_stored_pc = -1;
6785ae2012-06-15Per Hedbor  branch_check_threads_update_etc = -1;
54a26b2011-05-11Henrik Grubbström (Grubba) }
9030f62011-05-26Henrik Grubbström (Grubba) static void flush_dirty_regs(void) { /* NB: PIKE_FP_REG is currently never dirty. */ if (dirty_regs & (1 << PIKE_SP_REG)) {
719c3a2012-06-12Per Hedbor  mov_reg_mem(PIKE_SP_REG, Pike_interpreter_reg,
d97eb72011-07-10Henrik Grubbström (Grubba)  OFFSETOF(Pike_interpreter_struct, stack_pointer));
9030f62011-05-26Henrik Grubbström (Grubba)  dirty_regs &= ~(1 << PIKE_SP_REG); } if (dirty_regs & (1 << PIKE_MARK_SP_REG)) {
719c3a2012-06-12Per Hedbor  mov_reg_mem(PIKE_MARK_SP_REG, Pike_interpreter_reg,
d97eb72011-07-10Henrik Grubbström (Grubba)  OFFSETOF(Pike_interpreter_struct, mark_stack_pointer));
9030f62011-05-26Henrik Grubbström (Grubba)  dirty_regs &= ~(1 << PIKE_MARK_SP_REG); } }
7990392006-04-27Tor Edvardsson 
54a26b2011-05-11Henrik Grubbström (Grubba) /* NB: We load Pike_fp et al into registers that * are persistent across function calls. */ void amd64_load_fp_reg(void) { if (!fp_reg) {
719c3a2012-06-12Per Hedbor  mov_mem_reg(Pike_interpreter_reg,
6785ae2012-06-15Per Hedbor  OFFSETOF(Pike_interpreter_struct, frame_pointer), PIKE_FP_REG);
54a26b2011-05-11Henrik Grubbström (Grubba)  fp_reg = PIKE_FP_REG; } } void amd64_load_sp_reg(void) { if (!sp_reg) {
719c3a2012-06-12Per Hedbor  mov_mem_reg(Pike_interpreter_reg,
6785ae2012-06-15Per Hedbor  OFFSETOF(Pike_interpreter_struct, stack_pointer), PIKE_SP_REG);
54a26b2011-05-11Henrik Grubbström (Grubba)  sp_reg = PIKE_SP_REG; } } void amd64_load_mark_sp_reg(void) { if (!mark_sp_reg) {
719c3a2012-06-12Per Hedbor  mov_mem_reg(Pike_interpreter_reg,
6785ae2012-06-15Per Hedbor  OFFSETOF(Pike_interpreter_struct, mark_stack_pointer), PIKE_MARK_SP_REG);
54a26b2011-05-11Henrik Grubbström (Grubba)  mark_sp_reg = PIKE_MARK_SP_REG; } }
d1fa802011-05-09Henrik Grubbström (Grubba) static void update_arg1(INT32 value) {
719c3a2012-06-12Per Hedbor  mov_imm_reg(value, ARG1_REG);
d1fa802011-05-09Henrik Grubbström (Grubba)  /* FIXME: Alloc stack space on NT. */ }
7990392006-04-27Tor Edvardsson 
d1fa802011-05-09Henrik Grubbström (Grubba) static void update_arg2(INT32 value) {
719c3a2012-06-12Per Hedbor  mov_imm_reg(value, ARG2_REG);
d1fa802011-05-09Henrik Grubbström (Grubba)  /* FIXME: Alloc stack space on NT. */
7990392006-04-27Tor Edvardsson }
719c3a2012-06-12Per Hedbor static void amd64_add_sp( int num ) { amd64_load_sp_reg();
6785ae2012-06-15Per Hedbor  add_reg_imm( sp_reg, sizeof(struct svalue)*num);
719c3a2012-06-12Per Hedbor  dirty_regs |= 1 << PIKE_SP_REG; flush_dirty_regs(); /* FIXME: Why is this needed? */ } static void amd64_add_mark_sp( int num ) { amd64_load_mark_sp_reg(); add_reg_imm( mark_sp_reg, sizeof(struct svalue*)*num); dirty_regs |= 1 << PIKE_MARK_SP_REG; }
9030f62011-05-26Henrik Grubbström (Grubba) /* Note: Uses RAX and RCX internally. reg MUST not be REG_RAX. */ static void amd64_push_svaluep(int reg) {
719c3a2012-06-12Per Hedbor  LABELS();
9030f62011-05-26Henrik Grubbström (Grubba)  amd64_load_sp_reg();
719c3a2012-06-12Per Hedbor  mov_mem_reg(reg, OFFSETOF(svalue, type), REG_RAX); mov_mem_reg(reg, OFFSETOF(svalue, u.refs), REG_RCX); mov_reg_mem(REG_RAX, sp_reg, OFFSETOF(svalue, type)); and_reg_imm(REG_RAX, 0x1f); mov_reg_mem(REG_RCX, sp_reg, OFFSETOF(svalue, u.refs)); cmp_reg_imm(REG_RAX, MAX_REF_TYPE); jg(&label_A); add_imm_mem( 1, REG_RCX,OFFSETOF(pike_string, refs)); LABEL_A; amd64_add_sp( 1 );
9030f62011-05-26Henrik Grubbström (Grubba) }
eb42a12011-05-11Henrik Grubbström (Grubba) static void amd64_push_int(INT64 value, int subtype) { amd64_load_sp_reg();
719c3a2012-06-12Per Hedbor  mov_imm_mem((subtype<<16) + PIKE_T_INT, sp_reg, OFFSETOF(svalue, type)); mov_imm_mem(value, sp_reg, OFFSETOF(svalue, u.integer)); amd64_add_sp( 1 ); } static void amd64_push_int_reg(enum amd64_reg reg ) { if( reg == REG_RCX ) Pike_fatal( "Source clobbered in push_int_reg\n"); amd64_load_sp_reg(); mov_imm_mem( PIKE_T_INT, sp_reg, OFFSETOF(svalue, type)); mov_reg_mem( reg, sp_reg, OFFSETOF(svalue, u.integer)); amd64_add_sp( 1 );
eb42a12011-05-11Henrik Grubbström (Grubba) }
d1fa802011-05-09Henrik Grubbström (Grubba) 
ab7cf52011-05-24Henrik Grubbström (Grubba) static void amd64_mark(int offset) { amd64_load_sp_reg(); amd64_load_mark_sp_reg(); if (offset) {
719c3a2012-06-12Per Hedbor  add_reg_imm_reg(sp_reg, -offset * sizeof(struct svalue), REG_RAX); mov_reg_mem(REG_RAX, mark_sp_reg, 0);
ab7cf52011-05-24Henrik Grubbström (Grubba)  } else {
719c3a2012-06-12Per Hedbor  mov_reg_mem(sp_reg, mark_sp_reg, 0);
ab7cf52011-05-24Henrik Grubbström (Grubba)  }
719c3a2012-06-12Per Hedbor  amd64_add_mark_sp( 1 ); } static void mov_sval_type(enum amd64_reg src, enum amd64_reg dst ) { mov_mem32_reg( src, OFFSETOF(svalue,type), dst); and_reg_imm( dst, 0x1f ); } static void amd64_call_c_function(void *addr) { flush_dirty_regs(); call_imm(addr); } static void amd64_free_svalue(enum amd64_reg src, int guaranteed_ref ) { LABELS(); if( src == REG_RAX ) Pike_fatal("Clobbering RAX for free-svalue\n"); /* load type -> RAX */ mov_sval_type( src, REG_RAX ); /* if RAX > MAX_REF_TYPE+1 */ cmp_reg_imm( REG_RAX,MAX_REF_TYPE); jg( &label_A ); /* Load pointer to refs -> RAX */ mov_mem_reg( src, OFFSETOF(svalue, u.refs), REG_RAX); /* if( !--*RAX ) */ add_mem32_imm( REG_RAX, OFFSETOF(pike_string,refs), -1); if( !guaranteed_ref ) { /* We need to see if refs got to 0. */ jnz( &label_A );
99c1e92012-06-20Per Hedbor  /* if so, call really_free_svalue */ if( src != ARG1_REG ) mov_reg_reg( src, ARG1_REG ); amd64_call_c_function(really_free_svalue); } LABEL_A; } /* Type already in RAX */ static void amd64_free_svalue_type(enum amd64_reg src, enum amd64_reg type, int guaranteed_ref ) { LABELS(); /* if type > MAX_REF_TYPE+1 */ if( src == REG_RAX ) Pike_fatal("Clobbering RAX for free-svalue\n"); cmp_reg_imm(type,MAX_REF_TYPE); jg( &label_A ); /* Load pointer to refs -> RAX */ mov_mem_reg( src, OFFSETOF(svalue, u.refs), REG_RAX); /* if( !--*RAX ) */ add_mem32_imm( REG_RAX, OFFSETOF(pike_string,refs), -1); if( !guaranteed_ref ) { /* We need to see if refs got to 0. */ jnz( &label_A ); /* if so, call really_free_svalue */
719c3a2012-06-12Per Hedbor  if( src != ARG1_REG ) mov_reg_reg( src, ARG1_REG ); amd64_call_c_function(really_free_svalue);
ab7cf52011-05-24Henrik Grubbström (Grubba)  }
719c3a2012-06-12Per Hedbor  LABEL_A; }
b505a72012-06-13Per Hedbor void amd64_ref_svalue( enum amd64_reg src, int already_have_type )
719c3a2012-06-12Per Hedbor { LABELS(); if( src == REG_RAX ) Pike_fatal("Clobbering src in ref_svalue\n");
b505a72012-06-13Per Hedbor  if( !already_have_type ) mov_sval_type( src, REG_RAX ); else and_reg_imm( REG_RAX, 0x1f );
719c3a2012-06-12Per Hedbor  /* if RAX > MAX_REF_TYPE+1 */ cmp_reg_imm(REG_RAX, MAX_REF_TYPE ); jg( &label_A ); /* Load pointer to refs -> RAX */ mov_mem_reg( src, OFFSETOF(svalue, u.refs), REG_RAX); /* *RAX++ */ add_mem32_imm( REG_RAX, OFFSETOF(pike_string,refs), 1); LABEL_A; } void amd64_assign_local( int b ) { amd64_load_fp_reg(); amd64_load_sp_reg(); mov_mem_reg( fp_reg, OFFSETOF(pike_frame, locals), ARG1_REG); add_reg_imm( ARG1_REG,b*sizeof(struct svalue) ); mov_reg_reg( ARG1_REG, REG_RBX );
1c1a242012-06-13Henrik Grubbström (Grubba)  /* Free old svalue. */
719c3a2012-06-12Per Hedbor  amd64_free_svalue(ARG1_REG, 0);
1c1a242012-06-13Henrik Grubbström (Grubba)  /* Copy sp[-1] -> local */
719c3a2012-06-12Per Hedbor  mov_mem_reg(sp_reg, -1*sizeof(struct svalue), REG_RAX); mov_mem_reg(sp_reg, -1*sizeof(struct svalue)+sizeof(long), REG_RCX); mov_reg_mem( REG_RAX, REG_RBX, 0 ); mov_reg_mem( REG_RCX, REG_RBX, sizeof(long) );
ab7cf52011-05-24Henrik Grubbström (Grubba) } static void amd64_pop_mark(void) {
719c3a2012-06-12Per Hedbor  amd64_add_mark_sp( -1 );
ab7cf52011-05-24Henrik Grubbström (Grubba) } static void amd64_push_string(int strno, int subtype) { amd64_load_fp_reg(); amd64_load_sp_reg();
719c3a2012-06-12Per Hedbor  mov_mem_reg(fp_reg, OFFSETOF(pike_frame, context), REG_RAX); mov_mem_reg(REG_RAX, OFFSETOF(inherit, prog), REG_RAX); mov_mem_reg(REG_RAX, OFFSETOF(program, strings), REG_RAX); mov_mem_reg(REG_RAX, strno * sizeof(struct pike_string *), REG_RAX); mov_imm_mem((subtype<<16) | PIKE_T_STRING, sp_reg, OFFSETOF(svalue, type)); mov_reg_mem(REG_RAX, sp_reg,(INT32)OFFSETOF(svalue, u.string)); add_imm_mem( 1, REG_RAX, OFFSETOF(pike_string, refs)); amd64_add_sp(1);
ab7cf52011-05-24Henrik Grubbström (Grubba) }
9030f62011-05-26Henrik Grubbström (Grubba) static void amd64_push_local_function(int fun) { amd64_load_fp_reg(); amd64_load_sp_reg();
719c3a2012-06-12Per Hedbor  mov_mem_reg(fp_reg, OFFSETOF(pike_frame, context), REG_RAX); mov_mem_reg(fp_reg, OFFSETOF(pike_frame, current_object),
9030f62011-05-26Henrik Grubbström (Grubba)  REG_RCX);
719c3a2012-06-12Per Hedbor  mov_mem32_reg(REG_RAX, OFFSETOF(inherit, identifier_level),
9030f62011-05-26Henrik Grubbström (Grubba)  REG_RAX);
719c3a2012-06-12Per Hedbor  mov_reg_mem(REG_RCX, sp_reg, OFFSETOF(svalue, u.object)); add_reg_imm(REG_RAX, fun); add_imm_mem( 1, REG_RCX,(INT32)OFFSETOF(object, refs)); shl_reg_imm(REG_RAX, 16); add_reg_imm(REG_RAX, PIKE_T_FUNCTION); mov_reg_mem(REG_RAX, sp_reg, OFFSETOF(svalue, type)); amd64_add_sp(1);
d1fa802011-05-09Henrik Grubbström (Grubba) }
fa07a92011-05-16Henrik Grubbström (Grubba) static void amd64_stack_error(void) { Pike_fatal("Stack error\n"); }
d1fa802011-05-09Henrik Grubbström (Grubba) void amd64_update_pc(void) { INT32 tmp = PIKE_PC, disp;
719c3a2012-06-12Per Hedbor  if(amd64_prev_stored_pc == - 1) { enum amd64_reg tmp_reg = REG_RAX;
54a26b2011-05-11Henrik Grubbström (Grubba)  amd64_load_fp_reg();
719c3a2012-06-12Per Hedbor  mov_rip_imm_reg(tmp - PIKE_PC, tmp_reg); mov_reg_mem(tmp_reg, fp_reg, OFFSETOF(pike_frame, pc));
7990392006-04-27Tor Edvardsson #ifdef PIKE_DEBUG if (a_flag >= 60)
719c3a2012-06-12Per Hedbor  fprintf (stderr, "pc %d update pc via lea\n", tmp);
7990392006-04-27Tor Edvardsson #endif
bf49e12012-06-15Henrik Grubbström (Grubba)  amd64_prev_stored_pc = tmp;
7990392006-04-27Tor Edvardsson  }
719c3a2012-06-12Per Hedbor  else if ((disp = tmp - amd64_prev_stored_pc)) {
7990392006-04-27Tor Edvardsson #ifdef PIKE_DEBUG if (a_flag >= 60) fprintf (stderr, "pc %d update pc relative: %d\n", tmp, disp); #endif
54a26b2011-05-11Henrik Grubbström (Grubba)  amd64_load_fp_reg();
719c3a2012-06-12Per Hedbor  add_imm_mem(disp, fp_reg, OFFSETOF (pike_frame, pc));
bf49e12012-06-15Henrik Grubbström (Grubba)  amd64_prev_stored_pc += disp;
7990392006-04-27Tor Edvardsson  }
719c3a2012-06-12Per Hedbor  else {
7990392006-04-27Tor Edvardsson #ifdef PIKE_DEBUG if (a_flag >= 60) fprintf (stderr, "pc %d update pc - already up-to-date\n", tmp); #endif
719c3a2012-06-12Per Hedbor  } #if 0
fa07a92011-05-16Henrik Grubbström (Grubba) #ifdef PIKE_DEBUG if (d_flag) { /* Check that the stack keeps being 16 byte aligned. */
719c3a2012-06-12Per Hedbor  mov_reg_reg(REG_RSP, REG_RAX); and_reg_imm(REG_RAX, 0x08);
fa07a92011-05-16Henrik Grubbström (Grubba)  AMD64_JE(0x09);
719c3a2012-06-12Per Hedbor  call_imm(amd64_stack_error);
fa07a92011-05-16Henrik Grubbström (Grubba)  } #endif
719c3a2012-06-12Per Hedbor #endif
7990392006-04-27Tor Edvardsson }
d1fa802011-05-09Henrik Grubbström (Grubba) static void maybe_update_pc(void) { static int last_prog_id=-1; static size_t last_num_linenumbers=-1;
719c3a2012-06-12Per Hedbor 
d1fa802011-05-09Henrik Grubbström (Grubba)  if(
7990392006-04-27Tor Edvardsson #ifdef PIKE_DEBUG /* Update the pc more often for the sake of the opcode level trace. */ d_flag || #endif (amd64_prev_stored_pc == -1) || last_prog_id != Pike_compiler->new_program->id || last_num_linenumbers != Pike_compiler->new_program->num_linenumbers ) { last_prog_id=Pike_compiler->new_program->id; last_num_linenumbers = Pike_compiler->new_program->num_linenumbers; UPDATE_PC(); } }
719c3a2012-06-12Per Hedbor static void sync_registers(int flags) { maybe_update_pc(); flush_dirty_regs(); if (flags & I_UPDATE_SP) sp_reg = 0; if (flags & I_UPDATE_M_SP) mark_sp_reg = 0; if (flags & I_UPDATE_FP) fp_reg = 0; } static void amd64_call_c_opcode(void *addr, int flags) { sync_registers(flags); call_imm( addr ); }
54a26b2011-05-11Henrik Grubbström (Grubba) #ifdef PIKE_DEBUG static void ins_debug_instr_prologue (PIKE_INSTR_T instr, INT32 arg1, INT32 arg2) { int flags = instrs[instr].flags; maybe_update_pc(); if (flags & I_HASARG2)
719c3a2012-06-12Per Hedbor  mov_imm_reg(arg2, ARG3_REG);
54a26b2011-05-11Henrik Grubbström (Grubba)  if (flags & I_HASARG)
719c3a2012-06-12Per Hedbor  mov_imm_reg(arg1, ARG2_REG); mov_imm_reg(instr, ARG1_REG);
54a26b2011-05-11Henrik Grubbström (Grubba)  if (flags & I_HASARG2) amd64_call_c_function (simple_debug_instr_prologue_2); else if (flags & I_HASARG) amd64_call_c_function (simple_debug_instr_prologue_1); else amd64_call_c_function (simple_debug_instr_prologue_0); } #else /* !PIKE_DEBUG */ #define ins_debug_instr_prologue(instr, arg1, arg2) #endif
719c3a2012-06-12Per Hedbor static void amd64_push_this_object( ) { amd64_load_fp_reg(); amd64_load_sp_reg(); mov_imm_mem( PIKE_T_OBJECT, sp_reg, OFFSETOF(svalue,type)); mov_mem_reg( fp_reg, OFFSETOF(pike_frame, current_object), REG_RAX ); mov_reg_mem( REG_RAX, sp_reg, OFFSETOF(svalue,u.object) ); add_mem32_imm( REG_RAX, (INT32)OFFSETOF(object, refs), 1); amd64_add_sp( 1 ); } void amd64_ins_branch_check_threads_etc() { LABELS();
6785ae2012-06-15Per Hedbor #if 1 if( branch_check_threads_update_etc == -1 ) { /* Create update + call to branch_checl_threds_etc */ jmp( &label_A ); mov_imm_mem32( 0, REG_RSP, 0); branch_check_threads_update_etc = PIKE_PC;
5e064b2012-06-15Henrik Grubbström (Grubba)  if( (unsigned long long)&fast_check_threads_counter < 0x7fffffffULL )
6785ae2012-06-15Per Hedbor  { /* Short pointer. */ clear_reg( REG_RAX ); add_mem32_imm( REG_RAX,
5e064b2012-06-15Henrik Grubbström (Grubba)  (int)(ptrdiff_t)&fast_check_threads_counter,
6785ae2012-06-15Per Hedbor  0x80 ); } else { mov_imm_reg( (long)&fast_check_threads_counter, REG_RAX); add_mem_imm( REG_RAX, 0, 0x80 ); }
5e064b2012-06-15Henrik Grubbström (Grubba)  mov_imm_reg( (ptrdiff_t)branch_check_threads_etc, REG_RAX );
6785ae2012-06-15Per Hedbor  jmp_reg(REG_RAX); /* ret in BCTE will return to desired point. */ } LABEL_A; /* Use C-stack for counter. We have padding added in entry */ add_mem8_imm( REG_RSP, 0, 1 ); jno( &label_B ); call_rel_imm32( branch_check_threads_update_etc-PIKE_PC ); LABEL_B; #else call_imm( &branch_check_threads_etc ); #endif
719c3a2012-06-12Per Hedbor }
54a26b2011-05-11Henrik Grubbström (Grubba) void amd64_init_interpreter_state(void) { instrs[F_CATCH - F_OFFSET].address = inter_return_opcode_F_CATCH; }
6785ae2012-06-15Per Hedbor static void amd64_return_from_function() { if( ret_for_func ) { jmp_rel_imm( ret_for_func - PIKE_PC ); } else { ret_for_func = PIKE_PC; pop(REG_RBX); /* Stack padding. */ pop(REG_RBX); pop(REG_R12); pop(REG_R13); pop(REG_R14); pop(REG_R15); pop(REG_RBP); ret(); } }
d1fa802011-05-09Henrik Grubbström (Grubba) void ins_f_byte(unsigned int b) {
54a26b2011-05-11Henrik Grubbström (Grubba)  int flags;
d1fa802011-05-09Henrik Grubbström (Grubba)  void *addr;
48f5972011-05-23Henrik Grubbström (Grubba)  INT32 rel_addr = 0;
719c3a2012-06-12Per Hedbor  LABELS();
d1fa802011-05-09Henrik Grubbström (Grubba)  b-=F_OFFSET;
7990392006-04-27Tor Edvardsson #ifdef PIKE_DEBUG
d1fa802011-05-09Henrik Grubbström (Grubba)  if(b>255) Pike_error("Instruction too big %d\n",b);
7990392006-04-27Tor Edvardsson #endif
d1fa802011-05-09Henrik Grubbström (Grubba)  maybe_update_pc();
54a26b2011-05-11Henrik Grubbström (Grubba)  flags = instrs[b].flags;
eb42a12011-05-11Henrik Grubbström (Grubba)  addr=instrs[b].address; switch(b + F_OFFSET) {
b505a72012-06-13Per Hedbor  case F_DUP: amd64_load_sp_reg();
6785ae2012-06-15Per Hedbor  ins_debug_instr_prologue(b, 0, 0);
b505a72012-06-13Per Hedbor  add_reg_imm_reg(sp_reg, -sizeof(struct svalue), REG_R10 ); amd64_push_svaluep( REG_R10 ); return;
6785ae2012-06-15Per Hedbor  case F_ADD_INTS: { amd64_load_sp_reg(); ins_debug_instr_prologue(b, 0, 0); mov_mem32_reg( sp_reg, -sizeof(struct svalue)*2, REG_RAX ); shl_reg_imm( REG_RAX, 8 ); mov_mem32_reg( sp_reg,-sizeof(struct svalue), REG_RBX ); /* and_reg_imm( REG_RBX, 0x1f );*/ add_reg_reg( REG_RAX, REG_RBX ); and_reg_imm( REG_RAX, 0x1f1f ); cmp_reg_imm( REG_RAX, (PIKE_T_INT<<8)|PIKE_T_INT ); jne( &label_A ); /* So. Both are actually integers. */ mov_mem_reg( sp_reg, -sizeof(struct svalue)+OFFSETOF(svalue,u.integer), REG_RAX ); add_reg_mem( REG_RAX, sp_reg, -sizeof(struct svalue)*2+OFFSETOF(svalue,u.integer) ); jo( &label_A ); amd64_add_sp( -1 ); mov_reg_mem( REG_RAX, sp_reg, -sizeof(struct svalue)+OFFSETOF(svalue,u.integer)); jmp( &label_B ); LABEL_A; /* Fallback version */ update_arg1( 2 ); amd64_call_c_opcode( f_add, I_UPDATE_SP ); amd64_load_sp_reg(); LABEL_B; } return;
b505a72012-06-13Per Hedbor  case F_SWAP:
6785ae2012-06-15Per Hedbor  /* pike_sp[-1] = pike_sp[-2] FIXME: Can be changed to use movlq (128-bit mov, sse2) */ amd64_load_sp_reg(); add_reg_imm_reg( sp_reg, -2*sizeof(struct svalue), REG_R10); mov_mem_reg( REG_R10, 0, REG_RAX ); mov_mem_reg( REG_R10, 8, REG_RCX ); mov_mem_reg( REG_R10,16, REG_R8 ); mov_mem_reg( REG_R10,24, REG_R9 ); /* load done. */ mov_reg_mem(REG_R8, REG_R10,0); mov_reg_mem(REG_R9, REG_R10,8); mov_reg_mem(REG_RAX, REG_R10,sizeof(struct svalue)); mov_reg_mem(REG_RCX, REG_R10,8+sizeof(struct svalue)); /* save done. */ return;
b505a72012-06-13Per Hedbor  case F_POP_VALUE:
6785ae2012-06-15Per Hedbor  { ins_debug_instr_prologue(b, 0, 0); amd64_load_sp_reg(); amd64_add_sp( -1 ); amd64_free_svalue( sp_reg, 0 ); } return;
eb42a12011-05-11Henrik Grubbström (Grubba)  case F_CATCH:
21b9112011-05-23Henrik Grubbström (Grubba)  { /* Special argument for the F_CATCH instruction. */ addr = inter_return_opcode_F_CATCH;
719c3a2012-06-12Per Hedbor  mov_rip_imm_reg(0, ARG1_REG); /* Address for the POINTER. */
48f5972011-05-23Henrik Grubbström (Grubba)  rel_addr = PIKE_PC;
21b9112011-05-23Henrik Grubbström (Grubba)  }
eb42a12011-05-11Henrik Grubbström (Grubba)  break; case F_UNDEFINED: ins_debug_instr_prologue(b, 0, 0); amd64_push_int(0, 1); return; case F_CONST0: ins_debug_instr_prologue(b, 0, 0); amd64_push_int(0, 0); return; case F_CONST1: ins_debug_instr_prologue(b, 0, 0); amd64_push_int(1, 0); return; case F_CONST_1: ins_debug_instr_prologue(b, 0, 0); amd64_push_int(-1, 0); return; case F_BIGNUM: ins_debug_instr_prologue(b, 0, 0); amd64_push_int(0x7fffffff, 0); return; case F_RETURN_1: ins_f_byte(F_CONST1); ins_f_byte(F_RETURN); return; case F_RETURN_0: ins_f_byte(F_CONST0); ins_f_byte(F_RETURN); return; case F_ADD: ins_debug_instr_prologue(b, 0, 0); update_arg1(2); addr = f_add; break;
ab7cf52011-05-24Henrik Grubbström (Grubba)  case F_MARK: case F_SYNCH_MARK: ins_debug_instr_prologue(b, 0, 0); amd64_mark(0); return; case F_MARK2: ins_f_byte(F_MARK); ins_f_byte(F_MARK); return;
91080b2012-06-15Henrik Grubbström (Grubba)  case F_MARK_AND_CONST0: ins_f_byte(F_MARK); ins_f_byte(F_CONST0); return; case F_MARK_AND_CONST1: ins_f_byte(F_MARK); ins_f_byte(F_CONST1); return;
ab7cf52011-05-24Henrik Grubbström (Grubba)  case F_POP_MARK: ins_debug_instr_prologue(b, 0, 0); amd64_pop_mark(); return;
9853bd2012-06-13Henrik Grubbström (Grubba)  case F_POP_TO_MARK: ins_debug_instr_prologue(b, 0, 0); amd64_load_mark_sp_reg(); amd64_load_sp_reg(); amd64_pop_mark(); mov_mem_reg(mark_sp_reg, 0, REG_RBX); jmp(&label_A); LABEL_B; amd64_add_sp( -1 ); amd64_free_svalue( sp_reg, 0 ); LABEL_A; cmp_reg_reg(REG_RBX, sp_reg); jl(&label_B); return;
6785ae2012-06-15Per Hedbor #ifndef PIKE_DEBUG /* If we are compiling with debug, F_RETURN does extra checks */ case F_RETURN: #endif case F_DUMB_RETURN: { LABELS(); amd64_load_fp_reg(); /* Note: really mem16, but we & with PIKE_FRAME_RETURN_INTERNAL anyway */ mov_mem32_reg( fp_reg, OFFSETOF(pike_frame, flags), REG_RAX ); and_reg_imm( REG_RAX, PIKE_FRAME_RETURN_INTERNAL); jnz( &label_A ); /* So, it is just a normal return. */ LABEL_B; /* Actually return */ flush_dirty_regs(); amd64_return_from_function(); LABEL_A; amd64_call_c_opcode(addr,flags); #if 0 /* Can this happen? It seems to work without it, and from the code it looks like it should never happen, so.. */ cmp_reg_imm(REG_RAX, -1); je(&label_B); #endif jmp_reg(REG_RAX); } return;
91080b2012-06-15Henrik Grubbström (Grubba)  case F_CLEAR_STRING_SUBTYPE: ins_debug_instr_prologue(b, 0, 0); amd64_load_sp_reg(); mov_mem32_reg(sp_reg, OFFSETOF(svalue, type) - sizeof(struct svalue), REG_RAX); /* NB: We only care about subtype 1! */ cmp_reg_imm(REG_RAX, (1<<16)|PIKE_T_STRING); jne(&label_A); and_reg_imm(REG_RAX, 0x1f); mov_reg_mem32(REG_RAX, sp_reg, OFFSETOF(svalue, type) - sizeof(struct svalue)); LABEL_A; return;
eb42a12011-05-11Henrik Grubbström (Grubba)  }
719c3a2012-06-12Per Hedbor  amd64_call_c_opcode(addr,flags);
54a26b2011-05-11Henrik Grubbström (Grubba)  if (instrs[b].flags & I_RETURN) {
719c3a2012-06-12Per Hedbor  LABELS();
54a26b2011-05-11Henrik Grubbström (Grubba)  if ((b + F_OFFSET) == F_RETURN_IF_TRUE) { /* Kludge. We must check if the ret addr is
719c3a2012-06-12Per Hedbor  * PC + JUMP_EPILOGUE_SIZE. */ mov_rip_imm_reg(JUMP_EPILOGUE_SIZE, REG_RCX);
54a26b2011-05-11Henrik Grubbström (Grubba)  }
719c3a2012-06-12Per Hedbor  cmp_reg_imm(REG_RAX, -1);
6785ae2012-06-15Per Hedbor  jne(&label_A); amd64_return_from_function();
719c3a2012-06-12Per Hedbor  LABEL_A;
54a26b2011-05-11Henrik Grubbström (Grubba)  if ((b + F_OFFSET) == F_RETURN_IF_TRUE) { /* Kludge. We must check if the ret addr is * orig_addr + JUMP_EPILOGUE_SIZE. */
719c3a2012-06-12Per Hedbor  cmp_reg_reg( REG_RAX, REG_RCX );
15e8a02012-06-13Henrik Grubbström (Grubba)  je( &label_B );
719c3a2012-06-12Per Hedbor  jmp_reg(REG_RAX);
15e8a02012-06-13Henrik Grubbström (Grubba)  LABEL_B;
54a26b2011-05-11Henrik Grubbström (Grubba)  return; } }
f784252011-05-11Henrik Grubbström (Grubba)  if (flags & I_JUMP) {
719c3a2012-06-12Per Hedbor  jmp_reg(REG_RAX);
48f5972011-05-23Henrik Grubbström (Grubba)  if (b + F_OFFSET == F_CATCH) { upd_pointer(rel_addr - 4, PIKE_PC - rel_addr); }
d1fa802011-05-09Henrik Grubbström (Grubba)  }
7990392006-04-27Tor Edvardsson }
54a26b2011-05-11Henrik Grubbström (Grubba) int amd64_ins_f_jump(unsigned int op, int backward_jump) { int flags; void *addr; int off = op - F_OFFSET; int ret = -1;
b505a72012-06-13Per Hedbor  LABELS();
54a26b2011-05-11Henrik Grubbström (Grubba) #ifdef PIKE_DEBUG if(off>255) Pike_error("Instruction too big %d\n",off); #endif flags = instrs[off].flags; if (!(flags & I_BRANCH)) return -1;
6785ae2012-06-15Per Hedbor #define START_JUMP() do{ \ ins_debug_instr_prologue(off, 0, 0); \ if (backward_jump) { \ maybe_update_pc(); \ amd64_ins_branch_check_threads_etc(); \ } \ } while(0)
b505a72012-06-13Per Hedbor  switch( op ) {
99c1e92012-06-20Per Hedbor  case F_QUICK_BRANCH_WHEN_ZERO: case F_QUICK_BRANCH_WHEN_NON_ZERO: START_JUMP(); amd64_load_sp_reg(); amd64_add_sp( -1 ); mov_mem_reg( sp_reg, 8, REG_RAX ); test_reg(REG_RAX); if( op == F_QUICK_BRANCH_WHEN_ZERO ) return jz_imm_rel32(0); return jnz_imm_rel32(0); case F_BRANCH_WHEN_ZERO: case F_BRANCH_WHEN_NON_ZERO: START_JUMP(); amd64_load_sp_reg(); mov_mem16_reg( sp_reg, -sizeof(struct svalue), REG_RAX ); cmp_reg_imm( REG_RAX, PIKE_T_OBJECT ); je( &label_A ); amd64_add_sp( -1 ); mov_mem_reg( sp_reg, 8, REG_RBX ); amd64_free_svalue_type( sp_reg, REG_RAX, 0 ); test_reg( REG_RBX ); jmp( &label_B ); LABEL_A; /* It is an object. Use the C version. */ amd64_call_c_opcode(instrs[F_BRANCH_WHEN_ZERO-F_OFFSET].address, instrs[F_BRANCH_WHEN_ZERO-F_OFFSET].flags ); cmp_reg_imm( REG_RAX, -1 ); LABEL_B; /* Branch or not? */ if( op == F_BRANCH_WHEN_ZERO ) return jz_imm_rel32(0); return jnz_imm_rel32(0);
b505a72012-06-13Per Hedbor  case F_LOOP:
6785ae2012-06-15Per Hedbor  /* counter in pike_sp-1 */ /* decrement until 0. */ /* if not 0, branch */ /* otherwise, pop */ START_JUMP(); amd64_load_sp_reg(); mov_mem32_reg( sp_reg, -sizeof(struct svalue), REG_RAX ); /* Is it a normal integer? subtype -> 0, type -> PIKE_T_INT */ cmp_reg_imm( REG_RAX, PIKE_T_INT ); jne( &label_A ); /* if it is, is it 0? */ mov_mem_reg( sp_reg, -sizeof(struct svalue)+8, REG_RAX ); test_reg(REG_RAX); jz( &label_B ); /* it is. */ add_reg_imm( REG_RAX, -1 ); mov_reg_mem( REG_RAX, sp_reg, -sizeof(struct svalue)+8); mov_imm_reg( 1, REG_RAX ); /* decremented. Jump -> true. */ jmp( &label_C );
b505a72012-06-13Per Hedbor  LABEL_A; /* Not an integer. */
6785ae2012-06-15Per Hedbor  amd64_call_c_opcode(instrs[F_LOOP-F_OFFSET].address, instrs[F_LOOP-F_OFFSET].flags ); jmp( &label_C );
b505a72012-06-13Per Hedbor 
6785ae2012-06-15Per Hedbor  /* result in RAX */
b505a72012-06-13Per Hedbor  LABEL_B; /* loop done, inline. Known to be int, and 0 */
6785ae2012-06-15Per Hedbor  amd64_add_sp( -1 ); mov_imm_reg(0, REG_RAX );
b505a72012-06-13Per Hedbor  LABEL_C; /* Branch or not? */
6785ae2012-06-15Per Hedbor  test_reg( REG_RAX ); return jnz_imm_rel32(0); case F_BRANCH_WHEN_EQ: /* sp[-2] != sp[-1] */ case F_BRANCH_WHEN_NE: /* sp[-2] != sp[-1] */ /* START_JUMP();*/ amd64_load_sp_reg(); mov_mem16_reg( sp_reg, -sizeof(struct svalue), REG_RAX ); mov_mem16_reg( sp_reg, -sizeof(struct svalue)*2,REG_RBX ); cmp_reg_reg( REG_RAX, REG_RBX ); jnz( &label_A ); /* Types differ */ cmp_reg_imm( REG_RAX, PIKE_T_OBJECT ); je( &label_A ); /* Do not even bother with objects.. */ mov_mem_reg( sp_reg, -sizeof(struct svalue)+8, REG_RBX ); sub_reg_mem( REG_RBX, sp_reg, -sizeof(struct svalue)*2+8); /* RBX will now be 0 if they are equal.*/ /* Optimization: The types are equal, pop_stack can be greatly * simplified if they are < max_reg_type */ cmp_reg_imm( REG_RAX,MAX_REF_TYPE+1); jl( &label_B ); /* cheap pop. We know that both are >= max_ref_type */ amd64_add_sp( -2 ); jmp( &label_D ); LABEL_A; /* Fallback - call opcode. */ amd64_call_c_opcode( instrs[F_BRANCH_WHEN_NE-F_OFFSET].address, instrs[F_BRANCH_WHEN_NE-F_OFFSET].flags ); amd64_load_sp_reg(); /* Opcode returns 0 if equal, -1 if not. */ mov_reg_reg(REG_RAX, REG_RBX); jmp(&label_D); LABEL_B; /* comparison done, pop stack x2, reftypes */ amd64_add_sp( -2 ); mov_mem_reg( sp_reg, OFFSETOF(svalue,u.refs), REG_RAX ); add_mem32_imm( REG_RAX, OFFSETOF(pike_string,refs), -1 ); jnz( &label_C ); /* Free sp_reg */ mov_reg_reg( sp_reg, ARG1_REG ); amd64_call_c_function( really_free_svalue ); LABEL_C; add_reg_imm_reg( sp_reg, sizeof(struct svalue), ARG1_REG ); mov_mem_reg( ARG1_REG, OFFSETOF(svalue,u.refs), REG_RAX ); add_mem32_imm( REG_RAX, OFFSETOF(pike_string,refs), -1 ); jnz( &label_D ); amd64_call_c_function( really_free_svalue ); /* free sp[-2] */ LABEL_D; test_reg(REG_RBX); if( op == F_BRANCH_WHEN_EQ ) return jz_imm_rel32(0); return jnz_imm_rel32(0); #if 0 case F_BRANCH_WHEN_LT: /* sp[-2] < sp[-1] */ case F_BRANCH_WHEN_GE: /* sp[-2] >= sp[-1] */ case F_BRANCH_WHEN_GT: /* sp[-2] > sp[-1] */ case F_BRANCH_WHEN_LE: /* sp[-2] <= sp[-1] */ #endif case F_BRANCH: START_JUMP(); add_to_program(0xe9); ret=DO_NOT_WARN( (INT32) PIKE_PC ); PUSH_INT(0); return ret;
54a26b2011-05-11Henrik Grubbström (Grubba)  }
719c3a2012-06-12Per Hedbor  maybe_update_pc();
54a26b2011-05-11Henrik Grubbström (Grubba)  addr=instrs[off].address;
719c3a2012-06-12Per Hedbor  amd64_call_c_opcode(addr, flags);
6785ae2012-06-15Per Hedbor  amd64_load_sp_reg();
719c3a2012-06-12Per Hedbor  test_reg(REG_RAX);
54a26b2011-05-11Henrik Grubbström (Grubba)  if (backward_jump) {
48f5972011-05-23Henrik Grubbström (Grubba)  INT32 skip;
54a26b2011-05-11Henrik Grubbström (Grubba)  add_to_program (0x74); /* jz rel8 */
48f5972011-05-23Henrik Grubbström (Grubba)  add_to_program (0); /* Bytes to skip. */ skip = (INT32)PIKE_PC;
719c3a2012-06-12Per Hedbor  amd64_ins_branch_check_threads_etc(); /* amd64_call_c_function (branch_check_threads_etc); */
54a26b2011-05-11Henrik Grubbström (Grubba)  add_to_program (0xe9); /* jmp rel32 */ ret = DO_NOT_WARN ((INT32) PIKE_PC);
48f5972011-05-23Henrik Grubbström (Grubba)  PUSH_INT (0); /* Adjust the skip for the relative jump. */ Pike_compiler->new_program->program[skip-1] = ((INT32)PIKE_PC - skip);
54a26b2011-05-11Henrik Grubbström (Grubba)  } else { add_to_program (0x0f); /* jnz rel32 */ add_to_program (0x85); ret = DO_NOT_WARN ((INT32) PIKE_PC); PUSH_INT (0); } return ret; }
d1fa802011-05-09Henrik Grubbström (Grubba) void ins_f_byte_with_arg(unsigned int a, INT32 b) { maybe_update_pc();
eb42a12011-05-11Henrik Grubbström (Grubba)  switch(a) {
719c3a2012-06-12Per Hedbor  case F_THIS_OBJECT: if( b == 0 ) { amd64_push_this_object(); return; }
1c1a242012-06-13Henrik Grubbström (Grubba)  break; /* Fallback to C-version. */
6785ae2012-06-15Per Hedbor  case F_ADD_NEG_INT: b = -b; case F_ADD_INT: { LABELS(); amd64_load_sp_reg(); mov_mem16_reg( sp_reg, -sizeof(struct svalue), REG_RAX ); cmp_reg_imm( REG_RAX,PIKE_T_INT ); jne( &label_A ); mov_mem_reg(sp_reg, -sizeof(struct svalue)+OFFSETOF(svalue,u.integer), REG_RAX ); add_reg_imm( REG_RAX, b ); jo( &label_A ); /* if overflow, use f_add */ mov_reg_mem( REG_RAX,sp_reg, -sizeof(struct svalue)+OFFSETOF(svalue,u.integer)); jmp(&label_B); /* all done. */ LABEL_A; amd64_push_int(b,0); update_arg1(2); amd64_call_c_opcode( f_add, I_UPDATE_SP ); amd64_load_sp_reg(); LABEL_B; } return;
eb42a12011-05-11Henrik Grubbström (Grubba)  case F_NUMBER:
ab7cf52011-05-24Henrik Grubbström (Grubba)  ins_debug_instr_prologue(a-F_OFFSET, b, 0);
eb42a12011-05-11Henrik Grubbström (Grubba)  amd64_push_int(b, 0); return; case F_NEG_NUMBER:
ab7cf52011-05-24Henrik Grubbström (Grubba)  ins_debug_instr_prologue(a-F_OFFSET, b, 0);
eb42a12011-05-11Henrik Grubbström (Grubba)  amd64_push_int(-(INT64)b, 0); return;
ab7cf52011-05-24Henrik Grubbström (Grubba)  case F_STRING: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_push_string(b, 0); return; case F_ARROW_STRING: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_push_string(b, 1); return;
eb42a12011-05-11Henrik Grubbström (Grubba)  case F_POS_INT_INDEX: ins_f_byte_with_arg(F_NUMBER, b); ins_f_byte(F_INDEX); return; case F_NEG_INT_INDEX: ins_f_byte_with_arg(F_NEG_NUMBER, b); ins_f_byte(F_INDEX); return;
ab7cf52011-05-24Henrik Grubbström (Grubba)  case F_MARK_AND_CONST0: ins_f_byte(F_MARK); ins_f_byte(F_CONST0); return; case F_MARK_AND_CONST1: ins_f_byte(F_MARK); ins_f_byte(F_CONST0); return; case F_MARK_AND_STRING: ins_f_byte(F_MARK); ins_f_byte_with_arg(F_STRING, b); return; case F_MARK_AND_GLOBAL: ins_f_byte(F_MARK); ins_f_byte_with_arg(F_GLOBAL, b); return; case F_MARK_AND_LOCAL: ins_f_byte(F_MARK); ins_f_byte_with_arg(F_LOCAL, b); return; case F_MARK_X: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_mark(b); return;
9030f62011-05-26Henrik Grubbström (Grubba)  case F_LFUN: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_push_local_function(b); return;
719c3a2012-06-12Per Hedbor  case F_ASSIGN_LOCAL: ins_debug_instr_prologue(a-F_OFFSET, b, 0);
fd1b5a2012-06-19Henrik Grubbström (Grubba)  amd64_load_sp_reg();
719c3a2012-06-12Per Hedbor  amd64_assign_local(b); add_reg_imm_reg(sp_reg, -sizeof(struct svalue), ARG1_REG);
b505a72012-06-13Per Hedbor  amd64_ref_svalue(ARG1_REG, 0);
719c3a2012-06-12Per Hedbor  return; case F_ASSIGN_LOCAL_AND_POP: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_assign_local(b); amd64_add_sp(-1); return; case F_ASSIGN_GLOBAL: case F_ASSIGN_GLOBAL_AND_POP: /* arg1: pike_fp->current obj arg2: arg1+idenfier level arg3: Pike_sp-1 */ /* NOTE: We cannot simply do the same optimization as for ASSIGN_LOCAL_AND_POP with assign global, since assigning at times does not add references. We do know, however, that refs (should) never reach 0 when poping the stack. We can thus skip that part of pop_value */ ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg(); amd64_load_sp_reg(); mov_mem_reg(fp_reg, OFFSETOF(pike_frame, current_object), ARG1_REG); mov_mem_reg(fp_reg, OFFSETOF(pike_frame,context), ARG2_REG); mov_mem16_reg(ARG2_REG, OFFSETOF(inherit, identifier_level), ARG2_REG); add_reg_imm( ARG2_REG, b ); add_reg_imm_reg( sp_reg, -sizeof(struct svalue), ARG3_REG ); amd64_call_c_function( object_low_set_index ); if( a == F_ASSIGN_GLOBAL_AND_POP ) { /* assign done, pop. */ amd64_add_sp( -1 ); amd64_free_svalue( sp_reg, 1 ); } return; case F_SIZEOF_LOCAL: { LABELS();
5c82f42012-06-13Henrik Grubbström (Grubba)  ins_debug_instr_prologue(a-F_OFFSET, b, 0);
719c3a2012-06-12Per Hedbor  amd64_load_fp_reg(); amd64_load_sp_reg(); mov_mem_reg( fp_reg, OFFSETOF(pike_frame,locals), ARG1_REG); add_reg_imm( ARG1_REG, b*sizeof(struct svalue)); #if 0 mov_sval_type( ARG1_REG, REG_RAX ); /* type in RAX, svalue in ARG1 */ cmp_reg_imm( REG_RAX, PIKE_T_ARRAY ); jne( &label_A ); /* It's an array */ /* move arg to point to the array */ mov_mem_reg( ARG1_REG, OFFSETOF(svalue, u.array ), ARG1_REG); /* load size -> RAX*/ mov_mem32_reg( ARG1_REG,OFFSETOF(array, size), REG_RAX ); jmp( &label_C ); LABEL_A; cmp_reg_imm( REG_RAX, PIKE_T_STRING ); jne( &label_B ); /* It's a string */ /* move arg to point to the string */ mov_mem_reg( ARG1_REG, OFFSETOF(svalue, u.string ), ARG1_REG); /* load size ->RAX*/ mov_mem32_reg( ARG1_REG,OFFSETOF(pike_string, len ), REG_RAX ); jmp( &label_C ); LABEL_B; #endif /* It's something else, svalue already in ARG1. */ amd64_call_c_function( pike_sizeof ); amd64_load_sp_reg(); LABEL_C;/* all done, res in RAX */ /* Store result on stack */ amd64_push_int_reg( REG_RAX ); } return;
9030f62011-05-26Henrik Grubbström (Grubba)  case F_GLOBAL: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg(); amd64_load_sp_reg();
719c3a2012-06-12Per Hedbor  mov_mem_reg(fp_reg, OFFSETOF(pike_frame, context), ARG3_REG); mov_mem_reg(fp_reg, OFFSETOF(pike_frame, current_object),
9030f62011-05-26Henrik Grubbström (Grubba)  ARG2_REG);
719c3a2012-06-12Per Hedbor  mov_reg_reg(sp_reg, ARG1_REG); mov_mem32_reg(ARG3_REG, OFFSETOF(inherit, identifier_level), ARG3_REG); and_reg_imm(ARG3_REG, 0xffff); add_reg_imm(ARG3_REG, b);
9030f62011-05-26Henrik Grubbström (Grubba)  flush_dirty_regs(); /* In case an error is thrown. */
719c3a2012-06-12Per Hedbor  call_imm(low_object_index_no_free);
9030f62011-05-26Henrik Grubbström (Grubba)  /* NB: We know that low_object_index_no_free() doesn't * mess with the stack pointer. */
719c3a2012-06-12Per Hedbor  amd64_add_sp(1);
9030f62011-05-26Henrik Grubbström (Grubba)  return;
719c3a2012-06-12Per Hedbor 
9030f62011-05-26Henrik Grubbström (Grubba)  case F_LOCAL: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg(); amd64_load_sp_reg();
719c3a2012-06-12Per Hedbor  mov_mem_reg(fp_reg, OFFSETOF(pike_frame, locals), REG_RCX); add_reg_imm(REG_RCX, b*sizeof(struct svalue));
9030f62011-05-26Henrik Grubbström (Grubba)  amd64_push_svaluep(REG_RCX); return;
719c3a2012-06-12Per Hedbor 
91080b2012-06-15Henrik Grubbström (Grubba)  case F_CLEAR_LOCAL: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg(); mov_mem_reg(fp_reg, OFFSETOF(pike_frame, locals), REG_RBX); add_reg_imm(REG_RBX, b*sizeof(struct svalue)); amd64_free_svalue(REG_RBX, 0); mov_imm_mem(0, REG_RBX, OFFSETOF(svalue, u.integer)); mov_imm_mem32(PIKE_T_INT, REG_RBX, OFFSETOF(svalue, type)); return; case F_INC_LOCAL_AND_POP: { LABELS(); ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg(); mov_mem_reg(fp_reg, OFFSETOF(pike_frame, locals), REG_RCX); add_reg_imm(REG_RCX, b*sizeof(struct svalue)); mov_sval_type(REG_RCX, REG_RAX); cmp_reg_imm(REG_RAX, PIKE_T_INT); jne(&label_A); /* Integer - Zap subtype and try just incrementing it. */ mov_reg_mem32(REG_RAX, REG_RCX, OFFSETOF(svalue, type)); add_imm_mem(1, REG_RCX, OFFSETOF(svalue, u.integer));
ec96912012-06-17Henrik Grubbström (Grubba)  jno(&label_B);
91080b2012-06-15Henrik Grubbström (Grubba)  add_imm_mem(-1, REG_RCX, OFFSETOF(svalue, u.integer)); LABEL_A; /* Fallback to the C-implementation. */ update_arg1(b); amd64_call_c_opcode(instrs[a-F_OFFSET].address, instrs[a-F_OFFSET].flags); LABEL_B; } return;
b669592012-06-19Henrik Grubbström (Grubba)  case F_INC_LOCAL: ins_f_byte_with_arg(F_INC_LOCAL_AND_POP, b); ins_f_byte_with_arg(F_LOCAL, b); return; case F_POST_INC_LOCAL: ins_f_byte_with_arg(F_LOCAL, b); ins_f_byte_with_arg(F_INC_LOCAL_AND_POP, b); return;
91080b2012-06-15Henrik Grubbström (Grubba)  case F_DEC_LOCAL_AND_POP: { LABELS(); ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg(); mov_mem_reg(fp_reg, OFFSETOF(pike_frame, locals), REG_RCX); add_reg_imm(REG_RCX, b*sizeof(struct svalue)); mov_sval_type(REG_RCX, REG_RAX); cmp_reg_imm(REG_RAX, PIKE_T_INT); jne(&label_A); /* Integer - Zap subtype and try just decrementing it. */
ec96912012-06-17Henrik Grubbström (Grubba)  mov_reg_mem32(REG_RAX, REG_RCX, OFFSETOF(svalue, type));
91080b2012-06-15Henrik Grubbström (Grubba)  add_imm_mem(-1, REG_RCX, OFFSETOF(svalue, u.integer));
ec96912012-06-17Henrik Grubbström (Grubba)  jno(&label_B);
91080b2012-06-15Henrik Grubbström (Grubba)  add_imm_mem(1, REG_RCX, OFFSETOF(svalue, u.integer)); LABEL_A; /* Fallback to the C-implementation. */ update_arg1(b); amd64_call_c_opcode(instrs[a-F_OFFSET].address, instrs[a-F_OFFSET].flags); LABEL_B; } return;
b669592012-06-19Henrik Grubbström (Grubba)  case F_DEC_LOCAL: ins_f_byte_with_arg(F_DEC_LOCAL_AND_POP, b); ins_f_byte_with_arg(F_LOCAL, b); return; case F_POST_DEC_LOCAL: ins_f_byte_with_arg(F_LOCAL, b); ins_f_byte_with_arg(F_DEC_LOCAL_AND_POP, b); return;
719c3a2012-06-12Per Hedbor  case F_CONSTANT: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg(); amd64_load_sp_reg(); mov_mem_reg( fp_reg, OFFSETOF(pike_frame,context), REG_RCX ); mov_mem_reg( REG_RCX, OFFSETOF(inherit,prog), REG_RCX ); mov_mem_reg( REG_RCX, OFFSETOF(program,constants), REG_RCX ); add_reg_imm( REG_RCX, b*sizeof(struct program_constant) + OFFSETOF(program_constant,sval) ); amd64_push_svaluep( REG_RCX ); return; case F_GLOBAL_LVALUE: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg(); amd64_load_sp_reg(); amd64_push_this_object( ); mov_imm_mem( T_OBJ_INDEX, sp_reg, OFFSETOF(svalue,type)); mov_mem_reg(fp_reg, OFFSETOF(pike_frame, context), REG_RAX); mov_mem16_reg( REG_RAX,OFFSETOF(inherit, identifier_level), REG_RAX); add_reg_imm( REG_RAX, b ); mov_reg_mem( REG_RAX, sp_reg, OFFSETOF(svalue,u.identifier) ); amd64_add_sp( 1 ); return; case F_LOCAL_LVALUE: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg(); amd64_load_sp_reg(); /* &frame->locals[b] */ mov_mem_reg( fp_reg, OFFSETOF(pike_frame, locals), REG_RAX); add_reg_imm( REG_RAX, b*sizeof(struct svalue)); mov_imm_mem( T_SVALUE_PTR, sp_reg, OFFSETOF(svalue,type)); mov_reg_mem( REG_RAX, sp_reg, OFFSETOF(svalue,u.lval) ); mov_imm_mem( T_VOID, sp_reg, OFFSETOF(svalue,type)+sizeof(struct svalue)); amd64_add_sp( 2 ); return;
5151e52012-06-10Henrik Grubbström (Grubba)  case F_PROTECT_STACK: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg();
719c3a2012-06-12Per Hedbor  mov_mem_reg(fp_reg, OFFSETOF(pike_frame, locals), ARG1_REG);
5151e52012-06-10Henrik Grubbström (Grubba)  if (b) {
719c3a2012-06-12Per Hedbor  add_reg_imm_reg(ARG1_REG, sizeof(struct svalue) * b, ARG1_REG);
5151e52012-06-10Henrik Grubbström (Grubba)  }
719c3a2012-06-12Per Hedbor  mov_reg_mem(ARG1_REG, fp_reg, OFFSETOF(pike_frame, expendible));
5151e52012-06-10Henrik Grubbström (Grubba)  return; case F_MARK_AT: ins_debug_instr_prologue(a-F_OFFSET, b, 0); amd64_load_fp_reg(); amd64_load_mark_sp_reg();
719c3a2012-06-12Per Hedbor  mov_mem_reg(fp_reg, OFFSETOF(pike_frame, locals), ARG1_REG);
5151e52012-06-10Henrik Grubbström (Grubba)  if (b) {
719c3a2012-06-12Per Hedbor  add_reg_imm_reg(ARG1_REG, sizeof(struct svalue) * b, ARG1_REG);
5151e52012-06-10Henrik Grubbström (Grubba)  }
719c3a2012-06-12Per Hedbor  mov_reg_mem(ARG1_REG, mark_sp_reg, 0x00); add_reg_imm(mark_sp_reg, sizeof(struct svalue *));
5151e52012-06-10Henrik Grubbström (Grubba)  dirty_regs |= 1 << mark_sp_reg; /* FIXME: Deferred writing of Pike_mark_sp doen't seem to work reliably yet. */ if (dirty_regs & (1 << PIKE_MARK_SP_REG)) {
719c3a2012-06-12Per Hedbor  mov_reg_mem(PIKE_MARK_SP_REG, Pike_interpreter_reg,
5151e52012-06-10Henrik Grubbström (Grubba)  OFFSETOF(Pike_interpreter_struct, mark_stack_pointer)); dirty_regs &= ~(1 << PIKE_MARK_SP_REG); } return;
eb42a12011-05-11Henrik Grubbström (Grubba)  }
d1fa802011-05-09Henrik Grubbström (Grubba)  update_arg1(b); ins_f_byte(a);
7990392006-04-27Tor Edvardsson }
54a26b2011-05-11Henrik Grubbström (Grubba) int amd64_ins_f_jump_with_arg(unsigned int op, INT32 a, int backward_jump) {
719c3a2012-06-12Per Hedbor  LABELS();
54a26b2011-05-11Henrik Grubbström (Grubba)  if (!(instrs[op - F_OFFSET].flags & I_BRANCH)) return -1;
719c3a2012-06-12Per Hedbor  switch( op ) { case F_BRANCH_IF_NOT_LOCAL: case F_BRANCH_IF_LOCAL: ins_debug_instr_prologue(op-F_OFFSET, a, 0); amd64_load_fp_reg(); mov_mem_reg( fp_reg, OFFSETOF(pike_frame, locals), REG_RAX); add_reg_imm( REG_RAX, a*sizeof(struct svalue)); /* if( type == PIKE_T_OBJECT ) call c version... else u.integer -> RAX */ mov_sval_type( REG_RAX, REG_RCX ); cmp_reg_imm( REG_RCX, PIKE_T_OBJECT ); jne( &label_A ); update_arg1(a); /* Note: Always call IF_LOCAL, the negation is done below. */ amd64_call_c_opcode( instrs[F_BRANCH_IF_LOCAL-F_OFFSET].address, instrs[F_BRANCH_IF_LOCAL-F_OFFSET].flags ); jmp( &label_B ); LABEL_A; mov_mem_reg( REG_RAX, OFFSETOF(svalue, u.integer ), REG_RAX ); LABEL_B; test_reg( REG_RAX ); if( op == F_BRANCH_IF_LOCAL ) return jnz_imm_rel32(0); return jz_imm_rel32(0); }
54a26b2011-05-11Henrik Grubbström (Grubba)  maybe_update_pc(); update_arg1(a); return amd64_ins_f_jump(op, backward_jump); }
d1fa802011-05-09Henrik Grubbström (Grubba) void ins_f_byte_with_2_args(unsigned int a, INT32 b, INT32 c) { maybe_update_pc();
eb42a12011-05-11Henrik Grubbström (Grubba)  switch(a) { case F_NUMBER64:
ab7cf52011-05-24Henrik Grubbström (Grubba)  ins_debug_instr_prologue(a-F_OFFSET, b, c);
eb42a12011-05-11Henrik Grubbström (Grubba)  amd64_push_int((((unsigned INT64)b)<<32)|(unsigned INT32)c, 0); return;
ab7cf52011-05-24Henrik Grubbström (Grubba)  case F_MARK_AND_EXTERNAL: ins_f_byte(F_MARK); ins_f_byte_with_2_args(F_EXTERNAL, b, c); return;
f4107d2012-06-18Per Hedbor  case F_ADD_LOCAL_INT_AND_POP: { LABELS(); ins_debug_instr_prologue(a-F_OFFSET, b, 0);
fd1b5a2012-06-19Henrik Grubbström (Grubba)  amd64_load_fp_reg();
f4107d2012-06-18Per Hedbor  mov_mem_reg( fp_reg, OFFSETOF(pike_frame, locals), ARG1_REG); add_reg_imm( ARG1_REG, b*sizeof(struct svalue) ); /* arg1 = dst arg2 = int */ mov_sval_type( ARG1_REG, REG_RAX ); cmp_reg_imm( REG_RAX, PIKE_T_INT ); jne(&label_A); /* Fallback */ add_imm_mem( c, ARG1_REG,OFFSETOF(svalue,u.integer)); jno( &label_B); add_imm_mem( -c, ARG1_REG,OFFSETOF(svalue,u.integer)); LABEL_A; update_arg2(c); update_arg1(b); ins_f_byte(a); /* Will call C version */ LABEL_B; return; } case F_ADD_LOCALS_AND_POP: { LABELS(); ins_debug_instr_prologue(a-F_OFFSET, b, 0);
fd1b5a2012-06-19Henrik Grubbström (Grubba)  amd64_load_fp_reg();
f4107d2012-06-18Per Hedbor  mov_mem_reg( fp_reg, OFFSETOF(pike_frame, locals), ARG1_REG); add_reg_imm( ARG1_REG, b*sizeof(struct svalue) ); add_reg_imm_reg( ARG1_REG,(c-b)*sizeof(struct svalue), ARG2_REG ); /* arg1 = dst arg2 = src */ mov_sval_type( ARG1_REG, REG_RAX ); mov_sval_type( ARG2_REG, REG_RBX ); shl_reg_imm( REG_RAX, 8 ); add_reg_reg( REG_RAX, REG_RBX ); cmp_reg_imm( REG_RAX, (PIKE_T_INT<<8) | PIKE_T_INT ); jne(&label_A); /* Fallback */ mov_mem_reg( ARG2_REG, OFFSETOF(svalue,u.integer), REG_RAX ); add_reg_mem( REG_RAX, ARG1_REG, OFFSETOF(svalue,u.integer)); jo( &label_A); /* Clear subtype */ mov_imm_mem( PIKE_T_INT, ARG1_REG,OFFSETOF(svalue,type)); mov_reg_mem( REG_RAX, ARG1_REG, OFFSETOF(svalue,u.integer)); jmp( &label_B ); LABEL_A; update_arg2(c); update_arg1(b); ins_f_byte(a); /* Will call C version */ LABEL_B; return; } case F_ASSIGN_LOCAL_NUMBER_AND_POP: ins_debug_instr_prologue(a-F_OFFSET, b, 0);
fd1b5a2012-06-19Henrik Grubbström (Grubba)  amd64_load_fp_reg();
f4107d2012-06-18Per Hedbor  mov_mem_reg( fp_reg, OFFSETOF(pike_frame, locals), ARG1_REG); add_reg_imm( ARG1_REG,b*sizeof(struct svalue) ); mov_reg_reg( ARG1_REG, REG_RBX ); amd64_free_svalue(ARG1_REG, 0); mov_imm_mem(c, REG_RBX, OFFSETOF(svalue, u.integer)); mov_imm_mem32(PIKE_T_INT, REG_RBX, OFFSETOF(svalue, type)); return;
b505a72012-06-13Per Hedbor  case F_LOCAL_2_LOCAL:
5c82f42012-06-13Henrik Grubbström (Grubba)  ins_debug_instr_prologue(a-F_OFFSET, b, c);
b505a72012-06-13Per Hedbor  if( b != c ) { amd64_load_fp_reg();
373ff52012-06-13Henrik Grubbström (Grubba)  mov_mem_reg( fp_reg, OFFSETOF(pike_frame, locals), REG_RBX ); add_reg_imm( REG_RBX, b*sizeof(struct svalue) );
b505a72012-06-13Per Hedbor  /* RBX points to dst. */ amd64_free_svalue( REG_RBX, 0 );
373ff52012-06-13Henrik Grubbström (Grubba)  /* assign rbx[0] = rbx[c-b] */
b505a72012-06-13Per Hedbor  mov_mem_reg( REG_RBX, (c-b)*sizeof(struct svalue), REG_RAX ); mov_mem_reg( REG_RBX, (c-b)*sizeof(struct svalue)+8, REG_RCX ); mov_reg_mem( REG_RAX, REG_RBX, 0 ); mov_reg_mem( REG_RCX, REG_RBX, 8 ); amd64_ref_svalue( REG_RBX, 1 ); } return;
9030f62011-05-26Henrik Grubbström (Grubba)  case F_2_LOCALS:
719c3a2012-06-12Per Hedbor #if 1
5c82f42012-06-13Henrik Grubbström (Grubba)  ins_debug_instr_prologue(a-F_OFFSET, b, c);
719c3a2012-06-12Per Hedbor  amd64_load_fp_reg(); amd64_load_sp_reg(); mov_mem_reg(fp_reg, OFFSETOF(pike_frame, locals), REG_R8); add_reg_imm( REG_R8, b*sizeof(struct svalue) ); amd64_push_svaluep(REG_R8); add_reg_imm( REG_R8, (c-b)*sizeof(struct svalue) ); amd64_push_svaluep(REG_R8); #else ins_f_byte_with_arg( F_LOCAL, b ); ins_f_byte_with_arg( F_LOCAL, c ); #endif
9030f62011-05-26Henrik Grubbström (Grubba)  return;
719c3a2012-06-12Per Hedbor 
9853bd2012-06-13Henrik Grubbström (Grubba)  case F_FILL_STACK: { LABELS(); if (!b) return; ins_debug_instr_prologue(a-F_OFFSET, b, c); amd64_load_fp_reg(); amd64_load_sp_reg(); mov_mem_reg(fp_reg, OFFSETOF(pike_frame, locals), ARG1_REG); add_reg_imm(ARG1_REG, b*sizeof(struct svalue)); jmp(&label_A); LABEL_B; amd64_push_int(0, c); LABEL_A; cmp_reg_reg(sp_reg, ARG1_REG); jge(&label_B); } return;
5151e52012-06-10Henrik Grubbström (Grubba)  case F_INIT_FRAME: ins_debug_instr_prologue(a-F_OFFSET, b, c); amd64_load_fp_reg();
719c3a2012-06-12Per Hedbor  if(OFFSETOF(pike_frame, num_locals) != OFFSETOF(pike_frame, num_args)-2 ) Pike_fatal("This code does not with unless num_args\n" "directly follows num_locals in struct pike_frame\n"); mov_imm_mem32( (b<<16)|c, fp_reg, OFFSETOF(pike_frame, num_locals));
5151e52012-06-10Henrik Grubbström (Grubba)  return;
eb42a12011-05-11Henrik Grubbström (Grubba)  }
d1fa802011-05-09Henrik Grubbström (Grubba)  update_arg2(c); update_arg1(b); ins_f_byte(a);
7990392006-04-27Tor Edvardsson }
54a26b2011-05-11Henrik Grubbström (Grubba) int amd64_ins_f_jump_with_2_args(unsigned int op, INT32 a, INT32 b, int backward_jump) { if (!(instrs[op - F_OFFSET].flags & I_BRANCH)) return -1; maybe_update_pc(); update_arg2(b); update_arg1(a); return amd64_ins_f_jump(op, backward_jump); } void amd64_update_f_jump(INT32 offset, INT32 to_offset) { upd_pointer(offset, to_offset - offset - 4); } INT32 amd64_read_f_jump(INT32 offset) { return read_pointer(offset) + offset + 4; }