diff --git a/rtl/x86_fpdbgdisas.pp b/rtl/x86_fpdbgdisas.pp index 36e6cb6d..7a7d86a0 100644 --- a/rtl/x86_fpdbgdisas.pp +++ b/rtl/x86_fpdbgdisas.pp @@ -349,6 +349,7 @@ type CodeIdx: Byte; OperIdx: Integer; ModRMIdx: Byte; + mm: Byte; Flags: TFlags; SimdOpcode: TSimdOpcode; ModRM: record @@ -3538,13 +3539,16 @@ begin //--- $C0: begin DecodeSIMD([soNone]); - if SimdOpcode = soNone - then begin SetOpcode(OPxadd); AddEb; AddGb; CheckLock; end; + case SimdOpcode of + soNone: begin SetOpcode(OPxadd); AddEb; AddGb; CheckLock; end; + end; end; $C1: begin DecodeSIMD([soNone]); - if SimdOpcode = soNone - then begin SetOpcode(OPxadd); AddEv; AddGv; CheckLock; end; + case SimdOpcode of + soNone: begin SetOpcode(OPxadd); AddEv; AddGv; CheckLock; end; + so66: begin SetOpcode(OPxadd); AddEw; AddGw; CheckLock; end; + end; end; $C2: begin DecodeSIMD; @@ -3929,7 +3933,7 @@ const SIMDMAP: array[0..3] of TSimdOpcode = (soNone, so66, soF3, soF2); LENMAP: array[0..3] of TOperandSize = (os128, os256, os512, os0); var - idx, mm: Byte; + idx: Byte; begin Assert(ASize in [2..4], Format('Invalid VEX size: %u', [ASize])); @@ -4695,6 +4699,7 @@ begin OperIdx := 0; SimdOpcode := soInvalid; Vex.MaskIndex := 0; + mm:=0; Sib.Scale:=0; Sib.Index:=0; diff --git a/rtl/x86_jit.pas b/rtl/x86_jit.pas index 2c129323..db2d207b 100644 --- a/rtl/x86_jit.pas +++ b/rtl/x86_jit.pas @@ -88,8 +88,8 @@ type function is_valid:Boolean; function before:t_jit_i_link; function after :t_jit_i_link; - function prev(_before:boolean):t_jit_i_link; - function next(_before:boolean):t_jit_i_link; + function prev:t_jit_i_link; + function next:t_jit_i_link; property _node:p_jit_instruction read ALink; property _label:t_jit_i_link read get_label write set_label; end; @@ -99,6 +99,9 @@ const operator = (A,B:t_jit_i_link):Boolean; +//SimdOpcode (soNone=0, so66=1, soF3=2, soF2=3); +//mm (0F=1, 0F38=2, 0F3A=3) + type p_jit_builder=^t_jit_builder; t_jit_builder=object @@ -265,6 +268,7 @@ type function leaj(reg:TRegValue;mem:t_jit_regs;_label_id:t_jit_i_link):t_jit_i_link; // Procedure reta; + Procedure ud2; // Function GetInstructionsSize:Integer; Function GetDataSize:Integer; @@ -333,8 +337,8 @@ type procedure _VM (const desc:t_op_type;reg:TRegValue;mem:t_jit_regs;size:TOperandSize); procedure _VM_F3 (const desc:t_op_type;reg:TRegValue;mem:t_jit_regs;size:TOperandSize); procedure _VV_F3 (const desc:t_op_type;reg0,reg1:TRegValue;size:TOperandSize); - procedure _VVM (const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs); - procedure _VVMI8 (const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;imm8:Byte); + procedure _VVM (const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;size:TOperandSize); + procedure _VVMI8 (const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;size:TOperandSize;imm8:Byte); procedure _VVV (const desc:t_op_type;reg0,reg1,reg2:TRegValue); procedure _VVVI8 (const desc:t_op_type;reg0,reg1,reg2:TRegValue;imm8:Byte); procedure _VVI8 (const desc:t_op_type;reg0,reg1:TRegValue;imm8:Byte); @@ -649,6 +653,7 @@ begin begin Result.AType:=lnkLabelBefore; end; + else; end; end; @@ -661,48 +666,53 @@ begin begin Result.AType:=lnkLabelAfter; end; + else; end; end; -function t_jit_i_link.prev(_before:boolean):t_jit_i_link; +function t_jit_i_link.prev:t_jit_i_link; begin Result:=nil_link; case AType of lnkData: begin Result.AType:=lnkData; - Result.ALink:=TAILQ_PREV(ALink,@ALink^.link); + Result.ALink:=TAILQ_PREV(ALink,@p_jit_data(ALink)^.link); end; lnkLabelBefore, lnkLabelAfter: begin - if _before then - Result.AType:=lnkLabelBefore - else - Result.AType:=lnkLabelAfter; + Result.AType:=lnkLabelBefore; Result.ALink:=TAILQ_PREV(ALink,@ALink^.link); end; + else; + end; + if (Result.ALink=nil) then + begin + Result:=nil_link; end; end; -function t_jit_i_link.next(_before:boolean):t_jit_i_link; +function t_jit_i_link.next:t_jit_i_link; begin Result:=nil_link; case AType of lnkData: begin Result.AType:=lnkData; - Result.ALink:=TAILQ_NEXT(ALink,@ALink^.link); + Result.ALink:=TAILQ_NEXT(ALink,@p_jit_data(ALink)^.link); end; lnkLabelBefore, lnkLabelAfter: begin - if _before then - Result.AType:=lnkLabelBefore - else - Result.AType:=lnkLabelAfter; + Result.AType:=lnkLabelBefore; Result.ALink:=TAILQ_NEXT(ALink,@ALink^.link); end; + else; + end; + if (Result.ALink=nil) then + begin + Result:=nil_link; end; end; @@ -1139,6 +1149,19 @@ begin _add(ji); end; + +Procedure t_jit_builder.ud2; +var + ji:t_jit_instruction; +begin + ji:=default_jit_instruction; + + ji.EmitByte($0F); + ji.EmitByte($0B); + + _add(ji); +end; + Function t_jit_builder.GetInstructionsSize:Integer; begin Result:=AInstructionSize; @@ -3474,7 +3497,7 @@ begin _add(ji); end; -procedure t_jit_builder._VVM(const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs); +procedure t_jit_builder._VVM(const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;size:TOperandSize); var mreg:t_jit_reg; @@ -3518,10 +3541,17 @@ begin Vex.Index:=reg1.AIndex; - Vex.rexW:=False; - if (reg0.ASize=os64) then + if (size=os0) then + begin + size:=reg0.ASize; + end; + + if (size=os64) then begin Vex.rexW:=True; + end else + begin + Vex.rexW:=False; end; modrm_info:=Default(t_modrm_info); @@ -3563,7 +3593,7 @@ begin _add(ji); end; -procedure t_jit_builder._VVMI8(const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;imm8:Byte); +procedure t_jit_builder._VVMI8(const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;size:TOperandSize;imm8:Byte); var mreg:t_jit_reg; @@ -3607,10 +3637,17 @@ begin Vex.Index:=reg1.AIndex; - Vex.rexW:=False; - if (reg0.ASize=os64) then + if (size=os0) then + begin + size:=reg0.ASize; + end; + + if (size=os64) then begin Vex.rexW:=True; + end else + begin + Vex.rexW:=False; end; modrm_info:=Default(t_modrm_info); diff --git a/sys/kern/kern_exec.pas b/sys/kern/kern_exec.pas index 7a02894a..dbf036a4 100644 --- a/sys/kern/kern_exec.pas +++ b/sys/kern/kern_exec.pas @@ -937,12 +937,20 @@ var addr:Pointer; ST_TYPE:Integer; begin + if (obj=nil) then Exit; ctx:=Default(t_jit_context2); + ctx.text_start:=QWORD(obj^.map_base); + ctx.text___end:=ctx.text_start+obj^.text_size; + ctx.add_forward_point(obj^.entry_addr); - ctx.add_forward_point(obj^.init_proc_addr); - ctx.add_forward_point(obj^.fini_proc_addr); + + if (obj^.mainprog=0) then + begin + ctx.add_forward_point(obj^.init_proc_addr); + ctx.add_forward_point(obj^.fini_proc_addr); + end; lib_entry:=TAILQ_FIRST(@obj^.lib_table); while (lib_entry<>nil) do @@ -1020,6 +1028,8 @@ begin Writeln(StdErr,'preload_prx_modules:',str,' not loaded'); end; + pick_obj(obj); + obj:=TAILQ_FIRST(@dynlibs_info.obj_list); while (obj<>nil) do begin @@ -1055,6 +1065,8 @@ begin p_proc.libkernel___end_addr:=dynlibs_info.libkernel^.map_base + dynlibs_info.libkernel^.text_size; end; + //pick_obj(dynlibs_info.libprogram); + pick_obj(dynlibs_info.libkernel); _dyn_not_exist: diff --git a/sys/test/kern_jit2.pas b/sys/test/kern_jit2.pas index c78d96df..0e03581a 100644 --- a/sys/test/kern_jit2.pas +++ b/sys/test/kern_jit2.pas @@ -420,6 +420,12 @@ begin id:=PByte(ctx.code)[i]; case id of + $41: //assert? + begin + // + ctx.builder.call_far(nil); //TODO error dispatcher + end; + $44: //system error? begin // @@ -452,6 +458,12 @@ begin ctx.builder.call_far(nil); //TODO CPUID end; +procedure op_rdtsc(var ctx:t_jit_context2); +begin + add_orig(ctx); + op_save_rax(ctx,ctx.builder.rax); +end; + procedure op_nop(var ctx:t_jit_context2); begin //align? @@ -503,6 +515,7 @@ begin jit_cbs[OPPnone,OPiret,OPSx_q ]:=@op_iretq; jit_cbs[OPPnone,OPcpuid,OPSnone]:=@op_cpuid; + jit_cbs[OPPnone,OPrdtsc ,OPSnone]:=@op_rdtsc; jit_cbs[OPPnone,OPnop,OPSnone]:=@op_nop; @@ -510,6 +523,9 @@ begin end; procedure pick(var ctx:t_jit_context2); +const + SCODES:array[TSimdOpcode] of Byte=(0,0,1,3,2); + MCODES:array[0..3] of RawByteString=('','0F','0F38','0F3A'); label _next; var @@ -534,6 +550,9 @@ begin kern_jit2_ops.init_cbs; ctx.max:=QWORD(ctx.max_forward_point); + Writeln(' ctx.text_start:0x',HexStr(ctx.text_start,16)); + Writeln(' ctx.max :0x',HexStr(ctx.max,16)); + Writeln(' ctx.text___end:0x',HexStr(ctx.text___end,16)); links:=Default(t_jit_context2.t_forward_links); addr:=nil; @@ -567,6 +586,11 @@ begin begin //invalid writeln('invalid:0x',HexStr(ctx.ptr_curr)); + + ptr:=ctx.ptr_curr; + adec.Disassemble(ptr,ACodeBytes,ACode); + + goto _next; end; else; @@ -600,6 +624,9 @@ begin ctx.din.OpCode.Suffix,' ', ctx.din.Operand[1].Size,' ', ctx.din.Operand[2].Size); + Writeln('MIndex=',ctx.dis.ModRM.Index,' ', + 'SOpcode=',ctx.dis.SimdOpcode,':',SCODES[ctx.dis.SimdOpcode],' ', + 'mm=',ctx.dis.mm,':',MCODES[ctx.dis.mm and 3]); Assert(false); end; @@ -659,6 +686,8 @@ begin if not ctx.fetch_forward_point(links,addr) then begin + ctx.builder.ud2; + data:=AllocMem(ctx.builder.GetMemSize); ctx.builder.SaveTo(data,ctx.builder.GetMemSize); diff --git a/sys/test/kern_jit2_ctx.pas b/sys/test/kern_jit2_ctx.pas index f334f086..61982236 100644 --- a/sys/test/kern_jit2_ctx.pas +++ b/sys/test/kern_jit2_ctx.pas @@ -72,6 +72,9 @@ type forward_set:t_forward_set; label_set :t_label_set; + text_start:QWORD; + text___end:QWORD; + max:QWORD; Code :Pointer; @@ -105,6 +108,7 @@ function GetTargetOfs(var din:TInstruction;Code:PByte;id:Byte;var ofs:Int64):Boo function is_preserved(const r:TRegValue):Boolean; function is_preserved(const r:TOperand):Boolean; function is_preserved(const r:TInstruction):Boolean; +function is_memory(const r:TOperand):Boolean; function is_memory(const r:TInstruction):Boolean; function is_invalid(const r:TInstruction):Boolean; @@ -158,7 +162,6 @@ type t_op_hint=Set of (his_mov, his_xor, - his_cmp, his_xchg, his_rax, his_rd, @@ -179,7 +182,7 @@ type mem_one:t_op_type; //reg_one end; -procedure build_lea(var ctx:t_jit_context2;id:Byte;reg:TRegValue); +procedure build_lea(var ctx:t_jit_context2;id:Byte;reg:TRegValue;use_r_tmp1:Boolean=True); function cmp_reg(const r1,r2:TRegValue):Boolean; function new_reg(const Operand:TOperand):TRegValue; @@ -191,6 +194,8 @@ function flags(const i:TInstruction):t_jit_reg; function flags(const ctx:t_jit_context2):t_jit_reg; procedure add_orig(var ctx:t_jit_context2); +procedure op_load_rax(var ctx:t_jit_context2;reg:TRegValue); +procedure op_save_rax(var ctx:t_jit_context2;reg:TRegValue); procedure op_emit1(var ctx:t_jit_context2;const desc:t_op_type;hint:t_op_hint); procedure op_emit2(var ctx:t_jit_context2;const desc:t_op_desc); procedure op_emit_shift(var ctx:t_jit_context2;const desc:t_op_shift); @@ -255,6 +260,7 @@ var node:t_forward_point; begin if (dst=nil) then Exit; + node.dst:=dst; Result:=forward_set.Find(@node); if (Result=nil) then @@ -796,7 +802,7 @@ begin end; end; -procedure build_lea(var ctx:t_jit_context2;id:Byte;reg:TRegValue); +procedure build_lea(var ctx:t_jit_context2;id:Byte;reg:TRegValue;use_r_tmp1:Boolean=True); var RegValue:TRegValues; adr,new1,new2:TRegValue; @@ -851,6 +857,19 @@ begin end else if is_preserved(RegValue) then begin + + if (RegValue[0].AType<>regNone) and + (RegValue[1].AType<>regNone) and + (not is_preserved(RegValue[0])) and + (is_preserved(RegValue[1])) and + (RegValue[0].AScale<=1) then + begin + //optimal swap + new1:=RegValue[0]; + RegValue[0]:=RegValue[1]; + RegValue[1]:=new1; + end; + AScale:=RegValue[0].AScale; ofs:=0; @@ -875,15 +894,30 @@ begin if (RegValue[1].AType<>regNone) then begin + if is_preserved(RegValue[1]) then begin i:=GetFrameOffset(RegValue[1]); - addq(adr,[r_thrd+i]); + + if use_r_tmp1 then + begin + new2:=new_reg_size(r_tmp1,adr.ASize); + + movq(new2,[r_thrd+i]); + leaq(adr,[adr+new2]); + end else + begin + pushfq(os64); + addq(adr,[r_thrd+i]); + popfq(os64); + end; + end else begin new1:=RegValue[1]; - addq(adr,new1); + leaq(adr,[adr+new1]); end; + end; end else begin @@ -1052,6 +1086,28 @@ begin end; end; +procedure op_load_rax(var ctx:t_jit_context2;reg:TRegValue); +var + i:Integer; +begin + with ctx.builder do + begin + i:=GetFrameOffset(rax); + movq(reg,[r_thrd+i]); + end; +end; + +procedure op_save_rax(var ctx:t_jit_context2;reg:TRegValue); +var + i:Integer; +begin + with ctx.builder do + begin + i:=GetFrameOffset(rax); + movq([r_thrd+i],reg); + end; +end; + procedure op_emit1(var ctx:t_jit_context2;const desc:t_op_type;hint:t_op_hint); var i:Integer; @@ -1077,12 +1133,11 @@ var movq(r_tmp1,r_tmp0); - i:=GetFrameOffset(rax); - movq(rax,[r_thrd+i]); + op_load_rax(ctx,rax); _M(desc,mem_size,[flags(ctx)+r_tmp1]); - movq([r_thrd+i],rax); + op_save_rax(ctx,rax); end; end; @@ -1195,7 +1250,17 @@ begin mem_size:=os64; //fix size end; + if (his_rax in hint) then + begin + op_load_rax(ctx,rax); + end; + _M(desc,mem_size,[r_thrd+i]); + + if (his_rax in hint) then + begin + op_save_rax(ctx,rax); + end; end; else @@ -1232,12 +1297,11 @@ var movq(r_tmp1,r_tmp0); - i:=GetFrameOffset(rax); - movq(fix_size(new2),[r_thrd+i]); + op_load_rax(ctx,fix_size(new2)); _RM(desc.mem_reg,new1,[flags(ctx)+r_tmp1]); - movq([r_thrd+i],fix_size(new2)); + op_save_rax(ctx,fix_size(new2)); end else begin _RM(desc.mem_reg,new1,[flags(ctx)+r_tmp0]); @@ -1279,8 +1343,7 @@ var push(r15); - i:=GetFrameOffset(rax); - movq(r15,[r_thrd+i]); + op_load_rax(ctx,r15); xchgq(r15,rax); @@ -1288,7 +1351,7 @@ var pop(r15); - movq([r_thrd+i],rax); + op_save_rax(ctx,rax); end else begin new1:=new_reg_size(r_tmp1,ctx.din.Operand[2]); @@ -1375,13 +1438,77 @@ var _RM(desc.reg_mem,new1,[flags(ctx)+r_tmp0]); end; - if not (his_cmp in desc.hint) then + if not (his_rd in desc.hint) then begin i:=GetFrameOffset(ctx.din.Operand[1]); movq([r_thrd+i],fix_size(new1)); end; end; + + //read only swapped + + mo_mem_reg: + begin + //input:rax + + new1:=new_reg(ctx.din.Operand[2]); + + imm:=0; + if GetTargetOfs(ctx.din,ctx.code,3,imm) then + begin + imm_size:=ctx.din.Operand[3].Size; + mem_size:=ctx.din.Operand[2].Size; + + if (imm_size=os8) and + (mem_size<>os8) and + (not (not_impl in desc.reg_im8.opt)) then + begin + _RMI8(desc.reg_im8,new1,[flags(ctx)+r_tmp0],imm); + end else + begin + _RMI(desc.reg_imm,new1,[flags(ctx)+r_tmp0],imm); + end; + + end else + begin + _RM(desc.mem_reg,new1,[flags(ctx)+r_tmp0]); + end; + + end; + + mo_mem_ctx: + begin + //input:rax + + new1:=new_reg_size(r_tmp1,ctx.din.Operand[2]); + + i:=GetFrameOffset(ctx.din.Operand[2]); + movq(fix_size(new1),[r_thrd+i]); + + imm:=0; + if GetTargetOfs(ctx.din,ctx.code,3,imm) then + begin + imm_size:=ctx.din.Operand[3].Size; + mem_size:=ctx.din.Operand[2].Size; + + if (imm_size=os8) and + (mem_size<>os8) and + (not (not_impl in desc.reg_im8.opt)) then + begin + _RMI8(desc.reg_im8,new1,[flags(ctx)+r_tmp0],imm); + end else + begin + _RMI(desc.reg_imm,new1,[flags(ctx)+r_tmp0],imm); + end; + + end else + begin + _RM(desc.mem_reg,new1,[flags(ctx)+r_tmp0]); + end; + + end; + else; end; end; @@ -1409,6 +1536,26 @@ begin mo_mem_imm, mo_mem_ctx: begin + if (his_rd in desc.hint) then + begin + + if (mem_size=os8) or + (his_rw in desc.hint) then + begin + call_far(@uplift_jit); //in/out:rax uses:r14 + + mem_in; + end else + begin + //mem_size + movi(new_reg_size(r_tmp1,os8),OPERAND_BYTES[mem_size]); + + call_far(@copyin_mov); //in:rax(addr),r14:(size) out:rax + + mem_in; + end; + + end else if (mem_size=os8) or (his_rw in desc.hint) then begin @@ -1478,12 +1625,24 @@ begin movq([r_thrd+i],fix_size(new2)); end else + if (his_rax in desc.hint) then + begin + new2:=new_reg_size(rax,ctx.din.Operand[1]); + + op_load_rax(ctx,fix_size(new2)); + + i:=GetFrameOffset(ctx.din.Operand[1]); + _RM(desc.mem_reg,new1,[r_thrd+i]); + + op_save_rax(ctx,fix_size(new2)); + end else if (new1.ASize=os32) or (not_impl in desc.mem_reg.opt) then begin new2:=new_reg_size(r_tmp0,ctx.din.Operand[1]); - if (not (his_mov in desc.hint)) then + if (not (his_mov in desc.hint)) or + (his_rd in desc.hint) then begin i:=GetFrameOffset(ctx.din.Operand[1]); movq(fix_size(new2),[r_thrd+i]); @@ -1499,26 +1658,13 @@ begin _RR(desc.mem_reg,new2,new1,mem_size); end; - if not (his_cmp in desc.hint) then + if not (his_rd in desc.hint) then begin i:=GetFrameOffset(ctx.din.Operand[1]); movq([r_thrd+i],fix_size(new2)); end; end else - if (his_rax in desc.hint) then - begin - new2:=new_reg_size(rax,ctx.din.Operand[1]); - - i:=GetFrameOffset(rax); - movq(fix_size(new2),[r_thrd+i]); - - i:=GetFrameOffset(ctx.din.Operand[1]); - _RM(desc.mem_reg,new1,[r_thrd+i]); - - i:=GetFrameOffset(rax); - movq([r_thrd+i],fix_size(new2)); - end else begin i:=GetFrameOffset(ctx.din.Operand[1]); _RM(desc.mem_reg,new1,[r_thrd+i]); @@ -1615,7 +1761,8 @@ begin new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]); new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]); - if (not (his_mov in desc.hint)) then + if (not (his_mov in desc.hint)) or + (his_rd in desc.hint) then begin i:=GetFrameOffset(ctx.din.Operand[1]); movq(fix_size(new1),[r_thrd+i]); @@ -1642,7 +1789,7 @@ begin _RR(desc.mem_reg,new1,new2,mem_size); end; - if not (his_cmp in desc.hint) then + if not (his_rd in desc.hint) then begin i:=GetFrameOffset(ctx.din.Operand[1]); movq([r_thrd+i],fix_size(new1)); @@ -1964,6 +2111,7 @@ var mem_size:TOperandSize; link_next:t_jit_i_link; + i:Integer; imm:Int64; new1,new2:TRegValue; @@ -1974,16 +2122,18 @@ var begin //input:rax + mem_size:=ctx.din.Operand[1].Size; + new1:=new_reg(ctx.din.Operand[2]); new2:=new_reg(ctx.din.Operand[3]); imm:=0; if GetTargetOfs(ctx.din,ctx.code,4,imm) then begin - _VVMI8(desc,new2,new1,[flags(ctx)+r_tmp0],imm); + _VVMI8(desc,new2,new1,[flags(ctx)+r_tmp0],mem_size,imm); end else begin - _VVM(desc,new2,new1,[flags(ctx)+r_tmp0]); //[mem],arg2,arg3 -> arg3,arg2,[mem] + _VVM(desc,new2,new1,[flags(ctx)+r_tmp0],mem_size); //[mem],arg2,arg3 -> arg3,arg2,[mem] end; end; @@ -2001,10 +2151,10 @@ var imm:=0; if GetTargetOfs(ctx.din,ctx.code,4,imm) then begin - _VVMI8(desc,new1,new2,[flags(ctx)+r_tmp0],imm); + _VVMI8(desc,new1,new2,[flags(ctx)+r_tmp0],mem_size,imm); end else begin - _VVM(desc,new1,new2,[flags(ctx)+r_tmp0]); + _VVM(desc,new1,new2,[flags(ctx)+r_tmp0],mem_size); end; end; @@ -2038,6 +2188,32 @@ begin end; + end else + if is_preserved(ctx.din.Operand[3]) then + begin + //mo_reg_reg_ctx + + with ctx.builder do + begin + + mem_size:=ctx.din.Operand[3].Size; + + new1:=new_reg(ctx.din.Operand[1]); + new2:=new_reg(ctx.din.Operand[2]); + + i:=GetFrameOffset(ctx.din.Operand[3]); + + imm:=0; + if GetTargetOfs(ctx.din,ctx.code,4,imm) then + begin + _VVMI8(desc,new1,new2,[r_thrd+i],mem_size,imm); + end else + begin + _VVM(desc,new1,new2,[r_thrd+i],mem_size); + end; + + end; + end else if (ofMemory in ctx.din.Operand[1].Flags) then begin @@ -2078,10 +2254,11 @@ begin end; end; +//rri,mri procedure op_emit_avx3_imm8(var ctx:t_jit_context2;const desc:t_op_type); var i:Integer; - memop:t_memop_type1; + memop:t_memop_type2; mem_size:TOperandSize; link_next:t_jit_i_link; @@ -2095,6 +2272,8 @@ var begin //input:rax + //mem_reg + new2:=new_reg(ctx.din.Operand[2]); imm:=0; @@ -2104,12 +2283,31 @@ var end; end; + + procedure mem_in; + begin + with ctx.builder do + begin + //input:rax + + //reg_mem + + new1:=new_reg(ctx.din.Operand[1]); + + imm:=0; + GetTargetOfs(ctx.din,ctx.code,3,imm); + + _MVI8(desc,[flags(ctx)+r_tmp0],new2,imm); + end; + end; + + begin - memop:=classif_memop1(ctx.din); + memop:=classif_memop2(ctx.din); with ctx.builder do case memop of - mo_mem: + mo_mem_reg: begin build_lea(ctx,1,r_tmp0); mem_size:=ctx.din.Operand[1].Size; @@ -2136,7 +2334,28 @@ begin end; end; - mo_ctx: + mo_reg_mem: + begin + build_lea(ctx,2,r_tmp0); + mem_size:=ctx.din.Operand[2].Size; + + if (mem_size=os8) then + begin + call_far(@uplift_jit); //in/out:rax uses:r14 + + mem_in; + end else + begin + //mem_size + movi(new_reg_size(r_tmp1,os8),OPERAND_BYTES[mem_size]); + + call_far(@copyin_mov); //in:rax(addr),r14:(size) out:rax + + mem_in; + end; + end; + + mo_ctx_reg: begin mem_size:=ctx.din.Operand[1].Size; i:=GetFrameOffset(ctx.din.Operand[1]); @@ -2157,7 +2376,6 @@ begin begin _MVI8(desc,[r_thrd+i],new2,imm); end; - end; else diff --git a/sys/test/kern_jit2_ops.pas b/sys/test/kern_jit2_ops.pas index e9c571af..bc517d10 100644 --- a/sys/test/kern_jit2_ops.pas +++ b/sys/test/kern_jit2_ops.pas @@ -69,7 +69,7 @@ const reg_mem:(op:$33;index:0); reg_imm:(op:$81;index:4); reg_im8:(op:$83;index:4); - hint:[his_xor]; + hint:[]; ); procedure op_and(var ctx:t_jit_context2); @@ -187,7 +187,15 @@ begin end; end else begin - add_orig(ctx); + case ctx.din.OperCnt of + 1:begin + op_load_rax(ctx,ctx.builder.rax); + add_orig(ctx); + op_save_rax(ctx,ctx.builder.rax); + end; + else + add_orig(ctx); + end; end; end; @@ -201,7 +209,25 @@ begin op_emit1(ctx,mul_desc,[his_rax]); //R end else begin + op_load_rax(ctx,ctx.builder.rax); add_orig(ctx); + op_save_rax(ctx,ctx.builder.rax); + end; +end; + +const + idiv_desc1:t_op_type=(op:$F7;index:7); + +procedure op_idiv(var ctx:t_jit_context2); +begin + if is_preserved(ctx.din) or is_memory(ctx.din) then + begin + op_emit1(ctx,idiv_desc1,[his_rax]); //R + end else + begin + op_load_rax(ctx,ctx.builder.rax); + add_orig(ctx); + op_save_rax(ctx,ctx.builder.rax); end; end; @@ -215,7 +241,9 @@ begin op_emit1(ctx,div_desc,[his_rax]); //R end else begin + op_load_rax(ctx,ctx.builder.rax); add_orig(ctx); + op_save_rax(ctx,ctx.builder.rax); end; end; @@ -225,7 +253,7 @@ const reg_mem:(opt:[not_impl]); reg_imm:(opt:[not_impl]); reg_im8:(op:$0FBA;index:4); - hint:[]; + hint:[his_rd]; ); procedure op_bt(var ctx:t_jit_context2); @@ -512,6 +540,26 @@ begin end; end; +const + vmovupd_desc:t_op_desc=( + mem_reg:(op:$11;index:1;mm:1); + reg_mem:(op:$10;index:1;mm:1); + reg_imm:(opt:[not_impl]); + reg_im8:(opt:[not_impl]); + hint:[his_mov]; + ); + +procedure op_vmovupd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx2(ctx,vmovupd_desc); + end else + begin + add_orig(ctx); + end; +end; + const vmovaps_desc:t_op_desc=( mem_reg:(op:$29;index:0;mm:1); @@ -572,6 +620,26 @@ begin end; end; +const + vmovddup_desc:t_op_desc=( + mem_reg:(opt:[not_impl]); + reg_mem:(op:$12;index:3;mm:1); + reg_imm:(opt:[not_impl]); + reg_im8:(opt:[not_impl]); + hint:[his_mov]; + ); + +procedure op_vmovddup(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx2(ctx,vmovddup_desc); + end else + begin + add_orig(ctx); + end; +end; + const vmovq_desc:t_op_desc=( //vmovd_desc mem_reg:(op:$7E;index:1;mm:1); @@ -592,6 +660,96 @@ begin end; end; +const + vmovss_mrr_desc:t_op_type=(op:$11;index:2;mm:1;opt:[not_vex_len]); + vmovss_rrm_desc:t_op_type=(op:$10;index:2;mm:1;opt:[not_vex_len]); + +procedure op_vmovss(var ctx:t_jit_context2); +begin + if is_preserved(ctx.din) or is_memory(ctx.din) then + begin + if is_memory(ctx.din.Operand[1]) then + begin + op_emit_avx3(ctx,vmovss_mrr_desc); + end else + if is_memory(ctx.din.Operand[3]) then + begin + op_emit_avx3(ctx,vmovss_rrm_desc); + end else + begin + Assert(False); + end; + end else + begin + add_orig(ctx); + end; +end; + +const + vmovsd_mrr_desc:t_op_type=(op:$11;index:3;mm:1;opt:[not_vex_len]); + vmovsd_rrm_desc:t_op_type=(op:$10;index:3;mm:1;opt:[not_vex_len]); + +procedure op_vmovsd(var ctx:t_jit_context2); +begin + if is_preserved(ctx.din) or is_memory(ctx.din) then + begin + if is_memory(ctx.din.Operand[1]) then + begin + op_emit_avx3(ctx,vmovsd_mrr_desc); + end else + if is_memory(ctx.din.Operand[3]) then + begin + op_emit_avx3(ctx,vmovsd_rrm_desc); + end else + begin + Assert(False); + end; + end else + begin + add_orig(ctx); + end; +end; + +const + vucomiss_desc:t_op_desc=( + mem_reg:(opt:[not_impl]); + reg_mem:(op:$2E;index:0;mm:1); + reg_imm:(opt:[not_impl]); + reg_im8:(opt:[not_impl]); + hint:[]; + ); + +procedure op_vucomiss(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx2(ctx,vucomiss_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vucomisd_desc:t_op_desc=( + mem_reg:(opt:[not_impl]); + reg_mem:(op:$2E;index:1;mm:1); + reg_imm:(opt:[not_impl]); + reg_im8:(opt:[not_impl]); + hint:[]; + ); + +procedure op_vucomisd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx2(ctx,vucomisd_desc); + end else + begin + add_orig(ctx); + end; +end; + const vptest_desc:t_op_desc=( mem_reg:(opt:[not_impl]); @@ -656,7 +814,7 @@ const reg_mem:(op:$00;index:0); reg_imm:(op:$F7;index:0); reg_im8:(op:$00;index:0); - hint:[his_cmp]; + hint:[his_rd]; ); procedure op_test(var ctx:t_jit_context2); @@ -676,7 +834,7 @@ const reg_mem:(op:$3B;index:0); reg_imm:(op:$81;index:7); reg_im8:(op:$83;index:7); - hint:[his_cmp]; + hint:[his_rd]; ); procedure op_cmp(var ctx:t_jit_context2); @@ -707,7 +865,9 @@ begin op_emit2(ctx,cmpxchg_desc); end else begin + op_load_rax(ctx,ctx.builder.rax); add_orig(ctx); + op_save_rax(ctx,ctx.builder.rax); end; end; @@ -731,6 +891,26 @@ begin end; end; +const + lzcnt_desc:t_op_desc=( + mem_reg:(opt:[not_impl]); + reg_mem:(op:$F30FBD;index:0); + reg_imm:(opt:[not_impl]); + reg_im8:(opt:[not_impl]); + hint:[his_mov]; + ); + +procedure op_lzcnt(var ctx:t_jit_context2); +begin + if is_preserved(ctx.din) or is_memory(ctx.din) then + begin + op_emit2(ctx,lzcnt_desc); + end else + begin + add_orig(ctx); + end; +end; + const shl_desc:t_op_shift=( reg_im8:(op:$C1;index:4); @@ -920,18 +1100,18 @@ begin end; end; -procedure op_cdqe(var ctx:t_jit_context2); +procedure op_cdq(var ctx:t_jit_context2); var i:Integer; begin with ctx.builder do begin i:=GetFrameOffset(rax); - movq(r_tmp0,[r_thrd+i]); + movq(rax,[r_thrd+i]); add_orig(ctx); - movq([r_thrd+i],r_tmp0); + movq([r_thrd+i],rax); end; end; @@ -957,7 +1137,7 @@ end; const vxorps_desc:t_op_type=( - op:$57;index:1;mm:1; + op:$57;index:0;mm:1; ); procedure op_vxorps(var ctx:t_jit_context2); @@ -971,6 +1151,22 @@ begin end; end; +const + vxorpd_desc:t_op_type=( + op:$57;index:1;mm:1; + ); + +procedure op_vxorpd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vxorpd_desc); + end else + begin + add_orig(ctx); + end; +end; + const vpcmpeqd_desc:t_op_type=( op:$76;index:1;mm:1; @@ -1003,6 +1199,22 @@ begin end; end; +const + vpaddb_desc:t_op_type=( + op:$FC;index:1;mm:1; + ); + +procedure op_vpaddb(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vpaddb_desc); + end else + begin + add_orig(ctx); + end; +end; + const vpaddd_desc:t_op_type=( op:$FE;index:1;mm:1; @@ -1035,6 +1247,151 @@ begin end; end; +const + vaddps_desc:t_op_type=( + op:$58;index:0;mm:1; + ); + +procedure op_vaddps(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vaddps_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vaddpd_desc:t_op_type=( + op:$58;index:1;mm:1; + ); + +procedure op_vaddpd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vaddpd_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vaddss_desc:t_op_type=( + op:$58;index:2;mm:1; + ); + +procedure op_vaddss(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vaddss_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vaddsd_desc:t_op_type=( + op:$58;index:3;mm:1; + ); + +procedure op_vaddsd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vaddsd_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vdivss_desc:t_op_type=( + op:$5E;index:2;mm:1; + ); + +procedure op_vdivss(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vdivss_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vdivsd_desc:t_op_type=( + op:$5E;index:3;mm:1; + ); + +procedure op_vdivsd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vdivsd_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vmulps_desc:t_op_type=( + op:$59;index:0;mm:1; + ); + +procedure op_vmulps(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vmulps_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vmulpd_desc:t_op_type=( + op:$59;index:1;mm:1; + ); + +procedure op_vmulpd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vmulpd_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vmulsd_desc:t_op_type=( + op:$59;index:3;mm:1; + ); + +procedure op_vmulsd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vmulsd_desc); + end else + begin + add_orig(ctx); + end; +end; + + const vpunpcklqdq_desc:t_op_type=( op:$6C;index:1;mm:1; @@ -1051,6 +1408,22 @@ begin end; end; +const + vunpcklpd_desc:t_op_type=( + op:$14;index:1;mm:1; + ); + +procedure op_vunpcklpd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vunpcklpd_desc); + end else + begin + add_orig(ctx); + end; +end; + const vcmpps_desc:t_op_type=( op:$C2;index:0;mm:1; @@ -1099,6 +1472,75 @@ begin end; end; +const + vpshufb_desc:t_op_type=( + op:$00;index:1;mm:2; + ); + +procedure op_vpshufb(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vpshufb_desc); + end else + begin + add_orig(ctx); + end; +end; + + +const + vpermilps_rrm_desc:t_op_type=( + op:$0C;index:1;mm:2; + ); + + vpermilps_rmi_desc:t_op_type=( + op:$04;index:1;mm:3; + ); + +procedure op_vpermilps(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + if (ctx.din.Operand[3].ByteCount=0) then + begin + op_emit_avx3(ctx,vpermilps_rrm_desc); + end else + begin + op_emit_avx3_imm8(ctx,vpermilps_rmi_desc); + end; + end else + begin + add_orig(ctx); + end; +end; + +const + vpermilpd_rrm_desc:t_op_type=( + op:$0D;index:1;mm:2; + ); + + vpermilpd_rmi_desc:t_op_type=( + op:$05;index:1;mm:3; + ); + +procedure op_vpermilpd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + if (ctx.din.Operand[3].ByteCount=0) then + begin + op_emit_avx3(ctx,vpermilpd_rrm_desc); + end else + begin + op_emit_avx3_imm8(ctx,vpermilpd_rmi_desc); + end; + end else + begin + add_orig(ctx); + end; +end; + const vpminud_desc:t_op_type=( op:$3B;index:1;mm:2; @@ -1164,6 +1606,113 @@ begin end; end; +const + vblendpd_desc:t_op_type=( + op:$0D;index:1;mm:3;opt:[] + ); + +procedure op_vblendpd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vblendpd_desc); + end else + begin + add_orig(ctx); + end; +end; + + +const + vcvtsi2ss_desc:t_op_type=( + op:$2A;index:2;mm:1; + ); + +procedure op_vcvtsi2ss(var ctx:t_jit_context2); +begin + if is_preserved(ctx.din) or is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vcvtsi2ss_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vcvtsi2sd_desc:t_op_type=( + op:$2A;index:3;mm:1; + ); + +procedure op_vcvtsi2sd(var ctx:t_jit_context2); +begin + if is_preserved(ctx.din) or is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vcvtsi2sd_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vcvtsd2ss_desc:t_op_type=( + op:$5A;index:3;mm:1; + ); + +procedure op_vcvtsd2ss(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx3(ctx,vcvtsd2ss_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vcvtdq2pd_desc:t_op_desc=( + mem_reg:(opt:[not_impl]); + reg_mem:(op:$E6;index:2;mm:1); + reg_imm:(opt:[not_impl]); + reg_im8:(opt:[not_impl]); + hint:[]; + ); + +procedure op_vcvtdq2pd(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx2(ctx,vcvtdq2pd_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + vcvttsd2si_desc:t_op_desc=( + mem_reg:(opt:[not_impl]); + reg_mem:(op:$2C;index:3;mm:1); + reg_imm:(opt:[not_impl]); + reg_im8:(opt:[not_impl]); + hint:[]; + ); + +procedure op_vcvttsd2si(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_avx2(ctx,vcvttsd2si_desc); + end else + begin + add_orig(ctx); + end; +end; + +// + const bextr_desc:t_op_type=( op:$F7;index:0;mm:2; @@ -1221,9 +1770,9 @@ const procedure op_vinsertf128(var ctx:t_jit_context2); begin - if is_preserved(ctx.din) or is_memory(ctx.din) then + if is_memory(ctx.din) then begin - op_emit_avx3_imm8(ctx,vinsertf128_desc); + op_emit_avx3(ctx,vinsertf128_desc); end else begin add_orig(ctx); @@ -1246,22 +1795,6 @@ begin end; end; -const - fldcw_desc:t_op_type=( - op:$D9;index:5;opt:[not_prefix]; - ); - -procedure op_fldcw(var ctx:t_jit_context2); -begin - if is_memory(ctx.din) then - begin - op_emit1(ctx,fldcw_desc,[his_rd]); - end else - begin - add_orig(ctx); - end; -end; - const fxsave_desc:t_op_type=( op:$0FAE;index:0;opt:[not_prefix]; @@ -1294,6 +1827,22 @@ begin end; end; +const + fldcw_desc:t_op_type=( + op:$D9;index:5;opt:[not_prefix]; + ); + +procedure op_fldcw(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit1(ctx,fldcw_desc,[his_rd]); + end else + begin + add_orig(ctx); + end; +end; + const fld_32_desc:t_op_type=( op:$D9;index:0;opt:[not_prefix]; @@ -1324,6 +1873,38 @@ begin end; end; +const + fild_16_desc:t_op_type=( + op:$DF;index:0;opt:[not_prefix]; + ); + + fild_32_desc:t_op_type=( + op:$DB;index:0;opt:[not_prefix]; + ); + + fild_64_desc:t_op_type=( + op:$DF;index:5;opt:[not_prefix]; + ); + +procedure op_fild(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + case ctx.din.Operand[1].Size of + os16:op_emit1(ctx,fild_16_desc,[his_rd]); + os32:op_emit1(ctx,fild_32_desc,[his_rd]); + os64:op_emit1(ctx,fild_64_desc,[his_rd]); + else + Assert(false); + end; + end else + begin + add_orig(ctx); + end; +end; + +// + const fst_32_desc:t_op_type=( op:$D9;index:2;opt:[not_prefix]; @@ -1379,6 +1960,37 @@ begin end; end; +const + fisttp_16_desc:t_op_type=( + op:$DF;index:1;opt:[not_prefix]; + ); + + fisttp_32_desc:t_op_type=( + op:$DB;index:1;opt:[not_prefix]; + ); + + fisttp_64_desc:t_op_type=( + op:$DD;index:1;opt:[not_prefix]; + ); + +procedure op_fisttp(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + case ctx.din.Operand[1].Size of + os16:op_emit1(ctx,fisttp_16_desc,[]); + os32:op_emit1(ctx,fisttp_32_desc,[]); + os64:op_emit1(ctx,fisttp_64_desc,[]); + else + Assert(false); + end; + end else + begin + add_orig(ctx); + end; +end; + + var inited:Integer=0; @@ -1396,6 +2008,7 @@ begin jit_cbs[OPPnone,OPimul,OPSnone]:=@op_imul; jit_cbs[OPPnone,OPmul ,OPSnone]:=@op_mul; + jit_cbs[OPPnone,OPidiv,OPSnone]:=@op_idiv; jit_cbs[OPPnone,OPdiv ,OPSnone]:=@op_div; jit_cbs[OPPnone,OPbt ,OPSnone]:=@op_bt; @@ -1406,13 +2019,22 @@ begin jit_cbs[OPPnone,OPmov ,OPSnone]:=@op_mov; jit_cbs[OPPv,OPmovu,OPSx_ps ]:=@op_vmovups; + jit_cbs[OPPv,OPmovu,OPSx_pd ]:=@op_vmovupd; jit_cbs[OPPv,OPmova,OPSx_ps ]:=@op_vmovaps; jit_cbs[OPPv,OPmov ,OPSx_dqu]:=@op_vmovdqu; jit_cbs[OPPv,OPmov ,OPSx_dqa]:=@op_vmovdqa; + jit_cbs[OPPv,OPmovddup,OPSnone]:=@op_vmovddup; + jit_cbs[OPPv,OPmov ,OPSx_d ]:=@op_vmovq; jit_cbs[OPPv,OPmov ,OPSx_q ]:=@op_vmovq; + jit_cbs[OPPv,OPmov ,OPSx_ss ]:=@op_vmovss; + jit_cbs[OPPv,OPmov ,OPSx_sd ]:=@op_vmovsd; + + jit_cbs[OPPv,OPucomi,OPSx_ss]:=@op_vucomiss; + jit_cbs[OPPv,OPucomi,OPSx_sd]:=@op_vucomisd; + jit_cbs[OPPv,OPptest,OPSnone]:=@op_vptest; jit_cbs[OPPnone,OPblsr,OPSnone ]:=@op_blsr; @@ -1444,6 +2066,7 @@ begin jit_cbs[OPPnone,OPcmpxchg,OPSnone]:=@op_cmpxchg; jit_cbs[OPPnone,OPxadd ,OPSnone]:=@op_xadd; + jit_cbs[OPPnone,OPlzcnt ,OPSnone]:=@op_lzcnt; jit_cbs[OPPnone,OPshl ,OPSnone]:=@op_shl; jit_cbs[OPPnone,OPshr ,OPSnone]:=@op_shr; @@ -1469,38 +2092,109 @@ begin jit_cbs[OPPnone,OPpxor,OPSnone]:=@op_pxor; - jit_cbs[OPPnone,OPemms,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPemms ,OPSnone]:=@add_orig; jit_cbs[OPPnone,OPvzeroall,OPSnone]:=@add_orig; - jit_cbs[OPPnone,OPfninit,OPSnone]:=@add_orig; - jit_cbs[OPPnone,OPrdtsc ,OPSnone]:=@add_orig; - jit_cbs[OPPnone,OPpause ,OPSnone]:=@add_orig; + + jit_cbs[OPPnone,OPfninit ,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPfchs ,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPfxch ,OPSnone]:=@add_orig; + + jit_cbs[OPPnone,OPfcmov__ ,OPSc_b ]:=@add_orig; + jit_cbs[OPPnone,OPfcmov__ ,OPSc_e ]:=@add_orig; + jit_cbs[OPPnone,OPfcmov__ ,OPSc_be ]:=@add_orig; + jit_cbs[OPPnone,OPfcmov__ ,OPSc_u ]:=@add_orig; + jit_cbs[OPPnone,OPfcmov__ ,OPSc_nb ]:=@add_orig; + jit_cbs[OPPnone,OPfcmov__ ,OPSc_ne ]:=@add_orig; + jit_cbs[OPPnone,OPfcmov__ ,OPSc_nbe]:=@add_orig; + jit_cbs[OPPnone,OPfcmov__ ,OPSc_nu ]:=@add_orig; + + jit_cbs[OPPnone,OPfld1 ,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPfldl2e,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPfldl2t,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPfldlg2,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPfldln2,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPfldpi ,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPfldz ,OPSnone]:=@add_orig; + + jit_cbs[OPPnone,OPfucom,OPSx_i ]:=@add_orig; + jit_cbs[OPPnone,OPfucom,OPSx_ip]:=@add_orig; + jit_cbs[OPPnone,OPfucom,OPSx_p ]:=@add_orig; + + jit_cbs[OPPnone,OPfcom,OPSx_i ]:=@add_orig; + jit_cbs[OPPnone,OPfcom,OPSx_ip]:=@add_orig; + jit_cbs[OPPnone,OPfcom,OPSx_p ]:=@add_orig; + + jit_cbs[OPPnone,OPfsub,OPSx_p ]:=@add_orig; + + jit_cbs[OPPnone,OPpause ,OPSnone]:=@add_orig; + + jit_cbs[OPPnone,OPlfence ,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPmfence ,OPSnone]:=@add_orig; + jit_cbs[OPPnone,OPsfence ,OPSnone]:=@add_orig; jit_cbs[OPPv,OPxor ,OPSx_ps]:=@op_vxorps; + jit_cbs[OPPv,OPxor ,OPSx_pd]:=@op_vxorpd; + jit_cbs[OPPv,OPpcmpeq,OPSx_d ]:=@op_vpcmpeqd; jit_cbs[OPPv,OPpsub ,OPSx_q ]:=@op_vpsubq; + + jit_cbs[OPPv,OPpadd ,OPSx_b ]:=@op_vpaddb; jit_cbs[OPPv,OPpadd ,OPSx_d ]:=@op_vpaddd; jit_cbs[OPPv,OPpadd ,OPSx_q ]:=@op_vpaddq; + + jit_cbs[OPPv,OPadd ,OPSx_ps]:=@op_vaddps; + jit_cbs[OPPv,OPadd ,OPSx_pd]:=@op_vaddpd; + jit_cbs[OPPv,OPadd ,OPSx_ss]:=@op_vaddss; + jit_cbs[OPPv,OPadd ,OPSx_sd]:=@op_vaddsd; + + jit_cbs[OPPv,OPdiv ,OPSx_ss]:=@op_vdivss; + jit_cbs[OPPv,OPdiv ,OPSx_sd]:=@op_vdivsd; + + jit_cbs[OPPv,OPmul ,OPSx_ps]:=@op_vmulps; + jit_cbs[OPPv,OPmul ,OPSx_pd]:=@op_vmulpd; + jit_cbs[OPPv,OPmul ,OPSx_sd]:=@op_vmulsd; + jit_cbs[OPPv,OPpunpcklqdq,OPSnone]:=@op_vpunpcklqdq; + jit_cbs[OPPv,OPunpckl ,OPSx_pd]:=@op_vunpcklpd; jit_cbs[OPPv,OPcmp ,OPSx_ps]:=@op_vcmpps; jit_cbs[OPPv,OPcmp ,OPSx_pd]:=@op_vcmppd; jit_cbs[OPPv,OPpshuf ,OPSx_d ]:=@op_vpshufd; + jit_cbs[OPPv,OPpshuf ,OPSx_b ]:=@op_vpshufb; + + jit_cbs[OPPnone,OPvpermil,OPSx_ps]:=@op_vpermilps; + jit_cbs[OPPnone,OPvpermil,OPSx_pd]:=@op_vpermilpd; jit_cbs[OPPv,OPpminu ,OPSx_d ]:=@op_vpminud; jit_cbs[OPPv,OPmaskmov,OPSx_ps]:=@op_vmaskmovps; jit_cbs[OPPv,OPpxor ,OPSnone]:=@op_vpxor; jit_cbs[OPPv,OPpor ,OPSnone]:=@op_vpor; + jit_cbs[OPPv,OPblend ,OPSx_pd]:=@op_vblendpd; + + jit_cbs[OPPv,OPcvtsi2 ,OPSx_ss]:=@op_vcvtsi2ss; + jit_cbs[OPPv,OPcvtsi2 ,OPSx_sd]:=@op_vcvtsi2sd; + + jit_cbs[OPPv,OPcvtsd2 ,OPSx_ss]:=@op_vcvtsd2ss; + + jit_cbs[OPPv,OPcvtdq2 ,OPSx_pd]:=@op_vcvtdq2pd; + + jit_cbs[OPPv,OPcvttsd2,OPSx_si]:=@op_vcvttsd2si; + jit_cbs[OPPnone,OPbextr,OPSnone]:=@op_bextr; jit_cbs[OPPnone,OPandn ,OPSnone]:=@op_andn; jit_cbs[OPPv,OPpextr ,OPSx_q ]:=@op_vpextrq; jit_cbs[OPPv,OPinsert,OPSx_f128]:=@op_vinsertf128; - jit_cbs[OPPnone,OPcbw ,OPSnone]:=@op_cdqe; - jit_cbs[OPPnone,OPcwde,OPSnone]:=@op_cdqe; - jit_cbs[OPPnone,OPcdqe,OPSnone]:=@op_cdqe; + jit_cbs[OPPnone,OPcwd ,OPSnone]:=@op_cdq; + jit_cbs[OPPnone,OPcdq ,OPSnone]:=@op_cdq; + jit_cbs[OPPnone,OPcqo ,OPSnone]:=@op_cdq; + + jit_cbs[OPPnone,OPcbw ,OPSnone]:=@op_cdq; + jit_cbs[OPPnone,OPcwde,OPSnone]:=@op_cdq; + jit_cbs[OPPnone,OPcdqe,OPSnone]:=@op_cdq; jit_cbs[OPPnone,OPlea,OPSnone]:=@op_lea; @@ -1511,13 +2205,16 @@ begin jit_cbs[OPPnone,OPbswap,OPSnone]:=@op_bswap; jit_cbs[OPPnone,OPfnstcw,OPSnone]:=@op_fnstcw; + jit_cbs[OPPnone,OPfldcw ,OPSnone]:=@op_fldcw; + jit_cbs[OPPnone,OPfld ,OPSnone]:=@op_fld; + jit_cbs[OPPnone,OPfild ,OPSnone]:=@op_fild; jit_cbs[OPPnone,OPfxsave ,OPSnone]:=@op_fxsave; jit_cbs[OPPnone,OPfxrstor,OPSnone]:=@op_fxrstor; - jit_cbs[OPPnone,OPfld ,OPSnone]:=@op_fld; jit_cbs[OPPnone,OPfst ,OPSnone]:=@op_fst; jit_cbs[OPPnone,OPfst ,OPSx_p ]:=@op_fstp; + jit_cbs[OPPnone,OPfisttp ,OPSnone]:=@op_fisttp; inited:=1; end;