ShuriZma
/
FPPS4
mirror of https://github.com/red-prig/fpPS4.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

+

This commit is contained in:
Pavel 2023-09-22 23:55:00 +03:00
parent 1748a8d29f
commit 2ba7c4dc75
6 changed files with 1103 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
rtl/x86_fpdbgdisas.pp
View File

@ -349,6 +349,7 @@ type
CodeIdx: Byte;
OperIdx: Integer;
ModRMIdx: Byte;
mm: Byte;
Flags: TFlags;
SimdOpcode: TSimdOpcode;
ModRM: record
@ -3538,13 +3539,16 @@ begin
//---
$C0: begin
DecodeSIMD([soNone]);
if SimdOpcode = soNone
then begin SetOpcode(OPxadd); AddEb; AddGb; CheckLock; end;
case SimdOpcode of
soNone: begin SetOpcode(OPxadd); AddEb; AddGb; CheckLock; end;
end;
end;
$C1: begin
DecodeSIMD([soNone]);
if SimdOpcode = soNone
then begin SetOpcode(OPxadd); AddEv; AddGv; CheckLock; end;
case SimdOpcode of
soNone: begin SetOpcode(OPxadd); AddEv; AddGv; CheckLock; end;
so66: begin SetOpcode(OPxadd); AddEw; AddGw; CheckLock; end;
end;
end;
$C2: begin
DecodeSIMD;
@ -3929,7 +3933,7 @@ const
SIMDMAP: array[0..3] of TSimdOpcode = (soNone, so66, soF3, soF2);
LENMAP: array[0..3] of TOperandSize = (os128, os256, os512, os0);
var
idx, mm: Byte;
idx: Byte;
begin
Assert(ASize in [2..4], Format('Invalid VEX size: %u', [ASize]));
@ -4695,6 +4699,7 @@ begin
OperIdx := 0;
SimdOpcode := soInvalid;
Vex.MaskIndex := 0;
mm:=0;
Sib.Scale:=0;
Sib.Index:=0;

81
rtl/x86_jit.pas
View File

@ -88,8 +88,8 @@ type
function is_valid:Boolean;
function before:t_jit_i_link;
function after :t_jit_i_link;
function prev(_before:boolean):t_jit_i_link;
function next(_before:boolean):t_jit_i_link;
function prev:t_jit_i_link;
function next:t_jit_i_link;
property _node:p_jit_instruction read ALink;
property _label:t_jit_i_link read get_label write set_label;
end;
@ -99,6 +99,9 @@ const
operator = (A,B:t_jit_i_link):Boolean;
//SimdOpcode (soNone=0, so66=1, soF3=2, soF2=3);
//mm (0F=1, 0F38=2, 0F3A=3)
type
p_jit_builder=^t_jit_builder;
t_jit_builder=object
@ -265,6 +268,7 @@ type
function leaj(reg:TRegValue;mem:t_jit_regs;_label_id:t_jit_i_link):t_jit_i_link;
//
Procedure reta;
Procedure ud2;
//
Function GetInstructionsSize:Integer;
Function GetDataSize:Integer;
@ -333,8 +337,8 @@ type
procedure _VM (const desc:t_op_type;reg:TRegValue;mem:t_jit_regs;size:TOperandSize);
procedure _VM_F3 (const desc:t_op_type;reg:TRegValue;mem:t_jit_regs;size:TOperandSize);
procedure _VV_F3 (const desc:t_op_type;reg0,reg1:TRegValue;size:TOperandSize);
procedure _VVM (const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs);
procedure _VVMI8 (const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;imm8:Byte);
procedure _VVM (const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;size:TOperandSize);
procedure _VVMI8 (const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;size:TOperandSize;imm8:Byte);
procedure _VVV (const desc:t_op_type;reg0,reg1,reg2:TRegValue);
procedure _VVVI8 (const desc:t_op_type;reg0,reg1,reg2:TRegValue;imm8:Byte);
procedure _VVI8 (const desc:t_op_type;reg0,reg1:TRegValue;imm8:Byte);
@ -649,6 +653,7 @@ begin
begin
Result.AType:=lnkLabelBefore;
end;
else;
end;
end;
@ -661,48 +666,53 @@ begin
begin
Result.AType:=lnkLabelAfter;
end;
else;
end;
end;
function t_jit_i_link.prev(_before:boolean):t_jit_i_link;
function t_jit_i_link.prev:t_jit_i_link;
begin
Result:=nil_link;
case AType of
lnkData:
begin
Result.AType:=lnkData;
Result.ALink:=TAILQ_PREV(ALink,@ALink^.link);
Result.ALink:=TAILQ_PREV(ALink,@p_jit_data(ALink)^.link);
end;
lnkLabelBefore,
lnkLabelAfter:
begin
if _before then
Result.AType:=lnkLabelBefore
else
Result.AType:=lnkLabelAfter;
Result.AType:=lnkLabelBefore;
Result.ALink:=TAILQ_PREV(ALink,@ALink^.link);
end;
else;
end;
if (Result.ALink=nil) then
begin
Result:=nil_link;
end;
end;
function t_jit_i_link.next(_before:boolean):t_jit_i_link;
function t_jit_i_link.next:t_jit_i_link;
begin
Result:=nil_link;
case AType of
lnkData:
begin
Result.AType:=lnkData;
Result.ALink:=TAILQ_NEXT(ALink,@ALink^.link);
Result.ALink:=TAILQ_NEXT(ALink,@p_jit_data(ALink)^.link);
end;
lnkLabelBefore,
lnkLabelAfter:
begin
if _before then
Result.AType:=lnkLabelBefore
else
Result.AType:=lnkLabelAfter;
Result.AType:=lnkLabelBefore;
Result.ALink:=TAILQ_NEXT(ALink,@ALink^.link);
end;
else;
end;
if (Result.ALink=nil) then
begin
Result:=nil_link;
end;
end;
@ -1139,6 +1149,19 @@ begin
_add(ji);
end;
Procedure t_jit_builder.ud2;
var
ji:t_jit_instruction;
begin
ji:=default_jit_instruction;
ji.EmitByte($0F);
ji.EmitByte($0B);
_add(ji);
end;
Function t_jit_builder.GetInstructionsSize:Integer;
begin
Result:=AInstructionSize;
@ -3474,7 +3497,7 @@ begin
_add(ji);
end;
procedure t_jit_builder._VVM(const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs);
procedure t_jit_builder._VVM(const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;size:TOperandSize);
var
mreg:t_jit_reg;
@ -3518,10 +3541,17 @@ begin
Vex.Index:=reg1.AIndex;
Vex.rexW:=False;
if (reg0.ASize=os64) then
if (size=os0) then
begin
size:=reg0.ASize;
end;
if (size=os64) then
begin
Vex.rexW:=True;
end else
begin
Vex.rexW:=False;
end;
modrm_info:=Default(t_modrm_info);
@ -3563,7 +3593,7 @@ begin
_add(ji);
end;
procedure t_jit_builder._VVMI8(const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;imm8:Byte);
procedure t_jit_builder._VVMI8(const desc:t_op_type;reg0,reg1:TRegValue;mem:t_jit_regs;size:TOperandSize;imm8:Byte);
var
mreg:t_jit_reg;
@ -3607,10 +3637,17 @@ begin
Vex.Index:=reg1.AIndex;
Vex.rexW:=False;
if (reg0.ASize=os64) then
if (size=os0) then
begin
size:=reg0.ASize;
end;
if (size=os64) then
begin
Vex.rexW:=True;
end else
begin
Vex.rexW:=False;
end;
modrm_info:=Default(t_modrm_info);

16
sys/kern/kern_exec.pas
View File

@ -937,12 +937,20 @@ var
addr:Pointer;
ST_TYPE:Integer;
begin
if (obj=nil) then Exit;
ctx:=Default(t_jit_context2);
ctx.text_start:=QWORD(obj^.map_base);
ctx.text___end:=ctx.text_start+obj^.text_size;
ctx.add_forward_point(obj^.entry_addr);
ctx.add_forward_point(obj^.init_proc_addr);
ctx.add_forward_point(obj^.fini_proc_addr);
if (obj^.mainprog=0) then
begin
ctx.add_forward_point(obj^.init_proc_addr);
ctx.add_forward_point(obj^.fini_proc_addr);
end;
lib_entry:=TAILQ_FIRST(@obj^.lib_table);
while (lib_entry<>nil) do
@ -1020,6 +1028,8 @@ begin
Writeln(StdErr,'preload_prx_modules:',str,' not loaded');
end;
pick_obj(obj);
obj:=TAILQ_FIRST(@dynlibs_info.obj_list);
while (obj<>nil) do
begin
@ -1055,6 +1065,8 @@ begin
p_proc.libkernel___end_addr:=dynlibs_info.libkernel^.map_base + dynlibs_info.libkernel^.text_size;
end;
//pick_obj(dynlibs_info.libprogram);
pick_obj(dynlibs_info.libkernel);
_dyn_not_exist:

29
sys/test/kern_jit2.pas
View File

@ -420,6 +420,12 @@ begin
id:=PByte(ctx.code)[i];
case id of
$41: //assert?
begin
//
ctx.builder.call_far(nil); //TODO error dispatcher
end;
$44: //system error?
begin
//
@ -452,6 +458,12 @@ begin
ctx.builder.call_far(nil); //TODO CPUID
end;
procedure op_rdtsc(var ctx:t_jit_context2);
begin
add_orig(ctx);
op_save_rax(ctx,ctx.builder.rax);
end;
procedure op_nop(var ctx:t_jit_context2);
begin
//align?
@ -503,6 +515,7 @@ begin
jit_cbs[OPPnone,OPiret,OPSx_q ]:=@op_iretq;
jit_cbs[OPPnone,OPcpuid,OPSnone]:=@op_cpuid;
jit_cbs[OPPnone,OPrdtsc ,OPSnone]:=@op_rdtsc;
jit_cbs[OPPnone,OPnop,OPSnone]:=@op_nop;
@ -510,6 +523,9 @@ begin
end;
procedure pick(var ctx:t_jit_context2);
const
SCODES:array[TSimdOpcode] of Byte=(0,0,1,3,2);
MCODES:array[0..3] of RawByteString=('','0F','0F38','0F3A');
label
_next;
var
@ -534,6 +550,9 @@ begin
kern_jit2_ops.init_cbs;
ctx.max:=QWORD(ctx.max_forward_point);
Writeln(' ctx.text_start:0x',HexStr(ctx.text_start,16));
Writeln(' ctx.max :0x',HexStr(ctx.max,16));
Writeln(' ctx.text___end:0x',HexStr(ctx.text___end,16));
links:=Default(t_jit_context2.t_forward_links);
addr:=nil;
@ -567,6 +586,11 @@ begin
begin
//invalid
writeln('invalid:0x',HexStr(ctx.ptr_curr));
ptr:=ctx.ptr_curr;
adec.Disassemble(ptr,ACodeBytes,ACode);
goto _next;
end;
else;
@ -600,6 +624,9 @@ begin
ctx.din.OpCode.Suffix,' ',
ctx.din.Operand[1].Size,' ',
ctx.din.Operand[2].Size);
Writeln('MIndex=',ctx.dis.ModRM.Index,' ',
'SOpcode=',ctx.dis.SimdOpcode,':',SCODES[ctx.dis.SimdOpcode],' ',
'mm=',ctx.dis.mm,':',MCODES[ctx.dis.mm and 3]);
Assert(false);
end;
@ -659,6 +686,8 @@ begin
if not ctx.fetch_forward_point(links,addr) then
begin
ctx.builder.ud2;
data:=AllocMem(ctx.builder.GetMemSize);
ctx.builder.SaveTo(data,ctx.builder.GetMemSize);

300
sys/test/kern_jit2_ctx.pas
View File

@ -72,6 +72,9 @@ type
forward_set:t_forward_set;
label_set :t_label_set;
text_start:QWORD;
text___end:QWORD;
max:QWORD;
Code :Pointer;
@ -105,6 +108,7 @@ function GetTargetOfs(var din:TInstruction;Code:PByte;id:Byte;var ofs:Int64):Boo
function is_preserved(const r:TRegValue):Boolean;
function is_preserved(const r:TOperand):Boolean;
function is_preserved(const r:TInstruction):Boolean;
function is_memory(const r:TOperand):Boolean;
function is_memory(const r:TInstruction):Boolean;
function is_invalid(const r:TInstruction):Boolean;
@ -158,7 +162,6 @@ type
t_op_hint=Set of (his_mov,
his_xor,
his_cmp,
his_xchg,
his_rax,
his_rd,
@ -179,7 +182,7 @@ type
mem_one:t_op_type; //reg_one
end;
procedure build_lea(var ctx:t_jit_context2;id:Byte;reg:TRegValue);
procedure build_lea(var ctx:t_jit_context2;id:Byte;reg:TRegValue;use_r_tmp1:Boolean=True);
function cmp_reg(const r1,r2:TRegValue):Boolean;
function new_reg(const Operand:TOperand):TRegValue;
@ -191,6 +194,8 @@ function flags(const i:TInstruction):t_jit_reg;
function flags(const ctx:t_jit_context2):t_jit_reg;
procedure add_orig(var ctx:t_jit_context2);
procedure op_load_rax(var ctx:t_jit_context2;reg:TRegValue);
procedure op_save_rax(var ctx:t_jit_context2;reg:TRegValue);
procedure op_emit1(var ctx:t_jit_context2;const desc:t_op_type;hint:t_op_hint);
procedure op_emit2(var ctx:t_jit_context2;const desc:t_op_desc);
procedure op_emit_shift(var ctx:t_jit_context2;const desc:t_op_shift);
@ -255,6 +260,7 @@ var
node:t_forward_point;
begin
if (dst=nil) then Exit;
node.dst:=dst;
Result:=forward_set.Find(@node);
if (Result=nil) then
@ -796,7 +802,7 @@ begin
end;
end;
procedure build_lea(var ctx:t_jit_context2;id:Byte;reg:TRegValue);
procedure build_lea(var ctx:t_jit_context2;id:Byte;reg:TRegValue;use_r_tmp1:Boolean=True);
var
RegValue:TRegValues;
adr,new1,new2:TRegValue;
@ -851,6 +857,19 @@ begin
end else
if is_preserved(RegValue) then
begin
if (RegValue[0].AType<>regNone) and
(RegValue[1].AType<>regNone) and
(not is_preserved(RegValue[0])) and
(is_preserved(RegValue[1])) and
(RegValue[0].AScale<=1) then
begin
//optimal swap
new1:=RegValue[0];
RegValue[0]:=RegValue[1];
RegValue[1]:=new1;
end;
AScale:=RegValue[0].AScale;
ofs:=0;
@ -875,15 +894,30 @@ begin
if (RegValue[1].AType<>regNone) then
begin
if is_preserved(RegValue[1]) then
begin
i:=GetFrameOffset(RegValue[1]);
addq(adr,[r_thrd+i]);
if use_r_tmp1 then
begin
new2:=new_reg_size(r_tmp1,adr.ASize);
movq(new2,[r_thrd+i]);
leaq(adr,[adr+new2]);
end else
begin
pushfq(os64);
addq(adr,[r_thrd+i]);
popfq(os64);
end;
end else
begin
new1:=RegValue[1];
addq(adr,new1);
leaq(adr,[adr+new1]);
end;
end;
end else
begin
@ -1052,6 +1086,28 @@ begin
end;
end;
procedure op_load_rax(var ctx:t_jit_context2;reg:TRegValue);
var
i:Integer;
begin
with ctx.builder do
begin
i:=GetFrameOffset(rax);
movq(reg,[r_thrd+i]);
end;
end;
procedure op_save_rax(var ctx:t_jit_context2;reg:TRegValue);
var
i:Integer;
begin
with ctx.builder do
begin
i:=GetFrameOffset(rax);
movq([r_thrd+i],reg);
end;
end;
procedure op_emit1(var ctx:t_jit_context2;const desc:t_op_type;hint:t_op_hint);
var
i:Integer;
@ -1077,12 +1133,11 @@ var
movq(r_tmp1,r_tmp0);
i:=GetFrameOffset(rax);
movq(rax,[r_thrd+i]);
op_load_rax(ctx,rax);
_M(desc,mem_size,[flags(ctx)+r_tmp1]);
movq([r_thrd+i],rax);
op_save_rax(ctx,rax);
end;
end;
@ -1195,7 +1250,17 @@ begin
mem_size:=os64; //fix size
end;
if (his_rax in hint) then
begin
op_load_rax(ctx,rax);
end;
_M(desc,mem_size,[r_thrd+i]);
if (his_rax in hint) then
begin
op_save_rax(ctx,rax);
end;
end;
else
@ -1232,12 +1297,11 @@ var
movq(r_tmp1,r_tmp0);
i:=GetFrameOffset(rax);
movq(fix_size(new2),[r_thrd+i]);
op_load_rax(ctx,fix_size(new2));
_RM(desc.mem_reg,new1,[flags(ctx)+r_tmp1]);
movq([r_thrd+i],fix_size(new2));
op_save_rax(ctx,fix_size(new2));
end else
begin
_RM(desc.mem_reg,new1,[flags(ctx)+r_tmp0]);
@ -1279,8 +1343,7 @@ var
push(r15);
i:=GetFrameOffset(rax);
movq(r15,[r_thrd+i]);
op_load_rax(ctx,r15);
xchgq(r15,rax);
@ -1288,7 +1351,7 @@ var
pop(r15);
movq([r_thrd+i],rax);
op_save_rax(ctx,rax);
end else
begin
new1:=new_reg_size(r_tmp1,ctx.din.Operand[2]);
@ -1375,13 +1438,77 @@ var
_RM(desc.reg_mem,new1,[flags(ctx)+r_tmp0]);
end;
if not (his_cmp in desc.hint) then
if not (his_rd in desc.hint) then
begin
i:=GetFrameOffset(ctx.din.Operand[1]);
movq([r_thrd+i],fix_size(new1));
end;
end;
//read only swapped
mo_mem_reg:
begin
//input:rax
new1:=new_reg(ctx.din.Operand[2]);
imm:=0;
if GetTargetOfs(ctx.din,ctx.code,3,imm) then
begin
imm_size:=ctx.din.Operand[3].Size;
mem_size:=ctx.din.Operand[2].Size;
if (imm_size=os8) and
(mem_size<>os8) and
(not (not_impl in desc.reg_im8.opt)) then
begin
_RMI8(desc.reg_im8,new1,[flags(ctx)+r_tmp0],imm);
end else
begin
_RMI(desc.reg_imm,new1,[flags(ctx)+r_tmp0],imm);
end;
end else
begin
_RM(desc.mem_reg,new1,[flags(ctx)+r_tmp0]);
end;
end;
mo_mem_ctx:
begin
//input:rax
new1:=new_reg_size(r_tmp1,ctx.din.Operand[2]);
i:=GetFrameOffset(ctx.din.Operand[2]);
movq(fix_size(new1),[r_thrd+i]);
imm:=0;
if GetTargetOfs(ctx.din,ctx.code,3,imm) then
begin
imm_size:=ctx.din.Operand[3].Size;
mem_size:=ctx.din.Operand[2].Size;
if (imm_size=os8) and
(mem_size<>os8) and
(not (not_impl in desc.reg_im8.opt)) then
begin
_RMI8(desc.reg_im8,new1,[flags(ctx)+r_tmp0],imm);
end else
begin
_RMI(desc.reg_imm,new1,[flags(ctx)+r_tmp0],imm);
end;
end else
begin
_RM(desc.mem_reg,new1,[flags(ctx)+r_tmp0]);
end;
end;
else;
end;
end;
@ -1409,6 +1536,26 @@ begin
mo_mem_imm,
mo_mem_ctx:
begin
if (his_rd in desc.hint) then
begin
if (mem_size=os8) or
(his_rw in desc.hint) then
begin
call_far(@uplift_jit); //in/out:rax uses:r14
mem_in;
end else
begin
//mem_size
movi(new_reg_size(r_tmp1,os8),OPERAND_BYTES[mem_size]);
call_far(@copyin_mov); //in:rax(addr),r14:(size) out:rax
mem_in;
end;
end else
if (mem_size=os8) or
(his_rw in desc.hint) then
begin
@ -1478,12 +1625,24 @@ begin
movq([r_thrd+i],fix_size(new2));
end else
if (his_rax in desc.hint) then
begin
new2:=new_reg_size(rax,ctx.din.Operand[1]);
op_load_rax(ctx,fix_size(new2));
i:=GetFrameOffset(ctx.din.Operand[1]);
_RM(desc.mem_reg,new1,[r_thrd+i]);
op_save_rax(ctx,fix_size(new2));
end else
if (new1.ASize=os32) or
(not_impl in desc.mem_reg.opt) then
begin
new2:=new_reg_size(r_tmp0,ctx.din.Operand[1]);
if (not (his_mov in desc.hint)) then
if (not (his_mov in desc.hint)) or
(his_rd in desc.hint) then
begin
i:=GetFrameOffset(ctx.din.Operand[1]);
movq(fix_size(new2),[r_thrd+i]);
@ -1499,26 +1658,13 @@ begin
_RR(desc.mem_reg,new2,new1,mem_size);
end;
if not (his_cmp in desc.hint) then
if not (his_rd in desc.hint) then
begin
i:=GetFrameOffset(ctx.din.Operand[1]);
movq([r_thrd+i],fix_size(new2));
end;
end else
if (his_rax in desc.hint) then
begin
new2:=new_reg_size(rax,ctx.din.Operand[1]);
i:=GetFrameOffset(rax);
movq(fix_size(new2),[r_thrd+i]);
i:=GetFrameOffset(ctx.din.Operand[1]);
_RM(desc.mem_reg,new1,[r_thrd+i]);
i:=GetFrameOffset(rax);
movq([r_thrd+i],fix_size(new2));
end else
begin
i:=GetFrameOffset(ctx.din.Operand[1]);
_RM(desc.mem_reg,new1,[r_thrd+i]);
@ -1615,7 +1761,8 @@ begin
new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);
new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]);
if (not (his_mov in desc.hint)) then
if (not (his_mov in desc.hint)) or
(his_rd in desc.hint) then
begin
i:=GetFrameOffset(ctx.din.Operand[1]);
movq(fix_size(new1),[r_thrd+i]);
@ -1642,7 +1789,7 @@ begin
_RR(desc.mem_reg,new1,new2,mem_size);
end;
if not (his_cmp in desc.hint) then
if not (his_rd in desc.hint) then
begin
i:=GetFrameOffset(ctx.din.Operand[1]);
movq([r_thrd+i],fix_size(new1));
@ -1964,6 +2111,7 @@ var
mem_size:TOperandSize;
link_next:t_jit_i_link;
i:Integer;
imm:Int64;
new1,new2:TRegValue;
@ -1974,16 +2122,18 @@ var
begin
//input:rax
mem_size:=ctx.din.Operand[1].Size;
new1:=new_reg(ctx.din.Operand[2]);
new2:=new_reg(ctx.din.Operand[3]);
imm:=0;
if GetTargetOfs(ctx.din,ctx.code,4,imm) then
begin
_VVMI8(desc,new2,new1,[flags(ctx)+r_tmp0],imm);
_VVMI8(desc,new2,new1,[flags(ctx)+r_tmp0],mem_size,imm);
end else
begin
_VVM(desc,new2,new1,[flags(ctx)+r_tmp0]); //[mem],arg2,arg3 -> arg3,arg2,[mem]
_VVM(desc,new2,new1,[flags(ctx)+r_tmp0],mem_size); //[mem],arg2,arg3 -> arg3,arg2,[mem]
end;
end;
@ -2001,10 +2151,10 @@ var
imm:=0;
if GetTargetOfs(ctx.din,ctx.code,4,imm) then
begin
_VVMI8(desc,new1,new2,[flags(ctx)+r_tmp0],imm);
_VVMI8(desc,new1,new2,[flags(ctx)+r_tmp0],mem_size,imm);
end else
begin
_VVM(desc,new1,new2,[flags(ctx)+r_tmp0]);
_VVM(desc,new1,new2,[flags(ctx)+r_tmp0],mem_size);
end;
end;
@ -2038,6 +2188,32 @@ begin
end;
end else
if is_preserved(ctx.din.Operand[3]) then
begin
//mo_reg_reg_ctx
with ctx.builder do
begin
mem_size:=ctx.din.Operand[3].Size;
new1:=new_reg(ctx.din.Operand[1]);
new2:=new_reg(ctx.din.Operand[2]);
i:=GetFrameOffset(ctx.din.Operand[3]);
imm:=0;
if GetTargetOfs(ctx.din,ctx.code,4,imm) then
begin
_VVMI8(desc,new1,new2,[r_thrd+i],mem_size,imm);
end else
begin
_VVM(desc,new1,new2,[r_thrd+i],mem_size);
end;
end;
end else
if (ofMemory in ctx.din.Operand[1].Flags) then
begin
@ -2078,10 +2254,11 @@ begin
end;
end;
//rri,mri
procedure op_emit_avx3_imm8(var ctx:t_jit_context2;const desc:t_op_type);
var
i:Integer;
memop:t_memop_type1;
memop:t_memop_type2;
mem_size:TOperandSize;
link_next:t_jit_i_link;
@ -2095,6 +2272,8 @@ var
begin
//input:rax
//mem_reg
new2:=new_reg(ctx.din.Operand[2]);
imm:=0;
@ -2104,12 +2283,31 @@ var
end;
end;
procedure mem_in;
begin
with ctx.builder do
begin
//input:rax
//reg_mem
new1:=new_reg(ctx.din.Operand[1]);
imm:=0;
GetTargetOfs(ctx.din,ctx.code,3,imm);
_MVI8(desc,[flags(ctx)+r_tmp0],new2,imm);
end;
end;
begin
memop:=classif_memop1(ctx.din);
memop:=classif_memop2(ctx.din);
with ctx.builder do
case memop of
mo_mem:
mo_mem_reg:
begin
build_lea(ctx,1,r_tmp0);
mem_size:=ctx.din.Operand[1].Size;
@ -2136,7 +2334,28 @@ begin
end;
end;
mo_ctx:
mo_reg_mem:
begin
build_lea(ctx,2,r_tmp0);
mem_size:=ctx.din.Operand[2].Size;
if (mem_size=os8) then
begin
call_far(@uplift_jit); //in/out:rax uses:r14
mem_in;
end else
begin
//mem_size
movi(new_reg_size(r_tmp1,os8),OPERAND_BYTES[mem_size]);
call_far(@copyin_mov); //in:rax(addr),r14:(size) out:rax
mem_in;
end;
end;
mo_ctx_reg:
begin
mem_size:=ctx.din.Operand[1].Size;
i:=GetFrameOffset(ctx.din.Operand[1]);
@ -2157,7 +2376,6 @@ begin
begin
_MVI8(desc,[r_thrd+i],new2,imm);
end;
end;
else

767
sys/test/kern_jit2_ops.pas
View File

@ -69,7 +69,7 @@ const
reg_mem:(op:$33;index:0);
reg_imm:(op:$81;index:4);
reg_im8:(op:$83;index:4);
hint:[his_xor];
hint:[];
);
procedure op_and(var ctx:t_jit_context2);
@ -187,7 +187,15 @@ begin
end;
end else
begin
add_orig(ctx);
case ctx.din.OperCnt of
1:begin
op_load_rax(ctx,ctx.builder.rax);
add_orig(ctx);
op_save_rax(ctx,ctx.builder.rax);
end;
else
add_orig(ctx);
end;
end;
end;
@ -201,7 +209,25 @@ begin
op_emit1(ctx,mul_desc,[his_rax]); //R
end else
begin
op_load_rax(ctx,ctx.builder.rax);
add_orig(ctx);
op_save_rax(ctx,ctx.builder.rax);
end;
end;
const
idiv_desc1:t_op_type=(op:$F7;index:7);
procedure op_idiv(var ctx:t_jit_context2);
begin
if is_preserved(ctx.din) or is_memory(ctx.din) then
begin
op_emit1(ctx,idiv_desc1,[his_rax]); //R
end else
begin
op_load_rax(ctx,ctx.builder.rax);
add_orig(ctx);
op_save_rax(ctx,ctx.builder.rax);
end;
end;
@ -215,7 +241,9 @@ begin
op_emit1(ctx,div_desc,[his_rax]); //R
end else
begin
op_load_rax(ctx,ctx.builder.rax);
add_orig(ctx);
op_save_rax(ctx,ctx.builder.rax);
end;
end;
@ -225,7 +253,7 @@ const
reg_mem:(opt:[not_impl]);
reg_imm:(opt:[not_impl]);
reg_im8:(op:$0FBA;index:4);
hint:[];
hint:[his_rd];
);
procedure op_bt(var ctx:t_jit_context2);
@ -512,6 +540,26 @@ begin
end;
end;
const
vmovupd_desc:t_op_desc=(
mem_reg:(op:$11;index:1;mm:1);
reg_mem:(op:$10;index:1;mm:1);
reg_imm:(opt:[not_impl]);
reg_im8:(opt:[not_impl]);
hint:[his_mov];
);
procedure op_vmovupd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx2(ctx,vmovupd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vmovaps_desc:t_op_desc=(
mem_reg:(op:$29;index:0;mm:1);
@ -572,6 +620,26 @@ begin
end;
end;
const
vmovddup_desc:t_op_desc=(
mem_reg:(opt:[not_impl]);
reg_mem:(op:$12;index:3;mm:1);
reg_imm:(opt:[not_impl]);
reg_im8:(opt:[not_impl]);
hint:[his_mov];
);
procedure op_vmovddup(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx2(ctx,vmovddup_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vmovq_desc:t_op_desc=( //vmovd_desc
mem_reg:(op:$7E;index:1;mm:1);
@ -592,6 +660,96 @@ begin
end;
end;
const
vmovss_mrr_desc:t_op_type=(op:$11;index:2;mm:1;opt:[not_vex_len]);
vmovss_rrm_desc:t_op_type=(op:$10;index:2;mm:1;opt:[not_vex_len]);
procedure op_vmovss(var ctx:t_jit_context2);
begin
if is_preserved(ctx.din) or is_memory(ctx.din) then
begin
if is_memory(ctx.din.Operand[1]) then
begin
op_emit_avx3(ctx,vmovss_mrr_desc);
end else
if is_memory(ctx.din.Operand[3]) then
begin
op_emit_avx3(ctx,vmovss_rrm_desc);
end else
begin
Assert(False);
end;
end else
begin
add_orig(ctx);
end;
end;
const
vmovsd_mrr_desc:t_op_type=(op:$11;index:3;mm:1;opt:[not_vex_len]);
vmovsd_rrm_desc:t_op_type=(op:$10;index:3;mm:1;opt:[not_vex_len]);
procedure op_vmovsd(var ctx:t_jit_context2);
begin
if is_preserved(ctx.din) or is_memory(ctx.din) then
begin
if is_memory(ctx.din.Operand[1]) then
begin
op_emit_avx3(ctx,vmovsd_mrr_desc);
end else
if is_memory(ctx.din.Operand[3]) then
begin
op_emit_avx3(ctx,vmovsd_rrm_desc);
end else
begin
Assert(False);
end;
end else
begin
add_orig(ctx);
end;
end;
const
vucomiss_desc:t_op_desc=(
mem_reg:(opt:[not_impl]);
reg_mem:(op:$2E;index:0;mm:1);
reg_imm:(opt:[not_impl]);
reg_im8:(opt:[not_impl]);
hint:[];
);
procedure op_vucomiss(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx2(ctx,vucomiss_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vucomisd_desc:t_op_desc=(
mem_reg:(opt:[not_impl]);
reg_mem:(op:$2E;index:1;mm:1);
reg_imm:(opt:[not_impl]);
reg_im8:(opt:[not_impl]);
hint:[];
);
procedure op_vucomisd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx2(ctx,vucomisd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vptest_desc:t_op_desc=(
mem_reg:(opt:[not_impl]);
@ -656,7 +814,7 @@ const
reg_mem:(op:$00;index:0);
reg_imm:(op:$F7;index:0);
reg_im8:(op:$00;index:0);
hint:[his_cmp];
hint:[his_rd];
);
procedure op_test(var ctx:t_jit_context2);
@ -676,7 +834,7 @@ const
reg_mem:(op:$3B;index:0);
reg_imm:(op:$81;index:7);
reg_im8:(op:$83;index:7);
hint:[his_cmp];
hint:[his_rd];
);
procedure op_cmp(var ctx:t_jit_context2);
@ -707,7 +865,9 @@ begin
op_emit2(ctx,cmpxchg_desc);
end else
begin
op_load_rax(ctx,ctx.builder.rax);
add_orig(ctx);
op_save_rax(ctx,ctx.builder.rax);
end;
end;
@ -731,6 +891,26 @@ begin
end;
end;
const
lzcnt_desc:t_op_desc=(
mem_reg:(opt:[not_impl]);
reg_mem:(op:$F30FBD;index:0);
reg_imm:(opt:[not_impl]);
reg_im8:(opt:[not_impl]);
hint:[his_mov];
);
procedure op_lzcnt(var ctx:t_jit_context2);
begin
if is_preserved(ctx.din) or is_memory(ctx.din) then
begin
op_emit2(ctx,lzcnt_desc);
end else
begin
add_orig(ctx);
end;
end;
const
shl_desc:t_op_shift=(
reg_im8:(op:$C1;index:4);
@ -920,18 +1100,18 @@ begin
end;
end;
procedure op_cdqe(var ctx:t_jit_context2);
procedure op_cdq(var ctx:t_jit_context2);
var
i:Integer;
begin
with ctx.builder do
begin
i:=GetFrameOffset(rax);
movq(r_tmp0,[r_thrd+i]);
movq(rax,[r_thrd+i]);
add_orig(ctx);
movq([r_thrd+i],r_tmp0);
movq([r_thrd+i],rax);
end;
end;
@ -957,7 +1137,7 @@ end;
const
vxorps_desc:t_op_type=(
op:$57;index:1;mm:1;
op:$57;index:0;mm:1;
);
procedure op_vxorps(var ctx:t_jit_context2);
@ -971,6 +1151,22 @@ begin
end;
end;
const
vxorpd_desc:t_op_type=(
op:$57;index:1;mm:1;
);
procedure op_vxorpd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vxorpd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vpcmpeqd_desc:t_op_type=(
op:$76;index:1;mm:1;
@ -1003,6 +1199,22 @@ begin
end;
end;
const
vpaddb_desc:t_op_type=(
op:$FC;index:1;mm:1;
);
procedure op_vpaddb(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vpaddb_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vpaddd_desc:t_op_type=(
op:$FE;index:1;mm:1;
@ -1035,6 +1247,151 @@ begin
end;
end;
const
vaddps_desc:t_op_type=(
op:$58;index:0;mm:1;
);
procedure op_vaddps(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vaddps_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vaddpd_desc:t_op_type=(
op:$58;index:1;mm:1;
);
procedure op_vaddpd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vaddpd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vaddss_desc:t_op_type=(
op:$58;index:2;mm:1;
);
procedure op_vaddss(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vaddss_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vaddsd_desc:t_op_type=(
op:$58;index:3;mm:1;
);
procedure op_vaddsd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vaddsd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vdivss_desc:t_op_type=(
op:$5E;index:2;mm:1;
);
procedure op_vdivss(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vdivss_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vdivsd_desc:t_op_type=(
op:$5E;index:3;mm:1;
);
procedure op_vdivsd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vdivsd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vmulps_desc:t_op_type=(
op:$59;index:0;mm:1;
);
procedure op_vmulps(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vmulps_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vmulpd_desc:t_op_type=(
op:$59;index:1;mm:1;
);
procedure op_vmulpd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vmulpd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vmulsd_desc:t_op_type=(
op:$59;index:3;mm:1;
);
procedure op_vmulsd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vmulsd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vpunpcklqdq_desc:t_op_type=(
op:$6C;index:1;mm:1;
@ -1051,6 +1408,22 @@ begin
end;
end;
const
vunpcklpd_desc:t_op_type=(
op:$14;index:1;mm:1;
);
procedure op_vunpcklpd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vunpcklpd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vcmpps_desc:t_op_type=(
op:$C2;index:0;mm:1;
@ -1099,6 +1472,75 @@ begin
end;
end;
const
vpshufb_desc:t_op_type=(
op:$00;index:1;mm:2;
);
procedure op_vpshufb(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vpshufb_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vpermilps_rrm_desc:t_op_type=(
op:$0C;index:1;mm:2;
);
vpermilps_rmi_desc:t_op_type=(
op:$04;index:1;mm:3;
);
procedure op_vpermilps(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
if (ctx.din.Operand[3].ByteCount=0) then
begin
op_emit_avx3(ctx,vpermilps_rrm_desc);
end else
begin
op_emit_avx3_imm8(ctx,vpermilps_rmi_desc);
end;
end else
begin
add_orig(ctx);
end;
end;
const
vpermilpd_rrm_desc:t_op_type=(
op:$0D;index:1;mm:2;
);
vpermilpd_rmi_desc:t_op_type=(
op:$05;index:1;mm:3;
);
procedure op_vpermilpd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
if (ctx.din.Operand[3].ByteCount=0) then
begin
op_emit_avx3(ctx,vpermilpd_rrm_desc);
end else
begin
op_emit_avx3_imm8(ctx,vpermilpd_rmi_desc);
end;
end else
begin
add_orig(ctx);
end;
end;
const
vpminud_desc:t_op_type=(
op:$3B;index:1;mm:2;
@ -1164,6 +1606,113 @@ begin
end;
end;
const
vblendpd_desc:t_op_type=(
op:$0D;index:1;mm:3;opt:[]
);
procedure op_vblendpd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vblendpd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vcvtsi2ss_desc:t_op_type=(
op:$2A;index:2;mm:1;
);
procedure op_vcvtsi2ss(var ctx:t_jit_context2);
begin
if is_preserved(ctx.din) or is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vcvtsi2ss_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vcvtsi2sd_desc:t_op_type=(
op:$2A;index:3;mm:1;
);
procedure op_vcvtsi2sd(var ctx:t_jit_context2);
begin
if is_preserved(ctx.din) or is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vcvtsi2sd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vcvtsd2ss_desc:t_op_type=(
op:$5A;index:3;mm:1;
);
procedure op_vcvtsd2ss(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx3(ctx,vcvtsd2ss_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vcvtdq2pd_desc:t_op_desc=(
mem_reg:(opt:[not_impl]);
reg_mem:(op:$E6;index:2;mm:1);
reg_imm:(opt:[not_impl]);
reg_im8:(opt:[not_impl]);
hint:[];
);
procedure op_vcvtdq2pd(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx2(ctx,vcvtdq2pd_desc);
end else
begin
add_orig(ctx);
end;
end;
const
vcvttsd2si_desc:t_op_desc=(
mem_reg:(opt:[not_impl]);
reg_mem:(op:$2C;index:3;mm:1);
reg_imm:(opt:[not_impl]);
reg_im8:(opt:[not_impl]);
hint:[];
);
procedure op_vcvttsd2si(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit_avx2(ctx,vcvttsd2si_desc);
end else
begin
add_orig(ctx);
end;
end;
//
const
bextr_desc:t_op_type=(
op:$F7;index:0;mm:2;
@ -1221,9 +1770,9 @@ const
procedure op_vinsertf128(var ctx:t_jit_context2);
begin
if is_preserved(ctx.din) or is_memory(ctx.din) then
if is_memory(ctx.din) then
begin
op_emit_avx3_imm8(ctx,vinsertf128_desc);
op_emit_avx3(ctx,vinsertf128_desc);
end else
begin
add_orig(ctx);
@ -1246,22 +1795,6 @@ begin
end;
end;
const
fldcw_desc:t_op_type=(
op:$D9;index:5;opt:[not_prefix];
);
procedure op_fldcw(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit1(ctx,fldcw_desc,[his_rd]);
end else
begin
add_orig(ctx);
end;
end;
const
fxsave_desc:t_op_type=(
op:$0FAE;index:0;opt:[not_prefix];
@ -1294,6 +1827,22 @@ begin
end;
end;
const
fldcw_desc:t_op_type=(
op:$D9;index:5;opt:[not_prefix];
);
procedure op_fldcw(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
op_emit1(ctx,fldcw_desc,[his_rd]);
end else
begin
add_orig(ctx);
end;
end;
const
fld_32_desc:t_op_type=(
op:$D9;index:0;opt:[not_prefix];
@ -1324,6 +1873,38 @@ begin
end;
end;
const
fild_16_desc:t_op_type=(
op:$DF;index:0;opt:[not_prefix];
);
fild_32_desc:t_op_type=(
op:$DB;index:0;opt:[not_prefix];
);
fild_64_desc:t_op_type=(
op:$DF;index:5;opt:[not_prefix];
);
procedure op_fild(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
case ctx.din.Operand[1].Size of
os16:op_emit1(ctx,fild_16_desc,[his_rd]);
os32:op_emit1(ctx,fild_32_desc,[his_rd]);
os64:op_emit1(ctx,fild_64_desc,[his_rd]);
else
Assert(false);
end;
end else
begin
add_orig(ctx);
end;
end;
//
const
fst_32_desc:t_op_type=(
op:$D9;index:2;opt:[not_prefix];
@ -1379,6 +1960,37 @@ begin
end;
end;
const
fisttp_16_desc:t_op_type=(
op:$DF;index:1;opt:[not_prefix];
);
fisttp_32_desc:t_op_type=(
op:$DB;index:1;opt:[not_prefix];
);
fisttp_64_desc:t_op_type=(
op:$DD;index:1;opt:[not_prefix];
);
procedure op_fisttp(var ctx:t_jit_context2);
begin
if is_memory(ctx.din) then
begin
case ctx.din.Operand[1].Size of
os16:op_emit1(ctx,fisttp_16_desc,[]);
os32:op_emit1(ctx,fisttp_32_desc,[]);
os64:op_emit1(ctx,fisttp_64_desc,[]);
else
Assert(false);
end;
end else
begin
add_orig(ctx);
end;
end;
var
inited:Integer=0;
@ -1396,6 +2008,7 @@ begin
jit_cbs[OPPnone,OPimul,OPSnone]:=@op_imul;
jit_cbs[OPPnone,OPmul ,OPSnone]:=@op_mul;
jit_cbs[OPPnone,OPidiv,OPSnone]:=@op_idiv;
jit_cbs[OPPnone,OPdiv ,OPSnone]:=@op_div;
jit_cbs[OPPnone,OPbt ,OPSnone]:=@op_bt;
@ -1406,13 +2019,22 @@ begin
jit_cbs[OPPnone,OPmov ,OPSnone]:=@op_mov;
jit_cbs[OPPv,OPmovu,OPSx_ps ]:=@op_vmovups;
jit_cbs[OPPv,OPmovu,OPSx_pd ]:=@op_vmovupd;
jit_cbs[OPPv,OPmova,OPSx_ps ]:=@op_vmovaps;
jit_cbs[OPPv,OPmov ,OPSx_dqu]:=@op_vmovdqu;
jit_cbs[OPPv,OPmov ,OPSx_dqa]:=@op_vmovdqa;
jit_cbs[OPPv,OPmovddup,OPSnone]:=@op_vmovddup;
jit_cbs[OPPv,OPmov ,OPSx_d ]:=@op_vmovq;
jit_cbs[OPPv,OPmov ,OPSx_q ]:=@op_vmovq;
jit_cbs[OPPv,OPmov ,OPSx_ss ]:=@op_vmovss;
jit_cbs[OPPv,OPmov ,OPSx_sd ]:=@op_vmovsd;
jit_cbs[OPPv,OPucomi,OPSx_ss]:=@op_vucomiss;
jit_cbs[OPPv,OPucomi,OPSx_sd]:=@op_vucomisd;
jit_cbs[OPPv,OPptest,OPSnone]:=@op_vptest;
jit_cbs[OPPnone,OPblsr,OPSnone ]:=@op_blsr;
@ -1444,6 +2066,7 @@ begin
jit_cbs[OPPnone,OPcmpxchg,OPSnone]:=@op_cmpxchg;
jit_cbs[OPPnone,OPxadd ,OPSnone]:=@op_xadd;
jit_cbs[OPPnone,OPlzcnt ,OPSnone]:=@op_lzcnt;
jit_cbs[OPPnone,OPshl ,OPSnone]:=@op_shl;
jit_cbs[OPPnone,OPshr ,OPSnone]:=@op_shr;
@ -1469,38 +2092,109 @@ begin
jit_cbs[OPPnone,OPpxor,OPSnone]:=@op_pxor;
jit_cbs[OPPnone,OPemms,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPemms ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPvzeroall,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfninit,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPrdtsc ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPpause ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfninit ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfchs ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfxch ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfcmov__ ,OPSc_b ]:=@add_orig;
jit_cbs[OPPnone,OPfcmov__ ,OPSc_e ]:=@add_orig;
jit_cbs[OPPnone,OPfcmov__ ,OPSc_be ]:=@add_orig;
jit_cbs[OPPnone,OPfcmov__ ,OPSc_u ]:=@add_orig;
jit_cbs[OPPnone,OPfcmov__ ,OPSc_nb ]:=@add_orig;
jit_cbs[OPPnone,OPfcmov__ ,OPSc_ne ]:=@add_orig;
jit_cbs[OPPnone,OPfcmov__ ,OPSc_nbe]:=@add_orig;
jit_cbs[OPPnone,OPfcmov__ ,OPSc_nu ]:=@add_orig;
jit_cbs[OPPnone,OPfld1 ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfldl2e,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfldl2t,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfldlg2,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfldln2,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfldpi ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfldz ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPfucom,OPSx_i ]:=@add_orig;
jit_cbs[OPPnone,OPfucom,OPSx_ip]:=@add_orig;
jit_cbs[OPPnone,OPfucom,OPSx_p ]:=@add_orig;
jit_cbs[OPPnone,OPfcom,OPSx_i ]:=@add_orig;
jit_cbs[OPPnone,OPfcom,OPSx_ip]:=@add_orig;
jit_cbs[OPPnone,OPfcom,OPSx_p ]:=@add_orig;
jit_cbs[OPPnone,OPfsub,OPSx_p ]:=@add_orig;
jit_cbs[OPPnone,OPpause ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPlfence ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPmfence ,OPSnone]:=@add_orig;
jit_cbs[OPPnone,OPsfence ,OPSnone]:=@add_orig;
jit_cbs[OPPv,OPxor ,OPSx_ps]:=@op_vxorps;
jit_cbs[OPPv,OPxor ,OPSx_pd]:=@op_vxorpd;
jit_cbs[OPPv,OPpcmpeq,OPSx_d ]:=@op_vpcmpeqd;
jit_cbs[OPPv,OPpsub ,OPSx_q ]:=@op_vpsubq;
jit_cbs[OPPv,OPpadd ,OPSx_b ]:=@op_vpaddb;
jit_cbs[OPPv,OPpadd ,OPSx_d ]:=@op_vpaddd;
jit_cbs[OPPv,OPpadd ,OPSx_q ]:=@op_vpaddq;
jit_cbs[OPPv,OPadd ,OPSx_ps]:=@op_vaddps;
jit_cbs[OPPv,OPadd ,OPSx_pd]:=@op_vaddpd;
jit_cbs[OPPv,OPadd ,OPSx_ss]:=@op_vaddss;
jit_cbs[OPPv,OPadd ,OPSx_sd]:=@op_vaddsd;
jit_cbs[OPPv,OPdiv ,OPSx_ss]:=@op_vdivss;
jit_cbs[OPPv,OPdiv ,OPSx_sd]:=@op_vdivsd;
jit_cbs[OPPv,OPmul ,OPSx_ps]:=@op_vmulps;
jit_cbs[OPPv,OPmul ,OPSx_pd]:=@op_vmulpd;
jit_cbs[OPPv,OPmul ,OPSx_sd]:=@op_vmulsd;
jit_cbs[OPPv,OPpunpcklqdq,OPSnone]:=@op_vpunpcklqdq;
jit_cbs[OPPv,OPunpckl ,OPSx_pd]:=@op_vunpcklpd;
jit_cbs[OPPv,OPcmp ,OPSx_ps]:=@op_vcmpps;
jit_cbs[OPPv,OPcmp ,OPSx_pd]:=@op_vcmppd;
jit_cbs[OPPv,OPpshuf ,OPSx_d ]:=@op_vpshufd;
jit_cbs[OPPv,OPpshuf ,OPSx_b ]:=@op_vpshufb;
jit_cbs[OPPnone,OPvpermil,OPSx_ps]:=@op_vpermilps;
jit_cbs[OPPnone,OPvpermil,OPSx_pd]:=@op_vpermilpd;
jit_cbs[OPPv,OPpminu ,OPSx_d ]:=@op_vpminud;
jit_cbs[OPPv,OPmaskmov,OPSx_ps]:=@op_vmaskmovps;
jit_cbs[OPPv,OPpxor ,OPSnone]:=@op_vpxor;
jit_cbs[OPPv,OPpor ,OPSnone]:=@op_vpor;
jit_cbs[OPPv,OPblend ,OPSx_pd]:=@op_vblendpd;
jit_cbs[OPPv,OPcvtsi2 ,OPSx_ss]:=@op_vcvtsi2ss;
jit_cbs[OPPv,OPcvtsi2 ,OPSx_sd]:=@op_vcvtsi2sd;
jit_cbs[OPPv,OPcvtsd2 ,OPSx_ss]:=@op_vcvtsd2ss;
jit_cbs[OPPv,OPcvtdq2 ,OPSx_pd]:=@op_vcvtdq2pd;
jit_cbs[OPPv,OPcvttsd2,OPSx_si]:=@op_vcvttsd2si;
jit_cbs[OPPnone,OPbextr,OPSnone]:=@op_bextr;
jit_cbs[OPPnone,OPandn ,OPSnone]:=@op_andn;
jit_cbs[OPPv,OPpextr ,OPSx_q ]:=@op_vpextrq;
jit_cbs[OPPv,OPinsert,OPSx_f128]:=@op_vinsertf128;
jit_cbs[OPPnone,OPcbw ,OPSnone]:=@op_cdqe;
jit_cbs[OPPnone,OPcwde,OPSnone]:=@op_cdqe;
jit_cbs[OPPnone,OPcdqe,OPSnone]:=@op_cdqe;
jit_cbs[OPPnone,OPcwd ,OPSnone]:=@op_cdq;
jit_cbs[OPPnone,OPcdq ,OPSnone]:=@op_cdq;
jit_cbs[OPPnone,OPcqo ,OPSnone]:=@op_cdq;
jit_cbs[OPPnone,OPcbw ,OPSnone]:=@op_cdq;
jit_cbs[OPPnone,OPcwde,OPSnone]:=@op_cdq;
jit_cbs[OPPnone,OPcdqe,OPSnone]:=@op_cdq;
jit_cbs[OPPnone,OPlea,OPSnone]:=@op_lea;
@ -1511,13 +2205,16 @@ begin
jit_cbs[OPPnone,OPbswap,OPSnone]:=@op_bswap;
jit_cbs[OPPnone,OPfnstcw,OPSnone]:=@op_fnstcw;
jit_cbs[OPPnone,OPfldcw ,OPSnone]:=@op_fldcw;
jit_cbs[OPPnone,OPfld ,OPSnone]:=@op_fld;
jit_cbs[OPPnone,OPfild ,OPSnone]:=@op_fild;
jit_cbs[OPPnone,OPfxsave ,OPSnone]:=@op_fxsave;
jit_cbs[OPPnone,OPfxrstor,OPSnone]:=@op_fxrstor;
jit_cbs[OPPnone,OPfld ,OPSnone]:=@op_fld;
jit_cbs[OPPnone,OPfst ,OPSnone]:=@op_fst;
jit_cbs[OPPnone,OPfst ,OPSx_p ]:=@op_fstp;
jit_cbs[OPPnone,OPfisttp ,OPSnone]:=@op_fisttp;
inited:=1;
end;