diff --git a/rtl/x86_fpdbgdisas.pp b/rtl/x86_fpdbgdisas.pp index 3ea33915..36e6cb6d 100644 --- a/rtl/x86_fpdbgdisas.pp +++ b/rtl/x86_fpdbgdisas.pp @@ -589,6 +589,9 @@ type function GetCanReverseDisassemble: boolean; function ReadCodeAt(AnAddress: TDBGPtr; var ALen: Cardinal): Boolean; inline; public + Disassembler: TX86Disassembler; + Instr: TInstruction; + constructor Create(AProcess: TDbgProcess); procedure Disassemble(var AAddress: Pointer; out ACodeBytes: RawByteString; out ACode: RawByteString; out AnInfo: TDbgInstInfo); @@ -5061,8 +5064,6 @@ const OSTEXT: array[TOperandSize] of RawByteString = ('os8', 'os16', 'os32', 'os64', 'os48', 'os80', 'os128'); {$endif} var - Disassembler: TX86Disassembler; - Instr: TInstruction; S, Soper: RawByteString; n, i: Integer; TargetAddrOffs: Int64; diff --git a/rtl/x86_jit.pas b/rtl/x86_jit.pas index 42c083ea..c95ba0c0 100644 --- a/rtl/x86_jit.pas +++ b/rtl/x86_jit.pas @@ -151,6 +151,23 @@ type FS :t_jit_reg=(ARegValue:((AType:regNone),(AType:regNone));ASegment:4); GS :t_jit_reg=(ARegValue:((AType:regNone),(AType:regNone));ASegment:5); + mm0 :t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 0),(AType:regNone))); + mm1 :t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 1),(AType:regNone))); + mm2 :t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 2),(AType:regNone))); + mm3 :t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 3),(AType:regNone))); + mm4 :t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 4),(AType:regNone))); + mm5 :t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 5),(AType:regNone))); + mm6 :t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 6),(AType:regNone))); + mm7 :t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 7),(AType:regNone))); + mm8 :t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 8),(AType:regNone))); + mm9 :t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 9),(AType:regNone))); + mm10:t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 10),(AType:regNone))); + mm11:t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 11),(AType:regNone))); + mm12:t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 12),(AType:regNone))); + mm13:t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 13),(AType:regNone))); + mm14:t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 14),(AType:regNone))); + mm15:t_jit_reg=(ARegValue:((AType:regMm;ASize:os64;AIndex: 15),(AType:regNone))); + xmm0 :t_jit_reg=(ARegValue:((AType:regXmm;ASize:os128;AIndex: 0),(AType:regNone))); xmm1 :t_jit_reg=(ARegValue:((AType:regXmm;ASize:os128;AIndex: 1),(AType:regNone))); xmm2 :t_jit_reg=(ARegValue:((AType:regXmm;ASize:os128;AIndex: 2),(AType:regNone))); @@ -200,6 +217,8 @@ type // function call(_label_id:Integer):t_jit_i_link; function jmp (_label_id:Integer):t_jit_i_link; + function jmp8(_label_id:Integer):t_jit_i_link; + function jcc (op:TOpCodeSuffix;_label_id:Integer):t_jit_i_link; function movj(reg:t_jit_reg;mem:t_jit_regs;_label_id:Integer):t_jit_i_link; function leaj(reg:t_jit_reg;mem:t_jit_regs;_label_id:Integer):t_jit_i_link; // @@ -214,6 +233,7 @@ type // procedure _mov (op,op8:Byte;reg:t_jit_reg;mem:t_jit_regs); procedure _mov (op,op8:Byte;reg0:t_jit_reg;reg1:t_jit_reg); + procedure _mov (op,op8,index:Byte;reg:t_jit_reg); procedure _movi (op,op8,index:Byte;reg:t_jit_reg;imm:Int64); procedure _movi (op,op8,index:Byte;size:TOperandSize;mem:t_jit_regs;imm:Int64); procedure _movi8 (op,index:Byte;reg:t_jit_reg;imm:Byte); @@ -224,6 +244,7 @@ type procedure movq (reg0:t_jit_reg ;reg1:t_jit_reg); procedure movi (size:TOperandSize;mem:t_jit_regs;imm:Int64); procedure movi (reg:t_jit_reg ;imm:Int64); + procedure movi64 (reg:t_jit_reg ;imm:Int64); procedure movq (reg:t_jit_reg ;mem:t_jit_regs); procedure movq (mem:t_jit_regs ;reg:t_jit_reg); procedure leaq (reg:t_jit_reg ;mem:t_jit_regs); @@ -232,11 +253,13 @@ type procedure addq (reg0:t_jit_reg ;reg1:t_jit_reg); procedure addi (reg:t_jit_reg ;imm:Int64); procedure addi8 (reg:t_jit_reg ;imm:Byte); + procedure addi8 (size:TOperandSize;mem:t_jit_regs;imm:Byte); procedure subq (mem:t_jit_regs ;reg:t_jit_reg); procedure subq (reg:t_jit_reg ;mem:t_jit_regs); procedure subq (reg0:t_jit_reg ;reg1:t_jit_reg); procedure subi (reg:t_jit_reg ;imm:Int64); procedure subi8 (reg:t_jit_reg ;imm:Byte); + procedure subi8 (size:TOperandSize;mem:t_jit_regs;imm:Byte); procedure xorq (reg0:t_jit_reg ;reg1:t_jit_reg); procedure cmpq (mem:t_jit_regs ;reg:t_jit_reg); procedure cmpq (reg:t_jit_reg ;mem:t_jit_regs); @@ -254,7 +277,10 @@ type procedure pop16 (mem:t_jit_regs); procedure pop64 (mem:t_jit_regs); procedure pop (reg:t_jit_reg); - procedure _vmov (Op,SimdOpcode:Byte;reg:t_jit_reg;mem:t_jit_regs); + procedure pushfq; + procedure popfq; + procedure _pmov (Op0,Op1:Byte;reg:t_jit_reg;mem:t_jit_regs); + procedure _vmov (Op0,Op1:Byte;reg:t_jit_reg;mem:t_jit_regs); procedure vmovdqu (reg:t_jit_reg ;mem:t_jit_regs); procedure vmovdqu (mem:t_jit_regs;reg:t_jit_reg); procedure vmovdqa (reg:t_jit_reg ;mem:t_jit_regs); @@ -272,6 +298,10 @@ operator + (A:t_jit_reg;B:Integer):t_jit_reg; operator + (A:t_jit_reg;B:Int64):t_jit_reg; operator * (A:t_jit_reg;B:Integer):t_jit_reg; +function classif_offset_32(AOffset:Integer):Byte; +function classif_offset_64(AOffset:Int64):Byte; +function classif_offset_se64(AOffset:Int64):Byte; + implementation function is_valid_reg_type(reg:TRegValue):Boolean; inline; @@ -306,9 +336,9 @@ begin end; end; -function classif_offset_32(reg:t_jit_reg):Byte; inline; +function classif_offset_32(AOffset:Integer):Byte; begin - case reg.AOffset of + case AOffset of 0:Result:=0; -128..-1,1..127:Result:=1; else @@ -316,9 +346,9 @@ begin end; end; -function classif_offset_64(reg:t_jit_reg):Byte; inline; +function classif_offset_64(AOffset:Int64):Byte; begin - case reg.AOffset of + case AOffset of 0:Result:=0; -128 .. -1, 1..127 :Result:=1; -2147483648..-129,128..2147483647:Result:=2; @@ -327,6 +357,27 @@ begin end; end; +function classif_offset_se64(AOffset:Int64):Byte; +begin + case AOffset of + 0:Result:=0; + 1..127:Result:=1; + 128..2147483647:Result:=2; + else + Result:=3; + end; +end; + +function classif_offset_32(reg:t_jit_reg):Byte; inline; +begin + Result:=classif_offset_32(reg.AOffset); +end; + +function classif_offset_64(reg:t_jit_reg):Byte; inline; +begin + Result:=classif_offset_64(reg.AOffset); +end; + function is_not_reg(reg:t_jit_reg):Boolean; inline; begin Result:=(not is_valid_reg_type(reg.ARegValue[0])) and @@ -689,6 +740,60 @@ begin Result.inst_id:=High(AInstructions); end; +function t_jit_builder.jmp8(_label_id:Integer):t_jit_i_link; +var + ji:t_jit_instruction; +begin + ji:=default_jit_instruction; + + ji.EmitByte($EB); + + ji.ALink.AType:=lnkLabel; + ji.ALink.ADataOffset:=ji.ASize; + ji.ALink.ALinkId:=_label_id; + + ji.EmitByte(0); + + _add(ji); + + Result.builder:=@self; + Result.inst_id:=High(AInstructions); +end; + +const + COND_32:array[OPSc_o..OPSc_nle] of Byte=( + $80,$81,$82,$83,$84,$85,$86,$87, + $88,$89,$8A,$8B,$8C,$8D,$8E,$8F + ); + +function t_jit_builder.jcc(op:TOpCodeSuffix;_label_id:Integer):t_jit_i_link; +var + ji:t_jit_instruction; +begin + ji:=default_jit_instruction; + + case op of + OPSc_o..OPSc_nle:; + else + Assert(false); + end; + + ji.EmitByte($0F); + + ji.EmitByte(COND_32[op]); + + ji.ALink.AType:=lnkLabel; + ji.ALink.ADataOffset:=ji.ASize; + ji.ALink.ALinkId:=_label_id; + + ji.EmitInt32(0); + + _add(ji); + + Result.builder:=@self; + Result.inst_id:=High(AInstructions); +end; + function t_jit_builder.movj(reg:t_jit_reg;mem:t_jit_regs;_label_id:Integer):t_jit_i_link; var i:Integer; @@ -818,7 +923,9 @@ end; type t_modrm_info=object - rexF,rexB,rexX,rexR:Boolean; + RH,rexF:Boolean; + + rexB,rexX,rexR:Boolean; ModRM:record Mode,Index,RM:Byte; @@ -830,12 +937,15 @@ type AOffset:Int64; - procedure build(Index:Byte;mreg:t_jit_reg); - procedure build(reg,mreg:t_jit_reg); - procedure emit(var ji:t_jit_instruction); + procedure build_im(Index:Byte;mreg:t_jit_reg); + procedure build_rm(reg,mreg:t_jit_reg); + procedure build_rr(reg0,reg1:t_jit_reg); + procedure build_ir(Index:Byte;reg:t_jit_reg); + procedure emit_rex(var ji:t_jit_instruction;rexW:Boolean); + procedure emit_mrm(var ji:t_jit_instruction); end; -procedure t_modrm_info.build(Index:Byte;mreg:t_jit_reg); +procedure t_modrm_info.build_im(Index:Byte;mreg:t_jit_reg); var ubase:Boolean; begin @@ -882,6 +992,10 @@ begin SIB.Index:=mreg.ARegValue[0].AIndex; if (SIB.Index>=8) then begin + if RH then + begin + Assert(false,'imposible'); + end; rexX:=true; Dec(SIB.Index,8); end; @@ -891,6 +1005,10 @@ begin SIB.Base:=mreg.ARegValue[1].AIndex; if (SIB.Base>=8) then begin + if RH then + begin + Assert(false,'imposible'); + end; rexB:=true; Dec(SIB.Base,8); end; @@ -936,6 +1054,10 @@ begin ModRM.RM:=mreg.ARegValue[0].AIndex; if (ModRM.RM>=8) then begin + if RH then + begin + Assert(false,'imposible'); + end; rexB:=true; Dec(ModRM.RM,8); end; @@ -969,31 +1091,92 @@ begin end; end; -procedure t_modrm_info.build(reg,mreg:t_jit_reg); +procedure t_modrm_info.build_rm(reg,mreg:t_jit_reg); begin ModRM.Index:=reg.ARegValue[0].AIndex; - case reg.ARegValue[0].AType of - regGeneralH: - begin - ModRM.Index:=ModRM.Index+4; - end; - else - begin - if (ModRM.Index>=8) then - begin - rexR:=true; - Dec(ModRM.Index,8); - end; - end; + if (reg.ARegValue[0].AType=regGeneralH) then + begin + RH:=True; + Inc(ModRM.Index,4); + end else + if (ModRM.Index>=8) then + begin + rexR:=true; + Dec(ModRM.Index,8); end; rexF:=get_force_rex(reg); - build(ModRM.Index,mreg); + build_im(ModRM.Index,mreg); end; -procedure t_modrm_info.emit(var ji:t_jit_instruction); +procedure t_modrm_info.build_rr(reg0,reg1:t_jit_reg); +begin + ModRM.Index:=reg1.ARegValue[0].AIndex; + + if (reg1.ARegValue[0].AType=regGeneralH) then + begin + RH:=True; + Inc(ModRM.Index,4); + end else + if (ModRM.Index>=8) then + begin + rexR:=true; + Dec(ModRM.Index,8); + end; + + rexF:=get_force_rex(reg1); + + ModRM.RM:=reg0.ARegValue[0].AIndex; + + if (reg0.ARegValue[0].AType=regGeneralH) then + begin + if rexF or rexR then + begin + Assert(false,'imposible'); + end; + RH:=True; + Inc(ModRM.RM,4) + end else + if (ModRM.RM>=8) then + begin + if RH then + begin + Assert(false,'imposible'); + end; + rexB:=true; + Dec(ModRM.RM,8); + end; + + ModRM.Mode:=3; +end; + +procedure t_modrm_info.build_ir(Index:Byte;reg:t_jit_reg); +begin + ModRM.Index:=Index; + + ModRM.RM:=reg.ARegValue[0].AIndex; + if (ModRM.RM>=8) then + begin + rexB:=true; + Dec(ModRM.RM,8); + end; + + rexF:=get_force_rex(reg); + + ModRM.Mode:=3; +end; + +procedure t_modrm_info.emit_rex(var ji:t_jit_instruction;rexW:Boolean); +begin + if rexF or rexB or rexX or rexR or rexW then + begin + ji.EmitREX(rexB,rexX,rexR,rexW); + end; +end; + +procedure t_modrm_info.emit_mrm(var ji:t_jit_instruction); begin ji.EmitModRM(ModRM.Mode,ModRM.Index,ModRM.RM); @@ -1017,7 +1200,7 @@ begin end; end; end; - 1:ji.EmitByte (AOffset); //1 + 1:ji.EmitByte(AOffset); //1 2:begin ji.ALink.ADataOffset:=ji.ASize; ji.EmitInt32(AOffset); //4 @@ -1074,7 +1257,7 @@ begin modrm_info:=Default(t_modrm_info); - modrm_info.build(reg,mreg); + modrm_info.build_rm(reg,mreg); ji.EmitSelector(mreg.ASegment); @@ -1088,25 +1271,20 @@ begin ji.EmitByte($67); //Address-size override prefix (32) end; - if modrm_info.rexF or modrm_info.rexB or modrm_info.rexX or modrm_info.rexR or rexW then - begin - ji.EmitREX(modrm_info.rexB,modrm_info.rexX,modrm_info.rexR,rexW); - end; + modrm_info.emit_rex(ji,rexW); ji.EmitByte(op); - modrm_info.emit(ji); + modrm_info.emit_mrm(ji); _add(ji); end; procedure t_jit_builder._mov(op,op8:Byte;reg0:t_jit_reg;reg1:t_jit_reg); var - rexF,rexB,rexR,rexW:Boolean; + rexW:Boolean; - ModRM:record - Index,RM:Byte; - end; + modrm_info:t_modrm_info; Prefix:Byte; @@ -1118,21 +1296,16 @@ begin Assert(is_reg_size(reg0,[os8,os16,os32,os64])); Assert(is_reg_size(reg1,[os8,os16,os32,os64])); - Assert(is_reg_type(reg0,[regGeneral])); - Assert(is_reg_type(reg1,[regGeneral])); + Assert(is_reg_type(reg0,[regGeneral,regGeneralH])); + Assert(is_reg_type(reg1,[regGeneral,regGeneralH])); Assert(reg0.ARegValue[0].AScale<=1); Assert(reg1.ARegValue[0].AScale<=1); Assert(reg0.ARegValue[0].ASize=reg1.ARegValue[0].ASize); - Assert(get_force_rex(reg0)=get_force_rex(reg1)); - ji:=default_jit_instruction; - rexF:=false; - rexB:=false; - rexR:=false; rexW:=false; Prefix:=0; @@ -1153,35 +1326,80 @@ begin else; end; - ModRM.Index:=reg1.ARegValue[0].AIndex; - if (ModRM.Index>=8) then - begin - rexR:=true; - Dec(ModRM.Index,8); - end; + modrm_info:=Default(t_modrm_info); - ModRM.RM:=reg0.ARegValue[0].AIndex; - if (ModRM.RM>=8) then - begin - rexB:=true; - Dec(ModRM.RM,8); - end; - - rexF:=get_force_rex(reg0); + modrm_info.build_rr(reg0,reg1); if (Prefix<>0) then begin ji.EmitByte(Prefix); //Operand-size override prefix (16) end; - if rexF or rexB or rexR or rexW then - begin - ji.EmitREX(rexB,False,rexR,rexW); - end; + modrm_info.emit_rex(ji,rexW); ji.EmitByte(op); - ji.EmitModRM(3,ModRM.Index,ModRM.RM); + modrm_info.emit_mrm(ji); + + _add(ji); +end; + +//// + +procedure t_jit_builder._mov(op,op8,index:Byte;reg:t_jit_reg); +var + rexW:Boolean; + + modrm_info:t_modrm_info; + + Prefix:Byte; + + ji:t_jit_instruction; +begin + Assert(is_one_reg(reg)); + + Assert(is_reg_size(reg,[os8,os16,os32,os64])); + + Assert(is_reg_type(reg,[regGeneral,regGeneralH])); + + Assert(reg.ARegValue[0].AScale<=1); + + ji:=default_jit_instruction; + + rexW:=false; + Prefix:=0; + + case reg.ARegValue[0].ASize of + os8: + begin + op:=op8; + end; + os16: + begin + Prefix:=$66; + end; + os32:; + os64: + begin + rexW:=True; + end; + else; + end; + + modrm_info:=Default(t_modrm_info); + + modrm_info.build_ir(Index,reg); + + if (Prefix<>0) then + begin + ji.EmitByte(Prefix); //Operand-size override prefix (16) + end; + + modrm_info.emit_rex(ji,rexW); + + ji.EmitByte(op); + + modrm_info.emit_mrm(ji); _add(ji); end; @@ -1190,34 +1408,28 @@ end; procedure t_jit_builder._movi(op,op8,index:Byte;reg:t_jit_reg;imm:Int64); var - rexF,rexB,rexW:Boolean; + rexW:Boolean; - RM,Prefix:Byte; + Prefix:Byte; + + modrm_info:t_modrm_info; ji:t_jit_instruction; begin Assert(is_one_reg(reg)); Assert(is_reg_size(reg,[os8,os16,os32,os64])); - Assert(is_reg_type(reg,[regGeneral])); + Assert(is_reg_type(reg,[regGeneral,regGeneralH])); Assert(reg.ARegValue[0].AScale<=1); ji:=default_jit_instruction; - rexF:=false; - rexB:=false; - rexW:=false; Prefix:=0; - RM:=reg.ARegValue[0].AIndex; - if (RM>=8) then - begin - rexB:=true; - Dec(RM,8); - end; + modrm_info:=Default(t_modrm_info); - rexF:=get_force_rex(reg); + modrm_info.build_ir(Index,reg); case reg.ARegValue[0].ASize of os8: @@ -1241,14 +1453,11 @@ begin ji.EmitByte(Prefix); //Operand-size override prefix (16) end; - if rexF or rexB or rexW then - begin - ji.EmitREX(rexB,False,False,rexW); - end; + modrm_info.emit_rex(ji,rexW); ji.EmitByte(op); - ji.EmitModRM(3,index,RM); + modrm_info.emit_mrm(ji); case reg.ARegValue[0].ASize of os8:ji.EmitByte (imm); @@ -1304,7 +1513,7 @@ begin modrm_info:=Default(t_modrm_info); - modrm_info.build(Index,mreg); + modrm_info.build_im(Index,mreg); ji.EmitSelector(mreg.ASegment); @@ -1318,20 +1527,17 @@ begin ji.EmitByte($67); //Address-size override prefix (32) end; - if modrm_info.rexF or modrm_info.rexB or modrm_info.rexX or modrm_info.rexR or rexW then - begin - ji.EmitREX(modrm_info.rexB,modrm_info.rexX,modrm_info.rexR,rexW); - end; + modrm_info.emit_rex(ji,rexW); ji.EmitByte(op); - modrm_info.emit(ji); + modrm_info.emit_mrm(ji); case Size of os8:ji.EmitByte (imm); os16:ji.EmitWord (imm); os32:ji.EmitInt32(imm); - os64:ji.EmitInt64(imm); + os64:ji.EmitInt32(imm); else; end; @@ -1340,34 +1546,29 @@ end; procedure t_jit_builder._movi8(op,index:Byte;reg:t_jit_reg;imm:Byte); var - rexF,rexB,rexW:Boolean; + rexW:Boolean; - RM,Prefix:Byte; + Prefix:Byte; + + modrm_info:t_modrm_info; ji:t_jit_instruction; begin Assert(is_one_reg(reg)); Assert(is_reg_size(reg,[os16,os32,os64])); - Assert(is_reg_type(reg,[regGeneral])); + Assert(is_reg_type(reg,[regGeneral,regGeneralH])); Assert(reg.ARegValue[0].AScale<=1); ji:=default_jit_instruction; - rexF:=false; - rexB:=false; rexW:=false; Prefix:=0; - RM:=reg.ARegValue[0].AIndex; - if (RM>=8) then - begin - rexB:=true; - Dec(RM,8); - end; + modrm_info:=Default(t_modrm_info); - rexF:=get_force_rex(reg); + modrm_info.build_ir(Index,reg); case reg.ARegValue[0].ASize of os16: @@ -1387,14 +1588,11 @@ begin ji.EmitByte(Prefix); //Operand-size override prefix (16) end; - if rexF or rexB or rexW then - begin - ji.EmitREX(rexB,False,False,rexW); - end; + modrm_info.emit_rex(ji,rexW); ji.EmitByte(op); - ji.EmitModRM(3,index,RM); + modrm_info.emit_mrm(ji); ji.EmitByte(imm); @@ -1440,7 +1638,7 @@ begin modrm_info:=Default(t_modrm_info); - modrm_info.build(Index,mreg); + modrm_info.build_im(Index,mreg); ji.EmitSelector(mreg.ASegment); @@ -1454,14 +1652,11 @@ begin ji.EmitByte($67); //Address-size override prefix (32) end; - if modrm_info.rexF or modrm_info.rexB or modrm_info.rexX or modrm_info.rexR or rexW then - begin - ji.EmitREX(modrm_info.rexB,modrm_info.rexX,modrm_info.rexR,rexW); - end; + modrm_info.emit_rex(ji,rexW); ji.EmitByte(op); - modrm_info.emit(ji); + modrm_info.emit_mrm(ji); ji.EmitByte(imm); //1 @@ -1490,7 +1685,7 @@ begin modrm_info:=Default(t_modrm_info); - modrm_info.build(index,mreg); + modrm_info.build_im(index,mreg); ji.EmitSelector(mreg.ASegment); @@ -1504,14 +1699,11 @@ begin ji.EmitByte($67); //Address-size override prefix (32) end; - if modrm_info.rexF or modrm_info.rexB or modrm_info.rexX or modrm_info.rexR then - begin - ji.EmitREX(modrm_info.rexB,modrm_info.rexX,modrm_info.rexR,False); - end; + modrm_info.emit_rex(ji,False); ji.EmitByte(op); - modrm_info.emit(ji); + modrm_info.emit_mrm(ji); _add(ji); end; @@ -1624,6 +1816,11 @@ begin end; procedure t_jit_builder.movi(reg:t_jit_reg;imm:Int64); +begin + _movi($C7,$C6,0,reg,imm); +end; + +procedure t_jit_builder.movi64(reg:t_jit_reg;imm:Int64); var rexF,rexB,rexW:Boolean; @@ -1692,7 +1889,7 @@ begin os8:ji.EmitByte (imm); os16:ji.EmitWord (imm); os32:ji.EmitInt32(imm); - os64:ji.EmitInt32(imm); + os64:ji.EmitInt64(imm); else; end; @@ -1745,6 +1942,11 @@ begin _movi8($83,0,reg,imm); end; +procedure t_jit_builder.addi8(size:TOperandSize;mem:t_jit_regs;imm:Byte); +begin + _movi8($83,0,size,mem,imm); +end; + // procedure t_jit_builder.subq(mem:t_jit_regs;reg:t_jit_reg); @@ -1772,6 +1974,11 @@ begin _movi8($83,5,reg,imm); end; +procedure t_jit_builder.subi8(size:TOperandSize;mem:t_jit_regs;imm:Byte); +begin + _movi8($83,5,size,mem,imm); +end; + /// procedure t_jit_builder.xorq(reg0:t_jit_reg;reg1:t_jit_reg); @@ -1864,7 +2071,78 @@ begin _push($58,reg); end; -procedure t_jit_builder._vmov(Op,SimdOpcode:Byte;reg:t_jit_reg;mem:t_jit_regs); +procedure t_jit_builder.pushfq; +var + ji:t_jit_instruction; +begin + ji:=default_jit_instruction; + + ji.EmitByte($9C); + + _add(ji); +end; + +procedure t_jit_builder.popfq; +var + ji:t_jit_instruction; +begin + ji:=default_jit_instruction; + + ji.EmitByte($9D); + + _add(ji); +end; + +procedure t_jit_builder._pmov(Op0,Op1:Byte;reg:t_jit_reg;mem:t_jit_regs); +var + mreg:t_jit_reg; + + modrm_info:t_modrm_info; + + ji:t_jit_instruction; +begin + Assert(is_one_reg(reg)); + Assert(is_reg_size(reg,[os64,os128])); + + Assert(is_reg_type(reg,[regMm,regXmm])); + + Assert(reg.ARegValue[0].AScale<=1); + + mreg:=Sums(mem); + + Assert(is_reg_size(mreg,[os0,os32,os64])); + Assert(is_reg_type(mreg,[regNone,regGeneral,regRip])); + Assert(is_valid_scale(mreg)); + + ji:=default_jit_instruction; + + modrm_info:=Default(t_modrm_info); + + modrm_info.build_rm(reg,mreg); + + ji.EmitSelector(mreg.ASegment); + + if (mreg.ARegValue[0].ASize=os32) then + begin + ji.EmitByte($67); //Address-size override prefix (32) + end; + + if (reg.ARegValue[0].ASize=os128) then + begin + ji.EmitByte($66); //xmm prefix + end; + + modrm_info.emit_rex(ji,False); + + ji.EmitByte(Op0); + ji.EmitByte(Op1); + + modrm_info.emit_mrm(ji); + + _add(ji); +end; + +procedure t_jit_builder._vmov(Op0,Op1:Byte;reg:t_jit_reg;mem:t_jit_regs); var mreg:t_jit_reg; @@ -1899,7 +2177,7 @@ begin modrm_info:=Default(t_modrm_info); - modrm_info.build(reg,mreg); + modrm_info.build_rm(reg,mreg); ji.EmitSelector(mreg.ASegment); @@ -1913,17 +2191,17 @@ begin ji.EmitByte($C4); //VEX3 ji.EmitRXBm(modrm_info.rexB,modrm_info.rexX,modrm_info.rexR,1); - ji.EmitWvvv(False,0,Vex.Length,SimdOpcode); + ji.EmitWvvv(False,0,Vex.Length,Op1); end else begin ji.EmitByte($C5); //VEX2 - ji.EmitRvvv(modrm_info.rexR,0,Vex.Length,SimdOpcode); + ji.EmitRvvv(modrm_info.rexR,0,Vex.Length,Op1); end; - ji.EmitByte(Op); + ji.EmitByte(Op0); - modrm_info.emit(ji); + modrm_info.emit_mrm(ji); _add(ji); end; diff --git a/sys/kern/kern_exec.pas b/sys/kern/kern_exec.pas index c5a1bb94..4b154c77 100644 --- a/sys/kern/kern_exec.pas +++ b/sys/kern/kern_exec.pas @@ -59,7 +59,8 @@ uses machdep, kern_dlsym, kern_authinfo, - vfs_syscalls; + vfs_syscalls, + kern_jit2; function exec_alloc_args(args:p_image_args):Integer; begin @@ -244,7 +245,7 @@ begin } map:=@vmspace^.vm_map; - sv_minuser:=VM_MINUSER_ADDRESS; + sv_minuser:=pmap_mem[0].start; sv_maxuser:=VM_MAXUSER_ADDRESS; if (vm_map_min(map)=sv_minuser) and @@ -999,6 +1000,12 @@ begin p_proc.libkernel___end_addr:=dynlibs_info.libkernel^.map_base + dynlibs_info.libkernel^.text_size; end; + kern_jit2.add_entry_point(dynlibs_info.libkernel^.entry_addr); + kern_jit2.add_entry_point(dynlibs_info.libkernel^.init_proc_addr); + kern_jit2.add_entry_point(dynlibs_info.libkernel^.fini_proc_addr); + + kern_jit2.pick(); + _dyn_not_exist: obj:=TAILQ_FIRST(@dynlibs_info.obj_list); diff --git a/sys/kern/kern_jit.pas b/sys/kern/kern_jit.pas index 505a6f4a..4523d945 100644 --- a/sys/kern/kern_jit.pas +++ b/sys/kern/kern_jit.pas @@ -629,7 +629,7 @@ begin movi(edx,copy_size); - call(@copyout_mov); //rdi,rsi,edx + call(@copyin_mov); //rdi,rsi,edx reta; diff --git a/sys/test/kern_jit2.pas b/sys/test/kern_jit2.pas new file mode 100644 index 00000000..77467685 --- /dev/null +++ b/sys/test/kern_jit2.pas @@ -0,0 +1,1984 @@ +unit kern_jit2; + +{$mode ObjFPC}{$H+} +{$CALLING SysV_ABI_CDecl} + +interface + +uses + mqueue, + kern_thr, + ucontext, + vmparam, + vm_pmap, + systm, + trap, + x86_fpdbgdisas, + x86_jit, + kern_stub; + +procedure add_entry_point(dst:Pointer); + +procedure pick(); + +implementation + +type + p_jit_frame=^jit_frame; + jit_frame=packed record + tf_rax:QWORD; + tf_rsp:QWORD; + tf_rbp:QWORD; + tf_r14:QWORD; + tf_r15:QWORD; + end; + +function GetFrameOffset(const RegValue:TRegValue):Integer; inline; +begin + Result:=-1; + + case RegValue.AType of + regGeneral: + begin + case RegValue.ASize of + os8, + os16, + os32, + os64: + begin + case RegValue.AIndex of + 0:Result:=Integer(@p_jit_frame(nil)^.tf_rax); + 4:Result:=Integer(@p_jit_frame(nil)^.tf_rsp); + 5:Result:=Integer(@p_jit_frame(nil)^.tf_rbp); + 14:Result:=Integer(@p_jit_frame(nil)^.tf_r14); + 15:Result:=Integer(@p_jit_frame(nil)^.tf_r15); + else; + end; + end; + else; + end; + end; + regGeneralH: + begin + case RegValue.ASize of + os8: + begin + case RegValue.AIndex of + 0:Result:=Integer(@p_jit_frame(nil)^.tf_rax)+1; + else; + end; + end; + else; + end; + end; + else; + end; + + Assert(Result<>-1,'GetFrameOffset'); +end; + +function GetFrameOffset(const r:TOperand):Integer; inline; +begin + Result:=GetFrameOffset(r.RegValue[0]) +end; + +function GetTargetOfs(var din:TInstruction;Code:PByte;id:Byte;var ofs:Int64):Boolean; +var + i:Integer; +begin + Result:=True; + i:=din.Operand[id].CodeIndex; + case din.Operand[id].ByteCount of + 1: ofs:=PShortint(@Code[i])^; + 2: ofs:=PSmallint(@Code[i])^; + 4: ofs:=PInteger (@Code[i])^; + 8: ofs:=PInt64 (@Code[i])^; + else + Result:=False; + end; +end; + +{ +change: rsp,rbp,rip +rax? + +eflahs? temp change? + +change: push/pop + +thread: r15 + +} + +function is_preserved(const r:TRegValue):Boolean; +begin + Result:=False; + + case r.AType of + regRip:Result:=True; + regGeneral: + begin + case r.AIndex of + 0, //rax + 4, //rsp + 5, //rbp + 14, //r14 + 15: //r15 + Result:=True; + else; + end; + end; + else; + end; +end; + +function is_preserved(const r:TRegValues):Boolean; inline; +begin + Result:=is_preserved(r[0]) or is_preserved(r[1]); +end; + +function is_preserved(const r:TOperand):Boolean; inline; +begin + Result:=is_preserved(r.RegValue); +end; + +function is_preserved(const r:TInstruction):Boolean; +var + i:Integer; +begin + Result:=False; + if (r.OperCnt<>0) then + For i:=1 to r.OperCnt do + begin + Result:=is_preserved(r.Operand[i]); + if Result then Exit; + end; +end; + +function is_rip(const r:TRegValue):Boolean; inline; +begin + Result:=False; + case r.AType of + regRip:Result:=True; + else; + end; +end; + +function is_memory(const r:TOperand):Boolean; inline; +begin + Result:=ofMemory in r.Flags; +end; + +function is_memory(const r:TInstruction):Boolean; +var + i:Integer; +begin + Result:=False; + if (r.OperCnt<>0) then + For i:=1 to r.OperCnt do + begin + Result:=is_memory(r.Operand[i]); + if Result then Exit; + end; +end; + +type + p_jit_context2=^t_jit_context2; + t_jit_context2=record + ptr_curr:Pointer; + ptr_next:Pointer; + + dis:TX86Disassembler; + din:TInstruction; + + builder:t_jit_builder; + + end; + + t_jit_cb=procedure(var ctx:t_jit_context2); + +var + jit_cbs:array[TOpCode,TOpCodeSuffix] of t_jit_cb; + +procedure add_orig(var ctx:t_jit_context2); +var + ji:t_jit_instruction; +begin + ji:=default_jit_instruction; + + Move(ctx.ptr_curr^,ji.AData,ctx.dis.CodeIdx); + + ji.ASize:=ctx.dis.CodeIdx; + + ctx.builder._add(ji); +end; + +const + r_thrd:t_jit_reg=(ARegValue:((AType:regGeneral;ASize:os64;AIndex:15),(AType:regNone))); //r15 + r_tmp0:t_jit_reg=(ARegValue:((AType:regGeneral;ASize:os64;AIndex: 0),(AType:regNone))); //rax + r_tmp1:t_jit_reg=(ARegValue:((AType:regGeneral;ASize:os64;AIndex:14),(AType:regNone))); //r14 + +function new_reg(const r:TRegValue):t_jit_reg; inline; +begin + Result:=Default(t_jit_reg); + Result.ARegValue[0]:=r; +end; + +function new_reg(const Operand:TOperand):t_jit_reg; inline; +begin + Result:=Default(t_jit_reg); + Result.ARegValue[0]:=Operand.RegValue[0]; +end; + +function new_reg_size(const r:t_jit_reg;ASize:TOperandSize):t_jit_reg; inline; +begin + Result:=r; + Result.ARegValue[0].ASize:=ASize; +end; + +function new_reg_size(const r:t_jit_reg;const RegValue:TRegValues):t_jit_reg; inline; +begin + Result:=r; + Result.ARegValue[0].ASize:=RegValue[0].ASize; +end; + +function new_reg_size(const r:t_jit_reg;const Operand:TOperand):t_jit_reg; inline; +begin + Result:=r; + Result.ARegValue[0].ASize:=Operand.RegValue[0].ASize; +end; + +procedure build_lea(var ctx:t_jit_context2;id:Byte;reg:t_jit_reg); +var + RegValue:TRegValues; + adr,new:t_jit_reg; + i:Integer; + ofs:Int64; +begin + Assert(ctx.din.SegmentReg=-1); + + RegValue:=ctx.din.Operand[id].RegValue; + + adr:=new_reg_size(reg,RegValue); + + with ctx.builder do + begin + if (RegValue[0].ASize<>os64) then + begin + xorq(reg,reg); + end; + + if is_rip(RegValue[0]) then + begin + ofs:=0; + GetTargetOfs(ctx.din,ctx.ptr_curr,id,ofs); + ofs:=Int64(ctx.ptr_next)+ofs; + + if (classif_offset_se64(ofs)=3) then + begin + movi64(adr,ofs); + end else + begin + movi(adr,ofs); + end; + end else + begin + if is_preserved(RegValue[0]) then + begin + i:=GetFrameOffset(RegValue[0]); + movq(adr,[r_thrd+i]); + end else + begin + new:=new_reg(RegValue[0]); + movq(adr,new); + end; + + if (RegValue[0].AScale>1) then + begin + ofs:=0; + if GetTargetOfs(ctx.din,ctx.ptr_curr,id,ofs) then + begin + leaq(adr,[adr*RegValue[0].AScale+ofs]); + end else + begin + leaq(adr,[adr*RegValue[0].AScale]); + end; + end else + begin + ofs:=0; + if GetTargetOfs(ctx.din,ctx.ptr_curr,id,ofs) then + if (ofs<>0) then + begin + leaq(adr,[adr+ofs]); + end; + end; + + if (RegValue[1].AType<>regNone) then + begin + if is_preserved(RegValue[1]) then + begin + i:=GetFrameOffset(RegValue[1]); + addq(adr,[r_thrd+i]); + end else + begin + new:=new_reg(RegValue[1]); + addq(adr,new); + end; + end; + end; + + end; + +end; + +type + t_memop_type=(mo_reg_reg, + mo_mem_reg, + mo_reg_mem, + mo_reg_imm, + mo_mem_imm, + + mo_ctx_reg, + mo_reg_ctx, + mo_ctx_ctx, + + mo_mem_ctx, + + mo_ctx_mem, + mo_ctx_imm + ); + +function get_lea_id(memop:t_memop_type):Byte; +begin + case memop of + mo_mem_reg:Result:=1; + mo_reg_mem:Result:=2; + mo_mem_imm:Result:=1; + mo_mem_ctx:Result:=1; + mo_ctx_mem:Result:=2; + else + Assert(False); + end; +end; + +function classif_memop2(var din:TInstruction):t_memop_type; +begin + if (ofMemory in din.Operand[1].Flags) then + begin + if (din.Operand[2].ByteCount<>0) then + begin + Result:=mo_mem_imm; + end else + if is_preserved(din.Operand[2]) then + begin + Result:=mo_mem_ctx; + end else + begin + Result:=mo_mem_reg; + end; + end else + if (ofMemory in din.Operand[2].Flags) then + begin + if is_preserved(din.Operand[1]) then + begin + Result:=mo_ctx_mem; + end else + begin + Result:=mo_reg_mem; + end; + end else + if (din.Operand[2].ByteCount<>0) then + begin + if is_preserved(din.Operand[1]) then + begin + Result:=mo_ctx_imm; + end else + begin + Result:=mo_reg_imm; + end; + end else + if is_preserved(din.Operand[1]) and + is_preserved(din.Operand[2]) then + begin + Result:=mo_ctx_ctx; + end else + if is_preserved(din.Operand[1]) then + begin + Result:=mo_ctx_reg; + end else + if is_preserved(din.Operand[2]) then + begin + Result:=mo_reg_ctx; + end else + begin + Result:=mo_reg_reg; + end; +end; + +type + t_op_type=packed record + op,op8,index:Byte; + end; + + t_op_desc=packed record + mem_reg:t_op_type; //reg_reg + reg_mem:t_op_type; + reg_imm:t_op_type; //mem_imm + reg_im8:t_op_type; //mem_im8 + mem__cl:t_op_type; //reg__cl + mem___1:t_op_type; //reg___1 + hint:Set of (his_mov,his_xor,his_cmp,his_align); + end; + +//in/out:rax uses:r14 +procedure uplift_jit; assembler; nostackframe; +label + _exit; +asm + pushfq + // + //low addr (r14) + mov %rax,%r14 + and PAGE_MASK,%r14 + //high addr (rax) + shr PAGE_SHIFT ,%rax + and PAGE_MAP_MASK,%rax + //uplift (rax) + mov PAGE_MAP,%rax + mov (%rax,%rax,4),%edi + //filter (rax) + and PAGE_OFS_MASK,%rax + jz _exit + //combine (rax|r14) + shl PAGE_SHIFT,%rax + or %r14,%rax + _exit: + // + popfq +end; + +//in:rax(addr),r14b:(mem_size) out:ZF +procedure page_test; assembler; nostackframe; +label + _exit; +asm + push %rdi + push %rsi + // + mov %rax,%rdi + movzbq %r14b,%rsi + sub $1,%rsi + //addr2:=addr+mem_high (rsi) + add %rdi,%rsi + //high addr (rdi,rsi) + shr PAGE_SHIFT ,%rdi + shr PAGE_SHIFT ,%rsi + and PAGE_MAP_MASK,%rdi + and PAGE_MAP_MASK,%rsi + // + cmp %rdi,%rsi + je _exit + //uplift (rdi,rsi) + lea (,%rdi,4),%rdi + lea (,%rsi,4),%rsi + // + add PAGE_MAP,%rdi + add PAGE_MAP,%rsi + // + mov (%rdi),%edi + mov (%rsi),%esi + //filter (rdi,rsi) + and PAGE_OFS_MASK,%rdi + and PAGE_OFS_MASK,%rsi + // + cmp %rdi,%rsi + _exit: + // + pop %rsi + pop %rdi +end; + +//in:rax(addr),r14b:(size) +procedure copyout_mov; assembler; +label + _simple, + _exit; +var + data:array[0..31] of Byte; +asm + pushfq + // + call page_test + je _simple + + push %rdi + push %rsi + push %rdx + push %rcx + push %r8 + push %r9 + push %r10 + push %r11 + + push %rax + + mov data,%rax + + mov 8(%rbp),%rdi //ret addr + add $2,%rdi //jmp near + call %rdi //reg->data + + pop %rsi //vaddr + mov data,%rdi //data + movzbq %r14b,%rdx //size + + call copyout + + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rcx + pop %rdx + pop %rsi + pop %rdi + + jmp _exit + _simple: + + call uplift_jit + + mov 8(%rbp),%r14 //ret addr + add $2,%r14 //jmp near + call %r14 //reg->data + + _exit: + // + popfq +end; + +//in:rax(addr),r14b:(size) out:rax +procedure copyin_mov; assembler; +label + _simple, + _exit; +var + data:array[0..31] of Byte; +asm + pushfq + // + call page_test + je _simple + + push %rdi + push %rsi + push %rdx + push %rcx + push %r8 + push %r9 + push %r10 + push %r11 + + mov %rax,%rdi //vaddr + mov data,%rsi //data + movzbq %r14b,%rdx //size + + call copyin + + pop %r11 + pop %r10 + pop %r9 + pop %r8 + pop %rcx + pop %rdx + pop %rsi + pop %rdi + + mov data,%rax //vaddr:=data + + jmp _exit + _simple: + + call uplift_jit + + _exit: + // + popfq +end; + +function cmp_reg(const r1,r2:TRegValue):Boolean; inline; +begin + Result:=(r1.AType =r2.AType) and + (r1.ASize =r2.ASize) and + (r1.AIndex=r2.AIndex); +end; + +const + OPERAND_BYTES:array[TOperandSize] of Word=(0,1,2,4,8,6,10,16,32,64,512); + +procedure op_emit2(var ctx:t_jit_context2;const desc:t_op_desc); +var + i:Integer; + memop:t_memop_type; + mem_size:TOperandSize; + link_next:t_jit_i_link; + + imm:Int64; + imm_size:TOperandSize; + + new,new2:t_jit_reg; + + procedure mem_out; + begin + with ctx.builder do + case memop of + mo_mem_reg: + begin + //input:rax + + new:=new_reg(ctx.din.Operand[2]); + _mov(desc.mem_reg.op,desc.mem_reg.op8,new,[r_tmp0]); + end; + mo_mem_imm: + begin + //input:rax + + imm:=0; + if not GetTargetOfs(ctx.din,ctx.ptr_curr,2,imm) then + begin + Assert(false); + end; + + imm_size:=ctx.din.Operand[2].Size; + + Assert(imm_size<>os64); + + if (imm_size=os8) and (mem_size<>os8) and (desc.reg_im8.index<>255) then + begin + _movi8(desc.reg_im8.op,desc.reg_im8.index,mem_size,[r_tmp0],imm); + end else + begin + _movi(desc.reg_imm.op,desc.reg_imm.op8,desc.reg_imm.index,mem_size,[r_tmp0],imm); + end; + end; + mo_mem_ctx: + begin + //input:rax + + new:=new_reg_size(r_tmp1,ctx.din.Operand[2]); + + i:=GetFrameOffset(ctx.din.Operand[2]); + movq(new,[r_thrd+i]); + + _mov(desc.mem_reg.op,desc.mem_reg.op8,new,[r_tmp0]); + end; + else; + end; + end; + + procedure mem_in; + begin + with ctx.builder do + case memop of + mo_reg_mem: + begin + //input:rax + + new:=new_reg(ctx.din.Operand[1]); + _mov(desc.reg_mem.op,desc.reg_mem.op8,new,[r_tmp0]); + end; + mo_ctx_mem: + begin + //input:rax + + new:=new_reg_size(r_tmp1,ctx.din.Operand[1]); + + i:=GetFrameOffset(ctx.din.Operand[1]); + movq(new,[r_thrd+i]); + + new:=new_reg(ctx.din.Operand[1]); + _mov(desc.reg_mem.op,desc.reg_mem.op8,new,[r_tmp0]); + + if not (his_cmp in desc.hint) then + begin + movq([r_thrd+i],new); + end; + + end; + else; + end; + end; + +begin + memop:=classif_memop2(ctx.din); + + with ctx.builder do + case memop of + mo_mem_reg, + mo_reg_mem, + mo_mem_imm, + mo_mem_ctx, + mo_ctx_mem: + begin + build_lea(ctx,get_lea_id(memop),r_tmp0); + mem_size:=ctx.din.Operand[get_lea_id(memop)].Size; + end; + else; + end; + + with ctx.builder do + case memop of + mo_mem_reg, + mo_mem_imm, + mo_mem_ctx: + begin + if (mem_size=os8) then + begin + call(@uplift_jit); //in/out:rax + + mem_out; + end else + begin + //mem_size + movi(new_reg_size(r_tmp1,os8),OPERAND_BYTES[mem_size]); + + call(@copyout_mov); //in:rax(addr),r14:(size) + + link_next:=jmp8(0); + + mem_out; + + reta; + + link_next._label:=_label; + end; + end; + + mo_reg_mem, + mo_ctx_mem: + begin + if (mem_size=os8) then + begin + call(@uplift_jit); //in/out:rax + + mem_in; + end else + begin + //mem_size + movi(new_reg_size(r_tmp1,os8),OPERAND_BYTES[mem_size]); + + call(@copyin_mov); //in:rax(addr),r14:(size) out:rax + + mem_in; + end; + end; + + mo_ctx_reg: + begin + new:=new_reg(ctx.din.Operand[2]); + + i:=GetFrameOffset(ctx.din.Operand[1]); + _mov(desc.mem_reg.op,desc.mem_reg.op8,new,[r_thrd+i]); + end; + mo_reg_ctx: + begin + new:=new_reg(ctx.din.Operand[1]); + + i:=GetFrameOffset(ctx.din.Operand[2]); + _mov(desc.reg_mem.op,desc.reg_mem.op8,new,[r_thrd+i]); + end; + mo_ctx_ctx: + begin + + if cmp_reg(ctx.din.Operand[1].RegValue[0], + ctx.din.Operand[2].RegValue[0]) then + begin + new:=new_reg_size(r_tmp0,ctx.din.Operand[1]); + + new2:=new; + + if not (his_xor in desc.hint) then + begin + i:=GetFrameOffset(ctx.din.Operand[1]); + movq(new,[r_thrd+i]); + end; + + end else + begin + new:=new_reg_size(r_tmp0,ctx.din.Operand[1]); + + new2:=new_reg_size(r_tmp1,ctx.din.Operand[1]); + + if (not (his_mov in desc.hint)) or + (new.ARegValue[0].ASize<>new2.ARegValue[0].ASize) then + begin + i:=GetFrameOffset(ctx.din.Operand[1]); + movq(new,[r_thrd+i]); + end; + + i:=GetFrameOffset(ctx.din.Operand[2]); + movq(new2,[r_thrd+i]); + end; + + _mov(desc.mem_reg.op,desc.mem_reg.op8,new,new2); + + if not (his_cmp in desc.hint) then + begin + i:=GetFrameOffset(ctx.din.Operand[1]); + movq([r_thrd+i],new); + end; + end; + mo_ctx_imm: + begin + mem_size:=ctx.din.Operand[1].Size; + i:=GetFrameOffset(ctx.din.Operand[1]); + + imm:=0; + if not GetTargetOfs(ctx.din,ctx.ptr_curr,2,imm) then + begin + Assert(false); + end; + + imm_size:=ctx.din.Operand[2].Size; + + if (imm_size=os8) and (mem_size<>os8) and (desc.reg_im8.index<>255) then + begin + _movi8(desc.reg_im8.op,desc.reg_im8.index,mem_size,[r_thrd+i],imm); + end else + if (classif_offset_se64(imm)=3) then + begin + Assert(his_mov in desc.hint); + + new:=new_reg_size(r_tmp0,ctx.din.Operand[1]); + + movi64(new,imm); + + movq([r_thrd+i],new); + end else + begin + _movi(desc.reg_imm.op,desc.reg_imm.op8,desc.reg_imm.index,mem_size,[r_thrd+i],imm); + end; + + end; + else + Assert(false); + end; + +end; + +procedure op_emit_mmx2(var ctx:t_jit_context2;const desc:t_op_desc); +var + i:Integer; + memop:t_memop_type; + mem_size:TOperandSize; + link_next:t_jit_i_link; + + new:t_jit_reg; + + procedure mem_out; + begin + with ctx.builder do + case memop of + mo_mem_reg: + begin + //input:rax + + new:=new_reg(ctx.din.Operand[2]); + _pmov(desc.mem_reg.op,desc.mem_reg.op8,new,[r_tmp0]); + end; + else; + end; + end; + + procedure mem_in; + begin + with ctx.builder do + case memop of + mo_reg_mem: + begin + //input:rax + + new:=new_reg(ctx.din.Operand[1]); + _pmov(desc.reg_mem.op,desc.reg_mem.op8,new,[r_tmp0]); + end; + else; + end; + end; + +begin + memop:=classif_memop2(ctx.din); + + with ctx.builder do + case memop of + mo_mem_reg, + mo_reg_mem: + begin + build_lea(ctx,get_lea_id(memop),r_tmp0); + mem_size:=ctx.din.Operand[get_lea_id(memop)].Size; + end; + else; + end; + + with ctx.builder do + case memop of + mo_mem_reg: + begin + if (his_align in desc.hint) then + begin + call(@uplift_jit); //in/out:rax + + mem_out; + end else + begin + //mem_size + movi(new_reg_size(r_tmp1,os8),OPERAND_BYTES[mem_size]); + + call(@copyout_mov); //in:rax(addr),r14:(size) + + link_next:=jmp8(0); + + mem_out; + + reta; + + link_next._label:=_label; + end; + end; + + mo_reg_mem: + begin + if (his_align in desc.hint) then + begin + call(@uplift_jit); //in/out:rax + + mem_in; + end else + begin + //mem_size + movi(new_reg_size(r_tmp1,os8),OPERAND_BYTES[mem_size]); + + call(@copyin_mov); //in:rax(addr),r14:(size) out:rax + + mem_in; + end; + end; + + mo_ctx_reg: + begin + new:=new_reg(ctx.din.Operand[2]); + + i:=GetFrameOffset(ctx.din.Operand[1]); + _pmov(desc.mem_reg.op,desc.mem_reg.op8,new,[r_thrd+i]); + end; + mo_reg_ctx: + begin + new:=new_reg(ctx.din.Operand[1]); + + i:=GetFrameOffset(ctx.din.Operand[2]); + _pmov(desc.reg_mem.op,desc.reg_mem.op8,new,[r_thrd+i]); + end; + + else + Assert(false); + end; + +end; + +// + +procedure op_emit_avx2(var ctx:t_jit_context2;const desc:t_op_desc); +var + i:Integer; + memop:t_memop_type; + mem_size:TOperandSize; + link_next:t_jit_i_link; + + new:t_jit_reg; + + procedure mem_out; + begin + with ctx.builder do + case memop of + mo_mem_reg: + begin + //input:rax + + new:=new_reg(ctx.din.Operand[2]); + _vmov(desc.mem_reg.op,desc.mem_reg.op8,new,[r_tmp0]); + end; + else; + end; + end; + + procedure mem_in; + begin + with ctx.builder do + case memop of + mo_reg_mem: + begin + //input:rax + + new:=new_reg(ctx.din.Operand[1]); + _vmov(desc.reg_mem.op,desc.reg_mem.op8,new,[r_tmp0]); + end; + else; + end; + end; + +begin + memop:=classif_memop2(ctx.din); + + with ctx.builder do + case memop of + mo_mem_reg, + mo_reg_mem: + begin + build_lea(ctx,get_lea_id(memop),r_tmp0); + mem_size:=ctx.din.Operand[get_lea_id(memop)].Size; + end; + else; + end; + + with ctx.builder do + case memop of + mo_mem_reg: + begin + if (his_align in desc.hint) then + begin + call(@uplift_jit); //in/out:rax + + mem_out; + end else + begin + //mem_size + movi(new_reg_size(r_tmp1,os8),OPERAND_BYTES[mem_size]); + + call(@copyout_mov); //in:rax(addr),r14:(size) + + link_next:=jmp8(0); + + mem_out; + + reta; + + link_next._label:=_label; + end; + end; + + mo_reg_mem: + begin + if (his_align in desc.hint) then + begin + call(@uplift_jit); //in/out:rax + + mem_in; + end else + begin + //mem_size + movi(new_reg_size(r_tmp1,os8),OPERAND_BYTES[mem_size]); + + call(@copyin_mov); //in:rax(addr),r14:(size) out:rax + + mem_in; + end; + end; + + mo_ctx_reg: + begin + new:=new_reg(ctx.din.Operand[2]); + + i:=GetFrameOffset(ctx.din.Operand[1]); + _vmov(desc.mem_reg.op,desc.mem_reg.op8,new,[r_thrd+i]); + end; + mo_reg_ctx: + begin + new:=new_reg(ctx.din.Operand[1]); + + i:=GetFrameOffset(ctx.din.Operand[2]); + _vmov(desc.reg_mem.op,desc.reg_mem.op8,new,[r_thrd+i]); + end; + + else + Assert(false); + end; + +end; + +procedure print_disassemble(addr:Pointer;vsize:Integer); +var + proc:TDbgProcess; + adec:TX86AsmDecoder; + ptr,fin:Pointer; + ACodeBytes,ACode:RawByteString; +begin + ptr:=addr; + fin:=addr+vsize; + + proc:=TDbgProcess.Create(dm64); + adec:=TX86AsmDecoder.Create(proc); + + while (ptrnil) do + begin + if (entry^.src=src) then + begin + Exit(entry^.id); + end; + + entry:=TAILQ_NEXT(entry,@entry^.link); + end; + +end; + +procedure add_entry_point(head:P_TAILQ_HEAD;id:t_jit_i_link;dst:Pointer); +var + entry:p_jit_entry_point; +begin + entry:=AllocMem(Sizeof(t_jit_entry_point)); + entry^.id:=id; + entry^.dst:=dst; + TAILQ_INSERT_TAIL(head,entry,@entry^.link); +end; + +procedure add_entry_point(dst:Pointer); +begin + if (dst=nil) then Exit; + add_entry_point(@lfrwrd,Default(t_jit_i_link),dst); +end; + +function fetch_entry_point(head:P_TAILQ_HEAD;var id:t_jit_i_link;var dst:Pointer):Boolean; +var + entry:p_jit_entry_point; + min:p_jit_entry_point; +begin + Result:=false; + min:=nil; + + entry:=TAILQ_FIRST(head); + while (entry<>nil) do + begin + if (min=nil) then + begin + min:=entry; + end else + if (min^.dst>entry^.dst) then + begin + min:=entry; + end; + + entry:=TAILQ_NEXT(entry,@entry^.link); + end; + + if (min<>nil) then + begin + id :=min^.id; + dst:=min^.dst; + TAILQ_REMOVE(head,min,@min^.link); + FreeMem(min); + Result:=True; + end; +end; + +procedure op_jmp_dispatcher(var ctx:t_jit_context2); +begin + ctx.builder.call(nil); //input:rax TODO jmp dispatcher +end; + +procedure op_push_rip(var ctx:t_jit_context2;used_r_tmp0:Boolean); +var + i:Integer; + stack:t_jit_reg; + imm:Int64; +begin + with ctx.builder do + begin + stack:=r_tmp1; + + i:=GetFrameOffset(rsp.ARegValue[0]); + movq(stack,[r_thrd+i]); + leaq(stack,[stack+(-8)]); + movq([r_thrd+i],stack); + + imm:=Int64(ctx.ptr_next); + + if (classif_offset_se64(imm)=3) then + begin + if used_r_tmp0 then + begin + push(r_tmp0); + end; + + movi64(r_tmp0,imm); + movq([stack],r_tmp0); + + if used_r_tmp0 then + begin + pop(r_tmp0); + end; + end else + begin + movi(os64,[stack],imm); + end; + + end; +end; + +procedure op_pop_rip(var ctx:t_jit_context2); +var + i:Integer; + new,stack:t_jit_reg; +begin + with ctx.builder do + begin + new:=r_tmp0; + stack:=r_tmp1; + + i:=GetFrameOffset(rsp.ARegValue[0]); + movq(stack,[r_thrd+i]); + + movq(new,[stack]); + + leaq(stack,[stack+(+8)]); + movq([r_thrd+i],stack); + end; +end; + +function is_rsp(const r:TRegValue):Boolean; inline; +begin + Result:=False; + case r.AType of + regGeneral: + case r.AIndex of + 4:Result:=True; + else; + end; + else; + end; +end; + +function is_rsp(const r:TRegValues):Boolean; inline; +begin + Result:=is_rsp(r[0]) or is_rsp(r[1]); +end; + +procedure op_call(var ctx:t_jit_context2); +var + id:t_jit_i_link; + ofs:Int64; + dst:Pointer; + new,new2:t_jit_reg; + i:Integer; +begin + if (ctx.din.Operand[1].RegValue[0].AType=regNone) then + begin + ofs:=0; + if not GetTargetOfs(ctx.din,ctx.ptr_curr,1,ofs) then + begin + Assert(false); + end; + + dst:=ctx.ptr_next+ofs; + + i:=find_jit_label(@labels,dst); + + if (i<>-1) then + begin + op_push_rip(ctx,false); + // + ctx.builder.jmp(i); + end else + begin + op_push_rip(ctx,false); + // + id:=ctx.builder.jmp(-1); + add_entry_point(@lfrwrd,id,dst); + end; + end else + if is_memory(ctx.din) then + begin + new:=new_reg_size(r_tmp0,ctx.din.Operand[1]); + // + if is_rsp(ctx.din.Operand[1].RegValue) then + begin + build_lea(ctx,1,new); + // + op_push_rip(ctx,true); + end else + begin + op_push_rip(ctx,false); + // + build_lea(ctx,1,new); + end; + // + op_jmp_dispatcher(ctx); + end else + if is_preserved(ctx.din) then + begin + new:=new_reg_size(r_tmp0,ctx.din.Operand[1]); + // + op_push_rip(ctx,false); + // + i:=GetFrameOffset(ctx.din.Operand[1].RegValue[0]); + ctx.builder.movq(new,[r_thrd+i]); + // + op_jmp_dispatcher(ctx); + end else + begin + op_push_rip(ctx,false); + // + new:=new_reg_size(r_tmp0,ctx.din.Operand[1]); + new2:=new_reg(ctx.din.Operand[1]); + // + ctx.builder.movq(new,new2); + // + op_jmp_dispatcher(ctx); + end; +end; + +procedure op_ret(var ctx:t_jit_context2); +begin + Assert(ctx.din.Operand[1].ByteCount=0); + + op_pop_rip(ctx); //out:rax + // + op_jmp_dispatcher(ctx); + // + ctx.ptr_next:=nil; //trim +end; + +procedure op_jmp(var ctx:t_jit_context2); +var + id:t_jit_i_link; + ofs:Int64; + dst:Pointer; + new,new2:t_jit_reg; + i:Integer; +begin + if (ctx.din.Operand[1].RegValue[0].AType=regNone) then + begin + ofs:=0; + if not GetTargetOfs(ctx.din,ctx.ptr_curr,1,ofs) then + begin + Assert(false); + end; + + dst:=ctx.ptr_next+ofs; + + i:=find_jit_label(@labels,dst); + + if (i<>-1) then + begin + ctx.builder.jmp(i); + end else + begin + id:=ctx.builder.jmp(-1); + add_entry_point(@lfrwrd,id,dst); + end; + + end else + if is_memory(ctx.din) then + begin + new:=new_reg_size(r_tmp0,ctx.din.Operand[1]); + // + build_lea(ctx,1,new); + // + op_jmp_dispatcher(ctx); + end else + if is_preserved(ctx.din) then + begin + new:=new_reg_size(r_tmp0,ctx.din.Operand[1]); + // + i:=GetFrameOffset(ctx.din.Operand[1].RegValue[0]); + ctx.builder.movq(new,[r_thrd+i]); + // + op_jmp_dispatcher(ctx); + end else + begin + new:=new_reg_size(r_tmp0,ctx.din.Operand[1]); + new2:=new_reg(ctx.din.Operand[1]); + // + ctx.builder.movq(new,new2); + // + op_jmp_dispatcher(ctx); + end; + // + ctx.ptr_next:=nil; //trim +end; + +procedure op_jcc(var ctx:t_jit_context2); +var + id:t_jit_i_link; + ofs:Int64; + dst:Pointer; + i:Integer; +begin + ofs:=0; + if not GetTargetOfs(ctx.din,ctx.ptr_curr,1,ofs) then + begin + Assert(false); + end; + + dst:=ctx.ptr_next+ofs; + + i:=find_jit_label(@labels,dst); + + if (i<>-1) then + begin + ctx.builder.jcc(ctx.din.OpCode.Suffix,i); + end else + begin + id:=ctx.builder.jcc(ctx.din.OpCode.Suffix,-1); + add_entry_point(@lfrwrd,id,dst); + end; +end; + +const + test_desc:t_op_desc=( + mem_reg:(op:$85;op8:$84;index:0); + reg_mem:(op:$00;op8:$00;index:0); + reg_imm:(op:$F7;op8:$F6;index:0); + reg_im8:(op:$00;op8:$00;index:0); + mem__cl:(op:$00;op8:$00;index:255); + mem___1:(op:$00;op8:$00;index:255); + hint:[his_cmp]; + ); + +procedure op_test(var ctx:t_jit_context2); +begin + if is_preserved(ctx.din) or is_memory(ctx.din) then + begin + op_emit2(ctx,test_desc); + end else + begin + add_orig(ctx); + end; +end; + +const + cmp_desc:t_op_desc=( + mem_reg:(op:$39;op8:$38;index:0); + reg_mem:(op:$3B;op8:$3A;index:0); + reg_imm:(op:$81;op8:$80;index:7); + reg_im8:(op:$83;op8:$83;index:7); + mem__cl:(op:$00;op8:$00;index:255); + mem___1:(op:$00;op8:$00;index:255); + hint:[his_cmp]; + ); + +procedure op_cmp(var ctx:t_jit_context2); +begin + if is_preserved(ctx.din) or is_memory(ctx.din) then + begin + op_emit2(ctx,cmp_desc); + end else + begin + add_orig(ctx); + end; +end; + +procedure op_lea(var ctx:t_jit_context2); +var + new:t_jit_reg; + i:Integer; +begin + if is_preserved(ctx.din) then + begin + if is_preserved(ctx.din.Operand[1]) then + begin + new:=new_reg_size(r_tmp0,ctx.din.Operand[1]); + build_lea(ctx,2,new); + // + i:=GetFrameOffset(ctx.din.Operand[1].RegValue[0]); + ctx.builder.movq([r_thrd+i],new); + end else + begin + new:=new_reg(ctx.din.Operand[1]); + build_lea(ctx,2,new); + end; + end else + begin + add_orig(ctx); + end; +end; + +procedure op_push(var ctx:t_jit_context2); +var + i:Integer; + stack,new:t_jit_reg; +begin + with ctx.builder do + if is_memory(ctx.din) then + begin + Assert(false); + end else + begin + stack:=r_tmp0; + + if is_preserved(ctx.din) then + begin + new:=new_reg_size(r_tmp1,ctx.din.Operand[1]); + + i:=GetFrameOffset(rsp.ARegValue[0]); + movq(stack,[r_thrd+i]); + leaq(stack,[stack+(-OPERAND_BYTES[new.ARegValue[0].ASize])]); + movq([r_thrd+i],stack); + + i:=GetFrameOffset(ctx.din.Operand[1]); + movq(new,[r_thrd+i]); + + movq([stack],new); + end else + begin + new:=new_reg(ctx.din.Operand[1]); + + i:=GetFrameOffset(rsp.ARegValue[0]); + movq(stack,[r_thrd+i]); + leaq(stack,[stack+(-OPERAND_BYTES[new.ARegValue[0].ASize])]); + movq([r_thrd+i],stack); + + movq([stack],new); + end; + + end; +end; + +procedure op_pop(var ctx:t_jit_context2); +var + i:Integer; + stack,new:t_jit_reg; +begin + with ctx.builder do + if is_memory(ctx.din) then + begin + Assert(false); + end else + begin + stack:=r_tmp0; + + i:=GetFrameOffset(rsp.ARegValue[0]); + movq(stack,[r_thrd+i]); + + if is_preserved(ctx.din) then + begin + new:=new_reg_size(r_tmp1,ctx.din.Operand[1]); + + movq(new,[stack]); + + i:=GetFrameOffset(ctx.din.Operand[1]); + movq([r_thrd+i],new); + end else + begin + new:=new_reg(ctx.din.Operand[1]); + + movq(new,[stack]); + end; + + leaq(stack,[stack+OPERAND_BYTES[new.ARegValue[0].ASize]]); + movq([r_thrd+i],stack); + end; +end; + +procedure op_int(var ctx:t_jit_context2); +var + i:Integer; + id:Byte; +begin + i:=ctx.din.Operand[1].ByteCount; + Assert(i=1); + id:=PByte(ctx.ptr_curr)[i]; + + case id of + $44: //system error? + begin + // + ctx.builder.call(nil); //TODO error dispatcher + ctx.ptr_next:=nil; //trim + end; + else + begin + Assert(False); + end; + end; + +end; + +const + pxor_desc:t_op_desc=( + mem_reg:(op:$0F;op8:$EF;index:0); + reg_mem:(op:$00;op8:$00;index:255); + reg_imm:(op:$00;op8:$00;index:255); + reg_im8:(op:$00;op8:$00;index:255); + mem__cl:(op:$00;op8:$00;index:255); + mem___1:(op:$00;op8:$00;index:255); + hint:[his_xor]; + ); + +procedure op_pxor(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + op_emit_mmx2(ctx,pxor_desc); + end else + begin + add_orig(ctx); + end; +end; + +procedure op_vxorps(var ctx:t_jit_context2); +begin + if is_memory(ctx.din) then + begin + Assert(False); + end else + begin + add_orig(ctx); + end; +end; + +procedure pick(); +var + addr:Pointer; + id:t_jit_i_link; + + proc:TDbgProcess; + adec:TX86AsmDecoder; + ptr,fin:Pointer; + ACodeBytes,ACode:RawByteString; + ctx:t_jit_context2; + cb:t_jit_cb; + + len1,len2,i:Integer; +begin + + if not fetch_entry_point(@lfrwrd,id,addr) then + begin + Assert(false); + end; + + ptr:=addr; + fin:=Pointer(Int64(-1)); + + Writeln(SizeOf(TOpCode),' ',Succ(Ord(High(TOpCode)))); + Writeln(SizeOf(TOpCodeSuffix),' ',Succ(Ord(High(TOpCodeSuffix)))); + + jit_cbs[OPxor ,OPSnone]:=@op_xor; + jit_cbs[OPand ,OPSnone]:=@op_and; + jit_cbs[OPsub ,OPSnone]:=@op_sub; + jit_cbs[OPadd ,OPSnone]:=@op_add; + + jit_cbs[OPmov ,OPSnone]:=@op_mov; + jit_cbs[OPmovu,OPSx_ps]:=@op_vmovups; + + jit_cbs[OPcall,OPSnone]:=@op_call; + jit_cbs[OPjmp ,OPSnone]:=@op_jmp; + jit_cbs[OPret ,OPSnone]:=@op_ret; + + jit_cbs[OPtest,OPSnone]:=@op_test; + jit_cbs[OPcmp ,OPSnone]:=@op_cmp; + + jit_cbs[OPj__,OPSc_o ]:=@op_jcc; + jit_cbs[OPj__,OPSc_no ]:=@op_jcc; + jit_cbs[OPj__,OPSc_b ]:=@op_jcc; + jit_cbs[OPj__,OPSc_nb ]:=@op_jcc; + jit_cbs[OPj__,OPSc_z ]:=@op_jcc; + jit_cbs[OPj__,OPSc_nz ]:=@op_jcc; + jit_cbs[OPj__,OPSc_be ]:=@op_jcc; + jit_cbs[OPj__,OPSc_nbe]:=@op_jcc; + jit_cbs[OPj__,OPSc_s ]:=@op_jcc; + jit_cbs[OPj__,OPSc_ns ]:=@op_jcc; + jit_cbs[OPj__,OPSc_p ]:=@op_jcc; + jit_cbs[OPj__,OPSc_np ]:=@op_jcc; + jit_cbs[OPj__,OPSc_l ]:=@op_jcc; + jit_cbs[OPj__,OPSc_nl ]:=@op_jcc; + jit_cbs[OPj__,OPSc_le ]:=@op_jcc; + jit_cbs[OPj__,OPSc_nle]:=@op_jcc; + + jit_cbs[OPpush,OPSnone]:=@op_push; + jit_cbs[OPpop ,OPSnone]:=@op_pop; + + jit_cbs[OPpxor,OPSnone]:=@op_pxor; + jit_cbs[OPemms,OPSnone]:=@add_orig; + jit_cbs[OPvzeroall,OPSnone]:=@add_orig; + jit_cbs[OPxor,OPSx_ps]:=@op_vxorps; + + //OPmovu OPSx_ps + + jit_cbs[OPlea,OPSnone]:=@op_lea; + jit_cbs[OPint,OPSnone]:=@op_int; + + proc:=TDbgProcess.Create(dm64); + adec:=TX86AsmDecoder.Create(proc); + + add_jit_label(@labels,ptr,0); + + + while (ptrlen2) then + begin + Writeln('recompiled----------------------':32,' ',''); + For i:=len1 to len2-1 do + begin + + print_disassemble(@ctx.builder.AInstructions[i].AData, + ctx.builder.AInstructions[i].ASize); + + end; + Writeln('recompiled----------------------':32,' ',''); + end; + + if (len1<>len2) then + begin + i:=find_jit_label(@labels,ptr); + + if (i<>-1) then + begin + ctx.builder.jmp(i); + ctx.ptr_next:=nil; + end else + begin + add_jit_label(@labels,ptr,len2); + end; + + end; + + if (ctx.ptr_next=nil) then + begin + repeat + + if not fetch_entry_point(@lfrwrd,id,addr) then + begin + Assert(false); + end; + + i:=find_jit_label(@labels,addr); + if (i=-1) then + begin + Writeln('not found:0x',HexStr(addr)); + writeln; + Break; + end; + + until false; + + ptr:=addr; + end; + + //ptr:=ctx.ptr_next; //jmp switch + end; + + adec.Free; + proc.Free; +end; + + +end. + diff --git a/sys/test/project1.lpi b/sys/test/project1.lpi index 366c8310..86b3b423 100644 --- a/sys/test/project1.lpi +++ b/sys/test/project1.lpi @@ -738,6 +738,10 @@ + + + + diff --git a/sys/vm/vmparam.pas b/sys/vm/vmparam.pas index 23747cc6..93bd62f4 100644 --- a/sys/vm/vmparam.pas +++ b/sys/vm/vmparam.pas @@ -52,7 +52,7 @@ const SCE_KERNEL_GNMDRIVER =QWORD($00FE0000000); - VM_MINUSER_ADDRESS =QWORD($00200000000); + VM_MINUSER_ADDRESS =QWORD($00200000000); //(original:$000000000000) VM_MAXUSER_ADDRESS =QWORD($10000000000); //(original:$800000000000) VM_MIN_GPU_ADDRESS =QWORD($10000000000);