From ac50c4ca2b6607c26f8b4312f68681abccb5998b Mon Sep 17 00:00:00 2001 From: Pavel <68122101+red-prig@users.noreply.github.com> Date: Thu, 31 Aug 2023 12:08:16 +0300 Subject: [PATCH] + --- rtl/x86_jit.pas | 492 ++++++++++++++++++++++++++++++++-------- sys/kern/kern_dmem.pas | 2 +- sys/kern/trap.pas | 34 ++- sys/md/md_exception.pas | 3 + sys/vm/vm_fault.pas | 335 ++++++++++++++++++++++----- 5 files changed, 692 insertions(+), 174 deletions(-) diff --git a/rtl/x86_jit.pas b/rtl/x86_jit.pas index ad2cc810..1428445c 100644 --- a/rtl/x86_jit.pas +++ b/rtl/x86_jit.pas @@ -22,7 +22,6 @@ type TRegisterTypeSet=Set of TRegisterType; t_jit_regs =array of t_jit_reg; - t_jit_int64=array of Int64; t_jit_link_type=(lnkNone,lnkData,lnkLabel); @@ -74,6 +73,11 @@ type t_jit_builder=object Const + ah :t_jit_reg=(ARegValue:((AType:regGeneralH;ASize:os8;AIndex:0),(AType:regNone))); + ch :t_jit_reg=(ARegValue:((AType:regGeneralH;ASize:os8;AIndex:1),(AType:regNone))); + dh :t_jit_reg=(ARegValue:((AType:regGeneralH;ASize:os8;AIndex:2),(AType:regNone))); + bh :t_jit_reg=(ARegValue:((AType:regGeneralH;ASize:os8;AIndex:3),(AType:regNone))); + al :t_jit_reg=(ARegValue:((AType:regGeneral;ASize:os8;AIndex: 0),(AType:regNone))); cl :t_jit_reg=(ARegValue:((AType:regGeneral;ASize:os8;AIndex: 1),(AType:regNone))); dl :t_jit_reg=(ARegValue:((AType:regGeneral;ASize:os8;AIndex: 2),(AType:regNone))); @@ -208,32 +212,38 @@ type Procedure LinkData; Function SaveTo(ptr:PByte;size:Integer):Integer; // - procedure _mov (op:Byte;reg:t_jit_reg;mem:t_jit_regs); + procedure _mov (op,op8:Byte;reg:t_jit_reg;mem:t_jit_regs); procedure _mov (op,op8:Byte;reg0:t_jit_reg;reg1:t_jit_reg); - procedure _addi (op,op8,index:Byte;reg:t_jit_reg;imm:Int64); + procedure _movi (op,op8,index:Byte;reg:t_jit_reg;imm:Int64); + procedure _movi (op,op8,index:Byte;size:TOperandSize;mem:t_jit_regs;imm:Int64); + procedure _movi8 (op,index:Byte;reg:t_jit_reg;imm:Byte); + procedure _movi8 (op,index:Byte;size:TOperandSize;mem:t_jit_regs;imm:Byte); procedure _push (op,index:Byte;size:TOperandSize;mem:t_jit_regs); procedure _push (op:Byte;reg:t_jit_reg); procedure _pushi (size:TOperandSize;imm:Integer); procedure movq (reg0:t_jit_reg ;reg1:t_jit_reg); procedure movi (reg:t_jit_reg ;imm:Int64); procedure movq (reg:t_jit_reg ;mem:t_jit_regs); - procedure movq (reg:t_jit_reg ;mem:t_jit_int64); procedure movq (mem:t_jit_regs ;reg:t_jit_reg); - procedure movq (mem:t_jit_int64;reg:t_jit_reg); - procedure lea (reg:t_jit_reg ;mem:t_jit_regs); - procedure lea (reg:t_jit_reg ;mem:t_jit_int64); + procedure leaq (reg:t_jit_reg ;mem:t_jit_regs); procedure addq (mem:t_jit_regs ;reg:t_jit_reg); - procedure addq (mem:t_jit_int64;reg:t_jit_reg); procedure addq (reg:t_jit_reg ;mem:t_jit_regs); - procedure addq (reg:t_jit_reg ;mem:t_jit_int64); procedure addq (reg0:t_jit_reg ;reg1:t_jit_reg); procedure addi (reg:t_jit_reg ;imm:Int64); + procedure addi8 (reg:t_jit_reg ;imm:Byte); procedure subq (mem:t_jit_regs ;reg:t_jit_reg); - procedure subq (mem:t_jit_int64;reg:t_jit_reg); procedure subq (reg:t_jit_reg ;mem:t_jit_regs); - procedure subq (reg:t_jit_reg ;mem:t_jit_int64); procedure subq (reg0:t_jit_reg ;reg1:t_jit_reg); procedure subi (reg:t_jit_reg ;imm:Int64); + procedure subi8 (reg:t_jit_reg ;imm:Byte); + procedure xorq (reg0:t_jit_reg ;reg1:t_jit_reg); + procedure cmpq (mem:t_jit_regs ;reg:t_jit_reg); + procedure cmpq (reg:t_jit_reg ;mem:t_jit_regs); + procedure cmpq (reg0:t_jit_reg ;reg1:t_jit_reg); + procedure cmpi (reg:t_jit_reg ;imm:Int64); + procedure cmpi (size:TOperandSize;mem:t_jit_regs;imm:Int64); + procedure cmpi8 (reg:t_jit_reg;imm:Byte); + procedure cmpi8 (size:TOperandSize;mem:t_jit_regs;imm:Byte); procedure push16 (mem:t_jit_regs); procedure push64 (mem:t_jit_regs); procedure push (reg:t_jit_reg); @@ -252,6 +262,8 @@ type procedure vmovups (reg:t_jit_reg ;mem:t_jit_regs); procedure vmovups (mem:t_jit_regs;reg:t_jit_reg); procedure vmovdqa (reg0:t_jit_reg;reg1:t_jit_reg); + procedure sahf; + procedure lahf; end; operator + (A,B:t_jit_reg):t_jit_reg; @@ -439,23 +451,6 @@ begin end; end; -function Sums(mem:t_jit_int64):t_jit_reg; -var - i:Integer; -begin - Result:=Default(t_jit_reg); - if (Length(mem)=0) then - begin - // - end else - begin - For i:=0 to High(mem) do - begin - Result.AOffset:=Result.AOffset+mem[i]; - end; - end; -end; - //// procedure t_jit_i_link.set_label(id:Integer); @@ -697,7 +692,7 @@ function t_jit_builder.movj(reg:t_jit_reg;mem:t_jit_regs;_label_id:Integer):t_ji var i:Integer; begin - _mov($8B,reg,mem); //MOV r64, r/m64 + movq(reg,mem); i:=High(AInstructions); @@ -712,7 +707,7 @@ function t_jit_builder.leaj(reg:t_jit_reg;mem:t_jit_regs;_label_id:Integer):t_ji var i:Integer; begin - _mov($8D,reg,mem); //LEA r64,m + leaq(reg,mem); i:=High(AInstructions); @@ -822,7 +817,7 @@ end; type t_modrm_info=object - rexB,rexX,rexR:Boolean; + rexF,rexB,rexX,rexR:Boolean; ModRM:record Mode,Index,RM:Byte; @@ -837,6 +832,7 @@ type procedure build(Index:Byte;mreg:t_jit_reg); procedure build(reg,mreg:t_jit_reg); procedure emit(var ji:t_jit_instruction); + procedure emit8(var ji:t_jit_instruction); end; procedure t_modrm_info.build(Index:Byte;mreg:t_jit_reg); @@ -962,15 +958,38 @@ begin AOffset:=mreg.AOffset; end; +function get_force_rex(const reg:t_jit_reg):Boolean; inline; +begin + Result:=False; + if (reg.ARegValue[0].AType=regGeneral) then + if (reg.ARegValue[0].ASize=os8) then + case reg.ARegValue[0].AIndex of + 4..7:Result:=True; + else; + end; +end; + procedure t_modrm_info.build(reg,mreg:t_jit_reg); begin ModRM.Index:=reg.ARegValue[0].AIndex; - if (ModRM.Index>=8) then - begin - rexR:=true; - Dec(ModRM.Index,8); + + case reg.ARegValue[0].AType of + regGeneralH: + begin + ModRM.Index:=ModRM.Index+4; + end; + else + begin + if (ModRM.Index>=8) then + begin + rexR:=true; + Dec(ModRM.Index,8); + end; + end; end; + rexF:=get_force_rex(reg); + build(ModRM.Index,mreg); end; @@ -1007,20 +1026,33 @@ begin end; end; -procedure t_jit_builder._mov(op:Byte;reg:t_jit_reg;mem:t_jit_regs); +procedure t_modrm_info.emit8(var ji:t_jit_instruction); +begin + ji.EmitModRM(ModRM.Mode,ModRM.Index,ModRM.RM); + + if (ModRM.RM=4) then + begin + ji.EmitSIB(SIB.Scale,SIB.Index,SIB.Base); + end; + + ji.EmitByte(AOffset); //1 +end; + +procedure t_jit_builder._mov(op,op8:Byte;reg:t_jit_reg;mem:t_jit_regs); var mreg:t_jit_reg; rexW:Boolean; + Prefix:Byte; modrm_info:t_modrm_info; ji:t_jit_instruction; begin Assert(is_one_reg(reg)); - Assert(is_reg_size(reg,[os16,os32,os64])); + Assert(is_reg_size(reg,[os8,os16,os32,os64])); - Assert(is_reg_type(reg,[regGeneral])); + Assert(is_reg_type(reg,[regGeneral,regGeneralH])); Assert(reg.ARegValue[0].AScale<=1); @@ -1033,9 +1065,23 @@ begin ji:=default_jit_instruction; rexW:=False; - if (reg.ARegValue[0].ASize=os64) then - begin - rexW:=True; + Prefix:=0; + + case reg.ARegValue[0].ASize of + os8: + begin + op:=op8; + end; + os16: + begin + Prefix:=$66; + end; + os32:; + os64: + begin + rexW:=True; + end; + else; end; modrm_info:=Default(t_modrm_info); @@ -1044,9 +1090,9 @@ begin ji.EmitSelector(mreg.ASegment); - if (reg.ARegValue[0].ASize=os16) then + if (Prefix<>0) then begin - ji.EmitByte($66); //Operand-size override prefix (16) + ji.EmitByte(Prefix); //Operand-size override prefix (16) end; if (mreg.ARegValue[0].ASize=os32) then @@ -1054,12 +1100,8 @@ begin ji.EmitByte($67); //Address-size override prefix (32) end; - if modrm_info.rexB or modrm_info.rexX or modrm_info.rexR or rexW then + if modrm_info.rexF or modrm_info.rexB or modrm_info.rexX or modrm_info.rexR or rexW then begin - //rexB $4x bit 0: Extension of the ModR/M r/m field, SIB base field, or Opcode reg field - //rexX $4x bit 1: Extension of the SIB index field - //rexR $4x bit 2: Extension of the ModR/M reg field - //rexW $4x bit 3: 64 Bit Operand Size ji.EmitREX(modrm_info.rexB,modrm_info.rexX,modrm_info.rexR,rexW); end; @@ -1072,7 +1114,7 @@ end; procedure t_jit_builder._mov(op,op8:Byte;reg0:t_jit_reg;reg1:t_jit_reg); var - rexB,rexR,rexW:Boolean; + rexF,rexB,rexR,rexW:Boolean; ModRM:record Index,RM:Byte; @@ -1096,8 +1138,11 @@ begin Assert(reg0.ARegValue[0].ASize=reg1.ARegValue[0].ASize); + Assert(get_force_rex(reg0)=get_force_rex(reg1)); + ji:=default_jit_instruction; + rexF:=false; rexB:=false; rexR:=false; rexW:=false; @@ -1134,12 +1179,14 @@ begin Dec(ModRM.RM,8); end; + rexF:=get_force_rex(reg0); + if (Prefix<>0) then begin ji.EmitByte(Prefix); //Operand-size override prefix (16) end; - if rexB or rexR or rexW then + if rexF or rexB or rexR or rexW then begin ji.EmitREX(rexB,False,rexR,rexW); end; @@ -1153,9 +1200,9 @@ end; //// -procedure t_jit_builder._addi(op,op8,index:Byte;reg:t_jit_reg;imm:Int64); +procedure t_jit_builder._movi(op,op8,index:Byte;reg:t_jit_reg;imm:Int64); var - rexB,rexW:Boolean; + rexF,rexB,rexW:Boolean; RM,Prefix:Byte; @@ -1170,6 +1217,7 @@ begin ji:=default_jit_instruction; + rexF:=false; rexB:=false; rexW:=false; Prefix:=0; @@ -1181,6 +1229,8 @@ begin Dec(RM,8); end; + rexF:=get_force_rex(reg); + case reg.ARegValue[0].ASize of os8: begin @@ -1203,7 +1253,7 @@ begin ji.EmitByte(Prefix); //Operand-size override prefix (16) end; - if rexB or rexW then + if rexF or rexB or rexW then begin ji.EmitREX(rexB,False,False,rexW); end; @@ -1223,6 +1273,203 @@ begin _add(ji); end; +procedure t_jit_builder._movi(op,op8,index:Byte;size:TOperandSize;mem:t_jit_regs;imm:Int64); +var + mreg:t_jit_reg; + + rexW:Boolean; + Prefix:Byte; + + modrm_info:t_modrm_info; + + ji:t_jit_instruction; +begin + Assert(size in [os8,os16,os32,os64]); + + mreg:=Sums(mem); + + Assert(is_reg_size(mreg,[os0,os32,os64])); + Assert(is_reg_type(mreg,[regNone,regGeneral,regRip])); + Assert(is_valid_scale(mreg)); + + ji:=default_jit_instruction; + + rexW:=False; + Prefix:=0; + + case size of + os8: + begin + op:=op8; + end; + os16: + begin + Prefix:=$66; + end; + os32:; + os64: + begin + rexW:=True; + end; + else; + end; + + modrm_info:=Default(t_modrm_info); + + modrm_info.build(Index,mreg); + + ji.EmitSelector(mreg.ASegment); + + if (Prefix<>0) then + begin + ji.EmitByte(Prefix); //Operand-size override prefix (16) + end; + + if (mreg.ARegValue[0].ASize=os32) then + begin + ji.EmitByte($67); //Address-size override prefix (32) + end; + + if modrm_info.rexF or modrm_info.rexB or modrm_info.rexX or modrm_info.rexR or rexW then + begin + ji.EmitREX(modrm_info.rexB,modrm_info.rexX,modrm_info.rexR,rexW); + end; + + ji.EmitByte(op); + + modrm_info.emit(ji); + + _add(ji); +end; + +procedure t_jit_builder._movi8(op,index:Byte;reg:t_jit_reg;imm:Byte); +var + rexF,rexB,rexW:Boolean; + + RM,Prefix:Byte; + + ji:t_jit_instruction; +begin + Assert(is_one_reg(reg)); + Assert(is_reg_size(reg,[os16,os32,os64])); + + Assert(is_reg_type(reg,[regGeneral])); + + Assert(reg.ARegValue[0].AScale<=1); + + ji:=default_jit_instruction; + + rexF:=false; + rexB:=false; + rexW:=false; + Prefix:=0; + + RM:=reg.ARegValue[0].AIndex; + if (RM>=8) then + begin + rexB:=true; + Dec(RM,8); + end; + + rexF:=get_force_rex(reg); + + case reg.ARegValue[0].ASize of + os16: + begin + Prefix:=$66; + end; + os32:; + os64: + begin + rexW:=True; + end; + else; + end; + + if (Prefix<>0) then + begin + ji.EmitByte(Prefix); //Operand-size override prefix (16) + end; + + if rexF or rexB or rexW then + begin + ji.EmitREX(rexB,False,False,rexW); + end; + + ji.EmitByte(op); + + ji.EmitModRM(3,index,RM); + + ji.EmitByte(imm); + + _add(ji); +end; + +procedure t_jit_builder._movi8(op,index:Byte;size:TOperandSize;mem:t_jit_regs;imm:Byte); +var + mreg:t_jit_reg; + + rexW:Boolean; + Prefix:Byte; + + modrm_info:t_modrm_info; + + ji:t_jit_instruction; +begin + Assert(size in [os16,os32,os64]); + + mreg:=Sums(mem); + + Assert(is_reg_size(mreg,[os0,os32,os64])); + Assert(is_reg_type(mreg,[regNone,regGeneral,regRip])); + Assert(is_valid_scale(mreg)); + + ji:=default_jit_instruction; + + rexW:=False; + Prefix:=0; + + case size of + os16: + begin + Prefix:=$66; + end; + os32:; + os64: + begin + rexW:=True; + end; + else; + end; + + modrm_info:=Default(t_modrm_info); + + modrm_info.build(Index,mreg); + + ji.EmitSelector(mreg.ASegment); + + if (Prefix<>0) then + begin + ji.EmitByte(Prefix); //Operand-size override prefix (16) + end; + + if (mreg.ARegValue[0].ASize=os32) then + begin + ji.EmitByte($67); //Address-size override prefix (32) + end; + + if modrm_info.rexF or modrm_info.rexB or modrm_info.rexX or modrm_info.rexR or rexW then + begin + ji.EmitREX(modrm_info.rexB,modrm_info.rexX,modrm_info.rexR,rexW); + end; + + ji.EmitByte(op); + + modrm_info.emit8(ji); + + _add(ji); +end; + //// procedure t_jit_builder._push(op,index:Byte;size:TOperandSize;mem:t_jit_regs); @@ -1259,7 +1506,7 @@ begin ji.EmitByte($67); //Address-size override prefix (32) end; - if modrm_info.rexB or modrm_info.rexX or modrm_info.rexR then + if modrm_info.rexF or modrm_info.rexB or modrm_info.rexX or modrm_info.rexR then begin ji.EmitREX(modrm_info.rexB,modrm_info.rexX,modrm_info.rexR,False); end; @@ -1375,7 +1622,7 @@ end; procedure t_jit_builder.movi(reg:t_jit_reg;imm:Int64); var - rexB,rexW:Boolean; + rexF,rexB,rexW:Boolean; Index,Prefix,op:Byte; @@ -1390,6 +1637,7 @@ begin ji:=default_jit_instruction; + rexF:=false; rexB:=false; rexW:=false; Prefix:=0; @@ -1401,6 +1649,8 @@ begin Dec(Index,8); end; + rexF:=get_force_rex(reg); + case reg.ARegValue[0].ASize of os8: begin @@ -1428,7 +1678,7 @@ begin ji.EmitByte(Prefix); //Operand-size override prefix (16) end; - if rexB or rexW then + if rexF or rexB or rexW then begin ji.EmitREX(rexB,False,False,rexW); end; @@ -1448,56 +1698,33 @@ end; procedure t_jit_builder.movq(reg:t_jit_reg;mem:t_jit_regs); begin - _mov($8B,reg,mem); //MOV r64, r/m64 -end; - -procedure t_jit_builder.movq(reg:t_jit_reg;mem:t_jit_int64); -begin - movq(reg,[Sums(mem)]); + _mov($8B,$8A,reg,mem); //MOV r64, r/m64 end; procedure t_jit_builder.movq(mem:t_jit_regs;reg:t_jit_reg); begin - _mov($89,reg,mem); //MOV r/m64, r64 -end; - -procedure t_jit_builder.movq(mem:t_jit_int64;reg:t_jit_reg); -begin - movq([Sums(mem)],reg); + _mov($89,$88,reg,mem); //MOV r/m64, r64 end; // -procedure t_jit_builder.lea(reg:t_jit_reg;mem:t_jit_regs); +procedure t_jit_builder.leaq(reg:t_jit_reg;mem:t_jit_regs); begin - _mov($8D,reg,mem); //LEA r64,m -end; + Assert(is_reg_size(reg,[os16,os32,os64])); -procedure t_jit_builder.lea(reg:t_jit_reg;mem:t_jit_int64); -begin - lea(reg,[Sums(mem)]); + _mov($8D,$8D,reg,mem); //LEA r64,m end; // procedure t_jit_builder.addq(mem:t_jit_regs;reg:t_jit_reg); begin - _mov($01,reg,mem); //ADD r/m64, r64 -end; - -procedure t_jit_builder.addq(mem:t_jit_int64;reg:t_jit_reg); -begin - addq([Sums(mem)],reg); + _mov($01,$00,reg,mem); //ADD r/m64, r64 end; procedure t_jit_builder.addq(reg:t_jit_reg;mem:t_jit_regs); begin - _mov($03,reg,mem); //ADD r64, r/m64 -end; - -procedure t_jit_builder.addq(reg:t_jit_reg;mem:t_jit_int64); -begin - addq(reg,[Sums(mem)]); + _mov($03,$02,reg,mem); //ADD r64, r/m64 end; procedure t_jit_builder.addq(reg0:t_jit_reg;reg1:t_jit_reg); @@ -1507,29 +1734,24 @@ end; procedure t_jit_builder.addi(reg:t_jit_reg;imm:Int64); begin - _addi($81,$80,0,reg,imm); + _movi($81,$80,0,reg,imm); +end; + +procedure t_jit_builder.addi8(reg:t_jit_reg;imm:Byte); +begin + _movi8($83,0,reg,imm); end; // procedure t_jit_builder.subq(mem:t_jit_regs;reg:t_jit_reg); begin - _mov($29,reg,mem); //SUB r/m64, r64 -end; - -procedure t_jit_builder.subq(mem:t_jit_int64;reg:t_jit_reg); -begin - addq([Sums(mem)],reg); + _mov($29,$28,reg,mem); //SUB r/m64, r64 end; procedure t_jit_builder.subq(reg:t_jit_reg;mem:t_jit_regs); begin - _mov($2B,reg,mem); //SUB r64, r/m64 -end; - -procedure t_jit_builder.subq(reg:t_jit_reg;mem:t_jit_int64); -begin - addq(reg,[Sums(mem)]); + _mov($2B,$2A,reg,mem); //SUB r64, r/m64 end; procedure t_jit_builder.subq(reg0:t_jit_reg;reg1:t_jit_reg); @@ -1539,9 +1761,59 @@ end; procedure t_jit_builder.subi(reg:t_jit_reg;imm:Int64); begin - _addi($81,$80,5,reg,imm); + _movi($81,$80,5,reg,imm); end; +procedure t_jit_builder.subi8(reg:t_jit_reg;imm:Byte); +begin + _movi8($83,5,reg,imm); +end; + +/// + +procedure t_jit_builder.xorq(reg0:t_jit_reg;reg1:t_jit_reg); +begin + _mov($31,$30,reg0,reg1); +end; + +/// + +procedure t_jit_builder.cmpq(mem:t_jit_regs;reg:t_jit_reg); +begin + _mov($39,$38,reg,mem); +end; + +procedure t_jit_builder.cmpq(reg:t_jit_reg;mem:t_jit_regs); +begin + _mov($3B,$3A,reg,mem); +end; + +procedure t_jit_builder.cmpq(reg0:t_jit_reg;reg1:t_jit_reg); +begin + _mov($39,$38,reg0,reg1); +end; + +procedure t_jit_builder.cmpi(reg:t_jit_reg;imm:Int64); +begin + _movi($81,$80,7,reg,imm); +end; + +procedure t_jit_builder.cmpi(size:TOperandSize;mem:t_jit_regs;imm:Int64); +begin + _movi($81,$80,7,size,mem,imm); +end; + +procedure t_jit_builder.cmpi8(reg:t_jit_reg;imm:Byte); +begin + _movi8($83,7,reg,imm); +end; + +procedure t_jit_builder.cmpi8(size:TOperandSize;mem:t_jit_regs;imm:Byte); +begin + _movi8($83,7,size,mem,imm); +end; + + /// procedure t_jit_builder.push16(mem:t_jit_regs); @@ -1762,7 +2034,27 @@ begin _add(ji); end; +procedure t_jit_builder.sahf; +var + ji:t_jit_instruction; +begin + ji:=default_jit_instruction; + ji.EmitByte($9E); + + _add(ji); +end; + +procedure t_jit_builder.lahf; +var + ji:t_jit_instruction; +begin + ji:=default_jit_instruction; + + ji.EmitByte($9F); + + _add(ji); +end; end. diff --git a/sys/kern/kern_dmem.pas b/sys/kern/kern_dmem.pas index 47180975..a6c68657 100644 --- a/sys/kern/kern_dmem.pas +++ b/sys/kern/kern_dmem.pas @@ -286,7 +286,7 @@ begin addr:=SCE_REPLAY_EXEC_START; end; end else - if (p_proc.p_sce_replay_exec=0) and + if (p_proc.p_sce_replay_exec<>0) and (addrQWORD($7efffffff)) then begin diff --git a/sys/kern/trap.pas b/sys/kern/trap.pas index 8a02535d..29fa4e49 100644 --- a/sys/kern/trap.pas +++ b/sys/kern/trap.pas @@ -662,10 +662,8 @@ asm movqq %rax,%r11 //save rax movqq %rcx,%r10 //save rcx - lahf //load to AH - shr $8,%rax - andl $0xFF,%rax //filter flags - movqq %rax,%rcx //save flags + lahf //load to AH + movb %ah,%ch //save flags to CH movqq %gs:teb.thread,%rax //curkthread test %rax,%rax @@ -676,6 +674,9 @@ asm andl NOT_PCB_FULL_IRET,kthread.pcb_flags(%rax) //clear PCB_FULL_IRET + movqq $0 ,kthread.td_frame.tf_rflags(%rax) //clear + movb %ch ,kthread.td_frame.tf_rflags(%rax) //save flags + movqq %rdi,kthread.td_frame.tf_rdi (%rax) movqq %rsi,kthread.td_frame.tf_rsi (%rax) movqq %rdx,kthread.td_frame.tf_rdx (%rax) @@ -690,7 +691,6 @@ asm movqq %r13,kthread.td_frame.tf_r13 (%rax) movqq %r14,kthread.td_frame.tf_r14 (%rax) movqq %r15,kthread.td_frame.tf_r15 (%rax) - movqq %rcx,kthread.td_frame.tf_rflags(%rax) movqq $0 ,kthread.td_frame.tf_trapno(%rax) movqq $0 ,kthread.td_frame.tf_addr (%rax) @@ -720,8 +720,9 @@ asm jne _ast //Restore preserved registers. - movqq kthread.td_frame.tf_rflags(%rcx),%rax - shl $8,%rax + + //get flags + movb kthread.td_frame.tf_rflags(%rcx),%ah sahf //restore flags movqq kthread.td_frame.tf_rdi(%rcx),%rdi @@ -744,10 +745,9 @@ asm //fail (curkthread=nil) _fail: - movqq %rcx,%rax //get flags - shl $8,%rax - or $1,%ah //CF - sahf //restore flags + movb %ch,%ah //get flags + or $1,%ah //set CF + sahf //restore flags movqq $14,%rax //EFAULT movqq $0,%rdx @@ -874,9 +874,8 @@ asm test %rsp,%rsp jz _fail - shr $8,%rax - andl $0xFF,%rax //filter flags - movqq %rax,kthread.td_frame.tf_rflags(%rsp) //save flags + movqq $0 ,kthread.td_frame.tf_rflags(%rsp) //clear + movb %ah,kthread.td_frame.tf_rflags(%rsp) //save flags movqq %gs:teb.jit_rax,%rax //load %rax movqq %rax,kthread.td_frame.tf_rax(%rsp) //save %rax @@ -940,8 +939,7 @@ asm movqq %rax,%gs:teb.jitcall //save ret //get flags - movqq kthread.td_frame.tf_rflags(%rcx),%rax - shl $8,%rax + movb kthread.td_frame.tf_rflags(%rcx),%ah sahf //restore flags movqq kthread.td_frame.tf_rdi(%rcx),%rdi @@ -1008,11 +1006,11 @@ function IS_TRAP_FUNC(rip:qword):Boolean; begin Result:=( (rip>=QWORD(@fast_syscall)) and - (rip<=(QWORD(@fast_syscall)+$1AA)) //fast_syscall func size + (rip<=(QWORD(@fast_syscall)+$19C)) //fast_syscall func size ) or ( (rip>=QWORD(@jit_call)) and - (rip<=(QWORD(@jit_call)+$240)) //jit_call func size + (rip<=(QWORD(@jit_call)+$23C)) //jit_call func size ); end; diff --git a/sys/md/md_exception.pas b/sys/md/md_exception.pas index a99cf897..85f8c75e 100644 --- a/sys/md/md_exception.pas +++ b/sys/md/md_exception.pas @@ -178,6 +178,7 @@ begin case p^.ExceptionRecord^.ExceptionCode of FPC_EXCEPTION_CODE :Exit; + EXCEPTION_BREAKPOINT :Exit; EXCEPTION_SET_THREADNAME:Exit; end; @@ -206,6 +207,7 @@ begin case p^.ExceptionRecord^.ExceptionCode of FPC_EXCEPTION_CODE :Exit; + EXCEPTION_BREAKPOINT :Exit; EXCEPTION_SET_THREADNAME:Exit; end; @@ -252,6 +254,7 @@ begin write(stderr,msg); Writeln(stderr,' (',FName,', line ',LineNo,').'); print_backtrace(stderr,Get_pc_addr,get_frame,0); + DebugBreak; md_halt(217); end; diff --git a/sys/vm/vm_fault.pas b/sys/vm/vm_fault.pas index 27babc12..4fd27dbe 100644 --- a/sys/vm/vm_fault.pas +++ b/sys/vm/vm_fault.pas @@ -85,10 +85,10 @@ begin os8: begin case RegValue.AIndex of - 0:Result := @p_kthread(nil)^.td_frame.tf_rax+1; - 1:Result := @p_kthread(nil)^.td_frame.tf_rcx+1; - 2:Result := @p_kthread(nil)^.td_frame.tf_rdx+1; - 3:Result := @p_kthread(nil)^.td_frame.tf_rbx+1; + 0:Result := (@p_kthread(nil)^.td_frame.tf_rax)+1; + 1:Result := (@p_kthread(nil)^.td_frame.tf_rcx)+1; + 2:Result := (@p_kthread(nil)^.td_frame.tf_rdx)+1; + 3:Result := (@p_kthread(nil)^.td_frame.tf_rbx)+1; else; end; end; @@ -408,7 +408,7 @@ begin adr:=t_jit_builder.rdi; adr.ARegValue[0].ASize:=RegValue[0].ASize; - tdr:=t_jit_builder.rsi; + tdr:=t_jit_builder.rcx; with ctx.builder do begin @@ -417,13 +417,16 @@ begin i:=GetFrameOffsetInt(RegValue[0]); Assert(i<>0,'build_lea'); - //xor adr,adr needed + if (RegValue[0].ASize<>os64) then + begin + xorq(rdi,rdi); + end; movq(adr,[tdr+i]); if (RegValue[0].AScale>1) then begin - lea(adr,[adr*RegValue[0].AScale]) + leaq(adr,[adr*RegValue[0].AScale]) end; if (RegValue[1].AType<>regNone) then @@ -440,48 +443,55 @@ begin begin with ctx.builder do begin - lea(adr,[adr+ofs]); + leaq(adr,[adr+ofs]); end; end; end; // -procedure build_mov(var ctx:t_jit_context;id:Byte;memop:t_memop_type); -var - i,copy_size:Integer; - imm:Int64; - link:t_jit_i_link; - dst:t_jit_reg; +procedure get_size_info(Size:TOperandSize;var byte_size:Integer;var reg:t_jit_reg); begin - - case ctx.din.Operand[id].Size of + case Size of + os8: + begin + byte_size:=1; + reg:=t_jit_builder.sil; + end; + os16: + begin + byte_size:=2; + reg:=t_jit_builder.si; + end; os32: begin - copy_size:=4; - dst:=t_jit_builder.esi; + byte_size:=4; + reg:=t_jit_builder.esi; end; os64: begin - copy_size:=8; - dst:=t_jit_builder.rsi; - end + byte_size:=8; + reg:=t_jit_builder.rsi; + end; else begin - Writeln('build_mov (',ctx.din.Operand[id].Size,')'); - Assert(false,'build_mov (size)'); + Writeln('get_size_info (',Size,')'); + Assert(false,'get_size_info (size)'); end; end; +end; - if (ctx.din.Operand[1].Size<>ctx.din.Operand[2].Size) then - begin - Writeln(ctx.din.OpCode.Opcode,' ', - ctx.din.OpCode.Suffix,' ', - ctx.din.Operand[1].Size,' ', - ctx.din.Operand[2].Size); +procedure build_mov(var ctx:t_jit_context;id:Byte;memop:t_memop_type); +var + i,copy_size,reg_size:Integer; + imm:Int64; + link:t_jit_i_link; + mem,reg:t_jit_reg; +begin - Assert(false,'TODO'); - end; + get_size_info(ctx.din.Operand[get_lea_id(memop)].Size,copy_size,mem); + + get_size_info(ctx.din.Operand[id].Size,reg_size,reg); case memop of moCopyout: @@ -490,33 +500,54 @@ begin begin //input:rdi - link:=leaj(rsi,[rip+$FFFF],-1); + if (copy_size=1) then + begin + call(@uplift); //input:rdi output:rax=rdi + end else + begin + link:=leaj(rsi,[rip+$FFFF],-1); - movi(edx,copy_size); + movi(edx,copy_size); - call(@copyout_mov); //rdi,rsi,edx + call(@copyout_mov); //rdi,rsi,edx - reta; + reta; - //input:rdi + //input:rdi - link._label:=_label; + link._label:=_label; + end; imm:=0; if GetTargetOfs(ctx,id,imm) then begin //imm const - movi(dst,imm); - movq([rdi],dst); //TODO movi([rdi],imm); + + if (copy_size>reg_size) or (imm=0) then + begin + xorq(rsi,rsi); + end; + + if (imm<>0) then + begin + movi(reg,imm); + end; + + movq([rdi],mem); //TODO movi([rdi],imm); end else begin - movq(rax,[GS+Integer(teb_thread)]); + movq(rcx,[GS+Integer(teb_thread)]); i:=GetFrameOffsetInt(ctx.din.Operand[id].RegValue[0]); Assert(i<>0,'build_mov'); - movq(dst,[rax+i]); - movq([rdi],dst); + if (copy_size>reg_size) then + begin + xorq(rsi,rsi); + end; + + movq(reg,[rcx+i]); + movq([rdi],mem); end; reta; @@ -529,32 +560,209 @@ begin begin //input:rdi - link:=movj(rsi,[rip+$FFFF],-1); + if (copy_size=1) then + begin + call(@uplift); //input:rdi output:rax=rdi + end else + begin + link:=movj(rsi,[rip+$FFFF],-1); - movi(edx,copy_size); + movi(edx,copy_size); - call(@copyin_mov); //rdi,rsi,edx + call(@copyin_mov); //rdi,rsi,edx - reta; + reta; - //input:rdi + //input:rdi - link._label:=_label; + link._label:=_label; + end; - movq(rax,[GS+Integer(teb_thread)]); + movq(rcx,[GS+Integer(teb_thread)]); i:=GetFrameOffsetInt(ctx.din.Operand[id].RegValue[0]); Assert(i<>0,'build_mov'); - movq(dst,[rdi]); - movq([rax+i],dst); + if (copy_size1) then + begin + cmpi8(mem.ARegValue[0].ASize,[rdi],imm); + end else + begin + cmpi(mem.ARegValue[0].ASize,[rdi],imm); + end; + + end else + begin + i:=GetFrameOffsetInt(ctx.din.Operand[id].RegValue[0]); + Assert(i<>0,'build_cmp'); + + if (copy_size>reg_size) then + begin + xorq(rsi,rsi); + end; + + movq(reg,[rcx+i]); + cmpq([rdi],mem); + end; + + //save flags + build_save_flags(ctx); + //save flags + + reta; + end; + end; + moCopyin: + begin + with ctx.builder do + begin + + //input:rdi + + if (copy_size=1) then + begin + call(@uplift); //input:rdi output:rax=rdi + end else + begin + link:=leaj(rsi,[rip+$FFFF],-1); + + movi(edx,copy_size); + + call(@copyout_mov); //rdi,rsi,edx + + reta; + + //input:rdi + + link._label:=_label; + end; + + movq(rcx,[GS+Integer(teb_thread)]); + + i:=GetFrameOffsetInt(ctx.din.Operand[id].RegValue[0]); + Assert(i<>0,'build_cmp'); + + //load flags + build_load_flags(ctx); + //load flags + + if (copy_size