diff --git a/rtl/x86_jit.pas b/rtl/x86_jit.pas index 6dc64d7f..b00515cf 100644 --- a/rtl/x86_jit.pas +++ b/rtl/x86_jit.pas @@ -60,6 +60,18 @@ type t_jit_instructions=array of t_jit_instruction; t_jit_data =array of Pointer; + p_jit_builder=^t_jit_builder; + + t_jit_i_link=object + private + builder:p_jit_builder; + inst_id:Integer; + procedure set_label(id:Integer); + function get_label():Integer; + public + property _label:Integer read get_label write set_label; + end; + t_jit_builder=object Const al :t_jit_reg=(ARegValue:((AType:regGeneral;ASize:os8;AIndex: 0),(AType:regNone))); @@ -182,8 +194,12 @@ type Procedure call(P:Pointer); Procedure jmp (P:Pointer); // - Procedure call(_label_id:Integer); - Procedure jmp (_label_id:Integer); + function call(_label_id:Integer):t_jit_i_link; + function jmp (_label_id:Integer):t_jit_i_link; + function movj(reg:t_jit_reg;mem:t_jit_regs;_label_id:Integer):t_jit_i_link; + function leaj(reg:t_jit_reg;mem:t_jit_regs;_label_id:Integer):t_jit_i_link; + // + Procedure reta; // Function GetMemSize:Integer; Procedure RebuldInstructionOffset; @@ -191,6 +207,8 @@ type Function SaveTo(ptr:PByte;size:Integer):Integer; // procedure _mov (op:Byte;reg:t_jit_reg;mem:t_jit_regs); + procedure _mov (op,op8:Byte;reg0:t_jit_reg;reg1:t_jit_reg); + procedure _addi (op,op8,index:Byte;reg:t_jit_reg;imm:Int64); procedure _push (op,index:Byte;size:TOperandSize;mem:t_jit_regs); procedure _push (op:Byte;reg:t_jit_reg); procedure _pushi (size:TOperandSize;imm:Integer); @@ -208,6 +226,12 @@ type procedure addq (reg:t_jit_reg ;mem:t_jit_int64); procedure addq (reg0:t_jit_reg ;reg1:t_jit_reg); procedure addi (reg:t_jit_reg ;imm:Int64); + procedure subq (mem:t_jit_regs ;reg:t_jit_reg); + procedure subq (mem:t_jit_int64;reg:t_jit_reg); + procedure subq (reg:t_jit_reg ;mem:t_jit_regs); + procedure subq (reg:t_jit_reg ;mem:t_jit_int64); + procedure subq (reg0:t_jit_reg ;reg1:t_jit_reg); + procedure subi (reg:t_jit_reg ;imm:Int64); procedure push16 (mem:t_jit_regs); procedure push64 (mem:t_jit_regs); procedure push (reg:t_jit_reg); @@ -430,6 +454,18 @@ end; //// +procedure t_jit_i_link.set_label(id:Integer); +begin + builder^.AInstructions[inst_id].ALink.ALinkId:=id; +end; + +function t_jit_i_link.get_label():Integer; +begin + Result:=builder^.AInstructions[inst_id].ALink.ALinkId; +end; + +//// + procedure t_jit_instruction.EmitByte(b:byte); inline; begin AData[ASize]:=b; @@ -613,7 +649,7 @@ begin _add(ji); end; -Procedure t_jit_builder.call(_label_id:Integer); +function t_jit_builder.call(_label_id:Integer):t_jit_i_link; var ji:t_jit_instruction; begin @@ -628,9 +664,12 @@ begin ji.EmitInt32(0); _add(ji); + + Result.builder:=@self; + Result.inst_id:=High(AInstructions); end; -Procedure t_jit_builder.jmp(_label_id:Integer); +function t_jit_builder.jmp(_label_id:Integer):t_jit_i_link; var ji:t_jit_instruction; begin @@ -645,6 +684,50 @@ begin ji.EmitInt32(0); _add(ji); + + Result.builder:=@self; + Result.inst_id:=High(AInstructions); +end; + +function t_jit_builder.movj(reg:t_jit_reg;mem:t_jit_regs;_label_id:Integer):t_jit_i_link; +var + i:Integer; +begin + _mov($8B,reg,mem); //MOV r64, r/m64 + + i:=High(AInstructions); + + AInstructions[i].ALink.AType:=lnkLabel; + AInstructions[i].ALink.ALinkId:=_label_id; + + Result.builder:=@self; + Result.inst_id:=High(AInstructions); +end; + +function t_jit_builder.leaj(reg:t_jit_reg;mem:t_jit_regs;_label_id:Integer):t_jit_i_link; +var + i:Integer; +begin + _mov($8D,reg,mem); //LEA r64,m + + i:=High(AInstructions); + + AInstructions[i].ALink.AType:=lnkLabel; + AInstructions[i].ALink.ALinkId:=_label_id; + + Result.builder:=@self; + Result.inst_id:=High(AInstructions); +end; + +Procedure t_jit_builder.reta; +var + ji:t_jit_instruction; +begin + ji:=default_jit_instruction; + + ji.EmitByte($C3); + + _add(ji); end; Function t_jit_builder.GetMemSize:Integer; @@ -892,13 +975,20 @@ begin case ModRM.RM of 4:if (SIB.Base=5) then begin + ji.ALink.ADataOffset:=ji.ASize; + ji.EmitInt32(AOffset); //4 + end; + 5:begin + ji.ALink.ADataOffset:=ji.ASize; ji.EmitInt32(AOffset); //4 end; - 5:ji.EmitInt32(AOffset); //4 end; end; 1:ji.EmitByte (AOffset); //1 - 2:ji.EmitInt32(AOffset); //4 + 2:begin + ji.ALink.ADataOffset:=ji.ASize; + ji.EmitInt32(AOffset); //4 + end; else; end; end; @@ -966,6 +1056,159 @@ begin _add(ji); end; +procedure t_jit_builder._mov(op,op8:Byte;reg0:t_jit_reg;reg1:t_jit_reg); +var + rexB,rexR,rexW:Boolean; + + ModRM:record + Index,RM:Byte; + end; + + Prefix:Byte; + + ji:t_jit_instruction; +begin + Assert(is_one_reg(reg0)); + Assert(is_one_reg(reg1)); + + Assert(is_reg_size(reg0,[os8,os16,os32,os64])); + Assert(is_reg_size(reg1,[os8,os16,os32,os64])); + + Assert(is_reg_type(reg0,[regGeneral])); + Assert(is_reg_type(reg1,[regGeneral])); + + Assert(reg0.ARegValue[0].AScale<=1); + Assert(reg1.ARegValue[0].AScale<=1); + + Assert(reg0.ARegValue[0].ASize=reg1.ARegValue[0].ASize); + + ji:=default_jit_instruction; + + rexB:=false; + rexR:=false; + rexW:=false; + Prefix:=0; + + case reg0.ARegValue[0].ASize of + os8: + begin + op:=op8; + end; + os16: + begin + Prefix:=$66; + end; + os32:; + os64: + begin + rexW:=True; + end; + else; + end; + + ModRM.Index:=reg1.ARegValue[0].AIndex; + if (ModRM.Index>=8) then + begin + rexR:=true; + Dec(ModRM.Index,8); + end; + + ModRM.RM:=reg0.ARegValue[0].AIndex; + if (ModRM.RM>=8) then + begin + rexB:=true; + Dec(ModRM.RM,8); + end; + + if (Prefix<>0) then + begin + ji.EmitByte(Prefix); //Operand-size override prefix (16) + end; + + if rexB or rexR or rexW then + begin + ji.EmitREX(rexB,False,rexR,rexW); + end; + + ji.EmitByte(op); + + ji.EmitModRM(3,ModRM.Index,ModRM.RM); + + _add(ji); +end; + +//// + +procedure t_jit_builder._addi(op,op8,index:Byte;reg:t_jit_reg;imm:Int64); +var + rexB,rexW:Boolean; + + RM,Prefix:Byte; + + ji:t_jit_instruction; +begin + Assert(is_one_reg(reg)); + Assert(is_reg_size(reg,[os8,os16,os32,os64])); + + Assert(is_reg_type(reg,[regGeneral])); + + Assert(reg.ARegValue[0].AScale<=1); + + ji:=default_jit_instruction; + + rexB:=false; + rexW:=false; + Prefix:=0; + + RM:=reg.ARegValue[0].AIndex; + if (RM>=8) then + begin + rexB:=true; + Dec(RM,8); + end; + + case reg.ARegValue[0].ASize of + os8: + begin + op:=op8; + end; + os16: + begin + Prefix:=$66; + end; + os32:; + os64: + begin + rexW:=True; + end; + else; + end; + + if (Prefix<>0) then + begin + ji.EmitByte(Prefix); //Operand-size override prefix (16) + end; + + if rexB or rexW then + begin + ji.EmitREX(rexB,False,False,rexW); + end; + + ji.EmitByte(op); + + ji.EmitModRM(3,index,RM); + + case reg.ARegValue[0].ASize of + os8:ji.EmitByte (imm); + os16:ji.EmitWord (imm); + os32:ji.EmitInt32(imm); + os64:ji.EmitInt32(imm); + else; + end; + + _add(ji); +end; + //// procedure t_jit_builder._push(op,index:Byte;size:TOperandSize;mem:t_jit_regs); @@ -1112,89 +1355,8 @@ end; // procedure t_jit_builder.movq(reg0:t_jit_reg;reg1:t_jit_reg); -var - rexB,rexR,rexW:Boolean; - - ModRM:record - Index,RM:Byte; - end; - - Prefix,op:Byte; - - ji:t_jit_instruction; begin - Assert(is_one_reg(reg0)); - Assert(is_one_reg(reg1)); - - Assert(is_reg_size(reg0,[os8,os16,os32,os64])); - Assert(is_reg_size(reg1,[os8,os16,os32,os64])); - - Assert(is_reg_type(reg0,[regGeneral])); - Assert(is_reg_type(reg1,[regGeneral])); - - Assert(reg0.ARegValue[0].AScale<=1); - Assert(reg1.ARegValue[0].AScale<=1); - - Assert(reg0.ARegValue[0].ASize=reg1.ARegValue[0].ASize); - - ji:=default_jit_instruction; - - rexB:=false; - rexR:=false; - rexW:=false; - Prefix:=0; - - case reg0.ARegValue[0].ASize of - os8: - begin - op:=$88; - end; - os16: - begin - Prefix:=$66; - op:=$89; - end; - os32: - begin - op:=$89; - end; - os64: - begin - rexW:=True; - op:=$89; - end; - else; - end; - - ModRM.Index:=reg1.ARegValue[0].AIndex; - if (ModRM.Index>=8) then - begin - rexR:=true; - Dec(ModRM.Index,8); - end; - - ModRM.RM:=reg0.ARegValue[0].AIndex; - if (ModRM.RM>=8) then - begin - rexB:=true; - Dec(ModRM.RM,8); - end; - - if (Prefix<>0) then - begin - ji.EmitByte(Prefix); //Operand-size override prefix (16) - end; - - if rexB or rexR or rexW then - begin - ji.EmitREX(rexB,False,rexR,rexW); - end; - - ji.EmitByte(op); - - ji.EmitModRM(3,ModRM.Index,ModRM.RM); - - _add(ji); + _mov($89,$88,reg0,reg1); end; procedure t_jit_builder.movi(reg:t_jit_reg;imm:Int64); @@ -1272,7 +1434,7 @@ end; procedure t_jit_builder.movq(reg:t_jit_reg;mem:t_jit_regs); begin - _mov($8B,reg,[Sums(mem)]); //MOV r64, r/m64 + _mov($8B,reg,mem); //MOV r64, r/m64 end; procedure t_jit_builder.movq(reg:t_jit_reg;mem:t_jit_int64); @@ -1282,7 +1444,7 @@ end; procedure t_jit_builder.movq(mem:t_jit_regs;reg:t_jit_reg); begin - _mov($89,reg,[Sums(mem)]); //MOV r/m64, r64 + _mov($89,reg,mem); //MOV r/m64, r64 end; procedure t_jit_builder.movq(mem:t_jit_int64;reg:t_jit_reg); @@ -1290,9 +1452,11 @@ begin movq([Sums(mem)],reg); end; +// + procedure t_jit_builder.lea(reg:t_jit_reg;mem:t_jit_regs); begin - _mov($8D,reg,[Sums(mem)]); //LEA r64,m + _mov($8D,reg,mem); //LEA r64,m end; procedure t_jit_builder.lea(reg:t_jit_reg;mem:t_jit_int64); @@ -1300,9 +1464,11 @@ begin lea(reg,[Sums(mem)]); end; +// + procedure t_jit_builder.addq(mem:t_jit_regs;reg:t_jit_reg); begin - _mov($01,reg,[Sums(mem)]); //ADD r/m64, r64 + _mov($01,reg,mem); //ADD r/m64, r64 end; procedure t_jit_builder.addq(mem:t_jit_int64;reg:t_jit_reg); @@ -1312,7 +1478,7 @@ end; procedure t_jit_builder.addq(reg:t_jit_reg;mem:t_jit_regs); begin - _mov($03,reg,[Sums(mem)]); //ADD r64, r/m64 + _mov($03,reg,mem); //ADD r64, r/m64 end; procedure t_jit_builder.addq(reg:t_jit_reg;mem:t_jit_int64); @@ -1321,166 +1487,49 @@ begin end; procedure t_jit_builder.addq(reg0:t_jit_reg;reg1:t_jit_reg); -var - rexB,rexR,rexW:Boolean; - - ModRM:record - Index,RM:Byte; - end; - - Prefix,op:Byte; - - ji:t_jit_instruction; begin - Assert(is_one_reg(reg0)); - Assert(is_one_reg(reg1)); - - Assert(is_reg_size(reg0,[os8,os16,os32,os64])); - Assert(is_reg_size(reg1,[os8,os16,os32,os64])); - - Assert(is_reg_type(reg0,[regGeneral])); - Assert(is_reg_type(reg1,[regGeneral])); - - Assert(reg0.ARegValue[0].AScale<=1); - Assert(reg1.ARegValue[0].AScale<=1); - - Assert(reg0.ARegValue[0].ASize=reg1.ARegValue[0].ASize); - - ji:=default_jit_instruction; - - rexB:=false; - rexR:=false; - rexW:=false; - Prefix:=0; - - case reg0.ARegValue[0].ASize of - os8: - begin - op:=$00; - end; - os16: - begin - Prefix:=$66; - op:=$01; - end; - os32: - begin - op:=$01; - end; - os64: - begin - rexW:=True; - op:=$01; - end; - else; - end; - - ModRM.Index:=reg1.ARegValue[0].AIndex; - if (ModRM.Index>=8) then - begin - rexR:=true; - Dec(ModRM.Index,8); - end; - - ModRM.RM:=reg0.ARegValue[0].AIndex; - if (ModRM.RM>=8) then - begin - rexB:=true; - Dec(ModRM.RM,8); - end; - - if (Prefix<>0) then - begin - ji.EmitByte(Prefix); //Operand-size override prefix (16) - end; - - if rexB or rexR or rexW then - begin - ji.EmitREX(rexB,False,rexR,rexW); - end; - - ji.EmitByte(op); - - ji.EmitModRM(3,ModRM.Index,ModRM.RM); - - _add(ji); + _mov($01,$00,reg0,reg1); end; procedure t_jit_builder.addi(reg:t_jit_reg;imm:Int64); -var - rexB,rexW:Boolean; - - Index,Prefix,op:Byte; - - ji:t_jit_instruction; begin - Assert(is_one_reg(reg)); - Assert(is_reg_size(reg,[os8,os16,os32,os64])); - - Assert(is_reg_type(reg,[regGeneral])); - - Assert(reg.ARegValue[0].AScale<=1); - - ji:=default_jit_instruction; - - rexB:=false; - rexW:=false; - Prefix:=0; - - Index:=reg.ARegValue[0].AIndex; - if (Index>=8) then - begin - rexB:=true; - Dec(Index,8); - end; - - case reg.ARegValue[0].ASize of - os8: - begin - op:=$80; - end; - os16: - begin - Prefix:=$66; - op:=$81; - end; - os32: - begin - op:=$81; - end; - os64: - begin - rexW:=True; - op:=$81; - end; - else; - end; - - if (Prefix<>0) then - begin - ji.EmitByte(Prefix); //Operand-size override prefix (16) - end; - - if rexB or rexW then - begin - ji.EmitREX(rexB,False,False,rexW); - end; - - ji.EmitByte(op); - - ji.EmitModRM(3,0,Index); - - case reg.ARegValue[0].ASize of - os8:ji.EmitByte (imm); - os16:ji.EmitWord (imm); - os32:ji.EmitInt32(imm); - os64:ji.EmitInt32(imm); - else; - end; - - _add(ji); + _addi($81,$80,0,reg,imm); end; +// + +procedure t_jit_builder.subq(mem:t_jit_regs;reg:t_jit_reg); +begin + _mov($29,reg,mem); //SUB r/m64, r64 +end; + +procedure t_jit_builder.subq(mem:t_jit_int64;reg:t_jit_reg); +begin + addq([Sums(mem)],reg); +end; + +procedure t_jit_builder.subq(reg:t_jit_reg;mem:t_jit_regs); +begin + _mov($2B,reg,mem); //SUB r64, r/m64 +end; + +procedure t_jit_builder.subq(reg:t_jit_reg;mem:t_jit_int64); +begin + addq(reg,[Sums(mem)]); +end; + +procedure t_jit_builder.subq(reg0:t_jit_reg;reg1:t_jit_reg); +begin + _mov($29,$28,reg0,reg1); +end; + +procedure t_jit_builder.subi(reg:t_jit_reg;imm:Int64); +begin + _addi($81,$80,5,reg,imm); +end; + +/// + procedure t_jit_builder.push16(mem:t_jit_regs); begin _push($FF,6,os16,mem); diff --git a/sys/kern/kern_stub.pas b/sys/kern/kern_stub.pas index af2aeaa2..620caf2d 100644 --- a/sys/kern/kern_stub.pas +++ b/sys/kern/kern_stub.pas @@ -117,8 +117,8 @@ begin if (err<>0) then Exit; vm_map_lock(map); - vm_map_set_name_locked(map,start,start+size,'#patch'); - pmap_mark_flags(start,start+size,PAGE_PATCH_FLAG); + vm_map_set_name_locked(map,start,start+size,'#patch',VM_INHERIT_PATCH); + //pmap_mark_flags(start,start+size,PAGE_PATCH_FLAG); vm_map_unlock(map); Result:=Pointer(start); diff --git a/sys/md/vm_pmap.pas b/sys/md/vm_pmap.pas index e64fecd6..f81bc5c5 100644 --- a/sys/md/vm_pmap.pas +++ b/sys/md/vm_pmap.pas @@ -13,8 +13,8 @@ uses vm_object; const - PAGE_MAP_COUNT=(QWORD(VM_MAXUSER_ADDRESS) shr PAGE_SHIFT); - PAGE_MAP_MASK =PAGE_MAP_COUNT-1; + PAGE_MAP_COUNT =(QWORD(VM_MAXUSER_ADDRESS) shr PAGE_SHIFT); + PAGE_MAP_MASK =PAGE_MAP_COUNT-1; PAGE_PROT_EXECUTE=DWORD($80000000); PAGE_PROT_WRITE =DWORD($40000000); @@ -24,8 +24,10 @@ const PAGE_PROT_SHIFT =29; - PAGE_BUSY_FLAG =DWORD($10000000); - PAGE_PATCH_FLAG =DWORD($08000000); + PAGE_OFS_MASK =(1 shl PAGE_PROT_SHIFT)-1; //Possible addressing in 8TB + + //PAGE_BUSY_FLAG =DWORD($10000000); + //PAGE_PATCH_FLAG =DWORD($08000000); var PAGE_MAP:PDWORD=nil; @@ -268,7 +270,7 @@ begin __off:=OFF_TO_IDX(__off); while (start<__end) do begin - PAGE_MAP[start and PAGE_MAP_MASK]:=__off or flags; + PAGE_MAP[start and PAGE_MAP_MASK]:=(__off and PAGE_OFS_MASK) or flags; Inc(__off); Inc(start); end; @@ -314,7 +316,7 @@ begin addr:=OFF_TO_IDX(addr); addr:=addr and PAGE_MAP_MASK; Result:=PAGE_MAP[addr]; - Result:=Result and PAGE_MAP_MASK; + Result:=Result and PAGE_OFS_MASK; end; function pmap_test_cross(addr:vm_offset_t;h:Integer):Boolean; @@ -328,8 +330,8 @@ begin Result:=False; end else begin - page1:=PAGE_MAP[page1] and PAGE_MAP_MASK; - page2:=PAGE_MAP[page2] and PAGE_MAP_MASK; + page1:=PAGE_MAP[page1] and PAGE_OFS_MASK; + page2:=PAGE_MAP[page2] and PAGE_OFS_MASK; Result:=(page1<>page2); end; end; @@ -349,7 +351,7 @@ asm mov PAGE_MAP,%rax mov (%rax,%rdi,4),%edi //filter (rdi) - and PAGE_MAP_MASK,%rdi + and PAGE_OFS_MASK,%rdi jz _exit //combine (rdi|rsi) shl PAGE_SHIFT,%rdi @@ -404,6 +406,7 @@ begin //shift base:=base+VM_MIN_GPU_ADDRESS; prot:=prot or ((prot and VM_PROT_GPU_ALL) shr 4); + Writeln('pmap_enter_gpuobj:',HexStr(QWORD(base),11),':',HexStr(QWORD(base)+(__end-start),11),':',HexStr(prot,2)); end; r:=NtAllocateVirtualMemory( @@ -435,7 +438,7 @@ var begin old:=0; - pmap_mark_flags(start,start+size,PAGE_BUSY_FLAG); + //pmap_mark_flags(start,start+size,PAGE_BUSY_FLAG); //set old to readonly r:=NtProtectVirtualMemory( diff --git a/sys/vm/vm.pas b/sys/vm/vm.pas index f28ea0ca..2db03c01 100644 --- a/sys/vm/vm.pas +++ b/sys/vm/vm.pas @@ -35,6 +35,7 @@ const VM_INHERIT_SHARE =vm_inherit_t(0); VM_INHERIT_COPY =vm_inherit_t(1); VM_INHERIT_NONE =vm_inherit_t(2); + VM_INHERIT_PATCH =vm_inherit_t(3); VM_INHERIT_DEFAULT=VM_INHERIT_COPY; VM_PROT_NONE =vm_prot_t($00); diff --git a/sys/vm/vm_fault.pas b/sys/vm/vm_fault.pas index dd16cbb3..b0584da8 100644 --- a/sys/vm/vm_fault.pas +++ b/sys/vm/vm_fault.pas @@ -25,6 +25,7 @@ uses systm, trap, x86_fpdbgdisas, + x86_jit, kern_stub, ucontext, vm_patch_link; @@ -148,9 +149,24 @@ begin end; end; -function vm_check_patch(map:vm_map_t;vaddr:vm_offset_t):Boolean; +function GetFrameOffsetInt(RegValue:TRegValue):Integer; inline; begin - Result:=(pmap_get_raw(vaddr) and PAGE_PATCH_FLAG)<>0; + Result:=Integer(ptruint(GetFrameOffset(RegValue))); +end; + +function vm_check_patch(map:vm_map_t;vaddr:vm_offset_t):Boolean; +var + entry:vm_map_entry_t; +begin + if (vm_map_lookup_entry(map,vaddr,@entry)) then + begin + Result:=(entry^.inheritance=VM_INHERIT_PATCH); + end else + begin + Result:=False; + end; + + //Result:=(pmap_get_raw(vaddr) and PAGE_PATCH_FLAG)<>0; end; procedure test_jit; @@ -276,6 +292,59 @@ begin vm_add_jit_patch_link(entry^.vm_obj,vaddr,vsize,stub); end; +type + tcopy_cb=procedure(vaddr:Pointer); //rdi + +//rdi,rsi +procedure copyout_xmm(vaddr:Pointer;cb:tcopy_cb); +var + data:array[0..15] of Byte; +begin + if pmap_test_cross(QWORD(vaddr),15) then + begin + cb(@data); //xmm->data + copyout(@data,vaddr,16); + end else + begin + vaddr:=uplift(vaddr); + cb(vaddr); //xmm->vaddr + end; +end; + +//rdi,rsi +procedure copyin_xmm(vaddr:Pointer;cb:tcopy_cb); +var + data:array[0..15] of Byte; +begin + if pmap_test_cross(QWORD(vaddr),15) then + begin + copyin(vaddr,@data,16); + cb(@data); //data->xmm + end else + begin + vaddr:=uplift(vaddr); + cb(vaddr); //vaddr->xmm + end; +end; + +type + t_memop_type=(moCopyout,moCopyin); + +function classif_memop(var din:TInstruction):t_memop_type; +begin + if (ofMemory in din.Operand[1].Flags) then + begin + Result:=moCopyout; + end else + if (ofMemory in din.Operand[2].Flags) then + begin + Result:=moCopyin; + end else + begin + Assert(false,'classif_memop'); + end; +end; + type p_jit_code=^t_jit_code; t_jit_code=record @@ -283,16 +352,250 @@ type prolog:p_stub_chunk; o_len :Byte; o_data:t_data16; + code :record end; end; -//function pmap_test_cross(addr:vm_offset_t;h:Integer):Boolean; +type + t_jit_context=record + rip_addr:QWORD; -function generate_jit(var dis:TX86Disassembler; - var din:TInstruction):Pointer; + Code:t_data16; + + dis:TX86Disassembler; + din:TInstruction; + + builder:t_jit_builder; + + jit_code:p_jit_code; + end; + +procedure build_lea(var ctx:t_jit_context;id:Byte); +var + RegValue:TRegValues; + adr,tdr:t_jit_reg; + i:Integer; + ofs:Int64; +begin + RegValue:=ctx.din.Operand[id].RegValue; + + adr:=t_jit_builder.rdi; + adr.ARegValue[0].ASize:=RegValue[0].ASize; + + tdr:=t_jit_builder.rsi; + + with ctx.builder do + begin + movq(tdr,[GS+Integer(teb_thread)]); + + i:=GetFrameOffsetInt(RegValue[0]); + Assert(i<>0,'build_lea'); + + //xor adr,adr needed + + movq(adr,[tdr+i]); + + if (RegValue[0].AScale>1) then + begin + lea(adr,[adr*RegValue[0].AScale]) + end; + + if (RegValue[1].AType<>regNone) then + begin + i:=GetFrameOffsetInt(RegValue[1]); + Assert(i<>0,'build_lea'); + + addq(adr,[tdr+i]); + end; + end; + + i:=ctx.din.Operand[id].CodeIndex; + case ctx.din.Operand[id].ByteCount of + 1: ofs:=PShortint(@ctx.Code[i])^; + 2: ofs:=PSmallint(@ctx.Code[i])^; + 4: ofs:=PInteger (@ctx.Code[i])^; + 8: ofs:=PInt64 (@ctx.Code[i])^; + else + Exit; + end; + + with ctx.builder do + begin + lea(adr,[adr+ofs]); + end; +end; + +procedure build_vmovdqu(var ctx:t_jit_context;id:Byte;memop:t_memop_type); +var + link:t_jit_i_link; + dst:t_jit_reg; +begin + case memop of + moCopyout: + begin + with ctx.builder do + begin + //input:rdi + + link:=leaj(rsi,[rip+$FFFF],-1); + + call(@copyout_xmm); //rdi,rsi + + reta; + + //input:rdi + + link._label:=_label; + + dst:=Default(t_jit_reg); + dst.ARegValue:=ctx.din.Operand[id].RegValue; + + vmovdqu([rdi],dst); + + reta; + end; + end; + moCopyin: + begin + with ctx.builder do + begin + //input:rdi + + link:=movj(rsi,[rip+$FFFF],-1); + + call(@copyin_xmm); //rdi,rsi + + reta; + + //input:rdi + + link._label:=_label; + + dst:=Default(t_jit_reg); + dst.ARegValue:=ctx.din.Operand[id].RegValue; + + vmovdqu(dst,[rdi]); + + reta; + end; + end; + end; +end; + +procedure build_vmovdqa(var ctx:t_jit_context;id:Byte;memop:t_memop_type); +var + dst:t_jit_reg; +begin + case memop of + moCopyout: + begin + with ctx.builder do + begin + //input:rdi + + call(@uplift); //input:rdi output:rax + + dst:=Default(t_jit_reg); + dst.ARegValue:=ctx.din.Operand[id].RegValue; + + vmovdqa([rax],dst); + + reta; + end; + end; + moCopyin: + begin + with ctx.builder do + begin + //input:rdi + + call(@uplift); //input:rdi output:rax + + dst:=Default(t_jit_reg); + dst.ARegValue:=ctx.din.Operand[id].RegValue; + + vmovdqa(dst,[rax]); + + reta; + end; + end; + end; +end; + +function generate_jit(var ctx:t_jit_context):p_stub_chunk; +var + memop:t_memop_type; + code_size,size:Integer; begin + ctx.builder.call(@test_jit); ///test + case ctx.din.OpCode.Opcode of + OPmov: + begin + case ctx.din.OpCode.Suffix of + OPSx_dqu: + begin + memop:=classif_memop(ctx.din); + + case memop of + moCopyout: + begin + build_lea(ctx,1); + build_vmovdqu(ctx,2,memop); + end; + moCopyin: + begin + build_lea(ctx,2); + build_vmovdqu(ctx,1,memop); + end; + end; + end; + + OPSx_dqa: + begin + memop:=classif_memop(ctx.din); + + case memop of + moCopyout: + begin + build_lea(ctx,1); + build_vmovdqa(ctx,2,memop); + end; + moCopyin: + begin + build_lea(ctx,2); + build_vmovdqa(ctx,1,memop); + end; + end; + end + + else; + end; //case + end; //OPmov + + else; + Assert(false); + end; + + code_size:=ctx.builder.GetMemSize; + size:=SizeOf(t_jit_code)+code_size; + + Result:=p_alloc(nil,size); + + ctx.jit_code:=@Result^.body; + ctx.jit_code^:=Default(t_jit_code); + + ctx.jit_code^.frame.call:=@ctx.jit_code^.code; + ctx.jit_code^.frame.addr:=Pointer(ctx.rip_addr); + ctx.jit_code^.frame.reta:=Pointer(ctx.rip_addr+ctx.dis.CodeIdx); + + ctx.jit_code^.o_len :=ctx.dis.CodeIdx; + ctx.jit_code^.o_data:=ctx.Code; + + ctx.builder.SaveTo(@ctx.jit_code^.code,code_size); + + /// end; function vm_try_jit_patch(map:vm_map_t; @@ -300,16 +603,15 @@ function vm_try_jit_patch(map:vm_map_t; rip_addr:vm_offset_t):Integer; var err:Integer; - data:t_data16; - dis:TX86Disassembler; - din:TInstruction; + ctx:t_jit_context; ptr:Pointer; - chunk:p_stub_chunk; + chunk_prolog:p_stub_chunk; + chunk_jit:p_stub_chunk; jit_prolog:p_jit_prolog; - jit_frame:p_jit_frame; + //jit_frame:p_jit_frame; delta:Int64; rip_addr_jmp:vm_offset_t; @@ -324,29 +626,27 @@ begin //Did the exception happen inside a patch? just going out if vm_check_patch(map,rip_addr) then begin - vm_map_unlock(map); Exit(0); end; //Is the exception already patched? if vm_patch_exist(Pointer(rip_addr),0) then begin - vm_map_unlock(map); Exit(0); end; - data:=c_data16; + Writeln('mmaped addr 0x',HexStr(mem_addr,16),' to 0x',HexStr(uplift(Pointer(mem_addr)))); - err:=copyin_nofault(Pointer(rip_addr),@data,SizeOf(data)); + ctx:=Default(t_jit_context); + ctx.Code:=c_data16; + + err:=copyin_nofault(Pointer(rip_addr),@ctx.Code,SizeOf(ctx.Code)); if (err<>0) then Exit(KERN_PROTECTION_FAILURE); - dis:=Default(TX86Disassembler); - din:=Default(TInstruction); + ptr:=@ctx.Code; + ctx.dis.Disassemble(dm64,ptr,ctx.din); - ptr:=@data; - dis.Disassemble(dm64,ptr,din); - - if vm_patch_exist(Pointer(rip_addr+dis.CodeIdx),1) then + if vm_patch_exist(Pointer(rip_addr+ctx.dis.CodeIdx),1) then begin Assert(False,'patch on next instruction TODO'); end; @@ -359,32 +659,41 @@ begin //OPCODE: OPMOV //SUFFIX: OPSX_DQU - if (dis.CodeIdx=4) then + if (ctx.dis.CodeIdx=4) then begin - mask:=data[4]; + mask:=ctx.Code[4]; mask:=mask shl 24; rip_addr_jmp:=rip_addr+SizeOf(t_jmp32_trampoline); - chunk:=p_alloc_m(Pointer(rip_addr_jmp),SizeOf(t_jit_prolog),mask); + chunk_prolog:=p_alloc_m(Pointer(rip_addr_jmp),SizeOf(t_jit_prolog),mask); - if (chunk=nil) then Exit(KERN_NO_SPACE); + if (chunk_prolog=nil) then Exit(KERN_NO_SPACE); - jit_frame:=AllocMem(SizeOf(t_jit_frame)); + ctx.rip_addr:=rip_addr; - jit_frame^.call:=@test_jit; - jit_frame^.addr:=Pointer(rip_addr); - jit_frame^.reta:=Pointer(rip_addr+dis.CodeIdx); + chunk_jit:=generate_jit(ctx); - jit_prolog:=@chunk^.body; + //Writeln('vm_check_patch:',vm_check_patch(map,vm_offset_t(ctx.jit_code))); + + //jit_frame:=AllocMem(SizeOf(t_jit_frame)); + //jit_frame^.call:=@test_jit; + //jit_frame^.addr:=Pointer(rip_addr); + //jit_frame^.reta:=Pointer(rip_addr+ctx.dis.CodeIdx); + + jit_prolog:=@chunk_prolog^.body; jit_prolog^:=c_jit_prolog; - jit_prolog^.jframe:=jit_frame; + + //jit_prolog^.jframe:=jit_frame; + jit_prolog^.jframe:=@ctx.jit_code^.frame; + + ctx.jit_code^.prolog:=chunk_prolog; delta:=Int64(jit_prolog)-(Int64(rip_addr_jmp)); Assert(is_mask_valid(Pointer(rip_addr_jmp),jit_prolog,mask),'vm_try_jit_patch'); - patch_original(map,rip_addr,dis.CodeIdx,chunk,Integer(delta)); + patch_original(map,rip_addr,ctx.dis.CodeIdx,chunk_jit,Integer(delta)); end else begin Assert(False,'TODO'); diff --git a/sys/vm/vm_map.pas b/sys/vm/vm_map.pas index 1ac8e938..5ea7c457 100644 --- a/sys/vm/vm_map.pas +++ b/sys/vm/vm_map.pas @@ -248,6 +248,7 @@ function vm_map_remove(map:vm_map_t;start:vm_offset_t;__end:vm_offset_t):Intege procedure vm_map_set_name(map:vm_map_t;start,__end:vm_offset_t;name:PChar); procedure vm_map_set_name_locked(map:vm_map_t;start,__end:vm_offset_t;name:PChar); +procedure vm_map_set_name_locked(map:vm_map_t;start,__end:vm_offset_t;name:PChar;i:vm_inherit_t); function vmspace_pmap(vm:p_vmspace):pmap_t; inline; @@ -2821,6 +2822,7 @@ begin while ((current<>@map^.header) and (current^.start<__end)) do begin vm_map_clip_end(map,current,__end); + MoveChar0(name^,current^.name,32); vm_map_simplify_entry(map, current); @@ -2829,6 +2831,35 @@ begin end; end; +procedure vm_map_set_name_locked(map:vm_map_t;start,__end:vm_offset_t;name:PChar;i:vm_inherit_t); +var + current:vm_map_entry_t; + entry:vm_map_entry_t; +begin + VM_MAP_RANGE_CHECK(map, start, __end); + + if (vm_map_lookup_entry(map, start,@entry)) then + begin + vm_map_clip_start(map, entry, start); + end else + begin + entry:=entry^.next; + end; + + current:=entry; + while ((current<>@map^.header) and (current^.start<__end)) do + begin + vm_map_clip_end(map,current,__end); + + MoveChar0(name^,current^.name,32); + current^.inheritance:=i; + + vm_map_simplify_entry(map, current); + + current:=current^.next; + end; +end; + procedure vm_map_set_name(map:vm_map_t;start,__end:vm_offset_t;name:PChar); begin vm_map_lock(map);