From 27a85cf262eed07f9bd358ca46cb313671dc39b5 Mon Sep 17 00:00:00 2001 From: Pavel <68122101+red-prig@users.noreply.github.com> Date: Wed, 30 Aug 2023 09:43:22 +0300 Subject: [PATCH] + --- sys/kern/kern_stub.pas | 6 +- sys/kern/machdep.pas | 25 ++++++ sys/kern/trap.pas | 28 ++++++- sys/md/md_context.pas | 3 + sys/vm/vm_fault.pas | 163 ++++++++++++++++++++------------------- sys/vm/vm_patch_link.pas | 25 +++--- 6 files changed, 156 insertions(+), 94 deletions(-) diff --git a/sys/kern/kern_stub.pas b/sys/kern/kern_stub.pas index 620caf2d..170bac3e 100644 --- a/sys/kern/kern_stub.pas +++ b/sys/kern/kern_stub.pas @@ -249,13 +249,14 @@ var entry,next:p_stub_chunk; begin Result:=nil; + size:=size+SizeOf(stub_chunk); entry:=TAILQ_FIRST(@chunk_free); while (entry<>nil) do begin next:=TAILQ_NEXT(entry,@entry^.link); // - if (entry^.curr_size>=(size+SizeOf(stub_chunk))) then + if (entry^.curr_size>=size) then begin if (vaddr=nil) or is_near_valid(vaddr,@entry^.body) then begin @@ -274,13 +275,14 @@ var entry,next:p_stub_chunk; begin Result:=nil; + size:=size+SizeOf(stub_chunk); entry:=TAILQ_FIRST(@chunk_free); while (entry<>nil) do begin next:=TAILQ_NEXT(entry,@entry^.link); // - if (entry^.curr_size>=(size+SizeOf(stub_chunk))) then + if (entry^.curr_size>=size) then begin if is_mask_valid(vaddr,@entry^.body,mask) then begin diff --git a/sys/kern/machdep.pas b/sys/kern/machdep.pas index 744a3cf3..bd16c3a4 100644 --- a/sys/kern/machdep.pas +++ b/sys/kern/machdep.pas @@ -23,6 +23,8 @@ Procedure bmove(src,dst:Pointer;size:ptrint); function cpu_getstack(td:p_kthread):QWORD; procedure cpu_set_user_tls(td:p_kthread;base:Pointer); procedure cpu_set_gsbase(td:p_kthread;base:Pointer); +procedure cpu_init_jit(td:p_kthread); +procedure cpu_fini_jit(td:p_kthread); procedure cpu_fetch_syscall_args(td:p_kthread); procedure cpu_set_syscall_retval(td:p_kthread;error:Integer); procedure cpu_set_upcall_kse(td:p_kthread;entry,arg:Pointer;stack:p_stack_t); @@ -91,6 +93,29 @@ begin set_pcb_flags(td,PCB_FULL_IRET); end; +procedure cpu_init_jit(td:p_kthread); +begin + //teb stack + td^.td_teb^.sttop:=td^.td_kstack.sttop; + td^.td_teb^.stack:=td^.td_kstack.stack; + //teb stack +end; + +procedure cpu_fini_jit(td:p_kthread); +begin + //teb stack + if (sigonstack(td^.td_frame.tf_rsp)<>0) then + begin + td^.td_teb^.stack:=td^.td_sigstk.ss_sp; + td^.td_teb^.sttop:=td^.td_sigstk.ss_sp-td^.td_sigstk.ss_size; + end else + begin + td^.td_teb^.stack:=td^.td_ustack.stack; + td^.td_teb^.sttop:=td^.td_ustack.sttop; + end; + //teb stack +end; + procedure cpu_fetch_syscall_args(td:p_kthread); begin td^.td_retval[0]:=0; diff --git a/sys/kern/trap.pas b/sys/kern/trap.pas index 6685536e..62af9fea 100644 --- a/sys/kern/trap.pas +++ b/sys/kern/trap.pas @@ -148,6 +148,8 @@ const jitcall:@jit_call; ); +procedure switch_to_jit(frame:p_jit_frame); + function IS_TRAP_FUNC(rip:qword):Boolean; inline; function trap(frame:p_trapframe):Integer; @@ -816,6 +818,30 @@ end; end; } +procedure switch_to_jit(frame:p_jit_frame); +var + td:p_kthread; +begin + td:=curkthread; + if (td=nil) then Exit; + + td^.td_teb^.jit_rsp:=Pointer(td^.td_frame.tf_rsp); + td^.td_frame.tf_rsp:=QWORD(frame); + td^.td_frame.tf_rip:=QWORD(@jit_call); +end; + +type + t_proc=Procedure(); + +procedure jit_frame_call(frame:p_jit_frame); +begin + cpu_init_jit(curkthread); + + t_proc(frame^.call)(); + + cpu_fini_jit(curkthread); +end; + //input: // 1: %gs:teb.jit_rsp (original %rsp) // 2: %rsp (jitcall addr) @@ -889,7 +915,7 @@ asm movqq %rax,%gs:teb.jitcall movqq %rax,%gs:teb.jit_rsp - call t_jit_frame.call(%rdi) //call jit code + call jit_frame_call //call jit code _after_call: diff --git a/sys/md/md_context.pas b/sys/md/md_context.pas index c914af61..65df04a4 100644 --- a/sys/md/md_context.pas +++ b/sys/md/md_context.pas @@ -56,6 +56,9 @@ type YmmContext:TYMMCONTEXT; end; + TXmmSaveArea = Windows.TXmmSaveArea; + PXmmSaveArea = Windows.PXmmSaveArea; + PCONTEXT_CHUNK=^TCONTEXT_CHUNK; TCONTEXT_CHUNK=packed record Offset:LONG; diff --git a/sys/vm/vm_fault.pas b/sys/vm/vm_fault.pas index b0584da8..76b766ec 100644 --- a/sys/vm/vm_fault.pas +++ b/sys/vm/vm_fault.pas @@ -30,16 +30,6 @@ uses ucontext, vm_patch_link; -{ -TRegValue = object - AType : TRegisterType; - ASize : TOperandSize; - AIndex: Byte; - AScale: Byte; - // - function StrValue:RawByteString; -end; -} function GetFrameOffset(RegValue:TRegValue): Pointer; begin @@ -122,10 +112,6 @@ begin begin Result := nil; //Format('ymm%u', [AIndex]); end; - os512: - begin - Result := nil; //Format('zmm%u', [AIndex]); - end; else; end; end; @@ -154,19 +140,19 @@ begin Result:=Integer(ptruint(GetFrameOffset(RegValue))); end; -function vm_check_patch(map:vm_map_t;vaddr:vm_offset_t):Boolean; +function vm_check_patch_entry(map:vm_map_t;vaddr:vm_offset_t;p_entry:p_vm_map_entry_t):Boolean; var entry:vm_map_entry_t; begin if (vm_map_lookup_entry(map,vaddr,@entry)) then begin + p_entry^:=entry; Result:=(entry^.inheritance=VM_INHERIT_PATCH); end else begin - Result:=False; + p_entry^:=nil; + Result:=True; end; - - //Result:=(pmap_get_raw(vaddr) and PAGE_PATCH_FLAG)<>0; end; procedure test_jit; @@ -201,13 +187,6 @@ begin Result:=addr-(addr mod alignment); end; -procedure vm_add_jit_patch_link(obj:vm_map_object;vaddr:vm_offset_t;vsize:Integer;stub:p_stub_chunk); -begin - Assert(obj<>nil,'vm_add_jit_patch_link'); - - vm_add_patch_link(obj,Pointer(vaddr),vsize,pt_jit,stub); -end; - procedure copy_xchg(src,dst:Pointer;vsize:Integer); var data:t_data16; @@ -242,23 +221,16 @@ begin end; procedure patch_original(map:vm_map_t; + entry:vm_map_entry_t; vaddr:vm_offset_t; vsize:Integer; - stub:p_stub_chunk; delta:Integer); var trampoline:t_jmp32_trampoline; - entry:vm_map_entry_t; start :vm_offset_t; __end :vm_offset_t; new_prot:vm_prot_t; begin - entry:=nil; - if not vm_map_lookup_entry(map,vaddr,@entry) then - begin - Assert(False,'patch_original'); - end; - new_prot:=entry^.protection or VM_PROT_READ or VM_PROT_WRITE; if (new_prot<>entry^.protection) then @@ -288,8 +260,6 @@ begin entry^.protection, new_prot); end; - - vm_add_jit_patch_link(entry^.vm_obj,vaddr,vsize,stub); end; type @@ -603,37 +573,54 @@ function vm_try_jit_patch(map:vm_map_t; rip_addr:vm_offset_t):Integer; var err:Integer; + vsize:Integer; + + entry:vm_map_entry_t; + obj:vm_map_object; ctx:t_jit_context; ptr:Pointer; chunk_prolog:p_stub_chunk; - chunk_jit:p_stub_chunk; + chunk_jit :p_stub_chunk; jit_prolog:p_jit_prolog; - //jit_frame:p_jit_frame; + delta:Int64; rip_addr_jmp:vm_offset_t; - rv:Integer; mask:DWORD; info:t_patch_info; begin Result:=0; + info:=vm_get_patch_link(Pointer(rip_addr),0,pt_jit_frame); + + //Is the exception already patched? + if (info.stub<>nil) then + begin + ctx.jit_code:=@info.stub^.body; + if (ctx.jit_code^.prolog=nil) then + begin + //Prologue not created? Switch code on exit + switch_to_jit(@ctx.jit_code^.frame); + Exit(0); + end; + + p_dec_ref(info.stub); //release (vm_get_patch_link) + Exit(0); + end; + //Did the exception happen inside a patch? just going out - if vm_check_patch(map,rip_addr) then + if vm_check_patch_entry(map,rip_addr,@entry) then begin Exit(0); end; - //Is the exception already patched? - if vm_patch_exist(Pointer(rip_addr),0) then - begin - Exit(0); - end; + obj:=entry^.vm_obj; + Assert(obj<>nil,'vm_try_jit_patch (obj=nil)'); Writeln('mmaped addr 0x',HexStr(mem_addr,16),' to 0x',HexStr(uplift(Pointer(mem_addr)))); @@ -646,60 +633,78 @@ begin ptr:=@ctx.Code; ctx.dis.Disassemble(dm64,ptr,ctx.din); - if vm_patch_exist(Pointer(rip_addr+ctx.dis.CodeIdx),1) then + vsize:=ctx.dis.CodeIdx; + + info:=vm_get_patch_link(Pointer(rip_addr-1),1,pt_jit_frame); + + if (info.stub<>nil) then begin - Assert(False,'patch on next instruction TODO'); + + //Overlap? + if (info.vsize<5) then + begin + Assert(False,'Overlap patch on prev instruction TODO'); + end; + + p_dec_ref(info.stub); //release (vm_get_patch_link) end; - if vm_patch_exist(Pointer(rip_addr-1),1) then - begin - Assert(False,'patch on prev instruction TODO'); + rip_addr_jmp:=rip_addr+SizeOf(t_jmp32_trampoline); + + chunk_prolog:=nil; + + case vsize of + 4: + begin + //Overlapping jmp instructions [00 11 22 33] [MM] + mask:=ctx.Code[4]; + mask:=mask shl 24; + + chunk_prolog:=p_alloc_m(Pointer(rip_addr_jmp),SizeOf(t_jit_prolog),mask); + end; + 5: + begin + //Near 32bit jmp instructions + + chunk_prolog:=p_alloc(Pointer(rip_addr_jmp),SizeOf(t_jit_prolog)); + end; + else + Assert(false,'vm_try_jit_patch (vsize)'); end; - //OPCODE: OPMOV - //SUFFIX: OPSX_DQU + ctx.rip_addr:=rip_addr; + chunk_jit:=generate_jit(ctx); - if (ctx.dis.CodeIdx=4) then + if (chunk_prolog=nil) then + begin + //Prologue not created? + end else begin - mask:=ctx.Code[4]; - mask:=mask shl 24; - - rip_addr_jmp:=rip_addr+SizeOf(t_jmp32_trampoline); - - chunk_prolog:=p_alloc_m(Pointer(rip_addr_jmp),SizeOf(t_jit_prolog),mask); - - if (chunk_prolog=nil) then Exit(KERN_NO_SPACE); - - ctx.rip_addr:=rip_addr; - - chunk_jit:=generate_jit(ctx); - - //Writeln('vm_check_patch:',vm_check_patch(map,vm_offset_t(ctx.jit_code))); - - //jit_frame:=AllocMem(SizeOf(t_jit_frame)); - //jit_frame^.call:=@test_jit; - //jit_frame^.addr:=Pointer(rip_addr); - //jit_frame^.reta:=Pointer(rip_addr+ctx.dis.CodeIdx); - jit_prolog:=@chunk_prolog^.body; jit_prolog^:=c_jit_prolog; - - //jit_prolog^.jframe:=jit_frame; jit_prolog^.jframe:=@ctx.jit_code^.frame; ctx.jit_code^.prolog:=chunk_prolog; delta:=Int64(jit_prolog)-(Int64(rip_addr_jmp)); - Assert(is_mask_valid(Pointer(rip_addr_jmp),jit_prolog,mask),'vm_try_jit_patch'); + if (vsize<5) then + begin + Assert(is_mask_valid(Pointer(rip_addr_jmp),jit_prolog,mask),'vm_try_jit_patch (is_mask_valid)'); + end else + begin + Assert(is_near_valid(Pointer(rip_addr_jmp),jit_prolog),'vm_try_jit_patch (is_near_valid)'); + end; - patch_original(map,rip_addr,ctx.dis.CodeIdx,chunk_jit,Integer(delta)); - end else - begin - Assert(False,'TODO'); + patch_original(map,entry,rip_addr,vsize,Integer(delta)); + + vm_add_patch_link(obj,Pointer(rip_addr),vsize,pt_jit_prolog,chunk_prolog); end; + vm_add_patch_link(obj,Pointer(rip_addr),vsize,pt_jit_frame,chunk_jit); + //Switch code on exit + switch_to_jit(@ctx.jit_code^.frame); end; function vm_fault(map :vm_map_t; diff --git a/sys/vm/vm_patch_link.pas b/sys/vm/vm_patch_link.pas index bdf69ce2..789ed784 100644 --- a/sys/vm/vm_patch_link.pas +++ b/sys/vm/vm_patch_link.pas @@ -10,7 +10,7 @@ uses kern_stub; type - t_patch_type=(pt_fsbase,pt_gsbase,pt_syscall,pt_unresolve,pt_jit); + t_patch_type=(pt_none,pt_fsbase,pt_gsbase,pt_syscall,pt_unresolve,pt_jit_prolog,pt_jit_frame); p_patch_info=^t_patch_info; t_patch_info=record @@ -27,7 +27,7 @@ type info:t_patch_info; end; -function vm_get_patch_link(vaddr:Pointer;vsize:Integer):t_patch_info; +function vm_get_patch_link(vaddr:Pointer;vsize:Integer;ptype:t_patch_type):t_patch_info; function vm_patch_exist(vaddr:Pointer;vsize:Integer):Boolean; procedure vm_add_patch_link (_obj,vaddr:Pointer;vsize:Integer;ptype:t_patch_type;stub:p_stub_chunk); @@ -113,7 +113,7 @@ begin Result:=((vaddr+vsize)>info^.vaddr) and (vaddr<(info^.vaddr+info^.vsize)); end; -function _vm_get_patch_link(page:p_patch_page;vaddr:Pointer):t_patch_info; +function _vm_get_patch_link(page:p_patch_page;vaddr:Pointer;ptype:t_patch_type):t_patch_info; var entry,next:p_patch_node; begin @@ -125,7 +125,8 @@ begin begin next:=TAILQ_NEXT(entry,@entry^.link); // - if (entry^.info.vaddr=vaddr) then + if (entry^.info.vaddr=vaddr) and + ((ptype=pt_none) or (entry^.info.ptype=ptype)) then begin Result:=entry^.info; p_inc_ref(Result.stub); @@ -135,7 +136,7 @@ begin end; end; -function _vm_get_patch_link(page:p_patch_page;vaddr:Pointer;vsize:Integer):t_patch_info; +function _vm_get_patch_link(page:p_patch_page;vaddr:Pointer;vsize:Integer;ptype:t_patch_type):t_patch_info; var entry,next:p_patch_node; begin @@ -147,7 +148,8 @@ begin begin next:=TAILQ_NEXT(entry,@entry^.link); // - if info_cross(@entry^.info,vaddr,vsize) then + if info_cross(@entry^.info,vaddr,vsize) and + ((ptype=pt_none) or (entry^.info.ptype=ptype)) then begin Result:=entry^.info; p_inc_ref(Result.stub); @@ -157,12 +159,11 @@ begin end; end; -function vm_get_patch_link(vaddr:Pointer;vsize:Integer):t_patch_info; +function vm_get_patch_link(vaddr:Pointer;vsize:Integer;ptype:t_patch_type):t_patch_info; var off1,off2:DWORD; data:PPointer; page:p_patch_page; - entry,next:p_patch_node; begin Result:=Default(t_patch_info); @@ -182,10 +183,10 @@ begin if (vsize=0) then begin - Result:=_vm_get_patch_link(page,vaddr); + Result:=_vm_get_patch_link(page,vaddr,ptype); end else begin - Result:=_vm_get_patch_link(page,vaddr,vsize); + Result:=_vm_get_patch_link(page,vaddr,vsize,ptype); off2:=OFF_TO_IDX(vaddr+vsize-1); @@ -201,7 +202,7 @@ begin page:=data^; end; - Result:=_vm_get_patch_link(page,vaddr,vsize); + Result:=_vm_get_patch_link(page,vaddr,vsize,ptype); end; end; @@ -212,7 +213,7 @@ function vm_patch_exist(vaddr:Pointer;vsize:Integer):Boolean; var info:t_patch_info; begin - info:=vm_get_patch_link(vaddr,vsize); + info:=vm_get_patch_link(vaddr,vsize,pt_none); if (info.stub<>nil) then begin