diff --git a/sys/jit/kern_jit.pas b/sys/jit/kern_jit.pas index be7ad26a..db60738e 100644 --- a/sys/jit/kern_jit.pas +++ b/sys/jit/kern_jit.pas @@ -1067,7 +1067,7 @@ begin end; end; -procedure op_native2jit(var ctx:t_jit_context2;is_stack_restore:Boolean=True); +procedure op_native2jit(var ctx:t_jit_context2); begin with ctx.builder do begin @@ -1087,11 +1087,8 @@ begin movq([r13+Integer(@p_jit_frame(nil)^.tf_r13)],r14); //load rsp,rbp - if not is_stack_restore then - begin - movq([r13+Integer(@p_jit_frame(nil)^.tf_rsp)],rsp); - movq([r13+Integer(@p_jit_frame(nil)^.tf_rbp)],rbp); - end; + movq([r13+Integer(@p_jit_frame(nil)^.tf_rsp)],rsp); + movq([r13+Integer(@p_jit_frame(nil)^.tf_rbp)],rbp); //load internal stack movq(r14,[r13+Integer(@p_kthread(nil)^.td_kstack.stack)-Integer(@p_kthread(nil)^.td_frame.tf_r13)]); @@ -1370,7 +1367,7 @@ begin // op_jit2native(ctx); ctx.builder.call_far(node^.native); - op_native2jit(ctx,False); + op_native2jit(ctx); // op_pop_rip(ctx,0); //out:r14 op_jmp_dispatcher(ctx); diff --git a/sys/jit/kern_jit_dynamic.pas b/sys/jit/kern_jit_dynamic.pas index 18acff7d..58dc3a9b 100644 --- a/sys/jit/kern_jit_dynamic.pas +++ b/sys/jit/kern_jit_dynamic.pas @@ -13,9 +13,10 @@ uses murmurhash, x86_jit, kern_jit_ctx, + kern_jit_asm, kern_rwlock, kern_thr, - kern_jit_asm; + kern_stub; { entry_point -> +----------+ +---------+ @@ -86,6 +87,8 @@ type chunk_list:p_jcode_chunk; jpltc_list:t_jplt_cache_set; + mchunk:p_stub_chunk; + base:Pointer; size:ptruint; @@ -517,7 +520,7 @@ begin begin if exist_jit_host(from,@td^.td_frame.tf_rip) then begin - test_unresolve_symbol(td,addr,from); + test_unresolve_symbol(td,addr); end; end; @@ -690,9 +693,9 @@ begin //writeln('[0x',HexStr(start,16),':0x',HexStr(__end,16),':',count); end; -procedure _build(var ctx:t_jit_context2; - blob:p_jit_dynamic_blob; - start,__end:QWORD); +procedure build_blob(var ctx:t_jit_context2; + blob:p_jit_dynamic_blob; + start,__end:QWORD); var count:QWORD; @@ -827,7 +830,7 @@ begin if (start<>0) then begin //build prev saved - _build(ctx,blob,start,__end); + build_blob(ctx,blob,start,__end); end; //new start:=chunk^.start; @@ -841,7 +844,7 @@ begin if (start<>0) then begin //build prev saved - _build(ctx,blob,start,__end); + build_blob(ctx,blob,start,__end); end; blob^.attach; @@ -1089,15 +1092,21 @@ end; procedure t_jit_dynamic_blob.alloc_base(_size:ptruint); begin - //small chunk allocator TODO base:=nil; size:=_size; - md_mmap(base,size,MD_PROT_RWX); + + ///md_mmap(base,size,MD_PROT_RWX); + + mchunk:=p_alloc(nil,_size,False); + base:=@mchunk^.body; end; procedure t_jit_dynamic_blob.free_base; begin - md_unmap(base,size); + p_free(mchunk); + + ////md_unmap(base,size); + base:=nil; size:=0; end; diff --git a/sys/kern/kern_dlsym.pas b/sys/kern/kern_dlsym.pas index 843421d6..d1c09c02 100644 --- a/sys/kern/kern_dlsym.pas +++ b/sys/kern/kern_dlsym.pas @@ -32,7 +32,7 @@ type sym_out :p_elf64_sym; end; -function test_unresolve_symbol(td:p_kthread;addr,from:Pointer):Boolean; +function test_unresolve_symbol(td:p_kthread;addr:Pointer):Boolean; function do_dlsym(obj:p_lib_info;symbol,libname:pchar;flags:DWORD):Pointer; function name_dlsym(name,symbol:pchar;addrp:ppointer):Integer; @@ -441,7 +441,6 @@ end; procedure unresolve_symbol(data:p_jmpq64_trampoline); var td:p_kthread; - str:shortstring; begin td:=curkthread; jit_save_to_sys_save(td); @@ -465,7 +464,7 @@ function get_unresolve_ptr(refobj:p_lib_info;where:Pointer;nid:QWORD;libname:PCh var stub:p_stub_chunk; begin - stub:=p_alloc(nil,SizeOf(t_jmpq64_trampoline)); + stub:=p_alloc(nil,SizeOf(t_jmpq64_trampoline),False); p_jmpq64_trampoline(@stub^.body)^:=c_jmpq64_trampoline; p_jmpq64_trampoline(@stub^.body)^.addr :=QWORD(@_unresolve_symbol); @@ -494,7 +493,7 @@ begin end; } -function test_unresolve_symbol(td:p_kthread;addr,from:Pointer):Boolean; +function test_unresolve_symbol(td:p_kthread;addr:Pointer):Boolean; label _exit; var diff --git a/sys/kern/kern_patcher.pas b/sys/kern/kern_patcher.pas index e66541e4..95bbf7a0 100644 --- a/sys/kern/kern_patcher.pas +++ b/sys/kern/kern_patcher.pas @@ -407,7 +407,7 @@ var delta:Int64; begin - stub:=p_alloc(vaddr,SizeOf(t_base_data_trampoline)); + stub:=p_alloc(vaddr,SizeOf(t_base_data_trampoline),True); delta:=Int64(@stub^.body)-(Int64(vaddr)+SizeOf(t_call32_trampoline)); Assert(abs(delta)0) then Exit; + + vm_map_lock(map); + vm_map_set_name_locked(map,start,start+size,'#patch',VM_INHERIT_PATCH); + vm_map_unlock(map); + end else + begin + size:=AlignUp(size+SizeOf(stub_chunk),MD_PAGE_SIZE); + + err:=md_mmap(Pointer(start),size,MD_PROT_RWX); + + if (err<>0) then Exit; end; - err:=vm_mmap2(map, - @start, - size, - VM_PROT_RWX, - VM_PROT_RWX, - MAP_ANON or MAP_PRIVATE, - OBJT_DEFAULT, - nil, - 0); - - if (err<>0) then Exit; - - vm_map_lock(map); - vm_map_set_name_locked(map,start,start+size,'#patch',VM_INHERIT_PATCH); - vm_map_unlock(map); - Result:=Pointer(start); Result^.head :=m_header; @@ -138,13 +153,19 @@ begin Result^.link.tqe_prev:=nil; end; -procedure free_segment(chunk:p_stub_chunk); +procedure free_segment(chunk:p_stub_chunk;guest:Boolean); var map:vm_map_t; begin - map:=p_proc.p_vmspace; + if guest then + begin + map:=p_proc.p_vmspace; - vm_map_remove(map, qword(chunk), qword(chunk) + chunk^.curr_size); + vm_map_remove(map, qword(chunk), qword(chunk) + chunk^.curr_size); + end else + begin + md_unmap(chunk,chunk^.curr_size); + end; end; procedure fix_next_size(chunk:p_stub_chunk); @@ -158,7 +179,7 @@ begin end; end; -procedure split_chunk(chunk:p_stub_chunk;used_size:Integer); +procedure split_chunk(chunk:p_stub_chunk;used_size:Integer;guest:Boolean); var chunk_size:Integer; next:p_stub_chunk; @@ -184,15 +205,15 @@ begin if ((chunk^.flags and m_last__chunk)<>0) then begin chunk^.flags:=chunk^.flags and (not m_last__chunk); - next^.flags:=next^.flags or m_last__chunk; + next ^.flags:=next^.flags or m_last__chunk; end; - TAILQ_INSERT_TAIL(@chunk_free,next,@next^.link); +TAILQ_INSERT_TAIL(@chunk_free[guest],next,@next^.link); fix_next_size(next); end; -procedure merge_chunk(var chunk:p_stub_chunk); +procedure merge_chunk(var chunk:p_stub_chunk;guest:Boolean); var prev,next:p_stub_chunk; begin @@ -206,7 +227,8 @@ begin Assert(prev^.curr_size=chunk^.prev_size,'invalid prev chunk curr_size'); Assert(prev^.refs=0 ,'invalid prev chunk refs'); - TAILQ_REMOVE(@chunk_free,prev,@prev^.link); + TAILQ_REMOVE(@chunk_free[guest],prev,@prev^.link); + prev^.link:=Default(TAILQ_ENTRY); prev^.curr_size:=prev^.curr_size+chunk^.curr_size; @@ -231,7 +253,8 @@ begin Assert(next^.prev_size=chunk^.curr_size,'invalid next chunk prev_size'); Assert(next^.refs=0 ,'invalid next chunk refs'); - TAILQ_REMOVE(@chunk_free,next,@next^.link); + TAILQ_REMOVE(@chunk_free[guest],next,@next^.link); + next^.link:=Default(TAILQ_ENTRY); chunk^.curr_size:=chunk^.curr_size+next^.curr_size; @@ -248,13 +271,14 @@ begin end; end; -function find_free_chunk(vaddr:Pointer;size:Integer):p_stub_chunk; +function find_free_chunk(vaddr:Pointer;size:Integer;guest:Boolean):p_stub_chunk; var entry,next:p_stub_chunk; begin Result:=nil; size:=size+SizeOf(stub_chunk); - entry:=TAILQ_FIRST(@chunk_free); + + entry:=TAILQ_FIRST(@chunk_free[guest]); while (entry<>nil) do begin @@ -264,7 +288,7 @@ begin begin if (vaddr=nil) or is_near_valid(vaddr,@entry^.body) then begin - TAILQ_REMOVE(@chunk_free,entry,@entry^.link); + TAILQ_REMOVE(@chunk_free[guest],entry,@entry^.link); entry^.link:=Default(TAILQ_ENTRY); Exit(entry); end; @@ -274,13 +298,13 @@ begin end; end; -function find_free_chunk_m(vaddr:Pointer;size:Integer;mask:DWORD):p_stub_chunk; +function find_free_chunk_m(vaddr:Pointer;size:Integer;mask:DWORD;guest:Boolean):p_stub_chunk; var entry,next:p_stub_chunk; begin Result:=nil; size:=size+SizeOf(stub_chunk); - entry:=TAILQ_FIRST(@chunk_free); + entry:=TAILQ_FIRST(@chunk_free[guest]); while (entry<>nil) do begin @@ -290,7 +314,7 @@ begin begin if is_mask_valid(vaddr,@entry^.body,mask) then begin - TAILQ_REMOVE(@chunk_free,entry,@entry^.link); + TAILQ_REMOVE(@chunk_free[guest],entry,@entry^.link); entry^.link:=Default(TAILQ_ENTRY); Exit(entry); end; @@ -300,25 +324,33 @@ begin end; end; -function p_alloc(vaddr:Pointer;size:Integer):p_stub_chunk; +function p_alloc(vaddr:Pointer;size:Integer;guest:Boolean):p_stub_chunk; var chunk:p_stub_chunk; begin - rw_wlock(chunk_lock); + rw_wlock(chunk_lock[guest]); - chunk:=find_free_chunk(vaddr,size); + chunk:=find_free_chunk(vaddr,size,guest); if (chunk=nil) then begin - chunk:=alloc_segment(QWORD(vaddr),size); + chunk:=alloc_segment(QWORD(vaddr),size,guest); Assert(chunk<>nil,'p_alloc NOMEM'); if (vaddr<>nil) then if (not is_near_valid(vaddr,@chunk^.body)) then if (QWORD(vaddr)>High(Integer)) then begin - free_segment(chunk); - chunk:=alloc_segment(AlignUp(QWORD(vaddr)-High(Integer),PAGE_SIZE),size); + free_segment(chunk,guest); + + if guest then + begin + chunk:=alloc_segment(AlignUp(QWORD(vaddr)-High(Integer),PAGE_SIZE),size,guest); + end else + begin + chunk:=alloc_segment(AlignUp(QWORD(vaddr)-High(Integer),MD_PAGE_SIZE),size,guest); + end; + Assert(chunk<>nil,'p_alloc NOMEM'); end; @@ -328,29 +360,29 @@ begin end; end; - split_chunk(chunk,size); + split_chunk(chunk,size,guest); chunk^.flags:=chunk^.flags and (not m_free__chunk); {$IFDEF chunk_alloc} - HAMT_insert64(@chunk_alloc,QWORD(chunk),chunk); + HAMT_insert64(@chunk_alloc[guest],QWORD(chunk),chunk); {$ENDIF} - rw_wunlock(chunk_lock); + rw_wunlock(chunk_lock[guest]); Result:=chunk; end; // -function p_alloc_m(vaddr:Pointer;size:Integer;mask:DWORD):p_stub_chunk; +function p_alloc_m(vaddr:Pointer;size:Integer;mask:DWORD;guest:Boolean):p_stub_chunk; var chunk:p_stub_chunk; x:Integer; begin - rw_wlock(chunk_lock); + rw_wlock(chunk_lock[guest]); - chunk:=find_free_chunk_m(vaddr,size,mask); + chunk:=find_free_chunk_m(vaddr,size,mask,guest); if (chunk=nil) then begin @@ -359,33 +391,33 @@ begin if (x<0) and (QWORD(vaddr)