diff --git a/rtl/x86_jit.pas b/rtl/x86_jit.pas index e527f6ef..b8bd8745 100644 --- a/rtl/x86_jit.pas +++ b/rtl/x86_jit.pas @@ -86,10 +86,11 @@ type function get_label():t_jit_i_link; public function is_valid:Boolean; + function offset:Integer; function before:t_jit_i_link; function after :t_jit_i_link; - function prev:t_jit_i_link; - function next:t_jit_i_link; + function prev :t_jit_i_link; + function next :t_jit_i_link; property _node:p_jit_instruction read ALink; property _label:t_jit_i_link read get_label write set_label; end; @@ -103,6 +104,23 @@ operator = (A,B:t_jit_i_link):Boolean; //mm (0F=1, 0F38=2, 0F3A=3) type + t_jit_builder_allocator=object + type + PAllocNode=^TAllocNode; + TAllocNode=packed record + link:PAllocNode; + data:record end; + end; + var + pHead:SLIST_HEAD; + curr_apos:ptruint; //alloc pos in current node + curr_size:ptruint; //useable size of current node + used_size:ptruint; //full usable size + full_size:ptruint; //full alloc size + Function Alloc(Size:ptruint):Pointer; + Procedure Free; + end; + p_jit_builder=^t_jit_builder; t_jit_builder=object Const @@ -236,19 +254,13 @@ type ymm14:TRegValue=(AType:regXmm;ASize:os256;AIndex: 14); ymm15:TRegValue=(AType:regXmm;ASize:os256;AIndex: 15); var - AInstructions:TAILQ_HEAD; - ADataSet :t_jit_data_set; - ADataList :TAILQ_HEAD; + AInstructions :TAILQ_HEAD; + ADataSet :t_jit_data_set; + ADataList :TAILQ_HEAD; AInstructionSize:Integer; ADataCount :Integer; - Allocator:record - pHead:SLIST_HEAD; - curr_apos:ptruint; //alloc pos in current node - curr_size:ptruint; //useable size of current node - used_size:ptruint; //full usable size - full_size:ptruint; //full alloc size - end; + Allocator:t_jit_builder_allocator; // Function Alloc(Size:ptruint):Pointer; Procedure Free; @@ -658,6 +670,26 @@ begin Result:=(ALink<>nil); end; +function t_jit_i_link.offset:Integer; +begin + Result:=0; + if (ALink<>nil) then + begin + case AType of + lnkData: + begin + Result:=p_jit_data(ALink)^.pId*SizeOf(Pointer); + end; + lnkLabelBefore, + lnkLabelAfter: + begin + Result:=ALink^.AInstructionOffset; + end; + else; + end; + end; +end; + function t_jit_i_link.before:t_jit_i_link; begin Result:=Self; @@ -837,14 +869,7 @@ end; // -type - PAllocNode=^TAllocNode; - TAllocNode=packed record - link:PAllocNode; - data:record end; - end; - -Function t_jit_builder.Alloc(Size:ptruint):Pointer; +Function t_jit_builder_allocator.Alloc(Size:ptruint):Pointer; const asize=(1*1024*1024)-SizeOf(ptruint)*3; var @@ -863,49 +888,59 @@ var end; begin - if (Allocator.pHead.slh_first=nil) or (Size>Allocator.curr_size) then + if (pHead.slh_first=nil) or (Size>curr_size) then begin node:=_alloc; - SLIST_INSERT_HEAD(@Allocator.pHead,node,@node^.link); + SLIST_INSERT_HEAD(@pHead,node,@node^.link); //Push_head(_alloc); mem_size:=MemSize(node); - Allocator.curr_apos:=0; - Allocator.curr_size:=mem_size-SizeOf(Pointer); - Inc(Allocator.full_size,mem_size); + curr_apos:=0; + curr_size:=mem_size-SizeOf(Pointer); + Inc(full_size,mem_size); end; - node:=SLIST_FIRST(@Allocator.pHead); + node:=SLIST_FIRST(@pHead); - Result:=@PByte(@node^.data)[Allocator.curr_apos]; + Result:=@PByte(@node^.data)[curr_apos]; - Inc(Allocator.used_size,Size); + Inc(used_size,Size); Size:=Align(Size,SizeOf(ptruint)); - Inc(Allocator.curr_apos,Size); - Dec(Allocator.curr_size,Size); + Inc(curr_apos,Size); + Dec(curr_size,Size); end; -Procedure t_jit_builder.Free; +Procedure t_jit_builder_allocator.Free; var node:PAllocNode; begin //node:=Pop_head; - node:=Allocator.pHead.slh_first; + node:=pHead.slh_first; if (node<>nil) then begin - Allocator.pHead.slh_first:=node^.link; + pHead.slh_first:=node^.link; end; While (node<>nil) do begin FreeMem(node); //node:=Pop_head; - node:=Allocator.pHead.slh_first; + node:=pHead.slh_first; if (node<>nil) then begin - Allocator.pHead.slh_first:=node^.link; + pHead.slh_first:=node^.link; end; end; - Self:=Default(t_jit_builder); + Self:=Default(t_jit_builder_allocator); +end; + +Function t_jit_builder.Alloc(Size:ptruint):Pointer; +begin + Result:=Allocator.Alloc(Size); +end; + +Procedure t_jit_builder.Free; +begin + Allocator.Free; end; // @@ -1165,7 +1200,6 @@ begin _O($C3); end; - Procedure t_jit_builder.ud2; begin _O($0F0B); @@ -1233,12 +1267,44 @@ begin end; end; +function is_8bit_offset(d:Integer):Boolean; inline; +begin + case d of + -128..127:Result:=True; + else + Result:=False; + end; +end; + +function classif_instr(node:p_jit_instruction):Byte; inline; +begin + Result:=0; + case node^.AData[0] of + $EB:Result:=1; //jmp8 + $E9:Result:=2; //jmp32 + $70..$7F:Result:=3; //jcc8 + $0F: + case node^.AData[1] of + $80..$8F:Result:=4; //jcc32 + else; + end; + else; + end; +end; + Procedure t_jit_builder.LinkData; +label + _start; var node:p_jit_instruction; - d:Integer; + d,t:Integer; + is_change:Boolean; begin + + _start: + is_change:=False; + d:=0; node:=TAILQ_FIRST(@AInstructions); @@ -1247,27 +1313,81 @@ begin With node^ do case ALink.AType of lnkData: + if not is_change then begin d:=_get_data_offset(ALink.ALink,AInstructionOffset+ASize); _set_data(node,d); end; - lnkLabelBefore: - With node^ do - begin - d:=_get_label_before_offset(ALink.ALink,AInstructionOffset+ASize); - _set_data(node,d); - end; + lnkLabelBefore, lnkLabelAfter: - With node^ do begin - d:=_get_label_after_offset(ALink.ALink,AInstructionOffset+ASize); - _set_data(node,d); + + case ALink.AType of + lnkLabelBefore:d:=_get_label_before_offset(ALink.ALink,AInstructionOffset+ASize); + lnkLabelAfter :d:=_get_label_after_offset (ALink.ALink,AInstructionOffset+ASize); + end; + + t:=classif_instr(node); + + if (t<>0) then + begin + if (d=0) then + begin + //clear instr + + ALink.AType:=lnkNone; + ASize:=0; + + is_change:=True; + end; + end; + + if (ASize<>0) then + case t of + 2:if is_8bit_offset(d) then //jmp32 + begin + //set to jmp8 + + AData[0]:=$EB; + ASize:=2; + + ALink.ASize:=1; + + is_change:=True; + end; + 4:if is_8bit_offset(d) then //jcc32 + begin + t:=node^.AData[1] and $F; + + AData[0]:=$70 or t; + ASize:=2; + + ALink.ASize:=1; + ALink.AOffset:=1; + + is_change:=True; + end; + else; + end; + + if not is_change then + begin + _set_data(node,d); + end; + end; else; end; // node:=TAILQ_NEXT(node,@node^.link); end; + + if is_change then + begin + RebuldInstructionOffset; + goto _start; + end; + end; Function t_jit_builder.SaveTo(ptr:PByte;size:Integer):Integer; diff --git a/sys/kern/kern_dlsym.pas b/sys/kern/kern_dlsym.pas index 184e7ec6..92e3eaef 100644 --- a/sys/kern/kern_dlsym.pas +++ b/sys/kern/kern_dlsym.pas @@ -328,7 +328,7 @@ begin donelist_init(donelist); err:=0; - if (obj^.mainprog=0) then + if (obj^.rtld_flags.mainprog=0) then begin err:=symlook_list(@req,obj^.dagmembers,donelist); end else diff --git a/sys/kern/kern_dynlib.pas b/sys/kern/kern_dynlib.pas index 928e792b..66f330e2 100644 --- a/sys/kern/kern_dynlib.pas +++ b/sys/kern/kern_dynlib.pas @@ -299,7 +299,7 @@ begin begin dst^:=Default(SceKernelModuleInfo); - if ((flags and 1)=0) and (obj^.is_system<>0) then + if ((flags and 1)=0) and (obj^.rtld_flags.is_system<>0) then begin Result:=EPERM; Break; @@ -434,13 +434,13 @@ begin if ((flags and 1)<>0) then begin - tls_index:=((ord(obj^.is_system<>0) + ord(obj^.mainprog<>0)*2) * $10000) or tls_index; + tls_index:=((ord(obj^.rtld_flags.is_system<>0) + ord(obj^.rtld_flags.mainprog<>0)*2) * $10000) or tls_index; dst^.tls_index:=tls_index; end; if ((flags and 2)<>0) then begin - if (obj^.is_system<>0) {or (is_webkit)} then + if (obj^.rtld_flags.is_system<>0) {or (is_webkit)} then begin FillChar(dst^.name,SCE_DBG_MAX_NAME_LENGTH,0); end; @@ -461,7 +461,7 @@ begin //if not webkit then begin - if (obj^.not_get_proc=0) then + if (obj^.rtld_flags.not_get_proc=0) then begin dst^.init_proc_addr:=obj^.init_proc_addr; dst^.fini_proc_addr:=obj^.fini_proc_addr; @@ -551,7 +551,7 @@ begin obj:=TAILQ_FIRST(@dynlibs_info.obj_list); while (obj<>nil) and (i0) or (obj^.is_system=0) then + if ((flags and 1)<>0) or (obj^.rtld_flags.is_system=0) then begin if (numArray<=i) then begin @@ -621,11 +621,11 @@ begin dst:=nil; case num of - 1:if (obj^.not_get_proc=0) then + 1:if (obj^.rtld_flags.not_get_proc=0) then begin dst:=obj^.init_proc_addr; end; - 2:if (obj^.not_get_proc=0) then + 2:if (obj^.rtld_flags.not_get_proc=0) then begin dst:=obj^.fini_proc_addr; end; diff --git a/sys/kern/kern_exec.pas b/sys/kern/kern_exec.pas index dbf036a4..f3ce2989 100644 --- a/sys/kern/kern_exec.pas +++ b/sys/kern/kern_exec.pas @@ -60,8 +60,7 @@ uses kern_dlsym, kern_authinfo, vfs_syscalls, - kern_jit2, - kern_jit2_ctx; + kern_jit_dynamic; function exec_alloc_args(args:p_image_args):Integer; begin @@ -777,10 +776,9 @@ begin dynlibs_info.tls_static_space:=0; dynlibs_info.tls_count :=1; dynlibs_info.tls_max :=1; - //dynlibs_info.bits :=0; obj:=obj_new(); - obj^.mainprog:=1; + obj^.rtld_flags.mainprog:=1; obj^.relocbase:=imgp^.reloc_base; text_addr:=g_vmspace.vm_taddr; @@ -845,6 +843,7 @@ begin if (Result<>0) then begin obj_free(dynlibs_info.libprogram); + dynlibs_info.libprogram:=nil; end; end; end; @@ -927,68 +926,6 @@ begin end; end; -procedure pick_obj(obj:p_lib_info); -var - ctx:t_jit_context2; - - Lib_Entry:p_Lib_Entry; - h_entry:p_sym_hash_entry; - symp:p_elf64_sym; - addr:Pointer; - ST_TYPE:Integer; -begin - if (obj=nil) then Exit; - - ctx:=Default(t_jit_context2); - - ctx.text_start:=QWORD(obj^.map_base); - ctx.text___end:=ctx.text_start+obj^.text_size; - - ctx.add_forward_point(obj^.entry_addr); - - if (obj^.mainprog=0) then - begin - ctx.add_forward_point(obj^.init_proc_addr); - ctx.add_forward_point(obj^.fini_proc_addr); - end; - - lib_entry:=TAILQ_FIRST(@obj^.lib_table); - while (lib_entry<>nil) do - begin - if (Lib_Entry^.import=0) then //export - begin - h_entry:=TAILQ_FIRST(@Lib_Entry^.syms); - while (h_entry<>nil) do - begin - - symp:=@h_entry^.sym; - - ST_TYPE:=ELF64_ST_TYPE(symp^.st_info); - - Case ST_TYPE of - STT_NOTYPE, - STT_FUN, - STT_SCE: - if (symp^.st_value<>0) and - (symp^.st_shndx<>SHN_UNDEF) and - (symp^.st_valuenil) do begin @@ -1065,10 +1002,6 @@ begin p_proc.libkernel___end_addr:=dynlibs_info.libkernel^.map_base + dynlibs_info.libkernel^.text_size; end; - //pick_obj(dynlibs_info.libprogram); - - pick_obj(dynlibs_info.libkernel); - _dyn_not_exist: obj:=TAILQ_FIRST(@dynlibs_info.obj_list); @@ -1600,6 +1533,11 @@ done2: VFS_UNLOCK_GIANT(vfslocked); exec_free_args(args); + if (error=0) then + begin + kern_jit_dynamic.switch_to_jit(); + end; + Exit(error); end; diff --git a/sys/kern/kern_reloc.pas b/sys/kern/kern_reloc.pas index 7777eade..535a115d 100644 --- a/sys/kern/kern_reloc.pas +++ b/sys/kern/kern_reloc.pas @@ -97,7 +97,7 @@ begin R_X86_64_NONE:; //ignore R_X86_64_COPY: - if (obj^.mainprog=0) then + if (obj^.rtld_flags.mainprog=0) then begin Writeln(StdErr,'reloc_non_plt:','Unexpected R_X86_64_COPY relocation in shared library ',dynlib_basename(obj^.lib_path)); Exit(ENOEXEC); @@ -394,8 +394,8 @@ begin end; if (flags=1) and - (obj^.jmpslots_done=0) and - (defobj^.jmpslots_done=0) then + (obj^.rtld_flags.jmpslots_done=0) and + (defobj^.rtld_flags.jmpslots_done=0) then begin Exit(5); end; @@ -528,7 +528,7 @@ begin R_X86_64_NONE:; //ignore R_X86_64_COPY: - if (obj^.mainprog=0) then + if (obj^.rtld_flags.mainprog=0) then begin Writeln(StdErr,'dynlib_unlink_non_plt_reloc_each:','Unexpected R_X86_64_COPY relocation in dynamic library ',dynlib_basename(obj^.lib_path)); Exit(-1); diff --git a/sys/kern/kern_thr.pas b/sys/kern/kern_thr.pas index e968edb9..cd5fedc6 100644 --- a/sys/kern/kern_thr.pas +++ b/sys/kern/kern_thr.pas @@ -225,6 +225,7 @@ type td_sel :Pointer; td_vp_reserv :Int64; pcb_onfault :Pointer; + td_jit_ctx :Pointer; end; p_thr_param=^thr_param; diff --git a/sys/kern/subr_dynlib.pas b/sys/kern/subr_dynlib.pas index aef50909..1dbc25d9 100644 --- a/sys/kern/subr_dynlib.pas +++ b/sys/kern/subr_dynlib.pas @@ -116,20 +116,20 @@ type loaded:Integer; - //t_rtld_bits rtld_flags; - - mainprog :Byte; - tls_done :Byte; - init_scanned :Byte; - init_done :Byte; - on_fini_list :Byte; - not_get_proc :Byte; - textrel :Byte; - init_plt :Byte; - is_system :Byte; - dag_inited :Byte; - jmpslots_done:Byte; - internal :Byte; + rtld_flags:bitpacked record + mainprog :0..1; + tls_done :0..1; + init_scanned :0..1; + init_done :0..1; + on_fini_list :0..1; + not_get_proc :0..1; + textrel :0..1; + init_plt :0..1; + is_system :0..1; + dag_inited :0..1; + jmpslots_done:0..1; + internal :0..1; + end; dldags :TAILQ_HEAD; //Objlist_Entry dagmembers:TAILQ_HEAD; //Objlist_Entry @@ -327,6 +327,8 @@ procedure unload_object(root:p_lib_info); function relocate_object(root:p_lib_info):Integer; function dynlib_load_relocate():Integer; +procedure pick_obj(obj:p_lib_info); + function preload_prx_modules(path:pchar;flags:DWORD;var err:Integer):p_lib_info; function load_prx(path:pchar;flags:DWORD;var pobj:p_lib_info):Integer; function unload_prx(obj:p_lib_info):Integer; @@ -370,7 +372,9 @@ uses kern_proc, kern_reloc, kern_namedobj, - elf_nid_utils; + elf_nid_utils, + kern_jit2, + kern_jit2_ctx; procedure dynlibs_lock; begin @@ -567,7 +571,7 @@ end; function obj_new_int(mod_name:pchar):p_lib_info; begin Result:=obj_new(); - Result^.internal:=1; + Result^.rtld_flags.internal:=1; Result^.add_mod(mod_name); end; @@ -948,7 +952,7 @@ function allocate_tls_offset(obj:p_lib_info):Boolean; var off:Int64; begin - if (obj^.tls_done<>0) then + if (obj^.rtld_flags.tls_done<>0) then begin Exit(True); end; @@ -956,7 +960,7 @@ begin off:=obj^.tls_size; if (off=0) then begin - obj^.tls_done:=1; + obj^.rtld_flags.tls_done:=1; Exit(True); end; @@ -985,7 +989,7 @@ end; procedure free_tls_offset(obj:p_lib_info); begin - if (obj^.tls_done<>0) and (obj^.tls_offset=dynlibs_info.tls_last_offset) then + if (obj^.rtld_flags.tls_done<>0) and (obj^.tls_offset=dynlibs_info.tls_last_offset) then begin dynlibs_info.tls_last_offset:=obj^.tls_offset - obj^.tls_size; dynlibs_info.tls_last_size :=0; @@ -1269,8 +1273,9 @@ procedure initlist_add_objects(var fini_proc_list:TAILQ_HEAD; tail:p_lib_info; var init_proc_list:TAILQ_HEAD); begin - if (obj^.init_scanned<>0) or (obj^.init_done<>0) then Exit; - obj^.init_scanned:=1; + if (obj^.rtld_flags.init_scanned<>0) or + (obj^.rtld_flags.init_done<>0) then Exit; + obj^.rtld_flags.init_scanned:=1; if (obj<>tail) then begin @@ -1287,11 +1292,12 @@ begin objlist_push_tail(init_proc_list,obj); end; - if (obj^.fini_proc_addr<>nil) and (obj^.on_fini_list=0) then + if (obj^.fini_proc_addr<>nil) and + (obj^.rtld_flags.on_fini_list=0) then begin objlist_push_tail(fini_proc_list,obj); - obj^.on_fini_list:=1; + obj^.rtld_flags.on_fini_list:=1; end; end; @@ -1435,7 +1441,7 @@ begin DT_TEXTREL: begin - obj^.textrel:=1; + obj^.rtld_flags.textrel:=1; end; DT_FLAGS: @@ -1456,7 +1462,7 @@ begin if ((dval and DF_TEXTREL)<>0) then begin - obj^.textrel:=1; + obj^.rtld_flags.textrel:=1; end; end; @@ -2061,7 +2067,7 @@ begin if (budget=2) then begin - new^.is_system:=1; + new^.rtld_flags.is_system:=1; end; error:=acquire_per_file_info_obj(imgp,new); @@ -2122,7 +2128,7 @@ function relocate_text_or_data_segment(obj:p_lib_info;src,dst:Pointer;size:QWORD var map:vm_map_t; begin - if (obj^.textrel=0) or + if (obj^.rtld_flags.textrel=0) or (obj^.map_base > dst) or ((obj^.map_base + obj^.text_size) < (dst + size)) then begin @@ -2182,7 +2188,7 @@ var elm:p_Objlist_Entry; donelist:t_DoneList; begin - if (root^.dag_inited<>0) then Exit; + if (root^.rtld_flags.dag_inited<>0) then Exit; donelist:=Default(t_DoneList); donelist_init(donelist); @@ -2224,14 +2230,14 @@ begin elm:=TAILQ_NEXT(elm,@elm^.link); end; - root^.dag_inited:=1; + root^.rtld_flags.dag_inited:=1; end; procedure ref_dag(root:p_lib_info); var elm:p_Objlist_Entry; begin - Assert(root^.dag_inited<>0,'DAG is not initialized'); + Assert(root^.rtld_flags.dag_inited<>0,'DAG is not initialized'); elm:=TAILQ_FIRST(@root^.dagmembers); while (elm<>nil) do @@ -2246,7 +2252,7 @@ procedure unref_dag(root:p_lib_info); var elm:p_Objlist_Entry; begin - Assert(root^.dag_inited<>0,'DAG is not initialized'); + Assert(root^.rtld_flags.dag_inited<>0,'DAG is not initialized'); elm:=TAILQ_FIRST(@root^.dagmembers); while (elm<>nil) do @@ -2268,7 +2274,7 @@ var begin Result:=0; - if (obj^.init_plt<>0) then Exit; + if (obj^.rtld_flags.init_plt<>0) then Exit; Result:=change_relro_protection(obj,VM_PROT_RW); if (Result<>0) then Exit; @@ -2311,7 +2317,7 @@ begin err:=change_relro_protection(obj,VM_PROT_READ); - obj^.init_plt:=1; + obj^.rtld_flags.init_plt:=1; end; function DecodeSym(obj:p_lib_info; @@ -2547,7 +2553,7 @@ begin goto _error; end; - if (new^.textrel<>0) then + if (new^.rtld_flags.textrel<>0) then begin Writeln(StdErr,'do_load_object:',new^.lib_path,' has impure text'); err:=EINVAL; @@ -2678,7 +2684,7 @@ begin //reg lib dynlibs_add_obj(Result); // - Result^.internal:=1; + Result^.rtld_flags.internal:=1; Result^.loaded:=1; Writeln(' preload_prx_internal:',path); @@ -2692,6 +2698,75 @@ begin end; +procedure pick_obj(obj:p_lib_info); +var + ctx:t_jit_context2; + + Lib_Entry:p_Lib_Entry; + h_entry:p_sym_hash_entry; + symp:p_elf64_sym; + addr:Pointer; + ST_TYPE:Integer; +begin + if (obj=nil) then Exit; + + if (obj^.rtld_flags.internal=1) then + begin + Exit; + end; + + Writeln('pick_obj:',obj^.lib_path); + + ctx:=Default(t_jit_context2); + + ctx.text_start:=QWORD(obj^.map_base); + ctx.text___end:=ctx.text_start+obj^.text_size; + + ctx.add_forward_point(obj^.entry_addr); + + if (obj^.rtld_flags.mainprog=0) then + begin + ctx.add_forward_point(obj^.init_proc_addr); + ctx.add_forward_point(obj^.fini_proc_addr); + end; + + lib_entry:=TAILQ_FIRST(@obj^.lib_table); + while (lib_entry<>nil) do + begin + if (Lib_Entry^.import=0) then //export + begin + h_entry:=TAILQ_FIRST(@Lib_Entry^.syms); + while (h_entry<>nil) do + begin + + symp:=@h_entry^.sym; + + ST_TYPE:=ELF64_ST_TYPE(symp^.st_info); + + Case ST_TYPE of + STT_NOTYPE, + STT_FUN, + STT_SCE: + if (symp^.st_value<>0) and + (symp^.st_shndx<>SHN_UNDEF) and + (symp^.st_value0) then Exit; - Result:=relocate_one_object(root,ord(root^.jmpslots_done=0)); + Result:=relocate_one_object(root,ord(root^.rtld_flags.jmpslots_done=0)); if (Result=0) then begin @@ -2785,7 +2862,7 @@ begin begin if (obj<>root) then begin - Result:=relocate_one_object(obj,ord(root^.jmpslots_done=0)); + Result:=relocate_one_object(obj,ord(root^.rtld_flags.jmpslots_done=0)); end; obj:=TAILQ_NEXT(obj,@obj^.link); end; @@ -2844,12 +2921,12 @@ begin begin if ((flags and $20000)<>0) then //set jmpslots_done? begin - obj^.jmpslots_done:=1; + obj^.rtld_flags.jmpslots_done:=1; end; if ((flags and $40000)<>0) then //set not_get_proc? begin - obj^.not_get_proc:=1; + obj^.rtld_flags.not_get_proc:=1; end; init_dag(obj); diff --git a/sys/test/kern_jit2.pas b/sys/test/kern_jit2.pas index 4bddc86e..b5b489dd 100644 --- a/sys/test/kern_jit2.pas +++ b/sys/test/kern_jit2.pas @@ -25,11 +25,53 @@ implementation uses sysutils, - kern_jit2_ops; + kern_jit2_ops, + kern_jit_dynamic; + +procedure jit_before_start; assembler; nostackframe; +asm + nop +end; + +procedure jit_jmp_dispatch; +begin + Writeln('TODO:jit_jmp_dispatch'); + Assert(False); +end; + +procedure jit_syscall; +begin + Writeln('TODO:jit_syscall'); + Assert(False); +end; + +procedure jit_assert; +begin + Writeln('TODO:jit_assert'); + Assert(False); +end; + +procedure jit_system_error; +begin + Writeln('TODO:jit_system_error'); + Assert(False); +end; + +procedure jit_exit_proc; +begin + Writeln('TODO:jit_exit_proc'); + //Assert(False); +end; + +procedure jit_cpuid; +begin + Writeln('TODO:jit_cpuid'); + Assert(False); +end; procedure op_jmp_dispatcher(var ctx:t_jit_context2); begin - ctx.builder.call_far(nil); //input:rax TODO jmp dispatcher + ctx.builder.call_far(@jit_jmp_dispatch); //input:rax TODO jmp dispatcher end; procedure op_push_rip(var ctx:t_jit_context2;used_r_tmp0:Boolean); @@ -58,10 +100,20 @@ begin if (classif_offset_se64(imm)=os64) then begin - movi64(r_tmp1,imm); - movq([stack],r_tmp1); + if (classif_offset_u64(imm)=os64) then + begin + //64bit imm + movi64(r_tmp1,imm); + movq([stack],r_tmp1); + end else + begin + //32bit zero extend + movi(new_reg_size(r_tmp1,os32),imm); + movq([stack],r_tmp1); + end; end else begin + //32bit sign extend movi(os64,[stack],imm); end; @@ -115,6 +167,20 @@ begin Result:=is_rsp(r[0]) or is_rsp(r[1]); end; +procedure op_set_rax_imm(var ctx:t_jit_context2;imm:Int64); +begin + with ctx.builder do + if (classif_offset_u64(imm)=os64) then + begin + //64bit imm + movi64(r_tmp0,imm); + end else + begin + //32bit zero extend + movi(new_reg_size(r_tmp0,os32),imm); + end; +end; + procedure op_call(var ctx:t_jit_context2); var id:t_jit_i_link; @@ -134,20 +200,31 @@ begin dst:=ctx.ptr_next+ofs; - label_id:=ctx.find_label(dst); - - if (label_id<>nil_link) then + if ctx.is_curr_addr(QWORD(dst)) then begin - op_push_rip(ctx,false); - // - ctx.builder.jmp(label_id); + label_id:=ctx.find_label(dst); + + if (label_id<>nil_link) then + begin + op_push_rip(ctx,false); + // + ctx.builder.jmp(label_id); + end else + begin + op_push_rip(ctx,false); + // + id:=ctx.builder.jmp(nil_link); + ctx.add_forward_point(id,dst); + end; end else begin op_push_rip(ctx,false); // - id:=ctx.builder.jmp(nil_link); - ctx.add_forward_point(id,dst); + op_set_rax_imm(ctx,Int64(dst)); + // + op_jmp_dispatcher(ctx); end; + end else if is_memory(ctx.din) then begin @@ -220,15 +297,23 @@ begin dst:=ctx.ptr_next+ofs; - label_id:=ctx.find_label(dst); - - if (label_id<>nil_link) then + if ctx.is_curr_addr(QWORD(dst)) then begin - ctx.builder.jmp(label_id); + label_id:=ctx.find_label(dst); + + if (label_id<>nil_link) then + begin + ctx.builder.jmp(label_id); + end else + begin + id:=ctx.builder.jmp(nil_link); + ctx.add_forward_point(id,dst); + end; end else begin - id:=ctx.builder.jmp(nil_link); - ctx.add_forward_point(id,dst); + op_set_rax_imm(ctx,Int64(dst)); + // + op_jmp_dispatcher(ctx); end; end else @@ -263,7 +348,7 @@ end; procedure op_jcc(var ctx:t_jit_context2); var - id:t_jit_i_link; + id,id2:t_jit_i_link; ofs:Int64; dst:Pointer; label_id:t_jit_i_link; @@ -276,15 +361,31 @@ begin dst:=ctx.ptr_next+ofs; - label_id:=ctx.find_label(dst); - - if (label_id<>nil_link) then + if ctx.is_curr_addr(QWORD(dst)) then begin - ctx.builder.jcc(ctx.din.OpCode.Suffix,label_id); + label_id:=ctx.find_label(dst); + + if (label_id<>nil_link) then + begin + ctx.builder.jcc(ctx.din.OpCode.Suffix,label_id); + end else + begin + id:=ctx.builder.jcc(ctx.din.OpCode.Suffix,nil_link); + ctx.add_forward_point(id,dst); + end; end else begin - id:=ctx.builder.jcc(ctx.din.OpCode.Suffix,nil_link); - ctx.add_forward_point(id,dst); + id:=ctx.builder.jcc(ctx.din.OpCode.Suffix,nil_link,os8); + + id2:=ctx.builder.jmp(nil_link,os8); + + id._label:=ctx.builder.get_curr_label.after; + // + op_set_rax_imm(ctx,Int64(dst)); + // + op_jmp_dispatcher(ctx); + + id2._label:=ctx.builder.get_curr_label.after; end; end; @@ -437,7 +538,7 @@ end; procedure op_syscall(var ctx:t_jit_context2); begin - ctx.builder.call_far(nil); //TODO syscall dispatcher + ctx.builder.call_far(@jit_syscall); //TODO syscall dispatcher end; procedure op_int(var ctx:t_jit_context2); @@ -453,13 +554,13 @@ begin $41: //assert? begin // - ctx.builder.call_far(nil); //TODO error dispatcher + ctx.builder.call_far(@jit_assert); //TODO error dispatcher end; $44: //system error? begin // - ctx.builder.call_far(nil); //TODO error dispatcher + ctx.builder.call_far(@jit_system_error); //TODO error dispatcher ctx.ptr_next:=nil; //trim end; else @@ -472,26 +573,26 @@ end; procedure op_ud2(var ctx:t_jit_context2); begin //exit proc? - ctx.builder.call_far(nil); //TODO exit dispatcher + ctx.builder.call_far(@jit_exit_proc); //TODO exit dispatcher ctx.ptr_next:=nil; //trim end; procedure op_iretq(var ctx:t_jit_context2); begin //exit proc? - ctx.builder.call_far(nil); //TODO exit dispatcher + ctx.builder.call_far(@jit_exit_proc); //TODO exit dispatcher ctx.ptr_next:=nil; //trim end; procedure op_hlt(var ctx:t_jit_context2); begin //stop thread? - ctx.builder.call_far(nil); //TODO exit dispatcher + ctx.builder.call_far(@jit_exit_proc); //TODO exit dispatcher end; procedure op_cpuid(var ctx:t_jit_context2); begin - ctx.builder.call_far(nil); //TODO CPUID + ctx.builder.call_far(@jit_cpuid); //TODO CPUID end; procedure op_rdtsc(var ctx:t_jit_context2); @@ -505,9 +606,10 @@ begin //align? end; -procedure op_rep_cmps(var ctx:t_jit_context2); const test_desc:t_op_type=(op:$85;index:0); + +procedure op_rep_cmps(var ctx:t_jit_context2); var op:DWORD; size:TOperandSize; @@ -603,8 +705,6 @@ end; /// procedure op_rep_stos(var ctx:t_jit_context2); -const - test_desc:t_op_type=(op:$85;index:0); var i:Integer; size:TOperandSize; @@ -615,7 +715,6 @@ var link___end:t_jit_i_link; link_jmp0:t_jit_i_link; - link_jmp1:t_jit_i_link; begin //rdi,rsi //prefix $67 TODO @@ -635,7 +734,6 @@ begin begin link_jmp0:=nil_link; - link_jmp1:=nil_link; new:=new_reg_size(r_tmp1,size); @@ -658,17 +756,6 @@ begin leaq(rcx,[rcx-1]); leaq(rdi,[rdi+OPERAND_BYTES[size]]); - if (ifPrefixRepE in ctx.din.Flags) then - begin - //if a[i]<>b[i] then exit - link_jmp1:=jcc(OPSc_nz,nil_link,os8); - end else - if (ifPrefixRepNe in ctx.din.Flags) then - begin - //if a[i]=b[i] then exit - link_jmp1:=jcc(OPSc_z,nil_link,os8); - end; - //until jmp(link_start,os8); @@ -680,10 +767,6 @@ begin link___end:=ctx.builder.get_curr_label.before; //exit1 link_jmp0._label:=link___end; - - link___end:=link___end.after; //exit2 - - link_jmp1._label:=link___end; end; end; @@ -752,14 +835,17 @@ const SCODES:array[TSimdOpcode] of Byte=(0,0,1,3,2); MCODES:array[0..3] of RawByteString=('','0F','0F38','0F3A'); label - _next; + _next, + _build; var addr:Pointer; + ptr:Pointer; + links:t_jit_context2.t_forward_links; + entry_link:Pointer; proc:TDbgProcess; adec:TX86AsmDecoder; - ptr,fin:Pointer; ACodeBytes,ACode:RawByteString; cb:t_jit_cb; @@ -767,8 +853,10 @@ var node_new,node_curr:t_jit_i_link; node,node_code1,node_code2:p_jit_instruction; - data:Pointer; - F:THandle; + blob:p_jit_dynamic; + entry_point:t_jit_context2.p_entry_point; + + //F:THandle; begin init_cbs; @@ -783,21 +871,27 @@ begin if not ctx.fetch_forward_point(links,addr) then begin - Assert(false); + Exit; end; + entry_link:=addr; + + Writeln('0x',HexStr(entry_link)); + + ctx.builder.call_far(@jit_before_start); + ptr:=addr; - fin:=Pointer(Int64(-1)); proc:=TDbgProcess.Create(dm64); adec:=TX86AsmDecoder.Create(proc); - while (ptr []) or is_invalid(adec.Instr) then begin - writeln('invalid:0x',HexStr(ctx.ptr_curr)); + //writeln('invalid:0x',HexStr(ctx.ptr_curr)); + ctx.builder.ud2; goto _next; end; + { Writeln('original------------------------':32,' ','0x',HexStr(ptr-adec.Disassembler.CodeIdx)); Writeln(ACodeBytes:32,' ',ACode); Writeln('original------------------------':32,' ','0x',HexStr(ptr)); + } ctx.ptr_next:=ptr; @@ -856,6 +949,10 @@ begin if (cb=nil) then begin + Writeln('original------------------------':32,' ','0x',HexStr(ptr-adec.Disassembler.CodeIdx)); + Writeln(ACodeBytes:32,' ',ACode); + Writeln('original------------------------':32,' ','0x',HexStr(ptr)); + Writeln('Unhandled jit:', ctx.din.OpCode.Prefix,' ', ctx.din.OpCode.Opcode,' ', @@ -875,6 +972,7 @@ begin node_code2:=ctx.builder.get_curr_label._node; + { if (node_code1<>node_code2) and (node_code1<>nil) then begin @@ -891,8 +989,8 @@ begin end; Writeln('recompiled----------------------':32,' ',''); end; + } - //if (node_code1<>node_code2) then begin ctx.add_label(ctx.ptr_curr,node_curr); end; @@ -903,7 +1001,12 @@ begin links.root:=nil; end; - //if (len1<>len2) then + if (entry_link<>nil) then + begin + ctx.add_entry_point(entry_link,node_curr); + entry_link:=nil; + end; + begin node_new:=ctx.find_label(ptr); @@ -911,7 +1014,7 @@ begin begin ctx.builder.jmp(node_new); ctx.ptr_next:=nil; - Writeln('jmp next:0x',HexStr(ptr)); + //Writeln('jmp next:0x',HexStr(ptr)); end; end; @@ -923,39 +1026,63 @@ begin if not ctx.fetch_forward_point(links,addr) then begin - ctx.builder.ud2; - - data:=AllocMem(ctx.builder.GetMemSize); - ctx.builder.SaveTo(data,ctx.builder.GetMemSize); - - F:=FileCreate('recompile.bin'); - FileWrite(F,data^,ctx.builder.GetMemSize); - FileClose(F); - - Assert(false); + goto _build; end; node_new:=ctx.find_label(addr); if (node_new=nil_link) then begin - Writeln('not found:0x',HexStr(addr)); - writeln; + //Writeln('not found:0x',HexStr(addr)); Break; end else begin links.Resolve(node_new); links.root:=nil; + // + ctx.add_entry_point(addr,node_new); end; until false; + ctx.builder.call_far(@jit_before_start); + + entry_link:=addr; + ptr:=addr; end; end; + _build: + //build blob + + ctx.builder.ud2; + + blob:=new_blob(ctx.builder.GetMemSize); + + ctx.builder.SaveTo(blob^.base,ctx.builder.GetMemSize); + + //F:=FileCreate('recompile.bin'); + //FileWrite(F,data^,ctx.builder.GetMemSize); + //FileClose(F); + + //copy entrys + entry_point:=ctx.entry_list; + while (entry_point<>nil) do + begin + addr:=blob^.base+entry_point^.label_id.offset; + // + blob^.add_entry_point(entry_point^.src,addr); + // + entry_point:=entry_point^.next; + end; + + blob^.attach; + adec.Free; proc.Free; + + ctx.builder.Free; end; diff --git a/sys/test/kern_jit2_ctx.pas b/sys/test/kern_jit2_ctx.pas index 86dfbb9c..356e15c0 100644 --- a/sys/test/kern_jit2_ctx.pas +++ b/sys/test/kern_jit2_ctx.pas @@ -68,9 +68,18 @@ type function c(n1,n2:p_label):Integer; static; end; t_label_set=specialize TNodeSplay; + + p_entry_point=^t_entry_point; + t_entry_point=object + next :p_entry_point; + src :Pointer; + label_id:t_jit_i_link; + end; + var forward_set:t_forward_set; label_set :t_label_set; + entry_list :p_entry_point; text_start:QWORD; text___end:QWORD; @@ -86,6 +95,7 @@ type builder:t_jit_builder; + function is_curr_addr(addr:QWORD):Boolean; procedure add_forward_link(node:p_forward_point;label_id:t_jit_i_link); function add_forward_point(label_id:t_jit_i_link;dst:Pointer):p_forward_point; function add_forward_point(dst:Pointer):p_forward_point; @@ -93,6 +103,7 @@ type function fetch_forward_point(var links:t_forward_links;var dst:Pointer):Boolean; function add_label(src:Pointer;label_id:t_jit_i_link):p_label; function find_label(src:Pointer):t_jit_i_link; + procedure add_entry_point(src:Pointer;label_id:t_jit_i_link); end; const @@ -255,6 +266,11 @@ begin Result:=Integer(n1^.src>n2^.src)-Integer(n1^.src=text_start) and (addr +----------+ +---------+ + |code_chunk| -> |code_blob| + entry_point -> | | | | + +----------+ | | + entry_point -> +----------+ | | + |code_chunk| -> | | + entry_point -> | | | | + +----------+ +---------+ +} + +type + p_jit_dynamic=^t_jit_dynamic; + t_jit_dynamic=object + type + p_entry_point=^t_entry_point; + t_entry_point=object + next:p_entry_point; + blob:p_jit_dynamic; + src :Pointer; + dst :Pointer; + procedure inc_ref; + procedure dec_ref; + end; + var + entry_list:p_entry_point; + + base:Pointer; + size:ptruint; + + refs:Integer; + + procedure inc_ref; + procedure dec_ref; + procedure Free; + procedure add_entry_point(src,dst:Pointer); + procedure alloc_base(_size:ptruint); + procedure free_base; + procedure attach; + procedure detach; + end; + + p_jit_thr_ctx=^t_jit_thr_ctx; + t_jit_thr_ctx=object + frame:jit_frame; + cblob:p_jit_dynamic; + end; + +function new_blob(_size:ptruint):p_jit_dynamic; + +var + entry_lock:Pointer=nil; + entry_hamt:TSTUB_HAMT64; + +function fetch_entry(src:Pointer):t_jit_dynamic.p_entry_point; + +procedure switch_to_jit(); + +implementation + +uses + vmparam, + vm_pmap; + +procedure switch_to_jit(); +var + td:p_kthread; + node:t_jit_dynamic.p_entry_point; + jctx:p_jit_thr_ctx; +begin + td:=curkthread; + if (td=nil) then Exit; + + if not is_guest_addr(td^.td_frame.tf_rip) then + begin + Assert(False); + end; + + node:=fetch_entry(Pointer(td^.td_frame.tf_rip)); + + if (node=nil) then + begin + Writeln('tf_rip:0x',HexStr(td^.td_frame.tf_rip,16)); + Assert(False); + end; + + if (td^.td_jit_ctx=nil) then + begin + td^.td_jit_ctx:=AllocMem(SizeOf(t_jit_thr_ctx)); + end; + jctx:=td^.td_jit_ctx; + + if (jctx^.cblob<>nil) then + begin + jctx^.cblob^.dec_ref; + jctx^.cblob:=nil; + end; + + jctx^.cblob:=node^.blob; + + jctx^.frame.tf_rax:=td^.td_frame.tf_rax; + jctx^.frame.tf_rsp:=td^.td_frame.tf_rsp; + jctx^.frame.tf_rbp:=td^.td_frame.tf_rbp; + jctx^.frame.tf_r14:=td^.td_frame.tf_r14; + jctx^.frame.tf_r15:=td^.td_frame.tf_r15; + + td^.td_frame.tf_rsp:=QWORD(td^.td_kstack.stack); + td^.td_frame.tf_rbp:=QWORD(td^.td_kstack.stack); + + td^.td_frame.tf_rip:=QWORD(node^.dst); + td^.td_frame.tf_r15:=QWORD(jctx); +end; + +function fetch_entry(src:Pointer):t_jit_dynamic.p_entry_point; +var + data:PPointer; +begin + Result:=nil; + rw_rlock(entry_lock); + + data:=HAMT_search64(@entry_hamt,QWORD(src)); + if (data<>nil) then + begin + Result:=data^; + end; + + if (Result<>nil) then + begin + Result^.inc_ref; + end; + + rw_runlock(entry_lock); +end; + +// + +function new_blob(_size:ptruint):p_jit_dynamic; +begin + Result:=AllocMem(SizeOf(t_jit_dynamic)); + Result^.alloc_base(_size); +end; + +// + +procedure t_jit_dynamic.t_entry_point.inc_ref; +begin + blob^.inc_ref; +end; + +procedure t_jit_dynamic.t_entry_point.dec_ref; +begin + blob^.dec_ref; +end; + +// + +procedure t_jit_dynamic.inc_ref; +begin + System.InterlockedIncrement(refs); +end; + +procedure t_jit_dynamic.dec_ref; +begin + if (System.InterlockedDecrement(refs)=0) then + begin + Free; + end; +end; + +procedure t_jit_dynamic.Free; +var + node,next:p_entry_point; +begin + node:=entry_list; + while (node<>nil) do + begin + next:=node^.next; + FreeMem(node); + node:=next; + end; + + free_base; + + FreeMem(@Self); +end; + +procedure t_jit_dynamic.add_entry_point(src,dst:Pointer); +var + node:p_entry_point; +begin + if (src=nil) or (dst=nil) then Exit; + node:=AllocMem(Sizeof(t_entry_point)); + node^.next:=entry_list; + node^.blob:=@Self; + node^.src :=src; + node^.dst :=dst; + // + entry_list:=node; +end; + +procedure t_jit_dynamic.alloc_base(_size:ptruint); +begin + base:=md_mmap(nil,_size,MD_PROT_RWX); + size:=_size; +end; + +procedure t_jit_dynamic.free_base; +begin + md_unmap(base,size); + base:=nil; + size:=0; +end; + +procedure t_jit_dynamic.attach; +var + node,next:p_entry_point; +begin + rw_wlock(entry_lock); + + node:=entry_list; + while (node<>nil) do + begin + next:=node^.next; + + node^.inc_ref; + + HAMT_insert64(@entry_hamt,QWORD(node^.src),node); + + node:=next; + end; + + rw_wunlock(entry_lock); +end; + +procedure t_jit_dynamic.detach; +var + node,next:p_entry_point; +begin + rw_wlock(entry_lock); + + node:=entry_list; + while (node<>nil) do + begin + next:=node^.next; + + HAMT_delete64(@entry_hamt,QWORD(node^.src),nil); + + node^.dec_ref; + + node:=next; + end; + + rw_wunlock(entry_lock); +end; + + + +end. + + + + + + diff --git a/sys/test/project1.lpi b/sys/test/project1.lpi index ad98e21e..cb56e373 100644 --- a/sys/test/project1.lpi +++ b/sys/test/project1.lpi @@ -758,6 +758,10 @@ + + + +