diff --git a/sys/jit/kern_jit.pas b/sys/jit/kern_jit.pas index 0c451aeb..34fae224 100644 --- a/sys/jit/kern_jit.pas +++ b/sys/jit/kern_jit.pas @@ -98,185 +98,34 @@ asm jmp jit_exit_proc end; -procedure _jit_cpuid(tf_rip,rax:qword); -var - td:p_kthread; -begin - td:=curkthread; - jit_save_to_sys_save(td); - td^.td_frame.tf_rip:=tf_rip; - print_error_td('TODO:jit_cpuid:0x'+HexStr(rax,16)); - Assert(False); -end; - -//cpuid(0x0) :eax=0xd ebx=0x68747541 ecx=0x444d4163 edx=0x69746e65 -//cpuid(0x1) :eax=0x710f31 ebx=0x7080800 ecx=0x3ed8220b edx=0x178bfbff -//0x4 -//0x6 -//cpuid(0x7) :eax=0x0 ebx=0x0 ecx=0x0 edx=0x0 -//0xb - -//0x40000000 -//0x40000010 - -//cpuid(0x80000000):eax=0x8000001e ebx=0x68747541 ecx=0x444d4163 edx=0x69746e65 -//cpuid(0x80000001):eax=0x710f31 ebx=0x0 ecx=0x154837ff edx=0x2fd3fbff -//0x80000002 -//0x80000004 -//0x80000005 -//0x80000006 -//cpuid(0x80000008):eax=0x3028 ebx=0x0 ecx=0x3007 edx=0x0 - -//0xc0000000 -//0xc0000001 -procedure jit_cpuid; assembler; nostackframe; -label - _cpuid_0, - _cpuid_1, - _cpuid_7, - _cpuid_80000000, - _cpuid_80000001, - _cpuid_80000008, - _exit; -asm - movq %rax, %r14 - seto %al - lahf - xchg %rax, %r14 - - cmp $0,%eax - je _cpuid_0 - - cmp $1,%eax - je _cpuid_1 - - cmp $7,%eax - je _cpuid_7 - - cmp $0x80000000,%eax - je _cpuid_80000000 - - cmp $0x80000001,%eax - je _cpuid_80000001 - - cmp $0x80000008,%eax - je _cpuid_80000008 - - //unknow id - - xchg %r14, %rax - addb $127, %al - sahf - - mov %r14, %r15 - call jit_save_ctx - mov %r14, %rdi - mov %r15, %rsi - jmp _jit_cpuid - - //not reach - - _cpuid_0: - - //cpu_high - mov $0xD,%eax - - //cpu_vendor - mov $0x68747541,%ebx - mov $0x69746E65,%edx - mov $0x444D4163,%ecx - - jmp _exit - - _cpuid_1: - - //get host - cpuid - - //if ((cpu_id & 0xffffff80) == 0x740f00) then - //if "machdep.bootparams.base_ps4_mode" then sceKernelHasNeoMode - - //if ((cpu_id & 0xffffff80) == 0x740f00) then sceKernelIsAuthenticNeo - - mov p_cpuid ,%eax //cpu_id - - mov $0x178bfbff,%edx //cpu_feature - mov $0x3ed8220b,%ecx //cpu_feature2 - -// 0x07080800 -//CPUID_BRAND_INDEX 0x000000ff -//CPUID_CLFUSH_SIZE 0x0000ff00 -//CPUID_HTT_CORES 0x00ff0000 //sceKernelGetCurrentCpu 0..7 -//CPUID_LOCAL_APIC_ID 0xff000000 - - and $0xFF000000,%ebx //filter CPUID_LOCAL_APIC_ID - - or $0x00080800,%ebx //cpu_procinfo - - jmp _exit - - _cpuid_7: - - mov $0x0,%eax - mov $0x0,%ebx - mov $0x0,%edx - mov $0x0,%ecx - - jmp _exit - - _cpuid_80000000: - - //cpu_exthigh - mov $0x8000001E,%eax - - //cpu_vendor - mov $0x68747541,%ebx - mov $0x69746e65,%edx - mov $0x444d4163,%ecx - - jmp _exit - - _cpuid_80000001: - - mov $0x00710f31,%eax - mov $0x00000000,%ebx - mov $0x2fd3fbff,%edx //amd_feature - mov $0x154837ff,%ecx //amd_feature2 - - jmp _exit - - _cpuid_80000008: - - mov $0x00003028,%eax - mov $0x00000000,%ebx - mov $0x00000000,%edx - mov $0x00003007,%ecx //cpu_procinfo2 - - _exit: - - xchg %r14, %rax - addb $127, %al - sahf - movq %r14, %rax - -end; - -procedure op_jmp_dispatcher(var ctx:t_jit_context2); +procedure op_jmp_dispatcher(var ctx:t_jit_context2;cb:t_jit_cb); begin with ctx.builder do begin leap(r15); call_far(@jit_jmp_plt_cache); //input:r14,r15 out:r14 + + if (cb<>nil) then + begin + cb(ctx); + end; + jmp(r14); end; end; -procedure op_call_dispatcher(var ctx:t_jit_context2); +procedure op_call_dispatcher(var ctx:t_jit_context2;cb:t_jit_cb); begin with ctx.builder do begin leap(r15); call_far(@jit_jmp_plt_cache); //input:r14,r15 out:r14 + + if (cb<>nil) then + begin + cb(ctx); + end; + jmp(r14); end; end; @@ -286,7 +135,7 @@ begin ctx.trim:=True; end; -procedure op_push_rip(var ctx:t_jit_context2); +procedure op_push_rip_part0(var ctx:t_jit_context2); var stack:TRegValue; imm:Int64; @@ -301,7 +150,7 @@ begin op_load_rsp(ctx,stack); leaq(stack,[stack-8]); - op_uplift(ctx,os64); //in/out:r14 + op_uplift(ctx,stack,os64); //in/out:r14 imm:=Int64(ctx.ptr_next); @@ -329,12 +178,33 @@ begin //then we update the register //[op_uplift] op_load_rsp(ctx,stack); //[op_uplift] leaq(stack,[stack-8]); + //op_save_rsp(ctx,stack); + + end; +end; + +procedure op_push_rip_part1(var ctx:t_jit_context2); +var + stack:TRegValue; +begin + //lea rsp,[rsp-8] + //mov [rsp],r14 + + with ctx.builder do + begin + stack:=r_tmp1; + + //For transactionality, + //first we move the memory, + //then we update the register + op_load_rsp(ctx,stack); + leaq(stack,[stack-8]); op_save_rsp(ctx,stack); end; end; -procedure op_pop_rip(var ctx:t_jit_context2;imm:Word); //out:r14 +procedure op_pop_rip_part0(var ctx:t_jit_context2;imm:Word); //out:r14 var stack:TRegValue; begin @@ -347,20 +217,40 @@ begin op_load_rsp(ctx,stack); - op_uplift(ctx,os64); //in/out:r14 + op_uplift(ctx,stack,os64); //in/out:r14 //load to tmp - movq(r_tmp1,[stack]); + movq(r_tmp0,[stack]); //For transactionality, //first we move the memory, //then we update the register //[op_uplift] op_load_rsp(ctx,stack); - leaq(stack,[stack+8+imm]); - op_save_rsp(ctx,stack); + //leaq(stack,[stack+8+imm]); + //op_save_rsp(ctx,stack); //out:r14 - movq(r_tmp0,r_tmp1); + end; +end; + +procedure op_pop_rip_part1(var ctx:t_jit_context2); +var + stack:TRegValue; +begin + //mov r14,[rsp] + //lea rsp,[rsp+8+imm] + + with ctx.builder do + begin + stack:=r_tmp1; + + //For transactionality, + //first we move the memory, + //then we update the register + op_load_rsp(ctx,stack); + leaq(stack,[stack+8+ctx.imm]); + op_save_rsp(ctx,stack); + end; end; @@ -372,9 +262,9 @@ var new1,new2:TRegValue; link:t_jit_i_link; begin - op_jit_interrupt(ctx); + ctx.label_flags:=ctx.label_flags or LF_JMP; - op_push_rip(ctx); + op_push_rip_part0(ctx); if (ctx.din.Operand[1].RegValue[0].AType=regNone) then begin @@ -390,6 +280,8 @@ begin begin //near + op_push_rip_part1(ctx); + link:=ctx.get_link(dst); if (link<>nil_link) then @@ -405,7 +297,7 @@ begin begin op_set_r14_imm(ctx,Int64(dst)); // - op_call_dispatcher(ctx); + op_call_dispatcher(ctx,@op_push_rip_part1); end; end else @@ -413,13 +305,13 @@ begin begin new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]); // - build_lea(ctx,1,new1,[inc8_rsp,code_ref]); + build_lea(ctx,1,new1,[{inc8_rsp,}code_ref]); // - op_uplift(ctx,os64); //in/out:r14 + op_uplift(ctx,new1,os64); //in/out:r14 // ctx.builder.movq(new1,[new1]); // - op_call_dispatcher(ctx); + op_call_dispatcher(ctx,@op_push_rip_part1); end else if is_preserved(ctx.din) then begin @@ -432,7 +324,7 @@ begin ctx.builder.leaq(new1,[new1+8]); end; // - op_call_dispatcher(ctx); + op_call_dispatcher(ctx,@op_push_rip_part1); end else begin new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]); @@ -440,7 +332,7 @@ begin // ctx.builder.movq(new1,new2); // - op_call_dispatcher(ctx); + op_call_dispatcher(ctx,@op_push_rip_part1); end; // @@ -451,14 +343,16 @@ procedure op_ret(var ctx:t_jit_context2); var imm:Int64; begin - op_jit_interrupt(ctx); + ctx.label_flags:=ctx.label_flags or LF_JMP; imm:=0; GetTargetOfs(ctx.din,ctx.code,1,imm); // - op_pop_rip(ctx,imm); //out:r14 + op_pop_rip_part0(ctx,imm); //out:r14 // - op_jmp_dispatcher(ctx); + ctx.imm:=imm; + + op_jmp_dispatcher(ctx,@op_pop_rip_part1); // trim_flow(ctx); end; @@ -471,7 +365,7 @@ var new1,new2:TRegValue; link:t_jit_i_link; begin - op_jit_interrupt(ctx); + ctx.label_flags:=ctx.label_flags or LF_JMP; if (ctx.din.Operand[1].RegValue[0].AType=regNone) then begin @@ -502,7 +396,7 @@ begin begin op_set_r14_imm(ctx,Int64(dst)); // - op_jmp_dispatcher(ctx); + op_jmp_dispatcher(ctx,nil); end; end else @@ -512,11 +406,11 @@ begin // build_lea(ctx,1,new1,[code_ref]); // - op_uplift(ctx,os64); //in/out:r14 + op_uplift(ctx,new1,os64); //in/out:r14 // ctx.builder.movq(new1,[new1]); // - op_jmp_dispatcher(ctx); + op_jmp_dispatcher(ctx,nil); end else if is_preserved(ctx.din) then begin @@ -524,7 +418,7 @@ begin // op_load(ctx,new1,1); // - op_jmp_dispatcher(ctx); + op_jmp_dispatcher(ctx,nil); end else begin new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]); @@ -532,7 +426,7 @@ begin // ctx.builder.movq(new1,new2); // - op_jmp_dispatcher(ctx); + op_jmp_dispatcher(ctx,nil); end; // trim_flow(ctx); @@ -574,7 +468,7 @@ var dst:Pointer; link:t_jit_i_link; begin - op_jit_interrupt(ctx); + ctx.label_flags:=ctx.label_flags or LF_JMP; ofs:=0; GetTargetOfs(ctx.din,ctx.code,1,ofs); @@ -604,7 +498,7 @@ begin //invert cond jump id1:=ctx.builder.jcc(invert_cond(ctx.din.OpCode.Suffix),nil_link,os8); op_set_r14_imm(ctx,Int64(dst)); - op_jmp_dispatcher(ctx); + op_jmp_dispatcher(ctx,nil); id1._label:=ctx.builder.get_curr_label.after; { @@ -626,7 +520,7 @@ var dst:Pointer; link:t_jit_i_link; begin - op_jit_interrupt(ctx); + ctx.label_flags:=ctx.label_flags or LF_JMP; ofs:=0; GetTargetOfs(ctx.din,ctx.code,1,ofs); @@ -656,10 +550,12 @@ begin end; end else begin + //far + id2:=ctx.builder.jmp(nil_link,os8); id1._label:=ctx.builder.get_curr_label.after; op_set_r14_imm(ctx,Int64(dst)); - op_jmp_dispatcher(ctx); + op_jmp_dispatcher(ctx,nil); id2._label:=ctx.builder.get_curr_label.after; end; @@ -672,7 +568,7 @@ var dst:Pointer; link:t_jit_i_link; begin - op_jit_interrupt(ctx); + ctx.label_flags:=ctx.label_flags or LF_JMP; ofs:=0; GetTargetOfs(ctx.din,ctx.code,1,ofs); @@ -684,6 +580,8 @@ begin if ctx.is_text_addr(QWORD(dst)) and (not exist_entry(dst)) then begin + //near + link:=ctx.get_link(dst); id2:=ctx.builder.jmp(nil_link,os8); @@ -700,11 +598,12 @@ begin end; end else begin + //far id2:=ctx.builder.jmp(nil_link,os8); id1._label:=ctx.builder.get_curr_label.after; op_set_r14_imm(ctx,Int64(dst)); - op_jmp_dispatcher(ctx); + op_jmp_dispatcher(ctx,nil); id2._label:=ctx.builder.get_curr_label.after; end; @@ -730,7 +629,7 @@ begin begin build_lea(ctx,1,r_tmp0); - op_uplift(ctx,os64); //in/out:r14 + op_uplift(ctx,r_tmp0,os64); //in/out:r14 new:=new_reg_size(r_tmp1,ctx.din.Operand[1]); @@ -775,10 +674,10 @@ begin if (new.AIndex=r_tmp1.AIndex) then begin - op_uplift(ctx,new.ASize,[not_use_r_tmp1]); //in/out:r14 + op_uplift(ctx,stack,new.ASize,[not_use_r_tmp1]); //in/out:r14 end else begin - op_uplift(ctx,new.ASize); //in/out:r14 + op_uplift(ctx,stack,new.ASize); //in/out:r14 end; movq([stack],new); @@ -812,7 +711,7 @@ begin op_load_rsp(ctx,stack); leaq(stack,[stack-OPERAND_BYTES[mem_size]]); - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,stack,mem_size); //in/out:r14 //get all flags pushfq(mem_size); @@ -845,7 +744,7 @@ begin op_load_rbp(ctx,stack); - op_uplift(ctx,os64); //in/out:r14 + op_uplift(ctx,stack,os64); //in/out:r14 movq(new,[stack]); @@ -881,7 +780,7 @@ begin op_load_rsp(ctx,stack); - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,stack,mem_size); //in/out:r14 movq(new,[stack]); push(new); @@ -910,7 +809,7 @@ begin op_load_rsp(ctx,stack); - op_uplift(ctx,os64); //in/out:r14 + op_uplift(ctx,stack,os64); //in/out:r14 reload_rsp:=False; @@ -922,7 +821,7 @@ begin build_lea(ctx,1,stack,[not_use_r_tmp1]); - op_uplift(ctx,os64,[not_use_r_tmp1]); //in/out:r14 + op_uplift(ctx,stack,os64,[not_use_r_tmp1]); //in/out:r14 movq([stack],new); @@ -956,6 +855,8 @@ end; procedure op_syscall(var ctx:t_jit_context2); begin + ctx.label_flags:=ctx.label_flags or LF_JMP; + ctx.add_forward_point(fpCall,ctx.ptr_curr); ctx.add_forward_point(fpCall,ctx.ptr_next); // @@ -1077,6 +978,7 @@ begin begin link_jmp:=ctx.builder.jmp(nil_link,os8); // + //ctx.builder.int3; ctx.builder.cli; //op_set_r14_imm(ctx,$FACEADD7); op_set_r14_imm(ctx,Int64(ctx.ptr_curr)); @@ -1455,8 +1357,9 @@ begin ctx.builder.call_far(node^.native); op_native2jit(ctx,true); // - op_pop_rip(ctx,0); //out:r14 - op_jmp_dispatcher(ctx); + op_pop_rip_part0(ctx,0); //out:r14 + ctx.imm:=0; + op_jmp_dispatcher(ctx,@op_pop_rip_part1); // if (node^.dst<>nil) then begin @@ -1468,7 +1371,8 @@ begin ctx.add_label(ctx.ptr_curr, ctx.ptr_next, link_curr, - link_next); + link_next, + LF_JMP); // ctx.add_entry_point(ctx.ptr_curr,link_curr); // @@ -1571,6 +1475,8 @@ begin goto _invalid; end; + ctx.label_flags:=0; + ctx.ptr_curr:=ptr; //guest->host ptr @@ -1721,6 +1627,22 @@ begin end; } + { + The main idea of interrupting JIT code: + + If the LF_JMP flag is set for the label, + then it is enough to set the value to %gs:teb.jit_trp. + Otherwise, for each thread, generate + an individual lacuna with JIT code, + containing: + instructions to the end of the current guest instruction (start...rip...t_jinstr_len.recompil) + then jmp_dispatcher to return to the code. + If necessary, you can generate a page lock + recovery after the current command: + current instruction + then jmp_dispatcher + } + //op_jit_interrupt(ctx); cb(ctx); @@ -1830,7 +1752,7 @@ begin begin op_set_r14_imm(ctx,Int64(ctx.ptr_next)); // - op_jmp_dispatcher(ctx); + op_jmp_dispatcher(ctx,nil); // ctx.trim:=True; end; @@ -1843,7 +1765,10 @@ begin ctx.add_label(ctx.ptr_curr, ctx.ptr_next, link_curr, - link_next); + link_next, + ctx.label_flags); + + ctx.label_flags:=0; end; if ctx.trim then diff --git a/sys/jit/kern_jit_asm.pas b/sys/jit/kern_jit_asm.pas index eb80a500..420cdea0 100644 --- a/sys/jit/kern_jit_asm.pas +++ b/sys/jit/kern_jit_asm.pas @@ -66,6 +66,8 @@ procedure jit_load_ctx; procedure jit_save_to_sys_save(td:p_kthread); procedure sys_save_to_jit_save(td:p_kthread); +procedure jit_cpuid; assembler; + procedure strict_ps4_rdtsc_jit; assembler; procedure jit_interrupt_nop; assembler; @@ -78,6 +80,7 @@ uses trap, md_context, signal, + sys_bootparam, subr_backtrace; // @@ -449,7 +452,10 @@ asm movq %rsp,%rbp leaq 8(%rbp),%rbp - ret + //interrupt + jmp %gs:teb.jit_trp + + //ret _exit: @@ -487,6 +493,9 @@ asm //epilog movq %rbp,%rsp pop %rbp + + //interrupt + jmp %gs:teb.jit_trp end; procedure stack_set_user; assembler; nostackframe; @@ -568,6 +577,171 @@ asm jmp jit_jmp_dispatch end; +procedure _jit_cpuid(tf_rip,rax:qword); +var + td:p_kthread; +begin + td:=curkthread; + jit_save_to_sys_save(td); + td^.td_frame.tf_rip:=tf_rip; + print_error_td('TODO:jit_cpuid:0x'+HexStr(rax,16)); + Assert(False); +end; + +//cpuid(0x0) :eax=0xd ebx=0x68747541 ecx=0x444d4163 edx=0x69746e65 +//cpuid(0x1) :eax=0x710f31 ebx=0x7080800 ecx=0x3ed8220b edx=0x178bfbff +//0x4 +//0x6 +//cpuid(0x7) :eax=0x0 ebx=0x0 ecx=0x0 edx=0x0 +//0xb + +//0x40000000 +//0x40000010 + +//cpuid(0x80000000):eax=0x8000001e ebx=0x68747541 ecx=0x444d4163 edx=0x69746e65 +//cpuid(0x80000001):eax=0x710f31 ebx=0x0 ecx=0x154837ff edx=0x2fd3fbff +//0x80000002 +//0x80000004 +//0x80000005 +//0x80000006 +//cpuid(0x80000008):eax=0x3028 ebx=0x0 ecx=0x3007 edx=0x0 + +//0xc0000000 +//0xc0000001 +procedure jit_cpuid; assembler; nostackframe; +label + _cpuid_0, + _cpuid_1, + _cpuid_7, + _cpuid_80000000, + _cpuid_80000001, + _cpuid_80000008, + _exit; +asm + movq %rax, %r14 + seto %al + lahf + xchg %rax, %r14 + + cmp $0,%eax + je _cpuid_0 + + cmp $1,%eax + je _cpuid_1 + + cmp $7,%eax + je _cpuid_7 + + cmp $0x80000000,%eax + je _cpuid_80000000 + + cmp $0x80000001,%eax + je _cpuid_80000001 + + cmp $0x80000008,%eax + je _cpuid_80000008 + + //unknow id + + xchg %r14, %rax + addb $127, %al + sahf + + mov %r14, %r15 + call jit_save_ctx + mov %r14, %rdi + mov %r15, %rsi + jmp _jit_cpuid + + //not reach + + _cpuid_0: + + //cpu_high + mov $0xD,%eax + + //cpu_vendor + mov $0x68747541,%ebx + mov $0x69746E65,%edx + mov $0x444D4163,%ecx + + jmp _exit + + _cpuid_1: + + //get host + cpuid + + //if ((cpu_id & 0xffffff80) == 0x740f00) then + //if "machdep.bootparams.base_ps4_mode" then sceKernelHasNeoMode + + //if ((cpu_id & 0xffffff80) == 0x740f00) then sceKernelIsAuthenticNeo + + mov p_cpuid ,%eax //cpu_id + + mov $0x178bfbff,%edx //cpu_feature + mov $0x3ed8220b,%ecx //cpu_feature2 + +// 0x07080800 +//CPUID_BRAND_INDEX 0x000000ff +//CPUID_CLFUSH_SIZE 0x0000ff00 +//CPUID_HTT_CORES 0x00ff0000 //sceKernelGetCurrentCpu 0..7 +//CPUID_LOCAL_APIC_ID 0xff000000 + + and $0xFF000000,%ebx //filter CPUID_LOCAL_APIC_ID + + or $0x00080800,%ebx //cpu_procinfo + + jmp _exit + + _cpuid_7: + + mov $0x0,%eax + mov $0x0,%ebx + mov $0x0,%edx + mov $0x0,%ecx + + jmp _exit + + _cpuid_80000000: + + //cpu_exthigh + mov $0x8000001E,%eax + + //cpu_vendor + mov $0x68747541,%ebx + mov $0x69746e65,%edx + mov $0x444d4163,%ecx + + jmp _exit + + _cpuid_80000001: + + mov $0x00710f31,%eax + mov $0x00000000,%ebx + mov $0x2fd3fbff,%edx //amd_feature + mov $0x154837ff,%ecx //amd_feature2 + + jmp _exit + + _cpuid_80000008: + + mov $0x00003028,%eax + mov $0x00000000,%ebx + mov $0x00000000,%edx + mov $0x00003007,%ecx //cpu_procinfo2 + + _exit: + + xchg %r14, %rax + addb $127, %al + sahf + movq %r14, %rax + + //interrupt + jmp %gs:teb.jit_trp +end; + procedure strict_ps4_rdtsc_jit; assembler; nostackframe; asm seto %al diff --git a/sys/jit/kern_jit_ctx.pas b/sys/jit/kern_jit_ctx.pas index 2ceb332c..25610044 100644 --- a/sys/jit/kern_jit_ctx.pas +++ b/sys/jit/kern_jit_ctx.pas @@ -10,6 +10,9 @@ uses x86_fpdbgdisas, x86_jit; +const + LF_JMP=1; + type t_point_type=(fpCall,fpData,fpInvalid); @@ -42,12 +45,13 @@ type p_label=^t_label; t_label=object - pLeft :p_label; - pRight :p_label; - curr :Pointer; - next :Pointer; + pLeft :p_label; + pRight :p_label; + curr :Pointer; + next :Pointer; link_curr:t_jit_i_link; link_next:t_jit_i_link; + flags :Integer; function c(n1,n2:p_label):Integer; static; end; t_label_set=specialize TNodeSplay; @@ -86,6 +90,8 @@ type ptr_curr:Pointer; ptr_next:Pointer; + label_flags:Integer; + imm:Word; trim:Boolean; dis:TX86Disassembler; @@ -105,7 +111,7 @@ type procedure end_chunk(__end:Pointer); function max_forward_point():Pointer; function fetch_forward_point(var links:t_forward_links;var dst:Pointer):Boolean; - function add_label(curr,next:Pointer;link_curr,link_next:t_jit_i_link):p_label; + function add_label(curr,next:Pointer;link_curr,link_next:t_jit_i_link;flags:Integer):p_label; function get_label(src:Pointer):p_label; function get_link (src:Pointer):t_jit_i_link; procedure add_entry_point(src:Pointer;label_id:t_jit_i_link); @@ -246,7 +252,7 @@ procedure op_save_rbp(var ctx:t_jit_context2;reg:TRegValue); procedure op_load(var ctx:t_jit_context2;reg:TRegValue;opr:Byte); procedure op_save(var ctx:t_jit_context2;opr:Byte;reg:TRegValue); -procedure op_uplift(var ctx:t_jit_context2;mem_size:TOperandSize;hint:t_lea_hint=[]); +procedure op_uplift(var ctx:t_jit_context2;const dst:TRegValue;mem_size:TOperandSize;hint:t_lea_hint=[]); procedure add_orig(var ctx:t_jit_context2); procedure op_emit1(var ctx:t_jit_context2;const desc:t_op_type;hint:t_op_hint); @@ -457,7 +463,7 @@ begin end; end; -function t_jit_context2.add_label(curr,next:Pointer;link_curr,link_next:t_jit_i_link):p_label; +function t_jit_context2.add_label(curr,next:Pointer;link_curr,link_next:t_jit_i_link;flags:Integer):p_label; var node:t_label; begin @@ -471,6 +477,7 @@ begin Result^.next :=next; Result^.link_curr:=link_curr; Result^.link_next:=link_next; + Result^.flags :=flags; // label_set.Insert(Result); end; @@ -1470,7 +1477,9 @@ begin with ctx.builder do begin //[65 FF 14 25] [00 07 00 00] call gs:[$00000700] - call([GS+Integer(teb_jit_trp)]); + //call([GS+Integer(teb_jit_trp)]); + + //ctx.label_flags:=ctx.label_flags or LF_JMP_INTERRUPT; end; end; @@ -1600,7 +1609,7 @@ end; // $10000000000 = 1 shl 40 // 64-40 = 24 -procedure op_uplift(var ctx:t_jit_context2;mem_size:TOperandSize;hint:t_lea_hint=[]); +procedure op_uplift(var ctx:t_jit_context2;const dst:TRegValue;mem_size:TOperandSize;hint:t_lea_hint=[]); const shlx_desc:t_op_type=( op:$F7;simdop:1;mm:2;vw_mode:vwR64; @@ -1611,25 +1620,37 @@ const var rbits:TRegValue; begin - if jit_memory_guard then + if not jit_memory_guard then Exit; + + case dst.AIndex of + 14:hint:=hint+[not_use_r_tmp0]; //r14 + 15:hint:=hint+[not_use_r_tmp1]; //r15 + else; + end; + with ctx.builder do begin - if (not_use_r_tmp1 in hint) then + if (not_use_r_tmp0 in hint) and + (not_use_r_tmp1 in hint) then begin rbits:=rbp; end else + if (not_use_r_tmp0 in hint) then begin rbits:=r_tmp1; + end else + begin + rbits:=r_tmp0; end; //zero bits movi(new_reg_size(rbits,os8),24); //mov $24,%bpl //clear hi - _VVV(shlx_desc,r14,rbits,r14,os64); //1 3 2 | shlx %rbp,%r14,%r14 - _VVV(shrx_desc,r14,rbits,r14,os64); //1 3 2 | shrx %rbp,%r14,%r14 + _VVV(shlx_desc,dst,rbits,dst,os64); //1 3 2 | shlx %rbp,%r14,%r14 + _VVV(shrx_desc,dst,rbits,dst,os64); //1 3 2 | shrx %rbp,%r14,%r14 - if (not_use_r_tmp1 in hint) then + if (rbits.AIndex=rbp.AIndex) then begin //restore rbp movq(rbp,rsp); @@ -1717,7 +1738,7 @@ begin (mem_size=os4096) or (his_rw in hint) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_in; end else @@ -1740,7 +1761,7 @@ begin (his_rw in hint) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_out; end else @@ -2366,7 +2387,7 @@ begin if (mem_size=os8) or (his_rw in desc.hint) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_in; end else @@ -2380,7 +2401,7 @@ begin if (mem_size=os8) or (his_rw in desc.hint) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_out; end else @@ -2399,7 +2420,7 @@ begin if (mem_size=os8) or (his_rw in desc.hint) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_in; end else @@ -2715,7 +2736,7 @@ begin begin if true then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_out; end else @@ -2896,7 +2917,7 @@ begin begin if true then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_out; end else @@ -3093,7 +3114,7 @@ begin if (mem_size=os8) or (his_rw in hint) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_in; end else @@ -3115,7 +3136,7 @@ begin (his_rw in hint) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_out; end else @@ -3276,7 +3297,7 @@ begin begin if (his_align in desc.hint) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_out; end else @@ -3294,7 +3315,7 @@ begin begin if (his_align in desc.hint) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_in; end else @@ -3413,7 +3434,7 @@ begin if false then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_in; end else @@ -3466,7 +3487,7 @@ begin if false then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_out; end else @@ -3547,7 +3568,7 @@ begin if (mem_size=os8) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_out; end else @@ -3568,7 +3589,7 @@ begin if (mem_size=os8) then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_in; end else @@ -3680,7 +3701,7 @@ begin begin if false then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_in; end else @@ -3774,7 +3795,7 @@ begin if false then begin - op_uplift(ctx,mem_size); //in/out:r14 + op_uplift(ctx,r_tmp0,mem_size); //in/out:r14 mem_in; end else diff --git a/sys/jit/kern_jit_dynamic.pas b/sys/jit/kern_jit_dynamic.pas index 761bbf29..dfa98bb0 100644 --- a/sys/jit/kern_jit_dynamic.pas +++ b/sys/jit/kern_jit_dynamic.pas @@ -44,7 +44,10 @@ type p_jinstr_len=^t_jinstr_len; t_jinstr_len=packed record - original:Byte; + original:0..31; //5 + LF_JMP :0..1; + bit6 :0..1; + bit7 :0..1; recompil:Byte; end; @@ -693,7 +696,7 @@ begin original:=QWORD(next)-QWORD(curr); recompil:=link_next.offset-link_curr.offset; - if (original>255) or (recompil>255) then + if (original>16) or (recompil>255) then begin Writeln('0x',HexStr(curr)); Writeln(original,':',recompil); @@ -703,6 +706,8 @@ begin table[i].original:=Byte(original); table[i].recompil:=Byte(recompil); + table[i].LF_JMP :=ord((clabel^.flags and LF_JMP)<>0); + { writeln('|0x',HexStr(curr),'..',HexStr(next), ':0x',HexStr(link_curr.offset,8),'..',HexStr(link_next.offset,8), diff --git a/sys/jit/kern_jit_ops.pas b/sys/jit/kern_jit_ops.pas index 4f357444..89078e0b 100644 --- a/sys/jit/kern_jit_ops.pas +++ b/sys/jit/kern_jit_ops.pas @@ -59,12 +59,11 @@ begin //repeat link_jmp0:=jcxz(nil_link,ctx.dis.AddressSize); - movq(r_tmp0,rsi); - op_uplift(ctx,size); //in/out:r14 - movq(r_tmp1,r_tmp0); - movq(r_tmp0,rdi); - op_uplift(ctx,size,[not_use_r_tmp1]); //in/out:r14 + op_uplift(ctx,r_tmp0,size); //in/out:r14 + + movq(r_tmp1,rsi); + op_uplift(ctx,r_tmp1,size,[not_use_r_tmp0]); //in/out:r15 xchgq(rdi,r_tmp0); xchgq(rsi,r_tmp1); @@ -179,7 +178,7 @@ begin link_jmp0:=jcxz(nil_link,ctx.dis.AddressSize); movq(r_tmp0,rdi); - op_uplift(ctx,size); //in/out:r14 + op_uplift(ctx,r_tmp0,size); //in/out:r14 movq([r_tmp0],new); // mov [r14],rax @@ -270,12 +269,11 @@ begin //repeat link_jmp0:=jcxz(nil_link,ctx.dis.AddressSize); - movq(r_tmp0,rsi); - op_uplift(ctx,size); //in/out:r14 - movq(r_tmp1,r_tmp0); - movq(r_tmp0,rdi); - op_uplift(ctx,size,[not_use_r_tmp1]); //in/out:r14 + op_uplift(ctx,r_tmp0,size); //in/out:r14 + + movq(r_tmp1,rsi); + op_uplift(ctx,r_tmp1,size,[not_use_r_tmp0]); //in/out:r15 //[RSI] -> [RDI]. @@ -365,12 +363,11 @@ begin with ctx.builder do begin - movq(r_tmp0,rsi); - op_uplift(ctx,size); //in/out:r14 - movq(r_tmp1,r_tmp0); - movq(r_tmp0,rdi); - op_uplift(ctx,size,[not_use_r_tmp1]); //in/out:r14 + op_uplift(ctx,r_tmp0,size); //in/out:r14 + + movq(r_tmp1,rsi); + op_uplift(ctx,r_tmp1,size,[not_use_r_tmp0]); //in/out:r15 xchgq(rdi,r_tmp0); xchgq(rsi,r_tmp1); @@ -447,12 +444,11 @@ begin with ctx.builder do begin - movq(r_tmp0,rsi); - op_uplift(ctx,size); //in/out:r14 - movq(r_tmp1,r_tmp0); - movq(r_tmp0,rdi); - op_uplift(ctx,size,[not_use_r_tmp1]); //in/out:r14 + op_uplift(ctx,r_tmp0,size); //in/out:r14 + + movq(r_tmp1,rsi); + op_uplift(ctx,r_tmp1,size,[not_use_r_tmp0]); //in/out:r15 //[RSI] -> [RDI]. @@ -535,7 +531,7 @@ begin new:=new_reg_size(rax,size); movq(r_tmp0,rdi); - op_uplift(ctx,size); //in/out:r14 + op_uplift(ctx,r_tmp0,size); //in/out:r14 movq([r_tmp0],new); @@ -608,7 +604,7 @@ begin new:=new_reg_size(rax,size); movq(r_tmp0,rdi); - op_uplift(ctx,size); //in/out:r14 + op_uplift(ctx,r_tmp0,size); //in/out:r14 movq(new,[r_tmp0]); @@ -677,7 +673,7 @@ begin begin movq(r_tmp0,rdi); - op_uplift(ctx,size); //in/out:r14 + op_uplift(ctx,r_tmp0,size); //in/out:r14 xchgq(rdi,r_tmp0); diff --git a/sys/md/vm_pmap.pas b/sys/md/vm_pmap.pas index 3cb73bc6..77549d6a 100644 --- a/sys/md/vm_pmap.pas +++ b/sys/md/vm_pmap.pas @@ -1124,9 +1124,7 @@ begin lock:=pmap_wlock(pmap,start,__end); - pmap_unmark_rwx(start,__end); - pmap_untrack (start,__end,PAGE_TRACK_RWX); - //untrack? + pmap_unmark(start,__end); vm_track_map_remove_memory(@pmap^.tr_map,start,__end); diff --git a/sys/md/vm_pmap_prot.pas b/sys/md/vm_pmap_prot.pas index 9c20dad5..79fa2017 100644 --- a/sys/md/vm_pmap_prot.pas +++ b/sys/md/vm_pmap_prot.pas @@ -38,6 +38,7 @@ var PAGE_PROT:PBYTE=nil; procedure pmap_mark_rwx (start,__end:vm_offset_t;prots:Byte); +procedure pmap_unmark (start,__end:vm_offset_t); procedure pmap_unmark_rwx(start,__end:vm_offset_t); procedure pmap_track (start,__end:vm_offset_t;prots:Byte); procedure pmap_untrack (start,__end:vm_offset_t;prots:Byte); @@ -88,6 +89,20 @@ begin WriteBarrier; end; +procedure pmap_unmark(start,__end:vm_offset_t); +begin + start:=OFF_TO_IDX(start); + __end:=OFF_TO_IDX(__end); + start:=MAX_IDX(start); + __end:=MAX_IDX(__end); + while (start<__end) do + begin + //PAGE_PROT[start]:=0; + Inc(start); + end; + WriteBarrier; +end; + procedure pmap_unmark_rwx(start,__end:vm_offset_t); begin start:=OFF_TO_IDX(start); @@ -97,7 +112,6 @@ begin while (start<__end) do begin atomic_clear_byte(@PAGE_PROT[start],PAGE_PROT_RWX); - //PAGE_PROT[start]:=0; Inc(start); end; WriteBarrier; diff --git a/sys/vm/vm_tracking_map.pas b/sys/vm/vm_tracking_map.pas index 7b236635..47f1c90a 100644 --- a/sys/vm/vm_tracking_map.pas +++ b/sys/vm/vm_tracking_map.pas @@ -8,6 +8,7 @@ interface uses vm, mqueue, + vm_pmap_prot, kern_mtx; type @@ -36,6 +37,8 @@ type // on_destroy:t_on_destroy; on_trigger:t_on_trigger; + // + prot:Byte; end; pp_vm_track_map_entry=^p_vm_track_map_entry; @@ -48,6 +51,11 @@ type start :vm_offset_t; // start address __end :vm_offset_t; // end address instances:TAILQ_HEAD; // p_vm_track_object_instance + // + track_r :DWORD; + track_w :DWORD; + // + prot:Byte; end; p_vm_track_object_instance=^t_vm_track_object_instance; @@ -161,8 +169,6 @@ end; // -// - procedure _vm_track_entry_add_obj(entry:p_vm_track_map_entry;obj:p_vm_track_object); var node:p_vm_track_object_instance; @@ -172,6 +178,18 @@ begin node^.entry:=entry; node^.obj :=obj; + if (obj^.prot and PAGE_TRACK_R)<>0 then + begin + Inc(entry^.track_r); + end; + + if (obj^.prot and PAGE_TRACK_W)<>0 then + begin + Inc(entry^.track_w); + end; + + entry^.prot:=ord(entry^.track_r<>0)*PAGE_TRACK_R or ord(entry^.track_w<>0)*PAGE_TRACK_W; + TAILQ_INSERT_TAIL(@entry^.instances,node,@node^.entry_link); TAILQ_INSERT_TAIL(@obj^.instances,node,@node^.obj_link); @@ -200,12 +218,28 @@ begin end; function _vm_track_entry_del_node(entry:p_vm_track_map_entry;node:p_vm_track_object_instance):Boolean; +var + obj:p_vm_track_object; begin + obj:=node^.obj; + TAILQ_REMOVE(@entry^.instances,node,@node^.entry_link); - TAILQ_REMOVE(@node^.obj^.instances,node,@node^.obj_link); + TAILQ_REMOVE(@obj^.instances,node,@node^.obj_link); - vm_track_object_deallocate(node^.obj); + if (obj^.prot and PAGE_TRACK_R)<>0 then + begin + Dec(entry^.track_r); + end; + + if (obj^.prot and PAGE_TRACK_W)<>0 then + begin + Dec(entry^.track_w); + end; + + entry^.prot:=ord(entry^.track_r<>0)*PAGE_TRACK_R or ord(entry^.track_w<>0)*PAGE_TRACK_W; + + vm_track_object_deallocate(obj); FreeMem(node);