From 544d1f2f388486d00f7fe01118f9c3c9c8092173 Mon Sep 17 00:00:00 2001 From: Pavel <68122101+red-prig@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:17:25 +0300 Subject: [PATCH] + --- rtl/x86_jit.pas | 68 ++++++++++++++++++++++++++++++------ sys/jit/kern_jit.pas | 63 +++++++++++++++++---------------- sys/jit/kern_jit_dynamic.pas | 9 +++++ sys/jit/kern_jit_ops_sse.pas | 39 +++++++++++---------- sys/kern/kern_thr.pas | 3 +- sys/kern/kern_thread.pas | 5 +++ sys/kern/machdep.pas | 5 ++- sys/kern/trap.pas | 17 ++++++++- sys/md/md_thread.pas | 12 ++++++- 9 files changed, 155 insertions(+), 66 deletions(-) diff --git a/rtl/x86_jit.pas b/rtl/x86_jit.pas index 949b60a2..b2d1f2bc 100644 --- a/rtl/x86_jit.pas +++ b/rtl/x86_jit.pas @@ -379,20 +379,26 @@ type procedure addq (reg:TRegValue ;mem:t_jit_leas); procedure addq (reg0:TRegValue ;reg1:TRegValue); procedure addi (reg:TRegValue ;imm:Int64); - procedure addi8 (reg:TRegValue ;imm:Byte); - procedure addi8 (mem:t_jit_leas ;imm:Byte); + procedure addi8se (reg:TRegValue ;imm:ShortInt); + procedure addi8se (mem:t_jit_leas ;imm:ShortInt); procedure subq (mem:t_jit_leas ;reg:TRegValue); procedure subq (reg:TRegValue ;mem:t_jit_leas); procedure subq (reg0:TRegValue ;reg1:TRegValue); procedure subi (reg:TRegValue ;imm:Int64); - procedure subi8 (reg:TRegValue ;imm:Byte); - procedure subi8 (mem:t_jit_leas ;imm:Byte); + procedure subi8se (reg:TRegValue ;imm:ShortInt); + procedure subi8se (mem:t_jit_leas ;imm:ShortInt); procedure shli8 (reg:TRegValue ;imm:Byte); procedure shri8 (reg:TRegValue ;imm:Byte); procedure andi (reg:TRegValue ;imm:Int64); - procedure andi8 (reg:TRegValue ;imm:Byte); + procedure andi (mem:t_jit_leas;imm:Int64); + procedure andi8se (reg:TRegValue ;imm:ShortInt); + procedure andi8se (mem:t_jit_leas ;imm:ShortInt); procedure andq (reg0:TRegValue ;reg1:TRegValue); procedure orq (reg0:TRegValue ;reg1:TRegValue); + procedure ori (reg:TRegValue ;imm:Int64); + procedure ori (mem:t_jit_leas ;imm:Int64); + procedure ori8se (reg:TRegValue ;imm:ShortInt); + procedure ori8se (mem:t_jit_leas ;imm:ShortInt); procedure xorq (reg0:TRegValue ;reg1:TRegValue); procedure notq (reg:TRegValue); procedure cmpq (mem:t_jit_leas ;reg:TRegValue); @@ -3695,14 +3701,14 @@ begin _RI(desc,reg,imm); end; -procedure t_jit_builder.addi8(reg:TRegValue;imm:Byte); +procedure t_jit_builder.addi8se(reg:TRegValue;imm:ShortInt); const desc:t_op_type=(op:$83;index:0); begin _RI8(desc,reg,imm); end; -procedure t_jit_builder.addi8(mem:t_jit_leas;imm:Byte); +procedure t_jit_builder.addi8se(mem:t_jit_leas;imm:ShortInt); const desc:t_op_type=(op:$83;index:0); begin @@ -3739,14 +3745,14 @@ begin _RI(desc,reg,imm); end; -procedure t_jit_builder.subi8(reg:TRegValue;imm:Byte); +procedure t_jit_builder.subi8se(reg:TRegValue;imm:ShortInt); const desc:t_op_type=(op:$83;index:5); begin _RI8(desc,reg,imm); end; -procedure t_jit_builder.subi8(mem:t_jit_leas;imm:Byte); +procedure t_jit_builder.subi8se(mem:t_jit_leas;imm:ShortInt); const desc:t_op_type=(op:$83;index:5); begin @@ -3778,13 +3784,27 @@ begin _RI(desc,reg,imm); end; -procedure t_jit_builder.andi8(reg:TRegValue;imm:Byte); +procedure t_jit_builder.andi(mem:t_jit_leas;imm:Int64); +const + desc:t_op_type=(op:$81;index:4); +begin + _MI(desc,mem,imm); +end; + +procedure t_jit_builder.andi8se(reg:TRegValue;imm:ShortInt); const desc:t_op_type=(op:$83;index:4); begin _RI8(desc,reg,imm); end; +procedure t_jit_builder.andi8se(mem:t_jit_leas;imm:ShortInt); +const + desc:t_op_type=(op:$83;index:4); +begin + _MI8(desc,mem,imm); +end; + procedure t_jit_builder.andq(reg0:TRegValue;reg1:TRegValue); const desc:t_op_type=(op:$21;index:0); @@ -3792,6 +3812,34 @@ begin _RR(desc,reg0,reg1); end; +procedure t_jit_builder.ori(reg:TRegValue;imm:Int64); +const + desc:t_op_type=(op:$81;index:1); +begin + _RI(desc,reg,imm); +end; + +procedure t_jit_builder.ori(mem:t_jit_leas;imm:Int64); +const + desc:t_op_type=(op:$81;index:1); +begin + _MI(desc,mem,imm); +end; + +procedure t_jit_builder.ori8se(reg:TRegValue;imm:ShortInt); +const + desc:t_op_type=(op:$83;index:1); +begin + _RI8(desc,reg,imm); +end; + +procedure t_jit_builder.ori8se(mem:t_jit_leas;imm:ShortInt); +const + desc:t_op_type=(op:$83;index:1); +begin + _MI8(desc,mem,imm); +end; + procedure t_jit_builder.orq(reg0:TRegValue;reg1:TRegValue); const desc:t_op_type=(op:$09;index:0); diff --git a/sys/jit/kern_jit.pas b/sys/jit/kern_jit.pas index a8158e47..62e15726 100644 --- a/sys/jit/kern_jit.pas +++ b/sys/jit/kern_jit.pas @@ -994,32 +994,31 @@ begin end; procedure op_jit2native(var ctx:t_jit_context2;pcb:Boolean); -const - and_desc:t_op_type=(op:$80;index:4); begin with ctx.builder do begin - //reset PCB_IS_JIT + + //set PCB_IS_HLE if pcb then begin - _MI8(and_desc,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.pcb_flags),os8],not Byte(PCB_IS_JIT)); + ori([r13-jit_frame_offset+Integer(@p_kthread(nil)^.pcb_flags),os8],Byte(PCB_IS_HLE)); end; - //save internal stack - movq([r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_jctx.rsp)],rsp); - movq([r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_jctx.rbp)],rbp); - - //load guest stack - movq(r14,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_ustack.stack)]); - movq(r15,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_ustack.sttop)]); - - //set teb - movq([GS+teb_stack],r14); - movq([GS+teb_sttop],r15); + ////save internal stack + //movq([r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_jctx.rsp)],rsp); + //movq([r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_jctx.rbp)],rbp); + // + ////load guest stack + //movq(r14,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_ustack.stack)]); + //movq(r15,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_ustack.sttop)]); + // + ////set teb + //movq([GS+teb_stack],r14); + //movq([GS+teb_sttop],r15); //load rsp,rbp,r14,r15,r13 - movq(rsp,[r13+Integer(@p_jit_frame(nil)^.tf_rsp)]); - movq(rbp,[r13+Integer(@p_jit_frame(nil)^.tf_rbp)]); + //movq(rsp,[r13+Integer(@p_jit_frame(nil)^.tf_rsp)]); + //movq(rbp,[r13+Integer(@p_jit_frame(nil)^.tf_rbp)]); movq(r14,[r13+Integer(@p_jit_frame(nil)^.tf_r14)]); movq(r15,[r13+Integer(@p_jit_frame(nil)^.tf_r15)]); movq(r13,[r13+Integer(@p_jit_frame(nil)^.tf_r13)]); @@ -1027,17 +1026,16 @@ begin end; procedure op_native2jit(var ctx:t_jit_context2;pcb:Boolean); -const - or_desc:t_op_type=(op:$80;index:1); begin with ctx.builder do begin + //save r13 movq([GS+Integer(teb_jitcall)],r13); //load curkthread,jit ctx movq(r13,[GS +Integer(teb_thread)]); - leaq(r13,[r13+jit_frame_offset ]); + leaq(r13,[r13+jit_frame_offset ]); //load r14,r15 movq([r13+Integer(@p_jit_frame(nil)^.tf_r14)],r14); @@ -1048,26 +1046,27 @@ begin movq([r13+Integer(@p_jit_frame(nil)^.tf_r13)],r14); //load rsp,rbp - movq([r13+Integer(@p_jit_frame(nil)^.tf_rsp)],rsp); - movq([r13+Integer(@p_jit_frame(nil)^.tf_rbp)],rbp); + //movq([r13+Integer(@p_jit_frame(nil)^.tf_rsp)],rsp); + //movq([r13+Integer(@p_jit_frame(nil)^.tf_rbp)],rbp); //load host stack - movq(r14,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_kstack.stack)]); - movq(r15,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_kstack.sttop)]); + //movq(r14,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_kstack.stack)]); + //movq(r15,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_kstack.sttop)]); //set teb - movq([GS+teb_stack],r14); - movq([GS+teb_sttop],r15); + //movq([GS+teb_stack],r14); + //movq([GS+teb_sttop],r15); //load internal stack - movq(rsp,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_jctx.rsp)]); - movq(rbp,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_jctx.rbp)]); + //movq(rsp,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_jctx.rsp)]); + //movq(rbp,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.td_jctx.rbp)]); - //set PCB_IS_JIT + //reset PCB_IS_HLE if pcb then begin - _MI8(or_desc,[r13-jit_frame_offset+Integer(@p_kthread(nil)^.pcb_flags),os8],Byte(PCB_IS_JIT)); + andi([r13-jit_frame_offset+Integer(@p_kthread(nil)^.pcb_flags),os8],not Byte(PCB_IS_HLE)); end; + end; end; @@ -1382,7 +1381,6 @@ begin //debug end; - procedure pick_locked_internal(var ctx:t_jit_context2); var node:t_jit_context2.p_export_point; @@ -1415,10 +1413,13 @@ begin link_curr:=ctx.builder.get_curr_label.after; // op_jit2native(ctx,true); + //[JIT->HLE] + ctx.builder.call_far(node^.native); op_debug_info_addr(ctx,node^.native); + //[HLE->JIT] op_native2jit(ctx,true); // op_pop_rip_part0(ctx,0); //out:r14 diff --git a/sys/jit/kern_jit_dynamic.pas b/sys/jit/kern_jit_dynamic.pas index d0d774a0..14278ba3 100644 --- a/sys/jit/kern_jit_dynamic.pas +++ b/sys/jit/kern_jit_dynamic.pas @@ -271,6 +271,7 @@ end; procedure switch_to_jit(td:p_kthread); public; label + _host, _start; var node:p_jit_entry_point; @@ -279,6 +280,12 @@ var begin if (td=nil) then Exit; + if ((td^.pcb_flags and PCB_IS_HLE)<>0) then + begin + //hle mode + + goto _host; + end else if ((td^.pcb_flags and PCB_IS_JIT)<>0) then begin //jit mode @@ -311,6 +318,8 @@ begin begin //host mode + _host: + if is_guest_addr(td^.td_frame.tf_rip) then begin //host->jit diff --git a/sys/jit/kern_jit_ops_sse.pas b/sys/jit/kern_jit_ops_sse.pas index 33626716..54a7234d 100644 --- a/sys/jit/kern_jit_ops_sse.pas +++ b/sys/jit/kern_jit_ops_sse.pas @@ -470,36 +470,37 @@ begin lahf; //PEXTRQ r/m64, xmm2, imm8 - pextrq(ctx,m,xmm_b,1); + pextrq (ctx,m,xmm_b,1); - movq (b,m); - andi8(b,$3F); //b = len with m[0] + movq (b,m); + andi8se(b,$3F); // b = len = m[0] t:=new_reg_size(a,os32); - movi (t,64); //a = 64 (zero extended) - subq (a,b); //a = (64 - len) + movi (t,64); // a = 64 (zero extended) + subq (a,b); // a = (64 - len) - andi (m,$3F00); //filter + shri8 (m,8); // m[0] = 0 + shli8 (m,8); - movq (b,a); - andi8(b,$FF); - orq (m,b); // save (64 - len) to m[0] + movq (b,a); + andi8se(b,$3F); + orq (m,b); // m[0] = (64 - len) - movq (b,m); - shri8(b,8); - andi8(b,$3F); // b = idx with m[1] + movq (b,m); + shri8 (b,8); + andi8se(b,$3F); // b = idx = m[1] - subq (a,b); // a = (64 - len - idx) + subq (a,b); // a = (64 - len - idx) - movi (b,-1); // b = 0xFFFFFFFFFFFFFFFF (sign extended to 64-bit) + movi (b,-1); // b = 0xFFFFFFFFFFFFFFFF (sign extended to 64-bit) - shlx (b,b,a); // b = b shl (64 - idx - len) + shlx (b,b,a); // b = b shl (64 - idx - len) - shrx (b,b,m); // b = b shr (64 - len):[0x3F]; + shrx (b,b,m); // b = b shr (64 - len):[0x3F] - shli8(m,8); // m[0] = m[1] + shri8 (m,8); // m[0] = m[1] - shlx (b,b,m); // b = b shl idx + shlx (b,b,m); // b = b shl idx:[0x3F] //reassign //m -> a (idx) @@ -514,7 +515,7 @@ begin //b = xmm1[0:63] movq_r_xmm(ctx,b,xmm_b); - shlx (b,b,a); // b = b shl a (idx) + shlx (b,b,a); // b = b shl idx:[0x3F] end; //a = xmm0[0:63] diff --git a/sys/kern/kern_thr.pas b/sys/kern/kern_thr.pas index 7637cdfa..485dd138 100644 --- a/sys/kern/kern_thr.pas +++ b/sys/kern/kern_thr.pas @@ -130,6 +130,7 @@ const PCB_FULL_IRET=1; PCB_IS_JIT =2; + PCB_IS_HLE =4; type p_teb=^teb; @@ -381,7 +382,7 @@ end; procedure set_pcb_flags(td:p_kthread;f:Integer); begin - td^.pcb_flags:=f or (td^.pcb_flags and PCB_IS_JIT); + td^.pcb_flags:=f or (td^.pcb_flags and (PCB_IS_JIT or PCB_IS_HLE)); end; function TD_IS_SLEEPING(td:p_kthread):Boolean; diff --git a/sys/kern/kern_thread.pas b/sys/kern/kern_thread.pas index 33d33a34..252caa56 100644 --- a/sys/kern/kern_thread.pas +++ b/sys/kern/kern_thread.pas @@ -356,6 +356,8 @@ end; procedure before_start(td:p_kthread); begin + td:=curkthread; + InitThread(td^.td_ustack.stack-td^.td_ustack.sttop); Set8087CW(__INITIAL_FPUCW__); @@ -367,6 +369,7 @@ begin end; //switch + set_pcb_flags(curkthread,PCB_IS_JIT); //force JIT mode ipi_sigreturn; Writeln(stderr,'I''m a teapot!'); end; @@ -375,6 +378,8 @@ procedure before_start_kern(td:p_kthread); type t_cb=procedure(arg:QWORD); begin + td:=curkthread; + InitThread(td^.td_ustack.stack-td^.td_ustack.sttop); Set8087CW(__INITIAL_FPUCW__); diff --git a/sys/kern/machdep.pas b/sys/kern/machdep.pas index aab23c83..fcefd8f0 100644 --- a/sys/kern/machdep.pas +++ b/sys/kern/machdep.pas @@ -475,7 +475,6 @@ begin sp:=regs^.tf_rsp-128; end; - if ((td^.pcb_flags and PCB_IS_JIT)<>0) then if IS_SYSTEM_STACK(td,sp) then begin Assert(false,'System stack on guest context?'); @@ -515,7 +514,8 @@ begin regs^.tf_rsp:=QWORD(sfp); - if ((td^.pcb_flags and PCB_IS_JIT)=0) then + if ((td^.pcb_flags and PCB_IS_JIT)=0) or + ((td^.pcb_flags and PCB_IS_HLE)<>0) then begin regs^.tf_rip:=QWORD(@host_sigcode); end else @@ -542,7 +542,6 @@ begin Result:=copyin(sigcntxp,@uc,sizeof(ucontext_t)); if (Result<>0) then Exit; - if ((td^.pcb_flags and PCB_IS_JIT)<>0) then if IS_SYSTEM_STACK(td,uc.uc_mcontext.mc_rsp) then begin Assert(false,'System stack on guest context?'); diff --git a/sys/kern/trap.pas b/sys/kern/trap.pas index 5fd454b6..12df1b63 100644 --- a/sys/kern/trap.pas +++ b/sys/kern/trap.pas @@ -350,6 +350,8 @@ end; procedure fast_syscall; assembler; nostackframe; label + _align, + _restore, _after_call, _doreti, _fail, @@ -370,11 +372,16 @@ asm test %rax,%rax jz _fail + testl PCB_IS_HLE,kthread.pcb_flags(%rax) + jne _align + movqq kthread.td_kstack.stack(%rax),%rsp //td_kstack (Implicit lock interrupt) - andq $-16,%rsp //align stack andl NOT_PCB_FULL_IRET,kthread.pcb_flags(%rax) //clear PCB_FULL_IRET + _align: + andq $-16,%rsp //align stack + movqq $0 ,kthread.td_frame.tf_rflags(%rax) //clear movb %ch ,kthread.td_frame.tf_rflags(%rax) //save flags @@ -417,11 +424,16 @@ asm testl PCB_FULL_IRET,kthread.pcb_flags(%rcx) jnz _doreti + testl PCB_IS_HLE,kthread.pcb_flags(%rcx) + jne _restore + testl TDF_AST,kthread.td_flags(%rcx) jne _ast //Restore preserved registers. + _restore: + //get flags movqq kthread.td_frame.tf_rflags(%rcx),%rax push %rax @@ -468,6 +480,9 @@ asm //doreti _doreti: + testl PCB_IS_HLE,kthread.pcb_flags(%rcx) + jne _doreti_exit + //%rcx=curkthread testl TDF_AST,kthread.td_flags(%rcx) je _doreti_exit diff --git a/sys/md/md_thread.pas b/sys/md/md_thread.pas index 574184f4..e633e1f9 100644 --- a/sys/md/md_thread.pas +++ b/sys/md/md_thread.pas @@ -256,8 +256,17 @@ begin Writeln('NtCreateThread'); - //windows.CreateThread(); + td^.td_handle:=CreateThread(nil,4*1024,start_func,arg,CREATE_SUSPENDED,PDWORD(@ClientId^.UniqueThread)^); + if (td^.td_handle<>0) then + begin + Result:=0; + end else + begin + Result:=-1; + end; + + { Result:=NtCreateThread( @td^.td_handle, THREAD_ALL_ACCESS, @@ -267,6 +276,7 @@ begin Context, InitialTeb, True); + } if (Result=0) then begin