ShuriZma
/
FPPS4
mirror of https://github.com/red-prig/fpPS4.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

+

This commit is contained in:
Pavel 2023-09-28 17:23:30 +03:00
parent c57d813d10
commit 7d5f1ceb88
11 changed files with 831 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

214
rtl/x86_jit.pas
View File

@ -86,10 +86,11 @@ type
function get_label():t_jit_i_link; function get_label():t_jit_i_link;
public public
function is_valid:Boolean; function is_valid:Boolean;
function offset:Integer;
function before:t_jit_i_link; function before:t_jit_i_link;
function after :t_jit_i_link; function after :t_jit_i_link;
function prev:t_jit_i_link; function prev :t_jit_i_link;
function next:t_jit_i_link; function next :t_jit_i_link;
property _node:p_jit_instruction read ALink; property _node:p_jit_instruction read ALink;
property _label:t_jit_i_link read get_label write set_label; property _label:t_jit_i_link read get_label write set_label;
end; end;
@ -103,6 +104,23 @@ operator = (A,B:t_jit_i_link):Boolean;
//mm (0F=1, 0F38=2, 0F3A=3) //mm (0F=1, 0F38=2, 0F3A=3)
type type
t_jit_builder_allocator=object
type
PAllocNode=^TAllocNode;
TAllocNode=packed record
link:PAllocNode;
data:record end;
end;
var
pHead:SLIST_HEAD;
curr_apos:ptruint; //alloc pos in current node
curr_size:ptruint; //useable size of current node
used_size:ptruint; //full usable size
full_size:ptruint; //full alloc size
Function Alloc(Size:ptruint):Pointer;
Procedure Free;
end;
p_jit_builder=^t_jit_builder; p_jit_builder=^t_jit_builder;
t_jit_builder=object t_jit_builder=object
Const Const
@ -236,19 +254,13 @@ type
ymm14:TRegValue=(AType:regXmm;ASize:os256;AIndex: 14); ymm14:TRegValue=(AType:regXmm;ASize:os256;AIndex: 14);
ymm15:TRegValue=(AType:regXmm;ASize:os256;AIndex: 15); ymm15:TRegValue=(AType:regXmm;ASize:os256;AIndex: 15);
var var
AInstructions:TAILQ_HEAD; AInstructions :TAILQ_HEAD;
ADataSet :t_jit_data_set; ADataSet :t_jit_data_set;
ADataList :TAILQ_HEAD; ADataList :TAILQ_HEAD;
AInstructionSize:Integer; AInstructionSize:Integer;
ADataCount :Integer; ADataCount :Integer;
Allocator:record Allocator:t_jit_builder_allocator;
pHead:SLIST_HEAD;
curr_apos:ptruint; //alloc pos in current node
curr_size:ptruint; //useable size of current node
used_size:ptruint; //full usable size
full_size:ptruint; //full alloc size
end;
// //
Function Alloc(Size:ptruint):Pointer; Function Alloc(Size:ptruint):Pointer;
Procedure Free; Procedure Free;
@ -658,6 +670,26 @@ begin
Result:=(ALink<>nil); Result:=(ALink<>nil);
end; end;
function t_jit_i_link.offset:Integer;
begin
Result:=0;
if (ALink<>nil) then
begin
case AType of
lnkData:
begin
Result:=p_jit_data(ALink)^.pId*SizeOf(Pointer);
end;
lnkLabelBefore,
lnkLabelAfter:
begin
Result:=ALink^.AInstructionOffset;
end;
else;
end;
end;
end;
function t_jit_i_link.before:t_jit_i_link; function t_jit_i_link.before:t_jit_i_link;
begin begin
Result:=Self; Result:=Self;
@ -837,14 +869,7 @@ end;
// //
type Function t_jit_builder_allocator.Alloc(Size:ptruint):Pointer;
PAllocNode=^TAllocNode;
TAllocNode=packed record
link:PAllocNode;
data:record end;
end;
Function t_jit_builder.Alloc(Size:ptruint):Pointer;
const const
asize=(1*1024*1024)-SizeOf(ptruint)*3; asize=(1*1024*1024)-SizeOf(ptruint)*3;
var var
@ -863,49 +888,59 @@ var
end; end;
begin begin
if (Allocator.pHead.slh_first=nil) or (Size>Allocator.curr_size) then if (pHead.slh_first=nil) or (Size>curr_size) then
begin begin
node:=_alloc; node:=_alloc;
SLIST_INSERT_HEAD(@Allocator.pHead,node,@node^.link); SLIST_INSERT_HEAD(@pHead,node,@node^.link);
//Push_head(_alloc); //Push_head(_alloc);
mem_size:=MemSize(node); mem_size:=MemSize(node);
Allocator.curr_apos:=0; curr_apos:=0;
Allocator.curr_size:=mem_size-SizeOf(Pointer); curr_size:=mem_size-SizeOf(Pointer);
Inc(Allocator.full_size,mem_size); Inc(full_size,mem_size);
end; end;
node:=SLIST_FIRST(@Allocator.pHead); node:=SLIST_FIRST(@pHead);
Result:=@PByte(@node^.data)[Allocator.curr_apos]; Result:=@PByte(@node^.data)[curr_apos];
Inc(Allocator.used_size,Size); Inc(used_size,Size);
Size:=Align(Size,SizeOf(ptruint)); Size:=Align(Size,SizeOf(ptruint));
Inc(Allocator.curr_apos,Size); Inc(curr_apos,Size);
Dec(Allocator.curr_size,Size); Dec(curr_size,Size);
end; end;
Procedure t_jit_builder.Free; Procedure t_jit_builder_allocator.Free;
var var
node:PAllocNode; node:PAllocNode;
begin begin
//node:=Pop_head; //node:=Pop_head;
node:=Allocator.pHead.slh_first; node:=pHead.slh_first;
if (node<>nil) then if (node<>nil) then
begin begin
Allocator.pHead.slh_first:=node^.link; pHead.slh_first:=node^.link;
end; end;
While (node<>nil) do While (node<>nil) do
begin begin
FreeMem(node); FreeMem(node);
//node:=Pop_head; //node:=Pop_head;
node:=Allocator.pHead.slh_first; node:=pHead.slh_first;
if (node<>nil) then if (node<>nil) then
begin begin
Allocator.pHead.slh_first:=node^.link; pHead.slh_first:=node^.link;
end; end;
end; end;
Self:=Default(t_jit_builder); Self:=Default(t_jit_builder_allocator);
end;
Function t_jit_builder.Alloc(Size:ptruint):Pointer;
begin
Result:=Allocator.Alloc(Size);
end;
Procedure t_jit_builder.Free;
begin
Allocator.Free;
end; end;
// //
@ -1165,7 +1200,6 @@ begin
_O($C3); _O($C3);
end; end;
Procedure t_jit_builder.ud2; Procedure t_jit_builder.ud2;
begin begin
_O($0F0B); _O($0F0B);
@ -1233,12 +1267,44 @@ begin
end; end;
end; end;
function is_8bit_offset(d:Integer):Boolean; inline;
begin
case d of
-128..127:Result:=True;
else
Result:=False;
end;
end;
function classif_instr(node:p_jit_instruction):Byte; inline;
begin
Result:=0;
case node^.AData[0] of
$EB:Result:=1; //jmp8
$E9:Result:=2; //jmp32
$70..$7F:Result:=3; //jcc8
$0F:
case node^.AData[1] of
$80..$8F:Result:=4; //jcc32
else;
end;
else;
end;
end;
Procedure t_jit_builder.LinkData; Procedure t_jit_builder.LinkData;
label
_start;
var var
node:p_jit_instruction; node:p_jit_instruction;
d:Integer; d,t:Integer;
is_change:Boolean;
begin begin
_start:
is_change:=False;
d:=0; d:=0;
node:=TAILQ_FIRST(@AInstructions); node:=TAILQ_FIRST(@AInstructions);
@ -1247,27 +1313,81 @@ begin
With node^ do With node^ do
case ALink.AType of case ALink.AType of
lnkData: lnkData:
if not is_change then
begin begin
d:=_get_data_offset(ALink.ALink,AInstructionOffset+ASize); d:=_get_data_offset(ALink.ALink,AInstructionOffset+ASize);
_set_data(node,d); _set_data(node,d);
end; end;
lnkLabelBefore: lnkLabelBefore,
With node^ do
begin
d:=_get_label_before_offset(ALink.ALink,AInstructionOffset+ASize);
_set_data(node,d);
end;
lnkLabelAfter: lnkLabelAfter:
With node^ do
begin begin
d:=_get_label_after_offset(ALink.ALink,AInstructionOffset+ASize);
_set_data(node,d); case ALink.AType of
lnkLabelBefore:d:=_get_label_before_offset(ALink.ALink,AInstructionOffset+ASize);
lnkLabelAfter :d:=_get_label_after_offset (ALink.ALink,AInstructionOffset+ASize);
end;
t:=classif_instr(node);
if (t<>0) then
begin
if (d=0) then
begin
//clear instr
ALink.AType:=lnkNone;
ASize:=0;
is_change:=True;
end;
end;
if (ASize<>0) then
case t of
2:if is_8bit_offset(d) then //jmp32
begin
//set to jmp8
AData[0]:=$EB;
ASize:=2;
ALink.ASize:=1;
is_change:=True;
end;
4:if is_8bit_offset(d) then //jcc32
begin
t:=node^.AData[1] and $F;
AData[0]:=$70 or t;
ASize:=2;
ALink.ASize:=1;
ALink.AOffset:=1;
is_change:=True;
end;
else;
end;
if not is_change then
begin
_set_data(node,d);
end;
end; end;
else; else;
end; end;
// //
node:=TAILQ_NEXT(node,@node^.link); node:=TAILQ_NEXT(node,@node^.link);
end; end;
if is_change then
begin
RebuldInstructionOffset;
goto _start;
end;
end; end;
Function t_jit_builder.SaveTo(ptr:PByte;size:Integer):Integer; Function t_jit_builder.SaveTo(ptr:PByte;size:Integer):Integer;

2
sys/kern/kern_dlsym.pas
View File

@ -328,7 +328,7 @@ begin
donelist_init(donelist); donelist_init(donelist);
err:=0; err:=0;
if (obj^.mainprog=0) then if (obj^.rtld_flags.mainprog=0) then
begin begin
err:=symlook_list(@req,obj^.dagmembers,donelist); err:=symlook_list(@req,obj^.dagmembers,donelist);
end else end else

14
sys/kern/kern_dynlib.pas
View File

@ -299,7 +299,7 @@ begin
begin begin
dst^:=Default(SceKernelModuleInfo); dst^:=Default(SceKernelModuleInfo);
if ((flags and 1)=0) and (obj^.is_system<>0) then if ((flags and 1)=0) and (obj^.rtld_flags.is_system<>0) then
begin begin
Result:=EPERM; Result:=EPERM;
Break; Break;
@ -434,13 +434,13 @@ begin
if ((flags and 1)<>0) then if ((flags and 1)<>0) then
begin begin
tls_index:=((ord(obj^.is_system<>0) + ord(obj^.mainprog<>0)*2) * $10000) or tls_index; tls_index:=((ord(obj^.rtld_flags.is_system<>0) + ord(obj^.rtld_flags.mainprog<>0)*2) * $10000) or tls_index;
dst^.tls_index:=tls_index; dst^.tls_index:=tls_index;
end; end;
if ((flags and 2)<>0) then if ((flags and 2)<>0) then
begin begin
if (obj^.is_system<>0) {or (is_webkit)} then if (obj^.rtld_flags.is_system<>0) {or (is_webkit)} then
begin begin
FillChar(dst^.name,SCE_DBG_MAX_NAME_LENGTH,0); FillChar(dst^.name,SCE_DBG_MAX_NAME_LENGTH,0);
end; end;
@ -461,7 +461,7 @@ begin
//if not webkit then //if not webkit then
begin begin
if (obj^.not_get_proc=0) then if (obj^.rtld_flags.not_get_proc=0) then
begin begin
dst^.init_proc_addr:=obj^.init_proc_addr; dst^.init_proc_addr:=obj^.init_proc_addr;
dst^.fini_proc_addr:=obj^.fini_proc_addr; dst^.fini_proc_addr:=obj^.fini_proc_addr;
@ -551,7 +551,7 @@ begin
obj:=TAILQ_FIRST(@dynlibs_info.obj_list); obj:=TAILQ_FIRST(@dynlibs_info.obj_list);
while (obj<>nil) and (i<count) do while (obj<>nil) and (i<count) do
begin begin
if ((flags and 1)<>0) or (obj^.is_system=0) then if ((flags and 1)<>0) or (obj^.rtld_flags.is_system=0) then
begin begin
if (numArray<=i) then if (numArray<=i) then
begin begin
@ -621,11 +621,11 @@ begin
dst:=nil; dst:=nil;
case num of case num of
1:if (obj^.not_get_proc=0) then 1:if (obj^.rtld_flags.not_get_proc=0) then
begin begin
dst:=obj^.init_proc_addr; dst:=obj^.init_proc_addr;
end; end;
2:if (obj^.not_get_proc=0) then 2:if (obj^.rtld_flags.not_get_proc=0) then
begin begin
dst:=obj^.fini_proc_addr; dst:=obj^.fini_proc_addr;
end; end;

82
sys/kern/kern_exec.pas
View File

@ -60,8 +60,7 @@ uses
kern_dlsym, kern_dlsym,
kern_authinfo, kern_authinfo,
vfs_syscalls, vfs_syscalls,
kern_jit2, kern_jit_dynamic;
kern_jit2_ctx;
function exec_alloc_args(args:p_image_args):Integer; function exec_alloc_args(args:p_image_args):Integer;
begin begin
@ -777,10 +776,9 @@ begin
dynlibs_info.tls_static_space:=0; dynlibs_info.tls_static_space:=0;
dynlibs_info.tls_count :=1; dynlibs_info.tls_count :=1;
dynlibs_info.tls_max :=1; dynlibs_info.tls_max :=1;
//dynlibs_info.bits :=0;
obj:=obj_new(); obj:=obj_new();
obj^.mainprog:=1; obj^.rtld_flags.mainprog:=1;
obj^.relocbase:=imgp^.reloc_base; obj^.relocbase:=imgp^.reloc_base;
text_addr:=g_vmspace.vm_taddr; text_addr:=g_vmspace.vm_taddr;
@ -845,6 +843,7 @@ begin
if (Result<>0) then if (Result<>0) then
begin begin
obj_free(dynlibs_info.libprogram); obj_free(dynlibs_info.libprogram);
dynlibs_info.libprogram:=nil;
end; end;
end; end;
end; end;
@ -927,68 +926,6 @@ begin
end; end;
end; end;
procedure pick_obj(obj:p_lib_info);
var
ctx:t_jit_context2;
Lib_Entry:p_Lib_Entry;
h_entry:p_sym_hash_entry;
symp:p_elf64_sym;
addr:Pointer;
ST_TYPE:Integer;
begin
if (obj=nil) then Exit;
ctx:=Default(t_jit_context2);
ctx.text_start:=QWORD(obj^.map_base);
ctx.text___end:=ctx.text_start+obj^.text_size;
ctx.add_forward_point(obj^.entry_addr);
if (obj^.mainprog=0) then
begin
ctx.add_forward_point(obj^.init_proc_addr);
ctx.add_forward_point(obj^.fini_proc_addr);
end;
lib_entry:=TAILQ_FIRST(@obj^.lib_table);
while (lib_entry<>nil) do
begin
if (Lib_Entry^.import=0) then //export
begin
h_entry:=TAILQ_FIRST(@Lib_Entry^.syms);
while (h_entry<>nil) do
begin
symp:=@h_entry^.sym;
ST_TYPE:=ELF64_ST_TYPE(symp^.st_info);
Case ST_TYPE of
STT_NOTYPE,
STT_FUN,
STT_SCE:
if (symp^.st_value<>0) and
(symp^.st_shndx<>SHN_UNDEF) and
(symp^.st_value<obj^.text_size) then
begin
addr:=obj^.relocbase + symp^.st_value;
ctx.add_forward_point(addr);
end;
else;
end; //case
h_entry:=TAILQ_NEXT(h_entry,@h_entry^.link)
end;
end;
lib_entry:=TAILQ_NEXT(lib_entry,@lib_entry^.link)
end;
kern_jit2.pick(ctx);
end;
procedure dynlib_proc_initialize_step3(imgp:p_image_params); procedure dynlib_proc_initialize_step3(imgp:p_image_params);
label label
_dyn_not_exist; _dyn_not_exist;
@ -1011,6 +948,8 @@ begin
flags:=$40; //priv libs? flags:=$40; //priv libs?
if (budget_ptype_caller=0) then flags:=flags or $20; //vm_map_wire if (budget_ptype_caller=0) then flags:=flags or $20; //vm_map_wire
pick_obj(dynlibs_info.libprogram);
str:='libkernel.sprx'; str:='libkernel.sprx';
obj:=preload_prx_modules(pchar(str),flags,err); obj:=preload_prx_modules(pchar(str),flags,err);
dynlibs_info.libkernel:=obj; dynlibs_info.libkernel:=obj;
@ -1028,8 +967,6 @@ begin
Writeln(StdErr,'preload_prx_modules:',str,' not loaded'); Writeln(StdErr,'preload_prx_modules:',str,' not loaded');
end; end;
pick_obj(obj);
obj:=TAILQ_FIRST(@dynlibs_info.obj_list); obj:=TAILQ_FIRST(@dynlibs_info.obj_list);
while (obj<>nil) do while (obj<>nil) do
begin begin
@ -1065,10 +1002,6 @@ begin
p_proc.libkernel___end_addr:=dynlibs_info.libkernel^.map_base + dynlibs_info.libkernel^.text_size; p_proc.libkernel___end_addr:=dynlibs_info.libkernel^.map_base + dynlibs_info.libkernel^.text_size;
end; end;
//pick_obj(dynlibs_info.libprogram);
pick_obj(dynlibs_info.libkernel);
_dyn_not_exist: _dyn_not_exist:
obj:=TAILQ_FIRST(@dynlibs_info.obj_list); obj:=TAILQ_FIRST(@dynlibs_info.obj_list);
@ -1600,6 +1533,11 @@ done2:
VFS_UNLOCK_GIANT(vfslocked); VFS_UNLOCK_GIANT(vfslocked);
exec_free_args(args); exec_free_args(args);
if (error=0) then
begin
kern_jit_dynamic.switch_to_jit();
end;
Exit(error); Exit(error);
end; end;

8
sys/kern/kern_reloc.pas
View File

@ -97,7 +97,7 @@ begin
R_X86_64_NONE:; //ignore R_X86_64_NONE:; //ignore
R_X86_64_COPY: R_X86_64_COPY:
if (obj^.mainprog=0) then if (obj^.rtld_flags.mainprog=0) then
begin begin
Writeln(StdErr,'reloc_non_plt:','Unexpected R_X86_64_COPY relocation in shared library ',dynlib_basename(obj^.lib_path)); Writeln(StdErr,'reloc_non_plt:','Unexpected R_X86_64_COPY relocation in shared library ',dynlib_basename(obj^.lib_path));
Exit(ENOEXEC); Exit(ENOEXEC);
@ -394,8 +394,8 @@ begin
end; end;
if (flags=1) and if (flags=1) and
(obj^.jmpslots_done=0) and (obj^.rtld_flags.jmpslots_done=0) and
(defobj^.jmpslots_done=0) then (defobj^.rtld_flags.jmpslots_done=0) then
begin begin
Exit(5); Exit(5);
end; end;
@ -528,7 +528,7 @@ begin
R_X86_64_NONE:; //ignore R_X86_64_NONE:; //ignore
R_X86_64_COPY: R_X86_64_COPY:
if (obj^.mainprog=0) then if (obj^.rtld_flags.mainprog=0) then
begin begin
Writeln(StdErr,'dynlib_unlink_non_plt_reloc_each:','Unexpected R_X86_64_COPY relocation in dynamic library ',dynlib_basename(obj^.lib_path)); Writeln(StdErr,'dynlib_unlink_non_plt_reloc_each:','Unexpected R_X86_64_COPY relocation in dynamic library ',dynlib_basename(obj^.lib_path));
Exit(-1); Exit(-1);

1
sys/kern/kern_thr.pas
View File

@ -225,6 +225,7 @@ type
td_sel :Pointer; td_sel :Pointer;
td_vp_reserv :Int64; td_vp_reserv :Int64;
pcb_onfault :Pointer; pcb_onfault :Pointer;
td_jit_ctx :Pointer;
end; end;
p_thr_param=^thr_param; p_thr_param=^thr_param;

155
sys/kern/subr_dynlib.pas
View File

@ -116,20 +116,20 @@ type
loaded:Integer; loaded:Integer;
//t_rtld_bits rtld_flags; rtld_flags:bitpacked record
mainprog :0..1;
mainprog :Byte; tls_done :0..1;
tls_done :Byte; init_scanned :0..1;
init_scanned :Byte; init_done :0..1;
init_done :Byte; on_fini_list :0..1;
on_fini_list :Byte; not_get_proc :0..1;
not_get_proc :Byte; textrel :0..1;
textrel :Byte; init_plt :0..1;
init_plt :Byte; is_system :0..1;
is_system :Byte; dag_inited :0..1;
dag_inited :Byte; jmpslots_done:0..1;
jmpslots_done:Byte; internal :0..1;
internal :Byte; end;
dldags :TAILQ_HEAD; //Objlist_Entry dldags :TAILQ_HEAD; //Objlist_Entry
dagmembers:TAILQ_HEAD; //Objlist_Entry dagmembers:TAILQ_HEAD; //Objlist_Entry
@ -327,6 +327,8 @@ procedure unload_object(root:p_lib_info);
function relocate_object(root:p_lib_info):Integer; function relocate_object(root:p_lib_info):Integer;
function dynlib_load_relocate():Integer; function dynlib_load_relocate():Integer;
procedure pick_obj(obj:p_lib_info);
function preload_prx_modules(path:pchar;flags:DWORD;var err:Integer):p_lib_info; function preload_prx_modules(path:pchar;flags:DWORD;var err:Integer):p_lib_info;
function load_prx(path:pchar;flags:DWORD;var pobj:p_lib_info):Integer; function load_prx(path:pchar;flags:DWORD;var pobj:p_lib_info):Integer;
function unload_prx(obj:p_lib_info):Integer; function unload_prx(obj:p_lib_info):Integer;
@ -370,7 +372,9 @@ uses
kern_proc, kern_proc,
kern_reloc, kern_reloc,
kern_namedobj, kern_namedobj,
elf_nid_utils; elf_nid_utils,
kern_jit2,
kern_jit2_ctx;
procedure dynlibs_lock; procedure dynlibs_lock;
begin begin
@ -567,7 +571,7 @@ end;
function obj_new_int(mod_name:pchar):p_lib_info; function obj_new_int(mod_name:pchar):p_lib_info;
begin begin
Result:=obj_new(); Result:=obj_new();
Result^.internal:=1; Result^.rtld_flags.internal:=1;
Result^.add_mod(mod_name); Result^.add_mod(mod_name);
end; end;
@ -948,7 +952,7 @@ function allocate_tls_offset(obj:p_lib_info):Boolean;
var var
off:Int64; off:Int64;
begin begin
if (obj^.tls_done<>0) then if (obj^.rtld_flags.tls_done<>0) then
begin begin
Exit(True); Exit(True);
end; end;
@ -956,7 +960,7 @@ begin
off:=obj^.tls_size; off:=obj^.tls_size;
if (off=0) then if (off=0) then
begin begin
obj^.tls_done:=1; obj^.rtld_flags.tls_done:=1;
Exit(True); Exit(True);
end; end;
@ -985,7 +989,7 @@ end;
procedure free_tls_offset(obj:p_lib_info); procedure free_tls_offset(obj:p_lib_info);
begin begin
if (obj^.tls_done<>0) and (obj^.tls_offset=dynlibs_info.tls_last_offset) then if (obj^.rtld_flags.tls_done<>0) and (obj^.tls_offset=dynlibs_info.tls_last_offset) then
begin begin
dynlibs_info.tls_last_offset:=obj^.tls_offset - obj^.tls_size; dynlibs_info.tls_last_offset:=obj^.tls_offset - obj^.tls_size;
dynlibs_info.tls_last_size :=0; dynlibs_info.tls_last_size :=0;
@ -1269,8 +1273,9 @@ procedure initlist_add_objects(var fini_proc_list:TAILQ_HEAD;
tail:p_lib_info; tail:p_lib_info;
var init_proc_list:TAILQ_HEAD); var init_proc_list:TAILQ_HEAD);
begin begin
if (obj^.init_scanned<>0) or (obj^.init_done<>0) then Exit; if (obj^.rtld_flags.init_scanned<>0) or
obj^.init_scanned:=1; (obj^.rtld_flags.init_done<>0) then Exit;
obj^.rtld_flags.init_scanned:=1;
if (obj<>tail) then if (obj<>tail) then
begin begin
@ -1287,11 +1292,12 @@ begin
objlist_push_tail(init_proc_list,obj); objlist_push_tail(init_proc_list,obj);
end; end;
if (obj^.fini_proc_addr<>nil) and (obj^.on_fini_list=0) then if (obj^.fini_proc_addr<>nil) and
(obj^.rtld_flags.on_fini_list=0) then
begin begin
objlist_push_tail(fini_proc_list,obj); objlist_push_tail(fini_proc_list,obj);
obj^.on_fini_list:=1; obj^.rtld_flags.on_fini_list:=1;
end; end;
end; end;
@ -1435,7 +1441,7 @@ begin
DT_TEXTREL: DT_TEXTREL:
begin begin
obj^.textrel:=1; obj^.rtld_flags.textrel:=1;
end; end;
DT_FLAGS: DT_FLAGS:
@ -1456,7 +1462,7 @@ begin
if ((dval and DF_TEXTREL)<>0) then if ((dval and DF_TEXTREL)<>0) then
begin begin
obj^.textrel:=1; obj^.rtld_flags.textrel:=1;
end; end;
end; end;
@ -2061,7 +2067,7 @@ begin
if (budget=2) then if (budget=2) then
begin begin
new^.is_system:=1; new^.rtld_flags.is_system:=1;
end; end;
error:=acquire_per_file_info_obj(imgp,new); error:=acquire_per_file_info_obj(imgp,new);
@ -2122,7 +2128,7 @@ function relocate_text_or_data_segment(obj:p_lib_info;src,dst:Pointer;size:QWORD
var var
map:vm_map_t; map:vm_map_t;
begin begin
if (obj^.textrel=0) or if (obj^.rtld_flags.textrel=0) or
(obj^.map_base > dst) or (obj^.map_base > dst) or
((obj^.map_base + obj^.text_size) < (dst + size)) then ((obj^.map_base + obj^.text_size) < (dst + size)) then
begin begin
@ -2182,7 +2188,7 @@ var
elm:p_Objlist_Entry; elm:p_Objlist_Entry;
donelist:t_DoneList; donelist:t_DoneList;
begin begin
if (root^.dag_inited<>0) then Exit; if (root^.rtld_flags.dag_inited<>0) then Exit;
donelist:=Default(t_DoneList); donelist:=Default(t_DoneList);
donelist_init(donelist); donelist_init(donelist);
@ -2224,14 +2230,14 @@ begin
elm:=TAILQ_NEXT(elm,@elm^.link); elm:=TAILQ_NEXT(elm,@elm^.link);
end; end;
root^.dag_inited:=1; root^.rtld_flags.dag_inited:=1;
end; end;
procedure ref_dag(root:p_lib_info); procedure ref_dag(root:p_lib_info);
var var
elm:p_Objlist_Entry; elm:p_Objlist_Entry;
begin begin
Assert(root^.dag_inited<>0,'DAG is not initialized'); Assert(root^.rtld_flags.dag_inited<>0,'DAG is not initialized');
elm:=TAILQ_FIRST(@root^.dagmembers); elm:=TAILQ_FIRST(@root^.dagmembers);
while (elm<>nil) do while (elm<>nil) do
@ -2246,7 +2252,7 @@ procedure unref_dag(root:p_lib_info);
var var
elm:p_Objlist_Entry; elm:p_Objlist_Entry;
begin begin
Assert(root^.dag_inited<>0,'DAG is not initialized'); Assert(root^.rtld_flags.dag_inited<>0,'DAG is not initialized');
elm:=TAILQ_FIRST(@root^.dagmembers); elm:=TAILQ_FIRST(@root^.dagmembers);
while (elm<>nil) do while (elm<>nil) do
@ -2268,7 +2274,7 @@ var
begin begin
Result:=0; Result:=0;
if (obj^.init_plt<>0) then Exit; if (obj^.rtld_flags.init_plt<>0) then Exit;
Result:=change_relro_protection(obj,VM_PROT_RW); Result:=change_relro_protection(obj,VM_PROT_RW);
if (Result<>0) then Exit; if (Result<>0) then Exit;
@ -2311,7 +2317,7 @@ begin
err:=change_relro_protection(obj,VM_PROT_READ); err:=change_relro_protection(obj,VM_PROT_READ);
obj^.init_plt:=1; obj^.rtld_flags.init_plt:=1;
end; end;
function DecodeSym(obj:p_lib_info; function DecodeSym(obj:p_lib_info;
@ -2547,7 +2553,7 @@ begin
goto _error; goto _error;
end; end;
if (new^.textrel<>0) then if (new^.rtld_flags.textrel<>0) then
begin begin
Writeln(StdErr,'do_load_object:',new^.lib_path,' has impure text'); Writeln(StdErr,'do_load_object:',new^.lib_path,' has impure text');
err:=EINVAL; err:=EINVAL;
@ -2678,7 +2684,7 @@ begin
//reg lib //reg lib
dynlibs_add_obj(Result); dynlibs_add_obj(Result);
// //
Result^.internal:=1; Result^.rtld_flags.internal:=1;
Result^.loaded:=1; Result^.loaded:=1;
Writeln(' preload_prx_internal:',path); Writeln(' preload_prx_internal:',path);
@ -2692,6 +2698,75 @@ begin
end; end;
procedure pick_obj(obj:p_lib_info);
var
ctx:t_jit_context2;
Lib_Entry:p_Lib_Entry;
h_entry:p_sym_hash_entry;
symp:p_elf64_sym;
addr:Pointer;
ST_TYPE:Integer;
begin
if (obj=nil) then Exit;
if (obj^.rtld_flags.internal=1) then
begin
Exit;
end;
Writeln('pick_obj:',obj^.lib_path);
ctx:=Default(t_jit_context2);
ctx.text_start:=QWORD(obj^.map_base);
ctx.text___end:=ctx.text_start+obj^.text_size;
ctx.add_forward_point(obj^.entry_addr);
if (obj^.rtld_flags.mainprog=0) then
begin
ctx.add_forward_point(obj^.init_proc_addr);
ctx.add_forward_point(obj^.fini_proc_addr);
end;
lib_entry:=TAILQ_FIRST(@obj^.lib_table);
while (lib_entry<>nil) do
begin
if (Lib_Entry^.import=0) then //export
begin
h_entry:=TAILQ_FIRST(@Lib_Entry^.syms);
while (h_entry<>nil) do
begin
symp:=@h_entry^.sym;
ST_TYPE:=ELF64_ST_TYPE(symp^.st_info);
Case ST_TYPE of
STT_NOTYPE,
STT_FUN,
STT_SCE:
if (symp^.st_value<>0) and
(symp^.st_shndx<>SHN_UNDEF) and
(symp^.st_value<obj^.text_size) then
begin
addr:=obj^.relocbase + symp^.st_value;
ctx.add_forward_point(addr);
end;
else;
end; //case
h_entry:=TAILQ_NEXT(h_entry,@h_entry^.link)
end;
end;
lib_entry:=TAILQ_NEXT(lib_entry,@lib_entry^.link)
end;
kern_jit2.pick(ctx);
end;
function preload_prx_modules(path:pchar;flags:DWORD;var err:Integer):p_lib_info; function preload_prx_modules(path:pchar;flags:DWORD;var err:Integer):p_lib_info;
label label
_do_load; _do_load;
@ -2766,6 +2841,8 @@ begin
_do_load: _do_load:
Result:=do_load_object(pchar(fname),flags,err); Result:=do_load_object(pchar(fname),flags,err);
pick_obj(Result);
end; end;
function relocate_object(root:p_lib_info):Integer; function relocate_object(root:p_lib_info):Integer;
@ -2775,7 +2852,7 @@ begin
Result:=change_relro_protection_all(VM_PROT_RW); Result:=change_relro_protection_all(VM_PROT_RW);
if (Result<>0) then Exit; if (Result<>0) then Exit;
Result:=relocate_one_object(root,ord(root^.jmpslots_done=0)); Result:=relocate_one_object(root,ord(root^.rtld_flags.jmpslots_done=0));
if (Result=0) then if (Result=0) then
begin begin
@ -2785,7 +2862,7 @@ begin
begin begin
if (obj<>root) then if (obj<>root) then
begin begin
Result:=relocate_one_object(obj,ord(root^.jmpslots_done=0)); Result:=relocate_one_object(obj,ord(root^.rtld_flags.jmpslots_done=0));
end; end;
obj:=TAILQ_NEXT(obj,@obj^.link); obj:=TAILQ_NEXT(obj,@obj^.link);
end; end;
@ -2844,12 +2921,12 @@ begin
begin begin
if ((flags and $20000)<>0) then //set jmpslots_done? if ((flags and $20000)<>0) then //set jmpslots_done?
begin begin
obj^.jmpslots_done:=1; obj^.rtld_flags.jmpslots_done:=1;
end; end;
if ((flags and $40000)<>0) then //set not_get_proc? if ((flags and $40000)<>0) then //set not_get_proc?
begin begin
obj^.not_get_proc:=1; obj^.rtld_flags.not_get_proc:=1;
end; end;
init_dag(obj); init_dag(obj);

291
sys/test/kern_jit2.pas
View File

@ -25,11 +25,53 @@ implementation
uses uses
sysutils, sysutils,
kern_jit2_ops; kern_jit2_ops,
kern_jit_dynamic;
procedure jit_before_start; assembler; nostackframe;
asm
nop
end;
procedure jit_jmp_dispatch;
begin
Writeln('TODO:jit_jmp_dispatch');
Assert(False);
end;
procedure jit_syscall;
begin
Writeln('TODO:jit_syscall');
Assert(False);
end;
procedure jit_assert;
begin
Writeln('TODO:jit_assert');
Assert(False);
end;
procedure jit_system_error;
begin
Writeln('TODO:jit_system_error');
Assert(False);
end;
procedure jit_exit_proc;
begin
Writeln('TODO:jit_exit_proc');
//Assert(False);
end;
procedure jit_cpuid;
begin
Writeln('TODO:jit_cpuid');
Assert(False);
end;
procedure op_jmp_dispatcher(var ctx:t_jit_context2); procedure op_jmp_dispatcher(var ctx:t_jit_context2);
begin begin
ctx.builder.call_far(nil); //input:rax TODO jmp dispatcher ctx.builder.call_far(@jit_jmp_dispatch); //input:rax TODO jmp dispatcher
end; end;
procedure op_push_rip(var ctx:t_jit_context2;used_r_tmp0:Boolean); procedure op_push_rip(var ctx:t_jit_context2;used_r_tmp0:Boolean);
@ -58,10 +100,20 @@ begin
if (classif_offset_se64(imm)=os64) then if (classif_offset_se64(imm)=os64) then
begin begin
movi64(r_tmp1,imm); if (classif_offset_u64(imm)=os64) then
movq([stack],r_tmp1); begin
//64bit imm
movi64(r_tmp1,imm);
movq([stack],r_tmp1);
end else
begin
//32bit zero extend
movi(new_reg_size(r_tmp1,os32),imm);
movq([stack],r_tmp1);
end;
end else end else
begin begin
//32bit sign extend
movi(os64,[stack],imm); movi(os64,[stack],imm);
end; end;
@ -115,6 +167,20 @@ begin
Result:=is_rsp(r[0]) or is_rsp(r[1]); Result:=is_rsp(r[0]) or is_rsp(r[1]);
end; end;
procedure op_set_rax_imm(var ctx:t_jit_context2;imm:Int64);
begin
with ctx.builder do
if (classif_offset_u64(imm)=os64) then
begin
//64bit imm
movi64(r_tmp0,imm);
end else
begin
//32bit zero extend
movi(new_reg_size(r_tmp0,os32),imm);
end;
end;
procedure op_call(var ctx:t_jit_context2); procedure op_call(var ctx:t_jit_context2);
var var
id:t_jit_i_link; id:t_jit_i_link;
@ -134,20 +200,31 @@ begin
dst:=ctx.ptr_next+ofs; dst:=ctx.ptr_next+ofs;
label_id:=ctx.find_label(dst); if ctx.is_curr_addr(QWORD(dst)) then
if (label_id<>nil_link) then
begin begin
op_push_rip(ctx,false); label_id:=ctx.find_label(dst);
//
ctx.builder.jmp(label_id); if (label_id<>nil_link) then
begin
op_push_rip(ctx,false);
//
ctx.builder.jmp(label_id);
end else
begin
op_push_rip(ctx,false);
//
id:=ctx.builder.jmp(nil_link);
ctx.add_forward_point(id,dst);
end;
end else end else
begin begin
op_push_rip(ctx,false); op_push_rip(ctx,false);
// //
id:=ctx.builder.jmp(nil_link); op_set_rax_imm(ctx,Int64(dst));
ctx.add_forward_point(id,dst); //
op_jmp_dispatcher(ctx);
end; end;
end else end else
if is_memory(ctx.din) then if is_memory(ctx.din) then
begin begin
@ -220,15 +297,23 @@ begin
dst:=ctx.ptr_next+ofs; dst:=ctx.ptr_next+ofs;
label_id:=ctx.find_label(dst); if ctx.is_curr_addr(QWORD(dst)) then
if (label_id<>nil_link) then
begin begin
ctx.builder.jmp(label_id); label_id:=ctx.find_label(dst);
if (label_id<>nil_link) then
begin
ctx.builder.jmp(label_id);
end else
begin
id:=ctx.builder.jmp(nil_link);
ctx.add_forward_point(id,dst);
end;
end else end else
begin begin
id:=ctx.builder.jmp(nil_link); op_set_rax_imm(ctx,Int64(dst));
ctx.add_forward_point(id,dst); //
op_jmp_dispatcher(ctx);
end; end;
end else end else
@ -263,7 +348,7 @@ end;
procedure op_jcc(var ctx:t_jit_context2); procedure op_jcc(var ctx:t_jit_context2);
var var
id:t_jit_i_link; id,id2:t_jit_i_link;
ofs:Int64; ofs:Int64;
dst:Pointer; dst:Pointer;
label_id:t_jit_i_link; label_id:t_jit_i_link;
@ -276,15 +361,31 @@ begin
dst:=ctx.ptr_next+ofs; dst:=ctx.ptr_next+ofs;
label_id:=ctx.find_label(dst); if ctx.is_curr_addr(QWORD(dst)) then
if (label_id<>nil_link) then
begin begin
ctx.builder.jcc(ctx.din.OpCode.Suffix,label_id); label_id:=ctx.find_label(dst);
if (label_id<>nil_link) then
begin
ctx.builder.jcc(ctx.din.OpCode.Suffix,label_id);
end else
begin
id:=ctx.builder.jcc(ctx.din.OpCode.Suffix,nil_link);
ctx.add_forward_point(id,dst);
end;
end else end else
begin begin
id:=ctx.builder.jcc(ctx.din.OpCode.Suffix,nil_link); id:=ctx.builder.jcc(ctx.din.OpCode.Suffix,nil_link,os8);
ctx.add_forward_point(id,dst);
id2:=ctx.builder.jmp(nil_link,os8);
id._label:=ctx.builder.get_curr_label.after;
//
op_set_rax_imm(ctx,Int64(dst));
//
op_jmp_dispatcher(ctx);
id2._label:=ctx.builder.get_curr_label.after;
end; end;
end; end;
@ -437,7 +538,7 @@ end;
procedure op_syscall(var ctx:t_jit_context2); procedure op_syscall(var ctx:t_jit_context2);
begin begin
ctx.builder.call_far(nil); //TODO syscall dispatcher ctx.builder.call_far(@jit_syscall); //TODO syscall dispatcher
end; end;
procedure op_int(var ctx:t_jit_context2); procedure op_int(var ctx:t_jit_context2);
@ -453,13 +554,13 @@ begin
$41: //assert? $41: //assert?
begin begin
// //
ctx.builder.call_far(nil); //TODO error dispatcher ctx.builder.call_far(@jit_assert); //TODO error dispatcher
end; end;
$44: //system error? $44: //system error?
begin begin
// //
ctx.builder.call_far(nil); //TODO error dispatcher ctx.builder.call_far(@jit_system_error); //TODO error dispatcher
ctx.ptr_next:=nil; //trim ctx.ptr_next:=nil; //trim
end; end;
else else
@ -472,26 +573,26 @@ end;
procedure op_ud2(var ctx:t_jit_context2); procedure op_ud2(var ctx:t_jit_context2);
begin begin
//exit proc? //exit proc?
ctx.builder.call_far(nil); //TODO exit dispatcher ctx.builder.call_far(@jit_exit_proc); //TODO exit dispatcher
ctx.ptr_next:=nil; //trim ctx.ptr_next:=nil; //trim
end; end;
procedure op_iretq(var ctx:t_jit_context2); procedure op_iretq(var ctx:t_jit_context2);
begin begin
//exit proc? //exit proc?
ctx.builder.call_far(nil); //TODO exit dispatcher ctx.builder.call_far(@jit_exit_proc); //TODO exit dispatcher
ctx.ptr_next:=nil; //trim ctx.ptr_next:=nil; //trim
end; end;
procedure op_hlt(var ctx:t_jit_context2); procedure op_hlt(var ctx:t_jit_context2);
begin begin
//stop thread? //stop thread?
ctx.builder.call_far(nil); //TODO exit dispatcher ctx.builder.call_far(@jit_exit_proc); //TODO exit dispatcher
end; end;
procedure op_cpuid(var ctx:t_jit_context2); procedure op_cpuid(var ctx:t_jit_context2);
begin begin
ctx.builder.call_far(nil); //TODO CPUID ctx.builder.call_far(@jit_cpuid); //TODO CPUID
end; end;
procedure op_rdtsc(var ctx:t_jit_context2); procedure op_rdtsc(var ctx:t_jit_context2);
@ -505,9 +606,10 @@ begin
//align? //align?
end; end;
procedure op_rep_cmps(var ctx:t_jit_context2);
const const
test_desc:t_op_type=(op:$85;index:0); test_desc:t_op_type=(op:$85;index:0);
procedure op_rep_cmps(var ctx:t_jit_context2);
var var
op:DWORD; op:DWORD;
size:TOperandSize; size:TOperandSize;
@ -603,8 +705,6 @@ end;
/// ///
procedure op_rep_stos(var ctx:t_jit_context2); procedure op_rep_stos(var ctx:t_jit_context2);
const
test_desc:t_op_type=(op:$85;index:0);
var var
i:Integer; i:Integer;
size:TOperandSize; size:TOperandSize;
@ -615,7 +715,6 @@ var
link___end:t_jit_i_link; link___end:t_jit_i_link;
link_jmp0:t_jit_i_link; link_jmp0:t_jit_i_link;
link_jmp1:t_jit_i_link;
begin begin
//rdi,rsi //rdi,rsi
//prefix $67 TODO //prefix $67 TODO
@ -635,7 +734,6 @@ begin
begin begin
link_jmp0:=nil_link; link_jmp0:=nil_link;
link_jmp1:=nil_link;
new:=new_reg_size(r_tmp1,size); new:=new_reg_size(r_tmp1,size);
@ -658,17 +756,6 @@ begin
leaq(rcx,[rcx-1]); leaq(rcx,[rcx-1]);
leaq(rdi,[rdi+OPERAND_BYTES[size]]); leaq(rdi,[rdi+OPERAND_BYTES[size]]);
if (ifPrefixRepE in ctx.din.Flags) then
begin
//if a[i]<>b[i] then exit
link_jmp1:=jcc(OPSc_nz,nil_link,os8);
end else
if (ifPrefixRepNe in ctx.din.Flags) then
begin
//if a[i]=b[i] then exit
link_jmp1:=jcc(OPSc_z,nil_link,os8);
end;
//until //until
jmp(link_start,os8); jmp(link_start,os8);
@ -680,10 +767,6 @@ begin
link___end:=ctx.builder.get_curr_label.before; //exit1 link___end:=ctx.builder.get_curr_label.before; //exit1
link_jmp0._label:=link___end; link_jmp0._label:=link___end;
link___end:=link___end.after; //exit2
link_jmp1._label:=link___end;
end; end;
end; end;
@ -752,14 +835,17 @@ const
SCODES:array[TSimdOpcode] of Byte=(0,0,1,3,2); SCODES:array[TSimdOpcode] of Byte=(0,0,1,3,2);
MCODES:array[0..3] of RawByteString=('','0F','0F38','0F3A'); MCODES:array[0..3] of RawByteString=('','0F','0F38','0F3A');
label label
_next; _next,
_build;
var var
addr:Pointer; addr:Pointer;
ptr:Pointer;
links:t_jit_context2.t_forward_links; links:t_jit_context2.t_forward_links;
entry_link:Pointer;
proc:TDbgProcess; proc:TDbgProcess;
adec:TX86AsmDecoder; adec:TX86AsmDecoder;
ptr,fin:Pointer;
ACodeBytes,ACode:RawByteString; ACodeBytes,ACode:RawByteString;
cb:t_jit_cb; cb:t_jit_cb;
@ -767,8 +853,10 @@ var
node_new,node_curr:t_jit_i_link; node_new,node_curr:t_jit_i_link;
node,node_code1,node_code2:p_jit_instruction; node,node_code1,node_code2:p_jit_instruction;
data:Pointer; blob:p_jit_dynamic;
F:THandle; entry_point:t_jit_context2.p_entry_point;
//F:THandle;
begin begin
init_cbs; init_cbs;
@ -783,21 +871,27 @@ begin
if not ctx.fetch_forward_point(links,addr) then if not ctx.fetch_forward_point(links,addr) then
begin begin
Assert(false); Exit;
end; end;
entry_link:=addr;
Writeln('0x',HexStr(entry_link));
ctx.builder.call_far(@jit_before_start);
ptr:=addr; ptr:=addr;
fin:=Pointer(Int64(-1));
proc:=TDbgProcess.Create(dm64); proc:=TDbgProcess.Create(dm64);
adec:=TX86AsmDecoder.Create(proc); adec:=TX86AsmDecoder.Create(proc);
while (ptr<fin) do while True do
begin begin
if ((pmap_get_raw(QWORD(ptr)) and PAGE_PROT_EXECUTE)=0) then if ((pmap_get_raw(QWORD(ptr)) and PAGE_PROT_EXECUTE)=0) then
begin begin
writeln('not excec:0x',HexStr(ptr)); //writeln('not excec:0x',HexStr(ptr));
ctx.builder.ud2;
goto _next; goto _next;
end; end;
@ -809,12 +903,8 @@ begin
OPX_Invalid..OPX_GroupP: OPX_Invalid..OPX_GroupP:
begin begin
//invalid //invalid
writeln('invalid:0x',HexStr(ctx.ptr_curr)); //writeln('invalid:0x',HexStr(ctx.ptr_curr));
ctx.builder.ud2;
ptr:=ctx.ptr_curr;
adec.Disassemble(ptr,ACodeBytes,ACode);
goto _next; goto _next;
end; end;
else; else;
@ -823,13 +913,16 @@ begin
if (adec.Instr.Flags * [ifOnly32, ifOnly64, ifOnlyVex] <> []) or if (adec.Instr.Flags * [ifOnly32, ifOnly64, ifOnlyVex] <> []) or
is_invalid(adec.Instr) then is_invalid(adec.Instr) then
begin begin
writeln('invalid:0x',HexStr(ctx.ptr_curr)); //writeln('invalid:0x',HexStr(ctx.ptr_curr));
ctx.builder.ud2;
goto _next; goto _next;
end; end;
{
Writeln('original------------------------':32,' ','0x',HexStr(ptr-adec.Disassembler.CodeIdx)); Writeln('original------------------------':32,' ','0x',HexStr(ptr-adec.Disassembler.CodeIdx));
Writeln(ACodeBytes:32,' ',ACode); Writeln(ACodeBytes:32,' ',ACode);
Writeln('original------------------------':32,' ','0x',HexStr(ptr)); Writeln('original------------------------':32,' ','0x',HexStr(ptr));
}
ctx.ptr_next:=ptr; ctx.ptr_next:=ptr;
@ -856,6 +949,10 @@ begin
if (cb=nil) then if (cb=nil) then
begin begin
Writeln('original------------------------':32,' ','0x',HexStr(ptr-adec.Disassembler.CodeIdx));
Writeln(ACodeBytes:32,' ',ACode);
Writeln('original------------------------':32,' ','0x',HexStr(ptr));
Writeln('Unhandled jit:', Writeln('Unhandled jit:',
ctx.din.OpCode.Prefix,' ', ctx.din.OpCode.Prefix,' ',
ctx.din.OpCode.Opcode,' ', ctx.din.OpCode.Opcode,' ',
@ -875,6 +972,7 @@ begin
node_code2:=ctx.builder.get_curr_label._node; node_code2:=ctx.builder.get_curr_label._node;
{
if (node_code1<>node_code2) and if (node_code1<>node_code2) and
(node_code1<>nil) then (node_code1<>nil) then
begin begin
@ -891,8 +989,8 @@ begin
end; end;
Writeln('recompiled----------------------':32,' ',''); Writeln('recompiled----------------------':32,' ','');
end; end;
}
//if (node_code1<>node_code2) then
begin begin
ctx.add_label(ctx.ptr_curr,node_curr); ctx.add_label(ctx.ptr_curr,node_curr);
end; end;
@ -903,7 +1001,12 @@ begin
links.root:=nil; links.root:=nil;
end; end;
//if (len1<>len2) then if (entry_link<>nil) then
begin
ctx.add_entry_point(entry_link,node_curr);
entry_link:=nil;
end;
begin begin
node_new:=ctx.find_label(ptr); node_new:=ctx.find_label(ptr);
@ -911,7 +1014,7 @@ begin
begin begin
ctx.builder.jmp(node_new); ctx.builder.jmp(node_new);
ctx.ptr_next:=nil; ctx.ptr_next:=nil;
Writeln('jmp next:0x',HexStr(ptr)); //Writeln('jmp next:0x',HexStr(ptr));
end; end;
end; end;
@ -923,39 +1026,63 @@ begin
if not ctx.fetch_forward_point(links,addr) then if not ctx.fetch_forward_point(links,addr) then
begin begin
ctx.builder.ud2; goto _build;
data:=AllocMem(ctx.builder.GetMemSize);
ctx.builder.SaveTo(data,ctx.builder.GetMemSize);
F:=FileCreate('recompile.bin');
FileWrite(F,data^,ctx.builder.GetMemSize);
FileClose(F);
Assert(false);
end; end;
node_new:=ctx.find_label(addr); node_new:=ctx.find_label(addr);
if (node_new=nil_link) then if (node_new=nil_link) then
begin begin
Writeln('not found:0x',HexStr(addr)); //Writeln('not found:0x',HexStr(addr));
writeln;
Break; Break;
end else end else
begin begin
links.Resolve(node_new); links.Resolve(node_new);
links.root:=nil; links.root:=nil;
//
ctx.add_entry_point(addr,node_new);
end; end;
until false; until false;
ctx.builder.call_far(@jit_before_start);
entry_link:=addr;
ptr:=addr; ptr:=addr;
end; end;
end; end;
_build:
//build blob
ctx.builder.ud2;
blob:=new_blob(ctx.builder.GetMemSize);
ctx.builder.SaveTo(blob^.base,ctx.builder.GetMemSize);
//F:=FileCreate('recompile.bin');
//FileWrite(F,data^,ctx.builder.GetMemSize);
//FileClose(F);
//copy entrys
entry_point:=ctx.entry_list;
while (entry_point<>nil) do
begin
addr:=blob^.base+entry_point^.label_id.offset;
//
blob^.add_entry_point(entry_point^.src,addr);
//
entry_point:=entry_point^.next;
end;
blob^.attach;
adec.Free; adec.Free;
proc.Free; proc.Free;
ctx.builder.Free;
end; end;

38
sys/test/kern_jit2_ctx.pas
View File

@ -68,9 +68,18 @@ type
function c(n1,n2:p_label):Integer; static; function c(n1,n2:p_label):Integer; static;
end; end;
t_label_set=specialize TNodeSplay<t_label>; t_label_set=specialize TNodeSplay<t_label>;
p_entry_point=^t_entry_point;
t_entry_point=object
next :p_entry_point;
src :Pointer;
label_id:t_jit_i_link;
end;
var var
forward_set:t_forward_set; forward_set:t_forward_set;
label_set :t_label_set; label_set :t_label_set;
entry_list :p_entry_point;
text_start:QWORD; text_start:QWORD;
text___end:QWORD; text___end:QWORD;
@ -86,6 +95,7 @@ type
builder:t_jit_builder; builder:t_jit_builder;
function is_curr_addr(addr:QWORD):Boolean;
procedure add_forward_link(node:p_forward_point;label_id:t_jit_i_link); procedure add_forward_link(node:p_forward_point;label_id:t_jit_i_link);
function add_forward_point(label_id:t_jit_i_link;dst:Pointer):p_forward_point; function add_forward_point(label_id:t_jit_i_link;dst:Pointer):p_forward_point;
function add_forward_point(dst:Pointer):p_forward_point; function add_forward_point(dst:Pointer):p_forward_point;
@ -93,6 +103,7 @@ type
function fetch_forward_point(var links:t_forward_links;var dst:Pointer):Boolean; function fetch_forward_point(var links:t_forward_links;var dst:Pointer):Boolean;
function add_label(src:Pointer;label_id:t_jit_i_link):p_label; function add_label(src:Pointer;label_id:t_jit_i_link):p_label;
function find_label(src:Pointer):t_jit_i_link; function find_label(src:Pointer):t_jit_i_link;
procedure add_entry_point(src:Pointer;label_id:t_jit_i_link);
end; end;
const const
@ -255,6 +266,11 @@ begin
Result:=Integer(n1^.src>n2^.src)-Integer(n1^.src<n2^.src); Result:=Integer(n1^.src>n2^.src)-Integer(n1^.src<n2^.src);
end; end;
function t_jit_context2.is_curr_addr(addr:QWORD):Boolean;
begin
Result:=(addr>=text_start) and (addr<text___end);
end;
procedure t_jit_context2.add_forward_link(node:p_forward_point;label_id:t_jit_i_link); procedure t_jit_context2.add_forward_link(node:p_forward_point;label_id:t_jit_i_link);
var var
link:p_forward_link; link:p_forward_link;
@ -351,6 +367,19 @@ begin
Result:=entry^.label_id; Result:=entry^.label_id;
end; end;
procedure t_jit_context2.add_entry_point(src:Pointer;label_id:t_jit_i_link);
var
node:p_entry_point;
begin
if (src=nil) then Exit;
node:=builder.Alloc(Sizeof(t_entry_point));
node^.next :=entry_list;
node^.src :=src;
node^.label_id:=label_id;
//
entry_list:=node;
end;
// //
function GetFrameOffset(const RegValue:TRegValue):Integer; function GetFrameOffset(const RegValue:TRegValue):Integer;
@ -657,8 +686,9 @@ asm
shr PAGE_SHIFT ,%rax shr PAGE_SHIFT ,%rax
and PAGE_MAP_MASK,%rax and PAGE_MAP_MASK,%rax
//uplift (rax) //uplift (rax)
mov PAGE_MAP,%rax lea (,%rax,4),%rax
mov (%rax,%rax,4),%edi add PAGE_MAP(%rip),%rax
mov (%rax),%eax
//filter (rax) //filter (rax)
and PAGE_OFS_MASK,%rax and PAGE_OFS_MASK,%rax
jz _exit jz _exit
@ -696,8 +726,8 @@ asm
lea (,%rdi,4),%rdi lea (,%rdi,4),%rdi
lea (,%rsi,4),%rsi lea (,%rsi,4),%rsi
// //
add PAGE_MAP,%rdi add PAGE_MAP(%rip),%rdi
add PAGE_MAP,%rsi add PAGE_MAP(%rip),%rsi
// //
mov (%rdi),%edi mov (%rdi),%edi
mov (%rsi),%esi mov (%rsi),%esi

278
sys/test/kern_jit_dynamic.pas Normal file
View File

@ -0,0 +1,278 @@
unit kern_jit_dynamic;
{$mode ObjFPC}{$H+}
{$CALLING SysV_ABI_CDecl}
interface
uses
kern_jit2_ctx,
hamt,
kern_rwlock,
kern_thr;
{
entry_point -> +----------+ +---------+
|code_chunk| -> |code_blob|
entry_point -> | | | |
+----------+ | |
entry_point -> +----------+ | |
|code_chunk| -> | |
entry_point -> | | | |
+----------+ +---------+
}
type
p_jit_dynamic=^t_jit_dynamic;
t_jit_dynamic=object
type
p_entry_point=^t_entry_point;
t_entry_point=object
next:p_entry_point;
blob:p_jit_dynamic;
src :Pointer;
dst :Pointer;
procedure inc_ref;
procedure dec_ref;
end;
var
entry_list:p_entry_point;
base:Pointer;
size:ptruint;
refs:Integer;
procedure inc_ref;
procedure dec_ref;
procedure Free;
procedure add_entry_point(src,dst:Pointer);
procedure alloc_base(_size:ptruint);
procedure free_base;
procedure attach;
procedure detach;
end;
p_jit_thr_ctx=^t_jit_thr_ctx;
t_jit_thr_ctx=object
frame:jit_frame;
cblob:p_jit_dynamic;
end;
function new_blob(_size:ptruint):p_jit_dynamic;
var
entry_lock:Pointer=nil;
entry_hamt:TSTUB_HAMT64;
function fetch_entry(src:Pointer):t_jit_dynamic.p_entry_point;
procedure switch_to_jit();
implementation
uses
vmparam,
vm_pmap;
procedure switch_to_jit();
var
td:p_kthread;
node:t_jit_dynamic.p_entry_point;
jctx:p_jit_thr_ctx;
begin
td:=curkthread;
if (td=nil) then Exit;
if not is_guest_addr(td^.td_frame.tf_rip) then
begin
Assert(False);
end;
node:=fetch_entry(Pointer(td^.td_frame.tf_rip));
if (node=nil) then
begin
Writeln('tf_rip:0x',HexStr(td^.td_frame.tf_rip,16));
Assert(False);
end;
if (td^.td_jit_ctx=nil) then
begin
td^.td_jit_ctx:=AllocMem(SizeOf(t_jit_thr_ctx));
end;
jctx:=td^.td_jit_ctx;
if (jctx^.cblob<>nil) then
begin
jctx^.cblob^.dec_ref;
jctx^.cblob:=nil;
end;
jctx^.cblob:=node^.blob;
jctx^.frame.tf_rax:=td^.td_frame.tf_rax;
jctx^.frame.tf_rsp:=td^.td_frame.tf_rsp;
jctx^.frame.tf_rbp:=td^.td_frame.tf_rbp;
jctx^.frame.tf_r14:=td^.td_frame.tf_r14;
jctx^.frame.tf_r15:=td^.td_frame.tf_r15;
td^.td_frame.tf_rsp:=QWORD(td^.td_kstack.stack);
td^.td_frame.tf_rbp:=QWORD(td^.td_kstack.stack);
td^.td_frame.tf_rip:=QWORD(node^.dst);
td^.td_frame.tf_r15:=QWORD(jctx);
end;
function fetch_entry(src:Pointer):t_jit_dynamic.p_entry_point;
var
data:PPointer;
begin
Result:=nil;
rw_rlock(entry_lock);
data:=HAMT_search64(@entry_hamt,QWORD(src));
if (data<>nil) then
begin
Result:=data^;
end;
if (Result<>nil) then
begin
Result^.inc_ref;
end;
rw_runlock(entry_lock);
end;
//
function new_blob(_size:ptruint):p_jit_dynamic;
begin
Result:=AllocMem(SizeOf(t_jit_dynamic));
Result^.alloc_base(_size);
end;
//
procedure t_jit_dynamic.t_entry_point.inc_ref;
begin
blob^.inc_ref;
end;
procedure t_jit_dynamic.t_entry_point.dec_ref;
begin
blob^.dec_ref;
end;
//
procedure t_jit_dynamic.inc_ref;
begin
System.InterlockedIncrement(refs);
end;
procedure t_jit_dynamic.dec_ref;
begin
if (System.InterlockedDecrement(refs)=0) then
begin
Free;
end;
end;
procedure t_jit_dynamic.Free;
var
node,next:p_entry_point;
begin
node:=entry_list;
while (node<>nil) do
begin
next:=node^.next;
FreeMem(node);
node:=next;
end;
free_base;
FreeMem(@Self);
end;
procedure t_jit_dynamic.add_entry_point(src,dst:Pointer);
var
node:p_entry_point;
begin
if (src=nil) or (dst=nil) then Exit;
node:=AllocMem(Sizeof(t_entry_point));
node^.next:=entry_list;
node^.blob:=@Self;
node^.src :=src;
node^.dst :=dst;
//
entry_list:=node;
end;
procedure t_jit_dynamic.alloc_base(_size:ptruint);
begin
base:=md_mmap(nil,_size,MD_PROT_RWX);
size:=_size;
end;
procedure t_jit_dynamic.free_base;
begin
md_unmap(base,size);
base:=nil;
size:=0;
end;
procedure t_jit_dynamic.attach;
var
node,next:p_entry_point;
begin
rw_wlock(entry_lock);
node:=entry_list;
while (node<>nil) do
begin
next:=node^.next;
node^.inc_ref;
HAMT_insert64(@entry_hamt,QWORD(node^.src),node);
node:=next;
end;
rw_wunlock(entry_lock);
end;
procedure t_jit_dynamic.detach;
var
node,next:p_entry_point;
begin
rw_wlock(entry_lock);
node:=entry_list;
while (node<>nil) do
begin
next:=node^.next;
HAMT_delete64(@entry_hamt,QWORD(node^.src),nil);
node^.dec_ref;
node:=next;
end;
rw_wunlock(entry_lock);
end;
end.

4
sys/test/project1.lpi
View File

@ -758,6 +758,10 @@
<Filename Value="kern_jit2_ops_avx.pas"/> <Filename Value="kern_jit2_ops_avx.pas"/>
<IsPartOfProject Value="True"/> <IsPartOfProject Value="True"/>
</Unit> </Unit>
<Unit>
<Filename Value="kern_jit_dynamic.pas"/>
<IsPartOfProject Value="True"/>
</Unit>
</Units> </Units>
</ProjectOptions> </ProjectOptions>
<CompilerOptions> <CompilerOptions>