diff --git a/rtl/ntapi.pas b/rtl/ntapi.pas index 51a66885..f784fda8 100644 --- a/rtl/ntapi.pas +++ b/rtl/ntapi.pas @@ -891,7 +891,7 @@ function NtMapViewOfSection( function NtUnmapViewOfSection( ProcessHandle :THandle; - BaseAddress :PPointer + BaseAddress :Pointer ):DWORD; stdcall; external 'ntdll'; function NtAllocateVirtualMemory( diff --git a/sys/jit/kern_jit_asm.pas b/sys/jit/kern_jit_asm.pas index 69d889d0..b6e0cd6a 100644 --- a/sys/jit/kern_jit_asm.pas +++ b/sys/jit/kern_jit_asm.pas @@ -153,7 +153,7 @@ asm //uplift (r14) shl $2,%r14 add PAGE_MAP(%rip),%r14 - mov (%r14),%r14d + movslq (%r14),%r14 //sign extend int32->int64 //high addr (r14) shl PAGE_SHIFT,%r14 //combine (r14+origin) diff --git a/sys/kern/kern_authinfo.pas b/sys/kern/kern_authinfo.pas index 65435f83..e0ad7404 100644 --- a/sys/kern/kern_authinfo.pas +++ b/sys/kern/kern_authinfo.pas @@ -39,7 +39,7 @@ const // _sceKernelMapDirectMemory // sceKernelMapDirectMemory2 - //excp_flags + //attributeExe //bit 1 -> use in [libkernel_exception] -> // -> sceKernelInstallExceptionHandler // -> sceKernelRemoveExceptionHandler diff --git a/sys/kern/kern_exec.pas b/sys/kern/kern_exec.pas index 0a46e4e9..32f0f3df 100644 --- a/sys/kern/kern_exec.pas +++ b/sys/kern/kern_exec.pas @@ -1452,6 +1452,8 @@ begin g_authinfo:=imgp^.authinfo; //copy appinfo (TODO before execve) + g_appinfo.AppId:=$60000100; + g_appinfo.mmap_flags:=g_appinfo.mmap_flags or 1; //is_big_app ??? if (p_proc.p_sce_replay_exec<>0) then begin diff --git a/sys/kern/kern_rtld.pas b/sys/kern/kern_rtld.pas index b0f228a0..eda81b4e 100644 --- a/sys/kern/kern_rtld.pas +++ b/sys/kern/kern_rtld.pas @@ -209,6 +209,7 @@ uses vmparam, vm_map, vm_mmap, + vm_pmap, md_map, kern_proc, kern_budget, @@ -1140,7 +1141,7 @@ begin if ((prot and VM_PROT_EXECUTE)<>0) then begin - md_cacheflush(Pointer(vaddr_lo),memsz,ICACHE); + md_cacheflush(uplift(Pointer(vaddr_lo)),memsz,ICACHE); end; vm_map_unlock(map); diff --git a/sys/md/md_map.pas b/sys/md/md_map.pas index e7fd48a1..ff8fcdab 100644 --- a/sys/md/md_map.pas +++ b/sys/md/md_map.pas @@ -152,27 +152,17 @@ end; function md_file_mmap(handle:THandle;var base:Pointer;offset,size:QWORD;prot:Integer):Integer; var hSection:THandle; - DesiredAccess:DWORD; CommitSize:ULONG_PTR; SectionOffset:ULONG_PTR; - ViewSize:ULONG_PTR; begin - - case prot of - PAGE_READWRITE :DesiredAccess:=SECTION_MAP_READ or SECTION_MAP_WRITE; - PAGE_EXECUTE :DesiredAccess:=SECTION_MAP_EXECUTE; - PAGE_EXECUTE_READ :DesiredAccess:=SECTION_MAP_EXECUTE or SECTION_MAP_READ; - PAGE_EXECUTE_READWRITE:DesiredAccess:=SECTION_MAP_EXECUTE or SECTION_MAP_READ or SECTION_MAP_WRITE; - else - DesiredAccess:=SECTION_MAP_READ; - end; + CommitSize:=0; //full size hSection:=0; Result:=NtCreateSection( @hSection, - DesiredAccess, + SECTION_MAP_WRITE or SECTION_MAP_READ or SECTION_MAP_EXECUTE, nil, - @size, + @CommitSize, prot, SEC_COMMIT, handle @@ -182,9 +172,8 @@ begin base:=md_alloc_page(base); - CommitSize :=md_up_page(size); + CommitSize:=size; SectionOffset:=offset and (not (MD_ALLOC_GRANULARITY-1)); - ViewSize :=CommitSize; Result:=NtMapViewOfSection(hSection, NtCurrentProcess, @@ -192,9 +181,9 @@ begin 0, CommitSize, @SectionOffset, - @ViewSize, + @CommitSize, ViewUnmap, - 0 {MEM_COMMIT}, + 0, prot ); @@ -205,7 +194,7 @@ function md_file_unmap(base:Pointer;size:QWORD):Integer; begin base:=md_alloc_page(base); - Result:=NtUnmapViewOfSection(NtCurrentProcess,@base); + Result:=NtUnmapViewOfSection(NtCurrentProcess,base); end; // diff --git a/sys/md/md_vnops.pas b/sys/md/md_vnops.pas index 76f4c89b..c8ed3bbd 100644 --- a/sys/md/md_vnops.pas +++ b/sys/md/md_vnops.pas @@ -9,6 +9,7 @@ uses windows, ntapi, mqueue, + vm_pmap, kern_param, time, vfile, @@ -2292,6 +2293,13 @@ type Key :PULONG ):DWORD; stdcall; + t_iov_cb=procedure(iov:p_iovec); + +procedure iov_null(iov:p_iovec); assembler; nostackframe; +asm + // +end; + function md_io(vp:p_vnode;uio:p_uio;ioflag:Integer):Integer; var td:p_kthread; @@ -2305,7 +2313,8 @@ var locked:Boolean; iov:p_iovec; - cnt:Integer; + vec:iovec; + iol:t_iov_cb; OFFSET:Int64; BLK:IO_STATUS_BLOCK; @@ -2330,6 +2339,12 @@ begin UIO_WRITE:iocb:=@NtWriteFile; end; + iol:=nil; + case uio^.uio_segflg of + UIO_USERSPACE:iol:=@iov_uplift; + UIO_SYSSPACE :iol:=@iov_null; + end; + append:=(uio^.uio_rw=UIO_WRITE) and ((ioflag and IO_APPEND)<>0); locked:=((ioflag and IO_UNIT)<>0) and (not append); @@ -2342,9 +2357,11 @@ begin while (uio^.uio_iovcnt<>0) or (uio^.uio_resid<>0) do begin iov:=uio^.uio_iov; - cnt:=iov^.iov_len; - if (cnt=0) then + vec:=iov^; + iol(@vec); + + if (vec.iov_len=0) then begin Inc(uio^.uio_iov); Dec(uio^.uio_iovcnt); @@ -2361,7 +2378,7 @@ begin BLK:=Default(IO_STATUS_BLOCK); - R:=iocb(F,0,nil,nil,@BLK,iov^.iov_base,cnt,@OFFSET,nil); + R:=iocb(F,0,nil,nil,@BLK,vec.iov_base,vec.iov_len,@OFFSET,nil); if (R=STATUS_PENDING) then begin @@ -2372,19 +2389,19 @@ begin System.InterlockedIncrement64(ioin^); - if (Int64(BLK.Information)0) then + vec.iov_len:=BLK.Information; + if (vec.iov_len<>0) then begin - Inc(iov^.iov_base ,cnt); - Dec(iov^.iov_len ,cnt); - Dec(uio^.uio_resid ,cnt); + Inc(iov^.iov_base ,vec.iov_len); + Dec(iov^.iov_len ,vec.iov_len); + Dec(uio^.uio_resid ,vec.iov_len); if not append then begin - Inc(uio^.uio_offset,cnt); + Inc(uio^.uio_offset,vec.iov_len); end; end; Break; @@ -2392,17 +2409,20 @@ begin if (Result<>0) then Break; - Inc(iov^.iov_base ,cnt); - Dec(iov^.iov_len ,cnt); - Dec(uio^.uio_resid ,cnt); + Inc(iov^.iov_base ,vec.iov_len); + Dec(iov^.iov_len ,vec.iov_len); + Dec(uio^.uio_resid ,vec.iov_len); if not append then begin - Inc(uio^.uio_offset,cnt); + Inc(uio^.uio_offset,vec.iov_len); end; - Inc(uio^.uio_iov); - Dec(uio^.uio_iovcnt); + if (iov^.iov_len=0) then + begin + Inc(uio^.uio_iov); + Dec(uio^.uio_iovcnt); + end; end; if locked then diff --git a/sys/md/vm_pmap.pas b/sys/md/vm_pmap.pas index 3342f828..d5fc218c 100644 --- a/sys/md/vm_pmap.pas +++ b/sys/md/vm_pmap.pas @@ -10,7 +10,8 @@ uses vmparam, sys_vm_object, vm_file, - vnode; + vnode, + vuio; const PAGE_MAP_COUNT =(QWORD(VM_MAXUSER_ADDRESS) shr PAGE_SHIFT); @@ -38,6 +39,7 @@ function pmap_get_page (addr:vm_offset_t):DWORD; function pmap_test_cross(addr:vm_offset_t;h:Integer):Boolean; function uplift(addr:Pointer):Pointer; +procedure iov_uplift(iov:p_iovec); type p_pmap=^_pmap; @@ -132,7 +134,7 @@ begin if (r<>0) then begin - Writeln('failed md_reserve(',HexStr(base),',',HexStr(base+size),'):',HexStr(r,8)); + Writeln('failed md_reserve(',HexStr(base),',',HexStr(base+size),'):0x',HexStr(r,8)); //STATUS_COMMITMENT_LIMIT = $C000012D Assert(false,'pmap_init'); end; @@ -150,7 +152,7 @@ begin if (r<>0) then begin - Writeln('failed md_mmap(',HexStr(PAGE_MAP),',',HexStr(PAGE_MAP+size),'):',HexStr(r,8)); + Writeln('failed md_mmap(',HexStr(PAGE_MAP),',',HexStr(PAGE_MAP+size),'):0x',HexStr(r,8)); Assert(false,'pmap_init'); end; @@ -302,7 +304,7 @@ asm ja _exit //uplift (rdi) mov PAGE_MAP(%rip),%rax - mov (%rax,%rdi,4) ,%edi + movslq (%rax,%rdi,4),%rdi //sign extend int32->int64 //high addr (rdi) shl PAGE_SHIFT,%rdi //combine (rdi+rsi) @@ -317,6 +319,48 @@ asm xor %eax,%eax end; +procedure iov_uplift(iov:p_iovec); +var + base:Ptruint; + len:Ptruint; + i,p:DWORD; + v:Integer; +begin + base:=Ptruint(iov^.iov_base); + len:=iov^.iov_len; + + p:=base shr PAGE_SHIFT; + + if (p>=PAGE_MAP_COUNT) then + begin + //error + iov^.iov_base:=nil; + Exit; + end; + + i:=base and PAGE_MASK; + i:=PAGE_SIZE-i; + if (i>len) then i:=len; + Dec(len,i); + + v:=PAGE_MAP[p]; + + while (len<>0) and (pPAGE_MAP[p]) then Break; + + i:=PAGE_SIZE; + if (i>len) then i:=len; + Dec(len,i); + end; + + base:=base+(int64(v) shl PAGE_MASK); + + iov^.iov_base:=Pointer(base); + Dec(iov^.iov_len,len); +end; + const VM_RWX=VM_PROT_READ or VM_PROT_WRITE or VM_PROT_EXECUTE; @@ -349,6 +393,30 @@ begin end; end; +function vm_file_protect(start,__end:QWORD;base:Pointer;size:QWORD;prot:Integer):Integer; +begin + Result:=md_protect(base,size,wprots[prot and VM_RWX]); + + if (Result<>0) then + begin + Writeln('failed md_protect:0x',HexStr(Result,8)); + Assert(false,'md_protect'); + end; + + pmap_mark(start,__end,QWORD(base),prot); +end; + +function vm_file_unmap(base:Pointer;size:QWORD):Integer; +begin + Result:=md_file_unmap(base,size); + + if (Result<>0) then + begin + Writeln('failed md_file_unmap:0x',HexStr(Result,8)); + Assert(false,'md_file_unmap'); + end; +end; + function vm_file_map_fixed(map :p_vm_file_map; offset:vm_ooffset_t; start :vm_offset_t; @@ -360,7 +428,10 @@ var begin vm_file_map_delete(map,start,__end); - obj:=vm_file_obj_allocate(base,size,@md_file_unmap); + obj:=vm_file_obj_allocate(base,size); + + obj^.protect:=@vm_file_protect; + obj^.unmap :=@vm_file_unmap; Result:=vm_file_map_insert(map,obj,offset,start,__end); end; @@ -424,8 +495,11 @@ begin r:=md_enter(base,size,MD_PROT_RWX); end; end; - OBJT_VNODE: + OBJT_VNODE: begin + base:=nil; + delta:=0; + VM_OBJECT_LOCK(obj); fd:=get_vnode_handle(obj); @@ -447,8 +521,21 @@ begin base:=nil; r:=md_file_mmap(fd,base,offset,size,wprots[prot and VM_RWX]); + if (r<>0) then + begin + Writeln('failed md_file_mmap:0x',HexStr(r,8)); + Assert(false,'md_file_mmap'); + end; + + if (r=0) and (delta<>0) then + begin + md_protect(base,delta,MD_PROT_NONE); + end; + if (r=0) then begin + size:=(size+(MD_PAGE_SIZE-1)) and (not (MD_PAGE_SIZE-1)); + r:=vm_file_map_fixed(@obj^.un_pager.vnp.file_map, delta, start, @@ -460,6 +547,18 @@ begin end; VM_OBJECT_UNLOCK(obj); + + if (r<>0) then + begin + Writeln('failed vm_file_map_fixed:0x',HexStr(r,8)); + Assert(false,'vm_file_map_fixed'); + end; + + if (r=0) then + begin + base:=base+delta; + end; + end; else begin @@ -471,7 +570,7 @@ begin if (r<>0) then begin - Writeln('failed md_enter:',HexStr(r,8)); + Writeln('failed md_enter:0x',HexStr(r,8)); Assert(false,'pmap_enter_object'); end; @@ -494,7 +593,7 @@ begin if (r<>0) then begin - Writeln('failed md_protect:',HexStr(r,8)); + Writeln('failed md_protect:0x',HexStr(r,8)); Assert(false,'pmap_move'); end; @@ -503,7 +602,7 @@ begin if (r<>0) then begin - Writeln('failed md_enter:',HexStr(r,8)); + Writeln('failed md_enter:0x',HexStr(r,8)); Assert(false,'pmap_move'); end; @@ -517,7 +616,7 @@ begin if (r<>0) then begin - Writeln('failed md_protect:',HexStr(r,8)); + Writeln('failed md_protect:0x',HexStr(r,8)); Assert(false,'pmap_move'); end; end; @@ -529,7 +628,7 @@ begin if (r<>0) then begin - Writeln('failed md_remove:',HexStr(r,8)); + Writeln('failed md_remove:0x',HexStr(r,8)); Assert(false,'pmap_move'); end; end; @@ -579,6 +678,25 @@ begin r:=md_protect(base,size,MD_PROT_RWX); end; end; + OBJT_VNODE: + begin + VM_OBJECT_LOCK(obj); + + r:=vm_file_map_protect(@obj^.un_pager.vnp.file_map, + start, + __end, + prot); + + VM_OBJECT_UNLOCK(obj); + + if (r<>0) then + begin + Writeln('failed vm_file_map_protect:0x',HexStr(r,8)); + Assert(false,'vm_file_map_protect'); + end; + + Exit; + end; else begin Writeln('TODO:',vm_object_type(obj)); @@ -589,7 +707,7 @@ begin if (r<>0) then begin - Writeln('failed md_protect:',HexStr(r,8)); + Writeln('failed md_protect:0x',HexStr(r,8)); Assert(false,'pmap_protect'); end; @@ -636,6 +754,10 @@ begin begin //ignore end; + OBJT_VNODE: + begin + //ignore + end; else begin Writeln('TODO:',vm_object_type(obj)); @@ -646,7 +768,7 @@ begin if (r<>0) then begin - Writeln('failed md_reset:',HexStr(r,8)); + Writeln('failed md_reset:0x',HexStr(r,8)); Assert(false,'pmap_madv_free'); end; end; @@ -698,6 +820,24 @@ begin r:=md_remove(base,size); end; end; + OBJT_VNODE: + begin + VM_OBJECT_LOCK(obj); + + r:=vm_file_map_delete(@obj^.un_pager.vnp.file_map, + start, + __end); + + VM_OBJECT_UNLOCK(obj); + + if (r<>0) then + begin + Writeln('failed vm_file_map_delete:0x',HexStr(r,8)); + Assert(false,'vm_file_map_delete'); + end; + + Exit; + end; else begin Writeln('TODO:',vm_object_type(obj)); @@ -708,7 +848,7 @@ begin if (r<>0) then begin - Writeln('failed md_remove:',HexStr(r,8)); + Writeln('failed md_remove:0x',HexStr(r,8)); Assert(false,'pmap_remove'); end; end; diff --git a/sys/vm/vm_file.pas b/sys/vm/vm_file.pas index a01b79b1..e9452990 100644 --- a/sys/vm/vm_file.pas +++ b/sys/vm/vm_file.pas @@ -10,14 +10,16 @@ uses vmparam; type - t_uncb=function(base:Pointer;size:QWORD):Integer; + t_protect_cb=function(start,__end:QWORD;base:Pointer;size:QWORD;prot:Integer):Integer; + t_unmap_cb =function(base:Pointer;size:QWORD):Integer; p_vm_file_obj=^vm_file_obj; vm_file_obj=packed record - base:Pointer; - size:QWORD; - refs:QWORD; - uncb:t_uncb; + base :Pointer; + size :QWORD; + refs :QWORD; + protect:t_protect_cb; + unmap :t_unmap_cb; end; pp_vm_file_entry=^p_vm_file_entry; @@ -35,15 +37,11 @@ type p_vm_file_map=^vm_file_map; vm_file_map=packed object - header :vm_file_entry; - root :p_vm_file_entry; - nentries:DWORD; - pages :DWORD; - property min_offset:QWORD read header.start write header.start; - property max_offset:QWORD read header.__end write header.__end; + header:vm_file_entry; + root :p_vm_file_entry; end; -function vm_file_obj_allocate (base:Pointer;size:QWORD;uncb:t_uncb):p_vm_file_obj; +function vm_file_obj_allocate (base:Pointer;size:QWORD):p_vm_file_obj; procedure vm_file_obj_destroy (obj:p_vm_file_obj); procedure vm_file_obj_reference (obj:p_vm_file_obj); procedure vm_file_obj_deallocate(obj:p_vm_file_obj); @@ -63,7 +61,12 @@ function vm_file_map_insert( start :QWORD; __end :QWORD):Integer; -function vm_file_map_delete(map:p_vm_file_map; +function vm_file_map_protect(map :p_vm_file_map; + start:QWORD; + __end:QWORD; + prot :Integer):Integer; + +function vm_file_map_delete(map :p_vm_file_map; start:QWORD; __end:QWORD):Integer; @@ -74,21 +77,20 @@ begin Result:=QWORD(x) shr PAGE_SHIFT; end; -function vm_file_obj_allocate(base:Pointer;size:QWORD;uncb:t_uncb):p_vm_file_obj; +function vm_file_obj_allocate(base:Pointer;size:QWORD):p_vm_file_obj; begin Result:=AllocMem(SizeOf(vm_file_obj)); Result^.base:=base; Result^.size:=size; Result^.refs:=1; - Result^.uncb:=uncb; end; procedure vm_file_obj_destroy(obj:p_vm_file_obj); begin - if (obj^.uncb<>nil) then + if (obj^.unmap<>nil) then begin - obj^.uncb(obj^.base,obj^.size); + obj^.unmap(obj^.base,obj^.size); end; FreeMem(obj); @@ -113,18 +115,16 @@ end; procedure vm_file_map_init(map:p_vm_file_map;min,max:QWORD); begin - map^.header.next:=@map^.header; - map^.header.prev:=@map^.header; - map^.min_offset :=min; - map^.max_offset :=max; - map^.root :=nil; - map^.nentries :=0; - map^.pages :=0; + map^.header.next :=@map^.header; + map^.header.prev :=@map^.header; + map^.header.start:=min; + map^.header.__end:=max; + map^.root :=nil; end; procedure vm_file_map_free(map:p_vm_file_map); begin - vm_file_map_delete(map,map^.min_offset,map^.max_offset); + vm_file_map_delete(map,map^.header.start,map^.header.__end); end; procedure vm_file_entry_dispose(map:p_vm_file_map;entry:p_vm_file_entry); inline; @@ -223,8 +223,6 @@ procedure vm_file_map_entry_link( after_where:p_vm_file_entry; entry :p_vm_file_entry); begin - Inc(map^.nentries); - entry^.prev:=after_where; entry^.next:=after_where^.next; entry^.next^.prev:=entry; @@ -271,7 +269,6 @@ begin next:=entry^.next; next^.prev:=prev; prev^.next:=next; - Dec(map^.nentries); end; function vm_file_map_lookup_entry( @@ -347,19 +344,15 @@ begin new_entry^.obj :=obj; vm_file_map_entry_link(map, prev_entry, new_entry); - map^.pages:=map^.pages+OFF_TO_IDX(new_entry^.__end - new_entry^.start); end; procedure vm_file_map_entry_delete(map:p_vm_file_map;entry:p_vm_file_entry); begin vm_file_map_entry_unlink(map, entry); - - map^.pages:=map^.pages-OFF_TO_IDX(entry^.__end - entry^.start); - vm_file_entry_dispose(map,entry); end; -procedure _vm_map_clip_start(map:p_vm_file_map;entry:p_vm_file_entry;start:QWORD); +procedure _vm_file_map_clip_start(map:p_vm_file_map;entry:p_vm_file_entry;start:QWORD); var new_entry:p_vm_file_entry; begin @@ -375,15 +368,15 @@ begin vm_file_obj_reference(new_entry^.obj); end; -procedure vm_map_clip_start(map:p_vm_file_map;entry:p_vm_file_entry;start:QWORD); +procedure vm_file_map_clip_start(map:p_vm_file_map;entry:p_vm_file_entry;start:QWORD); begin if (start>entry^.start) then begin - _vm_map_clip_start(map,entry,start); + _vm_file_map_clip_start(map,entry,start); end; end; -procedure _vm_map_clip_end(map:p_vm_file_map;entry:p_vm_file_entry;__end:QWORD); +procedure _vm_file_map_clip_end(map:p_vm_file_map;entry:p_vm_file_entry;__end:QWORD); var new_entry:p_vm_file_entry; begin @@ -399,15 +392,80 @@ begin vm_file_obj_reference(new_entry^.obj); end; -procedure vm_map_clip_end(map:p_vm_file_map;entry:p_vm_file_entry;__end:QWORD); +procedure vm_file_map_clip_end(map:p_vm_file_map;entry:p_vm_file_entry;__end:QWORD); begin if (__end@map^.header) and (current^.start<__end)) do + begin + obj:=current^.obj; + + if (obj^.protect<>nil) then + begin + + m_start:=current^.start; + m___end:=current^.__end; + + if (start>m_start) then + begin + offset :=start-m_start; + m_start:=start; + end else + begin + offset:=0; + end; + + if (m___end>__end) then + begin + m___end:=__end; + end; + + offset:=offset+current^.offset; + + size:=(m___end-m_start)+offset; + if (size>obj^.size) then + begin + size:=obj^.size; + end; + size:=size-offset; + + obj^.protect(m_start,m___end,obj^.base+offset,size,prot); + end; + + current:=current^.next; + end; + + Result:=(KERN_SUCCESS); +end; + +function vm_file_map_delete(map :p_vm_file_map; start:QWORD; __end:QWORD):Integer; var @@ -427,12 +485,12 @@ begin begin entry:=first_entry; - vm_map_clip_start(map, entry, start); + vm_file_map_clip_start(map, entry, start); end; while (entry<>@map^.header) and (entry^.start<__end) do begin - vm_map_clip_end(map, entry, __end); + vm_file_map_clip_end(map, entry, __end); next:=entry^.next; vm_file_map_entry_delete(map, entry); entry:=next; @@ -442,3 +500,4 @@ end; end. +