From 2dd09a7e6fb27f892d51a2a950498bda2a4ccdba Mon Sep 17 00:00:00 2001 From: Pavel <68122101+red-prig@users.noreply.github.com> Date: Wed, 2 Aug 2023 16:53:09 +0300 Subject: [PATCH] + --- sys/dev/dev_dmem.pas | 18 +- sys/fs/devfs/devfs_devs.pas | 6 + sys/kern/kern_dmem.pas | 85 +++- sys/kern/kern_regmgr.pas | 14 +- sys/kern/kern_reloc.pas | 24 +- sys/kern/kern_rtld.pas | 12 +- sys/sys_mmap.pas | 6 +- sys/sys_sysinit.pas | 2 + sys/syscalls.pas | 8 + sys/sysent.pas | 2 +- sys/test/project1.lpi | 4 + sys/vfs/kern_conf.pas | 22 + sys/vfs/kern_descrip.pas | 4 + sys/vm/dmem_map.pas | 956 ++++++++++++++++++++++++++++++++++++ sys/vm/vm.pas | 6 +- sys/vm/vm_map.pas | 98 +++- 16 files changed, 1226 insertions(+), 41 deletions(-) create mode 100644 sys/vm/dmem_map.pas diff --git a/sys/dev/dev_dmem.pas b/sys/dev/dev_dmem.pas index b9664253..ff62e6c7 100644 --- a/sys/dev/dev_dmem.pas +++ b/sys/dev/dev_dmem.pas @@ -17,10 +17,17 @@ uses vuio, subr_uio, vm, + dmem_map, kern_dmem; -Const - SCE_KERNEL_MAIN_DMEM_SIZE=$180000000; //6GB +type + PAvailableDirectMemorySize=^TAvailableDirectMemorySize; + TAvailableDirectMemorySize=packed record + start:QWORD; //in,out + __end:QWORD; //in + align:QWORD; //in + osize:QWORD; //out + end; Function dmem_ioctl(dev:p_cdev;cmd:QWORD;data:Pointer;fflag:Integer):Integer; begin @@ -33,6 +40,13 @@ begin begin PQWORD(data)^:=SCE_KERNEL_MAIN_DMEM_SIZE; end; + $C0208016: //sceKernelAvailableDirectMemorySize + begin + with PAvailableDirectMemorySize(data)^ do + begin + Result:=dmem_map_query_available(@dmem,start,__end,align,start,osize); + end; + end; else Assert(False); end; diff --git a/sys/fs/devfs/devfs_devs.pas b/sys/fs/devfs/devfs_devs.pas index f2ead96f..e0e135ab 100644 --- a/sys/fs/devfs/devfs_devs.pas +++ b/sys/fs/devfs/devfs_devs.pas @@ -281,7 +281,9 @@ begin { Exit if the directory is not empty. } if (TAILQ_NEXT(de_dotdot,@de_dotdot^.de_list)<>nil) then + begin Exit; + end; dd:=devfs_parent_dirent(de); Assert(dd<>nil, 'devfs_rmdir_empty: nil dd'); @@ -329,7 +331,9 @@ begin devfs_dir_unref_de(dm, dd); end; end else + begin dd:=nil; + end; mtx_lock(devfs_de_interlock); vp:=de^.de_vnode; @@ -353,7 +357,9 @@ begin vdrop(vp); sx_xlock(@dm^.dm_lock); end else + begin mtx_unlock(devfs_de_interlock); + end; if (de^.de_symlink<>nil) then begin diff --git a/sys/kern/kern_dmem.pas b/sys/kern/kern_dmem.pas index 1d730b5a..1108df43 100644 --- a/sys/kern/kern_dmem.pas +++ b/sys/kern/kern_dmem.pas @@ -5,13 +5,35 @@ unit kern_dmem; interface +uses + dmem_map; + +var + dmem:t_dmem_map; + +procedure init_dmem_map; + function sys_dmem_container(d_pool_id:Integer):Integer; +// +function sys_blockpool_open(flags:Integer):Integer; implementation uses errno, - kern_thr; + kern_thr, + kern_descrip, + vfile, + vfcntl, + kern_conf, + vstat; + +////////// + +procedure init_dmem_map; +begin + dmem_map_init(@dmem,0,SCE_KERNEL_MAIN_DMEM_SIZE); +end; const default_pool_id=1; @@ -34,6 +56,67 @@ begin end; end; +//////////// + +function blockpool_ioctl(fp:p_file;com:QWORD;data:Pointer):Integer; +begin + Assert(False); + Result:=0; +end; + +function blockpool_stat(fp:p_file;sb:p_stat):Integer; +begin + Assert(False); + Result:=0; +end; + +function blockpool_close(fp:p_file):Integer; +begin + Assert(False); + Result:=0; +end; + +const + blockpool_ops:fileops=( + fo_read :fo_rdwr_t(@_enxio); + fo_write :fo_rdwr_t(@_enxio); + fo_truncate:fo_truncate_t(@_enxio); + fo_ioctl :@blockpool_ioctl; + fo_poll :fo_poll_t(@_eopnotsupp); + fo_kqfilter:fo_kqfilter_t(@_eopnotsupp); + fo_stat :@blockpool_stat; + fo_close :@blockpool_close; + fo_chmod :fo_chmod_t(@_einval); + fo_chown :fo_chown_t(@_einval); + fo_flags :0; + ); + +function sys_blockpool_open(flags:Integer):Integer; +var + td:p_kthread; + bp:Pointer; + fp:p_file; + fd:Integer; +begin + td:=curkthread; + if (td=nil) then Exit(-1); + //0x100000(O_CLOEXEC) | 0x400000(ASLR_FD????) + if ((flags and $ffafffff)<>0) then Exit(EINVAL); + + flags:=flags or FWRITE; + + fd:=0; + Result:=falloc(@fp,@fd,flags); + if (Result<>0) then Exit(); + + bp:=nil; ///// + + finit(fp, flags, DTYPE_BLOCKPOOL, bp, @blockpool_ops); + + fdrop(fp); + + td^.td_retval[0]:=fd; +end; end. diff --git a/sys/kern/kern_regmgr.pas b/sys/kern/kern_regmgr.pas index 71f36814..f77983b4 100644 --- a/sys/kern/kern_regmgr.pas +++ b/sys/kern/kern_regmgr.pas @@ -60,9 +60,19 @@ begin goto _err; end; - Writeln(' enc:0x',HexStr(qword(data.enc),16),' val1:0x',HexStr(data.val1,8),' val12:0x',HexStr(data.val2,8)); + case qword(data.enc) of + QWORD($0C82671ADF0EEB34):data.val2:=0; //_malloc_init_lv2 + QWORD($1AC46343411B3F40):data.val2:=0; //libSceSysmodule (bit 1,2 -> load debug lib) + QWORD($503F69BDE385A6AC):data.val2:=0; //libSceSysmodule (print errors?) + QWORD($2D946F62AEF8F878):data.val2:=0; //libSceSysmodule (preload module?) + QWORD($1BE26343C3D71F40):data.val2:=0; //libkernel + QWORD($68436EECF1CFD447):data.val2:=0; //libSceLibcInternal (sceLibcHeapGetTraceInfo -> get_segment_info) - data.val2:=0; + else + begin + Writeln(' enc:0x',HexStr(qword(data.enc),16),' val1:0x',HexStr(data.val1,8),' val12:0x',HexStr(data.val2,8)); + end; + end; Result:=copyout(@data,pvalue,vlen); if (Result<>0) then diff --git a/sys/kern/kern_reloc.pas b/sys/kern/kern_reloc.pas index bf1df91f..443fe9ef 100644 --- a/sys/kern/kern_reloc.pas +++ b/sys/kern/kern_reloc.pas @@ -173,7 +173,8 @@ begin Exit(ENOEXEC); end; - set_relo_bits(obj,i); + if (def<>@dynlibs_info.sym_nops) then + set_relo_bits(obj,i); end; //R_X86_64_DTPMOD64 R_X86_64_DTPOFF64: @@ -194,7 +195,8 @@ begin Exit(ENOEXEC); end; - set_relo_bits(obj,i); + if (def<>@dynlibs_info.sym_nops) then + set_relo_bits(obj,i); end; //R_X86_64_DTPOFF64 R_X86_64_TPOFF64: @@ -214,7 +216,8 @@ begin Exit(ENOEXEC); end; - set_relo_bits(obj,i); + if (def<>@dynlibs_info.sym_nops) then + set_relo_bits(obj,i); end; //R_X86_64_TPOFF64 else; @@ -260,7 +263,8 @@ begin Exit(ENOEXEC); end; - set_relo_bits(obj,i); + if (def<>@dynlibs_info.sym_nops) then + set_relo_bits(obj,i); end; //R_X86_64_PC32 R_X86_64_DTPOFF32: @@ -281,7 +285,8 @@ begin Exit(ENOEXEC); end; - set_relo_bits(obj,i); + if (def<>@dynlibs_info.sym_nops) then + set_relo_bits(obj,i); end; //R_X86_64_DTPOFF32 @@ -302,7 +307,8 @@ begin Exit(ENOEXEC); end; - set_relo_bits(obj,i); + if (def<>@dynlibs_info.sym_nops) then + set_relo_bits(obj,i); end; //R_X86_64_TPOFF32 else; @@ -339,7 +345,8 @@ begin Exit(ENOEXEC); end; - set_relo_bits(obj,i); + if (def<>@dynlibs_info.sym_nops) then + set_relo_bits(obj,i); goto _next; end; @@ -407,7 +414,8 @@ begin Exit(4); end; - set_relo_bits(obj,idofs); + if (def<>@dynlibs_info.sym_nops) then + set_relo_bits(obj,idofs); if (flags=0) then Exit; diff --git a/sys/kern/kern_rtld.pas b/sys/kern/kern_rtld.pas index 3b4fcf16..415790d7 100644 --- a/sys/kern/kern_rtld.pas +++ b/sys/kern/kern_rtld.pas @@ -394,7 +394,7 @@ begin s:=SELF_SEGMENT_INDEX(self_segs[i].flags); s:=elf_phdr[s].p_offset; MinSeg:=MinInt64(s,MinSeg); - s:=s+minInt64(self_segs[i].filesz,self_segs[i].filesz); + s:=s+minInt64(self_segs[i].filesz,self_segs[i].memsz); MaxSeg:=MaxInt64(s,MaxSeg); end; @@ -436,9 +436,9 @@ begin Assert(false,'src_ofs>=obj_size'); end; - if ((src_ofs+mem_size)>=obj_size) then + if ((src_ofs+mem_size)>obj_size) then begin - Assert(false,'(src_ofs+mem_size)>=obj_size'); + Assert(false,'(src_ofs+mem_size)>obj_size'); end; if (dst_ofs>=MaxSeg) then @@ -446,9 +446,9 @@ begin Assert(false,'dst_ofs>=MaxSeg'); end; - if ((dst_ofs+mem_size)>=MaxSeg) then + if ((dst_ofs+mem_size)>MaxSeg) then begin - Assert(false,'(dst_ofs+mem_size)>=MaxSeg'); + Assert(false,'(dst_ofs+mem_size)>MaxSeg'); end; Move( (Pointer(self_hdr) +src_ofs)^, //src @@ -633,7 +633,7 @@ begin addr^:=SCE_REPLAY_EXEC_START; end; - Result:=vm_mmap2(map,addr,size,0,0,MAP_ANON or MAP_PRIVATE,OBJT_DEFAULT,nil,0); + Result:=vm_mmap2(map,addr,size,0,0,MAP_ANON or MAP_PRIVATE or (21 shl MAP_ALIGNMENT_BIT),OBJT_DEFAULT,nil,0); end; procedure rtld_munmap(base:Pointer;size:QWORD); diff --git a/sys/sys_mmap.pas b/sys/sys_mmap.pas index 0b89b032..6bfd1ac5 100644 --- a/sys/sys_mmap.pas +++ b/sys/sys_mmap.pas @@ -38,9 +38,9 @@ const MAP_SELF =$80000; // map decryped SELF file MAP_ALIGNMENT_BIT =24; - MAP_ALIGNMENT_SHIFT=$1f000000; - MAP_ALIGNMENT_MASK =$ff shl MAP_ALIGNMENT_BIT; - MAP_ALIGNED_SUPER =$01 shl MAP_ALIGNMENT_BIT; + MAP_ALIGNMENT_SHIFT=24; + MAP_ALIGNMENT_MASK =$ff shl MAP_ALIGNMENT_SHIFT; + MAP_ALIGNED_SUPER =$01 shl MAP_ALIGNMENT_SHIFT; //MAP_ALIGNED(n) ((n) << MAP_ALIGNMENT_SHIFT) MCL_CURRENT=$0001; // Lock only current memory diff --git a/sys/sys_sysinit.pas b/sys/sys_sysinit.pas index 68a52351..79f424a4 100644 --- a/sys/sys_sysinit.pas +++ b/sys/sys_sysinit.pas @@ -24,6 +24,7 @@ uses vmount, vfiledesc, vm_map, + kern_dmem, kern_mtxpool, vsys_generic, vfs_subr, @@ -93,6 +94,7 @@ begin vmountinit; fd_table_init; vminit; + init_dmem_map; mtx_pool_setup_dynamic; selectinit; vntblinit; diff --git a/sys/syscalls.pas b/sys/syscalls.pas index 9f4f3afd..24a02998 100644 --- a/sys/syscalls.pas +++ b/sys/syscalls.pas @@ -234,6 +234,7 @@ function utc_to_localtime(time:QWORD;local_time,tsec:Pointer;dstsec:PInteger):I function localtime_to_utc(time:QWORD;tz_type:Integer;utc_time,tsec:Pointer;dstsec:PInteger):Integer; function dynlib_get_obj_member(handle:Integer;num:Byte;pout:PPointer):Integer; function budget_get_ptype_of_budget(key:Integer):Integer; +function blockpool_open(flags:Integer):Integer; function __sys_dynlib_get_info_for_libdbg(handle:Integer;info:Pointer):Integer; function fdatasync(fd:Integer):Integer; function __sys_dynlib_get_list2(pArray:PInteger;numArray:QWORD;pActualNum:PQWORD):Integer; @@ -1836,6 +1837,13 @@ asm jmp cerror end; +function blockpool_open(flags:Integer):Integer; assembler; nostackframe; +asm + movq $653,%rax + call fast_syscall + jmp cerror +end; + function __sys_dynlib_get_info_for_libdbg(handle:Integer;info:Pointer):Integer; assembler; nostackframe; asm movq $656,%rax diff --git a/sys/sysent.pas b/sys/sysent.pas index cb630622..01d1746d 100644 --- a/sys/sysent.pas +++ b/sys/sysent.pas @@ -3322,7 +3322,7 @@ const ), (//[653] sy_narg:1; - sy_call:nil; + sy_call:@sys_blockpool_open; sy_name:'sys_blockpool_open' ), (//[654] diff --git a/sys/test/project1.lpi b/sys/test/project1.lpi index cfd1755b..22747383 100644 --- a/sys/test/project1.lpi +++ b/sys/test/project1.lpi @@ -641,6 +641,10 @@ + + + + diff --git a/sys/vfs/kern_conf.pas b/sys/vfs/kern_conf.pas index 283db99b..cc532b6e 100644 --- a/sys/vfs/kern_conf.pas +++ b/sys/vfs/kern_conf.pas @@ -199,6 +199,7 @@ function _nullop():Integer; function _eopnotsupp():Integer; function _enxio():Integer; function _enodev():Integer; +function _einval():Integer; procedure dev_lock(); procedure dev_unlock(); @@ -503,6 +504,11 @@ begin Exit(ENODEV); end; +function _einval():Integer; +begin + Exit(EINVAL); +end; + procedure dead_strategy(bp:Pointer); begin //biofinish(bp, nil, ENXIO); @@ -896,7 +902,9 @@ begin len:=Length(R); if (len > sizeof(dev^.__si_namebuf) - 1) then + begin Exit(ENAMETOOLONG); + end; Move(PChar(R)^,dev^.__si_namebuf,len); @@ -909,10 +917,14 @@ begin begin { Treat multiple sequential slashes as single. } while (from[0]='/') and (from[1]='/') do + begin Inc(from); + end; { Trailing slash is considered invalid. } if (from[0]='/') and (from[1]=#0) then + begin Exit(EINVAL); + end; _to^:=from^; // Inc(from); @@ -921,7 +933,9 @@ begin _to^:=#0; if (dev^.__si_namebuf[0]=#0) then + begin Exit(EINVAL); + end; { Disallow '.' and '..' components. } s:=@dev^.__si_namebuf; @@ -931,16 +945,24 @@ begin while (q^<>'/') and (q^<>#0) do Inc(q); if (q - s=1) and (s[0]='.') then + begin Exit(EINVAL); + end; if (q - s=2) and (s[0]='.') and (s[1]='.') then + begin Exit(EINVAL); + end; if (q^<>'/') then + begin break; + end; s:=q + 1; end; if (devfs_dev_exists(dev^.__si_namebuf)<>0) then + begin Exit(EEXIST); + end; Exit(0); end; diff --git a/sys/vfs/kern_descrip.pas b/sys/vfs/kern_descrip.pas index baadc4cd..bbb71a0d 100644 --- a/sys/vfs/kern_descrip.pas +++ b/sys/vfs/kern_descrip.pas @@ -1217,7 +1217,9 @@ var begin error:=falloc_noinstall(@fp); if (error<>0) then + begin Exit(error); { no reference held on error } + end; error:=finstall(fp,@fd,flags); if (error<>0) then @@ -1232,7 +1234,9 @@ begin fdrop(fp); { release local reference } if (resultfd<>nil) then + begin resultfd^:=fd; + end; Exit(0); end; diff --git a/sys/vm/dmem_map.pas b/sys/vm/dmem_map.pas new file mode 100644 index 00000000..22814cc3 --- /dev/null +++ b/sys/vm/dmem_map.pas @@ -0,0 +1,956 @@ +unit dmem_map; + +{$mode ObjFPC}{$H+} + +interface + +uses + vmparam, + kern_mtx; + +Const + SCE_KERNEL_MAIN_DMEM_SIZE=$180000000; //6GB + + SCE_KERNEL_WB_ONION = 0; + SCE_KERNEL_WC_GARLIC = 3; + SCE_KERNEL_WB_GARLIC =10; + + //deprecated + SCE_KERNEL_WB_ONION_NONVOLATILE = 1; + SCE_KERNEL_WC_GARLIC_VOLATILE = 2; + SCE_KERNEL_WC_GARLIC_NONVOLATILE = 3; + SCE_KERNEL_WT_ONION_VOLATILE = 4; + SCE_KERNEL_WT_ONION_NONVOLATILE = 5; + SCE_KERNEL_WP_ONION_VOLATILE = 6; + SCE_KERNEL_WP_ONION_NONVOLATILE = 7; + SCE_KERNEL_UC_GARLIC_VOLATILE = 8; + SCE_KERNEL_UC_GARLIC_NONVOLATILE = 9; + +type + pp_dmem_map_entry=^p_dmem_map_entry; + p_dmem_map_entry=^t_dmem_map_entry; + t_dmem_map_entry=packed record + prev :p_dmem_map_entry; // previous entry + next :p_dmem_map_entry; // next entry + left :p_dmem_map_entry; // left child in binary search tree + right :p_dmem_map_entry; // right child in binary search tree + start :DWORD; // start address + __end :DWORD; // end address + avail_ssize :DWORD; // amt can grow if this is a stack + adj_free :DWORD; // amount of adjacent free space + max_free :DWORD; // max free space in subtree + m_type :DWORD; // memory type + end; + + p_dmem_map=^t_dmem_map; + t_dmem_map=packed object + header :t_dmem_map_entry; // List of entries + lock :mtx; // Lock for map data + nentries:DWORD; // Number of entries + size :DWORD; // size + root :p_dmem_map_entry; // Root of a binary search tree + property min_offset:DWORD read header.start write header.start; + property max_offset:DWORD read header.__end write header.__end; + end; + +procedure dmem_map_entry_deallocate(entry:p_dmem_map_entry); + +procedure dmem_map_lock(map:p_dmem_map); +function dmem_map_trylock(map:p_dmem_map):Boolean; +procedure dmem_map_unlock(map:p_dmem_map); +function dmem_map_locked(map:p_dmem_map):Boolean; inline; + +procedure dmem_map_init(map:p_dmem_map;min,max:QWORD); + +procedure dmem_map_entry_dispose(map:p_dmem_map;entry:p_dmem_map_entry); inline; +function dmem_map_entry_create(map:p_dmem_map):p_dmem_map_entry; + +function dmem_map_lookup_entry( + map :p_dmem_map; + address :DWORD; + entry :pp_dmem_map_entry):Boolean; + +function dmem_map_insert( + map :p_dmem_map; + start :DWORD; + __end :DWORD; + m_type :DWORD):Integer; + +Function dmem_map_query_available(map:p_dmem_map;start,__end,align:QWORD;var oaddr,osize:QWORD):Integer; + +function dmem_map_findspace(map :p_dmem_map; + start :DWORD; + length:DWORD; + addr :PDWORD):Integer; + +function dmem_map_fixed(map :p_dmem_map; + start :DWORD; + length :DWORD; + m_type :DWORD; + overwr :Integer):Integer; + +procedure dmem_map_simplify_entry(map:p_dmem_map;entry:p_dmem_map_entry); + +procedure dmem_map_entry_delete(map:p_dmem_map;entry:p_dmem_map_entry); + +function dmem_map_delete(map:p_dmem_map;start:DWORD;__end:DWORD):Integer; + +implementation + +uses + errno, + kern_thr; + +function IDX_TO_OFF(x:DWORD):QWORD; inline; +begin + Result:=QWORD(x) shl PAGE_SHIFT; +end; + +function OFF_TO_IDX(x:QWORD):DWORD; inline; +begin + Result:=QWORD(x) shr PAGE_SHIFT; +end; + +function IsPowerOfTwo(x:QWORD):Boolean; inline; +begin + Result:=(x and (x - 1))=0; +end; + +function fastIntLog2(i:QWORD):QWORD; inline; +begin + Result:=BsfQWORD(i); +end; + +function AlignUp(addr:PtrUInt;alignment:PtrUInt):PtrUInt; inline; +var + tmp:PtrUInt; +begin + if (alignment=0) then Exit(addr); + tmp:=addr+PtrUInt(alignment-1); + Result:=tmp-(tmp mod alignment) +end; + +function AlignDw(addr:PtrUInt;alignment:PtrUInt):PtrUInt; inline; +begin + Result:=addr-(addr mod alignment); +end; + +procedure dmem_map_entry_deallocate(entry:p_dmem_map_entry); +begin + //if ((entry^.eflags and MAP_ENTRY_IS_SUB_MAP)=0) then + //begin + // vm_object_deallocate(entry^.vm_obj); + //end; + Freemem(entry); +end; + +procedure DMEM_MAP_RANGE_CHECK(map:p_dmem_map;var start,__end:DWORD); +begin + if (startmap^.max_offset) then + begin + __end:=map^.max_offset; + end; + if (start>__end) then + begin + start:=__end; + end; +end; + +procedure dmem_map_lock(map:p_dmem_map); +begin + mtx_lock(map^.lock); +end; + +function dmem_map_trylock(map:p_dmem_map):Boolean; +begin + Result:=mtx_trylock(map^.lock); +end; + +procedure dmem_map_process_deferred; +var + td:p_kthread; + entry,next:p_dmem_map_entry; +begin + td:=curkthread; + if (td=nil) then Exit; + entry:=td^.td_map_def_user; + td^.td_map_def_user:=nil; + while (entry<>nil) do + begin + next:=entry^.next; + dmem_map_entry_deallocate(entry); + entry:=next; + end; +end; + +procedure dmem_map_unlock(map:p_dmem_map); +begin + mtx_unlock(map^.lock); + dmem_map_process_deferred(); +end; + +function dmem_map_locked(map:p_dmem_map):Boolean; inline; +begin + Result:=mtx_owned(map^.lock); +end; + +procedure DMEM_MAP_ASSERT_LOCKED(map:p_dmem_map); inline; +begin + Assert(dmem_map_locked(map)); +end; + +procedure _dmem_map_init(map:p_dmem_map;min,max:DWORD); +begin + map^.header.next:=@map^.header; + map^.header.prev:=@map^.header; + map^.min_offset :=min; + map^.max_offset :=max; + map^.header.adj_free:=(max-min); + map^.header.max_free:=(max-min); + map^.nentries:=0; + map^.size :=0; + map^.root:=nil; +end; + +procedure dmem_map_init(map:p_dmem_map;min,max:QWORD); +begin + _dmem_map_init(map, OFF_TO_IDX(min), OFF_TO_IDX(max)); + mtx_init(map^.lock,'dmem'); +end; + +procedure dmem_map_entry_dispose(map:p_dmem_map;entry:p_dmem_map_entry); inline; +begin + FreeMem(entry); +end; + +function dmem_map_entry_create(map:p_dmem_map):p_dmem_map_entry; +var + new_entry:p_dmem_map_entry; +begin + new_entry:=AllocMem(SizeOf(t_dmem_map_entry)); + Assert((new_entry<>nil),'dmem_map_entry_create: kernel resources exhausted'); + Result:=new_entry; +end; + +procedure dmem_map_entry_set_max_free(entry:p_dmem_map_entry); +begin + entry^.max_free:=entry^.adj_free; + if (entry^.left<>nil) then + if (entry^.left^.max_free>entry^.max_free) then + begin + entry^.max_free:=entry^.left^.max_free; + end; + if (entry^.right<>nil) then + if (entry^.right^.max_free>entry^.max_free) then + begin + entry^.max_free:=entry^.right^.max_free; + end; +end; + +function dmem_map_entry_splay(addr:DWORD;root:p_dmem_map_entry):p_dmem_map_entry; +var + llist,rlist:p_dmem_map_entry; + ltree,rtree:p_dmem_map_entry; + y :p_dmem_map_entry; +begin + { Special case of empty tree. } + if (root=nil) then Exit(root); + + llist:=nil; + rlist:=nil; + repeat + { root is never nil in here. } + if (addrnil) then + begin + { Rotate right and put y on rlist. } + root^.left:=y^.right; + y^.right:=root; + dmem_map_entry_set_max_free(root); + root:=y^.left; + y^.left:=rlist; + rlist:=y; + end else + begin + { Put root on rlist. } + root^.left:=rlist; + rlist:=root; + root:=y; + end; + end else + if (addr>=root^.__end) then + begin + y:=root^.right; + if (y=nil) then break; + if (addr>=y^.__end) and (y^.right<>nil) then + begin + { Rotate left and put y on llist. } + root^.right:=y^.left; + y^.left:=root; + dmem_map_entry_set_max_free(root); + root:=y^.right; + y^.right:=llist; + llist:=y; + end else + begin + { Put root on llist. } + root^.right:=llist; + llist:=root; + root:=y; + end; + end else + begin + break; + end; + until false; + + { + * Pass Two: Walk back up the two spines, flip the pointers + * and set max_free. The subtrees of the root go at the + * bottom of llist and rlist. + } + ltree:=root^.left; + while (llist<>nil) do + begin + y:=llist^.right; + llist^.right:=ltree; + dmem_map_entry_set_max_free(llist); + ltree:=llist; + llist:=y; + end; + rtree:=root^.right; + while (rlist<>nil) do + begin + y:=rlist^.left; + rlist^.left:=rtree; + dmem_map_entry_set_max_free(rlist); + rtree:=rlist; + rlist:=y; + end; + + { + * Final assembly: add ltree and rtree as subtrees of root. + } + root^.left:=ltree; + root^.right:=rtree; + dmem_map_entry_set_max_free(root); + + Result:=(root); +end; + +procedure dmem_map_entry_link( + map :p_dmem_map; + after_where:p_dmem_map_entry; + entry :p_dmem_map_entry); +begin + DMEM_MAP_ASSERT_LOCKED(map); + + Inc(map^.nentries); + entry^.prev:=after_where; + entry^.next:=after_where^.next; + entry^.next^.prev:=entry; + after_where^.next:=entry; + + if (after_where<>@map^.header) then + begin + if (after_where<>map^.root) then + begin + dmem_map_entry_splay(after_where^.start, map^.root); + end; + entry^.right:=after_where^.right; + entry^.left:=after_where; + after_where^.right:=nil; + after_where^.adj_free:=entry^.start - after_where^.__end; + dmem_map_entry_set_max_free(after_where); + end else + begin + entry^.right:=map^.root; + entry^.left:=nil; + end; + if (entry^.next=@map^.header) then + begin + entry^.adj_free:=map^.max_offset-entry^.__end; + end else + begin + entry^.adj_free:=entry^.next^.start-entry^.__end; + end; + dmem_map_entry_set_max_free(entry); + map^.root:=entry; +end; + +procedure dmem_map_entry_unlink( + map :p_dmem_map; + entry :p_dmem_map_entry); +var + next,prev,root:p_dmem_map_entry; +begin + DMEM_MAP_ASSERT_LOCKED(map); + + if (entry<>map^.root) then + begin + dmem_map_entry_splay(entry^.start, map^.root); + end; + if (entry^.left=nil) then + begin + root:=entry^.right; + end else + begin + root:=dmem_map_entry_splay(entry^.start, entry^.left); + root^.right:=entry^.right; + if (root^.next=@map^.header) then + begin + root^.adj_free:=map^.max_offset-root^.__end; + end else + begin + root^.adj_free:=entry^.next^.start-root^.__end; + end; + dmem_map_entry_set_max_free(root); + end; + map^.root:=root; + + prev:=entry^.prev; + next:=entry^.next; + next^.prev:=prev; + prev^.next:=next; + Dec(map^.nentries); +end; + +procedure dmem_map_entry_resize_free(map:p_dmem_map;entry:p_dmem_map_entry); +begin + if (entry<>map^.root) then + begin + map^.root:=dmem_map_entry_splay(entry^.start, map^.root); + end; + + if (entry^.next=@map^.header) then + begin + entry^.adj_free:=map^.max_offset-entry^.__end; + end else + begin + entry^.adj_free:=entry^.next^.start-entry^.__end; + end; + dmem_map_entry_set_max_free(entry); +end; + +function dmem_map_lookup_entry( + map :p_dmem_map; + address :DWORD; + entry :pp_dmem_map_entry):Boolean; +var + cur:p_dmem_map_entry; +begin + DMEM_MAP_ASSERT_LOCKED(map); + + { + * If the map is empty, then the map entry immediately preceding + * "address" is the map's header. + } + cur:=map^.root; + if (cur=nil) then + begin + entry^:=@map^.header; + end else + if (address>=cur^.start) and (cur^.__end>address) then + begin + entry^:=cur; + Exit(TRUE); + end else + begin + { + * Splay requires a write lock on the map. However, it only + * restructures the binary search tree; it does not otherwise + * change the map. Thus, the map's timestamp need not change + * on a temporary upgrade. + } + cur:=dmem_map_entry_splay(address,cur); + map^.root:=cur; + + { + * If "address" is contained within a map entry, the new root + * is that map entry. Otherwise, the new root is a map entry + * immediately before or after "address". + } + if (address>=cur^.start) then + begin + entry^:=cur; + if (cur^.__end>address) then + begin + Exit(TRUE); + end; + end else + begin + entry^:=cur^.prev; + end; + end; + Result:=(FALSE); +end; + +function dmem_map_insert( + map :p_dmem_map; + start :DWORD; + __end :DWORD; + m_type :DWORD):Integer; +var + new_entry :p_dmem_map_entry; + prev_entry:p_dmem_map_entry; + temp_entry:p_dmem_map_entry; +begin + DMEM_MAP_ASSERT_LOCKED(map); + + { + * Check that the start and end points are not bogus. + } + if (startmap^.max_offset) or (start>=__end) then + begin + Exit(EINVAL); + end; + + { + * Find the entry prior to the proposed starting address; if it's part + * of an existing entry, this range is bogus. + } + if dmem_map_lookup_entry(map,start,@temp_entry) then + begin + Exit(EAGAIN); + end; + + prev_entry:=temp_entry; + + { + * Assert that the next entry doesn't overlap the end point. + } + if (prev_entry^.next<>@map^.header) and + (prev_entry^.next^.start<__end) then + begin + Exit(EAGAIN); + end; + + if (prev_entry<>@map^.header) and + (prev_entry^.__end=start) then + begin + { + * We were able to extend the object. Determine if we + * can extend the previous map entry to include the + * new range as well. + } + if (prev_entry^.m_type=m_type) then + begin + map^.size:=map^.size+(__end - prev_entry^.__end); + prev_entry^.__end:=__end; + //change size + + //pmap_enter_object + + dmem_map_entry_resize_free(map, prev_entry); + dmem_map_simplify_entry(map, prev_entry); + Exit(0); + end; + + end; + + { + * NOTE: if conditionals fail, object can be nil here. This occurs + * in things like the buffer map where we manage kva but do not manage + * backing objects. + } + + { + * Create a new entry + } + new_entry:=dmem_map_entry_create(map); + new_entry^.start:=start; + new_entry^.__end:=__end; + + new_entry^.m_type:=m_type; + new_entry^.avail_ssize:=0; + + { + * Insert the new entry into the list + } + dmem_map_entry_link(map, prev_entry, new_entry); + map^.size:=map^.size+(new_entry^.__end - new_entry^.start); + + dmem_map_simplify_entry(map, new_entry); + + //pmap_enter_object + + Result:=0; +end; + +Function dmem_map_query_available(map:p_dmem_map;start,__end,align:QWORD;var oaddr,osize:QWORD):Integer; +const + max_valid=QWORD($5000000000); +var + entry:p_dmem_map_entry; + + r_addr,r_size:QWORD; + t_addr,t_size:QWORD; + t__end,t_free:QWORD; +begin + Result:=0; + + if not IsPowerOfTwo(align) then + begin + Exit(EINVAL); + end; + + if (alignmax_valid) then start:=max_valid; + + __end:=(not (__end shr 63)) and __end; + if (__end>max_valid) then __end:=max_valid; + + start:=AlignUp(start,align); + + dmem_map_lock(map); + + if (map^.root=nil) then + begin + if (start>=IDX_TO_OFF(map^.max_offset)) then + begin + r_addr:=0; + r_size:=0; + Result:=ENOMEM; + end else + begin + r_addr:=start; + r_size:=IDX_TO_OFF(map^.max_offset)-start; + Result:=0; + end; + end else + begin + r_addr:=0; + r_size:=0; + Result:=ENOMEM; + + map^.root:=dmem_map_entry_splay(OFF_TO_IDX(start), map^.root); + entry:=map^.root; + + while (entry<>nil) do + begin + if (entry^.adj_free<>0) then + begin + t__end:=IDX_TO_OFF(entry^.__end); + t_free:=IDX_TO_OFF(entry^.adj_free); + t_addr:=AlignUp(t__end,align); + if (__endr_size) then + begin + r_addr:=t_addr; + r_size:=t_size; + end; + Result:=0; + end; + end; + entry:=entry^.next; + end; + end; + + dmem_map_unlock(map); + + oaddr:=r_addr; + osize:=r_size; +end; + +function dmem_map_findspace(map :p_dmem_map; + start :DWORD; + length:DWORD; + addr :PDWORD):Integer; +label + _nxt; +var + entry:p_dmem_map_entry; + st:DWORD; +begin + { + * Request must fit within min/max VM address and must avoid + * address wrap. + } + if (startmap^.max_offset) or (start + lengthmap^.root^.__end) then + begin + st:=start; + end else + begin + st:=map^.root^.__end; + end; + + if (length<=map^.root^.__end + map^.root^.adj_free - st) then + begin + addr^:=st; + Exit(0); + end; + + { With max_free, can immediately tell if no solution. } + entry:=map^.root^.right; + + if (entry=nil) then + begin + Exit(1); + end; + + if (length>entry^.max_free) then + begin + Exit(1); + end; + + { + * Search the right subtree in the order: left subtree, root, + * right subtree (first fit). The previous splay implies that + * all regions in the right subtree have addresses>start. + } + while (entry<>nil) do + begin + if (entry^.left<>nil) then + begin + if not (entry^.left^.max_free>=length) then goto _nxt; + entry:=entry^.left; + end else + begin + _nxt: + if (entry^.adj_free>=length) then + begin + addr^:=entry^.__end; + Exit(0); + end else + begin + entry:=entry^.right; + end; + end; + end; + + { Can't get here, so panic if we do. } + Assert(false,'dmem_map_findspace: max_free corrupt'); +end; + +function dmem_map_fixed(map :p_dmem_map; + start :DWORD; + length :DWORD; + m_type :DWORD; + overwr :Integer):Integer; +var + __end:DWORD; +begin + __end:=start + length; + dmem_map_lock(map); + DMEM_MAP_RANGE_CHECK(map, start, __end); + if (overwr<>0) then + begin + dmem_map_delete(map, start, __end); + end; + Result:= dmem_map_insert(map, start, __end, m_type); + dmem_map_unlock(map); +end; + +procedure dmem_map_simplify_entry(map:p_dmem_map;entry:p_dmem_map_entry); +var + next,prev:p_dmem_map_entry; +begin + //if ((entry^.eflags and (MAP_ENTRY_IS_SUB_MAP))<>0) then + //begin + // Exit; + //end; + + prev:=entry^.prev; + if (prev<>@map^.header) then + begin + if (prev^.__end=entry^.start) and + (prev^.m_type=entry^.m_type) then + begin + dmem_map_entry_unlink(map, prev); + entry^.start:=prev^.start; + + //change + if (entry^.prev<>@map^.header) then + begin + dmem_map_entry_resize_free(map, entry^.prev); + end; + + dmem_map_entry_dispose(map, prev); + end; + end; + + next:=entry^.next; + if (next<>@map^.header) then + begin + if (entry^.__end=next^.start) and + (next^.m_type=entry^.m_type) then + begin + dmem_map_entry_unlink(map, next); + entry^.__end:=next^.__end; + //change + dmem_map_entry_resize_free(map, entry); + + dmem_map_entry_dispose(map, next); + end; + end; +end; + +procedure _dmem_map_clip_start(map:p_dmem_map;entry:p_dmem_map_entry;start:DWORD); +var + new_entry:p_dmem_map_entry; +begin + DMEM_MAP_ASSERT_LOCKED(map); + + dmem_map_simplify_entry(map, entry); + + new_entry:=dmem_map_entry_create(map); + new_entry^:=entry^; + + new_entry^.__end:=start; + entry^.start:=start; + + dmem_map_entry_link(map, entry^.prev, new_entry); + + //if ((entry^.eflags and MAP_ENTRY_IS_SUB_MAP)=0) then + //begin + // vm_object_reference(new_entry^.vm_obj); + //end; +end; + +procedure dmem_map_clip_start(map:p_dmem_map;entry:p_dmem_map_entry;start:DWORD); +begin + if (start>entry^.start) then + begin + _dmem_map_clip_start(map,entry,start); + end; +end; + +procedure _dmem_map_clip_end(map:p_dmem_map;entry:p_dmem_map_entry;__end:DWORD); +var + new_entry:p_dmem_map_entry; +begin + DMEM_MAP_ASSERT_LOCKED(map); + + { + * Create a new entry and insert it AFTER the specified entry + } + new_entry:=dmem_map_entry_create(map); + new_entry^:=entry^; + + new_entry^.start:=__end; + entry^.__end:=__end; + + dmem_map_entry_link(map, entry, new_entry); + + //if ((entry^.eflags and MAP_ENTRY_IS_SUB_MAP)=0) then + //begin + // vm_object_reference(new_entry^.vm_obj); + //end; +end; + +procedure dmem_map_clip_end(map:p_dmem_map;entry:p_dmem_map_entry;__end:DWORD); +begin + if (__end@map^.header) and (entry^.start<__end) do + begin + + dmem_map_clip_end(map, entry, __end); + + next:=entry^.next; + + //pmap_remove(map^.pmap,entry^.start,entry^.__end,entry^.protection); + + { + * Delete the entry only after removing all pmap + * entries pointing to its pages. (Otherwise, its + * page frames may be reallocated, and any modify bits + * will be set in the wrong object!) + } + dmem_map_entry_delete(map, entry); + entry:=next; + end; + Result:=(0); +end; + +end. + diff --git a/sys/vm/vm.pas b/sys/vm/vm.pas index 778ffbe1..f28ea0ca 100644 --- a/sys/vm/vm.pas +++ b/sys/vm/vm.pas @@ -81,9 +81,9 @@ const MAP_SELF =$80000; // map decryped SELF file MAP_ALIGNMENT_BIT =24; - MAP_ALIGNMENT_SHIFT=$1f000000; - MAP_ALIGNMENT_MASK =$ff shl MAP_ALIGNMENT_BIT; - MAP_ALIGNED_SUPER =$01 shl MAP_ALIGNMENT_BIT; + MAP_ALIGNMENT_SHIFT=24; + MAP_ALIGNMENT_MASK =$ff shl MAP_ALIGNMENT_SHIFT; + MAP_ALIGNED_SUPER =$01 shl MAP_ALIGNMENT_SHIFT; //MAP_ALIGNED(n) ((n) << MAP_ALIGNMENT_SHIFT) MCL_CURRENT=$0001; // Lock only current memory diff --git a/sys/vm/vm_map.pas b/sys/vm/vm_map.pas index 56f6325c..1a6d9948 100644 --- a/sys/vm/vm_map.pas +++ b/sys/vm/vm_map.pas @@ -54,8 +54,8 @@ type root:vm_map_entry_t; // Root of a binary search tree pmap:pmap_t; // (c) Physical map busy:Integer; - property min_offset:vm_offset_t read header.start; - property max_offset:vm_offset_t read header.__end; + property min_offset:vm_offset_t read header.start write header.start; + property max_offset:vm_offset_t read header.__end write header.__end; end; p_vmspace=^vmspace; @@ -212,6 +212,8 @@ function vm_map_find(map :vm_map_t; max :vm_prot_t; cow :Integer):Integer; +procedure vm_map_simplify_entry(map:vm_map_t;entry:vm_map_entry_t); + function vm_map_fixed(map :vm_map_t; vm_obj :vm_object_t; offset :vm_ooffset_t; @@ -254,6 +256,11 @@ var sgrowsiz:QWORD=vmparam.SGROWSIZ; stack_guard_page:Integer=0; +function OFF_TO_IDX(x:QWORD):DWORD; inline; +begin + Result:=QWORD(x) shr PAGE_SHIFT; +end; + function VMFS_ALIGNED_SPACE(x:QWORD):QWORD; inline; // find a range with fixed alignment begin Result:=x shl 8; @@ -331,13 +338,6 @@ begin FillChar(g_vmspace,SizeOf(vmspace),0); end; -procedure vm_map_zinit(map:vm_map_t); -begin - map^.nentries:=0; - map^.size:=0; - mtx_init(map^.lock,'vm map (system)'); -end; - function vmspace_pmap(vm:p_vmspace):pmap_t; inline; begin Result:=@vm^.vm_pmap; @@ -381,7 +381,9 @@ function vm_map_trylock(map:vm_map_t):Boolean; begin Result:=mtx_trylock(map^.lock); if Result then + begin Inc(map^.timestamp); + end; end; procedure vm_map_process_deferred; @@ -449,8 +451,8 @@ begin map^.header.next:=@map^.header; map^.header.prev:=@map^.header; map^.pmap:=pmap; - map^.header.start:=min; - map^.header.__end:=max; + map^.min_offset:=min; + map^.max_offset:=max; map^.header.adj_free:=(max-min); map^.header.max_free:=(max-min); map^.flags:=0; @@ -601,7 +603,9 @@ begin root:=y; end; end else + begin break; + end; until false; { @@ -639,7 +643,7 @@ begin end; { - * vm_map_entry_beginun,end;link: + * vm_map_entry_{un,}link: * * Insert/remove entries from maps. } @@ -649,6 +653,7 @@ procedure vm_map_entry_link( entry :vm_map_entry_t); begin VM_MAP_ASSERT_LOCKED(map); + Inc(map^.nentries); entry^.prev:=after_where; entry^.next:=after_where^.next; @@ -658,7 +663,9 @@ begin if (after_where<>@map^.header) then begin if (after_where<>map^.root) then + begin vm_map_entry_splay(after_where^.start, map^.root); + end; entry^.right:=after_where^.right; entry^.left:=after_where; after_where^.right:=nil; @@ -680,7 +687,6 @@ begin map^.root:=entry; end; - procedure vm_map_entry_unlink( map :vm_map_t; entry :vm_map_entry_t); @@ -688,6 +694,7 @@ var next,prev,root:vm_map_entry_t; begin VM_MAP_ASSERT_LOCKED(map); + if (entry<>map^.root) then begin vm_map_entry_splay(entry^.start, map^.root); @@ -806,13 +813,13 @@ begin Exit(TRUE); end; end else + begin entry^:=cur^.prev; + end; end; Result:=(FALSE); end; -procedure vm_map_simplify_entry(map:vm_map_t;entry:vm_map_entry_t); forward; - { * vm_map_insert: * @@ -1068,9 +1075,13 @@ begin * address wrap. } if (startmap^.max_offset) or (start + lengthentry^.max_free) then + begin Exit(1); + end; { * Search the right subtree in the order: left subtree, root, @@ -1138,7 +1153,9 @@ begin addr^:=entry^.__end; Exit(0); end else + begin entry:=entry^.right; + end; end; end; @@ -1208,7 +1225,9 @@ begin Assert((find_space and $ff)=0,'bad VMFS flags'); alignment:=vm_offset_t(1) shl (find_space shr 8); end else + begin alignment:=0; + end; initial_addr:=addr^; again: start:=initial_addr; @@ -1266,7 +1285,9 @@ var prevsize, esize:vm_size_t; begin if ((entry^.eflags and (MAP_ENTRY_IS_SUB_MAP))<>0) then + begin Exit; + end; prev:=entry^.prev; if (prev<>@map^.header) then @@ -1285,7 +1306,9 @@ begin entry^.offset:=prev^.offset; //change if (entry^.prev<>@map^.header) then + begin vm_map_entry_resize_free(map, entry^.prev); + end; { * If the backing object is a vnode object, @@ -1371,7 +1394,9 @@ end; procedure vm_map_clip_start(map:vm_map_t;entry:vm_map_entry_t;start:vm_offset_t); begin if (start>entry^.start) then + begin _vm_map_clip_start(map,entry,start); + end; end; { @@ -1413,7 +1438,9 @@ end; procedure vm_map_clip_end(map:vm_map_t;entry:vm_map_entry_t;__end:vm_offset_t); begin if (__end@map^.header) and (entry^.start<__end)) do begin @@ -2002,7 +2033,9 @@ var begin VM_MAP_ASSERT_LOCKED(map); if (start=__end) then + begin Exit(KERN_SUCCESS); + end; { * Find the start of the region, and clip it @@ -2077,24 +2110,32 @@ var tmp_entry:vm_map_entry_t; begin if (not vm_map_lookup_entry(map, start, @tmp_entry)) then + begin Exit(FALSE); + end; entry:=tmp_entry; while (start<__end) do begin if (entry=@map^.header) then + begin Exit (FALSE); + end; { * No holes allowed! } if (startprotection) then + begin Exit(FALSE); + end; { go to next entry } start:=entry^.__end; entry:=entry^.next; @@ -2127,7 +2168,9 @@ begin if (addrbosvm_map_max(map)) or (addrbos + max_ssize@map^.header) then + begin vm_map_clip_end(map, prev_entry, bot); + end; new_entry:=prev_entry^.next; if (new_entry^.__end<>top) or (new_entry^.start<>bot) then + begin Assert(false,'Bad entry start/end for new stack entry'); + end; new_entry^.avail_ssize:=max_ssize - init_ssize; if ((orient and MAP_STACK_GROWS_DOWN)<>0) then + begin new_entry^.eflags:=new_entry^.eflags or MAP_ENTRY_GROWS_DOWN; + end; if ((orient and MAP_STACK_GROWS_UP)<>0) then + begin new_entry^.eflags:=new_entry^.eflags or MAP_ENTRY_GROWS_UP; + end; end; vm_map_unlock(map); @@ -2293,8 +2344,11 @@ begin stack_entry:=prev_entry else stack_entry:=next_entry; + end else + begin stack_entry:=prev_entry; + end; end; if (stack_entry=next_entry) then @@ -2371,7 +2425,9 @@ begin growsize:=sgrowsiz; grow_amount:=roundup(grow_amount, growsize); if (grow_amount>stack_entry^.avail_ssize) then + begin grow_amount:=stack_entry^.avail_ssize; + end; if (is_procstack<>0) and (ctob(g_vmspace.vm_ssize) + grow_amount>stacklim) then begin grow_amount:=trunc_page(stacklim) - ctob(g_vmspace.vm_ssize); @@ -2402,7 +2458,9 @@ begin stack_entry^.avail_ssize:=max_grow; addr:=__end; if (stack_guard_page<>0) then + begin addr:=addr+PAGE_SIZE; + end; end; rv:=vm_map_insert(map, nil, 0, addr, stack_entry^.start, @@ -2412,7 +2470,9 @@ begin if (rv=KERN_SUCCESS) then begin if (prev_entry<>@map^.header) then + begin vm_map_clip_end(map, prev_entry, addr); + end; new_entry:=prev_entry^.next; Assert(new_entry=stack_entry^.prev, 'foo'); Assert(new_entry^.__end=stack_entry^.start, 'foo'); @@ -2438,7 +2498,9 @@ begin stack_entry^.avail_ssize:=__end - stack_entry^.__end; addr:=__end; if (stack_guard_page<>0) then + begin addr:=addr-PAGE_SIZE; + end; end; grow_amount:=addr - stack_entry^.__end; @@ -2460,13 +2522,19 @@ begin rv:=KERN_SUCCESS; if (next_entry<>@map^.header) then + begin vm_map_clip_start(map, next_entry, addr); + end; end else + begin rv:=KERN_FAILURE; + end; end; if (rv=KERN_SUCCESS) and (is_procstack<>0) then + begin g_vmspace.vm_ssize:=g_vmspace.vm_ssize+btoc(grow_amount); + end; vm_map_unlock(map);