diff --git a/chip/pm4_me.pas b/chip/pm4_me.pas index f126204e..9c5a2462 100644 --- a/chip/pm4_me.pas +++ b/chip/pm4_me.pas @@ -96,6 +96,21 @@ type procedure remove_stream(stream:p_pm4_stream); end; + PvCmdFreeNode=^TvCmdFreeNode; + TvCmdFreeNode=record + entry:STAILQ_ENTRY; + FCmd :TVkCommandBuffer; + end; + + TvCmdCachedPool=class(TvCmdPool) + FMemCache:STAILQ_HEAD; //PvCmdFreeNode + FDeffered:STAILQ_HEAD; //PvCmdFreeNode + FTrimCount:Integer; + Constructor Create(FFamily:TVkUInt32); + procedure Free(cmd:TVkCommandBuffer); register; override; + procedure Trim; register; override; + end; + t_pool_line=array[0..3] of TvCustomCmdPool; t_pool_cache=object queue:TvQueue; @@ -109,6 +124,8 @@ type entry :TAILQ_ENTRY; //stall stream:p_pm4_stream; // + function OnAlloc(size:Ptruint):Pointer; register; override; + Procedure OnFree (P:Pointer ); register; override; end; t_me_render_context=object @@ -130,6 +147,7 @@ type procedure BeginCmdBuffer; procedure FinishCmdBuffer; function CmdStatus(i:t_pm4_stream_type):TVkResult; + procedure PingCmd; function WaitConfirmOrSwitch:Boolean; // procedure switch_task; @@ -146,7 +164,8 @@ uses vmparam, kern_dmem, kern_proc, - vm_map; + vm_map, + vm_tracking_map; procedure StartFrameCapture; begin @@ -350,22 +369,6 @@ end; // -type - PvCmdFreeNode=^TvCmdFreeNode; - TvCmdFreeNode=record - entry:STAILQ_ENTRY; - FCmd :TVkCommandBuffer; - end; - - TvCmdCachedPool=class(TvCmdPool) - FMemCache:STAILQ_HEAD; //PvCmdFreeNode - FDeffered:STAILQ_HEAD; //PvCmdFreeNode - FTrimCount:Integer; - Constructor Create(FFamily:TVkUInt32); - procedure Free(cmd:TVkCommandBuffer); register; override; - procedure Trim; register; override; - end; - Constructor TvCmdCachedPool.Create(FFamily:TVkUInt32); begin inherited; @@ -446,6 +449,20 @@ begin Result:=last; end; +// + +function TvStreamCmdBuffer.OnAlloc(size:Ptruint):Pointer; register; +begin + Result:=stream^.allocator.Alloc(size); + FillChar(Result^,size,0); +end; + +Procedure TvStreamCmdBuffer.OnFree(P:Pointer); register; +begin + // +end; + + // procedure t_me_render_context.Init; @@ -579,6 +596,16 @@ begin Result:=VK_SUCCESS; end; +procedure t_me_render_context.PingCmd; +var + i:t_pm4_stream_type; +begin + for i:=Low(t_pm4_stream_type) to High(t_pm4_stream_type) do + begin + CmdStatus(i); + end; +end; + function t_me_render_context.WaitConfirmOrSwitch:Boolean; begin FinishCmdBuffer; @@ -1500,7 +1527,7 @@ begin Assert(false,'pm4_EventWriteEop'); end; - vm_map_track_trigger(p_proc.p_vmspace,QWORD(node^.addr),QWORD(node^.addr)+data_size,nil,0); + vm_map_track_trigger(p_proc.p_vmspace,QWORD(node^.addr),QWORD(node^.addr)+data_size,nil,M_DMEM_WRITE); end; if (node^.intSel=EVENTWRITEEOP_INT_SEL_SEND_INT) or @@ -1591,7 +1618,7 @@ begin PDWORD(addr_dmem)^:=node^.data; - vm_map_track_trigger(p_proc.p_vmspace,QWORD(node^.addr),QWORD(node^.addr)+4,nil,0); + vm_map_track_trigger(p_proc.p_vmspace,QWORD(node^.addr),QWORD(node^.addr)+4,nil,M_DMEM_WRITE); end; end; @@ -1641,7 +1668,7 @@ begin Move(src_dmem^,dst_dmem^,byteSize); - vm_map_track_trigger(p_proc.p_vmspace,QWORD(node^.dst),QWORD(node^.dst)+byteSize,nil,0); + vm_map_track_trigger(p_proc.p_vmspace,QWORD(node^.dst),QWORD(node^.dst)+byteSize,nil,M_DMEM_WRITE); end; end; else @@ -1792,6 +1819,8 @@ begin Continue; end; + ctx.PingCmd; + //stall is empty! me^.reset_sheduler; diff --git a/chip/pm4_pfp.pas b/chip/pm4_pfp.pas index ce49ec6b..4cc31003 100644 --- a/chip/pm4_pfp.pas +++ b/chip/pm4_pfp.pas @@ -88,7 +88,8 @@ uses sys_bootparam, kern_dmem, kern_proc, - vm_map; + vm_map, + vm_tracking_map; function PM4_TYPE(token:DWORD):Byte; inline; begin @@ -1074,14 +1075,14 @@ begin Move(Pointer(adrSrc_dmem)^,Pointer(adrDst_dmem)^,byteCount); - vm_map_track_trigger(p_proc.p_vmspace,QWORD(adrDst),QWORD(adrDst)+byteCount,nil,0); + vm_map_track_trigger(p_proc.p_vmspace,QWORD(adrDst),QWORD(adrDst)+byteCount,nil,M_DMEM_WRITE); end; (kDmaDataSrcData or (kDmaDataDstMemory shl 4)), (kDmaDataSrcData or (kDmaDataDstMemoryUsingL2 shl 4)): begin FillDWORD(Pointer(adrDst_dmem)^,(byteCount div 4),DWORD(adrSrc)); - vm_map_track_trigger(p_proc.p_vmspace,QWORD(adrDst),QWORD(adrDst)+byteCount,nil,0); + vm_map_track_trigger(p_proc.p_vmspace,QWORD(adrDst),QWORD(adrDst)+byteCount,nil,M_DMEM_WRITE); end; else Assert(false,'DmaData: srcSel=0x'+HexStr(srcSel,1)+' dstSel=0x'+HexStr(dstSel,1)); @@ -1145,7 +1146,7 @@ begin Move(src_dmem^,dst_dmem^,count*SizeOf(DWORD)); - vm_map_track_trigger(p_proc.p_vmspace,QWORD(dst),QWORD(dst)+count*SizeOf(DWORD),nil,0); + vm_map_track_trigger(p_proc.p_vmspace,QWORD(dst),QWORD(dst)+count*SizeOf(DWORD),nil,M_DMEM_WRITE); end; else Assert(false,'WriteData: dstSel=0x'+HexStr(dstSel,1)); diff --git a/chip/pm4_stream.pas b/chip/pm4_stream.pas index 675d146e..4d9ba5a7 100644 --- a/chip/pm4_stream.pas +++ b/chip/pm4_stream.pas @@ -1298,13 +1298,11 @@ Procedure t_pm4_allocator.Free; var node:PAllocNode; begin - node:=pHead.slh_first; - if (node<>nil) then - begin - pHead.slh_first:=node^.link; - end; + node:=SLIST_FIRST(@pHead); + While (node<>nil) do begin + SLIST_REMOVE(@pHead,node,@node^.link); if (node^.size=cache_block_allocator.mem_size) then begin @@ -1314,11 +1312,7 @@ begin md_unmap(node,node^.size); end; - node:=pHead.slh_first; - if (node<>nil) then - begin - pHead.slh_first:=node^.link; - end; + node:=SLIST_FIRST(@pHead); end; Self:=Default(t_pm4_allocator); end; diff --git a/sys/jit/kern_jit.pas b/sys/jit/kern_jit.pas index 6316c7b1..9fdd3d8d 100644 --- a/sys/jit/kern_jit.pas +++ b/sys/jit/kern_jit.pas @@ -1278,9 +1278,14 @@ begin Assert(false,'TODO: destroy in code analize'); end; -function pick_on_trigger(handle:Pointer;mode:Integer):Integer; +function pick_on_trigger(handle:Pointer;mode:T_TRIGGER_MODE):Integer; begin - if (mode=1) then Exit(DO_NOTHING); + case mode of + M_CPU_WRITE :; + M_DMEM_WRITE:; + else + Exit; + end; Result:=DO_NOTHING; diff --git a/sys/jit/kern_jit_dynamic.pas b/sys/jit/kern_jit_dynamic.pas index 4e4d487f..b37d2eeb 100644 --- a/sys/jit/kern_jit_dynamic.pas +++ b/sys/jit/kern_jit_dynamic.pas @@ -1612,9 +1612,15 @@ begin Result:=DO_DELETE; end; -function on_trigger(handle:Pointer;mode:Integer):Integer; +function on_trigger(handle:Pointer;mode:T_TRIGGER_MODE):Integer; begin - if (mode=1) then Exit(DO_NOTHING); + case mode of + M_CPU_WRITE :; + M_DMEM_WRITE:; + else + Exit; + end; + //TODO Result:=DO_INCREMENT; end; diff --git a/sys/kern/kern_dmem.pas b/sys/kern/kern_dmem.pas index 7f5fb2c2..ed1d54b9 100644 --- a/sys/kern/kern_dmem.pas +++ b/sys/kern/kern_dmem.pas @@ -140,8 +140,8 @@ procedure init_dmem_map; var vmap:vm_map_t; begin - dmem_map_init(@dmem,0,{SCE_KERNEL_MAIN_DMEM_SIZE} (VM_MAX_GPU_ADDRESS-VM_MIN_GPU_ADDRESS) ); - rmem_map_init(@rmap,0,{SCE_KERNEL_MAIN_DMEM_SIZE} (VM_MAX_GPU_ADDRESS-VM_MIN_GPU_ADDRESS) ); + dmem_map_init(@dmem,0,SCE_KERNEL_MAIN_DMEM_SIZE); + rmem_map_init(@rmap,0,(VM_MAX_GPU_ADDRESS-VM_MIN_GPU_ADDRESS)); vmap:=p_proc.p_vmspace; @@ -523,6 +523,8 @@ var start:QWORD; relofs:Int64; + offset:QWORD; + d_start,d_start2:QWORD; d_end,d_end2:QWORD; d_mtype:DWORD; @@ -564,20 +566,42 @@ begin if ((obj^.flags and OBJ_DMEM_EXT)<>0) then begin + offset:=entry^.offset; + + //transform by base addr + offset:=offset + (QWORD(obj^.un_pager.map_base) - VM_MIN_GPU_ADDRESS); + qinfo^.protection:=qinfo^.protection and (VM_PROT_GPU_ALL or VM_PROT_RW); start :=entry^.start; - relofs:=entry^.offset - start; + relofs:=offset - start; if (addr < start) then begin addr:=start; end; - ret:=dmem_map_get_mtype(dmem_maps[default_pool_id].dmem,obj,addr + relofs,@d_start2,@d_end2,@d_mtype); - if (ret<>0) then + if (offset>=(VM_MIN_DEV_ADDRESS-VM_MIN_GPU_ADDRESS)) then begin - Assert(false,'dmem_vmo_get_type error %d'); + //dev + + d_start2:=(VM_MIN_DEV_ADDRESS-VM_MIN_GPU_ADDRESS); + d_end2 :=(VM_MAX_DEV_ADDRESS-VM_MIN_GPU_ADDRESS); + d_mtype :=SCE_KERNEL_WB_ONION; + end else + begin + //dmem + + ret:=dmem_map_get_mtype(dmem_maps[default_pool_id].dmem, + obj, + addr + (entry^.offset - start), //send not transformed offset + @d_start2,@d_end2, + @d_mtype); + if (ret<>0) then + begin + Assert(false,'dmem_vmo_get_type error %d'); + end; + end; qinfo^.bits.isDirectMemory:=1; diff --git a/sys/md/vm_pmap.pas b/sys/md/vm_pmap.pas index 001837be..64cd1a6b 100644 --- a/sys/md/vm_pmap.pas +++ b/sys/md/vm_pmap.pas @@ -44,7 +44,6 @@ procedure iov_uplift(iov:p_iovec); type p_pmap=^t_pmap; t_pmap=packed object - vm_map:Pointer; rmlock:rangelock; rm_mtx:mtx; nt_map:t_vm_nt_map; @@ -63,7 +62,7 @@ function dev_mem_alloc(pages:Integer):Pointer; function pmap_reserve(wr:Boolean):DWORD; -procedure pmap_pinit(pmap:p_pmap;vm_map:Pointer); +procedure pmap_pinit(pmap:p_pmap); procedure pmap_align_superpage(obj :vm_object_t; offset:vm_ooffset_t; @@ -287,12 +286,10 @@ begin end; end; -procedure pmap_pinit(pmap:p_pmap;vm_map:Pointer); +procedure pmap_pinit(pmap:p_pmap); var i,r:Integer; begin - pmap^.vm_map:=vm_map; - r:=pmap_reserve(True); Assert(r=0,'pmap_pinit'); @@ -707,8 +704,6 @@ begin delta:=(info.__end-info.start); if (delta=0) then Break; - vm_map_lock(pmap^.vm_map); - r:=vm_nt_map_insert(@pmap^.nt_map, info.obj, info.start and PMAPP_BLK_MASK, //block local offset @@ -717,24 +712,18 @@ begin delta, (prot and VM_RW)); - vm_map_unlock(pmap^.vm_map,False); - if (r<>0) then begin Writeln('failed vm_nt_map_insert:0x',HexStr(r,8)); Assert(false,'pmap_enter_object'); end; - vm_map_lock(pmap^.vm_map); - //fill zero if needed vm_nt_map_madvise(@pmap^.nt_map, info.start, info.__end, MADV_NORMAL); - vm_map_unlock(pmap^.vm_map,False); - info.start :=info.start+delta; info.__end :=__end; end; @@ -775,8 +764,6 @@ begin Writeln('vm_nt_map_insert:',HexStr(info.start,11),':',HexStr(info.__end,11),':',HexStr(info.offset,11)); end; - vm_map_lock(pmap^.vm_map); - r:=vm_nt_map_insert(@pmap^.nt_map, info.obj, info.offset and PMAPP_BLK_MASK, //block local offset @@ -785,8 +772,6 @@ begin delta, (prot and VM_RW)); - vm_map_unlock(pmap^.vm_map,False); - if (r<>0) then begin Writeln('failed vm_nt_map_insert:0x',HexStr(r,8)); @@ -873,8 +858,6 @@ begin delta:=(info.__end-info.start); if (delta=0) then Break; - vm_map_lock(pmap^.vm_map); - r:=vm_nt_map_insert(@pmap^.nt_map, info.obj, info.start and PMAPP_BLK_MASK, //block local offset @@ -883,24 +866,18 @@ begin delta, (prot and VM_RW)); - vm_map_unlock(pmap^.vm_map,False); - if (r<>0) then begin Writeln('failed vm_nt_map_insert:0x',HexStr(r,8)); Assert(false,'pmap_enter_object'); end; - vm_map_lock(pmap^.vm_map); - //restore vm_nt_map_madvise(@pmap^.nt_map, info.start, info.__end, MADV_WILLNEED); - vm_map_unlock(pmap^.vm_map,False); - //copy pmap_copy(cow, info.offset, @@ -925,8 +902,6 @@ begin info.start :=start; info.__end :=start+paddi; - vm_map_lock(pmap^.vm_map); - r:=vm_nt_map_insert(@pmap^.nt_map, info.obj, info.offset, //offset in file @@ -935,8 +910,6 @@ begin size, (prot and VM_RW)); - vm_map_unlock(pmap^.vm_map,False); - if (r<>0) then begin Writeln('failed vm_nt_map_insert:0x',HexStr(r,8)); @@ -994,8 +967,6 @@ begin begin _default: - vm_map_lock(pmap^.vm_map); - vm_nt_map_prot_fix(@pmap^.nt_map, start, __end, @@ -1006,8 +977,6 @@ begin // __end, // (prot and VM_RW)); - vm_map_unlock(pmap^.vm_map,False); - end; OBJT_DEVICE: begin @@ -1051,14 +1020,10 @@ begin start:=start and (not PMAPP_MASK); __end:=(__end+PMAPP_MASK) and (not PMAPP_MASK); - vm_map_lock(pmap^.vm_map); - vm_nt_map_prot_fix(@pmap^.nt_map, start, __end, mode); - - vm_map_unlock(pmap^.vm_map,False); end; procedure _pmap_prot_int(pmap :pmap_t; @@ -1069,14 +1034,10 @@ begin start:=start and (not PMAPP_MASK); __end:=(__end+PMAPP_MASK) and (not PMAPP_MASK); - vm_map_lock(pmap^.vm_map); - vm_nt_map_protect(@pmap^.nt_map, start, __end, (prot and VM_RW)); - - vm_map_unlock(pmap^.vm_map,False); end; procedure pmap_prot_track(pmap :pmap_t; @@ -1087,16 +1048,12 @@ begin start:=start and (not PMAPP_MASK); __end:=(__end+PMAPP_MASK) and (not PMAPP_MASK); - vm_map_lock(pmap^.vm_map); - ppmap_track(start,__end,prots); vm_nt_map_prot_fix(@pmap^.nt_map, start, __end, TRACK_PROT or REMAP_PROT); - - vm_map_unlock(pmap^.vm_map,False); end; procedure pmap_madvise(pmap :pmap_t; @@ -1126,14 +1083,10 @@ begin begin _default: - vm_map_lock(pmap^.vm_map); - vm_nt_map_madvise(@pmap^.nt_map, start, __end, advise); - - vm_map_unlock(pmap^.vm_map,False); end; OBJT_DEVICE: begin @@ -1195,24 +1148,17 @@ begin OBJT_DEFAULT: begin - vm_map_lock(pmap^.vm_map); vm_nt_map_madvise(@pmap^.nt_map, start, __end, MADV_FREE); - vm_map_unlock(pmap^.vm_map,False); - _default: - vm_map_lock(pmap^.vm_map); - r:=vm_nt_map_delete(@pmap^.nt_map, start, __end); - - vm_map_unlock(pmap^.vm_map,False); end; OBJT_DEVICE: begin @@ -1262,14 +1208,10 @@ var begin lock:=pmap_rlock(pmap,start,__end); - vm_map_lock(pmap^.vm_map); - Result:=vm_nt_map_mirror(@pmap^.nt_map, start, __end); - vm_map_unlock(pmap^.vm_map,False); - pmap_unlock(pmap,lock); end; diff --git a/sys/vm/rmem_map.pas b/sys/vm/rmem_map.pas index 1ef63ddb..ce7dc02a 100644 --- a/sys/vm/rmem_map.pas +++ b/sys/vm/rmem_map.pas @@ -736,7 +736,7 @@ function rmem_map_insert(map:p_rmem_map; vaddr:QWORD; start,__end:DWORD):Integer; var - current,entry:p_rmem_map_entry; + entry:p_rmem_map_entry; begin if (start=__end) then begin @@ -760,19 +760,18 @@ begin entry:=rmem_map_insert_internal(map,entry,start,__end); end; - current:=entry; - while (current<>@map^.header) and (current^.start<__end) do + while (entry<>@map^.header) and (entry^.start<__end) do begin - rmem_map_clip_end(map, current, __end); + rmem_map_clip_end(map, entry, __end); - current:=rmem_map_insert_internal(map,current,current^.__end,__end); + entry:=rmem_map_insert_internal(map,entry,entry^.__end,__end); - if rmem_entry_add_vaddr(current,vaddr) then + if rmem_entry_add_vaddr(entry,vaddr) then begin rmem_entry_add_track(map^.tmap,entry,vaddr); end; - current:=current^.next; + entry:=entry^.next; end; rmem_map_unlock(map); diff --git a/sys/vm/vm_map.pas b/sys/vm/vm_map.pas index 9aa18f8e..89d190d8 100644 --- a/sys/vm/vm_map.pas +++ b/sys/vm/vm_map.pas @@ -15,7 +15,8 @@ uses kern_mtx, kern_thr, sys_resource, - kern_resource; + kern_resource, + vm_tracking_map; type vm_flags_t =type Byte; @@ -278,9 +279,9 @@ procedure vm_map_set_name_locked(map:vm_map_t;start,__end:vm_offset_t;name:PChar procedure vm_map_track_insert(map:vm_map_t;tobj:Pointer); procedure vm_map_track_remove(map:vm_map_t;tobj:Pointer); -function vm_map_track_next (map:vm_map_t;start:vm_offset_t;tobj:Pointer;htype:Byte):Pointer; +function vm_map_track_next (map:vm_map_t;start:vm_offset_t;tobj:Pointer;htype:T_THANDLE_TYPE):Pointer; procedure _vm_map_track_delete_deferred(map:vm_map_t;tobj:Pointer); -function vm_map_track_trigger(map:vm_map_t;start,__end:vm_offset_t;exclude:Pointer;mode:Integer):Integer; +function vm_map_track_trigger(map:vm_map_t;start,__end:vm_offset_t;exclude:Pointer;mode:T_TRIGGER_MODE):Integer; function vmspace_pmap(vm:p_vmspace):pmap_t; inline; @@ -294,7 +295,6 @@ uses md_map, kern_proc, rmem_map, - vm_tracking_map, kern_budget; var @@ -385,7 +385,7 @@ var begin vm:=@g_vmspace; - pmap_pinit(vmspace_pmap(vm),@vm^.vm_map); + pmap_pinit(vmspace_pmap(vm)); vm_map_init(@vm^.vm_map,vmspace_pmap(vm),VM_MINUSER_ADDRESS,VM_MAXUSER_ADDRESS); @@ -3408,14 +3408,10 @@ procedure vm_map_track_remove(map:vm_map_t;tobj:Pointer); begin if (tobj=nil) then Exit; - vm_map_lock(map); //lock to prevent deadlock - vm_track_map_remove_object(@map^.pmap^.tr_map,tobj); - - vm_map_unlock(map); end; -function vm_map_track_next(map:vm_map_t;start:vm_offset_t;tobj:Pointer;htype:Byte):Pointer; +function vm_map_track_next(map:vm_map_t;start:vm_offset_t;tobj:Pointer;htype:T_THANDLE_TYPE):Pointer; begin Result:=vm_track_map_next_object(@map^.pmap^.tr_map,start,tobj,htype); end; @@ -3425,7 +3421,7 @@ begin _vm_track_map_delete_deferred(@map^.pmap^.tr_map,tobj); end; -function vm_map_track_trigger(map:vm_map_t;start,__end:vm_offset_t;exclude:Pointer;mode:Integer):Integer; +function vm_map_track_trigger(map:vm_map_t;start,__end:vm_offset_t;exclude:Pointer;mode:T_TRIGGER_MODE):Integer; begin Result:=vm_track_map_trigger(@map^.pmap^.tr_map,start,__end,exclude,mode); end; diff --git a/sys/vm/vm_nt_map.pas b/sys/vm/vm_nt_map.pas index ba4e4057..562d2f95 100644 --- a/sys/vm/vm_nt_map.pas +++ b/sys/vm/vm_nt_map.pas @@ -65,6 +65,7 @@ type nentries :Integer; // Number of entries root :p_vm_nt_entry; // Root of a binary search tree danger_zone:t_danger_zone; + lock :mtx; property min_offset:vm_offset_t read header.start write header.start; property max_offset:vm_offset_t read header.__end write header.__end; end; @@ -79,11 +80,6 @@ function vm_nt_map_min(map:p_vm_nt_map):vm_offset_t; procedure vm_nt_map_init(map:p_vm_nt_map;min,max:vm_offset_t); -function vm_nt_map_lookup_entry( - map :p_vm_nt_map; - address:vm_offset_t; - entry :pp_vm_nt_entry):Boolean; - function vm_nt_map_insert( map :p_vm_nt_map; obj :p_vm_nt_file_obj; @@ -114,8 +110,6 @@ function vm_nt_map_mirror(map:p_vm_nt_map; start:vm_offset_t; __end:vm_offset_t):Pointer; -procedure vm_nt_entry_deallocate(entry:p_vm_nt_entry); - implementation uses @@ -678,6 +672,17 @@ begin map^.max_offset:=max; map^.root:=nil; map^.danger_zone.Init; + mtx_init(map^.lock,'vm_nt_map'); +end; + +procedure vm_nt_map_lock(map:p_vm_nt_map); inline; +begin + mtx_lock(map^.lock); +end; + +procedure vm_nt_map_unlock(map:p_vm_nt_map); inline; +begin + mtx_unlock(map^.lock); end; procedure vm_nt_entry_dispose(map:p_vm_nt_map;entry:p_vm_nt_entry); inline; @@ -863,7 +868,7 @@ end; function vm_nt_map_simplify_entry(map:p_vm_nt_map;entry:p_vm_nt_entry;var sb:t_range_stat):Boolean; forward; -function vm_nt_map_insert( +function _vm_nt_map_insert( map :p_vm_nt_map; obj :p_vm_nt_file_obj; offset:vm_ooffset_t; @@ -914,6 +919,34 @@ begin Result:=KERN_SUCCESS; end; +function vm_nt_map_insert( + map :p_vm_nt_map; + obj :p_vm_nt_file_obj; + offset:vm_ooffset_t; + start :vm_offset_t; + __end :vm_offset_t; + size :vm_offset_t; //unaligned size + prot :Integer):Integer; +begin + if (start=__end) then + begin + Exit(KERN_SUCCESS); + end; + + vm_nt_map_lock(map); + + Result:=_vm_nt_map_insert( + map , + obj , + offset, + start , + __end , + size , + prot ); + + vm_nt_map_unlock(map); +end; + function vm_nt_map_simplify_entry(map:p_vm_nt_map;entry:p_vm_nt_entry;var sb:t_range_stat):Boolean; var next,prev:p_vm_nt_entry; @@ -1124,6 +1157,8 @@ begin Exit(KERN_SUCCESS); end; + vm_nt_map_lock(map); + if (not vm_nt_map_lookup_entry(map, start, @first_entry)) then begin entry:=first_entry^.next; @@ -1148,6 +1183,9 @@ begin entry:=next; end; + + vm_nt_map_unlock(map); + Result:=(KERN_SUCCESS); end; @@ -1164,6 +1202,8 @@ var begin if (start=__end) then Exit; + vm_nt_map_lock(map); + if (not vm_nt_map_lookup_entry(map, start, @entry)) then begin entry:=entry^.next; @@ -1209,6 +1249,8 @@ begin entry:=entry^.next; end; + + vm_nt_map_unlock(map); end; procedure vm_nt_map_prot_fix(map:p_vm_nt_map; @@ -1222,6 +1264,8 @@ var begin if (start=__end) then Exit; + vm_nt_map_lock(map); + if (not vm_nt_map_lookup_entry(map, start, @entry)) then begin entry:=entry^.next; @@ -1256,6 +1300,8 @@ begin entry:=entry^.next; end; + + vm_nt_map_unlock(map); end; //rdi, rsi @@ -1292,6 +1338,8 @@ var begin if (start=__end) then Exit; + vm_nt_map_lock(map); + if (not vm_nt_map_lookup_entry(map, start, @entry)) then begin entry:=entry^.next; @@ -1343,6 +1391,8 @@ begin entry:=entry^.next; end; + + vm_nt_map_unlock(map); end; @@ -1369,6 +1419,8 @@ begin Exit; end; + vm_nt_map_lock(map); + if (not vm_nt_map_lookup_entry(map, start, @entry)) then begin entry:=entry^.next; @@ -1436,6 +1488,8 @@ begin entry:=entry^.next; end; + + vm_nt_map_unlock(map); end; diff --git a/sys/vm/vm_tracking_map.pas b/sys/vm/vm_tracking_map.pas index 2c61abbb..05098c99 100644 --- a/sys/vm/vm_tracking_map.pas +++ b/sys/vm/vm_tracking_map.pas @@ -26,27 +26,21 @@ const DO_DELETE =1; DO_INCREMENT=2; - H_ZERO =0; - H_JIT_CHUNK=1; - H_GPU_IMAGE=2; - H_GPU_BUF =3; +type + T_THANDLE_TYPE=(H_ZERO,H_JIT_CHUNK,H_GPU_IMAGE,H_GPU_BUF); + T_TRIGGER_MODE=(M_CPU_WRITE,M_DMEM_WRITE,M_GPU_PLANNED,M_GPU_APPLY); type t_on_destroy=function(handle:Pointer):Integer; - t_on_trigger=function(handle:Pointer;mode:Integer):Integer; + t_on_trigger=function(handle:Pointer;mode:T_TRIGGER_MODE):Integer; p_vm_track_object=^t_vm_track_object; - t_vm_track_object=record + t_vm_track_object=packed record del_link :TAILQ_ENTRY; //p_vm_track_map->delete_deferred //iter_link :TAILQ_ENTRY; // handle :Pointer; // - ref_count :DWORD; - htype :Byte; - mark_del :Byte; - prot :Byte; - // main :t_vm_track_interval; align :t_vm_track_interval; instances :TAILQ_HEAD; //p_vm_track_object_instance @@ -54,6 +48,10 @@ type on_destroy:t_on_destroy; on_trigger:t_on_trigger; // + ref_count :DWORD; + htype :T_THANDLE_TYPE; + mark_del :Byte; + prot :Byte; end; pp_vm_track_map_entry=^p_vm_track_map_entry; @@ -65,7 +63,6 @@ type right :p_vm_track_map_entry; // right child in binary search tree start :vm_offset_t; // start address __end :vm_offset_t; // end address - source :vm_offset_t; // source of mirror instances:Pointer; // p_vm_track_object_instance instcount:QWORD; // @@ -82,6 +79,7 @@ type obj_link :TAILQ_ENTRY; //p_vm_track_object ->instances entry :p_vm_track_map_entry; obj :p_vm_track_object; + source :vm_offset_t; // source of mirror end; p_vm_track_deferred=^t_vm_track_deferred; @@ -117,7 +115,7 @@ procedure vm_track_map_init(map:p_vm_track_map;min,max:vm_offset_t); // -function vm_track_object_allocate (handle:Pointer;start,__end:vm_offset_t;htype,prot:Byte):p_vm_track_object; +function vm_track_object_allocate (handle:Pointer;start,__end:vm_offset_t;htype:T_THANDLE_TYPE;prot:Byte):p_vm_track_object; procedure vm_track_object_deallocate(obj:p_vm_track_object); procedure vm_track_object_reference (obj:p_vm_track_object); @@ -137,9 +135,9 @@ function _vm_track_map_insert_mirror(map:p_vm_track_map;start,__end,dst:vm_offs function vm_track_map_remove_object(map:p_vm_track_map;obj:p_vm_track_object):Integer; function vm_track_map_remove_memory(map:p_vm_track_map;start,__end:vm_offset_t):Integer; -function vm_track_map_trigger (map:p_vm_track_map;start,__end:vm_offset_t;exclude:Pointer;mode:Integer):Integer; +function vm_track_map_trigger (map:p_vm_track_map;start,__end:vm_offset_t;exclude:Pointer;mode:T_TRIGGER_MODE):Integer; -function vm_track_map_next_object (map:p_vm_track_map;start:vm_offset_t;obj:p_vm_track_object;htype:Byte):p_vm_track_object; +function vm_track_map_next_object (map:p_vm_track_map;start:vm_offset_t;obj:p_vm_track_object;htype:T_THANDLE_TYPE):p_vm_track_object; implementation @@ -148,7 +146,7 @@ procedure pmap_prot_track(pmap :Pointer; __end:vm_offset_t; prots:Byte); external; -function vm_track_object_allocate(handle:Pointer;start,__end:vm_offset_t;htype,prot:Byte):p_vm_track_object; +function vm_track_object_allocate(handle:Pointer;start,__end:vm_offset_t;htype:T_THANDLE_TYPE;prot:Byte):p_vm_track_object; begin Result:=AllocMem(SizeOf(t_vm_track_object)); @@ -211,7 +209,7 @@ begin end; } -function vm_track_object_trigger(map:p_vm_track_map;obj:p_vm_track_object;start,__end:vm_offset_t;mode:Integer):Integer; +function vm_track_object_trigger(obj:p_vm_track_object;start,__end:vm_offset_t;mode:T_TRIGGER_MODE):Integer; begin Result:=DO_NOTHING; @@ -430,15 +428,16 @@ begin Exit(y); end; -procedure _vm_track_entry_add_obj(pmap:Pointer;entry:p_vm_track_map_entry;obj:p_vm_track_object); +procedure _vm_track_entry_add_obj(pmap:Pointer;entry:p_vm_track_map_entry;obj:p_vm_track_object;source:vm_offset_t); var node:p_vm_track_object_instance; prot:Byte; begin node:=AllocMem(SizeOf(t_vm_track_object_instance)); - node^.entry:=entry; - node^.obj :=obj; + node^.entry :=entry; + node^.obj :=obj; + node^.source:=source; vm_track_insert_instance(entry^.instances,node); Inc(entry^.instcount); @@ -475,7 +474,7 @@ begin // end; -procedure vm_track_entry_add_obj(pmap:Pointer;entry:p_vm_track_map_entry;obj:p_vm_track_object); +procedure vm_track_entry_add_obj(pmap:Pointer;entry:p_vm_track_map_entry;obj:p_vm_track_object;source:vm_offset_t); var root:p_vm_track_object_instance; begin @@ -488,7 +487,7 @@ begin Exit; end; - _vm_track_entry_add_obj(pmap,entry,obj); + _vm_track_entry_add_obj(pmap,entry,obj,source); end; function _vm_track_entry_del_node(pmap:Pointer;entry:p_vm_track_map_entry;node:p_vm_track_object_instance):Boolean; @@ -577,7 +576,7 @@ end; // -function in_obj_list(b:p_vm_track_map_entry;obj:p_vm_track_object):Boolean; +function in_obj_list(b:p_vm_track_map_entry;obj:p_vm_track_object;source:vm_offset_t):Boolean; var root:p_vm_track_object_instance; begin @@ -591,10 +590,10 @@ begin Exit; end; - Result:=(root^.obj=obj); + Result:=(root^.obj=obj) and (root^.source=source); end; -function compare_obj_list(a,b:p_vm_track_map_entry):Boolean; +function compare_obj_list(a:p_vm_track_map_entry;offset:vm_offset_t;b:p_vm_track_map_entry):Boolean; var node:p_vm_track_object_instance; begin @@ -608,7 +607,7 @@ begin while (node<>nil) do begin - if not in_obj_list(b,node^.obj) then + if not in_obj_list(b,node^.obj,node^.source+offset) then begin Exit(False); end; @@ -619,7 +618,35 @@ begin Result:=True; end; -procedure copy_obj_list(src,dst:p_vm_track_map_entry); +procedure inc_obj_list(src:p_vm_track_map_entry;offset:vm_offset_t); +var + node:p_vm_track_object_instance; +begin + node:=vm_track_first_instance(src^.instances); + + while (node<>nil) do + begin + node^.source:=node^.source+offset; + + node:=vm_track_next_instance(src^.instances,node); + end; +end; + +procedure dec_obj_list(src:p_vm_track_map_entry;offset:vm_offset_t); +var + node:p_vm_track_object_instance; +begin + node:=vm_track_first_instance(src^.instances); + + while (node<>nil) do + begin + node^.source:=node^.source-offset; + + node:=vm_track_next_instance(src^.instances,node); + end; +end; + +procedure copy_obj_list(src,dst:p_vm_track_map_entry;offset:vm_offset_t); var node:p_vm_track_object_instance; begin @@ -630,7 +657,7 @@ begin while (node<>nil) do begin - _vm_track_entry_add_obj(nil,dst,node^.obj); + _vm_track_entry_add_obj(nil,dst,node^.obj,node^.source+offset); node:=vm_track_next_instance(src^.instances,node); end; @@ -921,8 +948,7 @@ function vm_track_map_insert_internal( map :p_vm_track_map; after :p_vm_track_map_entry; start :vm_offset_t; - __end :vm_offset_t; - source:vm_offset_t):p_vm_track_map_entry; + __end :vm_offset_t):p_vm_track_map_entry; var new_entry:p_vm_track_map_entry; begin @@ -931,8 +957,6 @@ begin if (after<>@map^.header) and (start=__end) then begin @@ -1080,23 +1102,21 @@ begin vm_track_map_clip_start(map, entry, start); end else begin - entry:=vm_track_map_insert_internal(map,entry,start,__end,source); + entry:=vm_track_map_insert_internal(map,entry,start,__end); end; - current:=entry; - while (current<>@map^.header) and (current^.start<__end) do + while (entry<>@map^.header) and (entry^.start<__end) do begin - vm_track_map_clip_end(map, current, __end); + vm_track_map_clip_end(map, entry, __end); - current:=vm_track_map_insert_internal(map,current, - current^.__end, //start - __end, //__end - source+(current^.__end-start) + entry:=vm_track_map_insert_internal(map,entry, + entry^.__end, //start + __end //__end ); - vm_track_entry_add_obj(map^.pmap,current,obj); + vm_track_entry_add_obj(map^.pmap,entry,obj,source+(entry^.start-start)); - current:=current^.next; + entry:=entry^.next; end; Result:=(KERN_SUCCESS); @@ -1417,9 +1437,9 @@ begin Result:=(KERN_SUCCESS); end; -function vm_track_map_trigger(map:p_vm_track_map;start,__end:vm_offset_t;exclude:Pointer;mode:Integer):Integer; +function vm_track_map_trigger(map:p_vm_track_map;start,__end:vm_offset_t;exclude:Pointer;mode:T_TRIGGER_MODE):Integer; var - current,entry:p_vm_track_map_entry; + entry:p_vm_track_map_entry; node:p_vm_track_object_instance; diff:vm_offset_t; @@ -1456,20 +1476,10 @@ begin entry:=entry^.next; end; - current:=entry; - while (current<>@map^.header) and (current^.start<__end) do + while (entry<>@map^.header) and (entry^.start<__end) do begin node:=vm_track_first_instance(entry^.instances); - //remap with source - //diff:=entry^.start-start; - // - //s_start:=entry^.source-diff; - //s___end:=s_start+size; - - s_start:=start; - s___end:=__end; - while (node<>nil) do begin //vm_track_list_add_obj(list,node^.obj); //deferred @@ -1477,7 +1487,14 @@ begin //remap with source if (node^.obj<>exclude) then begin - ret:=vm_track_object_trigger(map,node^.obj,s_start,s___end,mode); + + //remap with source + diff:=entry^.start-start; + // + s_start:=node^.source-diff; + s___end:=s_start+size; + + ret:=vm_track_object_trigger(node^.obj,s_start,s___end,mode); if ((ret and DO_DELETE)<>0) then begin @@ -1494,7 +1511,7 @@ begin node:=vm_track_next_instance(entry^.instances,node); end; - current:=current^.next; + entry:=entry^.next; end; //iterate @@ -1532,7 +1549,7 @@ begin vm_track_map_unlock(map); end; -function vm_track_map_next_object(map:p_vm_track_map;start:vm_offset_t;obj:p_vm_track_object;htype:Byte):p_vm_track_object; +function vm_track_map_next_object(map:p_vm_track_map;start:vm_offset_t;obj:p_vm_track_object;htype:T_THANDLE_TYPE):p_vm_track_object; var entry:p_vm_track_map_entry; node:p_vm_track_object_instance; diff --git a/vulkan/vCmdBuffer.pas b/vulkan/vCmdBuffer.pas index 08fa469c..1b406312 100644 --- a/vulkan/vCmdBuffer.pas +++ b/vulkan/vCmdBuffer.pas @@ -12,6 +12,7 @@ uses kern_proc, vm_map, + vm_tracking_map, Vulkan, vDependence, @@ -690,7 +691,7 @@ begin begin //deffered trigger - vm_map_track_trigger(p_proc.p_vmspace,node^.start,node^.__end,node^.exclude,2); + vm_map_track_trigger(p_proc.p_vmspace,node^.start,node^.__end,node^.exclude,M_GPU_APPLY); FPlannedTriggers.delete(node); OnFree(node); @@ -706,7 +707,7 @@ Var node:p_cmd_track_deferred; begin //planned trigger - vm_map_track_trigger(p_proc.p_vmspace,start,__end,exclude,1); + vm_map_track_trigger(p_proc.p_vmspace,start,__end,exclude,M_GPU_PLANNED); tmp:=Default(t_cmd_track_deferred); tmp.start :=start; diff --git a/vulkan/vImageManager.pas b/vulkan/vImageManager.pas index 0a3f8b29..ac93eddf 100644 --- a/vulkan/vImageManager.pas +++ b/vulkan/vImageManager.pas @@ -211,7 +211,7 @@ begin Result:=DO_DELETE; end; -function on_trigger(handle:Pointer;mode:Integer):Integer; SysV_ABI_CDecl; +function on_trigger(handle:Pointer;mode:T_TRIGGER_MODE):Integer; SysV_ABI_CDecl; var image:TvCustomImage2; i:Integer; @@ -223,15 +223,17 @@ begin //Writeln('on_trigger image'); case mode of - 0://direct + M_CPU_WRITE, + M_DMEM_WRITE: + //direct begin System.InterlockedIncrement(image.change_rate.trigger); end; - 1://planned + M_GPU_PLANNED://planned begin System.InterlockedIncrement(image.change_rate.planned); end; - 2://differed + M_GPU_APPLY://differed begin i:=System.InterlockedExchangeAdd(image.change_rate.planned,0);