From d52673c1edabc9993fdabfe85757111373cd7ddd Mon Sep 17 00:00:00 2001 From: Pavel <68122101+red-prig@users.noreply.github.com> Date: Fri, 21 Jun 2024 14:35:06 +0300 Subject: [PATCH] + --- sys/jit/kern_jit.pas | 2 +- sys/jit/kern_jit_dynamic.pas | 233 ++++++++++++++++++++++++++++------- sys/kern/kern_rangelock.pas | 172 +++++++++++++++++--------- sys/md/vm_pmap.pas | 6 + 4 files changed, 309 insertions(+), 104 deletions(-) diff --git a/sys/jit/kern_jit.pas b/sys/jit/kern_jit.pas index 4b730c29..9c3f0faa 100644 --- a/sys/jit/kern_jit.pas +++ b/sys/jit/kern_jit.pas @@ -1311,7 +1311,7 @@ begin node:=preload_entry(preload); if (node<>nil) then begin - node^.dec_ref; + node^.dec_ref('preload_entry'); goto _exit; end; end; diff --git a/sys/jit/kern_jit_dynamic.pas b/sys/jit/kern_jit_dynamic.pas index 94e55d8d..5a3cd1ec 100644 --- a/sys/jit/kern_jit_dynamic.pas +++ b/sys/jit/kern_jit_dynamic.pas @@ -38,8 +38,8 @@ type blob:p_jit_dynamic_blob; src :Pointer; //<-guest dst :Pointer; //<-host - procedure inc_ref; - procedure dec_ref; + procedure inc_ref(name:pchar); + procedure dec_ref(name:pchar); end; p_jinstr_len=^t_jinstr_len; @@ -68,8 +68,8 @@ type count :QWORD; table :record end; //p_jinstr_len[] function c(n1,n2:p_jcode_chunk):Integer; static; - procedure inc_ref; - procedure dec_ref; + procedure inc_ref(name:pchar); + procedure dec_ref(name:pchar); function is_mark_del:Boolean; function find_host_by_guest(addr:QWORD):QWORD; function find_guest_by_host(addr:QWORD):QWORD; @@ -81,8 +81,10 @@ type p_jplt_cache=^t_jplt_cache; t_jplt_cache=object(t_jplt_cache_asm) - pLeft :p_jplt_cache; - pRight:p_jplt_cache; + pLeft :p_jplt_cache; //jpltc_curr + pRight:p_jplt_cache; //jpltc_curr + // + entry:TAILQ_ENTRY; //jpltc_attc function c(n1,n2:p_jplt_cache):Integer; static; end; @@ -92,7 +94,9 @@ type var entry_list:p_jit_entry_point; chunk_list:p_jcode_chunk; - jpltc_list:t_jplt_cache_set; + + jpltc_curr:t_jplt_cache_set; + jpltc_attc:TAILQ_HEAD; mchunk:p_stub_chunk; @@ -109,8 +113,8 @@ type attach_count:Integer; - procedure inc_ref; - procedure dec_ref; + procedure inc_ref(name:pchar); + procedure dec_ref(name:pchar); procedure inc_attach_count; function dec_attach_count:Boolean; function find_guest_by_host(addr:QWORD):QWORD; @@ -119,6 +123,10 @@ type function add_entry_point(src,dst:Pointer):p_jit_entry_point; procedure free_entry_point(node:p_jit_entry_point); procedure init_plt; + procedure attach_plt_cache(node:p_jplt_cache); + procedure detach_plt_cache(node:p_jplt_cache); + procedure detach_all_attc; + procedure detach_all_curr; function add_plt_cache(plt:p_jit_plt;src,dst:Pointer;blk:p_jit_dynamic_blob):p_jplt_cache; function new_chunk(count:QWORD):p_jcode_chunk; procedure alloc_base(_size:ptruint); @@ -128,10 +136,10 @@ type procedure attach_chunk; procedure attach; function detach_entry(node:p_jit_entry_point):Boolean; - procedure detach_entry; + procedure detach_all_entry; procedure detach_entry(c_start,c___end:QWORD); procedure detach_chunk(node:p_jcode_chunk); - procedure detach_chunk; + procedure detach_all_chunk; procedure detach; end; @@ -252,7 +260,7 @@ begin pick(ctx,addr); end else begin - node^.dec_ref; + node^.dec_ref('preload_entry'); end; end; @@ -352,6 +360,9 @@ begin td^.td_teb^.sttop:=td^.td_kstack.sttop; td^.td_teb^.stack:=td^.td_kstack.stack; //teb stack + + // + node^.dec_ref('fetch_entry') end; function fetch_chunk_by_guest(src:Pointer):p_jcode_chunk; @@ -373,7 +384,7 @@ begin if (Result<>nil) then begin - Result^.inc_ref; + Result^.inc_ref('fetch_chunk_by_guest'); end; vm_track_object_deallocate(obj); //<-vm_track_map_next_object @@ -420,7 +431,7 @@ begin if (Result<>nil) then begin - Result^.inc_ref; + Result^.inc_ref('next_chunk'); end; vm_track_object_deallocate(obj); //<-vm_track_map_next_object @@ -470,18 +481,23 @@ begin //while (i.Item<>nil) do - while (node>nil) do + while (node<>nil) do begin //node:=i.Item^; //if (node<>nil) then + + if not node^.is_mark_del then begin blob:=node^.blob; if (blob<>nil) and (blob<>prev) then begin if blob^.cross_host(QWORD(src),QWORD(src)+1) then begin - blob^.inc_ref; + blob^.inc_ref('fetch_blob_by_host'); + + rw_runlock(entry_chunk_lock); + Exit(blob); end; prev:=blob; @@ -509,7 +525,7 @@ begin tf_tip^:=blob^.find_guest_by_host(QWORD(src)); rw_runlock(blob^.lock); end; - blob^.dec_ref; + blob^.dec_ref('fetch_blob_by_host'); Result:=True; end else begin @@ -580,7 +596,10 @@ begin blob:=curr^.blob; rw_wlock(blob^.lock); - Result:=blob^.add_entry_point(addr,Pointer(dest)); + if not curr^.is_mark_del then + begin + Result:=blob^.add_entry_point(addr,Pointer(dest)); + end; rw_wunlock(blob^.lock); blob^.attach_entry(Result); @@ -589,13 +608,13 @@ begin end; next:=next_chunk(curr,addr); - curr^.dec_ref; + curr^.dec_ref('preload_entry'); curr:=next; end; if (curr<>nil) then begin - curr^.dec_ref; + curr^.dec_ref('preload_entry'); end; end; @@ -667,6 +686,9 @@ begin end; Result:=node^.dst; + + // + node^.dec_ref('fetch_entry') end; function t_jcode_chunk.c(n1,n2:p_jcode_chunk):Integer; @@ -969,7 +991,7 @@ begin if (Result<>nil) then begin - Result^.inc_ref; + Result^.inc_ref('fetch_entry'); end; rw_runlock(entry_hamt_lock); @@ -982,7 +1004,7 @@ begin entry:=fetch_entry(src); if (entry<>nil) then begin - entry^.dec_ref; + entry^.dec_ref('fetch_entry'); Result:=True; end else begin @@ -996,31 +1018,33 @@ function new_blob(_size:ptruint):p_jit_dynamic_blob; begin Result:=AllocMem(SizeOf(t_jit_dynamic_blob)); Result^.alloc_base(_size); + + TAILQ_INIT(@Result^.jpltc_attc); end; // -procedure t_jit_entry_point.inc_ref; +procedure t_jit_entry_point.inc_ref(name:pchar); begin - blob^.inc_ref; + blob^.inc_ref(name); end; -procedure t_jit_entry_point.dec_ref; +procedure t_jit_entry_point.dec_ref(name:pchar); begin - blob^.dec_ref; + blob^.dec_ref(name); end; // -procedure t_jcode_chunk.inc_ref; +procedure t_jcode_chunk.inc_ref(name:pchar); begin - blob^.inc_ref; + blob^.inc_ref(name); end; -procedure t_jcode_chunk.dec_ref; +procedure t_jcode_chunk.dec_ref(name:pchar); begin - blob^.dec_ref; + blob^.dec_ref(name); end; function t_jcode_chunk.is_mark_del:Boolean; @@ -1030,13 +1054,17 @@ end; // -procedure t_jit_dynamic_blob.inc_ref; +procedure t_jit_dynamic_blob.inc_ref(name:pchar); begin + //Writeln('inc_ref:0x',HexStr(@self),' ',name); + System.InterlockedIncrement(refs); end; -procedure t_jit_dynamic_blob.dec_ref; +procedure t_jit_dynamic_blob.dec_ref(name:pchar); begin + //Writeln('dec_ref:0x',HexStr(@self),' ',name); + if (System.InterlockedDecrement(refs)=0) then begin Free; @@ -1122,6 +1150,95 @@ begin end; end; +procedure t_jit_dynamic_blob.attach_plt_cache(node:p_jplt_cache); +begin + rw_wlock(lock); + + TAILQ_INSERT_TAIL(@jpltc_attc,node,@node^.entry); + + rw_wunlock(lock); +end; + +procedure t_jit_dynamic_blob.detach_plt_cache(node:p_jplt_cache); +begin + rw_wlock(lock); + + if (node^.entry.tqe_prev<>nil) then + begin + TAILQ_REMOVE(@jpltc_attc,node,@node^.entry); + + node^.entry:=Default(TAILQ_ENTRY); + end; + + rw_wunlock(lock); +end; + +procedure _reset_plt(node:p_jplt_cache); +var + plt:p_jit_plt; +begin + plt:=node^.plt; + if (plt<>nil) then + begin + //one element plt reset + System.InterlockedCompareExchange(plt^.cache,nil,node); + end; +end; + +procedure t_jit_dynamic_blob.detach_all_attc; +var + node,next:p_jplt_cache; +begin + node:=TAILQ_FIRST(@jpltc_attc); + + while (node<>nil) do + begin + next:=TAILQ_NEXT(node,@node^.entry); + + TAILQ_REMOVE(@jpltc_attc,node,@node^.entry); + + node^.entry:=Default(TAILQ_ENTRY); + + _reset_plt(node); + + //force deref + if (System.InterlockedCompareExchange(node^.blk,nil,@Self)=@Self) then + begin + Self.dec_ref('add_plt_cache'); + end; + + node:=next; + end; +end; + +procedure t_jit_dynamic_blob.detach_all_curr; +var + node:p_jplt_cache; + blk :p_jit_dynamic_blob; +begin + node:=jpltc_curr.Min; + + while (node<>nil) do + begin + jpltc_curr.Delete(node); + + _reset_plt(node); + + blk:=node^.blk; + + if (blk<>nil) then + begin + blk^.detach_plt_cache(node); + blk^.dec_ref('add_plt_cache'); + node^.blk:=nil; + end; + + FreeMem(node); + + node:=jpltc_curr.Min; + end; +end; + function t_jit_dynamic_blob.add_plt_cache(plt:p_jit_plt;src,dst:Pointer;blk:p_jit_dynamic_blob):p_jplt_cache; var node:t_jplt_cache; @@ -1139,7 +1256,7 @@ begin repeat rw_wlock(lock); - Result:=jpltc_list.Find(@node); + Result:=jpltc_curr.Find(@node); if (Result<>nil) then begin //update @@ -1147,16 +1264,23 @@ begin if (Result^.blk<>blk) then begin dec_blk:=Result^.blk; + Result^.blk:=blk; // - blk^.inc_ref; + blk^.inc_ref('add_plt_cache'); end; end; rw_wunlock(lock); if (dec_blk<>nil) then begin - dec_blk^.dec_ref; + //del + if (Result<>nil) then + begin + dec_blk^.detach_plt_cache(Result); + end; + + dec_blk^.dec_ref('add_plt_cache'); dec_blk:=nil; end; @@ -1172,21 +1296,28 @@ begin Result^.blk:=blk; // rw_wlock(lock); - _insert:=jpltc_list.Insert(Result); + _insert:=jpltc_curr.Insert(Result); if _insert then begin - blk^.inc_ref; + blk^.inc_ref('add_plt_cache'); end; rw_wunlock(lock); // if _insert then begin + //add + Break; end; end; until false; + if (Result<>nil) then + begin + blk^.attach_plt_cache(Result); + end; + end; function t_jit_dynamic_blob.new_chunk(count:QWORD):p_jcode_chunk; @@ -1297,7 +1428,7 @@ end; procedure t_jit_dynamic_blob.attach_entry(node:p_jit_entry_point); begin - node^.inc_ref; + node^.inc_ref('attach_entry'); self.inc_attach_count; rw_wlock(entry_hamt_lock); @@ -1356,10 +1487,10 @@ begin rw_wunlock(entry_hamt_lock); Result:=self.dec_attach_count; - node^.dec_ref; + node^.dec_ref('attach_entry'); end; -procedure t_jit_dynamic_blob.detach_entry; +procedure t_jit_dynamic_blob.detach_all_entry; var node,next:p_jit_entry_point; begin @@ -1416,10 +1547,10 @@ begin //entry_chunk.Delete(node); rw_wunlock(entry_chunk_lock); - node^.dec_ref; + node^.dec_ref('blob_track'); end; -procedure t_jit_dynamic_blob.detach_chunk; +procedure t_jit_dynamic_blob.detach_all_chunk; var node,next:p_jcode_chunk; begin @@ -1436,10 +1567,18 @@ end; procedure t_jit_dynamic_blob.detach; begin - inc_ref; - detach_entry; - detach_chunk; - dec_ref; + inc_ref('detach'); + + rw_wlock(lock); + + detach_all_entry; + detach_all_attc; + detach_all_curr; + detach_all_chunk; + + rw_wunlock(lock); + + dec_ref('detach'); end; procedure delete_jit_cache(chunk:p_jcode_chunk); @@ -1489,7 +1628,7 @@ begin if (node^.start<>node^.__end) then begin - node^.inc_ref; + node^.inc_ref('blob_track'); rw_wlock(entry_chunk_lock); diff --git a/sys/kern/kern_rangelock.pas b/sys/kern/kern_rangelock.pas index f54ca093..2032a646 100644 --- a/sys/kern/kern_rangelock.pas +++ b/sys/kern/kern_rangelock.pas @@ -35,10 +35,15 @@ type procedure rangelock_init (lock:p_rangelock); procedure rangelock_destroy (lock:p_rangelock); + procedure rangelock_unlock (lock:p_rangelock;cookie:Pointer;ilk:p_mtx); function rangelock_unlock_range(lock:p_rangelock;cookie:Pointer;start,__end:off_t;ilk:p_mtx):Pointer; + function rangelock_rlock (lock:p_rangelock;start,__end:off_t;ilk:p_mtx):Pointer; +function rangelock_tryrlock (lock:p_rangelock;start,__end:off_t;ilk:p_mtx):Pointer; function rangelock_wlock (lock:p_rangelock;start,__end:off_t;ilk:p_mtx):Pointer; +function rangelock_trywlock (lock:p_rangelock;start,__end:off_t;ilk:p_mtx):Pointer; + procedure rlqentry_free (rleq:p_rl_q_entry); implementation @@ -80,19 +85,11 @@ begin end; { - * Verifies the supplied rl_q_entries for compatibility. Returns true - * if the rangelock queue entries are not compatible, false if they are. - * * Two entries are compatible if their ranges do not overlap, or both * entries are for read. } -function rangelock_incompatible(e1,e2:p_rl_q_entry):Integer; +function ranges_overlap(e1,e2:p_rl_q_entry):Integer; begin - if ((e1^.rl_q_flags and RL_LOCK_TYPE_MASK)=RL_LOCK_READ) and - ((e2^.rl_q_flags and RL_LOCK_TYPE_MASK)=RL_LOCK_READ) then - begin - Exit(0); - end; if (e1^.rl_q_start < e2^.rl_q_end) and (e1^.rl_q_end > e2^.rl_q_start) then begin Exit(1); @@ -107,68 +104,102 @@ procedure rangelock_calc_block(lock:p_rangelock); label _out; var - entry,entry1,whead:p_rl_q_entry; + entry,nextentry,entry1:p_rl_q_entry; begin - if (lock^.rl_currdep=TAILQ_FIRST(@lock^.rl_waiters)) and - (lock^.rl_currdep<>nil) then - begin - lock^.rl_currdep:=TAILQ_NEXT(lock^.rl_currdep, @lock^.rl_currdep^.rl_q_link); - end; - + //for entry:=lock^.rl_currdep; - while (entry<>nil) do begin - entry1:=TAILQ_FIRST(@lock^.rl_waiters); + nextentry:=TAILQ_NEXT(entry, @entry^.rl_q_link); - while (entry1<>nil) do + if (entry^.rl_q_flags and RL_LOCK_READ)<>0 then begin - if (rangelock_incompatible(entry, entry1)<>0) then + // Reads must not overlap with granted writes. + + //for + entry1:=TAILQ_FIRST(@lock^.rl_waiters); + while ((entry1^.rl_q_flags and RL_LOCK_READ)=0) do begin - goto _out; + if (ranges_overlap(entry, entry1)<>0) then + begin + goto _out; + end; + // + entry1:=TAILQ_NEXT(entry1, @entry1^.rl_q_link); end; - if (entry1=entry) then + //for + + end else + begin + // Write must not overlap with any granted locks. + + //for + entry1:=TAILQ_FIRST(@lock^.rl_waiters); + while (entry1<>entry) do begin - break; + if (ranges_overlap(entry, entry1)<>0) then + begin + goto _out; + end; + // + entry1:=TAILQ_NEXT(entry1, @entry1^.rl_q_link); end; - // - entry1:=TAILQ_NEXT(entry1, @entry1^.rl_q_link) + //for + + // Move grantable write locks to the front. + TAILQ_REMOVE (@lock^.rl_waiters, entry, @entry^.rl_q_link); + TAILQ_INSERT_HEAD(@lock^.rl_waiters, entry, @entry^.rl_q_link); end; - //next - entry:=TAILQ_NEXT(entry, @entry^.rl_q_link); + // Grant this lock. + entry^.rl_q_flags:=entry^.rl_q_flags or RL_LOCK_GRANTED; + wakeup(entry); + + entry:=nextentry; end; + //for -_out: - lock^.rl_currdep:=entry; - - whead:=TAILQ_FIRST(@lock^.rl_waiters); - - while (whead<>nil) do - begin - if (whead=lock^.rl_currdep) then - begin - break; - end; - if ((whead^.rl_q_flags and RL_LOCK_GRANTED)=0) then - begin - whead^.rl_q_flags:=whead^.rl_q_flags or RL_LOCK_GRANTED; - wakeup(whead); - end; - // - whead:=TAILQ_NEXT(whead, @whead^.rl_q_link) - end; + _out: + lock^.rl_currdep:=entry; end; -procedure rangelock_unlock_locked(lock:p_rangelock;entry:p_rl_q_entry;ilk:p_mtx); +procedure rangelock_unlock_locked(lock:p_rangelock;entry:p_rl_q_entry;ilk:p_mtx;do_calc_block:Boolean); begin Assert((lock<>nil) and (entry<>nil) and (ilk<>nil)); mtx_assert(ilk^); - Assert(entry<>lock^.rl_currdep, 'stuck currdep'); + + if (not do_calc_block) then + begin + { + * This is the case where rangelock_enqueue() has been called + * with trylock=true and just inserted this entry in the + * queue. + * If rl_currdep is this entry, rl_currdep needs to + * be set to the next entry in the rl_waiters list. + * However, since this entry is the last entry in the + * list, the next entry is NULL. + } + + if (lock^.rl_currdep=entry) then + begin + Assert(TAILQ_NEXT(lock^.rl_currdep, @lock^.rl_currdep^.rl_q_link)=nil, 'rangelock_enqueue: next entry not NULL'); + + lock^.rl_currdep:=nil; + end; + end else + begin + Assert(entry<>lock^.rl_currdep, 'stuck currdep'); + end; TAILQ_REMOVE(@lock^.rl_waiters, entry, @entry^.rl_q_link); - rangelock_calc_block(lock); + entry^.rl_q_link:=Default(TAILQ_ENTRY); + + if (do_calc_block) then + begin + rangelock_calc_block(lock); + end; + mtx_unlock(ilk^); if (curkthread^.td_rlqe=nil) then @@ -182,7 +213,7 @@ begin Assert((lock<>nil) and (cookie<>nil) and (ilk<>nil)); mtx_lock(ilk^); - rangelock_unlock_locked(lock, cookie, ilk); + rangelock_unlock_locked(lock, cookie, ilk, true); end; { @@ -204,7 +235,7 @@ begin if (entry^.rl_q_end=__end) then begin - rangelock_unlock_locked(lock, cookie, ilk); + rangelock_unlock_locked(lock, cookie, ilk, true); Exit(nil); end; @@ -215,10 +246,10 @@ begin end; { - * Add the lock request to the queue of the pending requests for - * rangelock. Sleep until the request can be granted. +* Add the lock request to the queue of the pending requests for +* rangelock. Sleep until the request can be granted unless trylock=true. } -function rangelock_enqueue(lock:p_rangelock;start,__end:off_t;mode:Integer;ilk:p_mtx):Pointer; +function rangelock_enqueue(lock:p_rangelock;start,__end:off_t;mode:Integer;ilk:p_mtx;trylock:Boolean):Pointer; var entry:p_rl_q_entry; td:p_kthread; @@ -249,6 +280,13 @@ begin TAILQ_INSERT_TAIL(@lock^.rl_waiters, entry, @entry^.rl_q_link); + { + * If rl_currdep=NULL, there is no entry waiting for a conflicting + * range to be resolved, so set rl_currdep to this entry. If there is + * no conflicting entry for this entry, rl_currdep will be set back to + * NULL by rangelock_calc_block(). + } + if (lock^.rl_currdep=nil) then begin lock^.rl_currdep:=entry; @@ -258,6 +296,19 @@ begin while ((entry^.rl_q_flags and RL_LOCK_GRANTED)=0) do begin + + if (trylock) then + begin + { + * For this case, the range is not actually locked + * yet, but removal from the list requires the same + * steps, except for not doing a rangelock_calc_block() + * call, since rangelock_calc_block() was called above. + } + rangelock_unlock_locked(lock, entry, ilk, false); + Exit(nil); + end; + msleep(entry, ilk, 0, 'range', 0); end; @@ -267,14 +318,23 @@ end; function rangelock_rlock(lock:p_rangelock;start,__end:off_t;ilk:p_mtx):Pointer; begin - Result:=rangelock_enqueue(lock, start, __end, RL_LOCK_READ, ilk); + Result:=rangelock_enqueue(lock, start, __end, RL_LOCK_READ, ilk, false); +end; + +function rangelock_tryrlock(lock:p_rangelock;start,__end:off_t;ilk:p_mtx):Pointer; +begin + Result:=rangelock_enqueue(lock, start, __end, RL_LOCK_READ, ilk, true); end; function rangelock_wlock(lock:p_rangelock;start,__end:off_t;ilk:p_mtx):Pointer; begin - Result:=rangelock_enqueue(lock, start, __end, RL_LOCK_WRITE, ilk); + Result:=rangelock_enqueue(lock, start, __end, RL_LOCK_WRITE, ilk, false); end; +function rangelock_trywlock(lock:p_rangelock;start,__end:off_t;ilk:p_mtx):Pointer; +begin + Result:=rangelock_enqueue(lock, start, __end, RL_LOCK_WRITE, ilk, true); +end; end. diff --git a/sys/md/vm_pmap.pas b/sys/md/vm_pmap.pas index 3f5b2eae..3de8cbd3 100644 --- a/sys/md/vm_pmap.pas +++ b/sys/md/vm_pmap.pas @@ -552,6 +552,8 @@ function pmap_wlock(pmap :pmap_t; start:vm_offset_t; __end:vm_offset_t):Pointer; begin + //Writeln('pmap_wlock:',HexStr(start,10),'..',HexStr(__end,10)); + Result:=rangelock_wlock(@pmap^.rmlock,start,__end,@pmap^.rm_mtx); end; @@ -559,11 +561,15 @@ function pmap_rlock(pmap :pmap_t; start:vm_offset_t; __end:vm_offset_t):Pointer; begin + //Writeln('pmap_rlock:',HexStr(start,10),'..',HexStr(__end,10)); + Result:=rangelock_rlock(@pmap^.rmlock,start,__end,@pmap^.rm_mtx); end; procedure pmap_unlock(pmap:pmap_t;cookie:Pointer); begin + //Writeln('pmap_unlock:',HexStr(p_rl_q_entry(cookie)^.rl_q_start,10),'..',HexStr(p_rl_q_entry(cookie)^.rl_q_end,10)); + rangelock_unlock(@pmap^.rmlock,cookie,@pmap^.rm_mtx); end;