mirror of https://github.com/red-prig/fpPS4.git
4141 lines
76 KiB
Plaintext
4141 lines
76 KiB
Plaintext
unit kern_umtx;
|
|
|
|
{$mode ObjFPC}{$H+}
|
|
{$CALLING SysV_ABI_CDecl}
|
|
|
|
interface
|
|
|
|
uses
|
|
atomic,
|
|
mqueue,
|
|
kern_mtx,
|
|
time,
|
|
kern_thr,
|
|
_umtx,
|
|
rtprio;
|
|
|
|
procedure _umutex_init(mtx:p_umutex); inline;
|
|
|
|
procedure umtx_thread_init(td:p_kthread);
|
|
procedure umtx_thread_exit(td:p_kthread);
|
|
procedure umtx_thread_fini(td:p_kthread);
|
|
|
|
function sys__umtx_lock(mtx:Pointer):Integer;
|
|
function sys__umtx_unlock(mtx:Pointer):Integer;
|
|
function sys__umtx_op(obj:Pointer;op:Integer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
|
|
//
|
|
|
|
function kern_umtx_wake(td:p_kthread;umtx:p_umtx;n_wake,priv:Integer):Integer;
|
|
|
|
function umtx_copyin_timeout(addr:Pointer;tsp:p_timespec):Integer;
|
|
|
|
procedure umtxq_sysinit; //SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL);
|
|
|
|
const
|
|
_UMUTEX_TRY =1;
|
|
_UMUTEX_WAIT=2;
|
|
|
|
UQF_UMTXQ=$0001;
|
|
|
|
UMTX_SHARED_QUEUE =0;
|
|
UMTX_EXCLUSIVE_QUEUE=1;
|
|
|
|
GOLDEN_RATIO_PRIME=2654404609;
|
|
UMTX_CHAINS =512;
|
|
UMTX_SHIFTS =(32 - 9);
|
|
|
|
BUSY_SPINS =200;
|
|
|
|
type
|
|
// Priority inheritance mutex info.
|
|
p_umtx_pi=^umtx_pi;
|
|
umtx_pi=record
|
|
// Owner thread
|
|
pi_owner:p_kthread;
|
|
|
|
// Reference count
|
|
pi_refcount:Integer;
|
|
|
|
// List entry to link umtx holding by thread
|
|
pi_link:TAILQ_ENTRY; //umtx_pi
|
|
|
|
// List entry in hash
|
|
pi_hashlink:TAILQ_ENTRY; //umtx_pi
|
|
|
|
// List for waiters
|
|
pi_blocked:TAILQ_HEAD; //umtx_q
|
|
|
|
// Identify a userland lock object
|
|
pi_key:umtx_key;
|
|
end;
|
|
|
|
p_umtxq_queue=^umtxq_queue;
|
|
|
|
// A userland synchronous object user.
|
|
pp_umtx_q=^p_umtx_q;
|
|
p_umtx_q=^umtx_q;
|
|
umtx_q=record
|
|
// Linked list for the hash.
|
|
uq_link:TAILQ_ENTRY; //umtx_q
|
|
|
|
// Umtx key.
|
|
uq_key:umtx_key;
|
|
|
|
// Umtx flags.
|
|
uq_flags:Integer;
|
|
|
|
// Inherited priority from PP mutex
|
|
uq_inherited_pri:Integer;
|
|
|
|
// The thread waits on.
|
|
uq_thread:p_kthread;
|
|
|
|
{
|
|
Blocked on PI mutex. read can use chain lock
|
|
or umtx_lock, write must have both chain lock and
|
|
umtx_lock being hold.
|
|
}
|
|
|
|
uq_pi_blocked:p_umtx_pi;
|
|
|
|
// On blocked list
|
|
uq_lockq:TAILQ_ENTRY; //umtx_q
|
|
|
|
// Thread contending with us
|
|
uq_pi_contested:TAILQ_HEAD; //umtx_pi
|
|
|
|
// Spare queue ready to be reused
|
|
uq_spare_queue:p_umtxq_queue;
|
|
|
|
// The queue we on
|
|
uq_cur_queue:p_umtxq_queue;
|
|
end;
|
|
|
|
umtxq_head=TAILQ_HEAD; //umtx_q
|
|
|
|
// Per-key wait-queue
|
|
umtxq_queue=record
|
|
head :umtxq_head;
|
|
key :umtx_key;
|
|
link :LIST_ENTRY; //umtxq_queue
|
|
length:Integer;
|
|
end;
|
|
|
|
umtxq_list=LIST_HEAD; //umtxq_queue
|
|
|
|
// Userland lock object's wait-queue chain
|
|
p_umtxq_chain=^umtxq_chain;
|
|
umtxq_chain=record
|
|
// Lock for this chain.
|
|
uc_lock:mtx;
|
|
|
|
// List of sleep queues.
|
|
uc_queue:array[0..1] of umtxq_list;
|
|
|
|
uc_spare_queue:LIST_HEAD; //umtxq_queue
|
|
|
|
// Busy flag
|
|
uc_busy:Integer;
|
|
|
|
// Chain lock waiters
|
|
uc_waiters:Integer;
|
|
|
|
// All PI in the list
|
|
uc_pi_list:TAILQ_HEAD; //umtx_pi
|
|
end;
|
|
|
|
var
|
|
umtxq_chains:array[0..1,0..UMTX_CHAINS-1] of umtxq_chain;
|
|
umtx_pi_allocated:Integer=0;
|
|
|
|
size_of_umtx_q:Integer=sizeof(umtx_q); public;
|
|
|
|
implementation
|
|
|
|
uses
|
|
errno,
|
|
systm,
|
|
md_time,
|
|
kern_proc,
|
|
kern_param,
|
|
sched_ule,
|
|
vm,
|
|
vm_map,
|
|
sys_vm_object;
|
|
|
|
procedure UMTXQ_LOCKED_ASSERT(uc:p_umtxq_chain); inline;
|
|
begin
|
|
mtx_assert(uc^.uc_lock);
|
|
end;
|
|
|
|
procedure UMTXQ_BUSY_ASSERT(uc:p_umtxq_chain); inline;
|
|
begin
|
|
Assert(uc^.uc_busy<>0,'umtx chain is not busy');
|
|
end;
|
|
|
|
function UPRI(td:p_kthread):Integer; inline;
|
|
begin
|
|
Result:=td^.td_user_pri;
|
|
if ((Result and Integer($ffffff80)) = 768) then
|
|
begin
|
|
Result:=895;
|
|
end;
|
|
end;
|
|
|
|
function GET_SHARE(flags:Integer):Integer; inline;
|
|
begin
|
|
if ((flags and USYNC_PROCESS_SHARED)=0) then
|
|
begin
|
|
Result:=THREAD_SHARE;
|
|
end else
|
|
begin
|
|
Result:=PROCESS_SHARE;
|
|
end;
|
|
end;
|
|
|
|
function GET_PRIV_SHARE(priv:Integer):Integer; inline;
|
|
begin
|
|
if (priv<>0) then
|
|
begin
|
|
Result:=THREAD_SHARE;
|
|
end else
|
|
begin
|
|
Result:=AUTO_SHARE;
|
|
end;
|
|
end;
|
|
|
|
//
|
|
|
|
procedure umtxq_insert_queue(uq:p_umtx_q;q:Integer); forward;
|
|
procedure umtxq_remove_queue(uq:p_umtx_q;q:Integer); forward;
|
|
function umtxq_signal_queue(key:p_umtx_key;n_wake,q:Integer):Integer; forward;
|
|
procedure umtx_thread_cleanup(td:p_kthread); forward;
|
|
procedure umtx_pi_clearowner(pi:p_umtx_pi); forward;
|
|
|
|
//
|
|
|
|
function umtxq_signal(key:p_umtx_key;n_wake:Integer):Integer; inline;
|
|
begin
|
|
Result:=umtxq_signal_queue(key,n_wake,UMTX_SHARED_QUEUE);
|
|
end;
|
|
|
|
procedure umtxq_insert(uq:p_umtx_q); inline;
|
|
begin
|
|
umtxq_insert_queue(uq,UMTX_SHARED_QUEUE);
|
|
end;
|
|
|
|
procedure umtxq_remove(uq:p_umtx_q); inline;
|
|
begin
|
|
umtxq_remove_queue(uq,UMTX_SHARED_QUEUE);
|
|
end;
|
|
|
|
var
|
|
umtx_lock:mtx;
|
|
|
|
procedure umtxq_sysinit;
|
|
var
|
|
i,j:Integer;
|
|
begin
|
|
For i:=0 to 1 do
|
|
begin
|
|
For j:=0 to UMTX_CHAINS-1 do
|
|
begin
|
|
mtx_init ( umtxq_chains[i][j].uc_lock, 'umtxql');
|
|
LIST_INIT (@umtxq_chains[i][j].uc_queue[0]);
|
|
LIST_INIT (@umtxq_chains[i][j].uc_queue[1]);
|
|
LIST_INIT (@umtxq_chains[i][j].uc_spare_queue);
|
|
TAILQ_INIT(@umtxq_chains[i][j].uc_pi_list);
|
|
umtxq_chains[i][j].uc_busy :=0;
|
|
umtxq_chains[i][j].uc_waiters:=0;
|
|
end;
|
|
end;
|
|
|
|
mtx_init(umtx_lock, 'umtx lock');
|
|
//EVENTHANDLER_REGISTER(process_exec, umtx_exec_hook, NULL, EVENTHANDLER_PRI_ANY);
|
|
end;
|
|
|
|
procedure umtxq_alloc(uq:p_umtx_q);
|
|
begin
|
|
Assert(uq<>nil);
|
|
//uq:=AllocMem(sizeof(umtx_q));
|
|
|
|
//do not allocate uq_spare_queue satically, can migrate to another thread
|
|
uq^.uq_spare_queue:=AllocMem(sizeof(umtxq_queue));
|
|
//
|
|
TAILQ_INIT(@uq^.uq_spare_queue^.head);
|
|
TAILQ_INIT(@uq^.uq_pi_contested);
|
|
uq^.uq_inherited_pri:=PRI_MAX;
|
|
end;
|
|
|
|
procedure umtxq_free(uq:p_umtx_q);
|
|
begin
|
|
Assert(uq^.uq_spare_queue<>nil);
|
|
FreeMem(uq^.uq_spare_queue);
|
|
//FreeMem(uq);
|
|
end;
|
|
|
|
procedure umtx_thread_init(td:p_kthread); public;
|
|
var
|
|
uq:p_umtx_q;
|
|
begin
|
|
uq:=td^.td_umtxq;
|
|
umtxq_alloc(uq);
|
|
uq^.uq_thread:=td;
|
|
end;
|
|
|
|
procedure umtx_thread_exit(td:p_kthread); public;
|
|
begin
|
|
umtx_thread_cleanup(td);
|
|
end;
|
|
|
|
procedure umtx_thread_fini(td:p_kthread); public;
|
|
begin
|
|
umtxq_free(td^.td_umtxq);
|
|
end;
|
|
|
|
procedure umtx_thread_cleanup(td:p_kthread);
|
|
var
|
|
uq:p_umtx_q;
|
|
pi:p_umtx_pi;
|
|
begin
|
|
uq:=td^.td_umtxq;
|
|
|
|
if (uq=nil) then Exit;
|
|
|
|
mtx_lock(umtx_lock);
|
|
|
|
uq^.uq_inherited_pri:=PRI_MAX;
|
|
|
|
pi:=TAILQ_FIRST(@uq^.uq_pi_contested);
|
|
while (pi<>nil) do
|
|
begin
|
|
umtx_pi_clearowner(pi);
|
|
//
|
|
TAILQ_REMOVE(@uq^.uq_pi_contested, pi, @pi^.pi_link);
|
|
//
|
|
pi:=TAILQ_FIRST(@uq^.uq_pi_contested);
|
|
end;
|
|
|
|
mtx_unlock(umtx_lock);
|
|
|
|
thread_lock(td);
|
|
sched_lend_user_prio(td, PRI_MAX);
|
|
thread_unlock(td);
|
|
end;
|
|
|
|
procedure umtxq_hash(key:p_umtx_key);
|
|
var
|
|
n:QWORD;
|
|
begin
|
|
n:=QWORD(key^.info.both.a) + key^.info.both.b;
|
|
key^.hash:=((n * GOLDEN_RATIO_PRIME) shr UMTX_SHIFTS) mod UMTX_CHAINS;
|
|
end;
|
|
|
|
function umtxq_getchain(key:p_umtx_key):p_umtxq_chain;
|
|
begin
|
|
if (key^._type <= TYPE_SEM) then
|
|
begin
|
|
Exit(@umtxq_chains[1][key^.hash]);
|
|
end;
|
|
Exit(@umtxq_chains[0][key^.hash]);
|
|
end;
|
|
|
|
procedure umtxq_lock(key:p_umtx_key); inline;
|
|
var
|
|
uc:p_umtxq_chain;
|
|
begin
|
|
uc:=umtxq_getchain(key);
|
|
mtx_lock(uc^.uc_lock);
|
|
end;
|
|
|
|
procedure umtxq_unlock(key:p_umtx_key); inline;
|
|
var
|
|
uc:p_umtxq_chain;
|
|
begin
|
|
uc:=umtxq_getchain(key);
|
|
mtx_unlock(uc^.uc_lock);
|
|
end;
|
|
|
|
procedure umtxq_busy(key:p_umtx_key);
|
|
var
|
|
uc:p_umtxq_chain;
|
|
count:Integer;
|
|
begin
|
|
uc:=umtxq_getchain(key);
|
|
mtx_assert(uc^.uc_lock);
|
|
if (uc^.uc_busy<>0) then
|
|
begin
|
|
|
|
//if (smp_cpus > 1) then
|
|
begin
|
|
count:=BUSY_SPINS;
|
|
if (count > 0) then
|
|
begin
|
|
umtxq_unlock(key);
|
|
while (uc^.uc_busy<>0) and (count > 0) do
|
|
begin
|
|
Dec(count);
|
|
spin_pause;
|
|
end;
|
|
umtxq_lock(key);
|
|
end;
|
|
end;
|
|
|
|
while (uc^.uc_busy<>0) do
|
|
begin
|
|
Inc(uc^.uc_waiters);
|
|
msleep(uc, @uc^.uc_lock, 0, 'umtxqb', 0);
|
|
Dec(uc^.uc_waiters);
|
|
end;
|
|
end;
|
|
uc^.uc_busy:=1;
|
|
end;
|
|
|
|
procedure umtxq_unbusy(key:p_umtx_key);
|
|
var
|
|
uc:p_umtxq_chain;
|
|
begin
|
|
uc:=umtxq_getchain(key);
|
|
mtx_assert(uc^.uc_lock);
|
|
Assert(uc^.uc_busy<>0, 'not busy');
|
|
uc^.uc_busy:=0;
|
|
if (uc^.uc_waiters<>0) then
|
|
begin
|
|
wakeup_one(uc);
|
|
end;
|
|
end;
|
|
|
|
function umtxq_queue_lookup(key:p_umtx_key;q:Integer):p_umtxq_queue;
|
|
var
|
|
uh:p_umtxq_queue;
|
|
uc:p_umtxq_chain;
|
|
begin
|
|
uc:=umtxq_getchain(key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
|
|
uh:=LIST_FIRST(@uc^.uc_queue[q]);
|
|
while (uh<>nil) do
|
|
begin
|
|
if ((umtx_key_match(@uh^.key, key))<>0) then
|
|
begin
|
|
Exit(uh);
|
|
end;
|
|
uh:=LIST_NEXT(uh,@uh^.link);
|
|
end;
|
|
|
|
Exit(nil);
|
|
end;
|
|
|
|
procedure umtxq_insert_queue(uq:p_umtx_q;q:Integer);
|
|
var
|
|
uh:p_umtxq_queue;
|
|
uc:p_umtxq_chain;
|
|
begin
|
|
uc:=umtxq_getchain(@uq^.uq_key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
Assert((uq^.uq_flags and UQF_UMTXQ)=0, 'umtx_q is already on queue');
|
|
uh:=umtxq_queue_lookup(@uq^.uq_key, q);
|
|
if (uh<>nil) then
|
|
begin
|
|
LIST_INSERT_HEAD(@uc^.uc_spare_queue, uq^.uq_spare_queue, @uq^.uq_spare_queue^.link);
|
|
end else
|
|
begin
|
|
uh:=uq^.uq_spare_queue;
|
|
uh^.key:=uq^.uq_key;
|
|
LIST_INSERT_HEAD(@uc^.uc_queue[q], uh, @uh^.link);
|
|
end;
|
|
uq^.uq_spare_queue:=nil;
|
|
|
|
TAILQ_INSERT_TAIL(@uh^.head, uq, @uq^.uq_link);
|
|
Inc(uh^.length);
|
|
|
|
uq^.uq_flags:=uq^.uq_flags or UQF_UMTXQ;
|
|
uq^.uq_cur_queue:=uh;
|
|
end;
|
|
|
|
procedure umtxq_remove_queue(uq:p_umtx_q;q:Integer);
|
|
var
|
|
uc:p_umtxq_chain;
|
|
uh:p_umtxq_queue;
|
|
begin
|
|
uc:=umtxq_getchain(@uq^.uq_key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
if ((uq^.uq_flags and UQF_UMTXQ)<>0) then
|
|
begin
|
|
uh:=uq^.uq_cur_queue;
|
|
TAILQ_REMOVE(@uh^.head, uq, @uq^.uq_link);
|
|
Dec(uh^.length);
|
|
|
|
uq^.uq_flags:=uq^.uq_flags and (not UQF_UMTXQ);
|
|
if (TAILQ_EMPTY(@uh^.head)) then
|
|
begin
|
|
Assert(uh^.length=0,'inconsistent umtxq_queue length');
|
|
LIST_REMOVE(uh,@uh^.link);
|
|
end else
|
|
begin
|
|
uh:=LIST_FIRST(@uc^.uc_spare_queue);
|
|
Assert(uh<>nil,'uc_spare_queue is empty');
|
|
LIST_REMOVE(uh,@uh^.link);
|
|
end;
|
|
uq^.uq_spare_queue:=uh;
|
|
uq^.uq_cur_queue :=nil;
|
|
end;
|
|
end;
|
|
|
|
function umtxq_count(key:p_umtx_key):Integer;
|
|
var
|
|
uc:p_umtxq_chain;
|
|
uh:p_umtxq_queue;
|
|
begin
|
|
uc:=umtxq_getchain(key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
uh:=umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
|
|
if (uh<>nil) then
|
|
begin
|
|
Exit(uh^.length);
|
|
end;
|
|
Exit(0);
|
|
end;
|
|
|
|
function umtxq_count_pi(key:p_umtx_key;first:pp_umtx_q):Integer;
|
|
var
|
|
uc:p_umtxq_chain;
|
|
uh:p_umtxq_queue;
|
|
begin
|
|
first^:=nil;
|
|
uc:=umtxq_getchain(key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
uh:=umtxq_queue_lookup(key, UMTX_SHARED_QUEUE);
|
|
if (uh<>nil) then
|
|
begin
|
|
first^:=TAILQ_FIRST(@uh^.head);
|
|
Exit(uh^.length);
|
|
end;
|
|
Exit(0);
|
|
end;
|
|
|
|
function umtxq_check_susp(td:p_kthread):Integer;
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
|
|
function umtxq_signal_queue(key:p_umtx_key;n_wake,q:Integer):Integer;
|
|
var
|
|
uc:p_umtxq_chain;
|
|
uh:p_umtxq_queue;
|
|
uq:p_umtx_q;
|
|
ret:Integer;
|
|
begin
|
|
ret:=0;
|
|
uc:=umtxq_getchain(key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
uh:=umtxq_queue_lookup(key, q);
|
|
if (uh<>nil) then
|
|
begin
|
|
uq:=TAILQ_FIRST(@uh^.head);
|
|
while (uq<>nil) do
|
|
begin
|
|
umtxq_remove_queue(uq, q);
|
|
wakeup(uq);
|
|
Inc(ret);
|
|
if (ret >= n_wake) then
|
|
begin
|
|
Exit(ret);
|
|
end;
|
|
//
|
|
uq:=TAILQ_FIRST(@uh^.head);
|
|
end;
|
|
end;
|
|
Exit(ret);
|
|
end;
|
|
|
|
function umtxq_signal_by_tid(key:p_umtx_key;tid,q:Integer):Boolean;
|
|
var
|
|
uc:p_umtxq_chain;
|
|
uh:p_umtxq_queue;
|
|
uq:p_umtx_q;
|
|
begin
|
|
Result:=False;
|
|
uc:=umtxq_getchain(key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
uh:=umtxq_queue_lookup(key, q);
|
|
if (uh<>nil) then
|
|
begin
|
|
uq:=TAILQ_FIRST(@uh^.head);
|
|
while (uq<>nil) do
|
|
begin
|
|
if (uq^.uq_thread^.td_tid=tid) then
|
|
begin
|
|
umtxq_remove_queue(uq, q);
|
|
wakeup(uq);
|
|
Result:=True;
|
|
Break;
|
|
end;
|
|
//
|
|
uq:=TAILQ_FIRST(@uh^.head);
|
|
end;
|
|
end;
|
|
end;
|
|
|
|
function umtxq_first_tid(key:p_umtx_key;q:Integer):Integer;
|
|
var
|
|
uc:p_umtxq_chain;
|
|
uh:p_umtxq_queue;
|
|
uq:p_umtx_q;
|
|
begin
|
|
Result:=-1;
|
|
uc:=umtxq_getchain(key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
uh:=umtxq_queue_lookup(key, q);
|
|
if (uh<>nil) then
|
|
begin
|
|
uq:=TAILQ_FIRST(@uh^.head);
|
|
if (uq<>nil) then
|
|
begin
|
|
Result:=uq^.uq_thread^.td_tid;
|
|
end;
|
|
end;
|
|
end;
|
|
|
|
procedure umtxq_signal_thread(uq:p_umtx_q);
|
|
var
|
|
uc:p_umtxq_chain;
|
|
begin
|
|
uc:=umtxq_getchain(@uq^.uq_key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
umtxq_remove(uq);
|
|
wakeup(uq);
|
|
end;
|
|
|
|
function umtxq_sleep(uq:p_umtx_q;wmesg:pchar;timo:Int64):Integer;
|
|
var
|
|
uc:p_umtxq_chain;
|
|
error:Integer;
|
|
begin
|
|
uc:=umtxq_getchain(@uq^.uq_key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
if ((uq^.uq_flags and UQF_UMTXQ)=0) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
error:=msleep(uq, @uc^.uc_lock, PCATCH, wmesg, timo);
|
|
if (error=EWOULDBLOCK) then
|
|
begin
|
|
error:=ETIMEDOUT;
|
|
end;
|
|
Exit(error);
|
|
end;
|
|
|
|
function umtx_key_get(addr:Pointer;_type,share:Integer;key:p_umtx_key):Integer;
|
|
var
|
|
td:p_kthread;
|
|
map:vm_map_t;
|
|
entry:vm_map_entry_t;
|
|
pindex:vm_pindex_t;
|
|
prot:vm_prot_t;
|
|
begin
|
|
td:=curkthread;
|
|
if (td=nil) then Exit(-1);
|
|
|
|
if (ptrint(addr)<$1000) then Exit(EFAULT);
|
|
|
|
key^._type:=_type;
|
|
|
|
if (share=THREAD_SHARE) then
|
|
begin
|
|
key^.shared:=0;
|
|
key^.info.private.vs :=p_proc.p_vmspace;
|
|
key^.info.private.addr:=QWORD(addr);
|
|
end else
|
|
begin
|
|
Assert((share=PROCESS_SHARE) or (share=AUTO_SHARE));
|
|
map:=@p_vmspace(p_proc.p_vmspace)^.vm_map;
|
|
if (vm_map_lookup(@map, vm_offset_t(addr), VM_PROT_WRITE,
|
|
@entry, @key^.info.shared.vm_obj, @pindex, @prot
|
|
)<>KERN_SUCCESS) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (share=PROCESS_SHARE) or
|
|
((share=AUTO_SHARE) and
|
|
(VM_INHERIT_SHARE=entry^.inheritance)) then
|
|
begin
|
|
key^.shared:=1;
|
|
key^.info.shared.offset:=entry^.offset + entry^.start - vm_offset_t(addr);
|
|
vm_object_reference(key^.info.shared.vm_obj);
|
|
end else
|
|
begin
|
|
key^.shared:=0;
|
|
key^.info.private.vs :=p_proc.p_vmspace;
|
|
key^.info.private.addr:=QWORD(addr);
|
|
end;
|
|
vm_map_lookup_done(map, entry);
|
|
end;
|
|
|
|
umtxq_hash(key);
|
|
Exit(0);
|
|
end;
|
|
|
|
procedure umtx_key_release(key:p_umtx_key); inline;
|
|
begin
|
|
if (key^.shared<>0) then
|
|
begin
|
|
vm_object_deallocate(key^.info.shared.vm_obj);
|
|
end;
|
|
end;
|
|
|
|
//
|
|
|
|
function _do_lock_umtx(td:p_kthread;umtx:p_umtx;id:QWORD;timo:Int64):Integer;
|
|
var
|
|
uq:p_umtx_q;
|
|
owner,old:QWORD;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('_do_lock_umtx(',HexStr(umtx),',',id,',',timo,')');
|
|
|
|
uq:=td^.td_umtxq;
|
|
|
|
repeat
|
|
owner:=fuword64(umtx^.u_owner);
|
|
|
|
if (owner=QWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
owner:=casuword64(umtx^.u_owner,UMTX_UNOWNED,id);
|
|
|
|
if (owner=UMTX_UNOWNED) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
|
|
if (owner=QWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (owner=QWORD($40000000)) then
|
|
begin
|
|
Exit(EINVAL);
|
|
end;
|
|
|
|
if (owner=UMTX_CONTESTED) then
|
|
begin
|
|
owner:=casuword64(umtx^.u_owner,UMTX_CONTESTED,id or UMTX_CONTESTED);
|
|
|
|
if (owner=UMTX_CONTESTED) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
|
|
if (owner=QWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Continue;
|
|
end;
|
|
|
|
if (Result<>0) then Exit;
|
|
|
|
Result:=umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE, @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_busy (@uq^.uq_key);
|
|
umtxq_insert(uq);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
old:=casuword64(umtx^.u_owner,owner,owner or UMTX_CONTESTED);
|
|
|
|
if (old=QWORD(-1)) then
|
|
begin
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
umtxq_lock(@uq^.uq_key);
|
|
|
|
if (old=owner) then
|
|
begin
|
|
Result:=umtxq_sleep(uq,'umtx',timo);
|
|
end;
|
|
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
until false;
|
|
end;
|
|
|
|
function do_lock_umtx(td:p_kthread;umtx:p_umtx;id:QWORD;timeout:p_timespec):Integer;
|
|
var
|
|
tv,ts,ts2:Int64;
|
|
begin
|
|
Result:=0;
|
|
|
|
if (timeout=nil) then
|
|
begin
|
|
Result:=_do_lock_umtx(td,umtx,id,0);
|
|
if (Result=EINTR) then
|
|
begin
|
|
Result:=ERESTART;
|
|
end;
|
|
end else
|
|
begin
|
|
tv:=TIMESPEC_TO_UNIT(timeout);
|
|
ts:=get_unit_uptime;
|
|
ts:=ts+tv;
|
|
|
|
repeat
|
|
Result:=_do_lock_umtx(td,umtx,id,tvtohz(tv));
|
|
if (Result<>ETIMEDOUT) then Break;
|
|
|
|
ts2:=get_unit_uptime;
|
|
if (ts2>=ts) then
|
|
begin
|
|
Result:=ETIMEDOUT;
|
|
Break;
|
|
end;
|
|
|
|
tv:=ts-ts2;
|
|
until false;
|
|
|
|
if (Result=ERESTART) then
|
|
begin
|
|
Result:=EINTR;
|
|
end;
|
|
end;
|
|
end;
|
|
|
|
function do_unlock_umtx(td:p_kthread;umtx:p_umtx;id:QWORD):Integer;
|
|
var
|
|
key:umtx_key;
|
|
owner,old,t:QWORD;
|
|
count:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_unlock_umtx(',HexStr(umtx),',',id,')');
|
|
|
|
owner:=fuword64(umtx^.u_owner);
|
|
|
|
if (owner=QWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (owner=QWORD($40000000)) then
|
|
begin
|
|
Exit(EINVAL);
|
|
end;
|
|
|
|
if ((owner and (not UMTX_CONTESTED))<>id) then
|
|
begin
|
|
Exit(EPERM);
|
|
end;
|
|
|
|
if ((owner and UMTX_CONTESTED)=0) then
|
|
begin
|
|
old:=casuword64(umtx^.u_owner,owner,UMTX_UNOWNED);
|
|
|
|
if (old=QWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (old=owner) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
|
|
owner:=old;
|
|
end;
|
|
|
|
Result:=umtx_key_get(umtx, TYPE_SIMPLE_LOCK, AUTO_SHARE, @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock(@key);
|
|
umtxq_busy(@key);
|
|
count:=umtxq_count(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
if (count<=1) then
|
|
begin
|
|
t:=UMTX_UNOWNED;
|
|
end else
|
|
begin
|
|
t:=UMTX_CONTESTED;
|
|
end;
|
|
|
|
old:=casuword64(umtx^.u_owner,owner,t);
|
|
|
|
umtxq_lock (@key);
|
|
umtxq_signal(@key,1);
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
|
|
if (old=QWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (old<>owner) then
|
|
begin
|
|
Exit(EINVAL);
|
|
end;
|
|
end;
|
|
|
|
function do_wait(td :p_kthread;
|
|
addr :Pointer;
|
|
id :QWORD;
|
|
timeout :p_timespec;
|
|
compat32:Integer;
|
|
priv :Integer
|
|
):Integer;
|
|
var
|
|
uq:p_umtx_q;
|
|
tv,ts,ts2:Int64;
|
|
tmp:QWORD;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_wait(',HexStr(addr),',',id,')');
|
|
|
|
uq:=td^.td_umtxq;
|
|
|
|
Result:=umtx_key_get(addr, TYPE_SIMPLE_WAIT, GET_PRIV_SHARE(priv), @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_insert(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
if (compat32=0) then
|
|
begin
|
|
tmp:=fuword64(PQWORD(addr)^);
|
|
end else
|
|
begin
|
|
tmp:=fuword32(PDWORD(addr)^);
|
|
end;
|
|
|
|
if (tmp<>id) then
|
|
begin
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
end else
|
|
if (timeout=nil) then
|
|
begin
|
|
umtxq_lock(@uq^.uq_key);
|
|
Result:=umtxq_sleep(uq, 'uwait', 0);
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
end else
|
|
begin
|
|
tv:=TIMESPEC_TO_UNIT(timeout);
|
|
ts:=get_unit_uptime;
|
|
ts:=ts+tv;
|
|
|
|
repeat
|
|
umtxq_lock(@uq^.uq_key);
|
|
|
|
Result:=umtxq_sleep(uq,'uwait',tvtohz(tv));
|
|
|
|
if ((uq^.uq_flags and UQF_UMTXQ)=0) then
|
|
begin
|
|
Result:=0;
|
|
Break;
|
|
end;
|
|
|
|
if (Result<>ETIMEDOUT) then Break;
|
|
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
ts2:=get_unit_uptime;
|
|
if (ts2>=ts) then
|
|
begin
|
|
Result:=ETIMEDOUT;
|
|
umtxq_lock(@uq^.uq_key);
|
|
Break;
|
|
end;
|
|
|
|
tv:=ts-ts2;
|
|
until false;
|
|
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
end;
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
|
|
if (Result=ERESTART) then
|
|
begin
|
|
Result:=EINTR;
|
|
end;
|
|
end;
|
|
|
|
function kern_umtx_wake(td:p_kthread;umtx:p_umtx;n_wake,priv:Integer):Integer; public;
|
|
var
|
|
key:umtx_key;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('kern_umtx_wake(',HexStr(umtx),',',n_wake,')');
|
|
|
|
Result:=umtx_key_get(umtx, TYPE_SIMPLE_WAIT, GET_PRIV_SHARE(priv), @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock(@key);
|
|
|
|
Result:=umtxq_signal(@key, n_wake);
|
|
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
Exit(0);
|
|
end;
|
|
|
|
//
|
|
|
|
function _do_lock_normal(td:p_kthread;m:p_umutex;flags:Integer;timo:Int64;mode:Integer):Integer;
|
|
var
|
|
uq:p_umtx_q;
|
|
id,owner,old:DWORD;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('_do_lock_normal(',HexStr(m),',',flags,',',timo,',',mode,')');
|
|
|
|
id:=td^.td_tid;
|
|
uq:=td^.td_umtxq;
|
|
|
|
repeat
|
|
owner:=fuword32(m^.m_owner);
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (owner and (not UMUTEX_CONTESTED) = id) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
|
|
if (owner=DWORD($40000000)) then
|
|
begin
|
|
Exit(EINVAL);
|
|
end;
|
|
|
|
if (mode=_UMUTEX_WAIT) then
|
|
begin
|
|
if (owner=UMUTEX_UNOWNED) or (owner=UMUTEX_CONTESTED) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
end else
|
|
begin
|
|
owner:=casuword32(m^.m_owner,UMUTEX_UNOWNED,id);
|
|
|
|
if (owner=UMUTEX_UNOWNED) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (owner=UMUTEX_CONTESTED) then
|
|
begin
|
|
owner:=casuword32(m^.m_owner,UMUTEX_CONTESTED,id or UMUTEX_CONTESTED);
|
|
|
|
if (owner=UMUTEX_CONTESTED) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Continue;
|
|
end;
|
|
end;
|
|
|
|
if ((flags and UMUTEX_ERROR_CHECK)<>0) and
|
|
((owner and (not UMUTEX_CONTESTED)) = id) then
|
|
begin
|
|
Exit(EDEADLK);
|
|
end;
|
|
|
|
if (mode=_UMUTEX_TRY) then
|
|
begin
|
|
Exit(EBUSY);
|
|
end;
|
|
|
|
if (Result<>0) then Exit;
|
|
|
|
Result:=umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_busy (@uq^.uq_key);
|
|
umtxq_insert(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
old:=casuword32(m^.m_owner,owner,owner or UMUTEX_CONTESTED);
|
|
|
|
if (old=DWORD(-1)) then
|
|
begin
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_remove(uq);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
|
|
if (old=owner) then
|
|
begin
|
|
Result:=umtxq_sleep(uq,'umtxn',timo);
|
|
end;
|
|
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
|
|
until false;
|
|
|
|
Exit(0);
|
|
end;
|
|
|
|
function do_unlock_normal(td:p_kthread;m:p_umutex;flags:Integer):Integer;
|
|
var
|
|
key:umtx_key;
|
|
id,owner,old,t:DWORD;
|
|
count:Integer;
|
|
|
|
tid:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_unlock_normal(',HexStr(m),',',flags,')');
|
|
|
|
id:=td^.td_tid;
|
|
|
|
owner:=fuword32(m^.m_owner);
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (owner=DWORD($40000000)) then
|
|
begin
|
|
Exit(EINVAL);
|
|
end;
|
|
|
|
if ((owner and (not UMUTEX_CONTESTED))<>id) then
|
|
begin
|
|
Exit(EPERM);
|
|
end;
|
|
|
|
if ((owner and UMUTEX_CONTESTED)=0) then
|
|
begin
|
|
old:=casuword32(m^.m_owner,owner,UMUTEX_UNOWNED);
|
|
|
|
if (old=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (old=owner) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
|
|
owner:=old;
|
|
end;
|
|
|
|
Result:=umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock(@key);
|
|
umtxq_busy(@key);
|
|
count:=umtxq_count(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
if (count<=1) then
|
|
begin
|
|
t:=UMUTEX_UNOWNED;
|
|
end else
|
|
begin
|
|
t:=UMUTEX_CONTESTED;
|
|
end;
|
|
|
|
if ((flags and $80) <> 0) and (count <> 0) then
|
|
begin
|
|
umtxq_lock(@key);
|
|
|
|
tid:=umtxq_first_tid(@key,UMTX_SHARED_QUEUE);
|
|
|
|
umtxq_unlock(@key);
|
|
|
|
if (tid=-1) then
|
|
begin
|
|
old:=0;
|
|
end else
|
|
begin
|
|
old:=tid;
|
|
end;
|
|
|
|
old:=casuword32(m^.m_owner,owner,old or t);
|
|
|
|
umtxq_lock(@key);
|
|
|
|
if (tid <> -1) and (old <> DWORD(-1)) then
|
|
begin
|
|
umtxq_signal_by_tid(@key,tid,UMTX_SHARED_QUEUE);
|
|
end;
|
|
|
|
end else
|
|
begin
|
|
old:=casuword32(m^.m_owner,owner,t);
|
|
|
|
umtxq_lock (@key);
|
|
umtxq_signal(@key,1);
|
|
end;
|
|
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
|
|
if (old=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (old<>owner) then
|
|
begin
|
|
Exit(EINVAL);
|
|
end;
|
|
|
|
Exit(0);
|
|
end;
|
|
|
|
function do_wake_umutex(td:p_kthread;m:p_umutex):Integer;
|
|
var
|
|
key:umtx_key;
|
|
owner:DWORD;
|
|
count:Integer;
|
|
flags:DWORD;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_wake_umutex(',HexStr(m),')');
|
|
|
|
owner:=fuword32(m^.m_owner);
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if ((owner and (not UMUTEX_CONTESTED))<>0) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
|
|
flags:=fuword32(m^.m_flags);
|
|
|
|
if (flags=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Result:=umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags), @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock(@key);
|
|
umtxq_busy(@key);
|
|
count:=umtxq_count(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
if (count <= 1) then
|
|
begin
|
|
owner:=casuword32(m^.m_owner,UMUTEX_CONTESTED,UMUTEX_UNOWNED);
|
|
end;
|
|
|
|
umtxq_lock(@key);
|
|
|
|
if (count<>0) and ((owner and (not UMUTEX_CONTESTED))=0) then
|
|
begin
|
|
umtxq_signal(@key, 1);
|
|
end;
|
|
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
|
|
Exit(0);
|
|
end;
|
|
|
|
function do_cv_signalto(td:p_kthread;cv:p_ucond;tid:Integer):Integer;
|
|
var
|
|
key:umtx_key;
|
|
count,flags:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_cv_signalto(',HexStr(cv),',',tid,')');
|
|
|
|
flags:=fuword32(cv^.c_flags);
|
|
|
|
if (flags=-1) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Result:=umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock(@key);
|
|
umtxq_busy(@key);
|
|
|
|
count:=umtxq_count(@key);
|
|
|
|
if (count <> 0) then
|
|
begin
|
|
if umtxq_signal_by_tid(@key,tid,UMTX_SHARED_QUEUE) then
|
|
if (count = 1) then
|
|
begin
|
|
umtxq_unlock(@key);
|
|
Result:=suword32(cv^.c_has_waiters,0);
|
|
umtxq_lock(@key);
|
|
if (Result<>0) then Result:=EFAULT;
|
|
end;
|
|
end;
|
|
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
end;
|
|
|
|
function do_wake3_umutex(td:p_kthread;m:p_umutex;flags:DWORD):Integer;
|
|
label
|
|
_owning,
|
|
_signaling,
|
|
_umtxq_count_end;
|
|
var
|
|
key:umtx_key;
|
|
ktype:Integer;
|
|
owner,old:DWORD;
|
|
count:ptrint;
|
|
|
|
uc:p_umtxq_chain;
|
|
uh:p_umtxq_queue;
|
|
uq:p_umtx_q;
|
|
|
|
tid:Integer;
|
|
|
|
is_signal:Boolean;
|
|
is_cur_td:Boolean;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_wake3_umutex(',HexStr(m),',',flags,')');
|
|
|
|
Case (flags and (UMUTEX_PRIO_INHERIT or UMUTEX_PRIO_PROTECT)) of
|
|
0:ktype:=TYPE_NORMAL_UMUTEX;
|
|
UMUTEX_PRIO_INHERIT:ktype:=TYPE_PI_UMUTEX;
|
|
UMUTEX_PRIO_PROTECT:ktype:=TYPE_PP_UMUTEX;
|
|
else
|
|
Exit(EINVAL);
|
|
end;
|
|
|
|
Result:=umtx_key_get(m, ktype, GET_SHARE(flags), @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
owner:=0;
|
|
tid :=0;
|
|
|
|
umtxq_lock(@key);
|
|
umtxq_busy(@key);
|
|
|
|
//[umtxq_count] analog
|
|
uc:=umtxq_getchain(@key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
uh:=umtxq_queue_lookup(@key, UMTX_SHARED_QUEUE);
|
|
if (uh<>nil) then
|
|
begin
|
|
count :=uh^.length;
|
|
tid :=-1;
|
|
is_signal:=(count<>0);
|
|
|
|
if ((flags and $80) = 0) or (count = 0) then
|
|
begin
|
|
goto _umtxq_count_end;
|
|
end;
|
|
|
|
uh:=uc^.uc_queue[0].lh_first;
|
|
|
|
if (uh=nil) then
|
|
begin
|
|
is_signal:=true;
|
|
flags :=flags or $80;
|
|
goto _umtxq_count_end;
|
|
end;
|
|
|
|
while (uh<>nil) do
|
|
begin
|
|
if ((umtx_key_match(@uh^.key, @key))<>0) then
|
|
begin
|
|
uq:=TAILQ_FIRST(@uh^.head);
|
|
if (uq<>nil) then
|
|
begin
|
|
tid:=uq^.uq_thread^.td_tid;
|
|
end;
|
|
Break;
|
|
end;
|
|
uh:=LIST_NEXT(uh,@uh^.link);
|
|
end;
|
|
|
|
is_signal:=true;
|
|
flags :=flags or $80;
|
|
goto _umtxq_count_end;
|
|
end;
|
|
|
|
tid :=-1;
|
|
is_signal:=False;
|
|
count :=0;
|
|
//
|
|
_umtxq_count_end:
|
|
//[umtxq_count]
|
|
|
|
is_cur_td:=false;
|
|
|
|
umtxq_unlock(@key);
|
|
|
|
if (count < 2) then
|
|
begin
|
|
if (count = 1) then
|
|
begin
|
|
if ((flags and $80) = 0) then
|
|
begin
|
|
owner:=fuword32(m^.m_owner);
|
|
if (owner = DWORD(-1)) then
|
|
begin
|
|
Result:=EFAULT;
|
|
owner :=DWORD(-1);
|
|
goto _signaling;
|
|
end;
|
|
end else
|
|
begin
|
|
owner:=casuword32(m^.m_owner,0,tid);
|
|
if (owner = DWORD(-1)) then
|
|
begin
|
|
is_cur_td:=false;
|
|
Result :=EFAULT;
|
|
goto _signaling;
|
|
end;
|
|
if (owner = 0) then
|
|
begin
|
|
is_cur_td:=true;
|
|
owner :=0;
|
|
Result :=0;
|
|
goto _signaling;
|
|
end;
|
|
end;
|
|
is_cur_td:=false;
|
|
|
|
old:=owner;
|
|
|
|
repeat
|
|
if ((old and UMUTEX_CONTESTED) <> 0) or ((old and (not UMUTEX_CONTESTED)) = 0) then
|
|
begin
|
|
owner :=old;
|
|
Result :=0;
|
|
is_cur_td:=false;
|
|
goto _signaling;
|
|
end;
|
|
|
|
owner:=casuword32(m^.m_owner,old,old or UMUTEX_CONTESTED);
|
|
|
|
if (owner = DWORD(-1)) then
|
|
begin
|
|
owner :=old;
|
|
Result :=EFAULT;
|
|
is_cur_td:=false;
|
|
goto _signaling;
|
|
end;
|
|
|
|
if (owner = old) then
|
|
begin
|
|
owner :=old;
|
|
Result :=0;
|
|
is_cur_td:=false;
|
|
goto _signaling;
|
|
end;
|
|
|
|
Result:=umtxq_check_susp(td);
|
|
|
|
old:=owner;
|
|
until not (Result = 0);
|
|
|
|
end else
|
|
begin //if (count = 1)
|
|
is_cur_td:=false;
|
|
owner :=0;
|
|
Result :=0;
|
|
end;
|
|
end else //(count < 2)
|
|
if ((flags and $80) = 0) then
|
|
begin
|
|
owner:=fuword32(m^.m_owner);
|
|
if (owner <> DWORD(-1)) then
|
|
begin
|
|
goto _owning;
|
|
end;
|
|
Result:=EFAULT;
|
|
owner :=DWORD(-1);
|
|
end else
|
|
begin
|
|
owner:=casuword32(m^.m_owner,0,tid or UMUTEX_CONTESTED);
|
|
|
|
if (owner = DWORD(-1)) then
|
|
begin
|
|
is_cur_td:=false;
|
|
Result :=EFAULT;
|
|
end else
|
|
begin
|
|
if (owner = 0) then
|
|
begin
|
|
is_cur_td:=true;
|
|
owner :=0;
|
|
Result :=0;
|
|
goto _signaling;
|
|
end;
|
|
|
|
///////
|
|
_owning:
|
|
///////
|
|
|
|
is_cur_td:=false;
|
|
|
|
old:=owner;
|
|
repeat
|
|
if ((old and UMUTEX_CONTESTED) <> 0) then
|
|
begin
|
|
owner :=old;
|
|
count :=0;
|
|
is_cur_td:=false;
|
|
goto _signaling;
|
|
end;
|
|
|
|
owner:=casuword32(m^.m_owner,old,old or UMUTEX_CONTESTED);
|
|
|
|
if (owner = DWORD(-1)) then
|
|
begin
|
|
owner :=old;
|
|
Result :=EFAULT;
|
|
is_cur_td:=false;
|
|
goto _signaling;
|
|
end;
|
|
|
|
if (owner = old) then
|
|
begin
|
|
owner :=old;
|
|
Result :=0;
|
|
is_cur_td:=false;
|
|
goto _signaling;
|
|
end;
|
|
|
|
Result:=umtxq_check_susp(td);
|
|
old:=owner;
|
|
until not (Result = 0);
|
|
|
|
end;
|
|
|
|
end;
|
|
|
|
_signaling:
|
|
|
|
umtxq_lock(@key);
|
|
|
|
if (Result = EFAULT) then
|
|
begin
|
|
//umtxq_signal_queue all
|
|
umtxq_signal(@key,High(Integer));
|
|
end else
|
|
if (is_cur_td) then
|
|
begin
|
|
//umtxq_signal_queue by tid
|
|
umtxq_signal_by_tid(@key,tid,UMTX_SHARED_QUEUE);
|
|
end else
|
|
if is_signal and ((owner and (not UMUTEX_CONTESTED)) = 0) then
|
|
begin
|
|
//umtxq_signal_queue one
|
|
umtxq_signal(@key,1);
|
|
end;
|
|
|
|
//unlock and release
|
|
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
end;
|
|
|
|
//
|
|
|
|
function umtx_pi_alloc():p_umtx_pi; inline;
|
|
var
|
|
pi:p_umtx_pi;
|
|
begin
|
|
pi:=AllocMem(sizeof(umtx_pi));
|
|
TAILQ_INIT(@pi^.pi_blocked);
|
|
fetch_add(umtx_pi_allocated, 1);
|
|
Exit(pi);
|
|
end;
|
|
|
|
procedure umtx_pi_free(pi:p_umtx_pi); inline;
|
|
begin
|
|
FreeMem(pi);
|
|
fetch_add(umtx_pi_allocated, -1);
|
|
end;
|
|
|
|
function umtx_pi_adjust_thread(pi:p_umtx_pi;td:p_kthread):Integer;
|
|
var
|
|
uq,uq1,uq2:p_umtx_q;
|
|
td1:p_kthread;
|
|
begin
|
|
mtx_assert(umtx_lock);
|
|
if (pi=nil) then Exit(0);
|
|
|
|
uq:=td^.td_umtxq;
|
|
|
|
uq1:=TAILQ_PREV(uq, @uq^.uq_lockq);
|
|
uq2:=TAILQ_NEXT(uq, @uq^.uq_lockq);
|
|
|
|
if ((uq1<>nil) and (UPRI(td) < UPRI(uq1^.uq_thread))) or
|
|
((uq2<>nil) and (UPRI(td) > UPRI(uq2^.uq_thread))) then
|
|
begin
|
|
TAILQ_REMOVE(@pi^.pi_blocked, uq, @uq^.uq_lockq);
|
|
|
|
uq1:=TAILQ_FIRST(@pi^.pi_blocked);
|
|
while (uq1<>nil) do
|
|
begin
|
|
td1:=uq1^.uq_thread;
|
|
//Assert(td1^.td_proc^.p_magic=P_MAGIC);
|
|
if (UPRI(td1) > UPRI(td)) then Break;
|
|
//
|
|
uq1:=TAILQ_NEXT(uq1,@uq1^.uq_lockq);
|
|
end;
|
|
|
|
if (uq1=nil) then
|
|
begin
|
|
TAILQ_INSERT_TAIL(@pi^.pi_blocked, uq, @uq^.uq_lockq);
|
|
end else
|
|
begin
|
|
TAILQ_INSERT_BEFORE(uq1, uq, @uq^.uq_lockq);
|
|
end;
|
|
end;
|
|
Exit(1);
|
|
end;
|
|
|
|
procedure umtx_propagate_priority(td:p_kthread);
|
|
var
|
|
uq:p_umtx_q;
|
|
pi:p_umtx_pi;
|
|
pri:Integer;
|
|
begin
|
|
mtx_assert(umtx_lock);
|
|
|
|
pri:=UPRI(td);
|
|
uq:=td^.td_umtxq;
|
|
pi:=uq^.uq_pi_blocked;
|
|
if (pi=nil) then Exit;
|
|
|
|
while (true) do
|
|
begin
|
|
td:=pi^.pi_owner;
|
|
if (td=nil) or (td=curkthread) then Exit;
|
|
|
|
//Assert(td^.td_proc<>nil);
|
|
//Assert(td^.td_proc^.p_magic=P_MAGIC);
|
|
|
|
thread_lock(td);
|
|
if (td^.td_lend_user_pri > pri) then
|
|
begin
|
|
sched_lend_user_prio(td, pri);
|
|
end else
|
|
begin
|
|
thread_unlock(td);
|
|
Break;
|
|
end;
|
|
thread_unlock(td);
|
|
|
|
uq:=td^.td_umtxq;
|
|
pi:=uq^.uq_pi_blocked;
|
|
if (pi=nil) then Break;
|
|
|
|
umtx_pi_adjust_thread(pi, td);
|
|
end;
|
|
end;
|
|
|
|
procedure umtx_repropagate_priority(pi:p_umtx_pi);
|
|
var
|
|
uq,uq_owner:p_umtx_q;
|
|
pi2:p_umtx_pi;
|
|
pri:Integer;
|
|
begin
|
|
mtx_assert(umtx_lock);
|
|
|
|
while (pi<>nil) and (pi^.pi_owner<>nil) do
|
|
begin
|
|
pri:=PRI_MAX;
|
|
uq_owner:=pi^.pi_owner^.td_umtxq;
|
|
|
|
pi2:=TAILQ_FIRST(@uq_owner^.uq_pi_contested);
|
|
while (pi2<>nil) do
|
|
begin
|
|
uq:=TAILQ_FIRST(@pi2^.pi_blocked);
|
|
if (uq<>nil) then
|
|
begin
|
|
if (pri > UPRI(uq^.uq_thread)) then
|
|
begin
|
|
pri:=UPRI(uq^.uq_thread);
|
|
end;
|
|
end;
|
|
//
|
|
pi2:=TAILQ_NEXT(pi2,@pi2^.pi_link);
|
|
end;
|
|
|
|
if (pri > uq_owner^.uq_inherited_pri) then
|
|
begin
|
|
pri:=uq_owner^.uq_inherited_pri;
|
|
end;
|
|
|
|
thread_lock(pi^.pi_owner);
|
|
sched_lend_user_prio(pi^.pi_owner, pri);
|
|
thread_unlock(pi^.pi_owner);
|
|
|
|
pi:=uq_owner^.uq_pi_blocked;
|
|
if (pi<>nil) then
|
|
begin
|
|
umtx_pi_adjust_thread(pi, uq_owner^.uq_thread);
|
|
end;
|
|
end;
|
|
end;
|
|
|
|
procedure umtx_pi_setowner(pi:p_umtx_pi;owner:p_kthread);
|
|
var
|
|
uq_owner:p_umtx_q;
|
|
begin
|
|
uq_owner:=owner^.td_umtxq;
|
|
mtx_assert(umtx_lock);
|
|
if (pi^.pi_owner<>nil) then
|
|
begin
|
|
Assert(false,'pi_owner<>nil');
|
|
end;
|
|
thread_inc_ref(owner);
|
|
pi^.pi_owner:=owner;
|
|
TAILQ_INSERT_TAIL(@uq_owner^.uq_pi_contested, pi, @pi^.pi_link);
|
|
end;
|
|
|
|
procedure umtx_pi_clearowner(pi:p_umtx_pi);
|
|
var
|
|
owner:p_kthread;
|
|
begin
|
|
if (pi^.pi_owner<>nil) then
|
|
begin
|
|
owner:=pi^.pi_owner;
|
|
pi^.pi_owner:=nil;
|
|
thread_dec_ref(owner);
|
|
end;
|
|
end;
|
|
|
|
function umtx_pi_claim(pi:p_umtx_pi;owner:p_kthread):Integer;
|
|
var
|
|
uq:p_umtx_q;
|
|
pri:Integer;
|
|
begin
|
|
mtx_lock(umtx_lock);
|
|
|
|
if (pi^.pi_owner=owner) then
|
|
begin
|
|
mtx_unlock(umtx_lock);
|
|
Exit (0);
|
|
end;
|
|
|
|
if (pi^.pi_owner<>nil) then
|
|
begin
|
|
mtx_unlock(umtx_lock);
|
|
Exit(EPERM);
|
|
end;
|
|
|
|
umtx_pi_setowner(pi, owner);
|
|
uq:=TAILQ_FIRST(@pi^.pi_blocked);
|
|
|
|
if (uq<>nil) then
|
|
begin
|
|
pri:=UPRI(uq^.uq_thread);
|
|
thread_lock(owner);
|
|
if (pri < UPRI(owner)) then
|
|
begin
|
|
sched_lend_user_prio(owner, pri);
|
|
end;
|
|
thread_unlock(owner);
|
|
end;
|
|
mtx_unlock(umtx_lock);
|
|
Exit(0);
|
|
end;
|
|
|
|
procedure umtx_pi_adjust(td:p_kthread;oldpri:Integer); public;
|
|
var
|
|
uq:p_umtx_q;
|
|
pi:p_umtx_pi;
|
|
begin
|
|
uq:=td^.td_umtxq;
|
|
mtx_lock(umtx_lock);
|
|
|
|
pi:=uq^.uq_pi_blocked;
|
|
if (pi<>nil) then
|
|
begin
|
|
umtx_pi_adjust_thread(pi, td);
|
|
umtx_repropagate_priority(pi);
|
|
end;
|
|
mtx_unlock(umtx_lock);
|
|
end;
|
|
|
|
function umtxq_sleep_pi(uq:p_umtx_q;
|
|
pi:p_umtx_pi;
|
|
owner:DWORD;
|
|
wmesg:pchar;
|
|
timo:Int64;
|
|
flags:DWORD):Integer;
|
|
var
|
|
uc:p_umtxq_chain;
|
|
td,td1:p_kthread;
|
|
uq1:p_umtx_q;
|
|
pri:Integer;
|
|
begin
|
|
td:=uq^.uq_thread;
|
|
Assert(td=curkthread,'inconsistent uq_thread');
|
|
|
|
uc:=umtxq_getchain(@uq^.uq_key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
UMTXQ_BUSY_ASSERT(uc);
|
|
umtxq_insert(uq);
|
|
|
|
mtx_lock(umtx_lock);
|
|
if (pi^.pi_owner=nil) then
|
|
begin
|
|
mtx_unlock(umtx_lock);
|
|
|
|
td1:=tdfind(owner);
|
|
|
|
mtx_lock(umtx_lock);
|
|
if (td1<>nil) then
|
|
begin
|
|
if (pi^.pi_owner=nil) then
|
|
begin
|
|
umtx_pi_setowner(pi, td1);
|
|
end;
|
|
//
|
|
thread_dec_ref(td1);
|
|
end;
|
|
end;
|
|
|
|
uq1:=TAILQ_FIRST(@pi^.pi_blocked);
|
|
while (uq1<>nil) do
|
|
begin
|
|
pri:=UPRI(uq1^.uq_thread);
|
|
if (pri > UPRI(td)) then Break;
|
|
//
|
|
uq1:=TAILQ_NEXT(uq1,@uq1^.uq_lockq);
|
|
end;
|
|
|
|
if (uq1<>nil) then
|
|
begin
|
|
TAILQ_INSERT_BEFORE(uq1, uq, @uq^.uq_lockq);
|
|
end else
|
|
begin
|
|
TAILQ_INSERT_TAIL(@pi^.pi_blocked, uq, @uq^.uq_lockq);
|
|
end;
|
|
|
|
uq^.uq_pi_blocked:=pi;
|
|
|
|
thread_lock(td);
|
|
td^.td_flags:=td^.td_flags or TDF_UPIBLOCKED;
|
|
thread_unlock(td);
|
|
|
|
umtx_propagate_priority(td);
|
|
mtx_unlock(umtx_lock);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
|
|
if ((uq^.uq_flags and UQF_UMTXQ)<>0) then
|
|
begin
|
|
Result:=msleep(uq, @uc^.uc_lock, PCATCH, wmesg, timo);
|
|
if (Result=EWOULDBLOCK) then
|
|
begin
|
|
Result:=ETIMEDOUT;
|
|
end;
|
|
if ((uq^.uq_flags and UQF_UMTXQ)<>0) then
|
|
begin
|
|
umtxq_remove(uq);
|
|
end;
|
|
end;
|
|
mtx_lock(umtx_lock);
|
|
|
|
if ((flags and $280) = $280) and
|
|
((td^.td_flags and TDF_UPIBLOCKED) = 0) then
|
|
begin
|
|
mtx_unlock(umtx_lock);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
Exit;
|
|
end;
|
|
|
|
uq^.uq_pi_blocked:=nil;
|
|
|
|
thread_lock(td);
|
|
td^.td_flags:=td^.td_flags and (not TDF_UPIBLOCKED);
|
|
thread_unlock(td);
|
|
|
|
TAILQ_REMOVE(@pi^.pi_blocked, uq, @uq^.uq_lockq);
|
|
umtx_repropagate_priority(pi);
|
|
|
|
mtx_unlock(umtx_lock);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
end;
|
|
|
|
procedure umtx_pi_ref(pi:p_umtx_pi);
|
|
var
|
|
uc:p_umtxq_chain;
|
|
begin
|
|
uc:=umtxq_getchain(@pi^.pi_key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
Inc(pi^.pi_refcount);
|
|
end;
|
|
|
|
procedure umtx_pi_unref(pi:p_umtx_pi);
|
|
var
|
|
uq:p_umtx_q;
|
|
uc:p_umtxq_chain;
|
|
begin
|
|
uc:=umtxq_getchain(@pi^.pi_key);
|
|
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
Assert(pi^.pi_refcount > 0,'invalid reference count');
|
|
|
|
Dec(pi^.pi_refcount);
|
|
if (pi^.pi_refcount=0) then
|
|
begin
|
|
mtx_lock(umtx_lock);
|
|
|
|
if (pi^.pi_owner<>nil) then
|
|
begin
|
|
uq:=pi^.pi_owner^.td_umtxq;
|
|
TAILQ_REMOVE(@uq^.uq_pi_contested, pi, @pi^.pi_link);
|
|
umtx_pi_clearowner(pi);
|
|
end;
|
|
|
|
Assert(TAILQ_EMPTY(@pi^.pi_blocked),'blocked queue not empty');
|
|
mtx_unlock(umtx_lock);
|
|
|
|
TAILQ_REMOVE(@uc^.uc_pi_list, pi, @pi^.pi_hashlink);
|
|
|
|
umtx_pi_free(pi);
|
|
end;
|
|
end;
|
|
|
|
function umtx_pi_lookup(key:p_umtx_key):p_umtx_pi;
|
|
var
|
|
uc:p_umtxq_chain;
|
|
pi:p_umtx_pi;
|
|
begin
|
|
uc:=umtxq_getchain(key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
|
|
pi:=TAILQ_FIRST(@uc^.uc_pi_list);
|
|
while (pi<>nil) do
|
|
begin
|
|
if (umtx_key_match(@pi^.pi_key, key)<>0) then
|
|
begin
|
|
Exit(pi);
|
|
end;
|
|
//
|
|
pi:=TAILQ_NEXT(pi,@pi^.pi_hashlink);
|
|
end;
|
|
|
|
Exit(nil);
|
|
end;
|
|
|
|
procedure umtx_pi_insert(pi:p_umtx_pi);
|
|
var
|
|
uc:p_umtxq_chain;
|
|
begin
|
|
uc:=umtxq_getchain(@pi^.pi_key);
|
|
UMTXQ_LOCKED_ASSERT(uc);
|
|
TAILQ_INSERT_TAIL(@uc^.uc_pi_list, pi, @pi^.pi_hashlink);
|
|
end;
|
|
|
|
function _do_lock_pi(td:p_kthread;m:p_umutex;flags:Integer;timo:Int64;mode:Integer):Integer;
|
|
var
|
|
uq:p_umtx_q;
|
|
pi,new_pi:p_umtx_pi;
|
|
id,owner,old:DWORD;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('_do_lock_pi(',HexStr(m),',',flags,',',timo,',',mode,')');
|
|
|
|
id:=td^.td_tid;
|
|
uq:=td^.td_umtxq;
|
|
|
|
Result:=umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock(@uq^.uq_key);
|
|
pi:=umtx_pi_lookup(@uq^.uq_key);
|
|
|
|
if (pi=nil) then
|
|
begin
|
|
new_pi:=umtx_pi_alloc();
|
|
|
|
if (new_pi=nil) then
|
|
begin
|
|
umtxq_unlock(@uq^.uq_key);
|
|
new_pi:=umtx_pi_alloc();
|
|
umtxq_lock(@uq^.uq_key);
|
|
|
|
pi:=umtx_pi_lookup(@uq^.uq_key);
|
|
if (pi<>nil) then
|
|
begin
|
|
umtx_pi_free(new_pi);
|
|
new_pi:=nil;
|
|
end;
|
|
end;
|
|
|
|
if (new_pi<>nil) then
|
|
begin
|
|
new_pi^.pi_key:=uq^.uq_key;
|
|
umtx_pi_insert(new_pi);
|
|
pi:=new_pi;
|
|
end;
|
|
end;
|
|
|
|
umtx_pi_ref(pi);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
while (true) do
|
|
begin
|
|
owner:=casuword32(m^.m_owner, UMUTEX_UNOWNED, id);
|
|
|
|
if (owner=UMUTEX_UNOWNED) then
|
|
begin
|
|
Result:=0;
|
|
Break;
|
|
end;
|
|
|
|
if (owner=DWORD($40000000)) then
|
|
begin
|
|
Result:=EINVAL;
|
|
Break;
|
|
end;
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Result:=EFAULT;
|
|
Break;
|
|
end;
|
|
|
|
if (owner=UMUTEX_CONTESTED) then
|
|
begin
|
|
owner:=casuword32(m^.m_owner, UMUTEX_CONTESTED, id or UMUTEX_CONTESTED);
|
|
|
|
if (owner=UMUTEX_CONTESTED) then
|
|
begin
|
|
umtxq_lock(@uq^.uq_key);
|
|
umtxq_busy(@uq^.uq_key);
|
|
Result:=umtx_pi_claim(pi, td);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
Break;
|
|
end;
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Result:=EFAULT;
|
|
Break;
|
|
end;
|
|
|
|
continue;
|
|
end;
|
|
|
|
if (owner and (not UMUTEX_CONTESTED) = id) then
|
|
begin
|
|
Result:=0;
|
|
Break;
|
|
end;
|
|
|
|
if ((flags and UMUTEX_ERROR_CHECK)<>0) and
|
|
((owner and (not UMUTEX_CONTESTED))=id) then
|
|
begin
|
|
Result:=EDEADLK;
|
|
Break;
|
|
end;
|
|
|
|
if (mode<>0) then
|
|
begin
|
|
Result:=EBUSY;
|
|
Break;
|
|
end;
|
|
|
|
if (Result<>0) then Break;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_busy (@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
old:=casuword32(m^.m_owner, owner, owner or UMUTEX_CONTESTED);
|
|
|
|
if (old=DWORD(-1)) then
|
|
begin
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
Result:=EFAULT;
|
|
Break;
|
|
end;
|
|
|
|
umtxq_lock(@uq^.uq_key);
|
|
|
|
if (old=owner) then
|
|
begin
|
|
Result:=umtxq_sleep_pi(uq, pi, owner and (not UMUTEX_CONTESTED),'umtxpi', timo, flags);
|
|
end else
|
|
begin
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
end;
|
|
|
|
Result:=umtxq_check_susp(td);
|
|
if (Result<>0) then Break;
|
|
end; //while
|
|
|
|
if (Result<>0) then
|
|
begin
|
|
casuword32(m^.m_owner, id or UMUTEX_CONTESTED, UMUTEX_CONTESTED);
|
|
end;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtx_pi_unref(pi);
|
|
umtxq_unlock (@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
end;
|
|
|
|
function do_unlock_pi(td:p_kthread;m:p_umutex;flags:Integer):Integer;
|
|
var
|
|
uq_thread:p_kthread;
|
|
key:umtx_key;
|
|
uq_first,uq_first2,uq_me:p_umtx_q;
|
|
pi,pi2:p_umtx_pi;
|
|
owner,old,id,t:DWORD;
|
|
count,pri:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_unlock_pi(',HexStr(m),',',flags,')');
|
|
|
|
id:=td^.td_tid;
|
|
|
|
owner:=fuword32(m^.m_owner);
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (owner=DWORD($40000000)) then
|
|
begin
|
|
Exit(EINVAL);
|
|
end;
|
|
|
|
if ((owner and (not UMUTEX_CONTESTED))<>id) then
|
|
begin
|
|
Exit(EPERM);
|
|
end;
|
|
|
|
if ((owner and UMUTEX_CONTESTED)=0) then
|
|
begin
|
|
old:=casuword32(m^.m_owner, owner, UMUTEX_UNOWNED);
|
|
|
|
if (old=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (old=owner) then
|
|
begin
|
|
Exit(0);
|
|
end;
|
|
|
|
owner:=old;
|
|
end;
|
|
|
|
Result:=umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock(@key);
|
|
umtxq_busy(@key);
|
|
count:=umtxq_count_pi(@key, @uq_first);
|
|
|
|
if (count<=1) then
|
|
begin
|
|
t:=UMUTEX_UNOWNED;
|
|
end else
|
|
begin
|
|
t:=UMUTEX_CONTESTED;
|
|
end;
|
|
|
|
if (uq_first<>nil) then
|
|
begin
|
|
mtx_lock(umtx_lock);
|
|
|
|
pi:=uq_first^.uq_pi_blocked;
|
|
Assert(pi<>nil,'pi=nil?');
|
|
|
|
if (pi^.pi_owner<>td) then
|
|
begin
|
|
mtx_unlock(umtx_lock);
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
umtx_key_release(@key);
|
|
|
|
Exit(EPERM);
|
|
end;
|
|
|
|
uq_me:=td^.td_umtxq;
|
|
|
|
umtx_pi_clearowner(pi);
|
|
|
|
TAILQ_REMOVE(@uq_me^.uq_pi_contested, pi, @pi^.pi_link);
|
|
|
|
uq_first:=TAILQ_FIRST(@pi^.pi_blocked);
|
|
while (uq_first<>nil) and
|
|
((uq_first^.uq_flags and UQF_UMTXQ)=0) do
|
|
begin
|
|
uq_first:=TAILQ_NEXT(uq_first, @uq_first^.uq_lockq);
|
|
end;
|
|
|
|
pri:=1023;
|
|
|
|
pi2:=TAILQ_FIRST(@uq_me^.uq_pi_contested);
|
|
while (pi2<>nil) do
|
|
begin
|
|
uq_first2:=TAILQ_FIRST(@pi2^.pi_blocked);
|
|
if (uq_first2<>nil) then
|
|
begin
|
|
if (pri > UPRI(uq_first2^.uq_thread)) then
|
|
begin
|
|
pri:=UPRI(uq_first2^.uq_thread);
|
|
end;
|
|
end;
|
|
//
|
|
pi2:=TAILQ_NEXT(pi2,@pi2^.pi_link);
|
|
end;
|
|
|
|
thread_lock(td);
|
|
sched_lend_user_prio(td, pri);
|
|
thread_unlock(td);
|
|
|
|
mtx_unlock(umtx_lock);
|
|
|
|
if (uq_first=nil) then
|
|
begin
|
|
umtxq_unlock(@key);
|
|
end else
|
|
begin
|
|
//umtxq_signal_thread (extended)
|
|
|
|
if ((flags and $80) = 0) then
|
|
begin
|
|
umtxq_signal_thread(uq_first);
|
|
umtxq_unlock(@key);
|
|
end else
|
|
begin
|
|
uq_thread:=uq_first^.uq_thread;
|
|
id:=uq_thread^.td_tid;
|
|
|
|
if ((flags and $200) <> 0) then
|
|
begin
|
|
mtx_lock(umtx_lock);
|
|
|
|
pi:=uq_first^.uq_pi_blocked;
|
|
Assert(pi<>nil,'pi=nil?');
|
|
|
|
uq_first^.uq_pi_blocked:=nil;
|
|
|
|
thread_lock(uq_thread);
|
|
uq_thread^.td_flags:=uq_thread^.td_flags and (not TDF_UPIBLOCKED);
|
|
thread_unlock(uq_thread);
|
|
|
|
TAILQ_REMOVE(@pi^.pi_blocked, uq_first, @uq_first^.uq_lockq);
|
|
|
|
umtx_repropagate_priority(pi);
|
|
|
|
mtx_unlock(umtx_lock);
|
|
|
|
if (count > 1) then
|
|
begin
|
|
Result:=umtx_pi_claim(pi, uq_thread);
|
|
|
|
if (Result <> 0) then
|
|
begin
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
|
|
Exit(EINVAL);
|
|
end;
|
|
end;
|
|
|
|
end;//(flags and $200) <> 0)
|
|
|
|
umtxq_signal_thread(uq_first);
|
|
umtxq_unlock(@key);
|
|
|
|
if (count < 2) then
|
|
begin
|
|
t:=id;
|
|
end else
|
|
begin
|
|
t:=id or UMUTEX_CONTESTED;
|
|
end;
|
|
|
|
end; //((flags and $80) <> 0)
|
|
|
|
end; //(uq_first<>nil)
|
|
|
|
end;
|
|
|
|
old:=casuword32(m^.m_owner, owner, t);
|
|
|
|
umtxq_lock (@key);
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
|
|
if (old=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (old<>owner) then
|
|
begin
|
|
Exit(EINVAL);
|
|
end;
|
|
|
|
Exit(0);
|
|
end;
|
|
|
|
//
|
|
|
|
function _do_lock_pp(td:p_kthread;m:p_umutex;flags:Integer;timo:Int64;mode:Integer):Integer;
|
|
label
|
|
_out;
|
|
var
|
|
uq,uq2:p_umtx_q;
|
|
pi:p_umtx_pi;
|
|
ceiling,owner,id:DWORD;
|
|
pri,old_inherited_pri,su:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('_do_lock_pp(',HexStr(m),',',flags,',',timo,',',mode,')');
|
|
|
|
id:=td^.td_tid;
|
|
uq:=td^.td_umtxq;
|
|
|
|
Result:=umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
su:=1;
|
|
//su:=(priv_check(td, PRIV_SCHED_RTPRIO)=0);
|
|
|
|
while (true) do
|
|
begin
|
|
old_inherited_pri:=uq^.uq_inherited_pri;
|
|
umtxq_lock(@uq^.uq_key);
|
|
umtxq_busy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
ceiling:=fuword32(m^.m_ceilings[0]);
|
|
|
|
if (ceiling=DWORD(-1)) then
|
|
begin
|
|
Result:=EFAULT;
|
|
goto _out;
|
|
end;
|
|
|
|
if ((flags and $400) = 0) then
|
|
begin
|
|
if (ceiling > 767) then
|
|
begin
|
|
Result:=EINVAL;
|
|
goto _out;
|
|
end;
|
|
ceiling:=767 - ceiling;
|
|
end else
|
|
begin
|
|
if ((ceiling - 256) > 639) then
|
|
begin
|
|
Result:=EINVAL;
|
|
goto _out;
|
|
end;
|
|
end;
|
|
|
|
mtx_lock(umtx_lock);
|
|
|
|
if ((flags and $400) = 0) then
|
|
begin
|
|
ceiling:=ceiling + 256;
|
|
|
|
if (UPRI(td) < ceiling) then
|
|
begin
|
|
mtx_unlock(umtx_lock);
|
|
Result:=EINVAL;
|
|
goto _out;
|
|
end;
|
|
|
|
if (su<>0) and (ceiling < uq^.uq_inherited_pri) then
|
|
begin
|
|
uq^.uq_inherited_pri:=ceiling;
|
|
//
|
|
thread_lock(td);
|
|
//
|
|
if (uq^.uq_inherited_pri < UPRI(td)) then
|
|
begin
|
|
sched_lend_user_prio(td, uq^.uq_inherited_pri);
|
|
end;
|
|
//
|
|
thread_unlock(td);
|
|
end;
|
|
|
|
end else
|
|
begin
|
|
|
|
if (UPRI(td) < ceiling) then
|
|
begin
|
|
mtx_unlock(umtx_lock);
|
|
Result:=EINVAL;
|
|
goto _out;
|
|
end;
|
|
|
|
if (ceiling < uq^.uq_inherited_pri) then
|
|
begin
|
|
uq^.uq_inherited_pri:=ceiling;
|
|
//
|
|
thread_lock(td);
|
|
//
|
|
if (uq^.uq_inherited_pri < UPRI(td)) then
|
|
begin
|
|
sched_lend_user_prio(td, uq^.uq_inherited_pri);
|
|
end;
|
|
//
|
|
thread_unlock(td);
|
|
end;
|
|
|
|
end;
|
|
|
|
mtx_unlock(umtx_lock);
|
|
|
|
owner:=casuword32(m^.m_owner, UMUTEX_CONTESTED, id or UMUTEX_CONTESTED);
|
|
|
|
if (owner=UMUTEX_CONTESTED) then
|
|
begin
|
|
Result:=0;
|
|
Break;
|
|
end;
|
|
|
|
if (owner=DWORD($40000000)) then
|
|
begin
|
|
Result:=EINVAL;
|
|
Break;
|
|
end;
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Result:=EFAULT;
|
|
Break;
|
|
end;
|
|
|
|
if ((flags and UMUTEX_ERROR_CHECK)<>0) and
|
|
((owner and (not UMUTEX_CONTESTED))=id) then
|
|
begin
|
|
Result:=EDEADLK;
|
|
Break;
|
|
end;
|
|
|
|
if (mode<>0) then
|
|
begin
|
|
Result:=EBUSY;
|
|
Break;
|
|
end;
|
|
|
|
if (Result<>0) then Break;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_insert(uq);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
Result:=umtxq_sleep(uq, 'umtxpp', timo);
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
mtx_lock(umtx_lock);
|
|
uq^.uq_inherited_pri:=old_inherited_pri;
|
|
pri:=PRI_MAX;
|
|
|
|
pi:=TAILQ_FIRST(@uq^.uq_pi_contested);
|
|
while (pi<>nil) do
|
|
begin
|
|
uq2:=TAILQ_FIRST(@pi^.pi_blocked);
|
|
if (uq2<>nil) then
|
|
begin
|
|
if (pri > UPRI(uq2^.uq_thread)) then
|
|
begin
|
|
pri:=UPRI(uq2^.uq_thread);
|
|
end;
|
|
end;
|
|
//
|
|
pi:=TAILQ_NEXT(pi,@pi^.pi_link);
|
|
end;
|
|
|
|
if (pri > uq^.uq_inherited_pri) then
|
|
begin
|
|
pri:=uq^.uq_inherited_pri;
|
|
end;
|
|
|
|
thread_lock(td);
|
|
sched_lend_user_prio(td, pri);
|
|
thread_unlock(td);
|
|
|
|
mtx_unlock(umtx_lock);
|
|
end;
|
|
|
|
if (Result<>0) then
|
|
begin
|
|
mtx_lock(umtx_lock);
|
|
|
|
uq^.uq_inherited_pri:=old_inherited_pri;
|
|
pri:=PRI_MAX;
|
|
|
|
pi:=TAILQ_FIRST(@uq^.uq_pi_contested);
|
|
while (pi<>nil) do
|
|
begin
|
|
uq2:=TAILQ_FIRST(@pi^.pi_blocked);
|
|
if (uq2<>nil) then
|
|
begin
|
|
if (pri > UPRI(uq2^.uq_thread)) then
|
|
begin
|
|
pri:=UPRI(uq2^.uq_thread);
|
|
end;
|
|
end;
|
|
//
|
|
pi:=TAILQ_NEXT(pi,@pi^.pi_link);
|
|
end;
|
|
|
|
if (pri > uq^.uq_inherited_pri) then
|
|
begin
|
|
pri:=uq^.uq_inherited_pri;
|
|
end;
|
|
|
|
thread_lock(td);
|
|
sched_lend_user_prio(td, pri);
|
|
thread_unlock(td);
|
|
|
|
mtx_unlock(umtx_lock);
|
|
end;
|
|
|
|
_out:
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
end;
|
|
|
|
function do_unlock_pp(td:p_kthread;m:p_umutex;flags:Integer):Integer;
|
|
var
|
|
key:umtx_key;
|
|
uq,uq2:p_umtx_q;
|
|
pi:p_umtx_pi;
|
|
owner,id,rceiling:DWORD;
|
|
pri,new_inherited_pri,su:Integer;
|
|
|
|
tid:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_unlock_pp(',HexStr(m),',',flags,')');
|
|
|
|
id:=td^.td_tid;
|
|
uq:=td^.td_umtxq;
|
|
|
|
su:=1;
|
|
//su:=(priv_check(td, PRIV_SCHED_RTPRIO)=0);
|
|
|
|
owner:=fuword32(m^.m_owner);
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (owner=DWORD($40000000)) then
|
|
begin
|
|
Exit(EINVAL);
|
|
end;
|
|
|
|
if ((owner and (not UMUTEX_CONTESTED))<>id) then
|
|
begin
|
|
Exit(EPERM);
|
|
end;
|
|
|
|
Result:=copyin(@m^.m_ceilings[1], @rceiling, sizeof(DWORD));
|
|
if (Result<>0) then Exit;
|
|
|
|
if (rceiling=DWORD(-1)) then
|
|
begin
|
|
new_inherited_pri:=1023;
|
|
end else
|
|
begin
|
|
if (rceiling > 767) then
|
|
begin
|
|
Exit(EINVAL);
|
|
end;
|
|
rceiling:=767 - rceiling;
|
|
new_inherited_pri:=1023 - rceiling;
|
|
end;
|
|
|
|
Result:=umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock (@key);
|
|
umtxq_busy (@key);
|
|
|
|
if ((flags and $80) <> 0) then
|
|
begin
|
|
tid:=umtxq_first_tid(@key,UMTX_SHARED_QUEUE);
|
|
end;
|
|
|
|
umtxq_unlock(@key);
|
|
|
|
if ((flags and $80) <> 0) then
|
|
begin
|
|
|
|
if (tid <> -1) then
|
|
begin
|
|
id:=tid or UMUTEX_CONTESTED;
|
|
end else
|
|
begin
|
|
id:=UMUTEX_CONTESTED;
|
|
end;
|
|
|
|
Result:=suword32(m^.m_owner, id);
|
|
|
|
umtxq_lock(@key);
|
|
|
|
if (tid <> -1) and (Result = 0) then
|
|
begin
|
|
umtxq_signal_by_tid(@key,tid,UMTX_SHARED_QUEUE);
|
|
end;
|
|
|
|
end else
|
|
begin
|
|
|
|
Result:=suword32(m^.m_owner, UMUTEX_CONTESTED);
|
|
|
|
umtxq_lock(@key);
|
|
|
|
if (Result=0) then
|
|
begin
|
|
umtxq_signal(@key, 1);
|
|
end;
|
|
|
|
end;
|
|
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
if (Result=-1) then
|
|
begin
|
|
Result:=EFAULT;
|
|
end else
|
|
begin
|
|
mtx_lock(umtx_lock);
|
|
|
|
if ((flags and $400) <> 0) or (su<>0) then
|
|
begin
|
|
uq^.uq_inherited_pri:=new_inherited_pri;
|
|
end;
|
|
|
|
pri:=1023;
|
|
|
|
pi:=TAILQ_FIRST(@uq^.uq_pi_contested);
|
|
while (pi<>nil) do
|
|
begin
|
|
uq2:=TAILQ_FIRST(@pi^.pi_blocked);
|
|
if (uq2<>nil) then
|
|
begin
|
|
if (pri > UPRI(uq2^.uq_thread)) then
|
|
begin
|
|
pri:=UPRI(uq2^.uq_thread);
|
|
end;
|
|
end;
|
|
//
|
|
pi:=TAILQ_NEXT(pi,@pi^.pi_link);
|
|
end;
|
|
|
|
if (pri > uq^.uq_inherited_pri) then
|
|
begin
|
|
pri:=uq^.uq_inherited_pri;
|
|
end;
|
|
|
|
thread_lock(td);
|
|
sched_lend_user_prio(td, pri);
|
|
thread_unlock(td);
|
|
|
|
mtx_unlock(umtx_lock);
|
|
end;
|
|
|
|
umtx_key_release(@key);
|
|
end;
|
|
|
|
function do_set_ceiling(td:p_kthread;m:p_umutex;ceiling:DWORD;old_ceiling:PDWORD):Integer;
|
|
var
|
|
uq:p_umtx_q;
|
|
save_ceiling:DWORD;
|
|
id,owner:DWORD;
|
|
flags:DWORD;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_set_ceiling(',HexStr(m),',',ceiling,')');
|
|
|
|
flags:=fuword32(m^.m_flags);
|
|
|
|
if (flags=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if ((flags and UMUTEX_PRIO_PROTECT)=0) then Exit(EINVAL);
|
|
if (ceiling>PRI_MAX) then Exit(EINVAL);
|
|
|
|
id:=td^.td_tid;
|
|
uq:=td^.td_umtxq;
|
|
|
|
Result:=umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
repeat
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_busy (@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
save_ceiling:=fuword32(m^.m_ceilings[0]);
|
|
|
|
owner:=fuword32(m^.m_owner);
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Result:=EFAULT;
|
|
Break;
|
|
end;
|
|
|
|
owner:=casuword32(m^.m_owner,UMUTEX_CONTESTED,id or UMUTEX_CONTESTED);
|
|
|
|
if (owner=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (owner=UMUTEX_CONTESTED) then
|
|
begin
|
|
suword32(m^.m_ceilings[0],ceiling);
|
|
suword32(m^.m_owner,UMUTEX_CONTESTED);
|
|
Result:=0;
|
|
Break;
|
|
end;
|
|
|
|
if ((owner and (not UMUTEX_CONTESTED))=id) then
|
|
begin
|
|
suword32(m^.m_ceilings[0],ceiling);
|
|
Result:=0;
|
|
Break;
|
|
end;
|
|
|
|
if (Result<>0) then Break;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_insert(uq);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
|
|
Result:=umtxq_sleep(uq, 'umtxpp', 0);
|
|
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
until false;
|
|
|
|
umtxq_lock(@uq^.uq_key);
|
|
|
|
if (Result=0) then
|
|
begin
|
|
umtxq_signal(@uq^.uq_key,High(Integer));
|
|
end;
|
|
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
|
|
if (Result=0) and (old_ceiling<>nil) then
|
|
begin
|
|
suword32(old_ceiling^,save_ceiling);
|
|
end;
|
|
end;
|
|
|
|
//////
|
|
|
|
function _do_lock_umutex(td:p_kthread;m:p_umutex;flags:Integer;timo:Int64;mode:Integer):Integer;
|
|
begin
|
|
Case (flags and (UMUTEX_PRIO_INHERIT or UMUTEX_PRIO_PROTECT)) of
|
|
0:Result:=_do_lock_normal(td,m,flags,timo,mode);
|
|
UMUTEX_PRIO_INHERIT:Result:=_do_lock_pi(td,m,flags,timo,mode);
|
|
UMUTEX_PRIO_PROTECT:Result:=_do_lock_pp(td,m,flags,timo,mode);
|
|
else
|
|
Exit(EINVAL);
|
|
end;
|
|
end;
|
|
|
|
function do_lock_umutex(td:p_kthread;m:p_umutex;timeout:p_timespec;mode:Integer):Integer;
|
|
var
|
|
flags:DWORD;
|
|
tv,ts,ts2:Int64;
|
|
begin
|
|
Result:=0;
|
|
|
|
flags:=fuword32(m^.m_flags);
|
|
|
|
if (flags=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (timeout=nil) then
|
|
begin
|
|
Result:=_do_lock_umutex(td,m,flags,0,mode);
|
|
if (Result=EINTR) and (mode<>_UMUTEX_WAIT) then
|
|
begin
|
|
Result:=ERESTART;
|
|
end;
|
|
end else
|
|
begin
|
|
tv:=TIMESPEC_TO_UNIT(timeout);
|
|
ts:=get_unit_uptime;
|
|
ts:=ts+tv;
|
|
|
|
repeat
|
|
Result:=_do_lock_umutex(td,m,flags,tvtohz(tv),mode);
|
|
if (Result<>ETIMEDOUT) then Break;
|
|
|
|
ts2:=get_unit_uptime;
|
|
if (ts2>=ts) then
|
|
begin
|
|
Result:=ETIMEDOUT;
|
|
Break;
|
|
end;
|
|
|
|
tv:=ts-ts2;
|
|
until false;
|
|
|
|
if (Result=ERESTART) then
|
|
begin
|
|
Result:=EINTR;
|
|
end;
|
|
end;
|
|
end;
|
|
|
|
function do_unlock_umutex(td:p_kthread;m:p_umutex):Integer; inline;
|
|
var
|
|
flags:DWORD;
|
|
begin
|
|
Result:=0;
|
|
|
|
flags:=fuword32(m^.m_flags);
|
|
|
|
if (flags=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Case (flags and (UMUTEX_PRIO_INHERIT or UMUTEX_PRIO_PROTECT)) of
|
|
0:Result:=do_unlock_normal(td,m,flags);
|
|
UMUTEX_PRIO_INHERIT:Result:=do_unlock_pi(td,m,flags);
|
|
UMUTEX_PRIO_PROTECT:Result:=do_unlock_pp(td,m,flags);
|
|
else
|
|
Exit(EINVAL);
|
|
end;
|
|
end;
|
|
|
|
////
|
|
|
|
function do_cv_wait(td:p_kthread;cv:p_ucond;m:p_umutex;timeout:p_timespec;wflags:QWORD):Integer;
|
|
var
|
|
uq:p_umtx_q;
|
|
clockid:Integer;
|
|
oldlen:Integer;
|
|
flags:Integer;
|
|
cts,ets,tts:Int64;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_cv_wait(',HexStr(cv),',',HexStr(m),',',wflags,')');
|
|
|
|
uq:=td^.td_umtxq;
|
|
|
|
flags:=fuword32(cv^.c_flags);
|
|
|
|
if (flags=-1) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Result:=umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
if ((wflags and CVWAIT_CLOCKID)<>0) then
|
|
begin
|
|
clockid:=fuword32(cv^.c_clockid);
|
|
if (clockid<CLOCK_REALTIME) or
|
|
(clockid>CLOCK_SECOND) then
|
|
begin
|
|
umtx_key_release(@uq^.uq_key);
|
|
Exit(EINVAL);
|
|
end;
|
|
end else
|
|
if ((wflags and CVWAIT_ABSTIME)<>0) then
|
|
begin
|
|
clockid:=CLOCK_REALTIME;
|
|
end else
|
|
begin
|
|
clockid:=CLOCK_PROCTIME;
|
|
end;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_busy (@uq^.uq_key);
|
|
umtxq_insert(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
if (fuword32(cv^.c_has_waiters)=0) then
|
|
begin
|
|
suword32(cv^.c_has_waiters,1);
|
|
end;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
Result:=do_unlock_umutex(td,m);
|
|
|
|
umtxq_lock(@uq^.uq_key);
|
|
|
|
if (Result=0) then
|
|
begin
|
|
if (timeout=nil) then
|
|
begin
|
|
Result:=umtxq_sleep(uq, 'ucond', 0);
|
|
end else
|
|
begin
|
|
if ((wflags and CVWAIT_ABSTIME)=0) then
|
|
begin
|
|
ets:=0;
|
|
kern_clock_gettime_unit(clockid,@ets);
|
|
|
|
tts:=TIMESPEC_TO_UNIT(timeout);
|
|
ets:=ets+tts;
|
|
end else
|
|
begin
|
|
ets:=TIMESPEC_TO_UNIT(timeout);
|
|
|
|
cts:=0;
|
|
kern_clock_gettime_unit(clockid,@cts);
|
|
|
|
tts:=ets-cts;
|
|
end;
|
|
|
|
repeat
|
|
Result:=umtxq_sleep(uq, 'ucond', tvtohz(tts));
|
|
if (Result<>ETIMEDOUT) then Break;
|
|
|
|
cts:=0;
|
|
kern_clock_gettime_unit(clockid,@cts);
|
|
|
|
if (cts>=ets) then
|
|
begin
|
|
Result:=ETIMEDOUT;
|
|
Break;
|
|
end;
|
|
|
|
tts:=ets-cts;
|
|
until false;
|
|
end;
|
|
end;
|
|
|
|
if ((uq^.uq_flags and UQF_UMTXQ)=0) then
|
|
begin
|
|
Result:=0;
|
|
end else
|
|
begin
|
|
umtxq_busy(@uq^.uq_key);
|
|
|
|
if ((uq^.uq_flags and UQF_UMTXQ)<>0) then
|
|
begin
|
|
oldlen:=uq^.uq_cur_queue^.length;
|
|
umtxq_remove(uq);
|
|
|
|
if (oldlen=1) then
|
|
begin
|
|
umtxq_unlock(@uq^.uq_key);
|
|
suword32(cv^.c_has_waiters,0);
|
|
umtxq_lock(@uq^.uq_key);
|
|
end;
|
|
end;
|
|
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
|
|
if (Result=ERESTART) then
|
|
begin
|
|
Result:=EINTR;
|
|
end;
|
|
end;
|
|
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
end;
|
|
|
|
function do_cv_signal(td:p_kthread;cv:p_ucond):Integer;
|
|
var
|
|
key:umtx_key;
|
|
count,nwake,flags:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_cv_signal(',HexStr(cv),')');
|
|
|
|
flags:=fuword32(cv^.c_flags);
|
|
|
|
if (flags=-1) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Result:=umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock(@key);
|
|
umtxq_busy(@key);
|
|
|
|
count:=umtxq_count (@key);
|
|
nwake:=umtxq_signal(@key, 1);
|
|
|
|
if (count<=nwake) then
|
|
begin
|
|
umtxq_unlock(@key);
|
|
Result:=suword32(cv^.c_has_waiters,0);
|
|
umtxq_lock(@key);
|
|
if (Result<>0) then Result:=EFAULT;
|
|
end;
|
|
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
end;
|
|
|
|
function do_cv_broadcast(td:p_kthread;cv:p_ucond):Integer;
|
|
var
|
|
key:umtx_key;
|
|
flags:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_cv_broadcast(',HexStr(cv),')');
|
|
|
|
flags:=fuword32(cv^.c_flags);
|
|
|
|
if (flags=-1) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Result:=umtx_key_get(cv, TYPE_CV, GET_SHARE(flags), @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock (@key);
|
|
umtxq_busy (@key);
|
|
umtxq_signal(@key, High(Integer));
|
|
umtxq_unlock(@key);
|
|
|
|
Result:=suword32(cv^.c_has_waiters,0);
|
|
if (Result<>0) then Result:=EFAULT;
|
|
|
|
umtxq_lock (@key);
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
end;
|
|
|
|
////
|
|
|
|
function do_rw_rdlock(td:p_kthread;rwlock:p_urwlock;fflag:QWORD;timo:Int64):Integer;
|
|
label
|
|
_sleep;
|
|
var
|
|
uq:p_umtx_q;
|
|
flags,wrflags:DWORD;
|
|
state,oldstate:Integer;
|
|
blocked_readers:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
uq:=td^.td_umtxq;
|
|
|
|
flags:=fuword32(rwlock^.rw_flags);
|
|
|
|
if (flags=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Result:=umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
wrflags:=URWLOCK_WRITE_OWNER;
|
|
|
|
if ((fflag and URWLOCK_PREFER_READER)=0) and ((flags and URWLOCK_PREFER_READER)=0) then
|
|
begin
|
|
wrflags:=wrflags or URWLOCK_WRITE_WAITERS;
|
|
end;
|
|
|
|
repeat
|
|
state:=fuword32(rwlock^.rw_state);
|
|
|
|
While ((state and wrflags)=0) do
|
|
begin
|
|
if (URWLOCK_READER_COUNT(state)=URWLOCK_MAX_READERS) then
|
|
begin
|
|
umtx_key_release(@uq^.uq_key);
|
|
Exit(EAGAIN);
|
|
end;
|
|
|
|
oldstate:=casuword32(rwlock^.rw_state,state,state+1);
|
|
|
|
if (oldstate=-1) then
|
|
begin
|
|
umtx_key_release(@uq^.uq_key);
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (oldstate=state) then
|
|
begin
|
|
umtx_key_release(@uq^.uq_key);
|
|
Exit(0);
|
|
end;
|
|
|
|
state:=oldstate;
|
|
end;
|
|
|
|
if (Result<>0) then Break;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_busy (@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
state:=fuword32(rwlock^.rw_state);
|
|
|
|
While ((state and wrflags)<>0) and ((state and URWLOCK_READ_WAITERS)=0) do
|
|
begin
|
|
oldstate:=casuword32(rwlock^.rw_state,state,state or URWLOCK_READ_WAITERS);
|
|
|
|
if (oldstate=-1) then
|
|
begin
|
|
Result:=EFAULT;
|
|
Break;
|
|
end;
|
|
|
|
if (oldstate=state) then
|
|
begin
|
|
goto _sleep;
|
|
end;
|
|
|
|
state:=oldstate;
|
|
end;
|
|
|
|
if (Result<>0) then
|
|
begin
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
Break;
|
|
end;
|
|
|
|
if ((state and wrflags)=0) then
|
|
begin
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
Continue;
|
|
end;
|
|
|
|
_sleep:
|
|
|
|
blocked_readers:=fuword32(rwlock^.rw_blocked_readers);
|
|
suword32(rwlock^.rw_blocked_readers,blocked_readers+1);
|
|
|
|
While ((state and wrflags)<>0) do
|
|
begin
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_insert(uq);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
|
|
Result:= umtxq_sleep(uq, 'urdlck', timo);
|
|
|
|
umtxq_busy (@uq^.uq_key);
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
if (Result<>0) then Break;
|
|
|
|
state:=fuword32(rwlock^.rw_state);
|
|
end;
|
|
|
|
blocked_readers:=fuword32(rwlock^.rw_blocked_readers);
|
|
suword32(rwlock^.rw_blocked_readers,blocked_readers-1);
|
|
|
|
if (blocked_readers=1) then
|
|
begin
|
|
state:=fuword32(rwlock^.rw_state);
|
|
repeat
|
|
oldstate:=casuword32(rwlock^.rw_state,state,state and (not URWLOCK_READ_WAITERS));
|
|
|
|
if (oldstate=-1) then
|
|
begin
|
|
Result:=EFAULT;
|
|
Break;
|
|
end;
|
|
|
|
if (oldstate=state) then
|
|
begin
|
|
Break;
|
|
end;
|
|
|
|
state:=oldstate;
|
|
until false;
|
|
end;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
until (Result<>0);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
end;
|
|
|
|
function do_rw_rdlock2(td:p_kthread;rwlock:p_urwlock;fflag:QWORD;timeout:p_timespec):Integer;
|
|
var
|
|
ts,ts2,tv:Int64;
|
|
begin
|
|
Result:=0;
|
|
|
|
ts:=get_unit_uptime;
|
|
ts:=ts+TIMESPEC_TO_UNIT(timeout);
|
|
tv:=ts;
|
|
|
|
repeat
|
|
Result:=do_rw_rdlock(td,rwlock,fflag,tvtohz(tv));
|
|
|
|
if (Result<>ETIMEDOUT) then Break;
|
|
|
|
ts2:=get_unit_uptime;
|
|
|
|
if (ts2>=ts) then
|
|
begin
|
|
Result:=ETIMEDOUT;
|
|
Break;
|
|
end;
|
|
|
|
tv:=ts-ts2;
|
|
until false;
|
|
|
|
if (Result=ERESTART) then
|
|
begin
|
|
Result:=EINTR;
|
|
end;
|
|
end;
|
|
|
|
function do_rw_wrlock(td:p_kthread;rwlock:p_urwlock;fflag:QWORD;timo:Int64):Integer;
|
|
label
|
|
_sleep;
|
|
var
|
|
uq:p_umtx_q;
|
|
flags:Integer;
|
|
state,oldstate:Integer;
|
|
blocked_writers:Integer;
|
|
blocked_readers:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
uq:=td^.td_umtxq;
|
|
|
|
flags:=fuword32(rwlock^.rw_flags);
|
|
|
|
if (flags=-1) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Result:=umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
blocked_readers:=0;
|
|
|
|
repeat
|
|
state:=fuword32(rwlock^.rw_state);
|
|
|
|
while ((state and URWLOCK_WRITE_OWNER)=0) and (URWLOCK_READER_COUNT(state)=0) do
|
|
begin
|
|
oldstate:=casuword32(rwlock^.rw_state,state,state or URWLOCK_WRITE_OWNER);
|
|
|
|
if (oldstate=-1) then
|
|
begin
|
|
umtx_key_release(@uq^.uq_key);
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
if (oldstate=state) then
|
|
begin
|
|
umtx_key_release(@uq^.uq_key);
|
|
Exit(0);
|
|
end;
|
|
|
|
state:=oldstate;
|
|
end;
|
|
|
|
if (Result<>0) then
|
|
begin
|
|
if ((state and (URWLOCK_WRITE_OWNER or URWLOCK_WRITE_WAITERS))=0) and
|
|
(blocked_readers<>0) then
|
|
begin
|
|
umtxq_lock(@uq^.uq_key);
|
|
umtxq_busy(@uq^.uq_key);
|
|
umtxq_signal_queue(@uq^.uq_key, High(Integer), UMTX_SHARED_QUEUE);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
end;
|
|
Break;
|
|
end;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_busy (@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
state:=fuword32(rwlock^.rw_state);
|
|
|
|
while (((state and URWLOCK_WRITE_OWNER)<>0) or (URWLOCK_READER_COUNT(state)<>0)) and
|
|
((state and URWLOCK_WRITE_WAITERS)=0) do
|
|
begin
|
|
oldstate:=casuword32(rwlock^.rw_state,state,state or URWLOCK_WRITE_WAITERS);
|
|
|
|
if (oldstate=-1) then
|
|
begin
|
|
Result:=EFAULT;
|
|
Break;
|
|
end;
|
|
|
|
if (oldstate=state) then
|
|
begin
|
|
goto _sleep;
|
|
end;
|
|
|
|
state:=oldstate;
|
|
end;
|
|
|
|
if (Result<>0) then
|
|
begin
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
Break;
|
|
end;
|
|
|
|
if ((state and URWLOCK_WRITE_OWNER)=0) and (URWLOCK_READER_COUNT(state)=0) then
|
|
begin
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
Continue;
|
|
end;
|
|
|
|
_sleep:
|
|
|
|
blocked_writers:=fuword32(rwlock^.rw_blocked_writers);
|
|
suword32(rwlock^.rw_blocked_writers,blocked_writers+1);
|
|
|
|
While ((state and URWLOCK_WRITE_OWNER) or URWLOCK_READER_COUNT(state)<>0) do
|
|
begin
|
|
umtxq_lock(@uq^.uq_key);
|
|
umtxq_insert_queue(uq, UMTX_EXCLUSIVE_QUEUE);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
|
|
Result:=umtxq_sleep(uq, 'uwrlck', timo);
|
|
|
|
umtxq_busy(@uq^.uq_key);
|
|
umtxq_remove_queue(uq, UMTX_EXCLUSIVE_QUEUE);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
if (Result<>0) then Break;
|
|
|
|
state:=fuword32(rwlock^.rw_state);
|
|
end;
|
|
|
|
blocked_writers:=fuword32(rwlock^.rw_blocked_writers);
|
|
suword32(rwlock^.rw_blocked_writers,blocked_writers-1);
|
|
|
|
if (blocked_writers=1) then
|
|
begin
|
|
state:=fuword32(rwlock^.rw_state);
|
|
|
|
repeat
|
|
oldstate:=casuword32(rwlock^.rw_state,state,state and (not URWLOCK_WRITE_WAITERS));
|
|
|
|
if (oldstate=-1) then
|
|
begin
|
|
Result:=EFAULT;
|
|
Break;
|
|
end;
|
|
|
|
if (oldstate=state) then
|
|
begin
|
|
Break;
|
|
end;
|
|
|
|
state:=oldstate;
|
|
until (Result<>0);
|
|
|
|
blocked_readers:=fuword32(rwlock^.rw_blocked_readers);
|
|
end else
|
|
begin
|
|
blocked_readers:=0;
|
|
end;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
until false;
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
end;
|
|
|
|
function do_rw_wrlock2(td:p_kthread;rwlock:p_urwlock;fflag:QWORD;timeout:p_timespec):Integer;
|
|
var
|
|
ts,ts2,tv:Int64;
|
|
begin
|
|
Result:=0;
|
|
|
|
ts:=get_unit_uptime;
|
|
ts:=ts+TIMESPEC_TO_UNIT(timeout);
|
|
tv:=ts;
|
|
|
|
repeat
|
|
Result:=do_rw_wrlock(td,rwlock,fflag,tvtohz(tv));
|
|
|
|
if (Result<>ETIMEDOUT) then Break;
|
|
|
|
ts2:=get_unit_uptime;
|
|
|
|
if (ts2>=ts) then
|
|
begin
|
|
Result:=ETIMEDOUT;
|
|
Break;
|
|
end;
|
|
|
|
tv:=ts-ts2;
|
|
until false;
|
|
|
|
if (Result=ERESTART) then
|
|
begin
|
|
Result:=EINTR;
|
|
end;
|
|
end;
|
|
|
|
function do_rw_unlock(td:p_kthread;rwlock:p_urwlock):Integer;
|
|
label
|
|
_out;
|
|
var
|
|
uq:p_umtx_q;
|
|
flags:DWORD;
|
|
state,oldstate:Integer;
|
|
q,count:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
uq:=td^.td_umtxq;
|
|
|
|
flags:=fuword32(rwlock^.rw_flags);
|
|
|
|
if (flags=DWORD(-1)) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Result:=umtx_key_get(rwlock, TYPE_RWLOCK, GET_SHARE(flags), @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
state:=fuword32(rwlock^.rw_state);
|
|
|
|
if ((state and URWLOCK_WRITE_OWNER)<>0) then
|
|
begin
|
|
repeat
|
|
oldstate:=casuword32(rwlock^.rw_state,state,state and (not URWLOCK_WRITE_OWNER));
|
|
|
|
if (oldstate=-1) then
|
|
begin
|
|
Result:=EFAULT;
|
|
goto _out;
|
|
end;
|
|
|
|
if (oldstate<>state) then
|
|
begin
|
|
state:=oldstate;
|
|
if ((oldstate and URWLOCK_WRITE_OWNER)=0) then
|
|
begin
|
|
Result:=EPERM;
|
|
goto _out;
|
|
end;
|
|
end else
|
|
begin
|
|
Break;
|
|
end;
|
|
|
|
until false;
|
|
end else
|
|
if (URWLOCK_READER_COUNT(state)<>0) then
|
|
begin
|
|
repeat
|
|
oldstate:=casuword32(rwlock^.rw_state,state,state-1);
|
|
|
|
if (oldstate=-1) then
|
|
begin
|
|
Result:=EFAULT;
|
|
goto _out;
|
|
end;
|
|
|
|
if (oldstate<>state) then
|
|
begin
|
|
state:=oldstate;
|
|
if (URWLOCK_READER_COUNT(oldstate)=0) then
|
|
begin
|
|
Result:=EPERM;
|
|
goto _out;
|
|
end;
|
|
end else
|
|
begin
|
|
Break;
|
|
end;
|
|
|
|
until false;
|
|
end else
|
|
begin
|
|
Result:=EPERM;
|
|
goto _out;
|
|
end;
|
|
|
|
count:=0;
|
|
|
|
if ((flags and URWLOCK_PREFER_READER)=0) then
|
|
begin
|
|
if (state and URWLOCK_WRITE_WAITERS)<>0 then
|
|
begin
|
|
count:=1;
|
|
q:=UMTX_EXCLUSIVE_QUEUE;
|
|
end else
|
|
if (state and URWLOCK_READ_WAITERS)<>0 then
|
|
begin
|
|
count:=High(Integer);
|
|
q:=UMTX_SHARED_QUEUE;
|
|
end;
|
|
end else
|
|
begin
|
|
if ((state and URWLOCK_READ_WAITERS)<>0) then
|
|
begin
|
|
count:=High(Integer);
|
|
q:=UMTX_SHARED_QUEUE;
|
|
end else
|
|
if ((state and URWLOCK_WRITE_WAITERS)<>0) then
|
|
begin
|
|
count:=1;
|
|
q:=UMTX_EXCLUSIVE_QUEUE;
|
|
end;
|
|
end;
|
|
|
|
if (count<>0) then
|
|
begin
|
|
umtxq_lock(@uq^.uq_key);
|
|
umtxq_busy(@uq^.uq_key);
|
|
umtxq_signal_queue(@uq^.uq_key, count, q);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
end;
|
|
|
|
_out:
|
|
umtx_key_release(@uq^.uq_key);
|
|
end;
|
|
|
|
////
|
|
|
|
function do_sem_wait(td:p_kthread;sem:p__usem;timeout:p_timespec):Integer;
|
|
label
|
|
_fault;
|
|
var
|
|
uq:p_umtx_q;
|
|
flags:Integer;
|
|
has_waiters,count:DWORD;
|
|
cts,ets,tv:Int64;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_sem_wait(',HexStr(sem),')');
|
|
|
|
uq:=td^.td_umtxq;
|
|
|
|
flags:=fuword32(sem^._flags);
|
|
|
|
if (flags=-1) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Result:=umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), @uq^.uq_key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_busy (@uq^.uq_key);
|
|
umtxq_insert(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
has_waiters:=fuword32(sem^._has_waiters);
|
|
|
|
if (has_waiters=DWORD(-1)) then
|
|
begin
|
|
_fault:
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
has_waiters:=casuword32(sem^._has_waiters,0,1);
|
|
|
|
if (has_waiters=DWORD(-1)) then
|
|
begin
|
|
goto _fault;
|
|
end;
|
|
|
|
count:=fuword32(sem^._count);
|
|
|
|
if (count=DWORD(-1)) then
|
|
begin
|
|
goto _fault;
|
|
end;
|
|
|
|
umtxq_lock (@uq^.uq_key);
|
|
umtxq_unbusy(@uq^.uq_key);
|
|
|
|
if (count<>0) then
|
|
begin
|
|
umtxq_remove(uq);
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
Exit(0);
|
|
end;
|
|
|
|
if (timeout=nil) then
|
|
begin
|
|
Result:=umtxq_sleep(uq, 'usem', 0);
|
|
end else
|
|
begin
|
|
ets:=get_unit_uptime;
|
|
tv:=TIMESPEC_TO_UNIT(timeout);
|
|
ets:=ets+tv;
|
|
|
|
repeat
|
|
Result:=umtxq_sleep(uq, 'usem', tvtohz(tv));
|
|
if (Result<>ETIMEDOUT) then Break;
|
|
|
|
cts:=get_unit_uptime;
|
|
|
|
if (cts>=ets) then
|
|
begin
|
|
Result:=ETIMEDOUT;
|
|
Break;
|
|
end;
|
|
|
|
tv:=ets-cts;
|
|
until false;
|
|
end;
|
|
|
|
if ((uq^.uq_flags and UQF_UMTXQ)=0) then
|
|
begin
|
|
Result:=0;
|
|
end else
|
|
begin
|
|
umtxq_remove(uq);
|
|
|
|
if (Result=ERESTART) then
|
|
begin
|
|
Result:=EINTR;
|
|
end;
|
|
end;
|
|
|
|
umtxq_unlock(@uq^.uq_key);
|
|
|
|
umtx_key_release(@uq^.uq_key);
|
|
end;
|
|
|
|
function do_sem_wake(td:p_kthread;sem:p__usem):Integer;
|
|
var
|
|
key:umtx_key;
|
|
flags:Integer;
|
|
count:Integer;
|
|
begin
|
|
Result:=0;
|
|
|
|
//Writeln('do_sem_wake(',HexStr(sem),')');
|
|
|
|
flags:=fuword32(sem^._flags);
|
|
|
|
if (flags=-1) then
|
|
begin
|
|
Exit(EFAULT);
|
|
end;
|
|
|
|
Result:=umtx_key_get(sem, TYPE_SEM, GET_SHARE(flags), @key);
|
|
if (Result<>0) then Exit;
|
|
|
|
umtxq_lock(@key);
|
|
umtxq_busy(@key);
|
|
count:=umtxq_count(@key);
|
|
|
|
if (count>0) then
|
|
begin
|
|
if (count=1) then
|
|
begin
|
|
umtxq_unlock(@key);
|
|
Result:=suword32(sem^._has_waiters,0);
|
|
if (Result<>0) then Result:=EFAULT;
|
|
umtxq_lock(@key);
|
|
end;
|
|
umtxq_signal(@key, 1);
|
|
end;
|
|
|
|
umtxq_unbusy(@key);
|
|
umtxq_unlock(@key);
|
|
|
|
umtx_key_release(@key);
|
|
end;
|
|
|
|
////
|
|
|
|
function umtx_copyin_timeout(addr:Pointer;tsp:p_timespec):Integer; public;
|
|
begin
|
|
Result:=copyin(addr,tsp,SizeOf(timespec));
|
|
if (Result=0) then
|
|
begin
|
|
if (tsp^.tv_sec < 0) or
|
|
(tsp^.tv_nsec>=1000000000) or
|
|
(tsp^.tv_nsec< 0) then
|
|
begin
|
|
Result:=EINVAL;
|
|
end;
|
|
end else
|
|
begin
|
|
Result:=EFAULT;
|
|
end;
|
|
end;
|
|
|
|
function __umtx_op_lock_umtx(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
ts:p_timespec;
|
|
timeout:timespec;
|
|
begin
|
|
ts:=nil;
|
|
if (uaddr2<>nil) then
|
|
begin
|
|
Result:=umtx_copyin_timeout(uaddr2,@timeout);
|
|
if (Result<>0) then Exit;
|
|
ts:=@timeout;
|
|
end;
|
|
Result:=do_lock_umtx(td,obj,val,ts);
|
|
end;
|
|
|
|
function __umtx_op_unlock_umtx(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_unlock_umtx(td,obj,val);
|
|
end;
|
|
|
|
function __umtx_op_wait(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
ts:p_timespec;
|
|
timeout:timespec;
|
|
begin
|
|
ts:=nil;
|
|
if (uaddr2<>nil) then
|
|
begin
|
|
Result:=umtx_copyin_timeout(uaddr2,@timeout);
|
|
if (Result<>0) then Exit;
|
|
ts:=@timeout;
|
|
end;
|
|
Result:=do_wait(td,obj,val,ts,0,0);
|
|
end;
|
|
|
|
function __umtx_op_wait_uint(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
ts:p_timespec;
|
|
timeout:timespec;
|
|
begin
|
|
ts:=nil;
|
|
if (uaddr2<>nil) then
|
|
begin
|
|
Result:=umtx_copyin_timeout(uaddr2,@timeout);
|
|
if (Result<>0) then Exit;
|
|
ts:=@timeout;
|
|
end;
|
|
Result:=do_wait(td,obj,val,ts,1,0);
|
|
end;
|
|
|
|
function __umtx_op_wait_uint_private(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
ts:p_timespec;
|
|
timeout:timespec;
|
|
begin
|
|
ts:=nil;
|
|
if (uaddr2<>nil) then
|
|
begin
|
|
Result:=umtx_copyin_timeout(uaddr2,@timeout);
|
|
if (Result<>0) then Exit;
|
|
ts:=@timeout;
|
|
end;
|
|
Result:=do_wait(td,obj,val,ts,1,1);
|
|
end;
|
|
|
|
function __umtx_op_wake(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=kern_umtx_wake(td,obj,val,0);
|
|
end;
|
|
|
|
function __umtx_op_nwake_private(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
const
|
|
BATCH_SIZE=128;
|
|
var
|
|
count:Integer;
|
|
tocopy:Integer;
|
|
i,pos:Integer;
|
|
uaddrs:array[0..BATCH_SIZE-1] of Pointer;
|
|
upp:PPByte;
|
|
begin
|
|
Result:=0;
|
|
count :=val;
|
|
upp :=obj;
|
|
i :=0;
|
|
pos :=0;
|
|
|
|
While (count>0) do
|
|
begin
|
|
tocopy:=count;
|
|
if (tocopy>BATCH_SIZE) then
|
|
begin
|
|
tocopy:=BATCH_SIZE;
|
|
end;
|
|
|
|
Result:=copyin(upp+pos,@uaddrs,tocopy);
|
|
if (Result<>0) then Exit;
|
|
|
|
For i:=0 to tocopy-1 do
|
|
begin
|
|
kern_umtx_wake(td,uaddrs[i],High(Integer),1);
|
|
end;
|
|
|
|
count:=count-tocopy;
|
|
pos :=pos +tocopy;
|
|
end;
|
|
end;
|
|
|
|
function __umtx_op_wake_private(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=kern_umtx_wake(td,obj,val,1);
|
|
end;
|
|
|
|
function __umtx_op_lock_umutex(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
ts:p_timespec;
|
|
timeout:timespec;
|
|
begin
|
|
ts:=nil;
|
|
if (uaddr2<>nil) then
|
|
begin
|
|
Result:=umtx_copyin_timeout(uaddr2,@timeout);
|
|
if (Result<>0) then Exit;
|
|
ts:=@timeout;
|
|
end;
|
|
Result:=do_lock_umutex(td,obj,ts,0);
|
|
end;
|
|
|
|
function __umtx_op_trylock_umutex(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_lock_umutex(td,obj,nil,_UMUTEX_TRY);
|
|
end;
|
|
|
|
function __umtx_op_wait_umutex(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
ts:p_timespec;
|
|
timeout:timespec;
|
|
begin
|
|
ts:=nil;
|
|
if (uaddr2<>nil) then
|
|
begin
|
|
Result:=umtx_copyin_timeout(uaddr2,@timeout);
|
|
if (Result<>0) then Exit;
|
|
ts:=@timeout;
|
|
end;
|
|
Result:=do_lock_umutex(td,obj,ts,_UMUTEX_WAIT);
|
|
end;
|
|
|
|
function __umtx_op_wake_umutex(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_wake_umutex(td,obj);
|
|
end;
|
|
|
|
function __umtx_op_unlock_umutex(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_unlock_umutex(td,obj);
|
|
end;
|
|
|
|
function __umtx_op_set_ceiling(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_set_ceiling(td,obj,val,uaddr1);
|
|
end;
|
|
|
|
function __umtx_op_cv_wait(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
ts:p_timespec;
|
|
timeout:timespec;
|
|
begin
|
|
ts:=nil;
|
|
if (uaddr2<>nil) then
|
|
begin
|
|
Result:=umtx_copyin_timeout(uaddr2,@timeout);
|
|
if (Result<>0) then Exit;
|
|
ts:=@timeout;
|
|
end;
|
|
Result:=do_cv_wait(td,obj,uaddr1,ts,val);
|
|
end;
|
|
|
|
function __umtx_op_cv_signal(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_cv_signal(td,obj);
|
|
end;
|
|
|
|
function __umtx_op_cv_broadcast(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_cv_broadcast(td,obj);
|
|
end;
|
|
|
|
function __umtx_op_rw_rdlock(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
timeout:timespec;
|
|
begin
|
|
|
|
if (uaddr2=nil) then
|
|
begin
|
|
Result:=do_rw_rdlock(td,obj,val,0);
|
|
end else
|
|
begin
|
|
Result:=umtx_copyin_timeout(uaddr2,@timeout);
|
|
if (Result<>0) then Exit;
|
|
Result:=do_rw_rdlock2(td,obj,val,@timeout);
|
|
end;
|
|
end;
|
|
|
|
function __umtx_op_rw_wrlock(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
timeout:timespec;
|
|
begin
|
|
|
|
if (uaddr2=nil) then
|
|
begin
|
|
Result:=do_rw_wrlock(td,obj,val,0);
|
|
end else
|
|
begin
|
|
Result:=umtx_copyin_timeout(uaddr2,@timeout);
|
|
if (Result<>0) then Exit;
|
|
Result:=do_rw_wrlock2(td,obj,val,@timeout);
|
|
end;
|
|
end;
|
|
|
|
function __umtx_op_rw_unlock(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_rw_unlock(td,obj);
|
|
end;
|
|
|
|
function sys__umtx_lock(mtx:Pointer):Integer;
|
|
var
|
|
td:p_kthread;
|
|
begin
|
|
if (mtx=nil) then Exit(EINVAL);
|
|
td:=curkthread;
|
|
if (td=nil) then Exit(EFAULT);
|
|
Result:=_do_lock_umtx(td,mtx,td^.td_tid,0);
|
|
end;
|
|
|
|
function sys__umtx_unlock(mtx:Pointer):Integer;
|
|
var
|
|
td:p_kthread;
|
|
begin
|
|
if (mtx=nil) then Exit(EINVAL);
|
|
td:=curkthread;
|
|
if (td=nil) then Exit(EFAULT);
|
|
Result:=do_unlock_umtx(td,mtx,td^.td_tid);
|
|
end;
|
|
|
|
function __umtx_op_sem_wait(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
ts:p_timespec;
|
|
timeout:timespec;
|
|
begin
|
|
|
|
ts:=nil;
|
|
if (uaddr2<>nil) then
|
|
begin
|
|
Result:=umtx_copyin_timeout(uaddr2,@timeout);
|
|
if (Result<>0) then Exit;
|
|
ts:=@timeout;
|
|
end;
|
|
Result:=do_sem_wait(td,obj,ts);
|
|
end;
|
|
|
|
function __umtx_op_sem_wake(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_sem_wake(td,obj)
|
|
end;
|
|
|
|
function __umtx_op_sem_signalto(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_cv_signalto(td,obj,val);
|
|
end;
|
|
|
|
function __umtx_op_wake3_umutex(td:p_kthread;obj:Pointer;val:QWORD;uaddr1,uaddr2:Pointer):Integer; inline;
|
|
begin
|
|
Result:=do_wake3_umutex(td,obj,val);
|
|
end;
|
|
|
|
function sys__umtx_op(obj:Pointer;op:Integer;val:QWORD;uaddr1,uaddr2:Pointer):Integer;
|
|
var
|
|
td:p_kthread;
|
|
begin
|
|
if (obj=nil) then Exit(EINVAL);
|
|
td:=curkthread;
|
|
if (td=nil) then Exit(EFAULT);
|
|
|
|
Case op of
|
|
UMTX_OP_LOCK :Result:=__umtx_op_lock_umtx (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_UNLOCK :Result:=__umtx_op_unlock_umtx (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_WAIT :Result:=__umtx_op_wait (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_WAKE :Result:=__umtx_op_wake (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_MUTEX_TRYLOCK :Result:=__umtx_op_trylock_umutex (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_MUTEX_LOCK :Result:=__umtx_op_lock_umutex (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_MUTEX_UNLOCK :Result:=__umtx_op_unlock_umutex (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_SET_CEILING :Result:=__umtx_op_set_ceiling (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_CV_WAIT :Result:=__umtx_op_cv_wait (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_CV_SIGNAL :Result:=__umtx_op_cv_signal (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_CV_BROADCAST :Result:=__umtx_op_cv_broadcast (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_WAIT_UINT :Result:=__umtx_op_wait_uint (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_RW_RDLOCK :Result:=__umtx_op_rw_rdlock (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_RW_WRLOCK :Result:=__umtx_op_rw_wrlock (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_RW_UNLOCK :Result:=__umtx_op_rw_unlock (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_WAIT_UINT_PRIVATE:Result:=__umtx_op_wait_uint_private(td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_WAKE_PRIVATE :Result:=__umtx_op_wake_private (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_MUTEX_WAIT :Result:=__umtx_op_wait_umutex (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_MUTEX_WAKE :Result:=__umtx_op_wake_umutex (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_SEM_WAIT :Result:=__umtx_op_sem_wait (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_SEM_WAKE :Result:=__umtx_op_sem_wake (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_NWAKE_PRIVATE :Result:=__umtx_op_nwake_private (td,obj,val,uaddr1,uaddr2);
|
|
UMTX_OP_SEM_SIGNALTO :Result:=__umtx_op_sem_signalto (td,obj,val,uaddr1,uaddr2); //Sony extension
|
|
UMTX_OP_MUTEX_WAKE3 :Result:=__umtx_op_wake3_umutex (td,obj,val,uaddr1,uaddr2); //Sony extension
|
|
else
|
|
Exit(EINVAL);
|
|
end;
|
|
|
|
//Writeln('umtx_op(',HexStr(obj),',',op,',',val,',',HexStr(uaddr1),',',HexStr(uaddr2),')');
|
|
end;
|
|
|
|
procedure _umutex_init(mtx:p_umutex); inline;
|
|
begin
|
|
mtx^:=Default(umutex);
|
|
end;
|
|
|
|
|
|
end.
|
|
|