diff --git a/sys/kern/kern_patcher.pas b/sys/kern/kern_patcher.pas index de17cccb..c7473e74 100644 --- a/sys/kern/kern_patcher.pas +++ b/sys/kern/kern_patcher.pas @@ -5,23 +5,499 @@ unit kern_patcher; interface -uses - vm; - -procedure patcher_process_section(_obj:Pointer;data,vaddr:Pointer;filesz:QWORD); +procedure patcher_process_section(_obj,data,vaddr:Pointer;filesz:QWORD); implementation uses - vm_object; + windows, -procedure patcher_process_section(_obj:Pointer;data,vaddr:Pointer;filesz:QWORD); + mqueue, + hamt, + kern_rwlock, + kern_thr, + vm, + vm_map, + vm_mmap, + vm_object, + vm_pmap, + trap, + kern_stub; + +{ +64 48 A1 [0000000000000000] mov rax,fs:[$0000000000000000] -> 65 48 A1 [0807000000000000] mov rax,gs:[$0000000000000708] +64 48 8B 04 25 [00000000] mov rax,fs:[$00000000] -> 65 48 8B 04 25 [08070000] mov rax,gs:[$00000708] +64 48 8B 0C 25 [00000000] mov rcx,fs:[$00000000] -> 65 48 8B 0C 25 [08070000] mov rcx,gs:[$00000708] +64 48 8B 14 25 [00000000] mov rdx,fs:[$00000000] -> 65 48 8B 14 25 [08070000] mov rdx,gs:[$00000708] +64 48 8B 1C 25 [00000000] mov rbx,fs:[$00000000] -> 65 48 8B 1C 25 [08070000] mov rbx,gs:[$00000708] +64 48 8B 24 25 [00000000] mov rsp,fs:[$00000000] -> 65 48 8B 24 25 [08070000] mov rsp,gs:[$00000708] +64 48 8B 2C 25 [00000000] mov rbp,fs:[$00000000] -> 65 48 8B 2C 25 [08070000] mov rbp,gs:[$00000708] +64 48 8B 34 25 [00000000] mov rsi,fs:[$00000000] -> 65 48 8B 34 25 [08070000] mov rsi,gs:[$00000708] +64 48 8B 3C 25 [00000000] mov rdi,fs:[$00000000] -> 65 48 8B 3C 25 [08070000] mov rdi,gs:[$00000708] +64 4C 8B 04 25 [00000000] mov r8 ,fs:[$00000000] -> 65 4C 8B 04 25 [08070000] mov r8 ,gs:[$00000708] +64 4C 8B 0C 25 [00000000] mov r9 ,fs:[$00000000] -> 65 4C 8B 0C 25 [08070000] mov r9 ,gs:[$00000708] +64 4C 8B 14 25 [00000000] mov r10,fs:[$00000000] -> 65 4C 8B 14 25 [08070000] mov r10,gs:[$00000708] +64 4C 8B 1C 25 [00000000] mov r11,fs:[$00000000] -> 65 4C 8B 1C 25 [08070000] mov r11,gs:[$00000708] +64 4C 8B 24 25 [00000000] mov r12,fs:[$00000000] -> 65 4C 8B 24 25 [08070000] mov r12,gs:[$00000708] +64 4C 8B 2C 25 [00000000] mov r13,fs:[$00000000] -> 65 4C 8B 2C 25 [08070000] mov r13,gs:[$00000708] +64 4C 8B 34 25 [00000000] mov r14,fs:[$00000000] -> 65 4C 8B 34 25 [08070000] mov r14,gs:[$00000708] +64 4C 8B 3C 25 [00000000] mov r15,fs:[$00000000] -> 65 4C 8B 3C 25 [08070000] mov r15,gs:[$00000708] +} + +{ +[48 b8 | 10 01 00 00 00 00 00 00]:MOV RAX,0x110 + +[49 89 ca]:MOV R10,RCX + +[0f 05]:SYSCALL + +[72 01]:JC LAB + +[c3]:RET + +/////////////////// + +[48 c7 c0 | dd 01 00 00]:MOV RAX,0x1dd + +[49 89 ca]:MOV R10,RCX + +[0f 05]:SYSCALL + +[72 01]:JC LAB + +[c3]:RET + +////////// + +48 c7 c0 01 | 00 00 00 MOV RAX,0x1 +49 89 ca MOV R10,RCX +0f 05 SYSCALL +41 5c POP R12 +5b POP RBX + +[00 00 00 | 49 89 ca | 0f 05 | 41 5c] + +////////// + +48 c7 c0 fb | 00 00 00 MOV RAX,0xfb +49 89 ca MOV R10,RCX +0f 05 SYSCALL +72 1f JC LAB_ +83 fa 00 CMP EDX,0x0 +75 04 JNZ LAB_ +41 5c POP R12 + +[00 00 00 | 49 89 ca | 0f 05 | 72 1f] + +////////// + +48 c7 c0 01 | 00 00 00 MOV RAX,0x1 +49 89 ca MOV R10,RCX +0f 05 SYSCALL +41 5c POP R12 +5b POP RBX + +////////// + +48 c7 c0 3b | 00 00 00 MOV RAX,0x3b +49 89 ca MOV R10,RCX +0f 05 SYSCALL +48 8d 15 c0 LEA RDX,[FUN_] +ff ff ff +ff e2 JMP RDX=>FUN_ + +[00 00 00 | 49 89 ca | 0f 05 | 48 8d] + +////////// + +48 c7 c0 a5 | 01 00 00 MOV RAX,0x1a5 +49 89 ca MOV R10,RCX +0f 05 SYSCALL +72 06 JC LAB_ +48 83 c4 08 ADD RSP,0x8 +ff e6 JMP RSI + +[01 00 00 | 49 89 ca | 0f 05 | 72 06] + +////////// + +48 c7 c0 2a | 00 00 00 MOV RAX,0x2a +49 89 ca MOV R10,RCX +0f 05 SYSCALL +72 0d JC LAB_ +89 07 MOV dword ptr + +[00 00 00 | 49 89 ca | 0f 05 | 72 0d] + +////////// + +48 c7 c0 37 | 00 00 00 MOV RAX,0x37 +49 89 ca MOV R10,RCX +0f 05 SYSCALL +72 02 JC LAB_ +48 cf IRETQ + +[00 00 00 | 49 89 ca | 0f 05 | 72 02] + +////////// + +48 c7 c0 32 | 00 00 00 MOV RAX,0x32 +49 89 ca MOV R10,RCX +0f 05 SYSCALL +72 0e JC LAB_ +48 8d 15 a3 e1 08 00 LEA RDX,[DAT_] + +[00 00 00 | 49 89 ca | 0f 05 | 72 0e] + +////////// + +48 c7 c0 42 | 00 00 00 MOV RAX,0x42 +49 89 ca MOV R10,RCX +0f 05 SYSCALL +72 02 JC LAB_ +ff e6 JMP RSI + +[00 00 00 | 49 89 ca | 0f 05 | 72 02] + +////////// + +48 c7 c0 54 | 01 00 00 MOV RAX,0x154 +49 89 ca MOV R10,RCX +0f 05 SYSCALL +5f POP RDI +48 81 bf 18 CMP qword ptr [RDI + 0x118],0x20001 +01 00 00 01 +00 02 00 + +[01 00 00 | 49 89 ca | 0f 05 | 5f 48] + +////////// + +48 c7 c0 c6 | 01 00 00 MOV RAX,0x1c6 +49 89 ca MOV R10,RCX +0f 05 SYSCALL +c3 RET +90 ?? 90h + +[01 00 00 | 49 89 ca | 0f 05 | c3 90] + +////////// + + 0 1 2 3 4 5 6 7 8 9 +[0X 00 00 | 49 89 ca | 0f 05 | 41 5c] +[0X 00 00 | 49 89 ca | 0f 05 | 72 01] +[0X 00 00 | 49 89 ca | 0f 05 | 72 1f] +[0X 00 00 | 49 89 ca | 0f 05 | 48 8d] +[0X 00 00 | 49 89 ca | 0f 05 | 72 06] +[0X 00 00 | 49 89 ca | 0f 05 | 72 0d] +[0X 00 00 | 49 89 ca | 0f 05 | 72 02] +[0X 00 00 | 49 89 ca | 0f 05 | 72 0e] +[0X 00 00 | 49 89 ca | 0f 05 | 5f 48] +[0X 00 00 | 49 89 ca | 0f 05 | c3 90] + X=0,1,2 +} + +{ + jmpq 0(%rip) -> [FF 25 | 00 00 00 00] + callq 0(%rip) -> [FF 15 | 00 00 00 00] + + jmpl [32] -> [E9 | 00 00 00 00] + calll [32] -> [E8 | 00 00 00 00] +} + +type + t_patch_base_long=packed record + len :Byte ; //12 + inst:array[0..2] of Byte; + addr:QWORD; //teb_tcb/teb_gsbase + end; + + t_patch_base_short=packed record + len :Byte ; //9 + inst:array[0..4] of Byte; + addr:DWORD; //teb_tcb/teb_gsbase + end; + + t_patch_inst=packed record + case Byte of + 0:(A:t_patch_base_long); + 1:(B:t_patch_base_short); + 2:(C:array[0..11] of Byte); + end; + +const + patch_table:array[0..33] of t_patch_inst=( + (B:(len: 9;inst:($65,$48,$8B,$04,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$48,$8B,$0C,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$48,$8B,$14,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$48,$8B,$1C,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$48,$8B,$24,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$48,$8B,$2C,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$48,$8B,$34,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$48,$8B,$3C,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$4C,$8B,$04,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$4C,$8B,$0C,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$4C,$8B,$14,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$4C,$8B,$1C,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$4C,$8B,$24,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$4C,$8B,$2C,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$4C,$8B,$34,$25);addr:teb_tcb )), + (B:(len: 9;inst:($65,$4C,$8B,$3C,$25);addr:teb_tcb )), + (A:(len:12;inst:($65,$48,$A1 );addr:teb_tcb )), + // + (B:(len: 9;inst:($65,$48,$8B,$04,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$48,$8B,$0C,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$48,$8B,$14,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$48,$8B,$1C,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$48,$8B,$24,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$48,$8B,$2C,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$48,$8B,$34,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$48,$8B,$3C,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$4C,$8B,$04,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$4C,$8B,$0C,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$4C,$8B,$14,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$4C,$8B,$1C,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$4C,$8B,$24,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$4C,$8B,$2C,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$4C,$8B,$34,$25);addr:teb_gsbase)), + (B:(len: 9;inst:($65,$4C,$8B,$3C,$25);addr:teb_gsbase)), + (A:(len:12;inst:($65,$48,$A1 );addr:teb_gsbase)) + ); + +function IndexInstr(var pbuf:Pointer;pend:Pointer):Integer; +var + psrc:Pointer; + W:DWORD; +begin + Result:=-1; + psrc:=pbuf; + while (psrc$25) or + (PDWORD(@PBYTE(psrc)[5])^<>$00000000) then + begin + Inc(psrc,4); + Continue; + end; + end; + 16, + 33: + begin + if (PQWORD(@PBYTE(psrc)[3])^<>$0000000000000000) then + begin + Inc(psrc,4); + Continue; + end; + end; + 34: + begin + if (PDWORD(@PBYTE(psrc)[4])^<>$050FCA89) then + begin + Inc(psrc,4); + Continue; + end; + + Write(HexStr(PBYTE(psrc)[0],2),' '); + Write(HexStr(PBYTE(psrc)[1],2),' '); + Write(HexStr(PBYTE(psrc)[2],2),' '); + Write(HexStr(PBYTE(psrc)[3],2),' '); + Write(HexStr(PBYTE(psrc)[4],2),' '); + Write(HexStr(PBYTE(psrc)[5],2),' '); + Write(HexStr(PBYTE(psrc)[6],2),' '); + Write(HexStr(PBYTE(psrc)[7],2),' '); + Write(HexStr(PBYTE(psrc)[8],2),' '); + Write(HexStr(PBYTE(psrc)[9],2),' '); + + Case PBYTE(psrc)[8] of + $41, + $48, + $5f, + $72, + $c3:Writeln('True'); + else + begin + Writeln('False'); + Inc(psrc,4); + Continue; + end; + end; + end; + else; + end; + + if (Result<>-1) then + begin + pbuf:=psrc; + exit; + end; + + Inc(psrc); + end; + Result:=-1; +end; + +procedure patcher_process_section(_obj,data,vaddr:Pointer;filesz:QWORD); var obj:vm_object_t; + addr:Pointer; + pend:Pointer; + i,len:Integer; + + fs_count:Integer; + gs_count:Integer; + sv_count:Integer; + + procedure do_patch_base(P:PByte;i:Integer); inline; + begin + Move(patch_table[i].C[1],P^,patch_table[i].C[0]); + end; + begin Assert(_obj<>nil,'patcher_process_section'); obj:=_obj; + fs_count:=0; + gs_count:=0; + sv_count:=0; + + addr:=data; + pend:=addr+filesz; + repeat + i:=IndexInstr(addr,pend); + + if (i=-1) then Break; + + Case i of + 0..16: + begin + Inc(fs_count); + // + len:=patch_table[i].C[0]; + do_patch_base(addr,i); + end; + 17..33: + begin + Inc(gs_count); + // + len:=patch_table[i].C[0]; + do_patch_base(addr,i); + end; + 34: + begin + Inc(sv_count); + // + len:=9; + end + else + Assert(False); + end; + + Inc(addr,len); + + until false; + + Writeln('[patcher_vaddr]:0x',HexStr(vaddr)); + Writeln(' fs_count:',fs_count); + Writeln(' gs_count:',gs_count); + Writeln(' sv_count:',sv_count); + + Writeln; + + //readln; end; diff --git a/sys/kern/kern_rtld.pas b/sys/kern/kern_rtld.pas index 20cda95f..88141cff 100644 --- a/sys/kern/kern_rtld.pas +++ b/sys/kern/kern_rtld.pas @@ -223,6 +223,7 @@ uses vmparam, vm_map, vm_mmap, + vm_pmap, kern_patcher; function maxInt64(a,b:Int64):Int64; inline; @@ -1088,7 +1089,7 @@ begin //remove prev if exist vm_map_delete(map,vaddr_lo,vaddr_hi); - Result:=vm_map_insert(map,imgp^.obj,0,vaddr_lo,vaddr_hi,VM_PROT_RW,prot or VM_PROT_RW,0); + Result:=vm_map_insert(map,imgp^.obj,offset,vaddr_lo,vaddr_hi,VM_PROT_RW,prot or VM_PROT_RW,0); if (Result<>0) then begin vm_map_unlock(map); @@ -1102,20 +1103,21 @@ begin memsz:=vaddr_hi-vaddr_lo; cache:=ReAllocMem(cache,memsz); FillChar(cache^,memsz,0); - Move(Pointer(vaddr_lo)^,cache^,filesz); + Move(base^,cache^,filesz); if ((prot and VM_PROT_EXECUTE)<>0) then begin + Writeln('P_X:',HexStr(offset,8),' ',HexStr(memsz,8),' ',HexStr(filesz,8)); patcher_process_section(imgp^.obj,cache,Pointer(vaddr_lo),filesz); end; - Result:=copyout(base,cache,filesz); + Result:=copyout(Pointer(vaddr_lo),cache,memsz); if (Result<>0) then begin vm_map_unlock(map); // Writeln(StdErr,'[KERNEL] self_load_section: copyout failed ', - id,', ',HexStr(base),'->',HexStr(vaddr,8),':',HexStr(filesz,8)); + id,', ',HexStr(base),'->',HexStr(vaddr_lo,8),':',HexStr(memsz,8)); readln; Exit; end; @@ -1129,6 +1131,11 @@ begin Exit(vm_mmap_to_errno(Result)); end; + if ((prot and VM_PROT_EXECUTE)<>0) then + begin + md_cacheflush(Pointer(vaddr_lo),memsz,ICACHE); + end; + vm_map_unlock(map); end; diff --git a/sys/kern/kern_stub.pas b/sys/kern/kern_stub.pas new file mode 100644 index 00000000..a842a705 --- /dev/null +++ b/sys/kern/kern_stub.pas @@ -0,0 +1,246 @@ +unit kern_stub; + +{$mode ObjFPC}{$H+} +{$CALLING SysV_ABI_CDecl} + +interface + +uses + mqueue; + +const + m_header_alloc=Integer($C3C3C3C3); + m_header_free =Integer($C3C3C390); + + segment_size =64*1024; + +type + p_stub_chunk=^stub_chunk; + stub_chunk=packed record + head :Integer; + prev_size:Integer; + curr_size:Integer; + refs :Integer; + link :TAILQ_ENTRY; + body :record end; + end; + +function p_alloc (vaddr:Pointer;size:Integer):p_stub_chunk; +procedure p_free (chunk:p_stub_chunk); +procedure p_inc_ref(chunk:p_stub_chunk); +procedure p_dec_ref(chunk:p_stub_chunk); + +implementation + +uses + hamt, + kern_rwlock, + vm, + vm_map, + vm_mmap, + vm_object; + +var + chunk_alloc:TSTUB_HAMT64; + chunk_free :TAILQ_HEAD=(tqh_first:nil;tqh_last:@chunk_free .tqh_first); + + chunk_lock :Pointer=nil; + +function alloc_segment(start:Pointer):p_stub_chunk; +var + map:vm_map_t; + err:Integer; +begin + Result:=nil; + map:=@g_vmspace.vm_map; + + err:=_vm_mmap(map, + @start, + segment_size, + VM_PROT_RWX, + VM_PROT_RWX, + MAP_ANON or MAP_PRIVATE or (16 shl MAP_ALIGNMENT_BIT), + OBJT_DEFAULT, + nil, + 0); + + if (err<>0) then Exit; + + Result:=start; + + Result^.head :=m_header_free; + Result^.prev_size :=0; + Result^.curr_size :=segment_size; + Result^.refs :=0; + Result^.link.tqe_next:=nil; + Result^.link.tqe_prev:=nil; +end; + +procedure free_segment(chunk:p_stub_chunk); +var + map:vm_map_t; +begin + map:=@g_vmspace.vm_map; + + vm_map_lock (map); + vm_map_delete(map, qword(chunk), qword(chunk) + segment_size); + vm_map_unlock(map); +end; + +procedure split_chunk(chunk:p_stub_chunk;used_size:Integer); +var + chunk_size:Integer; + next:p_stub_chunk; +begin + chunk_size:=chunk^.curr_size; + + if ((used_size+SizeOf(stub_chunk)*2)>chunk_size) then Exit; + + used_size:=used_size+SizeOf(stub_chunk); + + next:=Pointer(chunk)+used_size; + + chunk^.curr_size:=used_size; + + next^.head :=m_header_free; + next^.prev_size :=used_size; + next^.curr_size :=chunk_size-used_size; + next^.refs :=0; + next^.link.tqe_next:=nil; + next^.link.tqe_prev:=nil; + + TAILQ_INSERT_TAIL(@chunk_free,next,@next^.link); +end; + +procedure merge_chunk(var chunk:p_stub_chunk); +var + prev,next:p_stub_chunk; +begin + + if (chunk^.prev_size<>0) then + begin + prev:=Pointer(chunk)-chunk^.prev_size; + if (prev^.head=m_header_free) then + begin + Assert(prev^.curr_size=chunk^.prev_size,'invalid prev chunk curr_size'); + Assert(prev^.refs=0 ,'invalid prev chunk refs'); + + prev^.curr_size:=prev^.curr_size+chunk^.curr_size; + chunk:=prev; + end; + end; + + if (chunk^.curr_sizenil) do + begin + next:=TAILQ_NEXT(entry,@entry^.link); + // + if (entry^.curr_size>=(size+SizeOf(stub_chunk))) then + begin + delta:=abs(Int64(vaddr)-Int64(@entry^.body)); + if (delta(segment_size-SizeOf(stub_chunk)),'p_alloc to big'); + + rw_wlock(chunk_lock); + + chunk:=find_free_chunk(vaddr,size); + + if (chunk=nil) then + begin + chunk:=alloc_segment(vaddr); + Assert(chunk<>nil,'p_alloc NOMEM'); + end; + + split_chunk(chunk,size); + + chunk^.head:=m_header_alloc; + HAMT_insert64(@chunk_alloc,QWORD(chunk),chunk); + + rw_wunlock(chunk_lock); + + Result:=chunk; +end; + +procedure p_free(chunk:p_stub_chunk); +begin + if (chunk=nil) then Exit; + + rw_wlock(chunk_lock); + + if (HAMT_search64(@chunk_alloc,QWORD(chunk))=nil) then + begin + rw_wunlock(chunk_lock); + Exit; + end; + + HAMT_delete64(@chunk_alloc,QWORD(chunk),nil); + chunk^.head:=m_header_free; + + merge_chunk(chunk); + + if (chunk^.curr_size>=segment_size) then + begin + free_segment(chunk); + end else + begin + TAILQ_INSERT_TAIL(@chunk_free,chunk,@chunk^.link); + end; + + rw_wunlock(chunk_lock); +end; + +procedure p_inc_ref(chunk:p_stub_chunk); +begin + if (chunk=nil) then Exit; + + System.InterlockedIncrement(chunk^.refs); +end; + +procedure p_dec_ref(chunk:p_stub_chunk); +begin + if (chunk=nil) then Exit; + + if (System.InterlockedDecrement(chunk^.refs)=0) then + begin + p_free(chunk); + end; +end; + +end. + diff --git a/sys/kern/kern_thread.pas b/sys/kern/kern_thread.pas index dce015d6..a85bcdd7 100644 --- a/sys/kern/kern_thread.pas +++ b/sys/kern/kern_thread.pas @@ -62,26 +62,6 @@ procedure kthread_exit(); implementation -{ -64 48 A1 [0000000000000000] mov rax,fs:[$0000000000000000] -> 65 48 A1 [0807000000000000] mov rax,gs:[$0000000000000708] -64 48 8B 04 25 [00000000] mov rax,fs:[$00000000] -> 65 48 8B 04 25 [08070000] mov rax,gs:[$00000708] -64 48 8B 0C 25 [00000000] mov rcx,fs:[$00000000] -> 65 48 8B 0C 25 [08070000] mov rcx,gs:[$00000708] -64 48 8B 14 25 [00000000] mov rdx,fs:[$00000000] -> 65 48 8B 14 25 [08070000] mov rdx,gs:[$00000708] -64 48 8B 1C 25 [00000000] mov rbx,fs:[$00000000] -> 65 48 8B 1C 25 [08070000] mov rbx,gs:[$00000708] -64 48 8B 24 25 [00000000] mov rsp,fs:[$00000000] -> 65 48 8B 24 25 [08070000] mov rsp,gs:[$00000708] -64 48 8B 2C 25 [00000000] mov rbp,fs:[$00000000] -> 65 48 8B 2C 25 [08070000] mov rbp,gs:[$00000708] -64 48 8B 34 25 [00000000] mov rsi,fs:[$00000000] -> 65 48 8B 34 25 [08070000] mov rsi,gs:[$00000708] -64 48 8B 3C 25 [00000000] mov rdi,fs:[$00000000] -> 65 48 8B 3C 25 [08070000] mov rdi,gs:[$00000708] -64 4C 8B 04 25 [00000000] mov r8 ,fs:[$00000000] -> 65 4C 8B 04 25 [08070000] mov r8 ,gs:[$00000708] -64 4C 8B 0C 25 [00000000] mov r9 ,fs:[$00000000] -> 65 4C 8B 0C 25 [08070000] mov r9 ,gs:[$00000708] -64 4C 8B 14 25 [00000000] mov r10,fs:[$00000000] -> 65 4C 8B 14 25 [08070000] mov r10,gs:[$00000708] -64 4C 8B 1C 25 [00000000] mov r11,fs:[$00000000] -> 65 4C 8B 1C 25 [08070000] mov r11,gs:[$00000708] -64 4C 8B 24 25 [00000000] mov r12,fs:[$00000000] -> 65 4C 8B 24 25 [08070000] mov r12,gs:[$00000708] -64 4C 8B 2C 25 [00000000] mov r13,fs:[$00000000] -> 65 4C 8B 2C 25 [08070000] mov r13,gs:[$00000708] -64 4C 8B 34 25 [00000000] mov r14,fs:[$00000000] -> 65 4C 8B 34 25 [08070000] mov r14,gs:[$00000708] -64 4C 8B 3C 25 [00000000] mov r15,fs:[$00000000] -> 65 4C 8B 3C 25 [08070000] mov r15,gs:[$00000708] -} - uses errno, systm, diff --git a/sys/md/vm_pmap.pas b/sys/md/vm_pmap.pas index c5e4a176..330578c4 100644 --- a/sys/md/vm_pmap.pas +++ b/sys/md/vm_pmap.pas @@ -54,6 +54,13 @@ procedure pmap_remove(pmap :pmap_t; __end:vm_offset_t; prot :vm_prot_t); +const + ICACHE=1; //Flush the instruction cache. + DCACHE=2; //Write back to memory and invalidate the affected valid cache lines. + BCACHE=ICACHE or DCACHE; + +procedure md_cacheflush(addr:Pointer;nbytes,cache:Integer); + implementation function atop(x:QWORD):DWORD; inline; @@ -387,6 +394,18 @@ begin end; end; +procedure md_cacheflush(addr:Pointer;nbytes,cache:Integer); +begin + if ((cache and ICACHE)<>0) then + begin + FlushInstructionCache(NtCurrentProcess,addr,nbytes); + end; + if ((cache and DCACHE)<>0) then + begin + FlushViewOfFile(addr,nbytes); + end; +end; + end. diff --git a/sys/test/project1.lpi b/sys/test/project1.lpi index d4a3ee20..49e81b43 100644 --- a/sys/test/project1.lpi +++ b/sys/test/project1.lpi @@ -593,6 +593,10 @@ + + + + diff --git a/sys/vfs/vfs_subr.pas b/sys/vfs/vfs_subr.pas index 3e4093b9..78c8d6b8 100644 --- a/sys/vfs/vfs_subr.pas +++ b/sys/vfs/vfs_subr.pas @@ -107,6 +107,8 @@ procedure insmntque_stddtr(vp:p_vnode;dtr_arg:Pointer); function insmntque1(vp:p_vnode;mp:p_mount;dtr:t_insmntque1_dtr;dtr_arg:Pointer):Integer; function insmntque(vp:p_vnode;mp:p_mount):Integer; +function vinvalbuf(vp:p_vnode;flags,slpflag,slptimeo:Integer):Integer; + function __mnt_vnode_next_all(mvp:pp_vnode;mp:p_mount):p_vnode; function __mnt_vnode_first_all(mvp:pp_vnode;mp:p_mount):p_vnode; procedure __mnt_vnode_markerfree_all(mvp:pp_vnode;mp:p_mount); @@ -725,7 +727,7 @@ begin end; { - * Exitthe next vnode from the free list. + * Return the next vnode from the free list. } function getnewvnode(tag:PChar;mp:p_mount;vops:p_vop_vector;vpp:pp_vnode):Integer; label @@ -868,7 +870,9 @@ begin { XXX non mp-safe fs may still call insmntque with vnode unlocked } if (VOP_ISLOCKED(vp)=0) then + begin vn_lock(vp, LK_EXCLUSIVE or LK_RETRY); + end; vgone(vp); vput(vp); end; @@ -895,8 +899,8 @@ begin MNT_ILOCK(mp); VI_LOCK(vp); if ((mp^.mnt_kern_flag and MNTK_NOINSMNTQ)<>0) and - (((mp^.mnt_kern_flag and MNTK_UNMOUNTF)<>0) or - (mp^.mnt_nvnodelistsize=0)) then + (((mp^.mnt_kern_flag and MNTK_UNMOUNTF)<>0) or + (mp^.mnt_nvnodelistsize=0)) then begin locked:=VOP_ISLOCKED(vp); if (locked=0) or @@ -906,7 +910,9 @@ begin VI_UNLOCK(vp); MNT_IUNLOCK(mp); if (dtr<>nil) then + begin dtr(vp, dtr_arg); + end; Exit(EBUSY); end; end; @@ -1941,7 +1947,9 @@ begin Assert((flags and LK_TYPE_MASK)<>0,'vget: invalid lock operation'); if ((flags and LK_INTERLOCK)=0) then + begin VI_LOCK(vp); + end; vholdl(vp); @@ -1951,8 +1959,10 @@ begin vdrop(vp); Exit(error); end; + if ((vp^.v_iflag and VI_DOOMED)<>0) and ((flags and LK_RETRY)=0) then Assert(false,'vget: vn_lock failed to Exit ENOENT'); + VI_LOCK(vp); { Upgrade our holdcnt to a usecount. } v_upgrade_usecount(vp); @@ -1966,7 +1976,10 @@ begin begin if (VOP_ISLOCKED(vp)=LK_EXCLUSIVE) and ((flags and LK_NOWAIT)=0) then + begin vinactive(vp); + end; + vp^.v_iflag:=vp^.v_iflag and (not VI_OWEINACT); end; VI_UNLOCK(vp); @@ -3169,10 +3182,12 @@ procedure vop_rename_fail(ap:p_vop_rename_args); begin if (ap^.a_tvp<>nil) then vput(ap^.a_tvp); + if (ap^.a_tdvp=ap^.a_tvp) then vrele(ap^.a_tdvp) else vput(ap^.a_tdvp); + vrele(ap^.a_fdvp); vrele(ap^.a_fvp); end; @@ -3181,9 +3196,12 @@ procedure vop_rename_pre(ap:p_vop_rename_args); begin if (ap^.a_tdvp<>ap^.a_fdvp) then vhold(ap^.a_fdvp); + if (ap^.a_tvp<>ap^.a_fvp) then vhold(ap^.a_fvp); + vhold(ap^.a_tdvp); + if (ap^.a_tvp<>nil) then vhold(ap^.a_tvp); end; diff --git a/sys/vm/vm_map.pas b/sys/vm/vm_map.pas index 29512271..56f6325c 100644 --- a/sys/vm/vm_map.pas +++ b/sys/vm/vm_map.pas @@ -1598,7 +1598,9 @@ begin MADV_CORE: begin if (start=__end) then + begin Exit(KERN_SUCCESS); + end; modify_map:=1; vm_map_lock(map); end; @@ -1607,7 +1609,9 @@ begin MADV_FREE: begin if (start=__end) then + begin Exit(KERN_SUCCESS); + end; vm_map_lock(map); end; else @@ -1622,7 +1626,9 @@ begin if (vm_map_lookup_entry(map,start,@entry)) then begin if (modify_map<>0) then + begin vm_map_clip_start(map, entry, start); + end; end else begin entry:=entry^.next; diff --git a/sys/vm/vm_mmap.pas b/sys/vm/vm_mmap.pas index b0b19699..bbd4d3e3 100644 --- a/sys/vm/vm_mmap.pas +++ b/sys/vm/vm_mmap.pas @@ -549,20 +549,27 @@ var begin size:=len; if (size=0) then + begin Exit(EINVAL); + end; pageoff:=(vm_size_t(addr) and PAGE_MASK); addr:=addr-pageoff; size:=size+pageoff; size:=round_page(size); + if (addr + size < addr) then + begin Exit(EINVAL); + end; { * Check for illegal addresses. Watch out for address wrap... } if (qword(addr) < vm_map_min(@g_vmspace.vm_map)) or (qword(addr) + size > vm_map_max(@g_vmspace.vm_map)) then + begin Exit(EINVAL); + end; vm_map_lock(@g_vmspace.vm_map); @@ -585,8 +592,11 @@ begin addr:=addr-pageoff; size:=size+pageoff; size:=round_page(size); + if (addr + size < addr) then + begin Exit(EINVAL); + end; case (vm_map_protect(@g_vmspace.vm_map, QWORD(addr), QWORD(addr) + size, prot, FALSE)) of KERN_SUCCESS :Exit(0); @@ -621,9 +631,14 @@ begin } if (vm_offset_t(addr) < vm_map_min(@g_vmspace.vm_map)) or (vm_offset_t(addr) + len > vm_map_max(@g_vmspace.vm_map)) then + begin Exit(EINVAL); + end; + if (vm_offset_t(addr) + len) < vm_offset_t(addr) then + begin Exit(EINVAL); + end; { * Since this routine is only advisory, we default to conservative @@ -633,7 +648,9 @@ begin __end:=round_page(vm_offset_t(addr) + len); if (vm_map_madvise(@g_vmspace.vm_map, start, __end, behav))<>0 then + begin Exit(EINVAL); + end; Exit(0); end; @@ -645,9 +662,14 @@ var begin if (vm_offset_t(addr) < vm_map_min(@g_vmspace.vm_map)) or (vm_offset_t(addr) + len > vm_map_max(@g_vmspace.vm_map)) then + begin Exit(EINVAL); + end; + if (vm_offset_t(addr) + len) < vm_offset_t(addr) then + begin Exit(EINVAL); + end; Result:=copyinstr(name,@_name,32,nil); if (Result<>0) then Exit; diff --git a/sys/vm/vm_object.pas b/sys/vm/vm_object.pas index 080a7f5b..fc48393c 100644 --- a/sys/vm/vm_object.pas +++ b/sys/vm/vm_object.pas @@ -61,14 +61,18 @@ const OBJPR_NOTMAPPED=$2; // Don't unmap pages. OBJPR_NOTWIRED =$4; // Don't remove wired pages. -procedure vm_object_reference (obj:vm_object_t); -function vm_object_allocate (t:objtype_t;size:vm_pindex_t):vm_object_t; -procedure vm_object_deallocate(obj:vm_object_t); -procedure vm_object_clear_flag(obj:vm_object_t;bits:Word); +procedure vm_object_reference (obj:vm_object_t); +function vm_object_allocate (t:objtype_t;size:vm_pindex_t):vm_object_t; +procedure vm_object_deallocate (obj:vm_object_t); + +procedure vm_object_set_flag (obj:vm_object_t;bits:Word); +procedure vm_object_clear_flag (obj:vm_object_t;bits:Word); procedure vm_object_pip_add (obj:vm_object_t;i:word); procedure vm_object_pip_subtract(obj:vm_object_t;i:word); procedure vm_object_pip_wakeup (obj:vm_object_t); +procedure vm_object_pip_wakeupn (obj:vm_object_t;i:word); +procedure vm_object_pip_wait (obj:vm_object_t;waitid:pchar); function vm_object_page_clean(obj:vm_object_t; start,__end:vm_ooffset_t; @@ -79,8 +83,9 @@ procedure vm_object_page_remove(obj:vm_object_t; __end:vm_pindex_t; options:Integer); -procedure vm_object_collapse(obj:vm_object_t); +procedure vm_object_collapse (obj:vm_object_t); +function VM_OBJECT_MTX (obj:vm_object_t):p_mtx; procedure VM_OBJECT_LOCK (obj:vm_object_t); function VM_OBJECT_TRYLOCK (obj:vm_object_t):Boolean; procedure VM_OBJECT_UNLOCK (obj:vm_object_t); @@ -99,7 +104,9 @@ function vm_object_coalesce(prev_object:vm_object_t; implementation uses - vmparam; + vmparam, + vnode, + vfs_subr; function IDX_TO_OFF(x:DWORD):QWORD; inline; begin @@ -111,6 +118,11 @@ begin Result:=QWORD(x) shr PAGE_SHIFT; end; +function VM_OBJECT_MTX(obj:vm_object_t):p_mtx; +begin + Result:=@obj^.mtx; +end; + procedure VM_OBJECT_LOCK(obj:vm_object_t); begin mtx_lock(obj^.mtx); @@ -136,6 +148,12 @@ begin Assert(mtx_owned(obj^.mtx)); end; +procedure vm_object_set_flag(obj:vm_object_t;bits:Word); +begin + VM_OBJECT_LOCK_ASSERT(obj); + obj^.flags:=obj^.flags or bits; +end; + procedure vm_object_clear_flag(obj:vm_object_t;bits:Word); begin VM_OBJECT_LOCK_ASSERT(obj); @@ -154,9 +172,14 @@ begin Result^.size :=size; Result^.generation:=1; Result^.ref_count :=1; + + if (t=OBJT_DEFAULT) then + begin + Result^.flags:=OBJ_ONEMAPPING; + end; end; -procedure _vm_object_deallocate(obj:vm_object_t); +procedure vm_object_destroy(obj:vm_object_t); begin Case obj^.otype of OBJT_DEFAULT:; @@ -175,19 +198,77 @@ begin System.InterlockedIncrement(obj^.ref_count); end; +{ + vm_object_terminate actually destroys the specified object, freeing + up all previously used resources. + + The object must be locked. + This routine may block. +} +procedure vm_object_terminate(obj:vm_object_t); +var + vp:p_vnode; +begin + VM_OBJECT_LOCK_ASSERT(obj); + + { + * Make sure no one uses us. + } + vm_object_set_flag(obj, OBJ_DEAD); + + { + * wait for the pageout daemon to be done with the obj + } + vm_object_pip_wait(obj, 'objtrm'); + + Assert(obj^.paging_in_progress=0,'vm_object_terminate: pageout in progress'); + + { + * Clean and free the pages, as appropriate. All references to the + * obj are gone, so we don't need to lock it. + } + if (obj^.otype=OBJT_VNODE) then + begin + vp:=obj^.handle; + + { + * Clean pages and flush buffers. + } + vm_object_page_clean(obj, 0, 0, OBJPC_SYNC); + VM_OBJECT_UNLOCK(obj); + + vinvalbuf(vp, V_SAVE, 0, 0); + + VM_OBJECT_LOCK(obj); + end; + + Assert(obj^.ref_count=0,'vm_object_terminate: obj with references'); + + VM_OBJECT_UNLOCK(obj); + + vm_object_destroy(obj); +end; + procedure vm_object_deallocate(obj:vm_object_t); +var + ref:Integer; begin if (obj=nil) then Exit; - if (System.InterlockedDecrement(obj^.ref_count)=0) then - begin - _vm_object_deallocate(obj); - end; -end; + ref:=System.InterlockedDecrement(obj^.ref_count); -procedure vm_object_destroy(obj:vm_object_t); -begin - vm_object_deallocate(obj); + if (ref=1) then + begin + VM_OBJECT_LOCK(obj); + vm_object_set_flag(obj, OBJ_ONEMAPPING); + VM_OBJECT_UNLOCK(obj); + end else + if (ref=0) then + if ((obj^.flags and OBJ_DEAD)=0) then + begin + VM_OBJECT_LOCK(obj); + vm_object_terminate(obj); + end; end; procedure vm_object_pip_add(obj:vm_object_t;i:word); @@ -219,6 +300,31 @@ begin end; end; +procedure vm_object_pip_wakeupn(obj:vm_object_t;i:word); +begin + if (obj=nil) then Exit; + + VM_OBJECT_LOCK_ASSERT(obj); + Dec(obj^.paging_in_progress,i); + if ((obj^.flags and OBJ_PIPWNT)<>0) and (obj^.paging_in_progress=0) then + begin + vm_object_clear_flag(obj, OBJ_PIPWNT); + wakeup(obj); + end; +end; + +procedure vm_object_pip_wait(obj:vm_object_t;waitid:pchar); +begin + if (obj=nil) then Exit; + + VM_OBJECT_LOCK_ASSERT(obj); + while (obj^.paging_in_progress<>0) do + begin + obj^.flags:=obj^.flags or OBJ_PIPWNT; + msleep(obj, VM_OBJECT_MTX(obj), PVM, waitid, 0); + end; +end; + { vm_object_page_clean @@ -277,7 +383,9 @@ procedure vm_object_page_remove(obj:vm_object_t; __end:vm_pindex_t; options:Integer); begin - + //OBJPR_CLEANONLY=$1; // Don't remove dirty pages. + //OBJPR_NOTMAPPED=$2; // Don't unmap pages. + //OBJPR_NOTWIRED =$4; // Don't remove wired pages. end; { @@ -348,15 +456,6 @@ begin Exit(FALSE); end; - { - * Remove any pages that may still be in the object from a previous - * deallocation. - } - if (next_pindex