unit kern_jit_ctx;

{$mode ObjFPC}{$H+}
{$CALLING SysV_ABI_CDecl}

interface

uses
 g_node_splay,
 x86_fpdbgdisas,
 x86_jit;

const
 LF_JMP=1;

type
 t_point_type=(fpCall,fpData,fpInvalid);

 t_ctx_modes=Set of (cmDontScanRipRel,cmDontScanSwitchTable,cmInternal);

 p_jit_context2=^t_jit_context2;
 t_jit_context2=object
  type
   p_forward_link=^t_forward_link;
   t_forward_link=object
    next       :p_forward_link;
    instruction:t_jit_i_link;
   end;

   t_forward_links=object
    root :p_forward_link;
    ptype:t_point_type;
   end;

   p_forward_point=^t_forward_point;
   t_forward_point=object
    pLeft :p_forward_point;
    pRight:p_forward_point;
    dst   :Pointer;
    links :t_forward_links;
    function c(n1,n2:p_forward_point):Integer; static;
   end;
   t_forward_set=specialize TNodeSplay<t_forward_point>;

   p_switchtable_point=^t_switchtable_point;
   t_switchtable_point=object
    pLeft :p_switchtable_point;
    pRight:p_switchtable_point;
    //
    table:PInteger;
    curr :PInteger;
    //
    function c(n1,n2:p_switchtable_point):Integer; static;
   end;
   t_switchtable_set=specialize TNodeSplay<t_switchtable_point>;

   p_jumpslot=^t_jumpslot;
   t_jumpslot=object
    pLeft :p_jumpslot;
    pRight:p_jumpslot;
    //
    addr:Pointer;
    //
    function c(n1,n2:p_jumpslot):Integer; static;
   end;
   t_jumpslot_set=specialize TNodeSplay<t_jumpslot>;

   p_label=^t_label;
   t_label=object
    pLeft    :p_label;
    pRight   :p_label;
    curr     :Pointer;
    next     :Pointer;
    link_curr:t_jit_i_link;
    link_next:t_jit_i_link;
    flags    :Integer;
    function c(n1,n2:p_label):Integer; static;
   end;
   t_label_set=specialize TNodeSplay<t_label>;

   p_entry_point=^t_entry_point;
   t_entry_point=object
    pLeft      :p_entry_point;
    pRight     :p_entry_point;
    //
    next       :p_entry_point;
    //
    src        :Pointer;
    instruction:t_jit_i_link;
    //
    function c(n1,n2:p_entry_point):Integer; static;
   end;
   t_entry_point_set=specialize TNodeSplay<t_entry_point>;

   p_export_point=^t_export_point;
   t_export_point=object
    next  :p_export_point;
    nid   :QWORD;
    native:Pointer;
    dst   :PPointer;
   end;

   p_import_point=^t_import_point;
   t_import_point=object
    next :p_import_point;
    guest:PPointer;
    dst  :PPointer;
   end;

  var
   forward_link_cache :p_forward_link;
   forward_point_cache:p_forward_point;
   //
   forward_set:t_forward_set;
   //
   switchtable_set:t_switchtable_set;
   min_switchtable:p_switchtable_point;
   //
   jumpslot_set:t_jumpslot_set;
   //
   label_set  :t_label_set;
   entry_list :p_entry_point;
   entry_set  :t_entry_point_set;

   export_list:p_export_point;
   import_list:p_import_point;

   obj:Pointer;

   text_start:QWORD;
   text___end:QWORD;
   map____end:QWORD;

   max_reloc :QWORD;

   modes:t_ctx_modes;

   Code    :Pointer;
   ptr_curr:Pointer;
   ptr_next:Pointer;

   label_flags:Integer;
   imm:Word;
   trim:Boolean;

   dis:TX86Disassembler;
   din:TInstruction;

   builder:t_jit_builder;

  function  is_text_addr(addr:QWORD):Boolean;
  function  is_map_addr (addr:QWORD):Boolean;
  procedure add_export_point (nid:QWORD;native:Pointer;dst:PPointer);
  procedure add_import_point (guest,dst:PPointer);
  procedure add_forward_link (node:p_forward_point;instruction:t_jit_i_link);
  procedure Resolve_forwards (var links:t_forward_links;target:t_jit_i_link);
  function  add_forward_point(ptype:t_point_type;instruction:t_jit_i_link;dst:Pointer):p_forward_point;
  function  add_forward_point(ptype:t_point_type;dst:Pointer):p_forward_point;
  function  add_switchtable  (table:Pointer):p_switchtable_point;
  procedure add_jumpslot     (addr:Pointer);
  function  is_jumpslot      (addr:Pointer):Boolean;
  function  fetch_switchtable(var out_table:p_switchtable_point;var out_next:PInteger):Boolean;
  Function  new_chunk(ptype:t_point_type;start:Pointer):p_jit_code_chunk;
  procedure mark_chunk(ptype:t_point_type);
  function  get_chunk_ptype():t_point_type;
  procedure end_chunk(__end:Pointer);
  function  max_forward_point():Pointer;
  function  fetch_forward_point(var links:t_forward_links;var dst:Pointer):Boolean;
  function  add_label(curr,next:Pointer;link_curr,link_next:t_jit_i_link;flags:Integer):p_label;
  function  get_label(src:Pointer):p_label;
  function  get_link (src:Pointer):t_jit_i_link;
  procedure add_entry_point(src:Pointer;label_id:t_jit_i_link);
  procedure Free;
 end;

const
 r_thrd:TRegValue=(AType:regGeneral;ASize:os64;AIndex:13); //r13
 r_tmp0:TRegValue=(AType:regGeneral;ASize:os64;AIndex:14); //r14
 r_tmp1:TRegValue=(AType:regGeneral;ASize:os64;AIndex:15); //r15

 OPERAND_BYTES:array[TOperandSize] of Word=(0,1,2,4,8,6,10,16,32,64,512);

 SCODES:array[TSimdOpcode] of Byte=(0,0,1,3,2);
 MCODES:array[0..3] of PChar=('_','0x0F','0x0F38','0x0F3A');

function GetFrameOffset(const RegValue:TRegValue):Integer;
function GetFrameOffset(const r:TOperand):Integer;
function GetTargetOfs(var din:TInstruction;Code:PByte;id:Byte;var ofs:Int64):Boolean;
function is_preserved(const r:TRegValue):Boolean;
function is_preserved(const r:TOperand):Boolean;
function is_preserved(const r:TInstruction):Boolean;
function is_rip(const r:TRegValue):Boolean;
function is_rip(const r:TRegValues):Boolean;
function is_rip(const r:TOperand):Boolean;
function is_rip(const r:TInstruction):Boolean;
function is_memory(const r:TOperand):Boolean;
function is_memory(const r:TInstruction):Boolean;
function is_xmm(const r:TOperand):Boolean;
function is_xmm(const r:TInstruction):Boolean;
function is_high(const r:TOperand):Boolean;
function is_rsp(const r:TRegValue):Boolean;
function is_rsp(const r:TRegValues):Boolean;
function is_invalid(const r:TRegValue):Boolean;
function is_invalid(const r:TInstruction):Boolean;

type
 t_memop_type1=(mo_reg,
                mo_mem,
                mo_ctx
               );

 t_memop_type2=(mo_reg_reg,
                mo_mem_reg,
                mo_reg_mem,
                mo_reg_imm,
                mo_mem_imm,

                mo_ctx_reg,
                mo_reg_ctx,
                mo_ctx_ctx,

                mo_mem_ctx,

                mo_ctx_mem,
                mo_ctx_imm
               );

 t_memop_shift=(mo_mem_imm8,
                mo_reg_imm8,

                mo_mem_cl,
                mo_reg_cl,

                mo_mem_one,
                mo_reg_one,

                mo_ctx_imm8,
                mo_ctx_cl,
                mo_ctx_one
               );

 t_op_hint=Set of (his_mov,
                   his_xor,
                   his_xchg,
                   his_bt,
                   his_mri8,   //mem-reg-mm8
                   his_ro,     //read only
                   his_wo,     //write only
                   his_rw,     //read-write
                   his_align,
                   his_unbs);  //unbalanced size r/m

 t_op_desc=packed record
  mem_reg:t_op_type; //reg_reg
  reg_mem:t_op_type; //reg_reg
  reg_imm:t_op_type; //mem_imm
  reg_im8:t_op_type; //mem_im8,reg_mem_im8,(his_mri8)->mem_reg_im8
  hint:t_op_hint;
 end;

 t_op_shift=packed record
  reg_im8:t_op_type; //mem_im8
  mem__cl:t_op_type; //reg__cl
  mem_one:t_op_type; //reg_one
 end;

 t_op_avx3_imm=packed record
  rmi:t_op_type;
  mri:t_op_type;
 end;

 t_lea_hint=Set Of (not_use_segment,
                    not_use_r_tmp0,
                    not_use_r_tmp1,
                    inc8_rsp,
                    code_ref);

procedure build_lea(var ctx:t_jit_context2;id:Byte;
                    reg:TRegValue;hint:t_lea_hint=[]);

function  cmp_reg(const r1,r2:TRegValue):Boolean;
function  cmp_reg(const r1,r2:TOperand):Boolean;
function  cmp_reg_cross(const r1,r2:TRegValue):Boolean;
function  new_reg(const Operand:TOperand):TRegValue;
function  new_reg_size(const r:TRegValue;const sizeof:TOperandSize):TRegValue;
function  new_reg_size(const r:TRegValue;const sizeof:TRegValues):TRegValue;
function  new_reg_size(const r:TRegValue;const sizeof:TOperand):TRegValue;
function  fix_size(const r:TRegValue):TRegValue;
function  is_rep_prefix(const i:TInstruction):Boolean;
function  is_segment(const r:TOperand):Boolean;
function  is_segment(const i:TInstruction):Boolean;
function  get_segment_value(const Operand:TOperand):Byte;
function  flags(const i:TInstruction):t_jit_lea;
function  flags(const ctx:t_jit_context2):t_jit_lea;

procedure op_jit_interrupt(var ctx:t_jit_context2);

procedure op_set_mem_imm(var ctx:t_jit_context2;mem:t_jit_leas;imm:Int64;hint:t_lea_hint=[]);
procedure op_set_reg_imm(var ctx:t_jit_context2;reg:TRegValue;imm:Int64);
procedure op_set_r14_imm(var ctx:t_jit_context2;imm:Int64);
procedure op_set_rip_imm(var ctx:t_jit_context2;imm:Int64);

procedure op_load_r14(var ctx:t_jit_context2;reg:TRegValue);
procedure op_save_r14(var ctx:t_jit_context2;reg:TRegValue);

procedure op_load_rsp(var ctx:t_jit_context2;reg:TRegValue);
procedure op_save_rsp(var ctx:t_jit_context2;reg:TRegValue);

procedure op_load_rbp(var ctx:t_jit_context2;reg:TRegValue);
procedure op_save_rbp(var ctx:t_jit_context2;reg:TRegValue);

procedure op_load(var ctx:t_jit_context2;reg:TRegValue;opr:Byte);
procedure op_save(var ctx:t_jit_context2;opr:Byte;reg:TRegValue);

procedure op_uplift(var ctx:t_jit_context2;const dst:TRegValue;mem_size:TOperandSize;hint:t_lea_hint=[]);

procedure add_orig(var ctx:t_jit_context2);
procedure op_emit1(var ctx:t_jit_context2;const desc:t_op_type;hint:t_op_hint);
procedure op_emit2(var ctx:t_jit_context2;const desc:t_op_desc);
procedure op_emit_shift2(var ctx:t_jit_context2;const desc:t_op_shift);
procedure op_emit_shift3(var ctx:t_jit_context2;const desc:t_op_shift);
procedure op_emit_avx1(var ctx:t_jit_context2;const desc:t_op_type;hint:t_op_hint);
procedure op_emit_avx2_rr(var ctx:t_jit_context2;const desc:t_op_type);
procedure op_emit_avx2(var ctx:t_jit_context2;const desc:t_op_desc);
procedure op_emit_avx3(var ctx:t_jit_context2;const desc:t_op_type);
procedure op_emit_avx3_imm8(var ctx:t_jit_context2;const desc:t_op_avx3_imm);
procedure op_emit_avx_F3(var ctx:t_jit_context2;const desc:t_op_type);
procedure op_emit_avx4(var ctx:t_jit_context2;const desc:t_op_type);
procedure op_emit_bmi_rrm(var ctx:t_jit_context2;const desc:t_op_type);

procedure print_disassemble(addr:Pointer;vsize:Integer);

type
 t_instruction_info=record
  code_size:Byte;
  mema_size:Byte;
 end;

function get_instruction_info(addr:Pointer):t_instruction_info;

var
 jit_relative_analize:Boolean=True;
 jit_memory_guard    :Boolean=False;

implementation

uses
 vm_pmap_prot,
 machdep,
 kern_thr,
 md_systm,
 systm,
 kern_jit_asm;

procedure print_disassemble(addr:Pointer;vsize:Integer);
var
 proc:TDbgProcess;
 adec:TX86AsmDecoder;
 ptr,fin:Pointer;
 ACodeBytes,ACode:RawByteString;
begin
 ptr:=addr;
 fin:=addr+vsize;

 proc:=TDbgProcess.Create(dm64);
 adec:=TX86AsmDecoder.Create(proc);

 while (ptr<fin) do
 begin
  adec.Disassemble(ptr,ACodeBytes,ACode);
  Writeln(ACodeBytes:32,' ',ACode);
 end;

 adec.Free;
 proc.Free;
end;


function t_jit_context2.t_forward_point.c(n1,n2:p_forward_point):Integer;
begin
 Result:=Integer(n1^.dst>n2^.dst)-Integer(n1^.dst<n2^.dst);
end;

function t_jit_context2.t_switchtable_point.c(n1,n2:p_switchtable_point):Integer;
begin
 Result:=Integer(n1^.table>n2^.table)-Integer(n1^.table<n2^.table);
end;

function t_jit_context2.t_jumpslot.c(n1,n2:p_jumpslot):Integer;
begin
 Result:=Integer(n1^.addr>n2^.addr)-Integer(n1^.addr<n2^.addr);
end;

function t_jit_context2.t_label.c(n1,n2:p_label):Integer;
begin
 Result:=Integer(n1^.curr>n2^.curr)-Integer(n1^.curr<n2^.curr);
end;

function t_jit_context2.t_entry_point.c(n1,n2:p_entry_point):Integer;
begin
 Result:=Integer(n1^.src>n2^.src)-Integer(n1^.src<n2^.src);
end;

function t_jit_context2.is_text_addr(addr:QWORD):Boolean;
begin
 Result:=(addr>=text_start) and (addr<text___end);
end;

function t_jit_context2.is_map_addr(addr:QWORD):Boolean;
begin
 Result:=(addr>=text_start) and (addr<map____end);
end;

procedure t_jit_context2.add_export_point(nid:QWORD;native:Pointer;dst:PPointer);
var
 node:p_export_point;
begin
 if (native=nil) or (dst=nil) then Exit;
 node:=builder.Alloc(Sizeof(t_export_point));
 node^.nid   :=nid;
 node^.native:=native;
 node^.dst   :=dst;
 node^.next  :=export_list;
 export_list :=node;
end;

procedure t_jit_context2.add_import_point(guest,dst:PPointer);
var
 node:p_import_point;
begin
 if (guest=nil) or (dst=nil) then Exit;
 node:=builder.Alloc(Sizeof(t_import_point));
 node^.guest:=guest;
 node^.dst  :=dst;
 node^.next :=import_list;
 import_list:=node;
end;

procedure t_jit_context2.add_forward_link(node:p_forward_point;instruction:t_jit_i_link);
var
 link:p_forward_link;
begin
 if (node=nil) or (instruction=nil_link) then Exit;
 //
 link:=forward_link_cache;
 if (link<>nil) then
 begin
  forward_link_cache:=link^.next;
  link^.next:=nil
 end else
 begin
  link:=builder.Alloc(Sizeof(t_forward_link));
 end;
 //
 link:=builder.Alloc(Sizeof(t_forward_link));
 //
 link^.instruction:=instruction;
 link^.next       :=node^.links.root;
 node^.links.root:=link;
end;

procedure t_jit_context2.Resolve_forwards(var links:t_forward_links;target:t_jit_i_link);
var
 node:p_forward_link;
begin
 //init
 node:=links.root;
 //
 While (node<>nil) do
 begin
  //extract
  links.root:=node^.next;
  //set node
  node^.instruction.target:=target;
  //cache
  node^.instruction:=Default(t_jit_i_link);
  node^.next       :=forward_link_cache;
  forward_link_cache:=node;
  //next
  node:=links.root;
 end;
end;


function t_jit_context2.add_forward_point(ptype:t_point_type;instruction:t_jit_i_link;dst:Pointer):p_forward_point;
var
 node:t_forward_point;
begin
 if (dst=nil) then Exit;

 case get_chunk_ptype of
  fpData   :if (ptype=fpCall) then ptype:=fpData;
  fpInvalid:ptype:=fpInvalid;
  else;
 end;

 node.dst:=dst;
 Result:=forward_set.Find(@node);
 if (Result=nil) then
 begin
  Result:=forward_point_cache;
  if (Result<>nil) then
  begin
   forward_point_cache:=Result^.pLeft;
   Result^.pLeft:=nil;
  end else
  begin
   Result:=builder.Alloc(Sizeof(t_forward_point));
  end;
  //
  Result^.dst:=dst;
  Result^.links.ptype:=ptype;
  //
  forward_set.Insert(Result);
 end;
 add_forward_link(Result,instruction);
end;

function t_jit_context2.add_forward_point(ptype:t_point_type;dst:Pointer):p_forward_point;
begin
 Result:=add_forward_point(ptype,nil_link,dst);
end;

//

procedure t_jit_context2.add_jumpslot(addr:Pointer);
var
 _node:t_jumpslot;
 pnode:p_jumpslot;
begin
 if (addr=nil) then Exit;

 _node.addr:=addr;

 if (jumpslot_set.Find(@_node)=nil) then
 begin
  pnode:=builder.Alloc(Sizeof(t_jumpslot));
  pnode^.addr:=addr;

  jumpslot_set.Insert(pnode);
 end;
end;

function t_jit_context2.is_jumpslot(addr:Pointer):Boolean;
var
 node:t_jumpslot;
begin
 node.addr:=addr;

 Result:=(jumpslot_set.Find(@node)<>nil);
end;

function t_jit_context2.add_switchtable(table:Pointer):p_switchtable_point;
var
 node:t_switchtable_point;
begin
 if (table=nil) then Exit;

 node.table:=table;
 //
 Result:=switchtable_set.Find(@node);
 if (Result=nil) then
 begin
  //
  Result:=builder.Alloc(Sizeof(t_switchtable_point));
  //
  Result^.table:=table;
  Result^.curr :=table;
  //
  switchtable_set.Insert(Result);
  //
  //mark minimum address for processing
  if (min_switchtable=nil) then
  begin
   min_switchtable:=Result;
  end else
  if (min_switchtable^.table>table) then
  begin
   min_switchtable:=Result;
  end;
  //
 end;
end;

function t_jit_context2.fetch_switchtable(var out_table:p_switchtable_point;var out_next:PInteger):Boolean;
var
 node,node_next:p_switchtable_point;
begin
 Result:=False;

 //move to min
 if (min_switchtable=nil) then Exit;
 switchtable_set._Splay(min_switchtable);
 node:=switchtable_set.pRoot;
 if (node=nil) then Exit;

 repeat

  node_next:=switchtable_set.Next(node);
  if (node_next<>nil) then
  begin
   if (node^.curr=nil) or
      (node^.curr>=node_next^.table) then
   begin
    //This is complete, let's move on to the next one.
    //
    min_switchtable:=node_next;
    node           :=node_next;
   end else
   begin
    out_table:=node;
    out_next :=node_next^.table;
    //
    Exit(True);
   end;
  end else
  if (node^.curr=nil) then
  begin
   //Terminated
   min_switchtable:=nil;
   //
   out_table:=nil;
   out_next :=nil;
   //
   Exit(False);
  end else
  begin
   //Last in list
   out_table:=node;
   out_next :=nil;
   //
   Exit(True);
  end;

 until false;

end;

//

Function t_jit_context2.new_chunk(ptype:t_point_type;start:Pointer):p_jit_code_chunk;
begin
 Result:=builder._new_chunk(QWORD(start));
 if (Result<>nil) then
 begin
  Result^.data:=QWORD(ptype);
 end;
end;

procedure t_jit_context2.mark_chunk(ptype:t_point_type);
var
 node:p_jit_code_chunk;
begin
 node:=builder.ACodeChunkCurr;
 if (node<>nil) then
 if (t_point_type(node^.data)=fpData) then
 begin
  node^.data:=QWORD(ptype);
 end;
end;

function t_jit_context2.get_chunk_ptype():t_point_type;
var
 node:p_jit_code_chunk;
begin
 Result:=fpCall;
 node:=builder.ACodeChunkCurr;
 if (node<>nil) then
 begin
  Result:=t_point_type(node^.data);
 end;
end;

procedure t_jit_context2.end_chunk(__end:Pointer);
begin
 builder._end_chunk(QWORD(__end));
end;

function t_jit_context2.max_forward_point():Pointer;
var
 entry:p_forward_point;
begin
 Result:=nil;
 entry:=forward_set.Max;
 if (entry=nil) then Exit;
 Result:=entry^.dst;
end;

function t_jit_context2.fetch_forward_point(var links:t_forward_links;var dst:Pointer):Boolean;
var
 min:p_forward_point;
begin
 Result:=False;
 //get min
 min:=forward_set.Min;
 if (min=nil) then Exit;
 //move to min
 forward_set._Splay(min);
 min:=forward_set.pRoot;
 //extract
 forward_set.Delete(min);
 //set
 dst  :=min^.dst;
 links:=min^.links;
 //cache
 min^:=Default(t_forward_point);
 min^.pLeft:=forward_point_cache;
 forward_point_cache:=min;
 //
 Result:=True;
end;

function t_jit_context2.add_label(curr,next:Pointer;link_curr,link_next:t_jit_i_link;flags:Integer):p_label;
var
 node:t_label;
begin
 if (curr=nil) then Exit;
 node.curr:=curr;
 Result:=label_set.Find(@node);
 if (Result<>nil) then Exit;
 Result:=builder.Alloc(Sizeof(t_label));
 //
 Result^.curr     :=curr;
 Result^.next     :=next;
 Result^.link_curr:=link_curr;
 Result^.link_next:=link_next;
 Result^.flags    :=flags;
 //
 label_set.Insert(Result);
end;

function t_jit_context2.get_label(src:Pointer):p_label;
var
 node:t_label;
begin
 Result:=nil;
 node.curr:=src;
 Result:=label_set.Find(@node);
end;

function t_jit_context2.get_link(src:Pointer):t_jit_i_link;
var
 node:p_label;
begin
 Result:=nil_link;
 node:=get_label(src);
 if (node=nil) then Exit;
 Result:=node^.link_curr;
end;

procedure t_jit_context2.add_entry_point(src:Pointer;label_id:t_jit_i_link);
var
 key :t_entry_point;
 node:p_entry_point;
begin
 if (src=nil) then Exit;
 //set key
 key:=Default(t_entry_point);
 key.src        :=src;
 key.instruction:=label_id;
 //find exists
 node:=entry_set.Find(@key);
 if (node<>nil) then Exit; //Already added
 //new
 node:=builder.Alloc(Sizeof(t_entry_point));
 node^:=key;
 //insert set
 entry_set.Insert(node);
 //insert list
 node^.next    :=entry_list;
 entry_list:=node;
end;

procedure t_jit_context2.Free;
begin
 builder.Free;
 Self:=Default(t_jit_context2);
end;

//

function GetFrameOffset(const RegValue:TRegValue):Integer;
begin
 Result:=-1;

 case RegValue.AType of
  regGeneral:
  begin
   case RegValue.AIndex of
     4:Result:=Integer(@p_jit_frame(nil)^.tf_rsp);
     5:Result:=Integer(@p_jit_frame(nil)^.tf_rbp);
    13:Result:=Integer(@p_jit_frame(nil)^.tf_r13);
    14:Result:=Integer(@p_jit_frame(nil)^.tf_r14);
    15:Result:=Integer(@p_jit_frame(nil)^.tf_r15);
    else;
   end;
  end;
 else;
 end;

 Assert(Result<>-1,'GetFrameOffset');
end;

function GetFrameOffset(const r:TOperand):Integer; inline;
begin
 Result:=GetFrameOffset(r.RegValue[0])
end;

function GetTargetOfs(var din:TInstruction;Code:PByte;id:Byte;var ofs:Int64):Boolean;
var
 i:Integer;
begin
 Result:=True;
 i:=din.Operand[id].CodeIndex;
 case din.Operand[id].ByteCount of
   1: ofs:=PShortint(@Code[i])^;
   2: ofs:=PSmallint(@Code[i])^;
   4: ofs:=PInteger (@Code[i])^;
   8: ofs:=PInt64   (@Code[i])^;
   else
      Result:=False;
 end;
end;

function is_preserved(AIndex:Byte):Boolean; inline;
begin
 Result:=AIndex in [4,5,13..15];
 //  4 rsp
 //  5 rbp
 // 13 r13
 // 14 r14
 // 15 r15
end;

function is_preserved(const r:TRegValue):Boolean;
begin
 Result:=False;

 case r.AType of
  regRip    :Result:=True;
  regGeneral:Result:=is_preserved(r.AIndex);
  else;
 end;

end;

function is_preserved(const r:TRegValues):Boolean; inline;
begin
 Result:=is_preserved(r[0]) or is_preserved(r[1]);
end;

function is_preserved(const r:TOperand):Boolean; inline;
begin
 Result:=is_preserved(r.RegValue);
end;

function is_preserved(const r:TInstruction):Boolean;
var
 i:Integer;
begin
 Result:=False;
 if (r.OperCnt<>0) then
 For i:=1 to r.OperCnt do
 begin
  Result:=is_preserved(r.Operand[i]);
  if Result then Exit;
 end;
end;

function is_rip(const r:TRegValue):Boolean; inline;
begin
 Result:=False;
 case r.AType of
  regRip:Result:=True;
  else;
 end;
end;

function is_rip(const r:TRegValues):Boolean; inline;
begin
 Result:=is_rip(r[0]) or is_rip(r[1]);
end;

function is_rip(const r:TOperand):Boolean; inline;
begin
 Result:=is_rip(r.RegValue);
end;

function is_rip(const r:TInstruction):Boolean;
var
 i:Integer;
begin
 Result:=False;
 if (r.OperCnt<>0) then
 For i:=1 to r.OperCnt do
 begin
  Result:=is_rip(r.Operand[i]);
  if Result then Exit;
 end;
end;

function is_memory(const r:TOperand):Boolean; inline;
begin
 Result:=ofMemory in r.Flags;
end;

function is_memory(const r:TInstruction):Boolean;
var
 i:Integer;
begin
 Result:=False;
 if (r.OperCnt<>0) then
 For i:=1 to r.OperCnt do
 begin
  Result:=is_memory(r.Operand[i]);
  if Result then Exit;
 end;
end;

function is_xmm(const r:TOperand):Boolean; inline;
begin
 Result:=r.RegValue[0].AType=regXmm;
end;

function is_xmm(const r:TInstruction):Boolean;
var
 i:Integer;
begin
 Result:=False;
 if (r.OperCnt<>0) then
 For i:=1 to r.OperCnt do
 begin
  Result:=is_xmm(r.Operand[i]);
  if Result then Exit;
 end;
end;

function is_rsp(const r:TRegValue):Boolean; inline;
begin
 Result:=False;
 case r.AType of
  regGeneral:
   case r.AIndex of
     4:Result:=True;
    else;
   end;
  else;
 end;
end;

function is_rsp(const r:TRegValues):Boolean; inline;
begin
 Result:=is_rsp(r[0]) or is_rsp(r[1]);
end;

function is_cl(const r:TRegValue):Boolean; inline;
begin
 Result:=False;
 if (r.ASize=os8) then
 case r.AType of
  regGeneral:
   case r.AIndex of
     1:Result:=True;
    else;
   end;
  else;
 end;
end;

function is_one(const r:TRegValue):Boolean; inline;
begin
 Result:=False;
 case r.AType of
  regOne:Result:=True;
  else;
 end;
end;

function is_high(const r:TOperand):Boolean; inline;
begin
 Result:=r.RegValue[0].AType=regGeneralH;
end;

function is_high(const r:TRegValue):Boolean; inline;
begin
 Result:=r.AType=regGeneralH;
end;

function cmp_reg(const r1,r2:TRegValue):Boolean; inline;
begin
 Result:=(r1.AType =r2.AType) and
         (r1.ASize =r2.ASize) and
         (r1.AIndex=r2.AIndex);
end;

function cmp_reg(const r1,r2:TOperand):Boolean; inline;
begin
 Result:=cmp_reg(r1.RegValue[0],r2.RegValue[0]);
end;

function cmp_reg_cross(const r1,r2:TRegValue):Boolean; inline;
begin
 Result:=(
          (r1.AType=r2.AType) or
          (
           (r1.AType in [regGeneral, regGeneralH]) and
           (r2.AType in [regGeneral, regGeneralH])
          )
         ) and
         (r1.AIndex=r2.AIndex);
end;

function cmp_reg_cross(const r1:TRegValue;const Operand:TOperand):Boolean; inline;
begin
 Result:=cmp_reg_cross(r1,Operand.RegValue[0]);
end;

function new_reg(const Operand:TOperand):TRegValue; inline;
begin
 Result:=Operand.RegValue[0];
end;

function new_reg_size(const r:TRegValue;const sizeof:TOperandSize):TRegValue; inline;
begin
 Result:=r;
 Result.ASize:=sizeof;
end;

function new_reg_size(const r:TRegValue;const sizeof:TRegValues):TRegValue; inline;
begin
 Result:=new_reg_size(r,sizeof[0].ASize);
end;

function new_reg_size(const r:TRegValue;const sizeof:TOperand):TRegValue; inline;
begin
 Result:=new_reg_size(r,sizeof.RegValue[0].ASize);
end;

function fix_size(const r:TRegValue):TRegValue; inline;
begin
 Result:=r;
 if (Result.ASize=os32) then
 begin
  Result.ASize:=os64;
 end;
end;

function fix_size8(const r:TRegValue):TRegValue; inline;
begin
 Result:=r;
 case Result.ASize of
  os8 :Result.ASize:=os16;
  os32:Result.ASize:=os64;
  else;
 end;
end;

function is_rep_prefix(const i:TInstruction):Boolean;
begin
 Result:=[ifPrefixRep,ifPrefixRepE,ifPrefixRepNe]*i.Flags<>[];
end;

function is_segment(const r:TOperand):Boolean; inline;
begin
 Result:=(r.RegValue[0].AType=regSegment) and
         (not (ofMemory in r.Flags));
end;

function get_segment_value(const Operand:TOperand):Byte;
const
 REG_ES = 0;
 REG_CS = 1;
 REG_SS = 2;
 REG_DS = 3;
 REG_FS = 4;
 REG_GS = 5;
begin
 case Operand.RegValue[0].AIndex of
  REG_ES:Result:=_udatasel;
  REG_CS:Result:=_ucodesel;
  REG_SS:Result:=_udatasel;
  REG_DS:Result:=_udatasel;
  REG_FS:Result:=_ufssel;
  REG_GS:Result:=_ugssel;
  else
         Result:=0;
 end;
end;

function flags(const i:TInstruction):t_jit_lea;
begin
 Result:=Default(t_jit_lea);

 if (ifPrefixLock in i.Flags) then
 begin
  Result:=Result+t_jit_builder.LOCK;
 end;
end;

function flags(const ctx:t_jit_context2):t_jit_lea; inline;
begin
 Result:=flags(ctx.din);
end;

procedure add_orig(var ctx:t_jit_context2);
var
 ji:t_jit_instruction;
begin
 ji:=default_jit_instruction;

 Move(ctx.code^,ji.AData,ctx.dis.CodeIdx);

 ji.AInstructionSize:=ctx.dis.CodeIdx;

 ctx.builder._add(ji);
end;

function is_invalid(const r:TRegValue):Boolean;
begin
 Result:=False;
 case r.AType of
  regInvalid:Result:=True;

  regGeneral:
    case r.ASize of
       os8,
      os16,
      os32,
      os64: Result:=(r.AIndex>=16);
    else
            Result:=True;
    end;

  regGeneralH:
    case r.ASize of
       os8: Result:=(r.AIndex>=4);
    else
            Result:=True;
    end;

  regMm,
  regX87: Result:=(r.AIndex>=8);

  regXmm:
    case r.ASize of
      os32,
      os64,
      os128,
      os256: Result:=(r.AIndex>=16);
    else
             Result:=True;
    end;

  regSegment: Result:=(r.AIndex>=6);

  regFlags:
   case r.ASize of
     os16,
     os32,
     os64:;
   else
     Result:=True;
   end;

  else;
 end;
end;

function is_invalid(const r:TOperand):Boolean; inline;
begin
 Result:=is_invalid(r.RegValue[0]) or
         is_invalid(r.RegValue[1]);
end;

function is_invalid(const r:TInstruction):Boolean;
var
 i:Integer;
begin
 Result:=False;
 if (r.OperCnt<>0) then
 For i:=1 to r.OperCnt do
 begin
  Result:=is_invalid(r.Operand[i]);
  if Result then Exit;
 end;
end;

function scan_switchtable(var ctx:t_jit_context2;start:Int64):Boolean;
var
 table:PInteger;
 rel:Integer;
begin
 Result:=False;
 //
 if (cmDontScanSwitchTable in ctx.modes) then Exit;
 //
 table:=PInteger(start);
 //
 if ctx.is_text_addr(QWORD(table)) then
 begin

  rel:=0;
  if (copyin(table,@rel,SizeOf(Integer))<>0) then
  begin
   Exit;
  end;

  if (DWORD(rel) and $FFFF0000)=$FFFF0000 then
  begin
   ctx.add_switchtable(table);
   Result:=True;
  end;

 end;
end;

procedure add_rip_entry(var ctx:t_jit_context2;ofs:Int64;hint:t_lea_hint);
var
 new_ofs:Int64;
begin
 if not jit_relative_analize then Exit;

 if (code_ref in hint) then
 begin
  //call [addr]
  //jmp  [addr]

  if not (cmDontScanRipRel in ctx.modes) then

  if ctx.is_map_addr(ofs) then
  if not ctx.is_jumpslot(Pointer(ofs)) then
  if ((ppmap_get_prot(QWORD(ofs)) and PAGE_PROT_READ)<>0) then
  begin
   new_ofs:=0;

   if (copyin(Pointer(ofs),@new_ofs,SizeOf(Pointer))=0) then
   begin
    if ctx.is_text_addr(new_ofs) and (new_ofs<=ctx.max_reloc) then
    begin
     ctx.add_forward_point(fpCall,Pointer(new_ofs));
    end;
   end;

  end;

 end else
 begin
  //lea

  if not (cmDontScanRipRel in ctx.modes) then

  if scan_switchtable(ctx,ofs) then
  begin
   //
  end else
  if ctx.is_text_addr(ofs) and (ofs<=ctx.max_reloc) then
  begin
   ctx.add_forward_point(fpData,Pointer(ofs));
  end;

 end;
end;

function is_segment(const i:TInstruction):Boolean;
begin
 Result:=False;

 case i.SegmentReg of
  4:Result:=True;
  5:Result:=True;
  else;
 end;
end;

function get_segment(const i:TInstruction):Integer;
begin
 Result:=0;

 case i.SegmentReg of
  4:Result:=teb_fsbase;
  5:Result:=teb_gsbase;
  else
    Assert(False);
 end;
end;

procedure optimal_swap(var RegValue:TRegValues);
var
 t:TRegValue;
begin
 if (RegValue[0].AType<>regNone) and
    (RegValue[1].AType<>regNone) and
    (not is_preserved(RegValue[0])) and
    (is_preserved(RegValue[1])) and
    (RegValue[0].AScale<=1) then
 begin
  //optimal swap
  t:=RegValue[0];
  RegValue[0]:=RegValue[1];
  RegValue[1]:=t;
 end;
end;

function lea_reg_is_used(var RegValue:TRegValues;reg:TRegValue):Boolean; inline;
begin
 Result:=(RegValue[1].AType<>regNone) and
         (RegValue[1].AIndex=reg.AIndex);
end;

procedure build_lea(var ctx:t_jit_context2;id:Byte;
                    reg:TRegValue;hint:t_lea_hint=[]);
var
 RegValue:TRegValues;
 adr:TRegValue;
 new1,new2:TRegValue;
 ofs:Int64;
 i:Integer;
 AScale:Byte;
 save_r_tmp0:Boolean;
 reg1_used  :Boolean;
 scale_ofs  :Boolean;
 fix_rsp    :Boolean;
begin
 RegValue:=ctx.din.Operand[id].RegValue;

 adr:=new_reg_size(reg,RegValue);

 if (adr.ASize=os0) then
 begin
  adr.ASize:=os64;
 end;

 with ctx.builder do
 begin
  if (not (not_use_segment in hint)) and
     is_segment(ctx.din) then
  begin

   if (RegValue[0].AType=regNone) then //absolute offset
   begin
    ofs:=0;
    GetTargetOfs(ctx.din,ctx.code,id,ofs);

    if (ofs=0) then
    begin
     movq(reg,[GS+get_segment(ctx.din)]); //endpoint
    end else
    begin
     movq(adr,[GS+get_segment(ctx.din)]);
     leaq(reg,[adr+ofs]); //endpoint
    end;

   end else
   begin
    Assert(false,'TODO');
   end;

  end else
  if (RegValue[0].AType=regNone) then //absolute offset
  begin
   ofs:=0;
   GetTargetOfs(ctx.din,ctx.code,id,ofs);

   //sign extend
   movi(reg,ofs); //endpoint
  end else
  if is_rip(RegValue[0]) then //rip relative
  begin
   ofs:=0;
   GetTargetOfs(ctx.din,ctx.code,id,ofs);
   ofs:=Int64(ctx.ptr_next)+ofs;

   add_rip_entry(ctx,ofs,hint);

   if (classif_offset_u64(ofs)=os64) then
   begin
    movi64(adr,ofs); //endpoint
   end else
   begin
    if (reg.ASize=os64) then
    begin
     //mov r14d,imm32   this is zero extend to 64bit
     movi(new_reg_size(reg,os32),ofs); //endpoint
    end else
    begin
     movi(reg,ofs); //endpoint
    end;
   end;
  end else
  if is_preserved(RegValue) then
  begin

   optimal_swap(RegValue);

   save_r_tmp0:=False;
   if lea_reg_is_used(RegValue,adr) then
   begin
    adr:=new_reg_size(r_tmp0,adr.ASize);
    save_r_tmp0:=(not_use_r_tmp0 in hint);
   end;

   if save_r_tmp0 then
   begin
    //use rbp
    push(rbp);
    adr:=new_reg_size(rbp,adr.ASize);
   end;

   AScale:=RegValue[0].AScale;

   ofs:=0;
   GetTargetOfs(ctx.din,ctx.code,id,ofs);

   reg1_used:=(RegValue[1].AType<>regNone);

   //1
   if is_preserved(RegValue[0]) then
   begin
    fix_rsp:=false;

    if (inc8_rsp in hint) and is_rsp(RegValue[0]) then
    begin
     //fix rsp relative
     if (AScale<=1) and
        (classif_offset_64(ofs+8)<>os64) then
     begin
      //shift ofs
      ofs:=ofs+8;
     end else
     begin
      //fix in lea
      fix_rsp:=true;
     end;
    end;

    scale_ofs:=(AScale>1) or (ofs<>0);

    i:=GetFrameOffset(RegValue[0]);

    if reg1_used or
       fix_rsp or
       scale_ofs then
    begin
     movq(adr,[r_thrd+i]);
    end else
    begin
     movq(reg,[r_thrd+i]); //endpoint
    end;

    //fix rsp relative
    if fix_rsp then
    begin
     if reg1_used or
        scale_ofs then
     begin
      leaq(adr,[adr+8]);
     end else
     begin
      leaq(reg,[adr+8]); //endpoint
     end;
    end;

    if scale_ofs then
    begin
     if reg1_used then
     begin
      leaq(adr,[adr*AScale+ofs]);
     end else
     begin
      leaq(reg,[adr*AScale+ofs]); //endpoint
     end;
    end;
   end else
   begin
    new1:=RegValue[0];
    //
    //AScale in new1
    if reg1_used then
    begin
     leaq(adr,[new1+ofs]);
    end else
    begin
     leaq(reg,[new1+ofs]); //endpoint
    end;
   end;
   //1

   //2
   if reg1_used then
   begin

    if is_preserved(RegValue[1]) then
    begin
     i:=GetFrameOffset(RegValue[1]);

     new2:=new_reg_size(r_tmp1,adr.ASize);

     if (not_use_r_tmp1 in hint) then
     begin
      if save_r_tmp0 then
      begin
       push(r_tmp1);
      end else
      begin
       //use rbp
       push(rbp);
       new2:=new_reg_size(rbp,adr.ASize);
      end;
     end;

     movq(new2,[r_thrd+i]);

     if (inc8_rsp in hint) and is_rsp(RegValue[1]) then
     begin
      //fix rsp relative
      leaq(reg,[adr+new2+8]); //endpoint
     end else
     begin
      leaq(reg,[adr+new2]); //endpoint
     end;

     if (not_use_r_tmp1 in hint) then
     begin
      if save_r_tmp0 then
      begin
       pop(r_tmp1);
      end else
      begin
       //restore rbp
       pop(rbp);
       //movq(rbp,rsp);
      end;
     end;

    end else
    begin
     new1:=RegValue[1];
     leaq(reg,[adr+new1]); //endpoint
    end;

   end;
   //2

   if save_r_tmp0 then
   begin
    //restore rbp
    pop(rbp);
    //movq(rbp,rsp);
   end;

   //is_preserved
  end else
  begin
   //direct
   ofs:=0;
   GetTargetOfs(ctx.din,ctx.code,id,ofs);

   new1:=RegValue[0];

   if (RegValue[1].AType<>regNone) then
   begin
    new2:=RegValue[1];
    //
    //AScale in new1
    leaq(reg,[new1+new2+ofs]); //endpoint
   end else
   begin
    //AScale in new1
    leaq(reg,[new1+ofs]); //endpoint
   end;

  end;

 end;

end;

function get_lea_id(memop:t_memop_type2):Byte;
begin
 case memop of
  mo_mem_reg:Result:=1;
  mo_reg_mem:Result:=2;
  mo_mem_imm:Result:=1;
  mo_mem_ctx:Result:=1;
  mo_ctx_mem:Result:=2;
  else
   Assert(False);
 end;
end;

function classif_memop1(var din:TInstruction):t_memop_type1;
begin
 if (ofMemory in din.Operand[1].Flags) then
 begin
  Result:=mo_mem;
 end else
 if is_preserved(din.Operand[1]) then
 begin
  Result:=mo_ctx;
 end else
 begin
  Result:=mo_reg;
 end;
end;

function classif_memop2(var din:TInstruction):t_memop_type2;
begin
 if (ofMemory in din.Operand[1].Flags) then
 begin
  if (din.Operand[2].ByteCount<>0) then
  begin
   Result:=mo_mem_imm;
  end else
  if is_preserved(din.Operand[2]) then
  begin
   Result:=mo_mem_ctx;
  end else
  begin
   Result:=mo_mem_reg;
  end;
 end else
 if (ofMemory in din.Operand[2].Flags) then
 begin
  if is_preserved(din.Operand[1]) then
  begin
   Result:=mo_ctx_mem;
  end else
  begin
   Result:=mo_reg_mem;
  end;
 end else
 if (din.Operand[2].ByteCount<>0) then
 begin
  if is_preserved(din.Operand[1]) then
  begin
   Result:=mo_ctx_imm;
  end else
  begin
   Result:=mo_reg_imm;
  end;
 end else
 if is_preserved(din.Operand[1]) and
    is_preserved(din.Operand[2]) then
 begin
  Result:=mo_ctx_ctx;
 end else
 if is_preserved(din.Operand[1]) then
 begin
  Result:=mo_ctx_reg;
 end else
 if is_preserved(din.Operand[2]) then
 begin
  Result:=mo_reg_ctx;
 end else
 begin
  Result:=mo_reg_reg;
 end;
end;

//

function classif_shift2(var din:TInstruction):t_memop_shift;
begin
 if (ofMemory in din.Operand[1].Flags) then
 begin
  if (din.Operand[2].ByteCount<>0) then
  begin
   Assert(din.Operand[2].Size=os8);
   Result:=mo_mem_imm8;
  end else
  if is_cl(din.Operand[2].RegValue[0]) then
  begin
   Result:=mo_mem_cl;
  end else
  if is_one(din.Operand[2].RegValue[0]) then
  begin
   Result:=mo_mem_one;
  end else
  begin
   Assert(false);
  end;
 end else
 if is_preserved(din.Operand[1]) then
 begin
  if (din.Operand[2].ByteCount<>0) then
  begin
   Assert(din.Operand[2].Size=os8);
   Result:=mo_ctx_imm8;
  end else
  if is_cl(din.Operand[2].RegValue[0]) then
  begin
   Result:=mo_ctx_cl;
  end else
  if is_one(din.Operand[2].RegValue[0]) then
  begin
   Result:=mo_ctx_one;
  end else
  begin
   Assert(false);
  end;
 end else
 if (din.Operand[2].ByteCount<>0) then
 begin
  Assert(din.Operand[2].Size=os8);
  Result:=mo_reg_imm8;
 end else
 if is_cl(din.Operand[2].RegValue[0]) then
 begin
  Result:=mo_reg_cl;
 end else
 if is_one(din.Operand[2].RegValue[0]) then
 begin
  Result:=mo_reg_one;
 end else
 begin
  Assert(false);
 end;
end;

function classif_shift3(var din:TInstruction):t_memop_shift;
begin
 if (ofMemory in din.Operand[1].Flags) then
 begin
  if (din.Operand[3].ByteCount<>0) then
  begin
   Assert(din.Operand[3].Size=os8);
   Result:=mo_mem_imm8;
  end else
  if is_cl(din.Operand[3].RegValue[0]) then
  begin
   Result:=mo_mem_cl;
  end else
  begin
   Assert(false);
  end;
 end else
 if is_preserved(din.Operand[1]) then
 begin
  if (din.Operand[3].ByteCount<>0) then
  begin
   Assert(din.Operand[3].Size=os8);
   Result:=mo_ctx_imm8;
  end else
  if is_cl(din.Operand[3].RegValue[0]) then
  begin
   Result:=mo_ctx_cl;
  end else
  begin
   Assert(false);
  end;
 end else
 if (din.Operand[3].ByteCount<>0) then
 begin
  Assert(din.Operand[3].Size=os8);
  Result:=mo_reg_imm8;
 end else
 if is_cl(din.Operand[3].RegValue[0]) then
 begin
  Result:=mo_reg_cl;
 end else
 begin
  Assert(false);
 end;
end;

//

procedure op_jit_interrupt(var ctx:t_jit_context2);
begin
 with ctx.builder do
 begin
  //[65 FF 14 25] [00 07 00 00] call gs:[$00000700]
  //call([GS+teb_jit_trp]);

  //ctx.label_flags:=ctx.label_flags or LF_JMP_INTERRUPT;
 end;
end;

//

procedure op_set_mem_imm(var ctx:t_jit_context2;mem:t_jit_leas;imm:Int64;hint:t_lea_hint=[]);
var
 mreg:t_jit_lea;
 reg:TRegValue;
begin
 with ctx.builder do
 begin
  mreg:=Sums(mem);
  if (mreg.AMemSize=os64) then
  begin
   //64
   if (classif_offset_se64(imm)=os64) then
   begin
    //64 imm
    if (not_use_r_tmp0 in hint) and
       (not_use_r_tmp1 in hint) then
    begin
     //32bit zero extend (+0) (+4)
     movi([mreg+0,os32],Lo(imm));
     movi([mreg+4,os32],Hi(imm));
     Exit;
    end else
    if (not_use_r_tmp0 in hint) then
    begin
     reg:=r_tmp1;
    end else
    begin
     reg:=r_tmp0;
    end;
    //
    if (classif_offset_u64(imm)=os64) then
    begin
     //64bit imm
     movi64(reg,imm);
     movq([mreg,os64],reg);
    end else
    begin
     //32bit zero extend
     movi(new_reg_size(reg,os32),imm);
     movq([mreg,os64],reg);
    end;
    //64 imm
   end else
   begin
    //32bit sign extend
    movi([mreg,os64],imm);
   end;
   //64
  end else
  begin
   movi(mem,imm);
  end;
 end;
end;

procedure op_set_reg_imm(var ctx:t_jit_context2;reg:TRegValue;imm:Int64);
begin
 with ctx.builder do
 begin
  if (reg.ASize=os64) then
  begin
   if (classif_offset_u64(imm)=os64) then
   begin
    if (imm and QWORD($FFFFFFFF80000000))=QWORD($FFFFFFFF80000000) then
    begin
     //32bit sign extend
     movi(reg,imm);
    end else
    begin
     //64bit imm
     movi64(reg,imm);
    end;
   end else
   begin
    //32bit zero extend
    movi(new_reg_size(reg,os32),imm);
   end;
  end else
  begin
   movi(reg,imm);
  end;
 end;
end;

procedure op_set_r14_imm(var ctx:t_jit_context2;imm:Int64);
begin
 op_set_reg_imm(ctx,r_tmp0,imm);
end;

procedure op_set_rip_imm(var ctx:t_jit_context2;imm:Int64);
begin
 op_set_r14_imm(ctx,imm);
 //
 with ctx.builder do
 begin
  movq([r_thrd+(@p_jit_frame(nil)^.tf_rip)],r_tmp0);
 end;
end;

procedure op_load_r14(var ctx:t_jit_context2;reg:TRegValue);
var
 i:Integer;
begin
 with ctx.builder do
 begin
  i:=GetFrameOffset(r14);
  movq(reg,[r_thrd+i]);
 end;
end;

procedure op_save_r14(var ctx:t_jit_context2;reg:TRegValue);
var
 i:Integer;
begin
 with ctx.builder do
 begin
  i:=GetFrameOffset(r14);
  movq([r_thrd+i],reg);
 end;
end;

//

procedure op_load_rsp(var ctx:t_jit_context2;reg:TRegValue);
var
 i:Integer;
begin
 with ctx.builder do
 begin
  i:=GetFrameOffset(rsp);
  movq(reg,[r_thrd+i]);
 end;
end;

procedure op_save_rsp(var ctx:t_jit_context2;reg:TRegValue);
var
 i:Integer;
begin
 with ctx.builder do
 begin
  i:=GetFrameOffset(rsp);
  movq([r_thrd+i],reg);
 end;
end;

//

procedure op_load_rbp(var ctx:t_jit_context2;reg:TRegValue);
var
 i:Integer;
begin
 with ctx.builder do
 begin
  i:=GetFrameOffset(rbp);
  movq(reg,[r_thrd+i]);
 end;
end;

procedure op_save_rbp(var ctx:t_jit_context2;reg:TRegValue);
var
 i:Integer;
begin
 with ctx.builder do
 begin
  i:=GetFrameOffset(rbp);
  movq([r_thrd+i],reg);
 end;
end;

//

procedure op_load(var ctx:t_jit_context2;reg:TRegValue;opr:Byte);
var
 i:Integer;
begin
 with ctx.builder do
 begin
  i:=GetFrameOffset(ctx.din.Operand[opr]);
  movq(reg,[r_thrd+i]);
 end;
end;

procedure op_save(var ctx:t_jit_context2;opr:Byte;reg:TRegValue);
var
 i:Integer;
begin
 with ctx.builder do
 begin
  i:=GetFrameOffset(ctx.din.Operand[opr]);
  movq([r_thrd+i],reg);
 end;
end;

// VM_MAXUSER_ADDRESS
// $80000000000 = 1 shl 43
// 64-43 = 21

procedure op_uplift(var ctx:t_jit_context2;const dst:TRegValue;mem_size:TOperandSize;hint:t_lea_hint=[]);
var
 rbits:TRegValue;
 instr:t_jit_i_link;
begin
 if not jit_memory_guard then Exit;

 case dst.AIndex of
  14:hint:=hint+[not_use_r_tmp0]; //r14
  15:hint:=hint+[not_use_r_tmp1]; //r15
  else;
 end;

 with ctx.builder do
 begin
  if (not_use_r_tmp0 in hint) and
     (not_use_r_tmp1 in hint) then
  begin
   rbits:=r13;
  end else
  if (not_use_r_tmp0 in hint) then
  begin
   rbits:=r_tmp1;
  end else
  begin
   rbits:=r_tmp0;
  end;

  xchgq(rcx,rbits);

  //addres bits
  movi(new_reg_size(rcx,os8),43);

  shrx(rcx,dst,rcx);

  instr:=jcxz(nil_link,as64,os8);

  //error
  xchgq(rcx,rbits);
  ud2;

  //next
  instr.target:=get_curr_label.after;

  xchgq(rcx,rbits);

  {
  //zero bits
  movi(new_reg_size(rbits,os8),21); //mov  $21,%bpl

  //clear hi
  shlx(dst,dst,rbits); //shlx %rbp,%r14,%r14
  shrx(dst,dst,rbits); //shrx %rbp,%r14,%r14
  }

  if (rbits.AIndex=r13.AIndex) then
  begin
   //restore jit_frame
   movq(r13,[GS +teb_thread]);
   leaq(r13,[r13+jit_frame_offset]);
  end;
 end;
end;

procedure op_copyin(var ctx:t_jit_context2;mem_size:TOperandSize); inline;
begin
 with ctx.builder do
 begin
  //call_far(copyin_mov_size[mem_size]); //in:r14(addr), out:r14
 end;
end;

procedure op_copyout_before(var ctx:t_jit_context2;var link_next:t_jit_i_link;mem_size:TOperandSize); inline;
begin
 with ctx.builder do
 begin
  ////call_far(copyout_mov_size[mem_size]); //in:r14(addr)
  //
  ////link_next:=jmp(nil_link,os8);
 end;
end;

procedure op_copyout_after(var ctx:t_jit_context2;var link_next:t_jit_i_link;mem_size:TOperandSize); inline;
begin
 with ctx.builder do
 begin
  ////reta;
  //
  ////link_next._label:=get_curr_label.after;
 end;
end;

//

procedure op_emit1(var ctx:t_jit_context2;const desc:t_op_type;hint:t_op_hint);
var
 i:Integer;
 memop:t_memop_type1;
 mem_size:TOperandSize;
 link_next:t_jit_i_link;

 new:TRegValue;

 procedure mem_out;
 begin
  with ctx.builder do
  begin
   //input:r14

   _M(desc,[flags(ctx)+r_tmp0,mem_size]);
  end;
 end;

 procedure mem_in;
 begin
  with ctx.builder do
  begin
   //input:r14

   _M(desc,[flags(ctx)+r_tmp0,mem_size]);
  end;
 end;

begin
 Assert(ctx.din.OperCnt=1);
 memop:=classif_memop1(ctx.din);

 with ctx.builder do
  case memop of
   mo_mem:
    begin

     if (his_ro in hint) then
     begin
      //DATA:=[mem]

      build_lea(ctx,1,r_tmp0);
      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      if (mem_size=os8) or
         (mem_size=os4096) or
         (his_rw in hint) then
      begin
       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_in;
      end else
      begin
       op_copyin(ctx,mem_size);

       mem_in;
      end;

     end else
     begin
      //[mem]:=DATA

      build_lea(ctx,1,r_tmp0);
      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      if (mem_size=os8) or
         (mem_size=os4096) or
         (his_rw in hint) then
      begin

       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_out;
      end else
      begin
       op_copyout_before(ctx,link_next,mem_size);

       mem_out;

       op_copyout_after(ctx,link_next,mem_size);
      end;

     end;

    end;

   mo_ctx:
    begin
     mem_size:=ctx.din.Operand[1].Size;
     Assert(mem_size<>os0);

     if (his_ro in hint) or
        (mem_size<>os32) then
     begin
      i:=GetFrameOffset(ctx.din.Operand[1]);
      _M(desc,[r_thrd+i,mem_size]);
     end else
     begin

      new:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

      if (not (his_wo in hint)) or
         (his_ro in hint) then
      begin
       op_load(ctx,new,1);
      end;

      _R(desc,new);

      if not (his_ro in hint) then
      begin
       op_save(ctx,1,fix_size(new));
      end;

     end;

    end;

   else
    Assert(false);
  end;

end;

procedure op_mi(var ctx:t_jit_context2;
                 const desc:t_op_desc;
                 mem:t_jit_leas;
                 imm:Int64;
                 imm_size:TOperandSize);
begin
 with ctx.builder do
  if (imm_size=os8) and
     (get_mem_size(mem)<>os8) and
     (not (not_impl in desc.reg_im8.opt)) then
  begin
   _MI8(desc.reg_im8,mem,imm);
  end else
  begin
   _MI(desc.reg_imm,mem,imm);
  end;
end;

procedure op_ri(var ctx:t_jit_context2;
                 const desc:t_op_desc;
                 reg:TRegValue;
                 imm:Int64;
                 mem_size,imm_size:TOperandSize);
begin
 with ctx.builder do
  if (imm_size=os8) and
     (mem_size<>os8) and
     (not (not_impl in desc.reg_im8.opt)) then
  begin
   _RI8(desc.reg_im8,reg,imm);
  end else
  begin
   _RI(desc.reg_imm,reg,imm);
  end;
end;

procedure op_rmi(var ctx:t_jit_context2;
                 const desc:t_op_desc;
                 reg:TRegValue;mem:t_jit_leas;
                 imm:Int64;
                 imm_size:TOperandSize);
begin
 with ctx.builder do
  if (not_impl in desc.reg_imm.opt) then
  begin
   _RMI8(desc.reg_im8,reg,mem,imm);
  end else
  if (imm_size=os8) and
    (get_mem_size(mem)<>os8) and
    (not (not_impl in desc.reg_im8.opt)) then
  begin
   _RMI8(desc.reg_im8,reg,mem,imm);
  end else
  begin
   _RMI(desc.reg_imm,reg,mem,imm);
  end;
end;

procedure op_rm_or_rmi(var ctx:t_jit_context2;
                       const desc:t_op_desc;
                       reg:TRegValue;mem:t_jit_leas);
var
 mem_size:TOperandSize;

 imm:Int64;
 imm_size:TOperandSize;
begin
 imm:=0;
 if GetTargetOfs(ctx.din,ctx.code,3,imm) then
 begin
  imm_size:=ctx.din.Operand[3].Size;
  mem_size:=get_mem_size(mem);
  Assert(mem_size<>os0);

  op_rmi(ctx,desc,reg,mem,imm,imm_size);
 end else
 begin
  ctx.builder._RM(desc.reg_mem,reg,mem);
 end;
end;

procedure op_mr_or_mri(var ctx:t_jit_context2;
                       const desc:t_op_desc;
                       reg:TRegValue;mem:t_jit_leas);
var
 mem_size:TOperandSize;

 imm:Int64;
 imm_size:TOperandSize;
begin
 imm:=0;
 if GetTargetOfs(ctx.din,ctx.code,3,imm) then
 begin
  imm_size:=ctx.din.Operand[3].Size;
  mem_size:=get_mem_size(mem);
  Assert(mem_size<>os0);

  op_rmi(ctx,desc,reg,mem,imm,imm_size);
 end else
 begin
  ctx.builder._RM(desc.mem_reg,reg,mem);
 end;
end;

procedure op_rri(var ctx:t_jit_context2;
                 const desc:t_op_desc;
                 reg1,reg2:TRegValue;
                 imm:Int64;
                 mem_size,imm_size:TOperandSize);
label
 _mri8;
begin
 with ctx.builder do
  if (not_impl in desc.reg_imm.opt) then
  begin
   goto _mri8;
  end else
  if (imm_size=os8) and
     (mem_size<>os8) and
     (not (not_impl in desc.reg_im8.opt)) then
  begin
   _mri8:
   if (his_mri8 in desc.hint) then
   begin
    //mri8
    _RRI8(desc.reg_im8,reg1,reg2,imm,mem_size);
   end else
   begin
    //rmi8
    _RRI8(desc.reg_im8,reg2,reg1,imm,mem_size); //swapped
   end;
  end else
  begin
   _RRI(desc.reg_imm,reg2,reg1,imm,mem_size); //swapped?
  end;
end;

procedure op_rr(var ctx:t_jit_context2;
                const desc:t_op_desc;
                reg1,reg2:TRegValue;
                mem_size:TOperandSize);
begin
 with ctx.builder do
  if (not_impl in desc.mem_reg.opt) then
  begin
   _RR(desc.reg_mem,reg2,reg1,mem_size); //swapped
  end else
  begin
   _RR(desc.mem_reg,reg1,reg2,mem_size);
  end;
end;

procedure op_rr_or_rri(var ctx:t_jit_context2;
                       const desc:t_op_desc;
                       reg1,reg2:TRegValue;
                       mem_size:TOperandSize);
var
 imm:Int64;
 imm_size:TOperandSize;
begin
 imm:=0;
 if GetTargetOfs(ctx.din,ctx.code,3,imm) then
 begin
  imm_size:=ctx.din.Operand[3].Size;

  op_rri(ctx,desc,reg1,reg2,imm,mem_size,imm_size);
 end else
 begin
  op_rr(ctx,desc,reg1,reg2,mem_size);
 end;
end;

const
 ax_id=0;
 cx_id=1;
 dx_id=2;
 bx_id=3;

type
 t_lo_regi=0..3;
 t_lo_regs=Set of t_lo_regi;

function get_implicit_regs(var ctx:t_jit_context2):t_lo_regs;
begin
 Result:=[];

 case ctx.din.OpCode.Opcode of
  OPcmpxchg:
    case ctx.din.OpCode.Suffix of
     OPSnone: Result:=[ax_id];
     else
              Result:=[ax_id,dx_id];
    end;

  OPmul:
    case ctx.din.OpCode.Suffix of
     OPSnone: Result:=[ax_id,dx_id];
     OPSx_x : Result:=[dx_id];
     else;
    end;

  OPimul:     Result:=[ax_id,dx_id];

  OPdiv:
    case ctx.din.OpCode.Suffix of
     OPSnone: Result:=[ax_id,dx_id];
     else;
    end;

  OPidiv:     Result:=[ax_id,dx_id];

  OPpcmpestri:Result:=[cx_id];
  OPpcmpistri:Result:=[cx_id];

  OProl:      Result:=[cx_id];
  OPror:      Result:=[cx_id];
  OPrcl:      Result:=[cx_id];
  OPrcr:      Result:=[cx_id];
  OPshl:      Result:=[cx_id];
  OPshr:      Result:=[cx_id];
  OPsar:      Result:=[cx_id];

  OPcbw :     Result:=[ax_id];
  OPcwde:     Result:=[ax_id];
  OPcdqe:     Result:=[ax_id];

  OPcwd:      Result:=[ax_id,dx_id];
  OPcdq:      Result:=[ax_id,dx_id];
  OPcqo:      Result:=[ax_id,dx_id];

  else;
 end;

end;

function alloc_tmp_lo(var ctx:t_jit_context2;ASize:TOperandSize):TRegValue;
var
 i,w:Byte;
 excl:t_lo_regs;
begin
 Result:=Default(TRegValue);
 Result.AType:=regGeneral;
 Result.ASize:=ASize;

 excl:=get_implicit_regs(ctx);

 Result.AIndex:=bx_id;

 repeat

  if (Result.AIndex in excl) then
  begin
   Inc(Result.AIndex);
   Continue;
  end;

  if (ctx.din.OperCnt<>0) then
  For i:=1 to ctx.din.OperCnt do
  For w:=0 to 1 do
  begin
   if (ctx.din.Operand[i].RegValue[w].AType in [regGeneral, regGeneralH]) then
   if (ctx.din.Operand[i].RegValue[w].AIndex=Result.AIndex) then
   begin
    if (Result.AIndex=0) then Break;
    Dec(Result.AIndex);
    Continue;
   end;
  end;

  Exit;
 until false;

 Assert(False);
end;

function alloc_tmp_lo(var ctx:t_jit_context2;i:Byte):TRegValue; inline;
begin
 Result:=alloc_tmp_lo(ctx,ctx.din.Operand[i].RegValue[0].ASize);
end;

type
 t_override_ctx=record
  original:TRegValue;
  enable  :Boolean;
  xchg    :Boolean;
 end;

procedure override_mem_in_beg(var ctx:t_jit_context2;
                              var ovr:t_override_ctx;
                              hint:t_op_hint;
                              var reg:TRegValue);
begin
 with ctx.builder do
 begin
  ovr.enable:=is_high(reg);

  if ovr.enable then
  begin
   ovr.original:=reg;

   //get override reg
   reg:=alloc_tmp_lo(ctx,1);

   ovr.xchg:=True;

   if ovr.xchg then
   begin
    //xchg hack
    xchgq(reg,ovr.original);
   end else
   begin
    //save reg
    push(fix_size8(reg));
    //
    if (not (his_wo in hint)) or
       (his_ro in hint) then
    begin
     movq(reg,ovr.original);
    end;
   end;

  end;

 end;
end;

procedure override_mem_in_fin(var ctx:t_jit_context2;
                              var ovr:t_override_ctx;
                              hint:t_op_hint;
                              var reg:TRegValue);
begin
 with ctx.builder do
 begin

  if (ovr.enable) then
  begin

   if ovr.xchg then
   begin
    //xchg hack
    xchgq(ovr.original,reg);
   end else
   begin
    if not (his_ro in hint) then
    begin
     movq(ovr.original,reg);
    end;
    //restore reg
    pop(fix_size8(reg));
   end;

   reg:=ovr.original;
  end;

 end;
end;

procedure override_mem_out_beg(var ctx:t_jit_context2;
                               var ovr:t_override_ctx;
                               hint:t_op_hint;
                               var reg:TRegValue);
begin
 with ctx.builder do
 begin
  ovr.enable:=is_high(reg);

  if ovr.enable then
  begin
   ovr.original:=reg;

   //get override reg
   reg:=alloc_tmp_lo(ctx,2);

   ovr.xchg:=True;

   if ovr.xchg then
   begin
    //xchg hack
    xchgq(reg,ovr.original);
   end else
   begin
    //save reg
    push(fix_size8(reg));
    //
    movq(reg,ovr.original);
   end;

  end;

 end;
end;

procedure override_mem_out_fin(var ctx:t_jit_context2;
                               var ovr:t_override_ctx;
                               hint:t_op_hint;
                               var reg:TRegValue);
begin
 with ctx.builder do
 begin

  if ovr.enable then
  begin

   if ovr.xchg then
   begin
    //xchg hack
    xchgq(ovr.original,reg);
   end else
   begin
    if (his_xchg in hint) then
    begin
     movq(ovr.original,reg);
    end;
    //restore reg
    pop(fix_size8(reg));
   end;

   reg:=ovr.original;
  end;

 end;
end;

procedure op_emit2(var ctx:t_jit_context2;const desc:t_op_desc);
var
 i:Integer;
 memop:t_memop_type2;
 mem_size:TOperandSize;
 link_next:t_jit_i_link;

 imm:Int64;
 imm_size:TOperandSize;

 new1,new2:TRegValue;

 cmp_opr:Boolean;

 new1_load:Boolean;

 ovr:t_override_ctx;

 procedure mem_out;
 begin
  with ctx.builder do
   case memop of
    mo_mem_reg:
      begin
       //input:r14

       new2:=new_reg(ctx.din.Operand[2]);

       override_mem_out_beg(ctx,ovr,desc.hint,new2);

       op_mr_or_mri(ctx,desc,new2,[flags(ctx)+r_tmp0,mem_size]);

       override_mem_out_fin(ctx,ovr,desc.hint,new2);

      end;
    mo_mem_imm:
      begin
       //input:r14

       imm:=0;
       GetTargetOfs(ctx.din,ctx.code,2,imm);

       imm_size:=ctx.din.Operand[2].Size;

       Assert(imm_size<>os64);

       op_mi(ctx,desc,[flags(ctx)+r_tmp0,mem_size],imm,imm_size);

      end;
    mo_mem_ctx:
      begin
       //input:r14

       new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]);

       op_load(ctx,new2,2);

       op_mr_or_mri(ctx,desc,new2,[flags(ctx)+r_tmp0,mem_size]);

       if (his_xchg in desc.hint) then
       begin
        op_save(ctx,2,fix_size(new2));
       end;

      end;
    else
     Assert(False);
   end;
 end;

 procedure mem_in;
 begin
  with ctx.builder do
   case memop of
    mo_reg_mem:
      begin
       //input:r14

       new1:=new_reg(ctx.din.Operand[1]);

       override_mem_in_beg(ctx,ovr,desc.hint,new1);

       op_rm_or_rmi(ctx,desc,new1,[flags(ctx)+r_tmp0,mem_size]);

       override_mem_in_fin(ctx,ovr,desc.hint,new1);

      end;
    mo_ctx_mem:
      begin
       //input:r14

       new1:=new_reg_size(r_tmp1,ctx.din.Operand[1]);

       if (not (his_wo in desc.hint)) or
          (his_ro in desc.hint) then
       begin
        op_load(ctx,new1,1);
       end;

       op_rm_or_rmi(ctx,desc,new1,[flags(ctx)+r_tmp0,mem_size]);

       if not (his_ro in desc.hint) then
       begin
        op_save(ctx,1,fix_size(new1));
       end;

      end;

    //read only swapped

    mo_mem_reg:
     begin
      //input:r14

      new2:=new_reg(ctx.din.Operand[2]);

      override_mem_out_beg(ctx,ovr,desc.hint,new2);

      op_mr_or_mri(ctx,desc,new2,[flags(ctx)+r_tmp0,mem_size]);

      override_mem_out_fin(ctx,ovr,desc.hint,new2);

     end;

    mo_mem_ctx:
     begin
      //input:r14

      new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]);

      op_load(ctx,new2,2);

      op_mr_or_mri(ctx,desc,new2,[flags(ctx)+r_tmp0,mem_size]);

     end;

    mo_mem_imm:
     begin
      //input:r14

      imm:=0;
      GetTargetOfs(ctx.din,ctx.code,2,imm);

      imm_size:=ctx.din.Operand[2].Size;

      Assert(imm_size<>os64);

      op_mi(ctx,desc,[flags(ctx)+r_tmp0,mem_size],imm,imm_size);

     end

    else
     Assert(False);
   end;
 end;

begin
 Assert(ctx.din.OperCnt in [2,3]);
 memop:=classif_memop2(ctx.din);

 with ctx.builder do
  case memop of
   mo_mem_reg,
   mo_reg_mem,
   mo_mem_imm,
   mo_mem_ctx,
   mo_ctx_mem:
     begin
      build_lea(ctx,get_lea_id(memop),r_tmp0);

      mem_size:=ctx.din.Operand[get_lea_id(memop)].Size;
     end;
   else;
  end;

 with ctx.builder do
  case memop of
   mo_mem_reg,
   mo_mem_imm,
   mo_mem_ctx:
     begin
      if (his_ro in desc.hint) then
      begin

       if (mem_size=os8) or
          (his_rw in desc.hint) then
       begin
        op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

        mem_in;
       end else
       begin
        op_copyin(ctx,mem_size);

        mem_in;
       end;

      end else
      if (mem_size=os8) or
         (his_rw in desc.hint) then
      begin
       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_out;
      end else
      begin
       op_copyout_before(ctx,link_next,mem_size);

       mem_out;

       op_copyout_after(ctx,link_next,mem_size);
      end;
     end;

   mo_reg_mem,
   mo_ctx_mem:
     begin
      if (mem_size=os8) or
         (his_rw in desc.hint) then
      begin
       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_in;
      end else
      begin
       op_copyin(ctx,mem_size);

       mem_in;
      end;
     end;

   mo_ctx_reg:
     begin
      new2:=new_reg(ctx.din.Operand[2]);

      override_mem_out_beg(ctx,ovr,desc.hint,new2);

      mem_size:=ctx.din.Operand[1].RegValue[0].ASize;
      Assert(mem_size<>os0);

      if ((his_ro in desc.hint) or (mem_size<>os32)) and
         (not (his_bt in desc.hint)) and
         (not (his_unbs in desc.hint)) and //TODO:is the second parameter always read only?
         (not (not_impl in desc.mem_reg.opt)) then
      begin
       i:=GetFrameOffset(ctx.din.Operand[1]);
       op_mr_or_mri(ctx,desc,new2,[r_thrd+i,mem_size]);
      end else
      begin

       new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

       if (not (his_wo in desc.hint)) or
          (his_ro in desc.hint) then
       begin
        op_load(ctx,new1,1);
       end;

       op_rr_or_rri(ctx,desc,new1,new2,mem_size);

       if not (his_ro in desc.hint) then
       begin
        op_save(ctx,1,fix_size(new1));
       end;

      end;

      override_mem_out_fin(ctx,ovr,desc.hint,new2);

     end;
   mo_reg_ctx:
     begin
      new1:=new_reg(ctx.din.Operand[1]);

      override_mem_in_beg(ctx,ovr,desc.hint,new1);

      mem_size:=ctx.din.Operand[1].RegValue[0].ASize;
      Assert(mem_size<>os0);

      if ((his_ro in desc.hint) or (mem_size<>os32)) and
         (not (his_bt in desc.hint)) and
         (not (his_unbs in desc.hint)) and
         (not (not_impl in desc.reg_mem.opt)) then
      begin
       i:=GetFrameOffset(ctx.din.Operand[2]);
       op_rm_or_rmi(ctx,desc,new1,[r_thrd+i,mem_size]);
      end else
      begin
       new2:=new_reg_size(r_tmp0,ctx.din.Operand[2]);

       op_load(ctx,new2,2);

       op_rr_or_rri(ctx,desc,new1,new2,mem_size);
      end;

      override_mem_in_fin(ctx,ovr,desc.hint,new1);

     end;
   mo_ctx_ctx:
     begin
      mem_size:=ctx.din.Operand[1].RegValue[0].ASize;
      Assert(mem_size<>os0);

      cmp_opr:=cmp_reg(ctx.din.Operand[1],ctx.din.Operand[2]);

      if ((his_ro in desc.hint) or (mem_size<>os32)) and
         (not (his_bt in desc.hint)) and
         (not (his_unbs in desc.hint)) and
         (not (not_impl in desc.mem_reg.opt)) and
         (not cmp_opr) then
      begin
       new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]);

       op_load(ctx,new2,2);

       i:=GetFrameOffset(ctx.din.Operand[1]);
       op_mr_or_mri(ctx,desc,new2,[r_thrd+i,mem_size]);
      end else
      begin

       new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

       new1_load:=False;

       if ((his_xor in desc.hint) and cmp_opr) then
       begin
        //fake load
        new1_load:=True;
       end else
       if (not (his_wo in desc.hint)) or
          (his_ro in desc.hint) then
       begin
        op_load(ctx,new1,1);
        new1_load:=True;
       end;

       if cmp_opr then
       begin
        new2:=new1;

        //preload if reg1=reg2
        if not new1_load then
        begin
         op_load(ctx,new2,2);
        end;
       end else
       begin
        new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]);

        op_load(ctx,new2,2);
       end;

       op_rr_or_rri(ctx,desc,new1,new2,mem_size);

       if not (his_ro in desc.hint) then
       begin
        op_save(ctx,1,fix_size(new1));
       end;
      end;


     end;
   mo_ctx_imm:
     begin
      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      imm:=0;
      GetTargetOfs(ctx.din,ctx.code,2,imm);

      imm_size:=ctx.din.Operand[2].Size;

      if (imm_size=os64) and
         (classif_offset_se64(imm)=os64) then
      begin
       Assert(his_mov in desc.hint);

       new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

       movi64(new1,imm);

       op_save(ctx,1,fix_size(new1));
      end else
      begin

       if ((his_ro in desc.hint) or (mem_size<>os32)) and
          (not (his_bt in desc.hint)) then
       begin
        i:=GetFrameOffset(ctx.din.Operand[1]);
        op_mi(ctx,desc,[r_thrd+i,mem_size],imm,imm_size);
       end else
       begin
        new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

        if (not (his_wo in desc.hint)) or
           (his_ro in desc.hint) then
        begin
         op_load(ctx,new1,1);
        end;

        op_ri(ctx,desc,new1,imm,mem_size,imm_size);

        if not (his_ro in desc.hint) then
        begin
         op_save(ctx,1,fix_size(new1));
        end;

       end;

      end;

     end;
   else
    Assert(false);
  end;

end;

//

procedure op_emit_shift2(var ctx:t_jit_context2;const desc:t_op_shift);
var
 i:Integer;
 memop:t_memop_shift;
 mem_size:TOperandSize;
 link_next:t_jit_i_link;

 imm:Int64;
 imm_size:TOperandSize;

 new:TRegValue;

 procedure mem_out;
 begin
  with ctx.builder do
   case memop of
    mo_mem_imm8:
      begin
       //input:r14

       imm:=0;
       GetTargetOfs(ctx.din,ctx.code,2,imm);

       imm_size:=ctx.din.Operand[2].Size;

       Assert(imm_size=os8);
       Assert(not (not_impl in desc.reg_im8.opt));

       _MI8(desc.reg_im8,[flags(ctx)+r_tmp0,mem_size],imm);
      end;
    mo_mem_cl:
      begin
       //input:r14

       _M(desc.mem__cl,[flags(ctx)+r_tmp0,mem_size]);
      end;
    mo_mem_one:
     begin
      //input:r14

      _M(desc.mem_one,[flags(ctx)+r_tmp0,mem_size]);
     end

    else
     Assert(False);
   end;
 end;

begin
 Assert(ctx.din.OperCnt=2);
 memop:=classif_shift2(ctx.din);

 with ctx.builder do
  case memop of
   mo_mem_imm8,
   mo_mem_cl,
   mo_mem_one:
     begin
      build_lea(ctx,1,r_tmp0);
      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);
     end;
   else;
  end;

 with ctx.builder do
  case memop of
   mo_mem_imm8,
   mo_mem_cl,
   mo_mem_one:
     begin
      if true then
      begin
       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_out;
      end else
      begin
       op_copyout_before(ctx,link_next,mem_size);

       mem_out;

       op_copyout_after(ctx,link_next,mem_size);
      end;
     end;

   mo_ctx_imm8:
     begin
      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      imm:=0;
      GetTargetOfs(ctx.din,ctx.code,2,imm);

      imm_size:=ctx.din.Operand[2].Size;
      Assert(imm_size=os8);

      if (mem_size<>os32) then
      begin
       i:=GetFrameOffset(ctx.din.Operand[1]);
       _MI8(desc.reg_im8,[r_thrd+i,mem_size],imm);
      end else
      begin
       new:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

       op_load(ctx,new,1);

       _RI8(desc.reg_im8,new,imm);

       op_save(ctx,1,fix_size(new));
      end;

     end;
    mo_ctx_cl:
     begin
      mem_size:=ctx.din.Operand[1].Size;

      if (mem_size<>os32) then
      begin
       i:=GetFrameOffset(ctx.din.Operand[1]);
       _M(desc.mem__cl,[r_thrd+i,mem_size]);
      end else
      begin
       new:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

       op_load(ctx,new,1);

       _R(desc.mem__cl,new);

       op_save(ctx,1,fix_size(new));
      end;

     end;
    mo_ctx_one:
     begin
      mem_size:=ctx.din.Operand[1].Size;

      if (mem_size<>os32) then
      begin
       i:=GetFrameOffset(ctx.din.Operand[1]);
       _M(desc.mem_one,[r_thrd+i,mem_size]);
      end else
      begin
       new:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

       op_load(ctx,new,1);

       _R(desc.mem_one,new);

       op_save(ctx,1,fix_size(new));
      end;

     end;

   else
    Assert(false);
  end;

end;

//

procedure op_emit_shift3(var ctx:t_jit_context2;const desc:t_op_shift);
var
 memop:t_memop_shift;
 mem_size:TOperandSize;
 link_next:t_jit_i_link;

 imm:Int64;
 imm_size:TOperandSize;

 new1,new2:TRegValue;

 procedure mem_out;
 begin
  with ctx.builder do
   case memop of
    mo_mem_imm8:
      begin
       //input:r14

       imm:=0;
       GetTargetOfs(ctx.din,ctx.code,3,imm);

       imm_size:=ctx.din.Operand[3].Size;

       Assert(imm_size=os8);
       Assert(not (not_impl in desc.reg_im8.opt));

       if is_preserved(ctx.din.Operand[2]) then
       begin
        //mem_ctx_imm
        new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]);

        op_load(ctx,new2,2);
       end else
       begin
        //mem_reg_imm
        new2:=new_reg(ctx.din.Operand[2]);
       end;

       Assert(not is_high(new2));

       _RMI8(desc.reg_im8,new2,[flags(ctx)+r_tmp0],imm);
      end;
    mo_mem_cl:
      begin
       //input:r14

       if is_preserved(ctx.din.Operand[2]) then
       begin
        //mem_ctx_cl
        new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]);

        op_load(ctx,new2,2);
       end else
       begin
        //mem_reg_cl
        new2:=new_reg(ctx.din.Operand[2]);
       end;

       Assert(not is_high(new2));

       _RM(desc.mem__cl,new2,[flags(ctx)+r_tmp0]);
      end;

    else
     Assert(False);
   end;
 end;

begin
 Assert(ctx.din.OperCnt=3);
 memop:=classif_shift3(ctx.din);

 with ctx.builder do
  case memop of
   mo_mem_imm8,
   mo_mem_cl:
     begin
      build_lea(ctx,1,r_tmp0);
      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);
     end;
   else;
  end;

 with ctx.builder do
  case memop of
   mo_mem_imm8,
   mo_mem_cl:
     begin
      if true then
      begin
       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_out;
      end else
      begin
       op_copyout_before(ctx,link_next,mem_size);

       mem_out;

       op_copyout_after(ctx,link_next,mem_size);
      end;
     end;

   mo_ctx_imm8:
     begin
      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      imm:=0;
      GetTargetOfs(ctx.din,ctx.code,3,imm);

      imm_size:=ctx.din.Operand[3].Size;
      Assert(imm_size=os8);

      if is_preserved(ctx.din.Operand[2]) then
      begin
       //ctx_ctx_imm

       new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

       op_load(ctx,new1,1);

       if cmp_reg(ctx.din.Operand[1],ctx.din.Operand[2]) then
       begin
        new2:=new1;
       end else
       begin
        new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]);

        op_load(ctx,new2,2);
       end;

      end else
      begin
       //ctx_reg_imm
       new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);
       new2:=new_reg(ctx.din.Operand[2]);

       op_load(ctx,new1,1);
      end;

      Assert(not is_high(new1));
      Assert(not is_high(new2));

      _RRI8(desc.reg_im8,new1,new2,imm,mem_size);

      op_save(ctx,1,fix_size(new1));
     end;
    mo_ctx_cl:
     begin
      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      if is_preserved(ctx.din.Operand[2]) then
      begin
       //ctx_ctx_cl

       new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

       op_load(ctx,new1,1);

       if cmp_reg(ctx.din.Operand[1],ctx.din.Operand[2]) then
       begin
        new2:=new1;
       end else
       begin
        new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]);

        op_load(ctx,new2,2);
       end;

      end else
      begin
       //ctx_reg_cl
       new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);
       new2:=new_reg(ctx.din.Operand[2]);

       op_load(ctx,new1,1);
      end;

      Assert(not is_high(new1));
      Assert(not is_high(new2));

      _RR(desc.mem__cl,new1,new2,mem_size);

      op_save(ctx,1,fix_size(new1));
     end;

    mo_reg_imm8:
     begin
      //reg_ctx_imm

      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      imm:=0;
      GetTargetOfs(ctx.din,ctx.code,3,imm);

      imm_size:=ctx.din.Operand[3].Size;
      Assert(imm_size=os8);

      new1:=new_reg(ctx.din.Operand[1]);
      new2:=new_reg_size(r_tmp0,ctx.din.Operand[2]);

      op_load(ctx,new2,2);

      Assert(not is_high(new1));
      Assert(not is_high(new2));

      _RRI8(desc.reg_im8,new1,new2,imm,mem_size);
     end;

    mo_reg_cl:
     begin
      //reg_ctx_cl

      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      new1:=new_reg(ctx.din.Operand[1]);
      new2:=new_reg_size(r_tmp0,ctx.din.Operand[2]);

      op_load(ctx,new2,2);

      Assert(not is_high(new1));
      Assert(not is_high(new2));

      _RR(desc.mem__cl,new1,new2,mem_size);
     end

   else
    Assert(false);
  end;

end;

//

procedure op_emit_avx1(var ctx:t_jit_context2;const desc:t_op_type;hint:t_op_hint);
var
 i:Integer;
 memop:t_memop_type1;
 mem_size:TOperandSize;
 link_next:t_jit_i_link;

 new:TRegValue;

 procedure mem_out;
 begin
  with ctx.builder do
  begin
   //input:r14

   _VM(desc,[flags(ctx)+r_tmp0,mem_size]);
  end;
 end;

 procedure mem_in;
 begin
  with ctx.builder do
  begin
   //input:r14

   _VM(desc,[flags(ctx)+r_tmp0,mem_size]);
  end;
 end;

begin
 Assert(ctx.din.OperCnt=1);
 memop:=classif_memop1(ctx.din);

 with ctx.builder do
  case memop of
   mo_mem:
    begin

     if (his_ro in hint) then
     begin
      //DATA:=[mem]

      build_lea(ctx,1,r_tmp0);
      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      if (mem_size=os8) or
         (his_rw in hint) then
      begin
       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_in;
      end else
      begin
       op_copyin(ctx,mem_size);

       mem_in;
      end;

     end else
     begin
      //[mem]:=DATA

      build_lea(ctx,1,r_tmp0);
      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      if (mem_size=os8) or
         (his_rw in hint) then
      begin

       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_out;
      end else
      begin
       op_copyout_before(ctx,link_next,mem_size);

       mem_out;

       op_copyout_after(ctx,link_next,mem_size);
      end;

     end;

    end;

   mo_ctx:
    begin
     mem_size:=ctx.din.Operand[1].Size;
     Assert(mem_size<>os0);

     if (his_ro in hint) or
        (mem_size<>os32) then
     begin
      i:=GetFrameOffset(ctx.din.Operand[1]);
      _VM(desc,[r_thrd+i,mem_size]);
     end else
     begin

      new:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

      if (not (his_wo in hint)) or
         (his_ro in hint) then
      begin
       op_load(ctx,new,1);
      end;

      Assert(false,'_V(desc,new);');

      if not (his_ro in hint) then
      begin
       op_save(ctx,1,fix_size(new));
      end;

     end;

    end;

   else
    Assert(false);
  end;

end;

//

procedure op_emit_avx2_rr(var ctx:t_jit_context2;const desc:t_op_type);
var
 new1,new2:TRegValue;
begin
 Assert(ctx.din.OperCnt=2);
 if is_preserved(ctx.din.Operand[1]) then
 begin
  with ctx.builder do
  begin
   new1:=r_tmp0;
   new2:=new_reg(ctx.din.Operand[2]);

   _VV(desc,new1,new2,new2.ASize);

   op_save(ctx,1,fix_size(new1));
  end;
 end else
 begin
  Assert(False);
 end;
end;

//

procedure op_emit_avx2(var ctx:t_jit_context2;const desc:t_op_desc);
var
 i:Integer;
 memop:t_memop_type2;
 mem_size:TOperandSize;
 link_next:t_jit_i_link;

 new1,new2:TRegValue;

 procedure mem_out;
 begin
  with ctx.builder do
  begin
   //input:r14

   new2:=new_reg(ctx.din.Operand[2]);
   _VM(desc.mem_reg,new2,[flags(ctx)+r_tmp0,mem_size]);
  end;
 end;

 procedure mem_in;
 begin
  with ctx.builder do
  begin
   case memop of
    mo_reg_mem:
      begin
       //input:r14

       new1:=new_reg(ctx.din.Operand[1]);
       _VM(desc.reg_mem,new1,[flags(ctx)+r_tmp0,mem_size]);
      end;

    mo_ctx_mem:
      begin
       //input:r14

       new1:=new_reg_size(r_tmp1,ctx.din.Operand[1]);

       if (not (his_wo in desc.hint)) or
          (his_ro in desc.hint) then
       begin
        op_load(ctx,new1,1);
       end;

       _VM(desc.reg_mem,new1,[flags(ctx)+r_tmp0,mem_size]);

       if not (his_ro in desc.hint) then
       begin
        op_save(ctx,1,fix_size(new1));
       end;

      end;
    else
     Assert(false);
    end;
  end;
 end;

begin
 Assert(ctx.din.OperCnt=2);
 memop:=classif_memop2(ctx.din);

 with ctx.builder do
  case memop of
   mo_mem_reg,
   mo_reg_mem,
   mo_ctx_mem:
     begin
      build_lea(ctx,get_lea_id(memop),r_tmp0);
      mem_size:=ctx.din.Operand[get_lea_id(memop)].Size;
      Assert(mem_size<>os0);
     end;
   else;
  end;

 with ctx.builder do
  case memop of
   mo_mem_reg:
     begin
      if (his_align in desc.hint) then
      begin
       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_out;
      end else
      begin
       op_copyout_before(ctx,link_next,mem_size);

       mem_out;

       op_copyout_after(ctx,link_next,mem_size);
      end;
     end;

   mo_reg_mem,
   mo_ctx_mem:
     begin
      if (his_align in desc.hint) then
      begin
       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_in;
      end else
      begin
       op_copyin(ctx,mem_size);

       mem_in;
      end;
     end;

   mo_ctx_reg:
     begin
      new1:=new_reg(ctx.din.Operand[2]);

      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      if (not (not_impl in desc.mem_reg.opt)) then
      begin

       if ((his_ro in desc.hint) or (mem_size<>os32)) then
       begin
        i:=GetFrameOffset(ctx.din.Operand[1]);
        _VM(desc.mem_reg,new1,[r_thrd+i,mem_size]);
       end else
       begin
        new2:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

        if (not (his_wo in desc.hint)) or
           (his_ro in desc.hint) then
        begin
         op_load(ctx,new2,1);
        end;

        _VV(desc.mem_reg,new1,new2,mem_size);

        if not (his_ro in desc.hint) then
        begin
         op_save(ctx,1,fix_size(new2));
        end;
       end;

      end else
      begin
       new2:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

       if (not (his_wo in desc.hint)) or
          (his_ro in desc.hint) then
       begin
        op_load(ctx,new2,1);
       end;

       _VV(desc.reg_mem,new2,new1,mem_size); //swapped

       if not (his_ro in desc.hint) then
       begin
        op_save(ctx,1,fix_size(new2));
       end;
      end;

     end;
   mo_reg_ctx:
     begin
      new1:=new_reg(ctx.din.Operand[1]);

      mem_size:=ctx.din.Operand[2].Size;
      Assert(mem_size<>os0);

      i:=GetFrameOffset(ctx.din.Operand[2]);
      _VM(desc.reg_mem,new1,[r_thrd+i,mem_size]);
     end;

   else
    Assert(false);
  end;

end;

procedure op_emit_avx3(var ctx:t_jit_context2;const desc:t_op_type);
var
 mem_size:TOperandSize;
 link_next:t_jit_i_link;

 i:Integer;
 imm:Int64;

 new1,new2:TRegValue;

 procedure mem_out;
 begin
  with ctx.builder do
  begin
   //input:r14

   mem_size:=ctx.din.Operand[1].Size;
   Assert(mem_size<>os0);

   new1:=new_reg(ctx.din.Operand[2]);
   new2:=new_reg(ctx.din.Operand[3]);

   imm:=0;
   if GetTargetOfs(ctx.din,ctx.code,4,imm) then
   begin
    _VVMI8(desc,new2,new1,[flags(ctx)+r_tmp0,mem_size],imm);
   end else
   begin
    _VVM(desc,new2,new1,[flags(ctx)+r_tmp0,mem_size]); //[mem],arg2,arg3 -> arg3,arg2,[mem]
   end;

  end;
 end;

 procedure mem_in;
 begin
  with ctx.builder do
  begin
   //input:r14

   new1:=new_reg(ctx.din.Operand[1]);
   new2:=new_reg(ctx.din.Operand[2]);

   imm:=0;
   if GetTargetOfs(ctx.din,ctx.code,4,imm) then
   begin
    _VVMI8(desc,new1,new2,[flags(ctx)+r_tmp0,mem_size],imm);
   end else
   begin
    _VVM(desc,new1,new2,[flags(ctx)+r_tmp0,mem_size]);
   end;

  end;
 end;

begin
 Assert(ctx.din.OperCnt in [3,4]);
 if (ofMemory in ctx.din.Operand[3].Flags) then
 begin
  //mo_reg_reg_mem

  build_lea(ctx,3,r_tmp0);
  mem_size:=ctx.din.Operand[3].Size;
  Assert(mem_size<>os0);

  with ctx.builder do
  begin

   if false then
   begin
    op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

    mem_in;
   end else
   begin
    op_copyin(ctx,mem_size);

    mem_in;
   end;

  end;

 end else
 if is_preserved(ctx.din.Operand[3]) then
 begin
  //mo_reg_reg_ctx

  with ctx.builder do
  begin

   mem_size:=ctx.din.Operand[3].Size;
   Assert(mem_size<>os0);

   new1:=new_reg(ctx.din.Operand[1]);
   new2:=new_reg(ctx.din.Operand[2]);

   imm:=0;
   if GetTargetOfs(ctx.din,ctx.code,4,imm) then
   begin
    i:=GetFrameOffset(ctx.din.Operand[3]);
    _VVMI8(desc,new1,new2,[r_thrd+i,mem_size],imm);
   end else
   begin
    i:=GetFrameOffset(ctx.din.Operand[3]);
    _VVM(desc,new1,new2,[r_thrd+i,mem_size]);
   end;

  end;

 end else
 if (ofMemory in ctx.din.Operand[1].Flags) then
 begin
  //mo_mem_reg_reg

  build_lea(ctx,1,r_tmp0);
  mem_size:=ctx.din.Operand[1].Size;
  Assert(mem_size<>os0);

  with ctx.builder do
  begin

   if false then
   begin
    op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

    mem_out;
   end else
   begin
    op_copyout_before(ctx,link_next,mem_size);

    mem_out;

    op_copyout_after(ctx,link_next,mem_size);
   end;

  end;

 end else
 begin
  Assert(False);
 end;
end;

//rri,mri
procedure op_emit_avx3_imm8(var ctx:t_jit_context2;const desc:t_op_avx3_imm);
var
 memop:t_memop_type2;
 mem_size:TOperandSize;
 link_next:t_jit_i_link;

 new1,new2:TRegValue;

 imm:Int64;

 procedure mem_out;
 begin
  with ctx.builder do
  begin
   //input:r14

   //mem_reg

   new2:=new_reg(ctx.din.Operand[2]);

   imm:=0;
   GetTargetOfs(ctx.din,ctx.code,3,imm);

   _VMI8(desc.mri,new2,[flags(ctx)+r_tmp0,mem_size],imm);
  end;
 end;


 procedure mem_in;
 begin
  with ctx.builder do
  begin
   //input:r14

   //reg_mem

   new1:=new_reg(ctx.din.Operand[1]);

   imm:=0;
   GetTargetOfs(ctx.din,ctx.code,3,imm);

   _VMI8(desc.rmi,new1,[flags(ctx)+r_tmp0,mem_size],imm);
  end;
 end;


begin
 Assert(ctx.din.OperCnt=3);
 memop:=classif_memop2(ctx.din);

 with ctx.builder do
  case memop of
   mo_mem_reg:
    begin
     build_lea(ctx,1,r_tmp0);
     mem_size:=ctx.din.Operand[1].Size;
     Assert(mem_size<>os0);

     if (mem_size=os8) then
     begin
      op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

      mem_out;
     end else
     begin
      op_copyout_before(ctx,link_next,mem_size);

      mem_out;

      op_copyout_after(ctx,link_next,mem_size);
     end;
    end;

   mo_reg_mem:
     begin
      build_lea(ctx,2,r_tmp0);
      mem_size:=ctx.din.Operand[2].Size;
      Assert(mem_size<>os0);

      if (mem_size=os8) then
      begin
       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_in;
      end else
      begin
       op_copyin(ctx,mem_size);

       mem_in;
      end;
     end;

   mo_ctx_reg:
    begin
     mem_size:=ctx.din.Operand[1].Size;
     Assert(mem_size<>os0);

     new2:=new_reg(ctx.din.Operand[2]);

     imm:=0;
     GetTargetOfs(ctx.din,ctx.code,3,imm);

     new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);

     //mri

     if (not (not_impl in desc.mri.opt)) then
     begin
      _VVI8(desc.mri,new2,new1,imm,mem_size); //swapped
     end else
     begin
      _VVI8(desc.rmi,new1,new2,imm,mem_size);
     end;

     op_save(ctx,1,fix_size(new1));
    end;

   mo_reg_ctx:
    begin
     mem_size:=ctx.din.Operand[2].Size;
     Assert(mem_size<>os0);

     new1:=new_reg(ctx.din.Operand[1]);
     new2:=new_reg_size(r_tmp0,ctx.din.Operand[2]);

     //rmi

     op_load(ctx,new2,2);

     _VVI8(desc.rmi,new1,new2,imm,mem_size);
    end


   else
    Assert(false);
  end;

end;

procedure op_emit_avx_F3(var ctx:t_jit_context2;const desc:t_op_type);
var
 memop:t_memop_type2;
 mem_size:TOperandSize;

 new1,new2:TRegValue;

 procedure mem_in;
 begin
  with ctx.builder do
  begin
   case memop of
    mo_reg_mem:
     begin
      //input:r14

      new1:=new_reg(ctx.din.Operand[1]);
      _VM_F3(desc,new1,[flags(ctx)+r_tmp0,mem_size]);
     end;
    mo_ctx_mem:
     begin
      //input:r14

      //load?

      new1:=new_reg_size(r_tmp1,ctx.din.Operand[1]);

      _VM_F3(desc,new1,[flags(ctx)+r_tmp0,mem_size]);

      op_save(ctx,1,fix_size(new1));
     end;
    else
     Assert(False);
   end;
  end;
 end;

begin
 Assert(ctx.din.OperCnt=2);
 memop:=classif_memop2(ctx.din);

 with ctx.builder do
  case memop of
   mo_reg_mem,
   mo_ctx_mem:
     begin
      build_lea(ctx,get_lea_id(memop),r_tmp0);
      mem_size:=ctx.din.Operand[get_lea_id(memop)].Size;
      Assert(mem_size<>os0);
     end;
   else;
  end;

 with ctx.builder do
  case memop of
   mo_reg_mem,
   mo_ctx_mem:
     begin
      if false then
      begin
       op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

       mem_in;
      end else
      begin
       op_copyin(ctx,mem_size);

       mem_in;
      end;
     end;

   mo_ctx_reg:
     begin
      new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);
      new2:=new_reg(ctx.din.Operand[2]);

      mem_size:=ctx.din.Operand[1].Size;
      Assert(mem_size<>os0);

      //load?

      _VV_F3(desc,new1,new2,mem_size);

      op_save(ctx,1,fix_size(new1));
     end;
   mo_reg_ctx:
     begin
      new1:=new_reg(ctx.din.Operand[1]);
      new2:=new_reg_size(r_tmp0,ctx.din.Operand[2]);

      mem_size:=ctx.din.Operand[2].Size;
      Assert(mem_size<>os0);

      op_load(ctx,new2,2);

      _VV_F3(desc,new1,new2,mem_size);
     end;

   mo_ctx_ctx:
     begin
      new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);
      new2:=new_reg_size(r_tmp1,ctx.din.Operand[2]);

      //load?

      op_load(ctx,new2,2);

      mem_size:=ctx.din.Operand[1].RegValue[0].ASize;
      Assert(mem_size<>os0);

      _VV_F3(desc,new1,new2,mem_size);

      op_save(ctx,1,fix_size(new1));
     end;

   else
    Assert(false);
  end;

end;

//rrrr,rrmr
procedure op_emit_avx4(var ctx:t_jit_context2;const desc:t_op_type);
var
 mem_size:TOperandSize;

 new1,new2,new3:TRegValue;

 procedure mem_in;
 begin
  with ctx.builder do
  begin
   //input:r14

   new1:=new_reg(ctx.din.Operand[1]);
   new2:=new_reg(ctx.din.Operand[2]);
   new3:=new_reg(ctx.din.Operand[4]);

   _VVMV(desc,new1,new2,[flags(ctx)+r_tmp0,mem_size],new3);
  end;
 end;

begin
 Assert(ctx.din.OperCnt=4);

 with ctx.builder do
  if is_memory(ctx.din.Operand[3]) then
  begin
   build_lea(ctx,3,r_tmp0);
   mem_size:=ctx.din.Operand[3].Size;
   Assert(mem_size<>os0);

   if false then
   begin
    op_uplift(ctx,r_tmp0,mem_size); //in/out:r14

    mem_in;
   end else
   begin
    op_copyin(ctx,mem_size);

    mem_in;
   end;
  end else
  begin
   Assert(false);
  end;
end;

//

procedure op_emit_bmi_rrm(var ctx:t_jit_context2;const desc:t_op_type);
var
 mem_size:TOperandSize;

 new1,new2,new3:TRegValue;

 pr_res,pr_mem:Boolean;

 tmp_count:Byte;

 function tmp_alloc:TRegValue; inline;
 begin
  case tmp_count of
   0:Result:=r_tmp0;
   1:Result:=r_tmp1;
   2:Result:=r_thrd;
   else
    Assert(False,'tmp_alloc');
  end;
  Inc(tmp_count);
 end;

begin
 Assert(ctx.din.OperCnt=3);

 tmp_count:=0;

 with ctx.builder do
 begin

  pr_res:=is_preserved(ctx.din.Operand[1]);
  pr_mem:=is_memory   (ctx.din.Operand[3]);

  //preload addr first
  if pr_mem then
  begin
   mem_size:=ctx.din.Operand[3].Size;
   Assert(mem_size<>os0);

   new3:=tmp_alloc;

   build_lea(ctx,3,new3);

   op_copyin(ctx,mem_size);
  end;

  if pr_res then
  begin
   //The value is always overwritten so it does not need to be allocated.
   new1:=new_reg_size(r_tmp0,ctx.din.Operand[1]);
   //not need load result
  end else
  begin
   new1:=new_reg(ctx.din.Operand[1]);
  end;

  if is_preserved(ctx.din.Operand[2]) then
  begin
   new2:=new_reg_size(tmp_alloc,ctx.din.Operand[2]);
   //
   op_load(ctx,new2,2);
  end else
  begin
   new2:=new_reg(ctx.din.Operand[2]);
  end;

  if pr_mem then
  begin
   _VVM(desc,new1,new2,[new3]); //1 2 3
  end else
  if is_preserved(ctx.din.Operand[3]) then
  begin
   new3:=new_reg_size(tmp_alloc,ctx.din.Operand[3]);
   //
   op_load(ctx,new3,3);
   //
   _VVV(desc,new1,new2,new3,new1.ASize); //1 2 3
  end else
  begin
   new3:=new_reg(ctx.din.Operand[3]);
   //
   _VVV(desc,new1,new2,new3,new1.ASize); //1 2 3
  end;

  if (tmp_count=3) then
  begin
   //restore jit_frame
   movq(r13,[GS +teb_thread]);
   leaq(r13,[r13+jit_frame_offset]);
  end;

  //store result
  if pr_res then
  begin
   op_save(ctx,1,fix_size(new1));
  end;

 end;
end;

//

function get_instruction_info(addr:Pointer):t_instruction_info;
type
 t_data_16=array[0..15] of Byte;
var
 ptr:Pointer;
 data:t_data_16;

 dis:TX86Disassembler;
 din:TInstruction;

 i:Byte;
begin
 dis:=Default(TX86Disassembler);
 din:=Default(TInstruction);

 data:=Default(t_data_16);
 md_copyin(addr,@data,16,nil);

 ptr:=@data;

 dis.Disassemble(dm64,ptr,din);

 Result.code_size:=(ptr-addr);
 Result.mema_size:=0;

 if (din.OperCnt<>0) then
 For i:=1 to din.OperCnt do
 if is_memory(din.Operand[i]) then
 begin
  Result.mema_size:=OPERAND_BYTES[din.Operand[i].Size];
  Exit;
 end;

 if (din.OpCode.Prefix=OPPnone) then
 begin
  case din.OpCode.Opcode of
   OPpush,
   OPpop,
   OPpushf,
   OPpopf:Result.mema_size:=OPERAND_BYTES[din.Operand[1].Size];
   //
   OPins ,
   OPouts,
   OPmovs,
   OPlods,
   OPstos,
   OPcmps,
   OPscas:
      case din.OpCode.Suffix of
       OPSx_b:begin Result.mema_size:=1; Exit; end;
       OPSx_w:begin Result.mema_size:=2; Exit; end;
       OPSx_d:begin Result.mema_size:=4; Exit; end;
       OPSx_q:begin Result.mema_size:=8; Exit; end;
       else;
      end;
   else;
  end;
 end;

 print_disassemble(@data,Result.code_size);

 Assert(false,'get_instruction_info');
end;

//

end.




