diff --git a/hw/intc/riscv_aplic.c b/hw/intc/riscv_aplic.c index 6a7fbfa861..fc5df0d598 100644 --- a/hw/intc/riscv_aplic.c +++ b/hw/intc/riscv_aplic.c @@ -488,6 +488,7 @@ static uint32_t riscv_aplic_idc_claimi(RISCVAPLICState *aplic, uint32_t idc) if (!topi) { aplic->iforce[idc] = 0; + riscv_aplic_idc_update(aplic, idc); return 0; } diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c index 72a55b8af1..d171e74f7b 100644 --- a/hw/riscv/virt.c +++ b/hw/riscv/virt.c @@ -711,6 +711,8 @@ static void create_fdt_sockets(RISCVVirtState *s, const MemMapEntry *memmap, qemu_fdt_add_subnode(ms->fdt, "/cpus"); qemu_fdt_setprop_cell(ms->fdt, "/cpus", "timebase-frequency", + kvm_enabled() ? + kvm_riscv_get_timebase_frequency(first_cpu) : RISCV_ACLINT_DEFAULT_TIMEBASE_FREQ); qemu_fdt_setprop_cell(ms->fdt, "/cpus", "#size-cells", 0x0); qemu_fdt_setprop_cell(ms->fdt, "/cpus", "#address-cells", 0x1); diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index c160b9216b..36e3e5fdaf 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -102,10 +102,10 @@ const RISCVIsaExtData isa_edata_arr[] = { ISA_EXT_DATA_ENTRY(zicbom, PRIV_VERSION_1_12_0, ext_zicbom), ISA_EXT_DATA_ENTRY(zicbop, PRIV_VERSION_1_12_0, ext_zicbop), ISA_EXT_DATA_ENTRY(zicboz, PRIV_VERSION_1_12_0, ext_zicboz), - ISA_EXT_DATA_ENTRY(ziccamoa, PRIV_VERSION_1_11_0, ext_always_enabled), - ISA_EXT_DATA_ENTRY(ziccif, PRIV_VERSION_1_11_0, ext_always_enabled), - ISA_EXT_DATA_ENTRY(zicclsm, PRIV_VERSION_1_11_0, ext_always_enabled), - ISA_EXT_DATA_ENTRY(ziccrse, PRIV_VERSION_1_11_0, ext_always_enabled), + ISA_EXT_DATA_ENTRY(ziccamoa, PRIV_VERSION_1_11_0, has_priv_1_11), + ISA_EXT_DATA_ENTRY(ziccif, PRIV_VERSION_1_11_0, has_priv_1_11), + ISA_EXT_DATA_ENTRY(zicclsm, PRIV_VERSION_1_11_0, has_priv_1_11), + ISA_EXT_DATA_ENTRY(ziccrse, PRIV_VERSION_1_11_0, has_priv_1_11), ISA_EXT_DATA_ENTRY(zicond, PRIV_VERSION_1_12_0, ext_zicond), ISA_EXT_DATA_ENTRY(zicntr, PRIV_VERSION_1_12_0, ext_zicntr), ISA_EXT_DATA_ENTRY(zicsr, PRIV_VERSION_1_10_0, ext_zicsr), @@ -114,7 +114,7 @@ const RISCVIsaExtData isa_edata_arr[] = { ISA_EXT_DATA_ENTRY(zihintpause, PRIV_VERSION_1_10_0, ext_zihintpause), ISA_EXT_DATA_ENTRY(zihpm, PRIV_VERSION_1_12_0, ext_zihpm), ISA_EXT_DATA_ENTRY(zmmul, PRIV_VERSION_1_12_0, ext_zmmul), - ISA_EXT_DATA_ENTRY(za64rs, PRIV_VERSION_1_12_0, ext_always_enabled), + ISA_EXT_DATA_ENTRY(za64rs, PRIV_VERSION_1_12_0, has_priv_1_11), ISA_EXT_DATA_ENTRY(zaamo, PRIV_VERSION_1_12_0, ext_zaamo), ISA_EXT_DATA_ENTRY(zacas, PRIV_VERSION_1_12_0, ext_zacas), ISA_EXT_DATA_ENTRY(zalrsc, PRIV_VERSION_1_12_0, ext_zalrsc), @@ -179,12 +179,12 @@ const RISCVIsaExtData isa_edata_arr[] = { ISA_EXT_DATA_ENTRY(smepmp, PRIV_VERSION_1_12_0, ext_smepmp), ISA_EXT_DATA_ENTRY(smstateen, PRIV_VERSION_1_12_0, ext_smstateen), ISA_EXT_DATA_ENTRY(ssaia, PRIV_VERSION_1_12_0, ext_ssaia), - ISA_EXT_DATA_ENTRY(ssccptr, PRIV_VERSION_1_11_0, ext_always_enabled), + ISA_EXT_DATA_ENTRY(ssccptr, PRIV_VERSION_1_11_0, has_priv_1_11), ISA_EXT_DATA_ENTRY(sscofpmf, PRIV_VERSION_1_12_0, ext_sscofpmf), - ISA_EXT_DATA_ENTRY(sscounterenw, PRIV_VERSION_1_12_0, ext_always_enabled), + ISA_EXT_DATA_ENTRY(sscounterenw, PRIV_VERSION_1_12_0, has_priv_1_12), ISA_EXT_DATA_ENTRY(sstc, PRIV_VERSION_1_12_0, ext_sstc), - ISA_EXT_DATA_ENTRY(sstvala, PRIV_VERSION_1_12_0, ext_always_enabled), - ISA_EXT_DATA_ENTRY(sstvecd, PRIV_VERSION_1_12_0, ext_always_enabled), + ISA_EXT_DATA_ENTRY(sstvala, PRIV_VERSION_1_12_0, has_priv_1_12), + ISA_EXT_DATA_ENTRY(sstvecd, PRIV_VERSION_1_12_0, has_priv_1_12), ISA_EXT_DATA_ENTRY(svade, PRIV_VERSION_1_11_0, ext_svade), ISA_EXT_DATA_ENTRY(svadu, PRIV_VERSION_1_12_0, ext_svadu), ISA_EXT_DATA_ENTRY(svinval, PRIV_VERSION_1_12_0, ext_svinval), @@ -1575,11 +1575,6 @@ const RISCVCPUMultiExtConfig riscv_cpu_experimental_exts[] = { DEFINE_PROP_END_OF_LIST(), }; -#define ALWAYS_ENABLED_FEATURE(_name) \ - {.name = _name, \ - .offset = CPU_CFG_OFFSET(ext_always_enabled), \ - .enabled = true} - /* * 'Named features' is the name we give to extensions that we * don't want to expose to users. They are either immutable @@ -1590,23 +1585,6 @@ const RISCVCPUMultiExtConfig riscv_cpu_experimental_exts[] = { const RISCVCPUMultiExtConfig riscv_cpu_named_features[] = { MULTI_EXT_CFG_BOOL("zic64b", ext_zic64b, true), - /* - * cache-related extensions that are always enabled - * in TCG since QEMU RISC-V does not have a cache - * model. - */ - ALWAYS_ENABLED_FEATURE("za64rs"), - ALWAYS_ENABLED_FEATURE("ziccif"), - ALWAYS_ENABLED_FEATURE("ziccrse"), - ALWAYS_ENABLED_FEATURE("ziccamoa"), - ALWAYS_ENABLED_FEATURE("zicclsm"), - ALWAYS_ENABLED_FEATURE("ssccptr"), - - /* Other named features that TCG always implements */ - ALWAYS_ENABLED_FEATURE("sstvecd"), - ALWAYS_ENABLED_FEATURE("sstvala"), - ALWAYS_ENABLED_FEATURE("sscounterenw"), - DEFINE_PROP_END_OF_LIST(), }; diff --git a/target/riscv/cpu_cfg.h b/target/riscv/cpu_cfg.h index 2040b90da0..cb750154bd 100644 --- a/target/riscv/cpu_cfg.h +++ b/target/riscv/cpu_cfg.h @@ -130,10 +130,12 @@ struct RISCVCPUConfig { bool ext_zic64b; /* - * Always 'true' boolean for named features - * TCG always implement/can't be disabled. + * Always 'true' booleans for named features + * TCG always implement/can't be user disabled, + * based on spec version. */ - bool ext_always_enabled; + bool has_priv_1_12; + bool has_priv_1_11; /* Vendor-specific custom extensions */ bool ext_xtheadba; diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c index ce7322011d..fc090d729a 100644 --- a/target/riscv/cpu_helper.c +++ b/target/riscv/cpu_helper.c @@ -1315,7 +1315,7 @@ bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size, bool two_stage_lookup = mmuidx_2stage(mmu_idx); bool two_stage_indirect_error = false; int ret = TRANSLATE_FAIL; - int mode = mmu_idx; + int mode = mmuidx_priv(mmu_idx); /* default TLB page size */ target_ulong tlb_size = TARGET_PAGE_SIZE; diff --git a/target/riscv/insn_trans/trans_rvbf16.c.inc b/target/riscv/insn_trans/trans_rvbf16.c.inc index 8ee99df3f3..0a9cd1ec31 100644 --- a/target/riscv/insn_trans/trans_rvbf16.c.inc +++ b/target/riscv/insn_trans/trans_rvbf16.c.inc @@ -71,11 +71,8 @@ static bool trans_vfncvtbf16_f_f_w(DisasContext *ctx, arg_vfncvtbf16_f_f_w *a) if (opfv_narrow_check(ctx, a) && (ctx->sew == MO_16)) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); gen_set_rm_chkfrm(ctx, RISCV_FRM_DYN); - tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, ctx->lmul); @@ -86,8 +83,7 @@ static bool trans_vfncvtbf16_f_f_w(DisasContext *ctx, arg_vfncvtbf16_f_f_w *a) ctx->cfg_ptr->vlenb, ctx->cfg_ptr->vlenb, data, gen_helper_vfncvtbf16_f_f_w); - mark_vs_dirty(ctx); - gen_set_label(over); + finalize_rvv_inst(ctx); return true; } return false; @@ -100,11 +96,8 @@ static bool trans_vfwcvtbf16_f_f_v(DisasContext *ctx, arg_vfwcvtbf16_f_f_v *a) if (opfv_widen_check(ctx, a) && (ctx->sew == MO_16)) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); gen_set_rm_chkfrm(ctx, RISCV_FRM_DYN); - tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, ctx->lmul); @@ -115,8 +108,7 @@ static bool trans_vfwcvtbf16_f_f_v(DisasContext *ctx, arg_vfwcvtbf16_f_f_v *a) ctx->cfg_ptr->vlenb, ctx->cfg_ptr->vlenb, data, gen_helper_vfwcvtbf16_f_f_v); - mark_vs_dirty(ctx); - gen_set_label(over); + finalize_rvv_inst(ctx); return true; } return false; @@ -130,11 +122,8 @@ static bool trans_vfwmaccbf16_vv(DisasContext *ctx, arg_vfwmaccbf16_vv *a) if (require_rvv(ctx) && vext_check_isa_ill(ctx) && (ctx->sew == MO_16) && vext_check_dss(ctx, a->rd, a->rs1, a->rs2, a->vm)) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); gen_set_rm_chkfrm(ctx, RISCV_FRM_DYN); - tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, ctx->lmul); @@ -146,8 +135,7 @@ static bool trans_vfwmaccbf16_vv(DisasContext *ctx, arg_vfwmaccbf16_vv *a) ctx->cfg_ptr->vlenb, ctx->cfg_ptr->vlenb, data, gen_helper_vfwmaccbf16_vv); - mark_vs_dirty(ctx); - gen_set_label(over); + finalize_rvv_inst(ctx); return true; } return false; diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index e42728990e..7d84e7d812 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -167,7 +167,7 @@ static bool do_vsetvl(DisasContext *s, int rd, int rs1, TCGv s2) gen_helper_vsetvl(dst, tcg_env, s1, s2); gen_set_gpr(s, rd, dst); - mark_vs_dirty(s); + finalize_rvv_inst(s); gen_update_pc(s, s->cur_insn_len); lookup_and_goto_ptr(s); @@ -187,7 +187,7 @@ static bool do_vsetivli(DisasContext *s, int rd, TCGv s1, TCGv s2) gen_helper_vsetvl(dst, tcg_env, s1, s2); gen_set_gpr(s, rd, dst); - mark_vs_dirty(s); + finalize_rvv_inst(s); gen_update_pc(s, s->cur_insn_len); lookup_and_goto_ptr(s); s->base.is_jmp = DISAS_NORETURN; @@ -616,9 +616,6 @@ static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data, TCGv base; TCGv_i32 desc; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); - dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); base = get_gpr(s, rs1, EXT_NONE); @@ -660,7 +657,7 @@ static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data, tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ); } - gen_set_label(over); + finalize_rvv_inst(s); return true; } @@ -802,9 +799,6 @@ static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2, TCGv base, stride; TCGv_i32 desc; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); - dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); base = get_gpr(s, rs1, EXT_NONE); @@ -819,7 +813,7 @@ static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2, fn(dest, mask, base, stride, tcg_env, desc); - gen_set_label(over); + finalize_rvv_inst(s); return true; } @@ -906,9 +900,6 @@ static bool ldst_index_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, TCGv base; TCGv_i32 desc; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); - dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); index = tcg_temp_new_ptr(); @@ -924,7 +915,7 @@ static bool ldst_index_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, fn(dest, mask, base, index, tcg_env, desc); - gen_set_label(over); + finalize_rvv_inst(s); return true; } @@ -1044,9 +1035,6 @@ static bool ldff_trans(uint32_t vd, uint32_t rs1, uint32_t data, TCGv base; TCGv_i32 desc; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); - dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); base = get_gpr(s, rs1, EXT_NONE); @@ -1058,8 +1046,7 @@ static bool ldff_trans(uint32_t vd, uint32_t rs1, uint32_t data, fn(dest, mask, base, tcg_env, desc); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } @@ -1097,13 +1084,9 @@ GEN_VEXT_TRANS(vle64ff_v, MO_64, r2nfvm, ldff_op, ld_us_check) typedef void gen_helper_ldst_whole(TCGv_ptr, TCGv, TCGv_env, TCGv_i32); static bool ldst_whole_trans(uint32_t vd, uint32_t rs1, uint32_t nf, - uint32_t width, gen_helper_ldst_whole *fn, + gen_helper_ldst_whole *fn, DisasContext *s) { - uint32_t evl = s->cfg_ptr->vlenb * nf / width; - TCGLabel *over = gen_new_label(); - tcg_gen_brcondi_tl(TCG_COND_GEU, cpu_vstart, evl, over); - TCGv_ptr dest; TCGv base; TCGv_i32 desc; @@ -1120,8 +1103,7 @@ static bool ldst_whole_trans(uint32_t vd, uint32_t rs1, uint32_t nf, fn(dest, base, tcg_env, desc); - gen_set_label(over); - + finalize_rvv_inst(s); return true; } @@ -1129,42 +1111,42 @@ static bool ldst_whole_trans(uint32_t vd, uint32_t rs1, uint32_t nf, * load and store whole register instructions ignore vtype and vl setting. * Thus, we don't need to check vill bit. (Section 7.9) */ -#define GEN_LDST_WHOLE_TRANS(NAME, ARG_NF, WIDTH) \ +#define GEN_LDST_WHOLE_TRANS(NAME, ARG_NF) \ static bool trans_##NAME(DisasContext *s, arg_##NAME * a) \ { \ if (require_rvv(s) && \ QEMU_IS_ALIGNED(a->rd, ARG_NF)) { \ - return ldst_whole_trans(a->rd, a->rs1, ARG_NF, WIDTH, \ + return ldst_whole_trans(a->rd, a->rs1, ARG_NF, \ gen_helper_##NAME, s); \ } \ return false; \ } -GEN_LDST_WHOLE_TRANS(vl1re8_v, 1, 1) -GEN_LDST_WHOLE_TRANS(vl1re16_v, 1, 2) -GEN_LDST_WHOLE_TRANS(vl1re32_v, 1, 4) -GEN_LDST_WHOLE_TRANS(vl1re64_v, 1, 8) -GEN_LDST_WHOLE_TRANS(vl2re8_v, 2, 1) -GEN_LDST_WHOLE_TRANS(vl2re16_v, 2, 2) -GEN_LDST_WHOLE_TRANS(vl2re32_v, 2, 4) -GEN_LDST_WHOLE_TRANS(vl2re64_v, 2, 8) -GEN_LDST_WHOLE_TRANS(vl4re8_v, 4, 1) -GEN_LDST_WHOLE_TRANS(vl4re16_v, 4, 2) -GEN_LDST_WHOLE_TRANS(vl4re32_v, 4, 4) -GEN_LDST_WHOLE_TRANS(vl4re64_v, 4, 8) -GEN_LDST_WHOLE_TRANS(vl8re8_v, 8, 1) -GEN_LDST_WHOLE_TRANS(vl8re16_v, 8, 2) -GEN_LDST_WHOLE_TRANS(vl8re32_v, 8, 4) -GEN_LDST_WHOLE_TRANS(vl8re64_v, 8, 8) +GEN_LDST_WHOLE_TRANS(vl1re8_v, 1) +GEN_LDST_WHOLE_TRANS(vl1re16_v, 1) +GEN_LDST_WHOLE_TRANS(vl1re32_v, 1) +GEN_LDST_WHOLE_TRANS(vl1re64_v, 1) +GEN_LDST_WHOLE_TRANS(vl2re8_v, 2) +GEN_LDST_WHOLE_TRANS(vl2re16_v, 2) +GEN_LDST_WHOLE_TRANS(vl2re32_v, 2) +GEN_LDST_WHOLE_TRANS(vl2re64_v, 2) +GEN_LDST_WHOLE_TRANS(vl4re8_v, 4) +GEN_LDST_WHOLE_TRANS(vl4re16_v, 4) +GEN_LDST_WHOLE_TRANS(vl4re32_v, 4) +GEN_LDST_WHOLE_TRANS(vl4re64_v, 4) +GEN_LDST_WHOLE_TRANS(vl8re8_v, 8) +GEN_LDST_WHOLE_TRANS(vl8re16_v, 8) +GEN_LDST_WHOLE_TRANS(vl8re32_v, 8) +GEN_LDST_WHOLE_TRANS(vl8re64_v, 8) /* * The vector whole register store instructions are encoded similar to * unmasked unit-stride store of elements with EEW=8. */ -GEN_LDST_WHOLE_TRANS(vs1r_v, 1, 1) -GEN_LDST_WHOLE_TRANS(vs2r_v, 2, 1) -GEN_LDST_WHOLE_TRANS(vs4r_v, 4, 1) -GEN_LDST_WHOLE_TRANS(vs8r_v, 8, 1) +GEN_LDST_WHOLE_TRANS(vs1r_v, 1) +GEN_LDST_WHOLE_TRANS(vs2r_v, 2) +GEN_LDST_WHOLE_TRANS(vs4r_v, 4) +GEN_LDST_WHOLE_TRANS(vs8r_v, 8) /* *** Vector Integer Arithmetic Instructions @@ -1195,10 +1177,6 @@ static inline bool do_opivv_gvec(DisasContext *s, arg_rmrr *a, GVecGen3Fn *gvec_fn, gen_helper_gvec_4_ptr *fn) { - TCGLabel *over = gen_new_label(); - - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); - if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2), vreg_ofs(s, a->rs1), @@ -1215,8 +1193,7 @@ do_opivv_gvec(DisasContext *s, arg_rmrr *a, GVecGen3Fn *gvec_fn, tcg_env, s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data, fn); } - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } @@ -1248,9 +1225,6 @@ static bool opivx_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, uint32_t vm, TCGv_i32 desc; uint32_t data = 0; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); - dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); src2 = tcg_temp_new_ptr(); @@ -1270,8 +1244,7 @@ static bool opivx_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, uint32_t vm, fn(dest, mask, src1, src2, tcg_env, desc); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } @@ -1296,7 +1269,7 @@ do_opivx_gvec(DisasContext *s, arg_rmrr *a, GVecGen2sFn *gvec_fn, gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2), src1, MAXSZ(s), MAXSZ(s)); - mark_vs_dirty(s); + finalize_rvv_inst(s); return true; } return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s); @@ -1410,9 +1383,6 @@ static bool opivi_trans(uint32_t vd, uint32_t imm, uint32_t vs2, uint32_t vm, TCGv_i32 desc; uint32_t data = 0; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); - dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); src2 = tcg_temp_new_ptr(); @@ -1432,8 +1402,7 @@ static bool opivi_trans(uint32_t vd, uint32_t imm, uint32_t vs2, uint32_t vm, fn(dest, mask, src1, src2, tcg_env, desc); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } @@ -1447,7 +1416,7 @@ do_opivi_gvec(DisasContext *s, arg_rmrr *a, GVecGen2iFn *gvec_fn, if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2), extract_imm(s, a->rs1, imm_mode), MAXSZ(s), MAXSZ(s)); - mark_vs_dirty(s); + finalize_rvv_inst(s); return true; } return opivi_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s, imm_mode); @@ -1495,8 +1464,6 @@ static bool do_opivv_widen(DisasContext *s, arg_rmrr *a, { if (checkfn(s, a)) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); @@ -1508,8 +1475,7 @@ static bool do_opivv_widen(DisasContext *s, arg_rmrr *a, tcg_env, s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data, fn); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } return false; @@ -1571,8 +1537,6 @@ static bool do_opiwv_widen(DisasContext *s, arg_rmrr *a, { if (opiwv_widen_check(s, a)) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); @@ -1583,8 +1547,7 @@ static bool do_opiwv_widen(DisasContext *s, arg_rmrr *a, vreg_ofs(s, a->rs2), tcg_env, s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data, fn); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } return false; @@ -1643,8 +1606,6 @@ static bool opivv_trans(uint32_t vd, uint32_t vs1, uint32_t vs2, uint32_t vm, gen_helper_gvec_4_ptr *fn, DisasContext *s) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); @@ -1654,8 +1615,7 @@ static bool opivv_trans(uint32_t vd, uint32_t vs1, uint32_t vs2, uint32_t vm, tcg_gen_gvec_4_ptr(vreg_ofs(s, vd), vreg_ofs(s, 0), vreg_ofs(s, vs1), vreg_ofs(s, vs2), tcg_env, s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data, fn); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } @@ -1788,7 +1748,7 @@ do_opivx_gvec_shift(DisasContext *s, arg_rmrr *a, GVecGen2sFn32 *gvec_fn, gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2), src1, MAXSZ(s), MAXSZ(s)); - mark_vs_dirty(s); + finalize_rvv_inst(s); return true; } return opivx_trans(a->rd, a->rs1, a->rs2, a->vm, fn, s); @@ -1834,8 +1794,6 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ gen_helper_##NAME##_h, \ gen_helper_##NAME##_w, \ }; \ - TCGLabel *over = gen_new_label(); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -1847,8 +1805,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ s->cfg_ptr->vlenb, \ s->cfg_ptr->vlenb, data, \ fns[s->sew]); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -2045,16 +2002,13 @@ static bool trans_vmv_v_v(DisasContext *s, arg_vmv_v_v *a) gen_helper_vmv_v_v_b, gen_helper_vmv_v_v_h, gen_helper_vmv_v_v_w, gen_helper_vmv_v_v_d, }; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs1), tcg_env, s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data, fns[s->sew]); - gen_set_label(over); } - mark_vs_dirty(s); + finalize_rvv_inst(s); return true; } return false; @@ -2068,8 +2022,6 @@ static bool trans_vmv_v_x(DisasContext *s, arg_vmv_v_x *a) /* vmv.v.x has rs2 = 0 and vm = 1 */ vext_check_ss(s, a->rd, 0, 1)) { TCGv s1; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); s1 = get_gpr(s, a->rs1, EXT_SIGN); @@ -2101,8 +2053,7 @@ static bool trans_vmv_v_x(DisasContext *s, arg_vmv_v_x *a) fns[s->sew](dest, s1_i64, tcg_env, desc); } - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } return false; @@ -2118,7 +2069,6 @@ static bool trans_vmv_v_i(DisasContext *s, arg_vmv_v_i *a) if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { tcg_gen_gvec_dup_imm(s->sew, vreg_ofs(s, a->rd), MAXSZ(s), MAXSZ(s), simm); - mark_vs_dirty(s); } else { TCGv_i32 desc; TCGv_i64 s1; @@ -2129,8 +2079,6 @@ static bool trans_vmv_v_i(DisasContext *s, arg_vmv_v_i *a) gen_helper_vmv_v_x_b, gen_helper_vmv_v_x_h, gen_helper_vmv_v_x_w, gen_helper_vmv_v_x_d, }; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); s1 = tcg_constant_i64(simm); dest = tcg_temp_new_ptr(); @@ -2138,10 +2086,8 @@ static bool trans_vmv_v_i(DisasContext *s, arg_vmv_v_i *a) s->cfg_ptr->vlenb, data)); tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd)); fns[s->sew](dest, s1, tcg_env, desc); - - mark_vs_dirty(s); - gen_set_label(over); } + finalize_rvv_inst(s); return true; } return false; @@ -2275,9 +2221,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ gen_helper_##NAME##_w, \ gen_helper_##NAME##_d, \ }; \ - TCGLabel *over = gen_new_label(); \ gen_set_rm(s, RISCV_FRM_DYN); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2291,8 +2235,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ s->cfg_ptr->vlenb, \ s->cfg_ptr->vlenb, data, \ fns[s->sew - 1]); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -2310,9 +2253,6 @@ static bool opfvf_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, TCGv_i32 desc; TCGv_i64 t1; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); - dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); src2 = tcg_temp_new_ptr(); @@ -2329,8 +2269,7 @@ static bool opfvf_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, fn(dest, mask, t1, src2, tcg_env, desc); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } @@ -2393,9 +2332,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ static gen_helper_gvec_4_ptr * const fns[2] = { \ gen_helper_##NAME##_h, gen_helper_##NAME##_w, \ }; \ - TCGLabel *over = gen_new_label(); \ gen_set_rm(s, RISCV_FRM_DYN); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);\ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2407,8 +2344,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ s->cfg_ptr->vlenb, \ s->cfg_ptr->vlenb, data, \ fns[s->sew - 1]); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -2467,9 +2403,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ static gen_helper_gvec_4_ptr * const fns[2] = { \ gen_helper_##NAME##_h, gen_helper_##NAME##_w, \ }; \ - TCGLabel *over = gen_new_label(); \ gen_set_rm(s, RISCV_FRM_DYN); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2481,8 +2415,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ s->cfg_ptr->vlenb, \ s->cfg_ptr->vlenb, data, \ fns[s->sew - 1]); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -2584,9 +2517,7 @@ static bool do_opfv(DisasContext *s, arg_rmr *a, { if (checkfn(s, a)) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); gen_set_rm_chkfrm(s, rm); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); @@ -2596,8 +2527,7 @@ static bool do_opfv(DisasContext *s, arg_rmr *a, vreg_ofs(s, a->rs2), tcg_env, s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data, fn); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } return false; @@ -2684,7 +2614,6 @@ static bool trans_vfmv_v_f(DisasContext *s, arg_vfmv_v_f *a) tcg_gen_gvec_dup_i64(s->sew, vreg_ofs(s, a->rd), MAXSZ(s), MAXSZ(s), t1); - mark_vs_dirty(s); } else { TCGv_ptr dest; TCGv_i32 desc; @@ -2696,8 +2625,6 @@ static bool trans_vfmv_v_f(DisasContext *s, arg_vfmv_v_f *a) gen_helper_vmv_v_x_w, gen_helper_vmv_v_x_d, }; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); t1 = tcg_temp_new_i64(); /* NaN-box f[rs1] */ @@ -2709,10 +2636,8 @@ static bool trans_vfmv_v_f(DisasContext *s, arg_vfmv_v_f *a) tcg_gen_addi_ptr(dest, tcg_env, vreg_ofs(s, a->rd)); fns[s->sew - 1](dest, t1, tcg_env, desc); - - mark_vs_dirty(s); - gen_set_label(over); } + finalize_rvv_inst(s); return true; } return false; @@ -2773,9 +2698,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ gen_helper_##HELPER##_h, \ gen_helper_##HELPER##_w, \ }; \ - TCGLabel *over = gen_new_label(); \ gen_set_rm_chkfrm(s, FRM); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2786,8 +2709,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ s->cfg_ptr->vlenb, \ s->cfg_ptr->vlenb, data, \ fns[s->sew - 1]); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -2824,9 +2746,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ gen_helper_##NAME##_h, \ gen_helper_##NAME##_w, \ }; \ - TCGLabel *over = gen_new_label(); \ gen_set_rm(s, RISCV_FRM_DYN); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2837,8 +2757,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ s->cfg_ptr->vlenb, \ s->cfg_ptr->vlenb, data, \ fns[s->sew]); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -2891,9 +2810,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ gen_helper_##HELPER##_h, \ gen_helper_##HELPER##_w, \ }; \ - TCGLabel *over = gen_new_label(); \ gen_set_rm_chkfrm(s, FRM); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2904,8 +2821,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ s->cfg_ptr->vlenb, \ s->cfg_ptr->vlenb, data, \ fns[s->sew - 1]); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -2940,9 +2856,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ gen_helper_##HELPER##_h, \ gen_helper_##HELPER##_w, \ }; \ - TCGLabel *over = gen_new_label(); \ gen_set_rm_chkfrm(s, FRM); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2953,8 +2867,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ s->cfg_ptr->vlenb, \ s->cfg_ptr->vlenb, data, \ fns[s->sew]); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -3031,8 +2944,6 @@ static bool trans_##NAME(DisasContext *s, arg_r *a) \ vext_check_isa_ill(s)) { \ uint32_t data = 0; \ gen_helper_gvec_4_ptr *fn = gen_helper_##NAME; \ - TCGLabel *over = gen_new_label(); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ data = \ @@ -3042,8 +2953,7 @@ static bool trans_##NAME(DisasContext *s, arg_r *a) \ vreg_ofs(s, a->rs2), tcg_env, \ s->cfg_ptr->vlenb, \ s->cfg_ptr->vlenb, data, fn); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -3131,8 +3041,6 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ s->vstart_eq_zero) { \ uint32_t data = 0; \ gen_helper_gvec_3_ptr *fn = gen_helper_##NAME; \ - TCGLabel *over = gen_new_label(); \ - tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -3144,8 +3052,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ tcg_env, s->cfg_ptr->vlenb, \ s->cfg_ptr->vlenb, \ data, fn); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -3171,8 +3078,6 @@ static bool trans_viota_m(DisasContext *s, arg_viota_m *a) require_align(a->rd, s->lmul) && s->vstart_eq_zero) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); - tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); @@ -3186,8 +3091,7 @@ static bool trans_viota_m(DisasContext *s, arg_viota_m *a) vreg_ofs(s, a->rs2), tcg_env, s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data, fns[s->sew]); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } return false; @@ -3201,8 +3105,6 @@ static bool trans_vid_v(DisasContext *s, arg_vid_v *a) require_align(a->rd, s->lmul) && require_vm(a->vm, a->rd)) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); @@ -3216,8 +3118,7 @@ static bool trans_vid_v(DisasContext *s, arg_vid_v *a) tcg_env, s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data, fns[s->sew]); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } return false; @@ -3373,6 +3274,8 @@ static bool trans_vmv_x_s(DisasContext *s, arg_vmv_x_s *a) vec_element_loadi(s, t1, a->rs2, 0, true); tcg_gen_trunc_i64_tl(dest, t1); gen_set_gpr(s, a->rd, dest); + tcg_gen_movi_tl(cpu_vstart, 0); + finalize_rvv_inst(s); return true; } return false; @@ -3399,8 +3302,9 @@ static bool trans_vmv_s_x(DisasContext *s, arg_vmv_s_x *a) s1 = get_gpr(s, a->rs1, EXT_NONE); tcg_gen_ext_tl_i64(t1, s1); vec_element_storei(s, a->rd, 0, t1); - mark_vs_dirty(s); gen_set_label(over); + tcg_gen_movi_tl(cpu_vstart, 0); + finalize_rvv_inst(s); return true; } return false; @@ -3427,6 +3331,8 @@ static bool trans_vfmv_f_s(DisasContext *s, arg_vfmv_f_s *a) } mark_fs_dirty(s); + tcg_gen_movi_tl(cpu_vstart, 0); + finalize_rvv_inst(s); return true; } return false; @@ -3452,8 +3358,10 @@ static bool trans_vfmv_s_f(DisasContext *s, arg_vfmv_s_f *a) do_nanbox(s, t1, cpu_fpr[a->rs1]); vec_element_storei(s, a->rd, 0, t1); - mark_vs_dirty(s); + gen_set_label(over); + tcg_gen_movi_tl(cpu_vstart, 0); + finalize_rvv_inst(s); return true; } return false; @@ -3559,7 +3467,7 @@ static bool trans_vrgather_vx(DisasContext *s, arg_rmrr *a) tcg_gen_gvec_dup_i64(s->sew, vreg_ofs(s, a->rd), MAXSZ(s), MAXSZ(s), dest); - mark_vs_dirty(s); + finalize_rvv_inst(s); } else { static gen_helper_opivx * const fns[4] = { gen_helper_vrgather_vx_b, gen_helper_vrgather_vx_h, @@ -3587,7 +3495,7 @@ static bool trans_vrgather_vi(DisasContext *s, arg_rmrr *a) endian_ofs(s, a->rs2, a->rs1), MAXSZ(s), MAXSZ(s)); } - mark_vs_dirty(s); + finalize_rvv_inst(s); } else { static gen_helper_opivx * const fns[4] = { gen_helper_vrgather_vx_b, gen_helper_vrgather_vx_h, @@ -3624,8 +3532,6 @@ static bool trans_vcompress_vm(DisasContext *s, arg_r *a) gen_helper_vcompress_vm_b, gen_helper_vcompress_vm_h, gen_helper_vcompress_vm_w, gen_helper_vcompress_vm_d, }; - TCGLabel *over = gen_new_label(); - tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); data = FIELD_DP32(data, VDATA, VTA, s->vta); @@ -3634,8 +3540,7 @@ static bool trans_vcompress_vm(DisasContext *s, arg_r *a) tcg_env, s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data, fns[s->sew]); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } return false; @@ -3656,15 +3561,11 @@ static bool trans_##NAME(DisasContext *s, arg_##NAME * a) \ if (s->vstart_eq_zero) { \ tcg_gen_gvec_mov(s->sew, vreg_ofs(s, a->rd), \ vreg_ofs(s, a->rs2), maxsz, maxsz); \ - mark_vs_dirty(s); \ } else { \ - TCGLabel *over = gen_new_label(); \ - tcg_gen_brcondi_tl(TCG_COND_GEU, cpu_vstart, maxsz, over); \ tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2), \ tcg_env, maxsz, maxsz, 0, gen_helper_vmvr_v); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ } \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -3692,8 +3593,6 @@ static bool int_ext_op(DisasContext *s, arg_rmr *a, uint8_t seq) { uint32_t data = 0; gen_helper_gvec_3_ptr *fn; - TCGLabel *over = gen_new_label(); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); static gen_helper_gvec_3_ptr * const fns[6][4] = { { @@ -3737,8 +3636,7 @@ static bool int_ext_op(DisasContext *s, arg_rmr *a, uint8_t seq) s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, data, fn); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } diff --git a/target/riscv/insn_trans/trans_rvvk.c.inc b/target/riscv/insn_trans/trans_rvvk.c.inc index a5cdd1b67f..ae1f40174a 100644 --- a/target/riscv/insn_trans/trans_rvvk.c.inc +++ b/target/riscv/insn_trans/trans_rvvk.c.inc @@ -164,8 +164,6 @@ GEN_OPIVX_GVEC_TRANS_CHECK(vandn_vx, andcs, zvkb_vx_check) gen_helper_##NAME##_w, \ gen_helper_##NAME##_d, \ }; \ - TCGLabel *over = gen_new_label(); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -176,8 +174,7 @@ GEN_OPIVX_GVEC_TRANS_CHECK(vandn_vx, andcs, zvkb_vx_check) vreg_ofs(s, a->rs2), tcg_env, \ s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, \ data, fns[s->sew]); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -249,14 +246,12 @@ GEN_OPIVI_WIDEN_TRANS(vwsll_vi, IMM_ZX, vwsll_vx, vwsll_vx_check) TCGv_ptr rd_v, rs2_v; \ TCGv_i32 desc, egs; \ uint32_t data = 0; \ - TCGLabel *over = gen_new_label(); \ \ if (!s->vstart_eq_zero || !s->vl_eq_vlmax) { \ /* save opcode for unwinding in case we throw an exception */ \ decode_save_opc(s); \ egs = tcg_constant_i32(EGS); \ gen_helper_egs_check(egs, tcg_env); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ } \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ @@ -271,8 +266,7 @@ GEN_OPIVI_WIDEN_TRANS(vwsll_vi, IMM_ZX, vwsll_vx, vwsll_vx_check) tcg_gen_addi_ptr(rd_v, tcg_env, vreg_ofs(s, a->rd)); \ tcg_gen_addi_ptr(rs2_v, tcg_env, vreg_ofs(s, a->rs2)); \ gen_helper_##NAME(rd_v, rs2_v, tcg_env, desc); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -325,14 +319,12 @@ GEN_V_UNMASKED_TRANS(vaesem_vs, vaes_check_vs, ZVKNED_EGS) TCGv_ptr rd_v, rs2_v; \ TCGv_i32 uimm_v, desc, egs; \ uint32_t data = 0; \ - TCGLabel *over = gen_new_label(); \ \ if (!s->vstart_eq_zero || !s->vl_eq_vlmax) { \ /* save opcode for unwinding in case we throw an exception */ \ decode_save_opc(s); \ egs = tcg_constant_i32(EGS); \ gen_helper_egs_check(egs, tcg_env); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ } \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ @@ -349,8 +341,7 @@ GEN_V_UNMASKED_TRANS(vaesem_vs, vaes_check_vs, ZVKNED_EGS) tcg_gen_addi_ptr(rd_v, tcg_env, vreg_ofs(s, a->rd)); \ tcg_gen_addi_ptr(rs2_v, tcg_env, vreg_ofs(s, a->rs2)); \ gen_helper_##NAME(rd_v, rs2_v, uimm_v, tcg_env, desc); \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -394,7 +385,6 @@ GEN_VI_UNMASKED_TRANS(vaeskf2_vi, vaeskf2_check, ZVKNED_EGS) { \ if (CHECK(s, a)) { \ uint32_t data = 0; \ - TCGLabel *over = gen_new_label(); \ TCGv_i32 egs; \ \ if (!s->vstart_eq_zero || !s->vl_eq_vlmax) { \ @@ -402,7 +392,6 @@ GEN_VI_UNMASKED_TRANS(vaeskf2_vi, vaeskf2_check, ZVKNED_EGS) decode_save_opc(s); \ egs = tcg_constant_i32(EGS); \ gen_helper_egs_check(egs, tcg_env); \ - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ } \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ @@ -416,8 +405,7 @@ GEN_VI_UNMASKED_TRANS(vaeskf2_vi, vaeskf2_check, ZVKNED_EGS) s->cfg_ptr->vlenb, s->cfg_ptr->vlenb, \ data, gen_helper_##NAME); \ \ - mark_vs_dirty(s); \ - gen_set_label(over); \ + finalize_rvv_inst(s); \ return true; \ } \ return false; \ @@ -448,7 +436,6 @@ static bool trans_vsha2cl_vv(DisasContext *s, arg_rmrr *a) { if (vsha_check(s, a)) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); TCGv_i32 egs; if (!s->vstart_eq_zero || !s->vl_eq_vlmax) { @@ -456,7 +443,6 @@ static bool trans_vsha2cl_vv(DisasContext *s, arg_rmrr *a) decode_save_opc(s); egs = tcg_constant_i32(ZVKNH_EGS); gen_helper_egs_check(egs, tcg_env); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); } data = FIELD_DP32(data, VDATA, VM, a->vm); @@ -471,8 +457,7 @@ static bool trans_vsha2cl_vv(DisasContext *s, arg_rmrr *a) s->sew == MO_32 ? gen_helper_vsha2cl32_vv : gen_helper_vsha2cl64_vv); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } return false; @@ -482,7 +467,6 @@ static bool trans_vsha2ch_vv(DisasContext *s, arg_rmrr *a) { if (vsha_check(s, a)) { uint32_t data = 0; - TCGLabel *over = gen_new_label(); TCGv_i32 egs; if (!s->vstart_eq_zero || !s->vl_eq_vlmax) { @@ -490,7 +474,6 @@ static bool trans_vsha2ch_vv(DisasContext *s, arg_rmrr *a) decode_save_opc(s); egs = tcg_constant_i32(ZVKNH_EGS); gen_helper_egs_check(egs, tcg_env); - tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); } data = FIELD_DP32(data, VDATA, VM, a->vm); @@ -505,8 +488,7 @@ static bool trans_vsha2ch_vv(DisasContext *s, arg_rmrr *a) s->sew == MO_32 ? gen_helper_vsha2ch32_vv : gen_helper_vsha2ch64_vv); - mark_vs_dirty(s); - gen_set_label(over); + finalize_rvv_inst(s); return true; } return false; diff --git a/target/riscv/kvm/kvm-cpu.c b/target/riscv/kvm/kvm-cpu.c index cda7d78a77..6a6c6cae80 100644 --- a/target/riscv/kvm/kvm-cpu.c +++ b/target/riscv/kvm/kvm-cpu.c @@ -739,6 +739,15 @@ static void kvm_riscv_put_regs_timer(CPUState *cs) env->kvm_timer_dirty = false; } +uint64_t kvm_riscv_get_timebase_frequency(CPUState *cs) +{ + uint64_t reg; + + KVM_RISCV_GET_TIMER(cs, frequency, reg); + + return reg; +} + static int kvm_riscv_get_regs_vector(CPUState *cs) { RISCVCPU *cpu = RISCV_CPU(cs); diff --git a/target/riscv/kvm/kvm_riscv.h b/target/riscv/kvm/kvm_riscv.h index 4bd98fddc7..5851898868 100644 --- a/target/riscv/kvm/kvm_riscv.h +++ b/target/riscv/kvm/kvm_riscv.h @@ -28,5 +28,6 @@ void kvm_riscv_aia_create(MachineState *machine, uint64_t group_shift, void riscv_kvm_aplic_request(void *opaque, int irq, int level); int kvm_riscv_sync_mpstate_to_kvm(RISCVCPU *cpu, int state); void riscv_kvm_cpu_finalize_features(RISCVCPU *cpu, Error **errp); +uint64_t kvm_riscv_get_timebase_frequency(CPUState *cs); #endif diff --git a/target/riscv/tcg/tcg-cpu.c b/target/riscv/tcg/tcg-cpu.c index ab6db817db..b5b95e052d 100644 --- a/target/riscv/tcg/tcg-cpu.c +++ b/target/riscv/tcg/tcg-cpu.c @@ -315,9 +315,19 @@ static void riscv_cpu_disable_priv_spec_isa_exts(RISCVCPU *cpu) static void riscv_cpu_update_named_features(RISCVCPU *cpu) { + if (cpu->env.priv_ver >= PRIV_VERSION_1_11_0) { + cpu->cfg.has_priv_1_11 = true; + } + + if (cpu->env.priv_ver >= PRIV_VERSION_1_12_0) { + cpu->cfg.has_priv_1_12 = true; + } + + /* zic64b is 1.12 or later */ cpu->cfg.ext_zic64b = cpu->cfg.cbom_blocksize == 64 && cpu->cfg.cbop_blocksize == 64 && - cpu->cfg.cboz_blocksize == 64; + cpu->cfg.cboz_blocksize == 64 && + cpu->cfg.has_priv_1_12; } static void riscv_cpu_validate_g(RISCVCPU *cpu) @@ -520,11 +530,6 @@ void riscv_cpu_validate_set_extensions(RISCVCPU *cpu, Error **errp) return; } - if (cpu->cfg.ext_zvfbfmin && !cpu->cfg.ext_zfbfmin) { - error_setg(errp, "Zvfbfmin extension depends on Zfbfmin extension"); - return; - } - if (cpu->cfg.ext_zvfbfmin && !cpu->cfg.ext_zve32f) { error_setg(errp, "Zvfbfmin extension depends on Zve32f extension"); return; @@ -1316,8 +1321,6 @@ static void riscv_tcg_cpu_instance_init(CPUState *cs) RISCVCPU *cpu = RISCV_CPU(cs); Object *obj = OBJECT(cpu); - cpu->cfg.ext_always_enabled = true; - misa_ext_user_opts = g_hash_table_new(NULL, g_direct_equal); multi_ext_user_opts = g_hash_table_new(NULL, g_direct_equal); riscv_cpu_add_user_properties(obj); diff --git a/target/riscv/translate.c b/target/riscv/translate.c index ea5d52b2ef..9d57089fcc 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -676,6 +676,12 @@ static void mark_vs_dirty(DisasContext *ctx) static inline void mark_vs_dirty(DisasContext *ctx) { } #endif +static void finalize_rvv_inst(DisasContext *ctx) +{ + mark_vs_dirty(ctx); + ctx->vstart_eq_zero = true; +} + static void gen_set_rm(DisasContext *ctx, int rm) { if (ctx->frm == rm) { diff --git a/target/riscv/vcrypto_helper.c b/target/riscv/vcrypto_helper.c index e2d719b13b..f7423df226 100644 --- a/target/riscv/vcrypto_helper.c +++ b/target/riscv/vcrypto_helper.c @@ -222,6 +222,8 @@ static inline void xor_round_key(AESState *round_state, AESState *round_key) uint32_t total_elems = vext_get_total_elems(env, desc, 4); \ uint32_t vta = vext_vta(desc); \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { \ AESState round_key; \ round_key.d[0] = *((uint64_t *)vs2 + H8(i * 2 + 0)); \ @@ -246,6 +248,8 @@ static inline void xor_round_key(AESState *round_state, AESState *round_key) uint32_t total_elems = vext_get_total_elems(env, desc, 4); \ uint32_t vta = vext_vta(desc); \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { \ AESState round_key; \ round_key.d[0] = *((uint64_t *)vs2 + H8(0)); \ @@ -305,6 +309,8 @@ void HELPER(vaeskf1_vi)(void *vd_vptr, void *vs2_vptr, uint32_t uimm, uint32_t total_elems = vext_get_total_elems(env, desc, 4); uint32_t vta = vext_vta(desc); + VSTART_CHECK_EARLY_EXIT(env); + uimm &= 0b1111; if (uimm > 10 || uimm == 0) { uimm ^= 0b1000; @@ -351,6 +357,8 @@ void HELPER(vaeskf2_vi)(void *vd_vptr, void *vs2_vptr, uint32_t uimm, uint32_t total_elems = vext_get_total_elems(env, desc, 4); uint32_t vta = vext_vta(desc); + VSTART_CHECK_EARLY_EXIT(env); + uimm &= 0b1111; if (uimm > 14 || uimm < 2) { uimm ^= 0b1000; @@ -457,6 +465,8 @@ void HELPER(vsha2ms_vv)(void *vd, void *vs1, void *vs2, CPURISCVState *env, uint32_t total_elems; uint32_t vta = vext_vta(desc); + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { if (sew == MO_32) { vsha2ms_e32(((uint32_t *)vd) + i * 4, ((uint32_t *)vs1) + i * 4, @@ -572,6 +582,8 @@ void HELPER(vsha2ch32_vv)(void *vd, void *vs1, void *vs2, CPURISCVState *env, uint32_t total_elems; uint32_t vta = vext_vta(desc); + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { vsha2c_32(((uint32_t *)vs2) + 4 * i, ((uint32_t *)vd) + 4 * i, ((uint32_t *)vs1) + 4 * i + 2); @@ -590,6 +602,8 @@ void HELPER(vsha2ch64_vv)(void *vd, void *vs1, void *vs2, CPURISCVState *env, uint32_t total_elems; uint32_t vta = vext_vta(desc); + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { vsha2c_64(((uint64_t *)vs2) + 4 * i, ((uint64_t *)vd) + 4 * i, ((uint64_t *)vs1) + 4 * i + 2); @@ -608,6 +622,8 @@ void HELPER(vsha2cl32_vv)(void *vd, void *vs1, void *vs2, CPURISCVState *env, uint32_t total_elems; uint32_t vta = vext_vta(desc); + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { vsha2c_32(((uint32_t *)vs2) + 4 * i, ((uint32_t *)vd) + 4 * i, (((uint32_t *)vs1) + 4 * i)); @@ -626,6 +642,8 @@ void HELPER(vsha2cl64_vv)(void *vd, void *vs1, void *vs2, CPURISCVState *env, uint32_t total_elems; uint32_t vta = vext_vta(desc); + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { vsha2c_64(((uint64_t *)vs2) + 4 * i, ((uint64_t *)vd) + 4 * i, (((uint64_t *)vs1) + 4 * i)); @@ -658,6 +676,8 @@ void HELPER(vsm3me_vv)(void *vd_vptr, void *vs1_vptr, void *vs2_vptr, uint32_t *vs1 = vs1_vptr; uint32_t *vs2 = vs2_vptr; + VSTART_CHECK_EARLY_EXIT(env); + for (int i = env->vstart / 8; i < env->vl / 8; i++) { uint32_t w[24]; for (int j = 0; j < 8; j++) { @@ -757,6 +777,8 @@ void HELPER(vsm3c_vi)(void *vd_vptr, void *vs2_vptr, uint32_t uimm, uint32_t *vs2 = vs2_vptr; uint32_t v1[8], v2[8], v3[8]; + VSTART_CHECK_EARLY_EXIT(env); + for (int i = env->vstart / 8; i < env->vl / 8; i++) { for (int k = 0; k < 8; k++) { v2[k] = bswap32(vd[H4(i * 8 + k)]); @@ -780,6 +802,8 @@ void HELPER(vghsh_vv)(void *vd_vptr, void *vs1_vptr, void *vs2_vptr, uint32_t vta = vext_vta(desc); uint32_t total_elems = vext_get_total_elems(env, desc, 4); + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { uint64_t Y[2] = {vd[i * 2 + 0], vd[i * 2 + 1]}; uint64_t H[2] = {brev8(vs2[i * 2 + 0]), brev8(vs2[i * 2 + 1])}; @@ -817,6 +841,8 @@ void HELPER(vgmul_vv)(void *vd_vptr, void *vs2_vptr, CPURISCVState *env, uint32_t vta = vext_vta(desc); uint32_t total_elems = vext_get_total_elems(env, desc, 4); + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = env->vstart / 4; i < env->vl / 4; i++) { uint64_t Y[2] = {brev8(vd[i * 2 + 0]), brev8(vd[i * 2 + 1])}; uint64_t H[2] = {brev8(vs2[i * 2 + 0]), brev8(vs2[i * 2 + 1])}; @@ -853,6 +879,8 @@ void HELPER(vsm4k_vi)(void *vd, void *vs2, uint32_t uimm5, CPURISCVState *env, uint32_t esz = sizeof(uint32_t); uint32_t total_elems = vext_get_total_elems(env, desc, esz); + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = group_start; i < group_end; ++i) { uint32_t vstart = i * egs; uint32_t vend = (i + 1) * egs; @@ -909,6 +937,8 @@ void HELPER(vsm4r_vv)(void *vd, void *vs2, CPURISCVState *env, uint32_t desc) uint32_t esz = sizeof(uint32_t); uint32_t total_elems = vext_get_total_elems(env, desc, esz); + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = group_start; i < group_end; ++i) { uint32_t vstart = i * egs; uint32_t vend = (i + 1) * egs; @@ -943,6 +973,8 @@ void HELPER(vsm4r_vs)(void *vd, void *vs2, CPURISCVState *env, uint32_t desc) uint32_t esz = sizeof(uint32_t); uint32_t total_elems = vext_get_total_elems(env, desc, esz); + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = group_start; i < group_end; ++i) { uint32_t vstart = i * egs; uint32_t vend = (i + 1) * egs; diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index fe56c007d5..fa139040f8 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -207,7 +207,9 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, uint32_t esz = 1 << log2_esz; uint32_t vma = vext_vma(desc); - for (i = env->vstart; i < env->vl; i++, env->vstart++) { + VSTART_CHECK_EARLY_EXIT(env); + + for (i = env->vstart; i < env->vl; env->vstart = ++i) { k = 0; while (k < nf) { if (!vm && !vext_elem_mask(v0, i)) { @@ -272,8 +274,10 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, uint32_t max_elems = vext_max_elems(desc, log2_esz); uint32_t esz = 1 << log2_esz; + VSTART_CHECK_EARLY_EXIT(env); + /* load bytes from guest memory */ - for (i = env->vstart; i < evl; i++, env->vstart++) { + for (i = env->vstart; i < evl; env->vstart = ++i) { k = 0; while (k < nf) { target_ulong addr = base + ((i * nf + k) << log2_esz); @@ -386,8 +390,10 @@ vext_ldst_index(void *vd, void *v0, target_ulong base, uint32_t esz = 1 << log2_esz; uint32_t vma = vext_vma(desc); + VSTART_CHECK_EARLY_EXIT(env); + /* load bytes from guest memory */ - for (i = env->vstart; i < env->vl; i++, env->vstart++) { + for (i = env->vstart; i < env->vl; env->vstart = ++i) { k = 0; while (k < nf) { if (!vm && !vext_elem_mask(v0, i)) { @@ -477,6 +483,8 @@ vext_ldff(void *vd, void *v0, target_ulong base, target_ulong addr, offset, remain; int mmu_index = riscv_env_mmu_index(env, false); + VSTART_CHECK_EARLY_EXIT(env); + /* probe every access */ for (i = env->vstart; i < env->vl; i++) { if (!vm && !vext_elem_mask(v0, i)) { @@ -572,6 +580,11 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, uint32_t vlenb = riscv_cpu_cfg(env)->vlenb; uint32_t max_elems = vlenb >> log2_esz; + if (env->vstart >= ((vlenb * nf) >> log2_esz)) { + env->vstart = 0; + return; + } + k = env->vstart / max_elems; off = env->vstart % max_elems; @@ -877,6 +890,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s1 = *((ETYPE *)vs1 + H(i)); \ ETYPE s2 = *((ETYPE *)vs2 + H(i)); \ @@ -909,6 +924,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s2 = *((ETYPE *)vs2 + H(i)); \ ETYPE carry = vext_elem_mask(v0, i); \ @@ -944,6 +961,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s1 = *((ETYPE *)vs1 + H(i)); \ ETYPE s2 = *((ETYPE *)vs2 + H(i)); \ @@ -982,6 +1001,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s2 = *((ETYPE *)vs2 + H(i)); \ ETYPE carry = !vm && vext_elem_mask(v0, i); \ @@ -1078,6 +1099,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \ @@ -1125,6 +1148,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \ @@ -1187,6 +1212,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s1 = *((ETYPE *)vs1 + H(i)); \ ETYPE s2 = *((ETYPE *)vs2 + H(i)); \ @@ -1252,6 +1279,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s2 = *((ETYPE *)vs2 + H(i)); \ if (!vm && !vext_elem_mask(v0, i)) { \ @@ -1799,6 +1828,8 @@ void HELPER(NAME)(void *vd, void *vs1, CPURISCVState *env, \ uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s1 = *((ETYPE *)vs1 + H(i)); \ *((ETYPE *)vd + H(i)) = s1; \ @@ -1823,6 +1854,8 @@ void HELPER(NAME)(void *vd, uint64_t s1, CPURISCVState *env, \ uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ *((ETYPE *)vd + H(i)) = (ETYPE)s1; \ } \ @@ -1846,6 +1879,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE *vt = (!vext_elem_mask(v0, i) ? vs2 : vs1); \ *((ETYPE *)vd + H(i)) = *(vt + H(i)); \ @@ -1870,6 +1905,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s2 = *((ETYPE *)vs2 + H(i)); \ ETYPE d = (!vext_elem_mask(v0, i) ? s2 : \ @@ -1915,6 +1952,8 @@ vext_vv_rm_1(void *vd, void *v0, void *vs1, void *vs2, uint32_t vl, uint32_t vm, int vxrm, opivv2_rm_fn *fn, uint32_t vma, uint32_t esz) { + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = env->vstart; i < vl; i++) { if (!vm && !vext_elem_mask(v0, i)) { /* set masked-off elements to 1s */ @@ -2040,6 +2079,8 @@ vext_vx_rm_1(void *vd, void *v0, target_long s1, void *vs2, uint32_t vl, uint32_t vm, int vxrm, opivx2_rm_fn *fn, uint32_t vma, uint32_t esz) { + VSTART_CHECK_EARLY_EXIT(env); + for (uint32_t i = env->vstart; i < vl; i++) { if (!vm && !vext_elem_mask(v0, i)) { /* set masked-off elements to 1s */ @@ -2837,6 +2878,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \ @@ -2880,6 +2923,8 @@ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \ @@ -3466,6 +3511,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ if (vl == 0) { \ return; \ } \ @@ -3987,6 +4034,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s1 = *((ETYPE *)vs1 + H(i)); \ ETYPE s2 = *((ETYPE *)vs2 + H(i)); \ @@ -4027,6 +4076,8 @@ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s2 = *((ETYPE *)vs2 + H(i)); \ if (!vm && !vext_elem_mask(v0, i)) { \ @@ -4220,6 +4271,8 @@ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ ETYPE s2 = *((ETYPE *)vs2 + H(i)); \ *((ETYPE *)vd + H(i)) = \ @@ -4544,6 +4597,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ uint32_t i; \ int a, b; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ a = vext_elem_mask(vs1, i); \ b = vext_elem_mask(vs2, i); \ @@ -4737,6 +4792,8 @@ void HELPER(NAME)(void *vd, void *v0, CPURISCVState *env, uint32_t desc) \ uint32_t vma = vext_vma(desc); \ int i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \ @@ -4772,6 +4829,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ uint32_t vma = vext_vma(desc); \ target_ulong offset = s1, i_min, i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ i_min = MAX(env->vstart, offset); \ for (i = i_min; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ @@ -4781,6 +4840,7 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ } \ *((ETYPE *)vd + H(i)) = *((ETYPE *)vs2 + H(i - offset)); \ } \ + env->vstart = 0; \ /* set tail elements to 1s */ \ vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } @@ -4804,6 +4864,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ uint32_t vma = vext_vma(desc); \ target_ulong i_max, i_min, i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ i_min = MIN(s1 < vlmax ? vlmax - s1 : 0, vl); \ i_max = MAX(i_min, env->vstart); \ for (i = env->vstart; i < i_max; ++i) { \ @@ -4846,6 +4908,8 @@ static void vslide1up_##BITWIDTH(void *vd, void *v0, uint64_t s1, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \ @@ -4895,6 +4959,8 @@ static void vslide1down_##BITWIDTH(void *vd, void *v0, uint64_t s1, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \ @@ -4970,6 +5036,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ uint64_t index; \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \ @@ -5013,6 +5081,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ uint64_t index = s1; \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \ @@ -5074,9 +5144,22 @@ void HELPER(vmvr_v)(void *vd, void *vs2, CPURISCVState *env, uint32_t desc) uint32_t startb = env->vstart * sewb; uint32_t i = startb; + if (startb >= maxsz) { + env->vstart = 0; + return; + } + + if (HOST_BIG_ENDIAN && i % 8 != 0) { + uint32_t j = ROUND_UP(i, 8); + memcpy((uint8_t *)vd + H1(j - 1), + (uint8_t *)vs2 + H1(j - 1), + j - i); + i = j; + } + memcpy((uint8_t *)vd + H1(i), (uint8_t *)vs2 + H1(i), - maxsz - startb); + maxsz - i); env->vstart = 0; } @@ -5094,6 +5177,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \ diff --git a/target/riscv/vector_internals.c b/target/riscv/vector_internals.c index 12f5964fbb..996c21eb31 100644 --- a/target/riscv/vector_internals.c +++ b/target/riscv/vector_internals.c @@ -44,6 +44,8 @@ void do_vext_vv(void *vd, void *v0, void *vs1, void *vs2, uint32_t vma = vext_vma(desc); uint32_t i; + VSTART_CHECK_EARLY_EXIT(env); + for (i = env->vstart; i < vl; i++) { if (!vm && !vext_elem_mask(v0, i)) { /* set masked-off elements to 1s */ @@ -68,6 +70,8 @@ void do_vext_vx(void *vd, void *v0, target_long s1, void *vs2, uint32_t vma = vext_vma(desc); uint32_t i; + VSTART_CHECK_EARLY_EXIT(env); + for (i = env->vstart; i < vl; i++) { if (!vm && !vext_elem_mask(v0, i)) { /* set masked-off elements to 1s */ diff --git a/target/riscv/vector_internals.h b/target/riscv/vector_internals.h index 842765f6c1..9e1e15b575 100644 --- a/target/riscv/vector_internals.h +++ b/target/riscv/vector_internals.h @@ -24,6 +24,13 @@ #include "tcg/tcg-gvec-desc.h" #include "internals.h" +#define VSTART_CHECK_EARLY_EXIT(env) do { \ + if (env->vstart >= env->vl) { \ + env->vstart = 0; \ + return; \ + } \ +} while (0) + static inline uint32_t vext_nf(uint32_t desc) { return FIELD_EX32(simd_data(desc), VDATA, NF); @@ -151,6 +158,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ uint32_t vma = vext_vma(desc); \ uint32_t i; \ \ + VSTART_CHECK_EARLY_EXIT(env); \ + \ for (i = env->vstart; i < vl; i++) { \ if (!vm && !vext_elem_mask(v0, i)) { \ /* set masked-off elements to 1s */ \