diff --git a/dv/cosim/spike_cosim.cc b/dv/cosim/spike_cosim.cc index 32537875ed..c36cd6f031 100644 --- a/dv/cosim/spike_cosim.cc +++ b/dv/cosim/spike_cosim.cc @@ -217,6 +217,10 @@ bool SpikeCosim::step(uint32_t write_reg, uint32_t write_reg_data, uint32_t pc, // (If the current step causes a synchronous trap, it will be // recorded against the current pc) initial_spike_pc = (processor->get_state()->pc & 0xffffffff); + // Debug-mode state the instruction executes in; a trap on a stepped + // instruction enters debug mode within this same step, but the cpuctrlsts + // flag update is decided by the mode at trap take. + bool initial_spike_debug_mode = processor->get_state()->debug_mode; processor->step(1); // ISS @@ -277,7 +281,7 @@ bool SpikeCosim::step(uint32_t write_reg, uint32_t write_reg_data, uint32_t pc, return false; } - handle_cpuctrl_exception_entry(); + handle_cpuctrl_exception_entry(initial_spike_debug_mode); // This is all the checking possible when consider a // synchronously-trapping instruction that never retired. @@ -533,8 +537,8 @@ void SpikeCosim::leave_nmi_mode() { #endif } -void SpikeCosim::handle_cpuctrl_exception_entry() { - if (!processor->get_state()->debug_mode) { +void SpikeCosim::handle_cpuctrl_exception_entry(bool debug_mode_at_trap) { + if (!debug_mode_at_trap) { bool old_sync_exc_seen = change_cpuctrlsts_sync_exc_seen(true); if (old_sync_exc_seen) { set_cpuctrlsts_double_fault_seen(); diff --git a/dv/cosim/spike_cosim.h b/dv/cosim/spike_cosim.h index 2849206e0c..d9f50f3f52 100644 --- a/dv/cosim/spike_cosim.h +++ b/dv/cosim/spike_cosim.h @@ -88,7 +88,7 @@ class SpikeCosim : public simif_t, public Cosim { bool change_cpuctrlsts_sync_exc_seen(bool flag); void set_cpuctrlsts_double_fault_seen(); - void handle_cpuctrl_exception_entry(); + void handle_cpuctrl_exception_entry(bool debug_mode_at_trap); void initial_proc_setup(uint32_t start_pc, uint32_t start_mtvec, uint32_t mhpm_counter_num); diff --git a/dv/uvm/core_ibex/directed_tests/directed_testlist.yaml b/dv/uvm/core_ibex/directed_tests/directed_testlist.yaml index 49eee3562a..9f5b120731 100644 --- a/dv/uvm/core_ibex/directed_tests/directed_testlist.yaml +++ b/dv/uvm/core_ibex/directed_tests/directed_testlist.yaml @@ -475,18 +475,18 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32mi/scall.S config: riscv-tests -- test: shamt +- test: sh-misaligned desc: > - riscv test - shamt + riscv test - sh-misaligned iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32mi/shamt.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32mi/sh-misaligned.S config: riscv-tests -- test: sh-misaligned +- test: shamt desc: > - riscv test - sh-misaligned + riscv test - shamt iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32mi/sh-misaligned.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32mi/shamt.S config: riscv-tests - test: sw-misaligned @@ -524,6 +524,13 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32um/divu.S config: riscv-tests +- test: mul + desc: > + riscv test - mul + iterations: 1 + test_srcs: ../../../../vendor/riscv-tests/isa/rv32um/mul.S + config: riscv-tests + - test: mulh desc: > riscv test - mulh @@ -545,13 +552,6 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32um/mulhu.S config: riscv-tests -- test: mul - desc: > - riscv test - mul - iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32um/mul.S - config: riscv-tests - - test: rem desc: > riscv test - rem @@ -566,13 +566,6 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32um/remu.S config: riscv-tests -- test: addi - desc: > - riscv test - addi - iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/addi.S - config: riscv-tests - - test: add desc: > riscv test - add @@ -580,11 +573,11 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/add.S config: riscv-tests -- test: andi +- test: addi desc: > - riscv test - andi + riscv test - addi iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/andi.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/addi.S config: riscv-tests - test: and @@ -594,6 +587,13 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/and.S config: riscv-tests +- test: andi + desc: > + riscv test - andi + iterations: 1 + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/andi.S + config: riscv-tests + - test: auipc desc: > riscv test - auipc @@ -650,18 +650,18 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/fence_i.S config: riscv-tests -- test: jalr +- test: jal desc: > - riscv test - jalr + riscv test - jal iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/jalr.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/jal.S config: riscv-tests -- test: jal +- test: jalr desc: > - riscv test - jal + riscv test - jalr iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/jal.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/jalr.S config: riscv-tests - test: lb @@ -706,18 +706,18 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/lw.S config: riscv-tests -- test: ori +- test: or desc: > - riscv test - ori + riscv test - or iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/ori.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/or.S config: riscv-tests -- test: or +- test: ori desc: > - riscv test - or + riscv test - ori iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/or.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/ori.S config: riscv-tests - test: sb @@ -741,6 +741,13 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/simple.S config: riscv-tests +- test: sll + desc: > + riscv test - sll + iterations: 1 + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/sll.S + config: riscv-tests + - test: slli desc: > riscv test - slli @@ -748,11 +755,11 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/slli.S config: riscv-tests -- test: sll +- test: slt desc: > - riscv test - sll + riscv test - slt iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/sll.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/slt.S config: riscv-tests - test: slti @@ -769,18 +776,18 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/sltiu.S config: riscv-tests -- test: slt +- test: sltu desc: > - riscv test - slt + riscv test - sltu iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/slt.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/sltu.S config: riscv-tests -- test: sltu +- test: sra desc: > - riscv test - sltu + riscv test - sra iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/sltu.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/sra.S config: riscv-tests - test: srai @@ -790,11 +797,11 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/srai.S config: riscv-tests -- test: sra +- test: srl desc: > - riscv test - sra + riscv test - srl iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/sra.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/srl.S config: riscv-tests - test: srli @@ -804,13 +811,6 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/srli.S config: riscv-tests -- test: srl - desc: > - riscv test - srl - iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/srl.S - config: riscv-tests - - test: sub desc: > riscv test - sub @@ -825,18 +825,18 @@ test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/sw.S config: riscv-tests -- test: xori +- test: xor desc: > - riscv test - xori + riscv test - xor iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/xori.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/xor.S config: riscv-tests -- test: xor +- test: xori desc: > - riscv test - xor + riscv test - xori iterations: 1 - test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/xor.S + test_srcs: ../../../../vendor/riscv-tests/isa/rv32ui/xori.S config: riscv-tests # Test-suite: riscv-arch-tests diff --git a/dv/uvm/core_ibex/directed_tests/gen_testlist.py b/dv/uvm/core_ibex/directed_tests/gen_testlist.py index 2ae1487121..c3bde9c97d 100644 --- a/dv/uvm/core_ibex/directed_tests/gen_testlist.py +++ b/dv/uvm/core_ibex/directed_tests/gen_testlist.py @@ -432,13 +432,11 @@ def append_directed_testlist(tests, test_suite, test_suite_name, is_assembly): # Test-suite: {test_suite_name} '''.format(test_suite_name = test_suite_name) extension = '.S' if is_assembly else '.c' - extension_grep = ' | egrep .S' if is_assembly else ' | egrep .c' for test_group_name in tests: - available_tests = os.popen('ls '+test_suite+test_group_name+extension_grep).read() - available_testlist = [] - for test in available_tests.split('\n')[:-1]: - available_testlist.append(test) + # Sort the directory listing so the output does not depend on the locale. + available_testlist = sorted(f for f in os.listdir(test_suite + test_group_name) + if f.endswith(extension)) for test_name_str in available_testlist: test_name = test_name_str.split(extension)[0] testlist_string = testlist_string + ''' @@ -455,11 +453,8 @@ def append_directed_testlist(tests, test_suite, test_suite_name, is_assembly): f.write(testlist_string) def list_tests(dir): - testlist_str = os.popen('ls '+dir).read() - testlist = [] - for test in testlist_str.split('\n')[:-1]: - testlist.append(test) - print(testlist) + testlist = sorted(os.listdir(dir)) + print(testlist) return testlist def _main() -> int: diff --git a/dv/uvm/core_ibex/riscv_dv_extension/ibex_asm_program_gen.sv b/dv/uvm/core_ibex/riscv_dv_extension/ibex_asm_program_gen.sv index 4f7544b09a..d8fa45f9fb 100644 --- a/dv/uvm/core_ibex/riscv_dv_extension/ibex_asm_program_gen.sv +++ b/dv/uvm/core_ibex/riscv_dv_extension/ibex_asm_program_gen.sv @@ -75,6 +75,9 @@ class ibex_asm_program_gen extends riscv_asm_program_gen; $sformatf("addi x%0d, x%0d, 4", cfg.gpr[0], cfg.gpr[0]), $sformatf("csrw 0x%0x, x%0d", MEPC, cfg.gpr[0]) }; + // The trap dispatcher pushed a kernel-stack frame on entry; a returning + // handler must pop it, or every ECALL leaks one frame. + pop_gpr_from_kernel_stack(MSTATUS, MSCRATCH, cfg.mstatus_mprv, cfg.sp, cfg.tp, instr); instr.push_back("mret"); gen_section(get_label("ecall_handler", hart), instr); endfunction diff --git a/dv/uvm/core_ibex/riscv_dv_extension/testlist.yaml b/dv/uvm/core_ibex/riscv_dv_extension/testlist.yaml index 3c2a447480..450bf739a5 100644 --- a/dv/uvm/core_ibex/riscv_dv_extension/testlist.yaml +++ b/dv/uvm/core_ibex/riscv_dv_extension/testlist.yaml @@ -653,11 +653,11 @@ +no_ebreak=0 +no_ecall=0 +no_branch_jump=0 - +instr_cnt=10000 + +instr_cnt=2000 +no_csr_instr=0 +randomize_csr=1 +gen_all_csrs_by_default=1 - +add_csr_write=MSTATUS,MEPC,MCAUSE,MTVAL,0x7c0,0x7c1 + +add_csr_write=0x7c0,0x7c1 +no_fence=0 +no_wfi=0 +num_of_sub_program=1 diff --git a/dv/uvm/core_ibex/scripts/directed_test_schema.py b/dv/uvm/core_ibex/scripts/directed_test_schema.py index 7cd4faf71f..3730ba7cb8 100755 --- a/dv/uvm/core_ibex/scripts/directed_test_schema.py +++ b/dv/uvm/core_ibex/scripts/directed_test_schema.py @@ -8,7 +8,7 @@ import sys import pydantic import pathlib3x as pathlib -from typing import List, Any +from typing import List, Any, Optional import scripts_lib @@ -52,6 +52,11 @@ class Config: # noqa rtl_test: str rtl_params: dict timeout_s: pydantic.conint(gt=0) + # Optional plusargs appended to the simulator command line. run_rtl.py + # already reads 'sim_opts' from the test options for both test types, but + # for a directed test the options come from this schema, so without this + # field pydantic silently dropped it. + sim_opts: Optional[str] = None # Directed Test Build Options gcc_opts: str # any options that don't specify a path eg. "-O3 -g -static" diff --git a/dv/uvm/core_ibex/scripts/ibex_cmd.py b/dv/uvm/core_ibex/scripts/ibex_cmd.py index 0d8c5346e2..9243bc56a1 100644 --- a/dv/uvm/core_ibex/scripts/ibex_cmd.py +++ b/dv/uvm/core_ibex/scripts/ibex_cmd.py @@ -182,8 +182,8 @@ def filter_tests_by_config(cfg: ibex_config.Config, f"rtl_params of {p_val}, which doesn't contain the " f"expected '{config_val}'.") break - - # The test is accepted if we got this far + else: + # The test is accepted if every parameter matched filtered_test_list.append(test) return filtered_test_list diff --git a/dv/uvm/core_ibex/tests/core_ibex_seq_lib.sv b/dv/uvm/core_ibex/tests/core_ibex_seq_lib.sv index 97be142bb2..7f828e9627 100644 --- a/dv/uvm/core_ibex/tests/core_ibex_seq_lib.sv +++ b/dv/uvm/core_ibex/tests/core_ibex_seq_lib.sv @@ -175,11 +175,17 @@ class debug_seq extends core_base_seq#(irq_seq_item); `uvm_object_new int unsigned drop_delay = 75; + // Longest time the request is held waiting for the core to enter debug mode. + int unsigned hold_timeout_cycles = 5000; + virtual core_ibex_rvfi_if rvfi_vif; virtual task body(); if (!uvm_config_db#(virtual core_ibex_dut_probe_if)::get(null, "", "dut_if", dut_vif)) begin `uvm_fatal(get_full_name(), "Cannot get dut_if") end + if (!uvm_config_db#(virtual core_ibex_rvfi_if)::get(null, "", "rvfi_if", rvfi_vif)) begin + `uvm_fatal(get_full_name(), "Cannot get rvfi_if") + end dut_vif.dut_cb.debug_req <= 1'b0; super.body(); endtask @@ -188,6 +194,22 @@ class debug_seq extends core_base_seq#(irq_seq_item); `uvm_info(get_full_name(), "Sending debug request", UVM_HIGH) dut_vif.dut_cb.debug_req <= 1'b1; clk_vif.wait_clks(drop_delay); + // A debug module keeps haltreq asserted until the hart reports halted, which the debug ROM does + // with its first instructions. Keep the request up until the core has retired an instruction + // that reports the request on RVFI (the first debug ROM instruction, or the next retirement if + // the core was already in debug mode), so a request that wakes the core from WFI is still + // asserted when that instruction enters ID, where RVFI samples it for the cosim. + `DV_SPINWAIT_EXIT(begin + wait (dut_vif.dut_cb.debug_mode == 1'b1); + do @(rvfi_vif.monitor_cb); + while (!(rvfi_vif.monitor_cb.valid && rvfi_vif.monitor_cb.ext_debug_req)); + end, + begin + clk_vif.wait_clks(hold_timeout_cycles); + `uvm_error(get_full_name(), + "No retirement reported the debug request before the hold timeout") + end, + "") dut_vif.dut_cb.debug_req <= 1'b0; endtask diff --git a/vendor/google_riscv-dv/src/riscv_debug_rom_gen.sv b/vendor/google_riscv-dv/src/riscv_debug_rom_gen.sv index 767540bd38..2b6f544894 100644 --- a/vendor/google_riscv-dv/src/riscv_debug_rom_gen.sv +++ b/vendor/google_riscv-dv/src/riscv_debug_rom_gen.sv @@ -116,9 +116,16 @@ class riscv_debug_rom_gen extends riscv_asm_program_gen; endfunction // Generate exception handling routine for debug ROM - // TODO(udinator) - remains empty for now, only a DRET + // Route through debug_end so the GPR frame pushed at debug_rom entry is + // unwound before dret; dpc is untouched by in-debug exceptions, so the + // interrupted program resumes correctly. virtual function void gen_debug_exception_handler(); - str = {"dret"}; + if (cfg.gen_debug_section) begin + str = {$sformatf("j %0sdebug_end", hart_prefix(hart))}; + end else begin + // No debug_end section exists in this configuration; nothing was pushed. + str = {"dret"}; + end gen_section($sformatf("%0sdebug_exception", hart_prefix(hart)), str); endfunction diff --git a/vendor/google_riscv-dv/src/riscv_instr_pkg.sv b/vendor/google_riscv-dv/src/riscv_instr_pkg.sv index 12f248b74e..0c356b51c2 100644 --- a/vendor/google_riscv-dv/src/riscv_instr_pkg.sv +++ b/vendor/google_riscv-dv/src/riscv_instr_pkg.sv @@ -1381,7 +1381,18 @@ package riscv_instr_pkg; riscv_reg_t tp, ref string instr[$]); string store_instr = (XLEN == 32) ? "sw" : "sd"; - if (scratch inside {implemented_csr}) begin + // On RV32 with BARE translation, claim the whole frame (USP slot plus 32 GPR slots) in a + // single instruction and publish the claim in gpr.TP before any slot is written, so a debug + // entry taken mid-push allocates below this frame instead of on top of it. Other + // configurations keep the original sequence. + bit atomic_claim = (scratch inside {implemented_csr}) && (XLEN == 32) && (SATP_MODE == BARE); + if (atomic_claim) begin + instr.push_back($sformatf("addi x%0d, x%0d, -132", tp, tp)); + // Push USP from gpr.SP onto the kernel stack (the top slot of the frame) + instr.push_back($sformatf("%0s x%0d, 128(x%0d)", store_instr, sp, tp)); + // Move KSP to gpr.SP + instr.push_back($sformatf("add x%0d, x%0d, zero", sp, tp)); + end else if (scratch inside {implemented_csr}) begin // Push USP from gpr.SP onto the kernel stack instr.push_back($sformatf("addi x%0d, x%0d, -4", tp, tp)); instr.push_back($sformatf("%0s x%0d, (x%0d)", store_instr, sp, tp)); @@ -1408,8 +1419,10 @@ package riscv_instr_pkg; end end // Push all GPRs (except for x0) to kernel stack - // (gpr.SP currently holds the KSP) - instr.push_back($sformatf("addi x%0d, x%0d, -%0d", sp, sp, 32 * (XLEN/8))); + // (gpr.SP currently holds the KSP; with an atomic claim the frame is already reserved) + if (!atomic_claim) begin + instr.push_back($sformatf("addi x%0d, x%0d, -%0d", sp, sp, 32 * (XLEN/8))); + end for(int i = 1; i < 32; i++) begin instr.push_back($sformatf("%0s x%0d, %0d(x%0d)", store_instr, i, i * (XLEN/8), sp)); end @@ -1426,12 +1439,21 @@ package riscv_instr_pkg; riscv_reg_t tp, ref string instr[$]); string load_instr = (XLEN == 32) ? "lw" : "ld"; + // Mirror of the claim made in push_gpr_to_kernel_stack. + bit atomic_claim = (scratch inside {implemented_csr}) && (XLEN == 32) && (SATP_MODE == BARE); // Move KSP to gpr.SP instr.push_back($sformatf("add x%0d, x%0d, zero", sp, tp)); // Pop GPRs from kernel stack for(int i = 1; i < 32; i++) begin instr.push_back($sformatf("%0s x%0d, %0d(x%0d)", load_instr, i, i * (XLEN/8), sp)); end + if (atomic_claim) begin + // Restore USP while the claim is still held, then release the USP slot and the frame in a + // single instruction (mirrors the push-side claim). + instr.push_back($sformatf("%0s x%0d, 128(x%0d)", load_instr, sp, tp)); + instr.push_back($sformatf("addi x%0d, x%0d, 132", tp, tp)); + return; + end instr.push_back($sformatf("addi x%0d, x%0d, %0d", sp, sp, 32 * (XLEN/8))); if (scratch inside {implemented_csr}) begin // Move KSP back to gpr.TP diff --git a/vendor/patches/google_riscv-dv/0006-pop-the-debug-frame-on-the-debug-exception-path.patch b/vendor/patches/google_riscv-dv/0006-pop-the-debug-frame-on-the-debug-exception-path.patch new file mode 100644 index 0000000000..fc9be8794d --- /dev/null +++ b/vendor/patches/google_riscv-dv/0006-pop-the-debug-frame-on-the-debug-exception-path.patch @@ -0,0 +1,17 @@ +--- a/src/riscv_debug_rom_gen.sv ++++ b/src/riscv_debug_rom_gen.sv +@@ -118,5 +118,12 @@ class riscv_debug_rom_gen extends riscv_asm_program_gen; + // Generate exception handling routine for debug ROM +- // TODO(udinator) - remains empty for now, only a DRET ++ // Route through debug_end so the GPR frame pushed at debug_rom entry is ++ // unwound before dret; dpc is untouched by in-debug exceptions, so the ++ // interrupted program resumes correctly. + virtual function void gen_debug_exception_handler(); +- str = {"dret"}; ++ if (cfg.gen_debug_section) begin ++ str = {$sformatf("j %0sdebug_end", hart_prefix(hart))}; ++ end else begin ++ // No debug_end section exists in this configuration; nothing was pushed. ++ str = {"dret"}; ++ end + gen_section($sformatf("%0sdebug_exception", hart_prefix(hart)), str); diff --git a/vendor/patches/google_riscv-dv/0007-make-the-kernel-stack-frame-claim-atomic.patch b/vendor/patches/google_riscv-dv/0007-make-the-kernel-stack-frame-claim-atomic.patch new file mode 100644 index 0000000000..7ce06c0162 --- /dev/null +++ b/vendor/patches/google_riscv-dv/0007-make-the-kernel-stack-frame-claim-atomic.patch @@ -0,0 +1,57 @@ +--- a/src/riscv_instr_pkg.sv ++++ b/src/riscv_instr_pkg.sv +@@ -1381,7 +1381,18 @@ package riscv_instr_pkg; + riscv_reg_t tp, + ref string instr[$]); + string store_instr = (XLEN == 32) ? "sw" : "sd"; +- if (scratch inside {implemented_csr}) begin ++ // On RV32 with BARE translation, claim the whole frame (USP slot plus 32 GPR slots) in a ++ // single instruction and publish the claim in gpr.TP before any slot is written, so a debug ++ // entry taken mid-push allocates below this frame instead of on top of it. Other ++ // configurations keep the original sequence. ++ bit atomic_claim = (scratch inside {implemented_csr}) && (XLEN == 32) && (SATP_MODE == BARE); ++ if (atomic_claim) begin ++ instr.push_back($sformatf("addi x%0d, x%0d, -132", tp, tp)); ++ // Push USP from gpr.SP onto the kernel stack (the top slot of the frame) ++ instr.push_back($sformatf("%0s x%0d, 128(x%0d)", store_instr, sp, tp)); ++ // Move KSP to gpr.SP ++ instr.push_back($sformatf("add x%0d, x%0d, zero", sp, tp)); ++ end else if (scratch inside {implemented_csr}) begin + // Push USP from gpr.SP onto the kernel stack + instr.push_back($sformatf("addi x%0d, x%0d, -4", tp, tp)); + instr.push_back($sformatf("%0s x%0d, (x%0d)", store_instr, sp, tp)); +@@ -1408,8 +1419,10 @@ package riscv_instr_pkg; + end + end + // Push all GPRs (except for x0) to kernel stack +- // (gpr.SP currently holds the KSP) +- instr.push_back($sformatf("addi x%0d, x%0d, -%0d", sp, sp, 32 * (XLEN/8))); ++ // (gpr.SP currently holds the KSP; with an atomic claim the frame is already reserved) ++ if (!atomic_claim) begin ++ instr.push_back($sformatf("addi x%0d, x%0d, -%0d", sp, sp, 32 * (XLEN/8))); ++ end + for(int i = 1; i < 32; i++) begin + instr.push_back($sformatf("%0s x%0d, %0d(x%0d)", store_instr, i, i * (XLEN/8), sp)); + end +@@ -1426,12 +1439,21 @@ package riscv_instr_pkg; + riscv_reg_t tp, + ref string instr[$]); + string load_instr = (XLEN == 32) ? "lw" : "ld"; ++ // Mirror of the claim made in push_gpr_to_kernel_stack. ++ bit atomic_claim = (scratch inside {implemented_csr}) && (XLEN == 32) && (SATP_MODE == BARE); + // Move KSP to gpr.SP + instr.push_back($sformatf("add x%0d, x%0d, zero", sp, tp)); + // Pop GPRs from kernel stack + for(int i = 1; i < 32; i++) begin + instr.push_back($sformatf("%0s x%0d, %0d(x%0d)", load_instr, i, i * (XLEN/8), sp)); + end ++ if (atomic_claim) begin ++ // Restore USP while the claim is still held, then release the USP slot and the frame in a ++ // single instruction (mirrors the push-side claim). ++ instr.push_back($sformatf("%0s x%0d, 128(x%0d)", load_instr, sp, tp)); ++ instr.push_back($sformatf("addi x%0d, x%0d, 132", tp, tp)); ++ return; ++ end + instr.push_back($sformatf("addi x%0d, x%0d, %0d", sp, sp, 32 * (XLEN/8))); + if (scratch inside {implemented_csr}) begin + // Move KSP back to gpr.TP