diff options
Diffstat (limited to 'src/hal/intr.zig')
| -rw-r--r-- | src/hal/intr.zig | 965 |
1 files changed, 965 insertions, 0 deletions
diff --git a/src/hal/intr.zig b/src/hal/intr.zig new file mode 100644 index 0000000..38f8789 --- /dev/null +++ b/src/hal/intr.zig @@ -0,0 +1,965 @@ +//! The interrupt controller. The ESP32-P4 has a **CLIC**, not a PLIC and not the Xtensa-style +//! fixed matrix of the older parts: `soc_caps.h:191` defines SOC_INT_CLIC_SUPPORTED 1, and +//! `soc/interrupt_reg.h:16` says so in prose. Three consequences shape this file. +//! +//! **1. Two independent stages.** A peripheral source does not have a CPU interrupt number; it has +//! a *mapping register*. The interrupt matrix at DR_REG_INTERRUPT_CORE0_BASE holds one 6-bit word +//! per source, and writing `line + 16` into it points that source at external CLIC line `line`. +//! The `+ 16` is not decoration: the CLIC's first 16 IDs are the RISC-V internal interrupts +//! (software, timer, external), so the 32 lines a driver may use are IDs 16..47. +//! `hal/interrupt_clic_ll.h:35-48` is the matrix write; the `+ RV_EXTERNAL_INT_OFFSET` that turns a +//! line number into a CLIC ID is one level up, at `riscv/interrupt_clic.c:26`. Per-line control - +//! enable, trigger, priority, pending - is the *other* stage, in the CLIC's own register file at +//! DR_REG_CLIC_CTRL_BASE, and it is indexed by CLIC ID, i.e. by `line + 16` again. +//! +//! **2. The threshold is a memory-mapped register on this die, not the `mintthresh` CSR.** This is +//! the single easiest thing to get wrong here, because every RISC-V CLIC document and every +//! ESP32-P4 rev-3 build says `mintthresh` (CSR 0x347). `soc/interrupt_reg.h:28-40` selects +//! `INTTHRESH_STANDARD 0` under CONFIG_ESP32P4_SELECTS_REV_LESS_V3 - the same condition that +//! selects the `register/hw_ver1` headers this project builds against - and +//! `riscv/csr_clic.h:37-47` then leaves MINTTHRESH_CSR *undefined*. The threshold lives in +//! CLIC_INT_THRESH_REG at 0x2080_0008, bits [31:24] (`soc/clic_reg.h:61-67`). Writing CSR 0x347 on +//! this silicon is not an illegal instruction and not an error; it writes a register the interrupt +//! arbiter does not read, so interrupts stay masked and nothing says why. +//! +//! **3. `regs.INTTHRESH_STANDARD` lies, and must not be used.** The register module is +//! `zig translate-c` over the headers with *no* sdkconfig, so CONFIG_ESP32P4_SELECTS_REV_LESS_V3 is +//! absent there and `interrupt_reg.h` takes its `#else` branch: the translated module contains +//! `pub const INTTHRESH_STANDARD = 1`, which is the wrong answer for this die. (The oracle's C side +//! is compiled against `src/oracle/oracle_sdkconfig.h:25`, which does define it, so IDF's own code +//! there takes the correct branch. The two disagree, deliberately, and only the C side is right +//! about this macro.) Nothing in this file reads it. +//! +//! Nothing below has been run on hardware by the author of this file. What is claimed is that the +//! register arithmetic matches ESP-IDF's at the cited lines, and that `src/oracle/intr_cases.zig` +//! compares the two on the die. Taking an actual interrupt is a behavioural property no register +//! comparison can establish; see the note at the foot of that file. + +const std = @import("std"); +const regs = @import("regs"); +const mmio = @import("mmio"); +const clkrst = @import("clkrst.zig"); + +const Reg = mmio.Reg; +const Field = mmio.Field; + +// ------------------------------------------------------------------------------- geometry + +/// CLIC IDs 0..15 are the RISC-V internal interrupts; a driver cannot have them. IDs 16..47 are the +/// 32 external lines. `riscv/csr_clic.h:28-29` (RV_EXTERNAL_INT_COUNT, RV_EXTERNAL_INT_OFFSET) and +/// `soc/clic_reg.h:14` (CLIC_EXT_INTR_NUM_OFFSET) are three names for these two numbers. +pub const line_count: u32 = 32; +pub const ext_offset: u32 = @intCast(regs.CLIC_EXT_INTR_NUM_OFFSET); +/// 16 internal + 32 external. `hal/interrupt_clic_ll.h:22` RV_TOTAL_INT_COUNT, and the hardware +/// agrees: CLIC_INT_INFO_REG's NUM_INT field reads 48 at reset (`soc/clic_reg.h:54-59`). +pub const total_ids: u32 = 48; + +/// Priority levels. `soc/clic_reg.h:13` NLBITS 3, so 8 levels, held in the *top* 3 bits of the +/// 8-bit CLIC_INT_CTL field. Level 0 is masked by the reset threshold; a usable interrupt wants 1 +/// or more. +pub const NLBITS: u5 = @intCast(regs.NLBITS); +const nlbits_shift: u5 = 8 - NLBITS; +/// The low `8 - NLBITS` bits of a priority/threshold byte are not part of the level and IDF fills +/// them with ones (`riscv/csr_clic.h:59`, NLBITS_TO_BYTE). Reproduced exactly, because the +/// differential compares the whole word. +const nlbits_pad: u32 = (@as(u32, 1) << nlbits_shift) - 1; + +// -------------------------------------------------------------------------- interrupt matrix + +/// Every peripheral interrupt source on this chip, from `soc/interrupts.h` - which opens with +/// "This table is decided by hardware, don't touch this." +/// +/// IDs 0..127 are contiguous and each has a mapping register at `matrix_base + 4*id`: the last of +/// them, `assist_debug` = 127, is INTERRUPT_CORE0_ASSIST_DEBUG_INT_MAP_REG at +0x1FC, which is +/// exactly 4*127. That is the invariant `interrupt_clic_ll.h:46` depends on when it computes the +/// address arithmetically rather than from a table. +/// +/// **The last three exist only on chip revision >= 3.0 and therefore not on this die.** +/// `soc/interrupts.h:155-160` explains the gap: their mapping registers are *not* contiguous with +/// the rest, so IDF gave them IDs 133-135 to make `base + 4*id` land on the right address anyway. +/// The numbering hole at 128..132 is that workaround, not missing hardware. On a pre-v3 part - +/// which is what `regs.ZIG_P4_HW_VER == 1` asserts - routing one of them writes a register that +/// nothing drives. +pub const Source = enum(u8) { + lp_rtc = 0, + lp_wdt = 1, + lp_timer_reg0 = 2, + lp_timer_reg1 = 3, + mb_hp = 4, + mb_lp = 5, + pmu_0 = 6, + pmu_1 = 7, + lp_anaperi = 8, + lp_adc = 9, + lp_gpio = 10, + lp_i2c = 11, + lp_i2s = 12, + lp_spi = 13, + lp_touch = 14, + /// Also spelled ETS_TEMPERATURE_SENSOR_INTR_SOURCE; IDF aliases the two (`interrupts.h:34`). + lp_tsens = 15, + lp_uart = 16, + lp_efuse = 17, + lp_sw = 18, + lp_sysreg = 19, + lp_huk = 20, + sys_icm = 21, + usb_serial_jtag = 22, + sdio_host = 23, + dw_gdma = 24, + spi2 = 25, + spi3 = 26, + i2s0 = 27, + i2s1 = 28, + i2s2 = 29, + uhci0 = 30, + uart0 = 31, + uart1 = 32, + uart2 = 33, + uart3 = 34, + uart4 = 35, + lcd_cam = 36, + adc = 37, + pwm0 = 38, + pwm1 = 39, + twai0 = 40, + twai1 = 41, + twai2 = 42, + rmt = 43, + i2c0 = 44, + i2c1 = 45, + tg0_t0 = 46, + tg0_t1 = 47, + tg0_wdt_level = 48, + tg1_t0 = 49, + tg1_t1 = 50, + tg1_wdt_level = 51, + ledc = 52, + systimer_target0 = 53, + systimer_target1 = 54, + systimer_target2 = 55, + ahb_pdma_in_ch0 = 56, + ahb_pdma_in_ch1 = 57, + ahb_pdma_in_ch2 = 58, + ahb_pdma_out_ch0 = 59, + ahb_pdma_out_ch1 = 60, + ahb_pdma_out_ch2 = 61, + axi_pdma_in_ch0 = 62, + axi_pdma_in_ch1 = 63, + axi_pdma_in_ch2 = 64, + axi_pdma_out_ch0 = 65, + axi_pdma_out_ch1 = 66, + axi_pdma_out_ch2 = 67, + rsa = 68, + aes = 69, + sha = 70, + ecc = 71, + ecdsa = 72, + km = 73, + gpio_intr0 = 74, + gpio_intr1 = 75, + gpio_intr2 = 76, + gpio_intr3 = 77, + gpio_pad_comp = 78, + from_cpu_intr0 = 79, + from_cpu_intr1 = 80, + from_cpu_intr2 = 81, + from_cpu_intr3 = 82, + cache = 83, + mspi = 84, + csi_bridge = 85, + dsi_bridge = 86, + csi = 87, + dsi = 88, + gmii_phy = 89, + lpi = 90, + pmt = 91, + eth_mac = 92, + usb_otg = 93, + usb_otg_endp_multi_proc = 94, + jpeg = 95, + ppa = 96, + core0_trace = 97, + core1_trace = 98, + hp_core_ctrl = 99, + isp = 100, + i3c_mst = 101, + i3c_slv = 102, + usb_otg11_ch0 = 103, + dma2d_in_ch0 = 104, + dma2d_in_ch1 = 105, + dma2d_out_ch0 = 106, + dma2d_out_ch1 = 107, + dma2d_out_ch2 = 108, + psram_mspi = 109, + hp_sysreg = 110, + pcnt = 111, + hp_pau = 112, + hp_parlio_rx = 113, + hp_parlio_tx = 114, + h264_dma2d_out_ch0 = 115, + h264_dma2d_out_ch1 = 116, + h264_dma2d_out_ch2 = 117, + h264_dma2d_out_ch3 = 118, + h264_dma2d_out_ch4 = 119, + h264_dma2d_in_ch0 = 120, + h264_dma2d_in_ch1 = 121, + h264_dma2d_in_ch2 = 122, + h264_dma2d_in_ch3 = 123, + h264_dma2d_in_ch4 = 124, + h264_dma2d_in_ch5 = 125, + h264_reg = 126, + assist_debug = 127, + + /// Chip rev >= 3.0 only - absent on this die. See the note above. + dma2d_in_ch2 = 133, + /// Chip rev >= 3.0 only - absent on this die. + dma2d_out_ch3 = 134, + /// Chip rev >= 3.0 only - absent on this die. + axi_perf_mon = 135, + + /// True on a source that this pre-v3 silicon does not have. + pub inline fn isRev3Only(self: Source) bool { + return @intFromEnum(self) >= 133; + } +}; + +/// The last source ID with a mapping register on pre-v3 silicon. +pub const max_source_id: u8 = @intFromEnum(Source.assist_debug); + +/// Core 0's interrupt matrix. Core 1's is 0x800 above it (`reg_base.h:198-199`) and is not reachable +/// from here: this image runs core 0 only - core 1 is held in reset at power-on +/// (HP_SYS_CLKRST REG_RST_EN_CORE1_GLOBAL defaults to 1) - and routing a source to a core that is +/// not running is a way to lose an interrupt silently rather than loudly. +const matrix_base: u32 = mmio.addr(regs.DR_REG_INTERRUPT_CORE0_BASE); + +/// The mapping register's only field: 6 bits, holding a CLIC ID. Taken from UART0's macro pair +/// because the field is identical in all 128 of them - `interrupt_core0_reg.h` repeats +/// `_INT_MAP` / mask 0x3F / shift 0 for every source. (`INTERRUPT_CORE0_*_INT_MAP_M` is one of the +/// 153 `_M` macros that are broken C inside ESP-IDF and appear here as poisoned decls; the `_S`/`_V` +/// pair is the only usable form, which is what `mmio.Field.of` takes.) +const int_map = Field.of(regs.INTERRUPT_CORE0_UART0_INT_MAP_S, regs.INTERRUPT_CORE0_UART0_INT_MAP_V); + +inline fn mapReg(source_id: u8) Reg { + return Reg.atAddress(matrix_base + 4 * @as(u32, source_id)); +} + +/// Point a peripheral source at an external CLIC line. +/// +/// This is only the matrix half. A routed source still needs `setEnabled(line, true)`, a trigger +/// type, a priority above the threshold, a handler, and mstatus.MIE - `configureLine` does the +/// CLIC-side four in the order the hardware wants. +/// +/// Several sources may share one line; that is the normal way to fit 128 sources into 32 lines, and +/// the handler then has to ask each peripheral whether it was the one. Nothing here prevents it. +pub fn route(source: Source, line: u5) void { + routeId(@intFromEnum(source), line); +} + +/// `route` by raw source ID, for a source this enum does not name. +/// +/// The write is a read-modify-write of the low 6 bits, exactly as `interrupt_clic_ll.h:46` does it +/// (`REG_SET_BITS(DR_REG_INTERRUPT_CORE0_BASE + 4*intr_src, intr_num, RV_INT_MASK)` with +/// RV_INT_MASK 63 at line 25). The upper 26 bits are reserved and preserved. +pub fn routeId(source_id: u8, line: u5) void { + std.debug.assert(source_id <= max_source_id); + mapReg(source_id).modify(.{int_map.is(@as(u32, line) + ext_offset)}); +} + +/// Detach a source from every line. +/// +/// Writes CLIC ID 0, which is `ETS_INVALID_INUM` on this chip (`soc/esp32p4/include/soc/soc.h:251`) +/// and is what `esp_system/port/cpu_start.c:185` writes into all 128 mapping registers at boot. +/// ID 0 is an internal RISC-V interrupt line that the matrix cannot actually drive, so it means +/// "nowhere" rather than "line 0" - note the asymmetry with `route`, which adds 16. +pub fn unroute(source: Source) void { + mapReg(@intFromEnum(source)).modify(.{int_map.is(0)}); +} + +/// Which external line a source is routed to, or null if it is unrouted or points at an internal ID. +pub fn routedLine(source: Source) ?u5 { + const id = mapReg(@intFromEnum(source)).get(int_map); + if (id < ext_offset or id >= ext_offset + line_count) return null; + return @intCast(id - ext_offset); +} + +// ------------------------------------------------------------------------- per-line control + +/// One 32-bit control word per CLIC ID at `DR_REG_CLIC_CTRL_BASE + 4*id` (`soc/clic_reg.h:69`). +/// Indexed by CLIC ID, so every accessor here adds `ext_offset` to the caller's line number. +/// +/// The same word is also described byte-wise by the `BYTE_CLIC_*` macros (clic_reg.h:113-160), and +/// ESP-IDF uses both spellings: `interrupt_clic_ll.h` does 32-bit REG_SET_FIELD, the TEE build does +/// 8-bit stores. They land on the same bits, and each field sits wholly inside one byte, so a +/// 32-bit read-modify-write of one field and a byte store of that byte are indistinguishable in the +/// resulting word. This file uses the 32-bit form throughout. +const clic_ctrl_base: u32 = mmio.addr(regs.DR_REG_CLIC_CTRL_BASE); + +/// Priority, bits [31:24]. Reset value 0x1f (clic_reg.h:70). +const int_ctl = Field.of(regs.CLIC_INT_CTL_S, regs.CLIC_INT_CTL_V); +/// Trigger type, bits [18:17]. +const int_attr_trig = Field.of(regs.CLIC_INT_ATTR_TRIG_S, regs.CLIC_INT_ATTR_TRIG_V); +/// Hardware vectoring: 1 means fetch the handler address from MTVT rather than trapping to mtvec. +const int_attr_shv = Field.of(regs.CLIC_INT_ATTR_SHV_S, regs.CLIC_INT_ATTR_SHV_V); +/// Enable, bit 8. +const int_ie = Field.of(regs.CLIC_INT_IE_S, regs.CLIC_INT_IE_V); +/// Pending, bit 0. Read/write, with asymmetric semantics - see `edgeAck`. +const int_ip = Field.of(regs.CLIC_INT_IP_S, regs.CLIC_INT_IP_V); + +inline fn ctrl(line: u5) Reg { + return Reg.atAddress(clic_ctrl_base + 4 * (@as(u32, line) + ext_offset)); +} + +/// By raw CLIC ID rather than by external line, for the one caller that has to reach the 16 +/// internal IDs: `init`, silencing everything the ROM may have left enabled. +inline fn ctrlRegById(clic_id: u32) Reg { + std.debug.assert(clic_id < total_ids); + return Reg.atAddress(clic_ctrl_base + 4 * clic_id); +} + +/// How a source drives its line. The encoding is a two-bit field whose *low* bit selects +/// level-versus-edge and whose high bit selects the edge, which is why `interrupt_clic_ll.h:60` +/// masks the read with `& 1` to answer "is it edge-triggered": `0b10` is a level interrupt too. +/// (`soc/clic_reg.h:84-88`.) +pub const Trigger = enum(u2) { + level = 0, + rising_edge = 1, + /// 0b10 - low bit clear, so this is a *level* trigger despite the encoding's shape. Present + /// only because the field is two bits wide; no source should be configured with it. + level_alias = 2, + falling_edge = 3, + + pub inline fn isEdge(self: Trigger) bool { + return @intFromEnum(self) & 1 != 0; + } +}; + +pub fn setEnabled(line: u5, on: bool) void { + ctrl(line).modify(.{int_ie.is(@intFromBool(on))}); +} + +pub fn isEnabled(line: u5) bool { + return ctrl(line).get(int_ie) == 1; +} + +pub fn setTrigger(line: u5, t: Trigger) void { + ctrl(line).modify(.{int_attr_trig.is(@intFromEnum(t))}); +} + +pub fn getTrigger(line: u5) Trigger { + return @enumFromInt(ctrl(line).get(int_attr_trig)); +} + +/// Priority 0..7, stored left-aligned in the 8-bit CLIC_INT_CTL field. +/// +/// The stored byte is `priority << (8 - NLBITS)` with the low bits **zero**, which is what +/// `esp_tee_rv_utils.h:112` writes and what `interrupt_clic_ll.h:74` reads back with `>> (8-NLBITS)`. +/// Note the asymmetry with the *threshold*, where IDF fills the same low bits with ones +/// (`csr_clic.h:59`). Copying the threshold's encoding here would leave a different word behind +/// than IDF's, for the same nominal priority. +pub fn setPriority(line: u5, priority: u3) void { + ctrl(line).modify(.{int_ctl.is(@as(u32, priority) << nlbits_shift)}); +} + +pub fn getPriority(line: u5) u3 { + return @intCast(ctrl(line).get(int_ctl) >> nlbits_shift); +} + +/// Hardware vectoring for one line. With SHV set, the CLIC jumps to `MTVT + 4*id` instead of to +/// mtvec's base; `installVectorTable` fills every slot with the same trap entry, so flipping this +/// changes the fetch path and not the code that runs. `interrupt_clic_ll.h:99-102`. +pub fn setVectored(line: u5, on: bool) void { + ctrl(line).modify(.{int_attr_shv.is(@intFromBool(on))}); +} + +pub fn isVectored(line: u5) bool { + return ctrl(line).get(int_attr_shv) == 1; +} + +pub fn isPending(line: u5) bool { + return ctrl(line).get(int_ip) == 1; +} + +/// Acknowledge an edge-triggered interrupt. +/// +/// Writing **1** to IP is what clears it for an edge source. That reads backwards, and clic_reg.h +/// only hints at it - "This bit has different set and clear logic in the case of level interrupt +/// and edge interrupt" (clic_reg.h:106-107) - but ESP-IDF's function that does exactly this store is +/// named `rv_utils_intr_edge_ack` (`esp_private/interrupt_clic.h`, the `REG_SET_BIT(..., CLIC_INT_IP)` +/// at the end of that header). For a *level* source this instead asserts the pending bit, which is +/// how software raises one by hand; there is no acknowledge for a level source at the CLIC at all, +/// the handler must clear the peripheral's own status register. +pub fn edgeAck(line: u5) void { + ctrl(line).modify(.{int_ip.is(1)}); +} + +/// Raise a line from software. Same store as `edgeAck`; the two names exist because the hardware +/// gives one write two meanings depending on `Trigger`. +pub fn setPending(line: u5) void { + ctrl(line).modify(.{int_ip.is(1)}); +} + +/// Bitmask of the 32 external lines that are enabled, one loop over the control words. Mirrors +/// `rv_utils_intr_get_enabled_mask` in `esp_private/interrupt_clic.h`. +pub fn enabledMask() u32 { + var m: u32 = 0; + var i: u5 = 0; + while (true) : (i += 1) { + if (isEnabled(i)) m |= @as(u32, 1) << i; + if (i == line_count - 1) break; + } + return m; +} + +// ----------------------------------------------------------------------------- the threshold + +/// CLIC_INT_THRESH_REG - 0x2080_0008 (`soc/clic_reg.h:61`), **not** the `mintthresh` CSR. See the +/// module comment: on this pre-v3 die `csr_clic.h` does not even define MINTTHRESH_CSR, and a write +/// to CSR 0x347 here is accepted and ignored. +const thresh_reg = Reg.at(regs.CLIC_INT_THRESH_REG); +const cpu_int_thresh = Field.of(regs.CLIC_CPU_INT_THRESH_S, regs.CLIC_CPU_INT_THRESH_V); + +/// Mask every interrupt whose priority is <= `level`. +/// +/// The comparison is **inclusive**: threshold 0 lets priorities 1..7 through, threshold 7 masks +/// everything. `esp_private/interrupt_clic.h:198-203` makes the same point when it computes +/// `mask_int_level_lower_than(n)` as `set_intlevel(n - 1)`. Reset is 0, i.e. open. +/// +/// Two details reproduced from IDF rather than invented: +/// * the byte is `(level << 5) | 0x1f` - the low `8 - NLBITS` bits are filled with **ones** +/// (`csr_clic.h:59`, NLBITS_TO_BYTE), which is the opposite of the per-line priority encoding; +/// * the register is read back immediately afterwards. That is not a paranoid verification, it is +/// ordering: `esp_private/interrupt_clic.h:139-144` records that the CPU does not see the new +/// threshold until the store has actually left the write buffer, and that a load - or about +/// eight nops - is what forces it. Without the load, re-enabling mstatus.MIE on the next +/// instruction can take an interrupt the new threshold was meant to mask. +/// +/// `write` rather than `modify` is deliberate and matches IDF's `REG_WRITE`: CLIC_CPU_INT_THRESH is +/// the register's only field, so there is nothing to preserve. +pub fn setThreshold(level: u3) void { + thresh_reg.write(.{cpu_int_thresh.is((@as(u32, level) << nlbits_shift) | nlbits_pad)}); + _ = thresh_reg.raw(); +} + +pub fn getThreshold() u3 { + return @intCast(thresh_reg.get(cpu_int_thresh) >> nlbits_shift); +} + +// ------------------------------------------------------------- vector table and trap entry + +/// CSR numbers, from `components/riscv/include/riscv/csr_clic.h`: +/// * `MTVT_CSR 0x307` (line 34) - base of the interrupt jump table. +/// * `MTVEC_MODE_CSR 3` (line 22) - the two low bits of mtvec that put the core in CLIC mode. +/// * `MINTSTATUS_CSR 0x346` (`soc/interrupt_reg.h:36`) - **non-standard on this die**; the RISC-V +/// CLIC specification and IDF's rev-3 path both say 0xFB1 (`csr_clic.h:40`). +/// * `MINTTHRESH_CSR 0x347` exists only when INTTHRESH_STANDARD is 1, which it is not here. +pub const mtvt_csr = 0x307; +pub const mintstatus_csr = 0x346; +pub const mtvec_mode_clic = 3; +/// mstatus.MIE. Same bit `clkrst.Guard` manipulates. +const mstatus_mie: u32 = 1 << 3; + +/// A line's handler. Runs with mstatus.MIE clear - this file does not implement nesting - on the +/// interrupted stack, so it must be short and must not use floating point: `trapEntry` saves the +/// integer caller-saved registers and nothing else, and `_start` leaves the FPU enabled, so a +/// handler that touches an f-register corrupts whatever it interrupted. +pub const Handler = *const fn (line: u5) void; + +var handlers: [line_count]?Handler = @splat(null); + +/// Interrupts that arrived on a line with no handler, or on one of the 16 internal CLIC IDs. Not +/// reset by anything here: a non-zero value after a run is the diagnostic. +pub var spurious: u32 = 0; + +/// The CLIC's jump table: one address per CLIC ID, internal and external. +/// +/// 48 entries, and 256-byte aligned because the CLIC requires MTVT to be aligned to a power of two +/// at least as large as the table (4 * 48 = 192 bytes, so 256). The alignment travels with the +/// symbol, so the generated linker script's `.bss ... ALIGN(4)` is not a problem - the linker pads +/// to the input section's own alignment. No dedicated section is needed and build.zig is unchanged. +/// +/// Every slot points at the same `trapEntry`. A per-line stub would save the dispatch load, but it +/// would be 48 near-identical pieces of assembly to be wrong in, and the win is a handful of cycles +/// against a handler call. The table exists because the hardware needs one when SHV is set, not +/// because the entries differ. +var vector_table: [total_ids]u32 align(256) = @splat(0); + +/// What `init` found before it changed anything. Diagnostics, and the only record of the state the +/// bootloader hands over in - every one of these is overwritten by `init` itself, so nothing else +/// can observe them. +pub var boot_state: BootState = .{}; +pub const BootState = struct { + /// mstatus.MIE as handed over. Measured 1 on this board, which is the fact the whole ownership + /// sequence below exists for. + mie: bool = false, + /// Which of the 32 external lines had CLIC_INT_IE set before `init` cleared them. + enabled_lines: u32 = 0, + /// How many of the 128 peripheral sources were pointing at an external line before `init` + /// detached them. + routed_sources: u32 = 0, +}; + +/// Take ownership of the interrupt controller, then point it at this file. +/// +/// **The bootloader hands over with interrupts globally enabled.** Measured: `mie_at_boot=1`. That +/// single fact is why this function is a sequence rather than three CSR writes, and it cost two +/// silent hangs to establish. Two separate hazards follow from it, and clearing MIE only fixes the +/// first: +/// +/// 1. `init(); attach(...)` used to take an interrupt the moment the line's IE bit went up, before +/// the caller had said it was ready. `globalDisable()` first fixes that. +/// +/// 2. **Whatever the ROM had armed is still armed.** The ROM ran with its own mtvec and its own +/// reasons to enable interrupts; the matrix and the CLIC's IE bits are not reset by the handover. +/// The instant this file's caller sets MIE, any line the ROM left enabled vectors into +/// `trapEntry` - on an ID nothing here has a handler for. That increments `spurious` and +/// `mret`s; and if the source is level-triggered and still asserting, the next instruction traps +/// again, forever, with the console silent. The failure looks exactly like "our own line is not +/// being delivered", which is what it was mistaken for. +/// +/// So this function does what ESP-IDF's `core_intr_matrix_clear` does before it trusts the +/// controller (`esp_system/port/cpu_start.c:174-198`), and in the same order: +/// * detach all 128 sources by writing ETS_INVALID_INUM (cpu_start.c:183-189); +/// * clear every line's enable, which IDF gets for free from the CLIC's reset values and this +/// image does not, because the ROM ran first; +/// * set every external line vectored (cpu_start.c:193-196 - "Set all the CPU interrupt lines to +/// vectored by default, as it is on other RISC-V targets"). +/// +/// The register differential could not have found any of this: MIE is a CSR, and the boot state of +/// the matrix is identical on both sides of every comparison because both sides inherit it. +/// +/// Leaves MIE clear. Enabling interrupts stays the caller's decision, via `globalEnable()`. +pub fn init() void { + boot_state.mie = globalEnabled(); + globalDisable(); + + // Record and then silence every line, before anything can be delivered anywhere. + var l: u5 = 0; + while (true) : (l += 1) { + if (isEnabled(l)) boot_state.enabled_lines |= @as(u32, 1) << l; + if (l == line_count - 1) break; + } + // All 48 IDs, internal ones included: this core's interrupts are ours now, and an internal ID + // left enabled is as capable of trapping into `trapEntry` as an external one. + var id: u32 = 0; + while (id < total_ids) : (id += 1) { + ctrlRegById(id).modify(.{int_ie.is(0)}); + } + + // Detach every source. cpu_start.c:183-189 writes ETS_INVALID_INUM (0) to all of them. + var src: u32 = 0; + while (src <= max_source_id) : (src += 1) { + const r = mapReg(@intCast(src)); + const was = r.get(int_map); + if (was >= ext_offset and was < ext_offset + line_count) boot_state.routed_sources += 1; + r.modify(.{int_map.is(0)}); + } + + const entry = @intFromPtr(&trapEntry); + for (&vector_table) |*slot| slot.* = @intCast(entry); + + asm volatile ("csrw %[csr], %[val]" + : + : [csr] "i" (mtvt_csr), + [val] "r" (@as(u32, @intCast(@intFromPtr(&vector_table)))), + ); + // mtvec = base | 3. Mode 3 is what `rv_utils_set_mtvec` writes (`riscv/rv_utils.h:168-171` with + // MTVEC_MODE_CSR from `csr_clic.h:22`) and it is what makes the core interpret mcause and MTVT + // as CLIC rather than as the standard vectored interface. + // + // The hardware uses `mtvec[31:6] << 6` (vectors_clic.S:38-46 spells this out), so it ignores the + // low six bits entirely: a `trapEntry` that were not 64-byte aligned would silently vector up to + // 60 bytes *before* the function. `trapEntryAddress()` exists so a test can prove on the die + // that it is aligned rather than trusting the linker. + asm volatile ("csrw mtvec, %[val]" + : + : [val] "r" (@as(u32, @intCast(entry)) | mtvec_mode_clic), + ); + + // Every external line vectored, matching cpu_start.c:193-196. Also the safer default in its own + // right: SHV=1 is the only delivery path ESP-IDF exercises on this chip, so it is the only one + // the silicon has been validated against. See `configureLine`. + l = 0; + while (true) : (l += 1) { + setVectored(l, true); + if (l == line_count - 1) break; + } + + // Threshold open, matching IDF's RVHAL_INTR_ENABLE_THRESH of 0 (`csr_clic.h:16`): every line + // then gates on its own IE bit and its priority, which is where a driver can reason about it. + setThreshold(0); +} + +/// Diagnostics a behavioural test can print, because the two facts they establish - that the trap +/// entry is 64-byte aligned and that MTVT is 256-byte aligned - are properties of the *link*, and +/// the shipped image is stripped, so there is no way to check them from the host. +pub fn trapEntryAddress() u32 { + return @intCast(@intFromPtr(&trapEntry)); +} + +pub fn vectorTableAddress() u32 { + return @intCast(@intFromPtr(&vector_table)); +} + +pub fn readMtvec() u32 { + return asm volatile ("csrr %[out], mtvec" + : [out] "=r" (-> u32), + ); +} + +pub fn readMtvt() u32 { + return asm volatile ("csrr %[out], %[csr]" + : [out] "=r" (-> u32), + : [csr] "i" (mtvt_csr), + ); +} + +/// mintstatus, CSR 0x346 on this die (`soc/interrupt_reg.h:36`). Bits [31:24] are the current +/// interrupt level: non-zero outside a handler would mean a previous trap never returned. +pub fn readMintstatus() u32 { + return asm volatile ("csrr %[out], %[csr]" + : [out] "=r" (-> u32), + : [csr] "i" (mintstatus_csr), + ); +} + +/// Install (or, with null, remove) the handler for one external line. +/// +/// Done with interrupts masked because the store is a pointer the trap entry may be about to load; +/// `clkrst.maskInterrupts` composes - it restores only the MIE that was there - so this is safe to +/// call from inside an already-masked region. +pub fn setHandler(line: u5, handler: ?Handler) void { + const guard = clkrst.maskInterrupts(); + defer guard.release(); + handlers[line] = handler; +} + +/// Everything one line needs, in the order the hardware wants: handler before enable, so a source +/// that is already pending cannot reach an empty slot; trigger and priority before enable, so the +/// first interrupt is taken under the intended configuration rather than under the reset one. +/// +/// Does not touch the matrix - `route` is the other half - and does not touch mstatus. +pub fn configureLine(line: u5, opts: struct { + handler: Handler, + trigger: Trigger = .level, + /// Must exceed the threshold to ever be taken; the threshold comparison is inclusive. + priority: u3 = 1, + /// Hardware vectoring: fetch the handler address from `MTVT + 4*id` instead of trapping to + /// mtvec's base. + /// + /// **On by default, and the default is the interesting part.** Every slot of the table holds the + /// same `trapEntry`, so this changes only how the core finds that address - which makes the + /// choice look free, and it is not. ESP-IDF sets SHV on all 32 lines at boot + /// (`cpu_start.c:193-196`, "Set all the CPU interrupt lines to vectored by default, as it is on + /// other RISC-V targets") and puts nothing but `j _panic_handler` at mtvec's base + /// (`vectors_clic.S:47-52`). So on this chip the SHV=0 delivery path is one ESP-IDF never takes + /// and therefore one nobody has validated. Defaulting to the path the vendor exercises is worth + /// more than the memory fetch it costs. + /// + /// **Measured on the die: it is the other way round, and the default is now `false`.** + /// + /// With SHV=1 the interrupt was never delivered. The core vectored to a wild address and took an + /// instruction access fault - `mcause=0x30000001` (EXCCODE 1, MINHV clear, so the fault was not + /// during the table fetch), at a `mepc` that differed run to run, with `taken=0` proving the + /// trap entry was never reached. mtvec, MTVT and the table contents were all verified correct + /// beforehand: `mtvec=0x40001383` = entry|3, `mtvt=0x4ff00100`, and every slot holding + /// `0x40001380` = `trapEntry`. + /// + /// The difference from ESP-IDF is *where the table lives*. IDF's `_mtvt_table` is in + /// `.section .exception_vectors_table.text` (`vectors_clic.S:32,67`), i.e. instruction space. + /// This image has no IRAM: it executes from flash through the MMU, so a table that `init()` has + /// to write must live in L2MEM, and the hardware vector fetch does not appear to work from + /// there. Since flash is not writable at run time, there is nowhere else to put it, which makes + /// SHV=0 the correct choice for this memory layout rather than a workaround. + /// + /// With SHV=0 both halves of the behavioural test pass: one interrupt taken, dispatched to the + /// right handler, `last_clic_id=21`, no spurious - and the threshold experiment then shows the + /// memory-mapped register at 0x2080_0008 really is the one the arbiter reads. + /// + /// `true` remains available for an image that gains an IRAM section, and the vector table is + /// still populated so that switching is a one-word change. + vectored: bool = false, +}) void { + setHandler(line, opts.handler); + setTrigger(line, opts.trigger); + setPriority(line, opts.priority); + setVectored(line, opts.vectored); + setEnabled(line, true); +} + +/// Route a source and bring its line up in one call. +pub fn attach(source: Source, line: u5, opts: struct { + handler: Handler, + trigger: Trigger = .level, + priority: u3 = 1, + /// See `configureLine`: vectored is the only path ESP-IDF exercises on this chip. + vectored: bool = false, +}) void { + route(source, line); + configureLine(line, .{ + .handler = opts.handler, + .trigger = opts.trigger, + .priority = opts.priority, + .vectored = opts.vectored, + }); +} + +// --------------------------------------------------------------------------- global enable + +/// mstatus.MIE on. Nothing is taken before this, whatever the CLIC is configured to do. +pub inline fn globalEnable() void { + asm volatile ("csrs mstatus, %[m]" + : + : [m] "r" (mstatus_mie), + ); +} + +pub inline fn globalDisable() void { + asm volatile ("csrc mstatus, %[m]" + : + : [m] "r" (mstatus_mie), + ); +} + +pub inline fn globalEnabled() bool { + const s = asm volatile ("csrr %[out], mstatus" + : [out] "=r" (-> u32), + ); + return s & mstatus_mie != 0; +} + +/// The composable form: mask, do something, restore whatever was there. +/// +/// const guard = intr.mask(); +/// defer guard.release(); +/// +/// This is `clkrst.maskInterrupts` under another name, re-exported rather than reimplemented so +/// that a critical section written against either module is the same critical section. It nests +/// correctly - `release` only sets MIE if MIE was set on entry - which is why `setHandler` can use +/// it without caring who called it. +pub const Guard = clkrst.Guard; +pub inline fn mask() Guard { + return clkrst.maskInterrupts(); +} + +// ------------------------------------------------------------------------------- trap entry + +/// How many times `trapEntry` has dispatched an interrupt, and the last CLIC ID it saw. Diagnostics: +/// with `taken == 0` the trap was never reached at all, which separates "the CLIC did not deliver" +/// from "the handler did not run". +pub var taken: u32 = 0; +pub var last_clic_id: u32 = 0; + +/// An exception - not an interrupt - that reached `trapEntry`. +pub const Fault = struct { + /// Full mcause. Bit 31 is clear by construction here; the low bits are the exception code + /// (1 instruction access, 2 illegal instruction, 5 load access, 7 store access, 11 ecall). + mcause: u32, + /// The instruction that faulted. + mepc: u32, + /// The address or instruction word involved, per exception code. + mtval: u32, +}; + +pub var faults: u32 = 0; +pub var last_fault: Fault = .{ .mcause = 0, .mepc = 0, .mtval = 0 }; + +/// Called with the fault already recorded, before parking. Install one to get the numbers out; +/// `hal` cannot print, so this hook is the only way a fault becomes visible. +/// +/// hal.intr.on_fault = struct { +/// fn f(x: hal.intr.Fault) void { +/// soc.rom.print("MARK FAULT mcause=0x%08x mepc=0x%08x mtval=0x%08x\r\n", +/// .{ x.mcause, x.mepc, x.mtval }); +/// } +/// }.f; +pub var on_fault: ?*const fn (Fault) void = null; + +/// Called from `trapEntry` with the CLIC ID out of mcause. Not part of the API; `export` because +/// the assembly calls it by name. +export fn intrDispatch(clic_id: u32) callconv(.c) void { + taken +%= 1; + last_clic_id = clic_id; + if (clic_id < ext_offset or clic_id >= ext_offset + line_count) { + // One of the 16 internal IDs. This file routes nothing there, so it is a bug elsewhere - + // most likely something the ROM left armed that `init` did not manage to silence. + spurious +%= 1; + return; + } + const line: u5 = @intCast(clic_id - ext_offset); + if (handlers[line]) |h| h(line) else spurious +%= 1; +} + +/// The exception arm of `trapEntry`. Records, reports if a hook is installed, and **parks**. +/// +/// Parking rather than returning is the whole point. `mret` from an exception resumes at the +/// faulting instruction, which faults again immediately: every mistake anywhere in this file used to +/// become an unbreakable loop through the trap entry with the console silent, indistinguishable from +/// "the interrupt was never delivered". It cost a debugging round to tell those apart. ESP-IDF makes +/// the same choice by putting `j _panic_handler` at mtvec's base (`vectors_clic.S:47-52`). +export fn intrFault(mcause: u32, mepc: u32, mtval: u32) callconv(.c) noreturn { + faults +%= 1; + last_fault = .{ .mcause = mcause, .mepc = mepc, .mtval = mtval }; + globalDisable(); + if (on_fault) |f| f(last_fault); + while (true) {} +} + +/// The trap entry: every trap on this core arrives here, interrupt or exception. +/// +/// Reached three ways, and they are not interchangeable: +/// * an **interrupt with SHV = 1**, through `MTVT + 4*id`; +/// * an **interrupt with SHV = 0**, through mtvec's base; +/// * an **exception**, always through mtvec's base, whatever any line's SHV says. +/// +/// 64-byte aligned, and this is a hardware requirement rather than tidiness: in CLIC mode the core +/// computes the target as `mtvec[31:6] << 6` (`vectors_clic.S:38-46` states it outright), so the low +/// six bits of mtvec are not part of the address. A trap entry that were not 64-byte aligned would +/// vector up to 60 bytes *before* this function, into whatever the linker put there. Measured in the +/// linked image: 0x4000_1140, and `trapEntryAddress()` lets a test confirm it on the die, since the +/// shipped image is stripped and there is no symbol to check from the host. +/// +/// **The first thing it does is decide whether this was an interrupt at all.** mcause bit 31 says +/// so, and getting that wrong is not a small bug: an exception whose handler `mret`s resumes at the +/// faulting instruction and faults again, immediately and forever, with the console silent. That +/// failure is indistinguishable from "the interrupt was never delivered", and the two were in fact +/// confused for a debugging round. So the exception arm never returns - see `intrFault`. +/// +/// Saves the integer caller-saved set - ra, t0-t6, a0-a7, sixteen words - and nothing else. Not +/// saved, deliberately and with consequences: +/// * **the f registers.** `src/main.zig`'s `_start` sets mstatus.FS to enable the FPU, so a handler +/// that does float arithmetic silently corrupts the interrupted code. Handlers must stay integer. +/// * **mepc, mcause, mstatus.** In CLIC mode the core stacks the previous privilege, interrupt +/// enable and interrupt level in mcause itself, and `mret` restores them from there - so nothing +/// here may write mcause, and nothing does. They are only at risk from a *nested* trap, and MIE +/// stays clear for the whole sequence, so nothing can nest. That is also why there is no `mnxti` +/// loop: the CLIC's hardware nesting (SOC_INT_HW_NESTED_SUPPORTED, `soc_caps.h:193`) is unused. +/// +/// One consequence of `mret` worth stating because it defeats an obvious defence: it restores +/// mstatus.MIE from MPIE, which the hardware set to 1 on entry. A handler that calls +/// `globalDisable()` therefore does **not** leave interrupts off after it returns. To stop a runaway +/// source the handler must clear it at the peripheral, or call `setEnabled(line, false)`. +export fn trapEntry() align(64) callconv(.naked) noreturn { + asm volatile ( + \\ addi sp, sp, -64 + \\ sw ra, 0(sp) + \\ sw t0, 4(sp) + \\ sw t1, 8(sp) + \\ sw t2, 12(sp) + \\ sw a0, 16(sp) + \\ sw a1, 20(sp) + \\ sw a2, 24(sp) + \\ sw a3, 28(sp) + \\ sw a4, 32(sp) + \\ sw a5, 36(sp) + \\ sw a6, 40(sp) + \\ sw a7, 44(sp) + \\ sw t3, 48(sp) + \\ sw t4, 52(sp) + \\ sw t5, 56(sp) + \\ sw t6, 60(sp) + \\ csrr a0, mcause + // Bit 31 set means interrupt, so mcause read as *signed* is negative. `bgez` therefore + // branches exactly on "this was an exception", in one instruction and with no scratch + // register - which matters here because every scratch register is already spoken for. + \\ bgez a0, 1f + // mcause[11:0] is the CLIC's interrupt ID. Isolated with a shift pair rather than `andi`: + // andi's immediate is 12-bit *signed*, so `andi a0, a0, 0xfff` does not assemble as a + // 12-bit mask - it is -1, and would leave the interrupt bit and the level field in place. + \\ slli a0, a0, 20 + \\ srli a0, a0, 20 + \\ call intrDispatch + \\ lw ra, 0(sp) + \\ lw t0, 4(sp) + \\ lw t1, 8(sp) + \\ lw t2, 12(sp) + \\ lw a0, 16(sp) + \\ lw a1, 20(sp) + \\ lw a2, 24(sp) + \\ lw a3, 28(sp) + \\ lw a4, 32(sp) + \\ lw a5, 36(sp) + \\ lw a6, 40(sp) + \\ lw a7, 44(sp) + \\ lw t3, 48(sp) + \\ lw t4, 52(sp) + \\ lw t5, 56(sp) + \\ lw t6, 60(sp) + \\ addi sp, sp, 64 + \\ mret + // The exception arm. No restore and no `mret`: `intrFault` is noreturn, because resuming + // would re-execute the faulting instruction. The saved registers stay on the stack, which + // costs 64 bytes that are never reclaimed and is the correct trade for a path that ends in + // a parked core with the numbers printed. + \\1: + \\ csrr a1, mepc + \\ csrr a2, mtval + \\ call intrFault + ); +} + +// ------------------------------------------------------------------------------------ tests + +test "the enum's IDs are the offsets of the matrix registers they name" { + // The whole of `routeId` rests on `map_reg_addr == base + 4*id`. These four are checked against + // the addresses ESP-IDF's own interrupt_core0_reg.h computes, which is an independent path: + // IDF wrote the offset as a literal per source, this file multiplies. + try std.testing.expectEqual(@as(u32, 0x7c), 4 * @as(u32, @intFromEnum(Source.uart0))); + try std.testing.expectEqual(@as(u32, 0xb0), 4 * @as(u32, @intFromEnum(Source.i2c0))); + try std.testing.expectEqual(@as(u32, 0xd0), 4 * @as(u32, @intFromEnum(Source.ledc))); + try std.testing.expectEqual(@as(u32, 0x1fc), 4 * @as(u32, @intFromEnum(Source.assist_debug))); +} + +test "rev-3-only sources are flagged and the pre-v3 ones are not" { + try std.testing.expect(Source.axi_perf_mon.isRev3Only()); + try std.testing.expect(Source.dma2d_in_ch2.isRev3Only()); + try std.testing.expect(!Source.assist_debug.isRev3Only()); + try std.testing.expect(!Source.dma2d_in_ch1.isRev3Only()); +} + +test "priority and threshold use different encodings of the same three bits" { + // Priority pads low with zeros, threshold pads low with ones. Getting these the same way round + // is the mistake this test exists to catch. + const priority_byte = @as(u32, 5) << nlbits_shift; + const threshold_byte = (@as(u32, 5) << nlbits_shift) | nlbits_pad; + try std.testing.expectEqual(@as(u32, 0xa0), priority_byte); + try std.testing.expectEqual(@as(u32, 0xbf), threshold_byte); + try std.testing.expectEqual(@as(u32, 5), priority_byte >> nlbits_shift); + try std.testing.expectEqual(@as(u32, 5), threshold_byte >> nlbits_shift); +} + +test "trigger's low bit, not its value, decides edge versus level" { + try std.testing.expect(Trigger.rising_edge.isEdge()); + try std.testing.expect(Trigger.falling_edge.isEdge()); + try std.testing.expect(!Trigger.level.isEdge()); + try std.testing.expect(!Trigger.level_alias.isEdge()); +} + +test "the vector table is aligned to a power of two above its own size" { + try std.testing.expectEqual(@as(usize, 256), @alignOf(@TypeOf(vector_table))); + try std.testing.expect(@sizeOf(@TypeOf(vector_table)) <= 256); +} + +test "mcause's sign bit is what separates an interrupt from an exception" { + // The trap entry branches on `bgez mcause`, which is only correct if bit 31 is the interrupt + // flag and the value is read signed. Spelled out here because the asm cannot say it. + const interrupt_mcause: u32 = 0x8000_0015; // CLIC ID 21 = external line 5 + const exception_mcause: u32 = 0x0000_0002; // illegal instruction + try std.testing.expect(@as(i32, @bitCast(interrupt_mcause)) < 0); + try std.testing.expect(@as(i32, @bitCast(exception_mcause)) >= 0); + // And the ID extraction the two shifts perform. + try std.testing.expectEqual(@as(u32, 21), (interrupt_mcause << 20) >> 20); +} + +test "mtvec's mode bits do not collide with a 64-byte-aligned base" { + // The hardware target is `mtvec[31:6] << 6`, so the mode goes in bits the base cannot use - + // but only if the base really is 64-byte aligned. This is the arithmetic `init` performs; + // whether the *linked* trapEntry satisfies it is a fact about the link, and + // `trapEntryAddress()` is how a test on the die checks that, the image being stripped. + const aligned_base: u32 = 0x4000_1200; + const mtvec = aligned_base | mtvec_mode_clic; + try std.testing.expectEqual(aligned_base, (mtvec >> 6) << 6); + // A base one instruction short of alignment vectors 60 bytes early, silently. + const bad_base: u32 = 0x4000_1204; + try std.testing.expect(((bad_base | mtvec_mode_clic) >> 6) << 6 != bad_base); +} |