//! The interrupt controller. The ESP32-P4 has a **CLIC**, not a PLIC and not the Xtensa-style //! fixed matrix of the older parts: `soc_caps.h:191` defines SOC_INT_CLIC_SUPPORTED 1, and //! `soc/interrupt_reg.h:16` says so in prose. Three consequences shape this file. //! //! **1. Two independent stages.** A peripheral source does not have a CPU interrupt number; it has //! a *mapping register*. The interrupt matrix at DR_REG_INTERRUPT_CORE0_BASE holds one 6-bit word //! per source, and writing `line + 16` into it points that source at external CLIC line `line`. //! The `+ 16` is not decoration: the CLIC's first 16 IDs are the RISC-V internal interrupts //! (software, timer, external), so the 32 lines a driver may use are IDs 16..47. //! `hal/interrupt_clic_ll.h:35-48` is the matrix write; the `+ RV_EXTERNAL_INT_OFFSET` that turns a //! line number into a CLIC ID is one level up, at `riscv/interrupt_clic.c:26`. Per-line control - //! enable, trigger, priority, pending - is the *other* stage, in the CLIC's own register file at //! DR_REG_CLIC_CTRL_BASE, and it is indexed by CLIC ID, i.e. by `line + 16` again. //! //! **2. The threshold is a memory-mapped register on this die, not the `mintthresh` CSR.** This is //! the single easiest thing to get wrong here, because every RISC-V CLIC document and every //! ESP32-P4 rev-3 build says `mintthresh` (CSR 0x347). `soc/interrupt_reg.h:28-40` selects //! `INTTHRESH_STANDARD 0` under CONFIG_ESP32P4_SELECTS_REV_LESS_V3 - the same condition that //! selects the `register/hw_ver1` headers this project builds against - and //! `riscv/csr_clic.h:37-47` then leaves MINTTHRESH_CSR *undefined*. The threshold lives in //! CLIC_INT_THRESH_REG at 0x2080_0008, bits [31:24] (`soc/clic_reg.h:61-67`). Writing CSR 0x347 on //! this silicon is not an illegal instruction and not an error; it writes a register the interrupt //! arbiter does not read, so interrupts stay masked and nothing says why. //! //! **3. `regs.INTTHRESH_STANDARD` lies, and must not be used.** The register module is //! `zig translate-c` over the headers with *no* sdkconfig, so CONFIG_ESP32P4_SELECTS_REV_LESS_V3 is //! absent there and `interrupt_reg.h` takes its `#else` branch: the translated module contains //! `pub const INTTHRESH_STANDARD = 1`, which is the wrong answer for this die. (The oracle's C side //! is compiled against `src/oracle/oracle_sdkconfig.h:25`, which does define it, so IDF's own code //! there takes the correct branch. The two disagree, deliberately, and only the C side is right //! about this macro.) Nothing in this file reads it. //! //! Nothing below has been run on hardware by the author of this file. What is claimed is that the //! register arithmetic matches ESP-IDF's at the cited lines, and that `src/oracle/intr_cases.zig` //! compares the two on the die. Taking an actual interrupt is a behavioural property no register //! comparison can establish; see the note at the foot of that file. const std = @import("std"); const regs = @import("regs"); const mmio = @import("mmio"); const clkrst = @import("clkrst.zig"); const Reg = mmio.Reg; const Field = mmio.Field; // ------------------------------------------------------------------------------- geometry /// CLIC IDs 0..15 are the RISC-V internal interrupts; a driver cannot have them. IDs 16..47 are the /// 32 external lines. `riscv/csr_clic.h:28-29` (RV_EXTERNAL_INT_COUNT, RV_EXTERNAL_INT_OFFSET) and /// `soc/clic_reg.h:14` (CLIC_EXT_INTR_NUM_OFFSET) are three names for these two numbers. pub const line_count: u32 = 32; pub const ext_offset: u32 = @intCast(regs.CLIC_EXT_INTR_NUM_OFFSET); /// 16 internal + 32 external. `hal/interrupt_clic_ll.h:22` RV_TOTAL_INT_COUNT, and the hardware /// agrees: CLIC_INT_INFO_REG's NUM_INT field reads 48 at reset (`soc/clic_reg.h:54-59`). pub const total_ids: u32 = 48; /// Priority levels. `soc/clic_reg.h:13` NLBITS 3, so 8 levels, held in the *top* 3 bits of the /// 8-bit CLIC_INT_CTL field. Level 0 is masked by the reset threshold; a usable interrupt wants 1 /// or more. pub const NLBITS: u5 = @intCast(regs.NLBITS); const nlbits_shift: u5 = 8 - NLBITS; /// The low `8 - NLBITS` bits of a priority/threshold byte are not part of the level and IDF fills /// them with ones (`riscv/csr_clic.h:59`, NLBITS_TO_BYTE). Reproduced exactly, because the /// differential compares the whole word. const nlbits_pad: u32 = (@as(u32, 1) << nlbits_shift) - 1; // -------------------------------------------------------------------------- interrupt matrix /// Every peripheral interrupt source on this chip, from `soc/interrupts.h` - which opens with /// "This table is decided by hardware, don't touch this." /// /// IDs 0..127 are contiguous and each has a mapping register at `matrix_base + 4*id`: the last of /// them, `assist_debug` = 127, is INTERRUPT_CORE0_ASSIST_DEBUG_INT_MAP_REG at +0x1FC, which is /// exactly 4*127. That is the invariant `interrupt_clic_ll.h:46` depends on when it computes the /// address arithmetically rather than from a table. /// /// **The last three exist only on chip revision >= 3.0 and therefore not on this die.** /// `soc/interrupts.h:155-160` explains the gap: their mapping registers are *not* contiguous with /// the rest, so IDF gave them IDs 133-135 to make `base + 4*id` land on the right address anyway. /// The numbering hole at 128..132 is that workaround, not missing hardware. On a pre-v3 part - /// which is what `regs.ZIG_P4_HW_VER == 1` asserts - routing one of them writes a register that /// nothing drives. pub const Source = enum(u8) { lp_rtc = 0, lp_wdt = 1, lp_timer_reg0 = 2, lp_timer_reg1 = 3, mb_hp = 4, mb_lp = 5, pmu_0 = 6, pmu_1 = 7, lp_anaperi = 8, lp_adc = 9, lp_gpio = 10, lp_i2c = 11, lp_i2s = 12, lp_spi = 13, lp_touch = 14, /// Also spelled ETS_TEMPERATURE_SENSOR_INTR_SOURCE; IDF aliases the two (`interrupts.h:34`). lp_tsens = 15, lp_uart = 16, lp_efuse = 17, lp_sw = 18, lp_sysreg = 19, lp_huk = 20, sys_icm = 21, usb_serial_jtag = 22, sdio_host = 23, dw_gdma = 24, spi2 = 25, spi3 = 26, i2s0 = 27, i2s1 = 28, i2s2 = 29, uhci0 = 30, uart0 = 31, uart1 = 32, uart2 = 33, uart3 = 34, uart4 = 35, lcd_cam = 36, adc = 37, pwm0 = 38, pwm1 = 39, twai0 = 40, twai1 = 41, twai2 = 42, rmt = 43, i2c0 = 44, i2c1 = 45, tg0_t0 = 46, tg0_t1 = 47, tg0_wdt_level = 48, tg1_t0 = 49, tg1_t1 = 50, tg1_wdt_level = 51, ledc = 52, systimer_target0 = 53, systimer_target1 = 54, systimer_target2 = 55, ahb_pdma_in_ch0 = 56, ahb_pdma_in_ch1 = 57, ahb_pdma_in_ch2 = 58, ahb_pdma_out_ch0 = 59, ahb_pdma_out_ch1 = 60, ahb_pdma_out_ch2 = 61, axi_pdma_in_ch0 = 62, axi_pdma_in_ch1 = 63, axi_pdma_in_ch2 = 64, axi_pdma_out_ch0 = 65, axi_pdma_out_ch1 = 66, axi_pdma_out_ch2 = 67, rsa = 68, aes = 69, sha = 70, ecc = 71, ecdsa = 72, km = 73, gpio_intr0 = 74, gpio_intr1 = 75, gpio_intr2 = 76, gpio_intr3 = 77, gpio_pad_comp = 78, from_cpu_intr0 = 79, from_cpu_intr1 = 80, from_cpu_intr2 = 81, from_cpu_intr3 = 82, cache = 83, mspi = 84, csi_bridge = 85, dsi_bridge = 86, csi = 87, dsi = 88, gmii_phy = 89, lpi = 90, pmt = 91, eth_mac = 92, usb_otg = 93, usb_otg_endp_multi_proc = 94, jpeg = 95, ppa = 96, core0_trace = 97, core1_trace = 98, hp_core_ctrl = 99, isp = 100, i3c_mst = 101, i3c_slv = 102, usb_otg11_ch0 = 103, dma2d_in_ch0 = 104, dma2d_in_ch1 = 105, dma2d_out_ch0 = 106, dma2d_out_ch1 = 107, dma2d_out_ch2 = 108, psram_mspi = 109, hp_sysreg = 110, pcnt = 111, hp_pau = 112, hp_parlio_rx = 113, hp_parlio_tx = 114, h264_dma2d_out_ch0 = 115, h264_dma2d_out_ch1 = 116, h264_dma2d_out_ch2 = 117, h264_dma2d_out_ch3 = 118, h264_dma2d_out_ch4 = 119, h264_dma2d_in_ch0 = 120, h264_dma2d_in_ch1 = 121, h264_dma2d_in_ch2 = 122, h264_dma2d_in_ch3 = 123, h264_dma2d_in_ch4 = 124, h264_dma2d_in_ch5 = 125, h264_reg = 126, assist_debug = 127, /// Chip rev >= 3.0 only - absent on this die. See the note above. dma2d_in_ch2 = 133, /// Chip rev >= 3.0 only - absent on this die. dma2d_out_ch3 = 134, /// Chip rev >= 3.0 only - absent on this die. axi_perf_mon = 135, /// True on a source that this pre-v3 silicon does not have. pub inline fn isRev3Only(self: Source) bool { return @intFromEnum(self) >= 133; } }; /// The last source ID with a mapping register on pre-v3 silicon. pub const max_source_id: u8 = @intFromEnum(Source.assist_debug); /// Core 0's interrupt matrix. Core 1's is 0x800 above it (`reg_base.h:198-199`) and is not reachable /// from here: this image runs core 0 only - core 1 is held in reset at power-on /// (HP_SYS_CLKRST REG_RST_EN_CORE1_GLOBAL defaults to 1) - and routing a source to a core that is /// not running is a way to lose an interrupt silently rather than loudly. const matrix_base: u32 = mmio.addr(regs.DR_REG_INTERRUPT_CORE0_BASE); /// The mapping register's only field: 6 bits, holding a CLIC ID. Taken from UART0's macro pair /// because the field is identical in all 128 of them - `interrupt_core0_reg.h` repeats /// `_INT_MAP` / mask 0x3F / shift 0 for every source. (`INTERRUPT_CORE0_*_INT_MAP_M` is one of the /// 153 `_M` macros that are broken C inside ESP-IDF and appear here as poisoned decls; the `_S`/`_V` /// pair is the only usable form, which is what `mmio.Field.of` takes.) const int_map = Field.of(regs.INTERRUPT_CORE0_UART0_INT_MAP_S, regs.INTERRUPT_CORE0_UART0_INT_MAP_V); inline fn mapReg(source_id: u8) Reg { return Reg.atAddress(matrix_base + 4 * @as(u32, source_id)); } /// Point a peripheral source at an external CLIC line. /// /// This is only the matrix half. A routed source still needs `setEnabled(line, true)`, a trigger /// type, a priority above the threshold, a handler, and mstatus.MIE - `configureLine` does the /// CLIC-side four in the order the hardware wants. /// /// Several sources may share one line; that is the normal way to fit 128 sources into 32 lines, and /// the handler then has to ask each peripheral whether it was the one. Nothing here prevents it. pub fn route(source: Source, line: u5) void { routeId(@intFromEnum(source), line); } /// `route` by raw source ID, for a source this enum does not name. /// /// The write is a read-modify-write of the low 6 bits, exactly as `interrupt_clic_ll.h:46` does it /// (`REG_SET_BITS(DR_REG_INTERRUPT_CORE0_BASE + 4*intr_src, intr_num, RV_INT_MASK)` with /// RV_INT_MASK 63 at line 25). The upper 26 bits are reserved and preserved. pub fn routeId(source_id: u8, line: u5) void { std.debug.assert(source_id <= max_source_id); mapReg(source_id).modify(.{int_map.is(@as(u32, line) + ext_offset)}); } /// Detach a source from every line. /// /// Writes CLIC ID 0, which is `ETS_INVALID_INUM` on this chip (`soc/esp32p4/include/soc/soc.h:251`) /// and is what `esp_system/port/cpu_start.c:185` writes into all 128 mapping registers at boot. /// ID 0 is an internal RISC-V interrupt line that the matrix cannot actually drive, so it means /// "nowhere" rather than "line 0" - note the asymmetry with `route`, which adds 16. pub fn unroute(source: Source) void { mapReg(@intFromEnum(source)).modify(.{int_map.is(0)}); } /// Which external line a source is routed to, or null if it is unrouted or points at an internal ID. pub fn routedLine(source: Source) ?u5 { const id = mapReg(@intFromEnum(source)).get(int_map); if (id < ext_offset or id >= ext_offset + line_count) return null; return @intCast(id - ext_offset); } // ------------------------------------------------------------------------- per-line control /// One 32-bit control word per CLIC ID at `DR_REG_CLIC_CTRL_BASE + 4*id` (`soc/clic_reg.h:69`). /// Indexed by CLIC ID, so every accessor here adds `ext_offset` to the caller's line number. /// /// The same word is also described byte-wise by the `BYTE_CLIC_*` macros (clic_reg.h:113-160), and /// ESP-IDF uses both spellings: `interrupt_clic_ll.h` does 32-bit REG_SET_FIELD, the TEE build does /// 8-bit stores. They land on the same bits, and each field sits wholly inside one byte, so a /// 32-bit read-modify-write of one field and a byte store of that byte are indistinguishable in the /// resulting word. This file uses the 32-bit form throughout. const clic_ctrl_base: u32 = mmio.addr(regs.DR_REG_CLIC_CTRL_BASE); /// Priority, bits [31:24]. Reset value 0x1f (clic_reg.h:70). const int_ctl = Field.of(regs.CLIC_INT_CTL_S, regs.CLIC_INT_CTL_V); /// Trigger type, bits [18:17]. const int_attr_trig = Field.of(regs.CLIC_INT_ATTR_TRIG_S, regs.CLIC_INT_ATTR_TRIG_V); /// Hardware vectoring: 1 means fetch the handler address from MTVT rather than trapping to mtvec. const int_attr_shv = Field.of(regs.CLIC_INT_ATTR_SHV_S, regs.CLIC_INT_ATTR_SHV_V); /// Enable, bit 8. const int_ie = Field.of(regs.CLIC_INT_IE_S, regs.CLIC_INT_IE_V); /// Pending, bit 0. Read/write, with asymmetric semantics - see `edgeAck`. const int_ip = Field.of(regs.CLIC_INT_IP_S, regs.CLIC_INT_IP_V); inline fn ctrl(line: u5) Reg { return Reg.atAddress(clic_ctrl_base + 4 * (@as(u32, line) + ext_offset)); } /// By raw CLIC ID rather than by external line, for the one caller that has to reach the 16 /// internal IDs: `init`, silencing everything the ROM may have left enabled. inline fn ctrlRegById(clic_id: u32) Reg { std.debug.assert(clic_id < total_ids); return Reg.atAddress(clic_ctrl_base + 4 * clic_id); } /// How a source drives its line. The encoding is a two-bit field whose *low* bit selects /// level-versus-edge and whose high bit selects the edge, which is why `interrupt_clic_ll.h:60` /// masks the read with `& 1` to answer "is it edge-triggered": `0b10` is a level interrupt too. /// (`soc/clic_reg.h:84-88`.) pub const Trigger = enum(u2) { level = 0, rising_edge = 1, /// 0b10 - low bit clear, so this is a *level* trigger despite the encoding's shape. Present /// only because the field is two bits wide; no source should be configured with it. level_alias = 2, falling_edge = 3, pub inline fn isEdge(self: Trigger) bool { return @intFromEnum(self) & 1 != 0; } }; pub fn setEnabled(line: u5, on: bool) void { ctrl(line).modify(.{int_ie.is(@intFromBool(on))}); } pub fn isEnabled(line: u5) bool { return ctrl(line).get(int_ie) == 1; } pub fn setTrigger(line: u5, t: Trigger) void { ctrl(line).modify(.{int_attr_trig.is(@intFromEnum(t))}); } pub fn getTrigger(line: u5) Trigger { return @enumFromInt(ctrl(line).get(int_attr_trig)); } /// Priority 0..7, stored left-aligned in the 8-bit CLIC_INT_CTL field. /// /// The stored byte is `priority << (8 - NLBITS)` with the low bits **zero**, which is what /// `esp_tee_rv_utils.h:112` writes and what `interrupt_clic_ll.h:74` reads back with `>> (8-NLBITS)`. /// Note the asymmetry with the *threshold*, where IDF fills the same low bits with ones /// (`csr_clic.h:59`). Copying the threshold's encoding here would leave a different word behind /// than IDF's, for the same nominal priority. pub fn setPriority(line: u5, priority: u3) void { ctrl(line).modify(.{int_ctl.is(@as(u32, priority) << nlbits_shift)}); } pub fn getPriority(line: u5) u3 { return @intCast(ctrl(line).get(int_ctl) >> nlbits_shift); } /// Hardware vectoring for one line. With SHV set, the CLIC jumps to `MTVT + 4*id` instead of to /// mtvec's base; `installVectorTable` fills every slot with the same trap entry, so flipping this /// changes the fetch path and not the code that runs. `interrupt_clic_ll.h:99-102`. pub fn setVectored(line: u5, on: bool) void { ctrl(line).modify(.{int_attr_shv.is(@intFromBool(on))}); } pub fn isVectored(line: u5) bool { return ctrl(line).get(int_attr_shv) == 1; } pub fn isPending(line: u5) bool { return ctrl(line).get(int_ip) == 1; } /// Acknowledge an edge-triggered interrupt. /// /// Writing **1** to IP is what clears it for an edge source. That reads backwards, and clic_reg.h /// only hints at it - "This bit has different set and clear logic in the case of level interrupt /// and edge interrupt" (clic_reg.h:106-107) - but ESP-IDF's function that does exactly this store is /// named `rv_utils_intr_edge_ack` (`esp_private/interrupt_clic.h`, the `REG_SET_BIT(..., CLIC_INT_IP)` /// at the end of that header). For a *level* source this instead asserts the pending bit, which is /// how software raises one by hand; there is no acknowledge for a level source at the CLIC at all, /// the handler must clear the peripheral's own status register. pub fn edgeAck(line: u5) void { ctrl(line).modify(.{int_ip.is(1)}); } /// Raise a line from software. Same store as `edgeAck`; the two names exist because the hardware /// gives one write two meanings depending on `Trigger`. pub fn setPending(line: u5) void { ctrl(line).modify(.{int_ip.is(1)}); } /// Bitmask of the 32 external lines that are enabled, one loop over the control words. Mirrors /// `rv_utils_intr_get_enabled_mask` in `esp_private/interrupt_clic.h`. pub fn enabledMask() u32 { var m: u32 = 0; var i: u5 = 0; while (true) : (i += 1) { if (isEnabled(i)) m |= @as(u32, 1) << i; if (i == line_count - 1) break; } return m; } // ----------------------------------------------------------------------------- the threshold /// CLIC_INT_THRESH_REG - 0x2080_0008 (`soc/clic_reg.h:61`), **not** the `mintthresh` CSR. See the /// module comment: on this pre-v3 die `csr_clic.h` does not even define MINTTHRESH_CSR, and a write /// to CSR 0x347 here is accepted and ignored. const thresh_reg = Reg.at(regs.CLIC_INT_THRESH_REG); const cpu_int_thresh = Field.of(regs.CLIC_CPU_INT_THRESH_S, regs.CLIC_CPU_INT_THRESH_V); /// Mask every interrupt whose priority is <= `level`. /// /// The comparison is **inclusive**: threshold 0 lets priorities 1..7 through, threshold 7 masks /// everything. `esp_private/interrupt_clic.h:198-203` makes the same point when it computes /// `mask_int_level_lower_than(n)` as `set_intlevel(n - 1)`. Reset is 0, i.e. open. /// /// Two details reproduced from IDF rather than invented: /// * the byte is `(level << 5) | 0x1f` - the low `8 - NLBITS` bits are filled with **ones** /// (`csr_clic.h:59`, NLBITS_TO_BYTE), which is the opposite of the per-line priority encoding; /// * the register is read back immediately afterwards. That is not a paranoid verification, it is /// ordering: `esp_private/interrupt_clic.h:139-144` records that the CPU does not see the new /// threshold until the store has actually left the write buffer, and that a load - or about /// eight nops - is what forces it. Without the load, re-enabling mstatus.MIE on the next /// instruction can take an interrupt the new threshold was meant to mask. /// /// `write` rather than `modify` is deliberate and matches IDF's `REG_WRITE`: CLIC_CPU_INT_THRESH is /// the register's only field, so there is nothing to preserve. pub fn setThreshold(level: u3) void { thresh_reg.write(.{cpu_int_thresh.is((@as(u32, level) << nlbits_shift) | nlbits_pad)}); _ = thresh_reg.raw(); } pub fn getThreshold() u3 { return @intCast(thresh_reg.get(cpu_int_thresh) >> nlbits_shift); } // ------------------------------------------------------------- vector table and trap entry /// CSR numbers, from `components/riscv/include/riscv/csr_clic.h`: /// * `MTVT_CSR 0x307` (line 34) - base of the interrupt jump table. /// * `MTVEC_MODE_CSR 3` (line 22) - the two low bits of mtvec that put the core in CLIC mode. /// * `MINTSTATUS_CSR 0x346` (`soc/interrupt_reg.h:36`) - **non-standard on this die**; the RISC-V /// CLIC specification and IDF's rev-3 path both say 0xFB1 (`csr_clic.h:40`). /// * `MINTTHRESH_CSR 0x347` exists only when INTTHRESH_STANDARD is 1, which it is not here. pub const mtvt_csr = 0x307; pub const mintstatus_csr = 0x346; pub const mtvec_mode_clic = 3; /// mstatus.MIE. Same bit `clkrst.Guard` manipulates. const mstatus_mie: u32 = 1 << 3; /// A line's handler. Runs with mstatus.MIE clear - this file does not implement nesting - on the /// interrupted stack, so it must be short and must not use floating point: `trapEntry` saves the /// integer caller-saved registers and nothing else, and `_start` leaves the FPU enabled, so a /// handler that touches an f-register corrupts whatever it interrupted. pub const Handler = *const fn (line: u5) void; var handlers: [line_count]?Handler = @splat(null); /// Interrupts that arrived on a line with no handler, or on one of the 16 internal CLIC IDs. Not /// reset by anything here: a non-zero value after a run is the diagnostic. pub var spurious: u32 = 0; /// The CLIC's jump table: one address per CLIC ID, internal and external. /// /// 48 entries, and 256-byte aligned because the CLIC requires MTVT to be aligned to a power of two /// at least as large as the table (4 * 48 = 192 bytes, so 256). The alignment travels with the /// symbol, so the generated linker script's `.bss ... ALIGN(4)` is not a problem - the linker pads /// to the input section's own alignment. No dedicated section is needed and build.zig is unchanged. /// /// Every slot points at the same `trapEntry`. A per-line stub would save the dispatch load, but it /// would be 48 near-identical pieces of assembly to be wrong in, and the win is a handful of cycles /// against a handler call. The table exists because the hardware needs one when SHV is set, not /// because the entries differ. var vector_table: [total_ids]u32 align(256) = @splat(0); /// What `init` found before it changed anything. Diagnostics, and the only record of the state the /// bootloader hands over in - every one of these is overwritten by `init` itself, so nothing else /// can observe them. pub var boot_state: BootState = .{}; pub const BootState = struct { /// mstatus.MIE as handed over. Measured 1 on this board, which is the fact the whole ownership /// sequence below exists for. mie: bool = false, /// Which of the 32 external lines had CLIC_INT_IE set before `init` cleared them. enabled_lines: u32 = 0, /// How many of the 128 peripheral sources were pointing at an external line before `init` /// detached them. routed_sources: u32 = 0, }; /// Take ownership of the interrupt controller, then point it at this file. /// /// **The bootloader hands over with interrupts globally enabled.** Measured: `mie_at_boot=1`. That /// single fact is why this function is a sequence rather than three CSR writes, and it cost two /// silent hangs to establish. Two separate hazards follow from it, and clearing MIE only fixes the /// first: /// /// 1. `init(); attach(...)` used to take an interrupt the moment the line's IE bit went up, before /// the caller had said it was ready. `globalDisable()` first fixes that. /// /// 2. **Whatever the ROM had armed is still armed.** The ROM ran with its own mtvec and its own /// reasons to enable interrupts; the matrix and the CLIC's IE bits are not reset by the handover. /// The instant this file's caller sets MIE, any line the ROM left enabled vectors into /// `trapEntry` - on an ID nothing here has a handler for. That increments `spurious` and /// `mret`s; and if the source is level-triggered and still asserting, the next instruction traps /// again, forever, with the console silent. The failure looks exactly like "our own line is not /// being delivered", which is what it was mistaken for. /// /// So this function does what ESP-IDF's `core_intr_matrix_clear` does before it trusts the /// controller (`esp_system/port/cpu_start.c:174-198`), and in the same order: /// * detach all 128 sources by writing ETS_INVALID_INUM (cpu_start.c:183-189); /// * clear every line's enable, which IDF gets for free from the CLIC's reset values and this /// image does not, because the ROM ran first; /// * set every external line vectored (cpu_start.c:193-196 - "Set all the CPU interrupt lines to /// vectored by default, as it is on other RISC-V targets"). /// /// The register differential could not have found any of this: MIE is a CSR, and the boot state of /// the matrix is identical on both sides of every comparison because both sides inherit it. /// /// Leaves MIE clear. Enabling interrupts stays the caller's decision, via `globalEnable()`. pub fn init() void { boot_state.mie = globalEnabled(); globalDisable(); // Record and then silence every line, before anything can be delivered anywhere. var l: u5 = 0; while (true) : (l += 1) { if (isEnabled(l)) boot_state.enabled_lines |= @as(u32, 1) << l; if (l == line_count - 1) break; } // All 48 IDs, internal ones included: this core's interrupts are ours now, and an internal ID // left enabled is as capable of trapping into `trapEntry` as an external one. var id: u32 = 0; while (id < total_ids) : (id += 1) { ctrlRegById(id).modify(.{int_ie.is(0)}); } // Detach every source. cpu_start.c:183-189 writes ETS_INVALID_INUM (0) to all of them. var src: u32 = 0; while (src <= max_source_id) : (src += 1) { const r = mapReg(@intCast(src)); const was = r.get(int_map); if (was >= ext_offset and was < ext_offset + line_count) boot_state.routed_sources += 1; r.modify(.{int_map.is(0)}); } const entry = @intFromPtr(&trapEntry); for (&vector_table) |*slot| slot.* = @intCast(entry); asm volatile ("csrw %[csr], %[val]" : : [csr] "i" (mtvt_csr), [val] "r" (@as(u32, @intCast(@intFromPtr(&vector_table)))), ); // mtvec = base | 3. Mode 3 is what `rv_utils_set_mtvec` writes (`riscv/rv_utils.h:168-171` with // MTVEC_MODE_CSR from `csr_clic.h:22`) and it is what makes the core interpret mcause and MTVT // as CLIC rather than as the standard vectored interface. // // The hardware uses `mtvec[31:6] << 6` (vectors_clic.S:38-46 spells this out), so it ignores the // low six bits entirely: a `trapEntry` that were not 64-byte aligned would silently vector up to // 60 bytes *before* the function. `trapEntryAddress()` exists so a test can prove on the die // that it is aligned rather than trusting the linker. asm volatile ("csrw mtvec, %[val]" : : [val] "r" (@as(u32, @intCast(entry)) | mtvec_mode_clic), ); // Every external line vectored, matching cpu_start.c:193-196. Also the safer default in its own // right: SHV=1 is the only delivery path ESP-IDF exercises on this chip, so it is the only one // the silicon has been validated against. See `configureLine`. l = 0; while (true) : (l += 1) { setVectored(l, true); if (l == line_count - 1) break; } // Threshold open, matching IDF's RVHAL_INTR_ENABLE_THRESH of 0 (`csr_clic.h:16`): every line // then gates on its own IE bit and its priority, which is where a driver can reason about it. setThreshold(0); } /// Diagnostics a behavioural test can print, because the two facts they establish - that the trap /// entry is 64-byte aligned and that MTVT is 256-byte aligned - are properties of the *link*, and /// the shipped image is stripped, so there is no way to check them from the host. pub fn trapEntryAddress() u32 { return @intCast(@intFromPtr(&trapEntry)); } pub fn vectorTableAddress() u32 { return @intCast(@intFromPtr(&vector_table)); } pub fn readMtvec() u32 { return asm volatile ("csrr %[out], mtvec" : [out] "=r" (-> u32), ); } pub fn readMtvt() u32 { return asm volatile ("csrr %[out], %[csr]" : [out] "=r" (-> u32), : [csr] "i" (mtvt_csr), ); } /// mintstatus, CSR 0x346 on this die (`soc/interrupt_reg.h:36`). Bits [31:24] are the current /// interrupt level: non-zero outside a handler would mean a previous trap never returned. pub fn readMintstatus() u32 { return asm volatile ("csrr %[out], %[csr]" : [out] "=r" (-> u32), : [csr] "i" (mintstatus_csr), ); } /// Install (or, with null, remove) the handler for one external line. /// /// Done with interrupts masked because the store is a pointer the trap entry may be about to load; /// `clkrst.maskInterrupts` composes - it restores only the MIE that was there - so this is safe to /// call from inside an already-masked region. pub fn setHandler(line: u5, handler: ?Handler) void { const guard = clkrst.maskInterrupts(); defer guard.release(); handlers[line] = handler; } /// Everything one line needs, in the order the hardware wants: handler before enable, so a source /// that is already pending cannot reach an empty slot; trigger and priority before enable, so the /// first interrupt is taken under the intended configuration rather than under the reset one. /// /// Does not touch the matrix - `route` is the other half - and does not touch mstatus. pub fn configureLine(line: u5, opts: struct { handler: Handler, trigger: Trigger = .level, /// Must exceed the threshold to ever be taken; the threshold comparison is inclusive. priority: u3 = 1, /// Hardware vectoring: fetch the handler address from `MTVT + 4*id` instead of trapping to /// mtvec's base. /// /// **On by default, and the default is the interesting part.** Every slot of the table holds the /// same `trapEntry`, so this changes only how the core finds that address - which makes the /// choice look free, and it is not. ESP-IDF sets SHV on all 32 lines at boot /// (`cpu_start.c:193-196`, "Set all the CPU interrupt lines to vectored by default, as it is on /// other RISC-V targets") and puts nothing but `j _panic_handler` at mtvec's base /// (`vectors_clic.S:47-52`). So on this chip the SHV=0 delivery path is one ESP-IDF never takes /// and therefore one nobody has validated. Defaulting to the path the vendor exercises is worth /// more than the memory fetch it costs. /// /// **Measured on the die: it is the other way round, and the default is now `false`.** /// /// With SHV=1 the interrupt was never delivered. The core vectored to a wild address and took an /// instruction access fault - `mcause=0x30000001` (EXCCODE 1, MINHV clear, so the fault was not /// during the table fetch), at a `mepc` that differed run to run, with `taken=0` proving the /// trap entry was never reached. mtvec, MTVT and the table contents were all verified correct /// beforehand: `mtvec=0x40001383` = entry|3, `mtvt=0x4ff00100`, and every slot holding /// `0x40001380` = `trapEntry`. /// /// The difference from ESP-IDF is *where the table lives*. IDF's `_mtvt_table` is in /// `.section .exception_vectors_table.text` (`vectors_clic.S:32,67`), i.e. instruction space. /// This image has no IRAM: it executes from flash through the MMU, so a table that `init()` has /// to write must live in L2MEM, and the hardware vector fetch does not appear to work from /// there. Since flash is not writable at run time, there is nowhere else to put it, which makes /// SHV=0 the correct choice for this memory layout rather than a workaround. /// /// With SHV=0 both halves of the behavioural test pass: one interrupt taken, dispatched to the /// right handler, `last_clic_id=21`, no spurious - and the threshold experiment then shows the /// memory-mapped register at 0x2080_0008 really is the one the arbiter reads. /// /// `true` remains available for an image that gains an IRAM section, and the vector table is /// still populated so that switching is a one-word change. vectored: bool = false, }) void { setHandler(line, opts.handler); setTrigger(line, opts.trigger); setPriority(line, opts.priority); setVectored(line, opts.vectored); setEnabled(line, true); } /// Route a source and bring its line up in one call. pub fn attach(source: Source, line: u5, opts: struct { handler: Handler, trigger: Trigger = .level, priority: u3 = 1, /// See `configureLine`: vectored is the only path ESP-IDF exercises on this chip. vectored: bool = false, }) void { route(source, line); configureLine(line, .{ .handler = opts.handler, .trigger = opts.trigger, .priority = opts.priority, .vectored = opts.vectored, }); } // --------------------------------------------------------------------------- global enable /// mstatus.MIE on. Nothing is taken before this, whatever the CLIC is configured to do. pub inline fn globalEnable() void { asm volatile ("csrs mstatus, %[m]" : : [m] "r" (mstatus_mie), ); } pub inline fn globalDisable() void { asm volatile ("csrc mstatus, %[m]" : : [m] "r" (mstatus_mie), ); } pub inline fn globalEnabled() bool { const s = asm volatile ("csrr %[out], mstatus" : [out] "=r" (-> u32), ); return s & mstatus_mie != 0; } /// The composable form: mask, do something, restore whatever was there. /// /// const guard = intr.mask(); /// defer guard.release(); /// /// This is `clkrst.maskInterrupts` under another name, re-exported rather than reimplemented so /// that a critical section written against either module is the same critical section. It nests /// correctly - `release` only sets MIE if MIE was set on entry - which is why `setHandler` can use /// it without caring who called it. pub const Guard = clkrst.Guard; pub inline fn mask() Guard { return clkrst.maskInterrupts(); } // ------------------------------------------------------------------------------- trap entry /// How many times `trapEntry` has dispatched an interrupt, and the last CLIC ID it saw. Diagnostics: /// with `taken == 0` the trap was never reached at all, which separates "the CLIC did not deliver" /// from "the handler did not run". pub var taken: u32 = 0; pub var last_clic_id: u32 = 0; /// An exception - not an interrupt - that reached `trapEntry`. pub const Fault = struct { /// Full mcause. Bit 31 is clear by construction here; the low bits are the exception code /// (1 instruction access, 2 illegal instruction, 5 load access, 7 store access, 11 ecall). mcause: u32, /// The instruction that faulted. mepc: u32, /// The address or instruction word involved, per exception code. mtval: u32, }; pub var faults: u32 = 0; pub var last_fault: Fault = .{ .mcause = 0, .mepc = 0, .mtval = 0 }; /// Called with the fault already recorded, before parking. Install one to get the numbers out; /// `hal` cannot print, so this hook is the only way a fault becomes visible. /// /// hal.intr.on_fault = struct { /// fn f(x: hal.intr.Fault) void { /// soc.rom.print("MARK FAULT mcause=0x%08x mepc=0x%08x mtval=0x%08x\r\n", /// .{ x.mcause, x.mepc, x.mtval }); /// } /// }.f; pub var on_fault: ?*const fn (Fault) void = null; /// Called from `trapEntry` with the CLIC ID out of mcause. Not part of the API; `export` because /// the assembly calls it by name. export fn intrDispatch(clic_id: u32) callconv(.c) void { taken +%= 1; last_clic_id = clic_id; if (clic_id < ext_offset or clic_id >= ext_offset + line_count) { // One of the 16 internal IDs. This file routes nothing there, so it is a bug elsewhere - // most likely something the ROM left armed that `init` did not manage to silence. spurious +%= 1; return; } const line: u5 = @intCast(clic_id - ext_offset); if (handlers[line]) |h| h(line) else spurious +%= 1; } /// The exception arm of `trapEntry`. Records, reports if a hook is installed, and **parks**. /// /// Parking rather than returning is the whole point. `mret` from an exception resumes at the /// faulting instruction, which faults again immediately: every mistake anywhere in this file used to /// become an unbreakable loop through the trap entry with the console silent, indistinguishable from /// "the interrupt was never delivered". It cost a debugging round to tell those apart. ESP-IDF makes /// the same choice by putting `j _panic_handler` at mtvec's base (`vectors_clic.S:47-52`). export fn intrFault(mcause: u32, mepc: u32, mtval: u32) callconv(.c) noreturn { faults +%= 1; last_fault = .{ .mcause = mcause, .mepc = mepc, .mtval = mtval }; globalDisable(); if (on_fault) |f| f(last_fault); while (true) {} } /// The trap entry: every trap on this core arrives here, interrupt or exception. /// /// Reached three ways, and they are not interchangeable: /// * an **interrupt with SHV = 1**, through `MTVT + 4*id`; /// * an **interrupt with SHV = 0**, through mtvec's base; /// * an **exception**, always through mtvec's base, whatever any line's SHV says. /// /// 64-byte aligned, and this is a hardware requirement rather than tidiness: in CLIC mode the core /// computes the target as `mtvec[31:6] << 6` (`vectors_clic.S:38-46` states it outright), so the low /// six bits of mtvec are not part of the address. A trap entry that were not 64-byte aligned would /// vector up to 60 bytes *before* this function, into whatever the linker put there. Measured in the /// linked image: 0x4000_1140, and `trapEntryAddress()` lets a test confirm it on the die, since the /// shipped image is stripped and there is no symbol to check from the host. /// /// **The first thing it does is decide whether this was an interrupt at all.** mcause bit 31 says /// so, and getting that wrong is not a small bug: an exception whose handler `mret`s resumes at the /// faulting instruction and faults again, immediately and forever, with the console silent. That /// failure is indistinguishable from "the interrupt was never delivered", and the two were in fact /// confused for a debugging round. So the exception arm never returns - see `intrFault`. /// /// Saves the integer caller-saved set - ra, t0-t6, a0-a7, sixteen words - and nothing else. Not /// saved, deliberately and with consequences: /// * **the f registers.** `src/main.zig`'s `_start` sets mstatus.FS to enable the FPU, so a handler /// that does float arithmetic silently corrupts the interrupted code. Handlers must stay integer. /// * **mepc, mcause, mstatus.** In CLIC mode the core stacks the previous privilege, interrupt /// enable and interrupt level in mcause itself, and `mret` restores them from there - so nothing /// here may write mcause, and nothing does. They are only at risk from a *nested* trap, and MIE /// stays clear for the whole sequence, so nothing can nest. That is also why there is no `mnxti` /// loop: the CLIC's hardware nesting (SOC_INT_HW_NESTED_SUPPORTED, `soc_caps.h:193`) is unused. /// /// One consequence of `mret` worth stating because it defeats an obvious defence: it restores /// mstatus.MIE from MPIE, which the hardware set to 1 on entry. A handler that calls /// `globalDisable()` therefore does **not** leave interrupts off after it returns. To stop a runaway /// source the handler must clear it at the peripheral, or call `setEnabled(line, false)`. export fn trapEntry() align(64) callconv(.naked) noreturn { asm volatile ( \\ addi sp, sp, -64 \\ sw ra, 0(sp) \\ sw t0, 4(sp) \\ sw t1, 8(sp) \\ sw t2, 12(sp) \\ sw a0, 16(sp) \\ sw a1, 20(sp) \\ sw a2, 24(sp) \\ sw a3, 28(sp) \\ sw a4, 32(sp) \\ sw a5, 36(sp) \\ sw a6, 40(sp) \\ sw a7, 44(sp) \\ sw t3, 48(sp) \\ sw t4, 52(sp) \\ sw t5, 56(sp) \\ sw t6, 60(sp) \\ csrr a0, mcause // Bit 31 set means interrupt, so mcause read as *signed* is negative. `bgez` therefore // branches exactly on "this was an exception", in one instruction and with no scratch // register - which matters here because every scratch register is already spoken for. \\ bgez a0, 1f // mcause[11:0] is the CLIC's interrupt ID. Isolated with a shift pair rather than `andi`: // andi's immediate is 12-bit *signed*, so `andi a0, a0, 0xfff` does not assemble as a // 12-bit mask - it is -1, and would leave the interrupt bit and the level field in place. \\ slli a0, a0, 20 \\ srli a0, a0, 20 \\ call intrDispatch \\ lw ra, 0(sp) \\ lw t0, 4(sp) \\ lw t1, 8(sp) \\ lw t2, 12(sp) \\ lw a0, 16(sp) \\ lw a1, 20(sp) \\ lw a2, 24(sp) \\ lw a3, 28(sp) \\ lw a4, 32(sp) \\ lw a5, 36(sp) \\ lw a6, 40(sp) \\ lw a7, 44(sp) \\ lw t3, 48(sp) \\ lw t4, 52(sp) \\ lw t5, 56(sp) \\ lw t6, 60(sp) \\ addi sp, sp, 64 \\ mret // The exception arm. No restore and no `mret`: `intrFault` is noreturn, because resuming // would re-execute the faulting instruction. The saved registers stay on the stack, which // costs 64 bytes that are never reclaimed and is the correct trade for a path that ends in // a parked core with the numbers printed. \\1: \\ csrr a1, mepc \\ csrr a2, mtval \\ call intrFault ); } // ------------------------------------------------------------------------------------ tests test "the enum's IDs are the offsets of the matrix registers they name" { // The whole of `routeId` rests on `map_reg_addr == base + 4*id`. These four are checked against // the addresses ESP-IDF's own interrupt_core0_reg.h computes, which is an independent path: // IDF wrote the offset as a literal per source, this file multiplies. try std.testing.expectEqual(@as(u32, 0x7c), 4 * @as(u32, @intFromEnum(Source.uart0))); try std.testing.expectEqual(@as(u32, 0xb0), 4 * @as(u32, @intFromEnum(Source.i2c0))); try std.testing.expectEqual(@as(u32, 0xd0), 4 * @as(u32, @intFromEnum(Source.ledc))); try std.testing.expectEqual(@as(u32, 0x1fc), 4 * @as(u32, @intFromEnum(Source.assist_debug))); } test "rev-3-only sources are flagged and the pre-v3 ones are not" { try std.testing.expect(Source.axi_perf_mon.isRev3Only()); try std.testing.expect(Source.dma2d_in_ch2.isRev3Only()); try std.testing.expect(!Source.assist_debug.isRev3Only()); try std.testing.expect(!Source.dma2d_in_ch1.isRev3Only()); } test "priority and threshold use different encodings of the same three bits" { // Priority pads low with zeros, threshold pads low with ones. Getting these the same way round // is the mistake this test exists to catch. const priority_byte = @as(u32, 5) << nlbits_shift; const threshold_byte = (@as(u32, 5) << nlbits_shift) | nlbits_pad; try std.testing.expectEqual(@as(u32, 0xa0), priority_byte); try std.testing.expectEqual(@as(u32, 0xbf), threshold_byte); try std.testing.expectEqual(@as(u32, 5), priority_byte >> nlbits_shift); try std.testing.expectEqual(@as(u32, 5), threshold_byte >> nlbits_shift); } test "trigger's low bit, not its value, decides edge versus level" { try std.testing.expect(Trigger.rising_edge.isEdge()); try std.testing.expect(Trigger.falling_edge.isEdge()); try std.testing.expect(!Trigger.level.isEdge()); try std.testing.expect(!Trigger.level_alias.isEdge()); } test "the vector table is aligned to a power of two above its own size" { try std.testing.expectEqual(@as(usize, 256), @alignOf(@TypeOf(vector_table))); try std.testing.expect(@sizeOf(@TypeOf(vector_table)) <= 256); } test "mcause's sign bit is what separates an interrupt from an exception" { // The trap entry branches on `bgez mcause`, which is only correct if bit 31 is the interrupt // flag and the value is read signed. Spelled out here because the asm cannot say it. const interrupt_mcause: u32 = 0x8000_0015; // CLIC ID 21 = external line 5 const exception_mcause: u32 = 0x0000_0002; // illegal instruction try std.testing.expect(@as(i32, @bitCast(interrupt_mcause)) < 0); try std.testing.expect(@as(i32, @bitCast(exception_mcause)) >= 0); // And the ID extraction the two shifts perform. try std.testing.expectEqual(@as(u32, 21), (interrupt_mcause << 20) >> 20); } test "mtvec's mode bits do not collide with a 64-byte-aligned base" { // The hardware target is `mtvec[31:6] << 6`, so the mode goes in bits the base cannot use - // but only if the base really is 64-byte aligned. This is the arithmetic `init` performs; // whether the *linked* trapEntry satisfies it is a fact about the link, and // `trapEntryAddress()` is how a test on the die checks that, the image being stripped. const aligned_base: u32 = 0x4000_1200; const mtvec = aligned_base | mtvec_mode_clic; try std.testing.expectEqual(aligned_base, (mtvec >> 6) << 6); // A base one instruction short of alignment vectors 60 bytes early, silently. const bad_base: u32 = 0x4000_1204; try std.testing.expect(((bad_base | mtvec_mode_clic) >> 6) << 6 != bad_base); }