mirror of
https://github.com/UniXNU/NativeAdreno.git
synced 2026-11-18 13:31:40 +03:00
172f53f6cf
A first-ring job that waits while the CP stays on the second ring after a switch back timed out is stranded, as a second-ring job already was: the restart replays every job and no client loses its device. The switch timeout record also carries CP_CONTEXT_SWITCH_CNTL and the two rings of the switch.
126 lines
8.3 KiB
C++
126 lines
8.3 KiB
C++
/* SPDX-License-Identifier: MIT */
|
|
#ifndef NATIVE_ADRENO_PREEMPT_H
|
|
#define NATIVE_ADRENO_PREEMPT_H
|
|
#include <IOKit/IOTypes.h>
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
/* A6XX..A7XX CP context switch between two rings, as msm's a6xx_preempt.c drives it: the CP keeps one ring loaded and saves
|
|
* and restores the other's state through a preemption record the CPU fills in. Every constant below is from the pinned
|
|
* registers/adreno/a6xx.xml and a6xx_gpu.c/a6xx_preempt.c, never guessed:
|
|
* CP_CONTEXT_SWITCH_CNTL 0x08a0 (STOP bit 0, LEVEL bits 6..7, USES_GMEM bit 8, SKIP_SAVE_RESTORE bit 9)
|
|
* CP_CONTEXT_SWITCH_SMMU_INFO 0x08a1 (64 bit): the a7xx_cp_smmu_info the CP reads on the way in
|
|
* CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR 0x08a3 (64 bit): the incoming ring's preemption record
|
|
* RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE_ENABLE 0x8e50
|
|
* GMU_PWR_COL_PREEMPT_KEEPALIVE: a6xx_gmu.xml dword offset 0x1f8c4 in the GMU's own register window
|
|
* RBBM_INT_0_MASK bit 8 CP_SW: the switch finished
|
|
* A record is 4192 KiB (the CP stores its state there) and the SMMU info page 4 KiB; a6xx_preempt.c allocates exactly these
|
|
* per ring. A record's CPU-aware fields are the first 56 bytes (a6xx_preempt_record in a6xx_gpu.h).
|
|
*
|
|
* This object writes the trigger registers and judges the completion; the caller owns the memory (records, SMMU info pages,
|
|
* ring and fence pages), fills a record through writeRecord() and an SMMU info page through writeSmmuInfo(), and gives the
|
|
* write pointer again after a switch, because only it knows which ring the CP now runs. */
|
|
enum : uint32_t {NativeAdrenoPreemptRingCount = 2};
|
|
|
|
/* The GPU addresses of the per-ring state. */
|
|
struct NativeAdrenoPreemptRings {
|
|
uint64_t record[NativeAdrenoPreemptRingCount]; /* the preemption record of each ring */
|
|
uint64_t smmu_info[NativeAdrenoPreemptRingCount]; /* the a7xx_cp_smmu_info page of each ring */
|
|
/* The CPU's own mappings of both, which the caller fills before each switch. */
|
|
uint8_t *record_cpu[NativeAdrenoPreemptRingCount];
|
|
uint8_t *smmu_cpu[NativeAdrenoPreemptRingCount];
|
|
};
|
|
|
|
struct NativeAdrenoPreemptIo {
|
|
void *cookie;
|
|
int (*write32)(void *, uint32_t reg, uint32_t value);
|
|
int (*read32)(void *, uint32_t reg, uint32_t *value);
|
|
int (*barrier)(void *);
|
|
/* The GMU's window: the preemption keepalive vote lives there, not in the GPU's register window. Its register is a dword index,
|
|
* as a6xx_gmu.xml writes it (0x1f8c4). */
|
|
void *gmu_cookie;
|
|
int (*gmu_write32)(void *, uint32_t dword_offset, uint32_t value);
|
|
};
|
|
|
|
struct NativeAdrenoPreemptStats {
|
|
uint32_t triggers; /* switches the CPU asked for */
|
|
uint32_t completed; /* switches whose STOP bit had cleared when the interrupt came */
|
|
uint32_t skipped; /* a higher-priority ring was waiting and no switch was possible */
|
|
uint32_t faulted; /* a switch that did not finish */
|
|
uint32_t disjoint_skips; /* switches not taken because the rings' buffers overlap */
|
|
uint32_t restored_wptr; /* write pointers given again after a switch */
|
|
uint32_t last_us; /* how long the last completed switch took */
|
|
};
|
|
|
|
/* The newest 16 switches, one row each (words): [0] sequence, [1..2] trigger time (us), [3..4] end time (us, 0 pending), [5] from | to << 8 |
|
|
* outcome << 16 (0 pending, 1 done, 2 timed out, 3 a register write refused), [6] CP_CONTEXT_SWITCH_CNTL read right after the trigger,
|
|
* [7] the same at the end, [8] RBBM_STATUS, [9] RBBM_INT_0_STATUS and [10] CP_HW_FAULT at the end, [11] CP_RB_RPTR | CP_RB_WPTR << 16 at the
|
|
* end, [12..15] ring 0's record before the trigger and [16..19] ring 1's (read pointer, write pointer, errno, info), [20..27] both again at the
|
|
* end, [28..33] the CP's IB state when a switch timed out (NativeAdrenoRing::progressState). native_adreno_switch_count is the total. */
|
|
enum : uint32_t {NativeAdrenoSwitchLogRows=16,NativeAdrenoSwitchLogWords=34};
|
|
extern "C" volatile uint32_t native_adreno_switch_log[NativeAdrenoSwitchLogRows][NativeAdrenoSwitchLogWords];
|
|
extern "C" volatile uint32_t native_adreno_switch_count;
|
|
|
|
class NativeAdrenoPreempt final {
|
|
NativeAdrenoPreemptIo io_{};
|
|
NativeAdrenoPreemptRings rings_{};
|
|
uint32_t loaded_=0; /* the ring the CP runs */
|
|
uint32_t next_=0; /* the ring a switch in flight goes to */
|
|
bool armed_=false; /* a switch was triggered and its STOP bit not yet seen cleared */
|
|
bool attached_=false,keepalive_=false;
|
|
uint64_t (*now_us_)(void *)=nullptr;
|
|
void *clock_cookie_=nullptr;
|
|
uint64_t trigger_us_=0;
|
|
NativeAdrenoPreemptStats stats_{};
|
|
volatile uint32_t *event_=nullptr; /* the log row of the switch in flight */
|
|
void snapshot(volatile uint32_t *row,unsigned first) const;
|
|
volatile uint32_t *openEvent(uint32_t to);
|
|
void closeEvent(uint32_t outcome);
|
|
public:
|
|
NativeAdrenoPreempt()=default;
|
|
~NativeAdrenoPreempt()=default;
|
|
NativeAdrenoPreempt(const NativeAdrenoPreempt &)=delete;
|
|
NativeAdrenoPreempt &operator=(const NativeAdrenoPreempt &)=delete;
|
|
/* Programmes RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE_ENABLE and clears CP_CONTEXT_SWITCH_SMMU_INFO, as a6xx_preempt_hw_init
|
|
* does, with `loaded` the ring the CP runs. The caller must already own a powered GPU with that ring up. `now` times the
|
|
* switches (optional). */
|
|
IOReturn attach(const NativeAdrenoPreemptIo *,const NativeAdrenoPreemptRings &,uint32_t loaded,uint64_t (*now)(void *),void *clock);
|
|
IOReturn detach();
|
|
bool attached() const {return attached_;}
|
|
/* The CPU-aware fields of a record (a6xx_preempt_record, first 56 bytes): the magic the CP checks, the ring's address, its
|
|
* read-pointer shadow and BV read-pointer shadow, its CP_RB_CNTL value and the write pointer the CP resumes at. */
|
|
static bool writeRecord(uint8_t *record,uint64_t rbase,uint64_t rptr_addr,uint64_t bv_rptr_addr,uint32_t cntl,uint32_t wptr);
|
|
/* An a7xx_cp_smmu_info page: its magic, the page table the ring's CP runs, the address-space id and context idr. */
|
|
static bool writeSmmuInfo(uint8_t *page,uint64_t ttbr0,uint32_t asid,uint32_t context_idr);
|
|
/* The write pointer a record resumes at (the CP writes it back after saving), for the caller to give again. */
|
|
static uint32_t recordWptr(const uint8_t *record);
|
|
/* Before each switch only the write pointer is given again: the read pointer, the info and errno words are the CP's (it saves them
|
|
* into the record when it switches the ring out), so rewriting the whole record would send the ring back to its start. */
|
|
static bool setRecordWptr(uint8_t *record,uint32_t wptr);
|
|
static uint32_t recordWord(const uint8_t *record,uint32_t byte_offset);
|
|
/* Switch the CP to `ring`, which must not be the loaded one: its SMMU info and record go out first, then the control word
|
|
* with STOP. `uses_gmem` asks for the GMEM save and restore. False where a switch is still in flight or a register refused. */
|
|
bool trigger(uint32_t ring,bool uses_gmem);
|
|
/* The interrupt's side: read CP_CONTEXT_SWITCH_CNTL; a cleared STOP bit completes the switch and makes its ring the loaded
|
|
* one, a set one is still pending. False where nothing was in flight. */
|
|
bool complete();
|
|
/* Release the power vote after the incoming ring's WPTR is restored. */
|
|
bool releaseKeepalive();
|
|
bool inFlight() const {return armed_;}
|
|
uint32_t loaded() const {return loaded_;}
|
|
uint32_t next() const {return next_;} /* the ring the switch in flight loads */
|
|
/* A switch triggered more than `limit_us` ago and still not complete. */
|
|
bool timedOut(uint32_t limit_us) const;
|
|
/* The switch in flight did not finish: the log row takes the CP's IB state (progress) and the registers, and the switch ends as timed out. */
|
|
void noteTimeout(const uint32_t progress[6]);
|
|
uint32_t readCntl() const; /* CP_CONTEXT_SWITCH_CNTL as the hardware holds it (all ones when unreadable); STOP set means a switch is not done */
|
|
uint32_t target() const {return next_;}
|
|
/* After a GPU reset: `ring` is the one the CP runs and nothing is in flight. */
|
|
void restart(uint32_t ring) {loaded_=ring;armed_=false;event_=nullptr;releaseKeepalive();}
|
|
NativeAdrenoPreemptStats stats() const {return stats_;}
|
|
void noteDisjointSkip() {stats_.disjoint_skips=stats_.disjoint_skips+1;}
|
|
void noteSkipped() {stats_.skipped=stats_.skipped+1;}
|
|
void noteRestoredWptr() {stats_.restored_wptr=stats_.restored_wptr+1;}
|
|
};
|
|
#endif
|