diff options
Diffstat (limited to 'drivers/pci/controller/pcie-brcmstb.c')
| -rw-r--r-- | drivers/pci/controller/pcie-brcmstb.c | 1070 |
1 files changed, 575 insertions, 495 deletions
diff --git a/drivers/pci/controller/pcie-brcmstb.c b/drivers/pci/controller/pcie-brcmstb.c index ed25178d02d7..169a4d82519a 100644 --- a/drivers/pci/controller/pcie-brcmstb.c +++ b/drivers/pci/controller/pcie-brcmstb.c @@ -5,6 +5,7 @@ #include <linux/bitops.h> #include <linux/clk.h> #include <linux/compiler.h> +#include <linux/debugfs.h> #include <linux/delay.h> #include <linux/init.h> #include <linux/interrupt.h> @@ -48,15 +49,6 @@ #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY 0x04dc #define PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK 0xc00 -#define PCIE_RC_TL_VDM_CTL0 0x0a20 -#define PCIE_RC_TL_VDM_CTL0_VDM_ENABLED_MASK 0x10000 -#define PCIE_RC_TL_VDM_CTL0_VDM_IGNORETAG_MASK 0x20000 -#define PCIE_RC_TL_VDM_CTL0_VDM_IGNOREVNDRID_MASK 0x40000 - -#define PCIE_RC_TL_VDM_CTL1 0x0a0c -#define PCIE_RC_TL_VDM_CTL1_VDM_VNDRID0_MASK 0x0000ffff -#define PCIE_RC_TL_VDM_CTL1_VDM_VNDRID1_MASK 0xffff0000 - #define PCIE_RC_CFG_PRIV1_ROOT_CAP 0x4f8 #define PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK 0xf8 @@ -68,6 +60,34 @@ #define PCIE_RC_PL_PHY_CTL_15_DIS_PLL_PD_MASK 0x400000 #define PCIE_RC_PL_PHY_CTL_15_PM_CLK_PERIOD_MASK 0xff +#define PCIE_RC_PL_STATS_CTRL 0x1940 +#define PCIE_RC_PL_STATS_CTRL_EN_MASK 0x1 +#define PCIE_RC_PL_STATS_CTRL_LEN_MASK 0xfffffff0 + +#define PCIE_RC_PL_STATS_TXTLP_LO 0x1944 +#define PCIE_RC_PL_STATS_TXTLP_HI 0x1948 +#define PCIE_RC_PL_STATS_TXDLLP_LO 0x194c +#define PCIE_RC_PL_STATS_TXDLLP_HI 0x1950 +#define PCIE_RC_PL_STATS_RXTLP_LO 0x195c +#define PCIE_RC_PL_STATS_RXTLP_HI 0x1960 +#define PCIE_RC_PL_STATS_RXDLLP_LO 0x1964 +#define PCIE_RC_PL_STATS_RXDLLP_HI 0x1968 +#define PCIE_RC_PL_STATS_RXPL_ERR 0x1974 +#define PCIE_RC_PL_STATS_RXDL_ERR 0x1978 +#define PCIE_RC_PL_STATS_RXTL_ERR 0x197c + +#define PCIE_RC_PL_LTSSM_STATS_3 0x19b0 +#define PCIE_RC_PL_LTSSM_STATS_3_TIME_L0S_MASK 0xffff0000 +#define PCIE_RC_PL_LTSSM_STATS_3_TIME_RECOV_MASK 0x0000ffff + +#define PCIE_RC_PL_LTSSM_STATS_CNT 0x19b4 +#define PCIE_RC_PL_LTSSM_STATS_CNT_L0S_FAIL_MASK 0xffff0000 +#define PCIE_RC_PL_LTSSM_STATS_CNT_RECOV_MASK 0x0000ffff + +#define PCIE_RC_PL_LTSSM_HIST_0 0x1cec +#define PCIE_RC_PL_LTSSM_HIST(n) \ + (PCIE_RC_PL_LTSSM_HIST_0 + ((n) * 4)) + #define PCIE_MISC_MISC_CTRL 0x4008 #define PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK 0x80 #define PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK 0x400 @@ -98,15 +118,9 @@ #define PCIE_BRCM_MAX_INBOUND_WINS 16 #define PCIE_MISC_RC_BAR1_CONFIG_LO 0x402c #define PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK 0x1f -#define PCIE_MISC_RC_BAR1_CONFIG_HI 0x4030 -#define PCIE_MISC_RC_BAR2_CONFIG_LO 0x4034 -#define PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK 0x1f -#define PCIE_MISC_RC_BAR2_CONFIG_HI 0x4038 +#define PCIE_MISC_RC_BAR4_CONFIG_LO 0x40d4 -#define PCIE_MISC_RC_BAR3_CONFIG_LO 0x403c -#define PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK 0x1f -#define PCIE_MISC_RC_BAR3_CONFIG_HI 0x4040 #define PCIE_MISC_MSI_BAR_CONFIG_LO 0x4044 #define PCIE_MISC_MSI_BAR_CONFIG_HI 0x4048 @@ -115,15 +129,12 @@ #define PCIE_MISC_MSI_DATA_CONFIG_VAL_32 0xffe06540 #define PCIE_MISC_MSI_DATA_CONFIG_VAL_8 0xfff86540 -#define PCIE_MISC_RC_CONFIG_RETRY_TIMEOUT 0x405c - #define PCIE_MISC_PCIE_CTRL 0x4064 #define PCIE_MISC_PCIE_CTRL_PCIE_L23_REQUEST_MASK 0x1 #define PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK 0x4 #define PCIE_MISC_PCIE_STATUS 0x4068 #define PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK 0x80 -#define PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK_2712 0x40 #define PCIE_MISC_PCIE_STATUS_PCIE_DL_ACTIVE_MASK 0x20 #define PCIE_MISC_PCIE_STATUS_PCIE_PHYLINKUP_MASK 0x10 #define PCIE_MISC_PCIE_STATUS_PCIE_LINK_IN_L23_MASK 0x40 @@ -150,81 +161,21 @@ #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK 0x2 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_PERST_ASSERT_MASK 0x8 +#define PCIE_MISC_HARD_PCIE_HARD_DEBUG_REFCLK_OVRD_ENABLE_MASK 0x10000 +#define PCIE_MISC_HARD_PCIE_HARD_DEBUG_REFCLK_OVRD_OUT_MASK 0x100000 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK 0x200000 #define PCIE_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x08000000 #define PCIE_BMIPS_MISC_HARD_PCIE_HARD_DEBUG_SERDES_IDDQ_MASK 0x00800000 #define PCIE_CLKREQ_MASK \ (PCIE_MISC_HARD_PCIE_HARD_DEBUG_CLKREQ_DEBUG_ENABLE_MASK | \ + PCIE_MISC_HARD_PCIE_HARD_DEBUG_REFCLK_OVRD_ENABLE_MASK | \ + PCIE_MISC_HARD_PCIE_HARD_DEBUG_REFCLK_OVRD_OUT_MASK | \ PCIE_MISC_HARD_PCIE_HARD_DEBUG_L1SS_ENABLE_MASK) - #define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP 0x40ac #define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_EN_MASK BIT(0) #define PCIE_MISC_UBUS_BAR4_CONFIG_REMAP 0x410c -#define PCIE_MISC_CTRL_1 0x40A0 -#define PCIE_MISC_CTRL_1_OUTBOUND_TC_MASK 0xf -#define PCIE_MISC_CTRL_1_OUTBOUND_NO_SNOOP_MASK BIT(3) -#define PCIE_MISC_CTRL_1_OUTBOUND_RO_MASK BIT(4) -#define PCIE_MISC_CTRL_1_EN_VDM_QOS_CONTROL_MASK BIT(5) - -#define PCIE_MISC_UBUS_CTRL 0x40a4 -#define PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_ERR_DIS_MASK BIT(13) -#define PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_DECERR_DIS_MASK BIT(19) - -#define PCIE_MISC_UBUS_TIMEOUT 0x40A8 - -#define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP 0x40ac -#define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_ENABLE_MASK BIT(0) -#define PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_HI 0x40b0 - -#define PCIE_MISC_UBUS_BAR2_CONFIG_REMAP 0x40b4 -#define PCIE_MISC_UBUS_BAR2_CONFIG_REMAP_ACCESS_ENABLE_MASK BIT(0) - -/* Additional RC BARs */ -#define PCIE_MISC_RC_BAR_CONFIG_LO_SIZE_MASK 0x1f -#define PCIE_MISC_RC_BAR4_CONFIG_LO 0x40d4 -#define PCIE_MISC_RC_BAR4_CONFIG_HI 0x40d8 -/* ... */ -#define PCIE_MISC_RC_BAR10_CONFIG_LO 0x4104 -#define PCIE_MISC_RC_BAR10_CONFIG_HI 0x4108 - -#define PCIE_MISC_UBUS_BAR_CONFIG_REMAP_ENABLE 0x1 -#define PCIE_MISC_UBUS_BAR_CONFIG_REMAP_LO_MASK 0xfffff000 -#define PCIE_MISC_UBUS_BAR_CONFIG_REMAP_HI_MASK 0xff -#define PCIE_MISC_UBUS_BAR4_CONFIG_REMAP_LO 0x410c -#define PCIE_MISC_UBUS_BAR4_CONFIG_REMAP_HI 0x4110 -/* ... */ -#define PCIE_MISC_UBUS_BAR10_CONFIG_REMAP_LO 0x413c -#define PCIE_MISC_UBUS_BAR10_CONFIG_REMAP_HI 0x4140 - -/* AXI priority forwarding - automatic level-based */ -#define PCIE_MISC_TC_QUEUE_TO_QOS_MAP(x) (0x4160 - (x) * 4) -/* Defined in quarter-fullness */ -#define QUEUE_THRESHOLD_34_TO_QOS_MAP_SHIFT 12 -#define QUEUE_THRESHOLD_23_TO_QOS_MAP_SHIFT 8 -#define QUEUE_THRESHOLD_12_TO_QOS_MAP_SHIFT 4 -#define QUEUE_THRESHOLD_01_TO_QOS_MAP_SHIFT 0 -#define QUEUE_THRESHOLD_MASK 0xf - -/* VDM messages indexing TCs to AXI priorities */ -/* Indexes 8-15 */ -#define PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_HI 0x4164 -/* Indexes 0-7 */ -#define PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_LO 0x4168 -#define VDM_PRIORITY_TO_QOS_MAP_SHIFT(x) (4 * (x)) -#define VDM_PRIORITY_TO_QOS_MAP_MASK 0xf - -#define PCIE_MISC_AXI_INTF_CTRL 0x416C -#define AXI_EN_RCLK_QOS_ARRAY_FIX BIT(13) -#define AXI_EN_QOS_UPDATE_TIMING_FIX BIT(12) -#define AXI_DIS_QOS_GATING_IN_MASTER BIT(11) -#define AXI_REQFIFO_EN_QOS_PROPAGATION BIT(7) -#define AXI_BRIDGE_LOW_LATENCY_MODE BIT(6) -#define AXI_MASTER_MAX_OUTSTANDING_REQUESTS_MASK 0x3f - -#define PCIE_MISC_AXI_READ_ERROR_DATA 0x4170 - #define PCIE_MSI_INTR2_BASE 0x4500 /* Offsets from INTR2_CPU and MSI_INTR2 BASE offsets */ @@ -292,6 +243,58 @@ #define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_MASK 0x1 #define PCIE_DVT_PMU_PCIE_PHY_CTRL_DAST_PWRDN_SHIFT 0x0 +/* BCM7712/2712-specific registers */ + +#define PCIE_RC_TL_VDM_CTL0 0x0a20 +#define PCIE_RC_TL_VDM_CTL0_VDM_ENABLED_MASK 0x10000 +#define PCIE_RC_TL_VDM_CTL0_VDM_IGNORETAG_MASK 0x20000 +#define PCIE_RC_TL_VDM_CTL0_VDM_IGNOREVNDRID_MASK 0x40000 + +#define PCIE_RC_TL_VDM_CTL1 0x0a0c +#define PCIE_RC_TL_VDM_CTL1_VDM_VNDRID0_MASK 0x0000ffff +#define PCIE_RC_TL_VDM_CTL1_VDM_VNDRID1_MASK 0xffff0000 + +#define PCIE_MISC_CTRL_1 0x40A0 +#define PCIE_MISC_CTRL_1_OUTBOUND_TC_MASK 0xf +#define PCIE_MISC_CTRL_1_OUTBOUND_NO_SNOOP_MASK BIT(3) +#define PCIE_MISC_CTRL_1_OUTBOUND_RO_MASK BIT(4) +#define PCIE_MISC_CTRL_1_EN_VDM_QOS_CONTROL_MASK BIT(5) + +#define PCIE_MISC_UBUS_CTRL 0x40a4 +#define PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_ERR_DIS_MASK BIT(13) +#define PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_DECERR_DIS_MASK BIT(19) + +#define PCIE_MISC_UBUS_TIMEOUT 0x40a8 + +#define PCIE_MISC_RC_CONFIG_RETRY_TIMEOUT 0x405c + +/* AXI priority forwarding - automatic level-based */ +#define PCIE_MISC_TC_QUEUE_TO_QOS_MAP(x) (0x4160 - (x) * 4) +/* Defined in quarter-fullness */ +#define QUEUE_THRESHOLD_34_TO_QOS_MAP_SHIFT 12 +#define QUEUE_THRESHOLD_23_TO_QOS_MAP_SHIFT 8 +#define QUEUE_THRESHOLD_12_TO_QOS_MAP_SHIFT 4 +#define QUEUE_THRESHOLD_01_TO_QOS_MAP_SHIFT 0 +#define QUEUE_THRESHOLD_MASK 0xf + +/* VDM messages indexing TCs to AXI priorities */ +/* Indexes 8-15 */ +#define PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_HI 0x4164 +/* Indexes 0-7 */ +#define PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_LO 0x4168 +#define VDM_PRIORITY_TO_QOS_MAP_SHIFT(x) (4 * (x)) +#define VDM_PRIORITY_TO_QOS_MAP_MASK 0xf + +#define PCIE_MISC_AXI_INTF_CTRL 0x416C +#define AXI_EN_RCLK_QOS_ARRAY_FIX BIT(13) +#define AXI_EN_QOS_UPDATE_TIMING_FIX BIT(12) +#define AXI_DIS_QOS_GATING_IN_MASTER BIT(11) +#define AXI_REQFIFO_EN_QOS_PROPAGATION BIT(7) +#define AXI_BRIDGE_LOW_LATENCY_MODE BIT(6) +#define AXI_MASTER_MAX_OUTSTANDING_REQUESTS_MASK 0x3f + +#define PCIE_MISC_AXI_READ_ERROR_DATA 0x4170 + /* Forward declarations */ struct brcm_pcie; @@ -311,7 +314,6 @@ enum pcie_soc_base { BCM7425, BCM7435, BCM7712, - BCM2712, }; struct inbound_win { @@ -320,6 +322,91 @@ struct inbound_win { u64 cpu_addr; }; +#define TRACE_BUF_LEN 64 +struct trace_entry { + u8 st; + ktime_t ts; +}; + +static const char *brcm_pcie_decode_ltssm_state(u8 state) +{ + switch (state) { + case 0x01: + return "L0"; + case 0x02: + return "L1"; + case 0x03: + return "L2"; + case 0x04: + return "RxL0s_TxL0s"; + case 0x05: + return "RxL0_TxL0s"; + case 0x09: + return "Polling.Active"; + case 0x0A: + return "Polling.Configuration"; + case 0x0C: + return "Polling.Compliance"; + case 0x10: + return "Configuration.Link"; + case 0x11: + return "Configuration.Linkwidth.Accept"; + case 0x12: + return "Configuration.Lanenum.Wait"; + case 0x13: + return "Confguration.Lanenum.Accept"; + case 0x14: + return "Confguration.Complete"; + case 0x15: + return "Configuration.Idle"; + case 0x20: + return "Recovery.RcvrLock"; + case 0x21: + return "Recovery.RcvrCfg"; + case 0x22: + return "Recovery.Idle"; + case 0x23: + return "Recovery.Speed"; + case 0x24: + return "Recovery.EQ_Phase0"; + case 0x25: + return "Recovery.EQ_Phase1"; + case 0x26: + return "Recovery.EQ_Phase2"; + case 0x27: + return "Recovery.EQ_Phase3"; + case 0x40: + return "Disable"; + case 0x43: + return "Reset"; + case 0x47: + return "Loopback.Entry"; + case 0x48: + return "Loopback.Active"; + case 0x49: + return "Loopback.Exit"; + case 0x4C: + return "Loopback.Master.Entry"; + case 0x4D: + return "Loopback.Master.Active"; + case 0x4E: + return "Loopback.Master.Exit"; + case 0x70: + return "Recovery.Speed_Change"; + case 0x80: + return "Detect.Quiet"; + case 0x81: + return "Detect.Active"; + case 0x82: + return "Detect.Wait"; + case 0x83: + return "Detect.Active2"; + default: + break; + } + return NULL; +}; + /* * The RESCAL block is tied to PCIe controller #1, regardless of the number of * controllers, and turning off PCIe controller #1 prevents access to the RESCAL @@ -329,6 +416,12 @@ struct inbound_win { */ #define CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN BIT(0) +/* + * MDIO register map differences and required changes to the defaults mean that refclk + * spread-spectrum clocking isn't supportable. + */ +#define CFG_QUIRK_NO_SSC BIT(1) + struct pcie_cfg_data { const int *offsets; const enum pcie_soc_base soc_base; @@ -354,7 +447,7 @@ struct brcm_msi { struct mutex lock; /* guards the alloc/free operations */ u64 target_addr; int irq; - DECLARE_BITMAP(used, 64); + DECLARE_BITMAP(used, BRCM_INT_PCI_MSI_NR); bool legacy; /* Some chips have MSIs in bits [31..24] of a shared register. */ int legacy_shift; @@ -383,6 +476,22 @@ struct brcm_pcie { struct subdev_regulators *sr; bool ep_wakeup_capable; const struct pcie_cfg_data *cfg; + u32 tperst_clk_ms; + bool trace_ltssm; + struct trace_entry *ltssm_trace_buf; + /* Statistics exposed in debugfs */ + struct dentry *debugfs_dir; + u64 tx_tlp; + u64 rx_tlp; + u64 tx_dllp; + u64 rx_dllp; + u32 pl_rx_err; + u32 dl_rx_err; + u32 tl_rx_err; + u16 l0s_exit_time; + u16 recov_time; + u16 l0s_fail_cnt; + u16 recov_cnt; }; static inline bool is_bmips(const struct brcm_pcie *pcie) @@ -492,35 +601,6 @@ static int brcm_pcie_set_ssc(struct brcm_pcie *pcie) return ssc && pll ? 0 : -EIO; } -static void brcm_pcie_munge_pll(struct brcm_pcie *pcie) -{ - //print "MDIO block 0x1600 written per Dannys instruction" - //tmp = pcie_mdio_write(phyad, &h16&, &h50b9&) - //tmp = pcie_mdio_write(phyad, &h17&, &hbd1a&) - //tmp = pcie_mdio_write(phyad, &h1b&, &h5030&) - //tmp = pcie_mdio_write(phyad, &h1e&, &h0007&) - - u32 tmp; - int ret, i; - u8 regs[] = { 0x16, 0x17, 0x18, 0x19, 0x1b, 0x1c, 0x1e }; - u16 data[] = { 0x50b9, 0xbda1, 0x0094, 0x97b4, 0x5030, 0x5030, 0x0007 }; - - ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET, - 0x1600); - for (i = 0; i < ARRAY_SIZE(regs); i++) { - brcm_pcie_mdio_read(pcie->base, MDIO_PORT0, regs[i], &tmp); - dev_dbg(pcie->dev, "PCIE MDIO pre_refclk 0x%02x = 0x%04x\n", - regs[i], tmp); - } - for (i = 0; i < ARRAY_SIZE(regs); i++) { - brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, regs[i], data[i]); - brcm_pcie_mdio_read(pcie->base, MDIO_PORT0, regs[i], &tmp); - dev_dbg(pcie->dev, "PCIE MDIO post_refclk 0x%02x = 0x%04x\n", - regs[i], tmp); - } - usleep_range(100, 200); -} - /* Limits operation to a specific generation (1, 2, or 3) */ static void brcm_pcie_set_gen(struct brcm_pcie *pcie, int gen) { @@ -578,73 +658,6 @@ static void brcm_pcie_set_outbound_win(struct brcm_pcie *pcie, writel(tmp, pcie->base + PCIE_MEM_WIN0_LIMIT_HI(win)); } -static void brcm_pcie_set_tc_qos(struct brcm_pcie *pcie) -{ - int i; - u32 reg; - - if (pcie->soc_base != BCM2712) - return; - - /* Disable broken QOS forwarding search. Set chicken bits for 2712D0 */ - reg = readl(pcie->base + PCIE_MISC_AXI_INTF_CTRL); - reg &= ~AXI_REQFIFO_EN_QOS_PROPAGATION; - reg |= AXI_EN_RCLK_QOS_ARRAY_FIX | AXI_EN_QOS_UPDATE_TIMING_FIX | - AXI_DIS_QOS_GATING_IN_MASTER; - writel(reg, pcie->base + PCIE_MISC_AXI_INTF_CTRL); - - /* - * If the QOS_UPDATE_TIMING_FIX bit is Reserved-0, then this is a - * 2712C1 chip, or a single-lane RC. Use the best-effort alternative - * which is to partially throttle AXI requests in-flight to the SDC. - */ - reg = readl(pcie->base + PCIE_MISC_AXI_INTF_CTRL); - if (!(reg & AXI_EN_QOS_UPDATE_TIMING_FIX)) { - reg &= ~AXI_MASTER_MAX_OUTSTANDING_REQUESTS_MASK; - reg |= 15; - writel(reg, pcie->base + PCIE_MISC_AXI_INTF_CTRL); - } - - /* Disable VDM reception by default - QoS map defaults to 0 */ - reg = readl(pcie->base + PCIE_MISC_CTRL_1); - reg &= ~PCIE_MISC_CTRL_1_EN_VDM_QOS_CONTROL_MASK; - writel(reg, pcie->base + PCIE_MISC_CTRL_1); - - if (!of_property_read_u32(pcie->np, "brcm,fifo-qos-map", &pcie->qos_map)) { - /* - * Backpressure mode - bottom 4 nibbles are QoS for each - * quartile of FIFO level. Each TC gets the same map, because - * this mode is intended for nonrealtime EPs. - */ - - pcie->qos_map &= 0x0000ffff; - for (i = 0; i < 8; i++) - writel(pcie->qos_map, pcie->base + PCIE_MISC_TC_QUEUE_TO_QOS_MAP(i)); - - return; - } - - if (!of_property_read_u32(pcie->np, "brcm,vdm-qos-map", &pcie->qos_map)) { - - reg = readl(pcie->base + PCIE_MISC_CTRL_1); - reg |= PCIE_MISC_CTRL_1_EN_VDM_QOS_CONTROL_MASK; - writel(reg, pcie->base + PCIE_MISC_CTRL_1); - - /* No forwarding means no point separating panic priorities from normal */ - writel(pcie->qos_map, pcie->base + PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_LO); - writel(pcie->qos_map, pcie->base + PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_HI); - - /* Match Vendor ID of 0 */ - writel(0, pcie->base + PCIE_RC_TL_VDM_CTL1); - /* Forward VDMs to priority interface - at least the rx counters work */ - reg = readl(pcie->base + PCIE_RC_TL_VDM_CTL0); - reg |= PCIE_RC_TL_VDM_CTL0_VDM_ENABLED_MASK | - PCIE_RC_TL_VDM_CTL0_VDM_IGNORETAG_MASK | - PCIE_RC_TL_VDM_CTL0_VDM_IGNOREVNDRID_MASK; - writel(reg, pcie->base + PCIE_RC_TL_VDM_CTL0); - } -} - static struct irq_chip brcm_msi_irq_chip = { .name = "BRCM STB PCIe MSI", .irq_ack = irq_chip_ack_parent, @@ -653,8 +666,8 @@ static struct irq_chip brcm_msi_irq_chip = { }; static struct msi_domain_info brcm_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), + .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_NO_AFFINITY | MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, .chip = &brcm_msi_irq_chip, }; @@ -673,23 +686,10 @@ static void brcm_pcie_msi_isr(struct irq_desc *desc) status = readl(msi->intr_base + MSI_INT_STATUS); status >>= msi->legacy_shift; - for_each_set_bit(bit, &status, BRCM_INT_PCI_MSI_NR/*msi->nr*/) { - unsigned long virq; - bool found = false; - - virq = irq_find_mapping(msi->inner_domain, bit); - if (virq) { - found = true; - dev_dbg(dev, "MSI -> %ld\n", virq); - generic_handle_irq(virq); - } - virq = irq_find_mapping(msi->inner_domain, bit + 32); - if (virq) { - found = true; - dev_dbg(dev, "MSI -> %ld\n", virq); - generic_handle_irq(virq); - } - if (!found) + for_each_set_bit(bit, &status, msi->nr) { + int ret; + ret = generic_handle_domain_irq(msi->inner_domain, bit); + if (ret) dev_dbg(dev, "unexpected MSI\n"); } @@ -702,13 +702,13 @@ static void brcm_msi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) msg->address_lo = lower_32_bits(msi->target_addr); msg->address_hi = upper_32_bits(msi->target_addr); - msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | (data->hwirq & 0x1f); + msg->data = (0xffff & PCIE_MISC_MSI_DATA_CONFIG_VAL_32) | data->hwirq; } static void brcm_msi_ack_irq(struct irq_data *data) { struct brcm_msi *msi = irq_data_get_irq_chip_data(data); - const int shift_amt = (data->hwirq & 0x1f) + msi->legacy_shift; + const int shift_amt = data->hwirq + msi->legacy_shift; writel(1 << shift_amt, msi->intr_base + MSI_INT_CLR); } @@ -868,7 +868,7 @@ static int brcm_pcie_enable_msi(struct brcm_pcie *pcie) msi->legacy_shift = 24; } else { msi->intr_base = msi->base + PCIE_MSI_INTR2_BASE; - msi->nr = 64; //BRCM_INT_PCI_MSI_NR; + msi->nr = BRCM_INT_PCI_MSI_NR; msi->legacy_shift = 0; } @@ -890,9 +890,6 @@ static bool brcm_pcie_rc_mode(struct brcm_pcie *pcie) void __iomem *base = pcie->base; u32 val = readl(base + PCIE_MISC_PCIE_STATUS); - if (pcie->soc_base == BCM2712) - return !!FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK_2712, val) | 1; //XXX - return !!FIELD_GET(PCIE_MISC_PCIE_STATUS_PCIE_PORT_MASK, val); } @@ -985,21 +982,6 @@ static int brcm_pcie_bridge_sw_init_set_7278(struct brcm_pcie *pcie, u32 val) return 0; } -static int brcm_pcie_bridge_sw_init_set_2712(struct brcm_pcie *pcie, u32 val) -{ - int ret; - - if (WARN_ONCE(!pcie->bridge_reset, "missing bridge reset controller\n")) - return -EINVAL; - - if (val) - ret = reset_control_assert(pcie->bridge_reset); - else - ret = reset_control_deassert(pcie->bridge_reset); - - return ret; -} - static int brcm_pcie_perst_set_4908(struct brcm_pcie *pcie, u32 val) { int ret; @@ -1030,18 +1012,6 @@ static int brcm_pcie_perst_set_7278(struct brcm_pcie *pcie, u32 val) return 0; } -static int brcm_pcie_perst_set_2712(struct brcm_pcie *pcie, u32 val) -{ - u32 tmp; - - /* Perst bit has moved and assert value is 0 */ - tmp = readl(pcie->base + PCIE_MISC_PCIE_CTRL); - u32p_replace_bits(&tmp, !val, PCIE_MISC_PCIE_CTRL_PCIE_PERSTB_MASK); - writel(tmp, pcie->base + PCIE_MISC_PCIE_CTRL); - - return 0; -} - static int brcm_pcie_perst_set_generic(struct brcm_pcie *pcie, u32 val) { u32 tmp; @@ -1060,6 +1030,7 @@ static int brcm_pcie_post_setup_bcm2712(struct brcm_pcie *pcie) static const u8 regs[] = { 0x16, 0x17, 0x18, 0x19, 0x1b, 0x1c, 0x1e }; int ret, i; u32 tmp; + u8 qos_map[8]; /* Allow a 54MHz (xosc) refclk source */ ret = brcm_pcie_mdio_write(pcie->base, MDIO_PORT0, SET_ADDR_OFFSET, 0x1600); @@ -1083,6 +1054,108 @@ static int brcm_pcie_post_setup_bcm2712(struct brcm_pcie *pcie) tmp |= 0x12; writel(tmp, pcie->base + PCIE_RC_PL_PHY_CTL_15); + /* + * BCM7712/2712 uses a UBUS-AXI bridge. + * Suppress AXI error responses and return 1s for read failures. + */ + tmp = readl(pcie->base + PCIE_MISC_UBUS_CTRL); + u32p_replace_bits(&tmp, 1, PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_ERR_DIS_MASK); + u32p_replace_bits(&tmp, 1, PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_DECERR_DIS_MASK); + writel(tmp, pcie->base + PCIE_MISC_UBUS_CTRL); + writel(0xffffffff, pcie->base + PCIE_MISC_AXI_READ_ERROR_DATA); + + /* + * Adjust timeouts. The UBUS timeout also affects Configuration Request + * Retry responses, as the request will get terminated if + * either timeout expires, so both have to be a large value + * (in clocks of 750MHz). + * Set UBUS timeout to 250ms, then set RC config retry timeout + * to be ~240ms. + * + * If CRSSVE=1 this will stop the core from blocking on a Retry + * response, but does require the device to be well-behaved... + */ + writel(0xB2D0000, pcie->base + PCIE_MISC_UBUS_TIMEOUT); + writel(0xABA0000, pcie->base + PCIE_MISC_RC_CONFIG_RETRY_TIMEOUT); + + /* + * BCM2712 has a configurable QoS mechanism that assigns TLP Traffic Classes + * to separate AXI IDs with a configurable priority scheme. + * Dynamic priority elevation is supported through reception of Type 1 + * Vendor Defined Messages, but several bugs make this largely ineffective. + */ + + /* Disable broken forwarding search. Set chicken bits for 2712D0 */ + tmp = readl(pcie->base + PCIE_MISC_AXI_INTF_CTRL); + tmp &= ~AXI_REQFIFO_EN_QOS_PROPAGATION; + tmp |= AXI_EN_RCLK_QOS_ARRAY_FIX | AXI_EN_QOS_UPDATE_TIMING_FIX | + AXI_DIS_QOS_GATING_IN_MASTER; + writel(tmp, pcie->base + PCIE_MISC_AXI_INTF_CTRL); + + /* + * Work around spurious QoS=0 assignments to inbound traffic. + * If the QOS_UPDATE_TIMING_FIX bit is Reserved-0, then this is a + * 2712C1 chip, or a single-lane RC. Use the best-effort alternative + * which is to partially throttle AXI requests in-flight to SDRAM. + */ + tmp = readl(pcie->base + PCIE_MISC_AXI_INTF_CTRL); + if (!(tmp & AXI_EN_QOS_UPDATE_TIMING_FIX)) { + tmp &= ~AXI_MASTER_MAX_OUTSTANDING_REQUESTS_MASK; + tmp |= 15; + writel(tmp, pcie->base + PCIE_MISC_AXI_INTF_CTRL); + } + + /* Disable VDM reception by default */ + tmp = readl(pcie->base + PCIE_MISC_CTRL_1); + tmp &= ~PCIE_MISC_CTRL_1_EN_VDM_QOS_CONTROL_MASK; + writel(tmp, pcie->base + PCIE_MISC_CTRL_1); + + if (!of_property_read_u8_array(pcie->np, "brcm,fifo-qos-map", qos_map, 4)) { + + /* + * Backpressure mode - each element is QoS for each + * quartile of FIFO level. Each TC gets the same map, because + * this mode is intended for nonrealtime EPs. + */ + tmp = 0; + for (i = 0; i < 4; i++) { + /* Priorities range from 0-15 */ + qos_map[i] &= 0x0f; + tmp |= qos_map[i] << (i * 4); + } + for (i = 0; i < 8; i++) + writel(tmp, pcie->base + PCIE_MISC_TC_QUEUE_TO_QOS_MAP(i)); + + } else if (!of_property_read_u8_array(pcie->np, "brcm,vdm-qos-map", qos_map, 8)) { + /* Enable VDM reception */ + tmp = readl(pcie->base + PCIE_MISC_CTRL_1); + tmp |= PCIE_MISC_CTRL_1_EN_VDM_QOS_CONTROL_MASK; + writel(tmp, pcie->base + PCIE_MISC_CTRL_1); + + tmp = 0; + for (i = 0; i < 8; i++) { + /* Priorities range from 0-15 */ + qos_map[i] &= 0x0f; + tmp |= qos_map[i] << (i * 4); + } + /* Broken forwarding means no point separating panic priorities from normal */ + writel(tmp, pcie->base + PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_LO); + writel(tmp, pcie->base + PCIE_MISC_VDM_PRIORITY_TO_QOS_MAP_HI); + + /* Match Vendor ID of 0 */ + writel(0, pcie->base + PCIE_RC_TL_VDM_CTL1); + + /* + * Forward VDMs to priority interface anyway - + * useful for debugging starvation through the received VDM count fields. + */ + tmp = readl(pcie->base + PCIE_RC_TL_VDM_CTL0); + tmp |= PCIE_RC_TL_VDM_CTL0_VDM_ENABLED_MASK | + PCIE_RC_TL_VDM_CTL0_VDM_IGNORETAG_MASK | + PCIE_RC_TL_VDM_CTL0_VDM_IGNOREVNDRID_MASK; + writel(tmp, pcie->base + PCIE_RC_TL_VDM_CTL0); + } + return 0; } @@ -1289,135 +1362,16 @@ static void set_inbound_win_registers(struct brcm_pcie *pcie, } } -static int brcm_pcie_get_rc_bar2_size_and_offset(struct brcm_pcie *pcie, - u64 *rc_bar2_size, - u64 *rc_bar2_offset) -{ - struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); - struct resource_entry *entry; - struct device *dev = pcie->dev; - u64 lowest_pcie_addr = ~(u64)0; - int ret, i = 0; - u64 size = 0; - - resource_list_for_each_entry(entry, &bridge->dma_ranges) { - u64 pcie_beg = entry->res->start - entry->offset; - - size += entry->res->end - entry->res->start + 1; - if (pcie_beg < lowest_pcie_addr) - lowest_pcie_addr = pcie_beg; - if (pcie->soc_base == BCM2711 || pcie->soc_base == BCM2712) - break; // Only consider the first entry - } - - if (lowest_pcie_addr == ~(u64)0) { - dev_err(dev, "DT node has no dma-ranges\n"); - return -EINVAL; - } - - ret = of_property_read_variable_u64_array(pcie->np, "brcm,scb-sizes", pcie->memc_size, 1, - PCIE_BRCM_MAX_MEMC); - - if (ret <= 0) { - /* Make an educated guess */ - pcie->num_memc = 1; - pcie->memc_size[0] = 1ULL << fls64(size - 1); - } else { - pcie->num_memc = ret; - } - - /* Each memc is viewed through a "port" that is a power of 2 */ - for (i = 0, size = 0; i < pcie->num_memc; i++) - size += pcie->memc_size[i]; - - /* System memory starts at this address in PCIe-space */ - *rc_bar2_offset = lowest_pcie_addr; - /* The sum of all memc views must also be a power of 2 */ - *rc_bar2_size = 1ULL << fls64(size - 1); - - /* - * We validate the inbound memory view even though we should trust - * whatever the device-tree provides. This is because of an HW issue on - * early Raspberry Pi 4's revisions (bcm2711). It turns out its - * firmware has to dynamically edit dma-ranges due to a bug on the - * PCIe controller integration, which prohibits any access above the - * lower 3GB of memory. Given this, we decided to keep the dma-ranges - * in check, avoiding hard to debug device-tree related issues in the - * future: - * - * The PCIe host controller by design must set the inbound viewport to - * be a contiguous arrangement of all of the system's memory. In - * addition, its size mut be a power of two. To further complicate - * matters, the viewport must start on a pcie-address that is aligned - * on a multiple of its size. If a portion of the viewport does not - * represent system memory -- e.g. 3GB of memory requires a 4GB - * viewport -- we can map the outbound memory in or after 3GB and even - * though the viewport will overlap the outbound memory the controller - * will know to send outbound memory downstream and everything else - * upstream. - * - * For example: - * - * - The best-case scenario, memory up to 3GB, is to place the inbound - * region in the first 4GB of pcie-space, as some legacy devices can - * only address 32bits. We would also like to put the MSI under 4GB - * as well, since some devices require a 32bit MSI target address. - * - * - If the system memory is 4GB or larger we cannot start the inbound - * region at location 0 (since we have to allow some space for - * outbound memory @ 3GB). So instead it will start at the 1x - * multiple of its size - */ - if (!*rc_bar2_size || (*rc_bar2_offset & (*rc_bar2_size - 1)) || - (*rc_bar2_offset < SZ_4G && *rc_bar2_offset > SZ_2G)) { - dev_err(dev, "Invalid rc_bar2_offset/size: size 0x%llx, off 0x%llx\n", - *rc_bar2_size, *rc_bar2_offset); - return -EINVAL; - } - - return 0; -} - -static int brcm_pcie_get_rc_bar_n(struct brcm_pcie *pcie, - int idx, - u64 *rc_bar_cpu, - u64 *rc_bar_size, - u64 *rc_bar_pci) -{ - struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie); - struct resource_entry *entry; - int i = 0; - - resource_list_for_each_entry(entry, &bridge->dma_ranges) { - if (i == idx) { - *rc_bar_cpu = entry->res->start; - *rc_bar_size = entry->res->end - entry->res->start + 1; - *rc_bar_pci = entry->res->start - entry->offset; - return 0; - } - - i++; - } - - return -EINVAL; -} - static int brcm_pcie_setup(struct brcm_pcie *pcie) { -#if 0 struct inbound_win inbound_wins[PCIE_BRCM_MAX_INBOUND_WINS]; -#endif - u64 rc_bar2_offset, rc_bar2_size; void __iomem *base = pcie->base; struct pci_host_bridge *bridge; struct resource_entry *entry; u32 tmp, burst, aspm_support; u8 num_out_wins = 0; -#if 0 int num_inbound_wins = 0; -#endif int memc, ret; - int count, i; /* Reset the bridge */ ret = pcie->cfg->bridge_sw_init_set(pcie, 1); @@ -1449,17 +1403,6 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) /* Wait for SerDes to be stable */ usleep_range(100, 200); - if (pcie->soc_base == BCM2712) { - /* Allow a 54MHz (xosc) refclk source */ - brcm_pcie_munge_pll(pcie); - /* Fix for L1SS errata */ - tmp = readl(base + PCIE_RC_PL_PHY_CTL_15); - tmp &= ~PCIE_RC_PL_PHY_CTL_15_PM_CLK_PERIOD_MASK; - /* PM clock period is 18.52ns (round down) */ - tmp |= 0x12; - writel(tmp, base + PCIE_RC_PL_PHY_CTL_15); - } - /* * SCB_MAX_BURST_SIZE is a two bit field. For GENERIC chips it * is encoded as 0=128, 1=256, 2=512, 3=Rsvd, for BCM7278 it @@ -1476,38 +1419,27 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) /* * Set SCB_MAX_BURST_SIZE, CFG_READ_UR_MODE, SCB_ACCESS_EN, - * RCB_MPS_MODE + * RCB_MPS_MODE, RCB_64B_MODE */ tmp = readl(base + PCIE_MISC_MISC_CTRL); u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_SCB_ACCESS_EN_MASK); u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_CFG_READ_UR_MODE_MASK); u32p_replace_bits(&tmp, burst, PCIE_MISC_MISC_CTRL_MAX_BURST_SIZE_MASK); - if (pcie->rcb_mps_mode) - u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK); + u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_MPS_MODE_MASK); + u32p_replace_bits(&tmp, 1, PCIE_MISC_MISC_CTRL_PCIE_RCB_64B_MODE_MASK); writel(tmp, base + PCIE_MISC_MISC_CTRL); - brcm_pcie_set_tc_qos(pcie); + num_inbound_wins = brcm_pcie_get_inbound_wins(pcie, inbound_wins); + if (num_inbound_wins < 0) + return num_inbound_wins; - ret = brcm_pcie_get_rc_bar2_size_and_offset(pcie, &rc_bar2_size, - &rc_bar2_offset); - if (ret) - return ret; - - tmp = lower_32_bits(rc_bar2_offset); - u32p_replace_bits(&tmp, brcm_pcie_encode_ibar_size(rc_bar2_size), - PCIE_MISC_RC_BAR2_CONFIG_LO_SIZE_MASK); - writel(tmp, base + PCIE_MISC_RC_BAR2_CONFIG_LO); - writel(upper_32_bits(rc_bar2_offset), - base + PCIE_MISC_RC_BAR2_CONFIG_HI); + set_inbound_win_registers(pcie, inbound_wins, num_inbound_wins); if (!brcm_pcie_rc_mode(pcie)) { dev_err(pcie->dev, "PCIe RC controller misconfigured as Endpoint\n"); return -EINVAL; } - tmp = readl(base + PCIE_MISC_UBUS_BAR2_CONFIG_REMAP); - u32p_replace_bits(&tmp, 1, PCIE_MISC_UBUS_BAR2_CONFIG_REMAP_ACCESS_ENABLE_MASK); - writel(tmp, base + PCIE_MISC_UBUS_BAR2_CONFIG_REMAP); tmp = readl(base + PCIE_MISC_MISC_CTRL); for (memc = 0; memc < pcie->num_memc; memc++) { u32 scb_size_val = ilog2(pcie->memc_size[memc]) - 15; @@ -1521,29 +1453,6 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) } writel(tmp, base + PCIE_MISC_MISC_CTRL); - if (pcie->soc_base == BCM2712) { - /* Suppress AXI error responses and return 1s for read failures */ - tmp = readl(base + PCIE_MISC_UBUS_CTRL); - u32p_replace_bits(&tmp, 1, PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_ERR_DIS_MASK); - u32p_replace_bits(&tmp, 1, PCIE_MISC_UBUS_CTRL_UBUS_PCIE_REPLY_DECERR_DIS_MASK); - writel(tmp, base + PCIE_MISC_UBUS_CTRL); - writel(0xffffffff, base + PCIE_MISC_AXI_READ_ERROR_DATA); - - /* - * Adjust timeouts. The UBUS timeout also affects CRS - * completion retries, as the request will get terminated if - * either timeout expires, so both have to be a large value - * (in clocks of 750MHz). - * Set UBUS timeout to 250ms, then set RC config retry timeout - * to be ~240ms. - * - * Setting CRSVis=1 will stop the core from blocking on a CRS - * response, but does require the device to be well-behaved... - */ - writel(0xB2D0000, base + PCIE_MISC_UBUS_TIMEOUT); - writel(0xABA0000, base + PCIE_MISC_RC_CONFIG_RETRY_TIMEOUT); - } - /* * We ideally want the MSI target address to be located in the 32bit * addressable memory area. Some devices might depend on it. This is @@ -1551,58 +1460,22 @@ static int brcm_pcie_setup(struct brcm_pcie *pcie) * 4GB or when the inbound area is smaller than 4GB (taking into * account the rounding-up we're forced to perform). */ - if (rc_bar2_offset >= SZ_4G || (rc_bar2_size + rc_bar2_offset) < SZ_4G) + if (inbound_wins[2].pci_offset >= SZ_4G || + (inbound_wins[2].size + inbound_wins[2].pci_offset) < SZ_4G) pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_LT_4GB; else pcie->msi_target_addr = BRCM_MSI_TARGET_ADDR_GT_4GB; - /* disable the PCIe->GISB memory window (RC_BAR1) */ - tmp = readl(base + PCIE_MISC_RC_BAR1_CONFIG_LO); - tmp &= ~PCIE_MISC_RC_BAR1_CONFIG_LO_SIZE_MASK; - writel(tmp, base + PCIE_MISC_RC_BAR1_CONFIG_LO); - - /* disable the PCIe->SCB memory window (RC_BAR3) */ - tmp = readl(base + PCIE_MISC_RC_BAR3_CONFIG_LO); - tmp &= ~PCIE_MISC_RC_BAR3_CONFIG_LO_SIZE_MASK; - writel(tmp, base + PCIE_MISC_RC_BAR3_CONFIG_LO); - /* Always advertise L1 capability */ - aspm_support = BIT(1); - /* Advertise L0s capability unless 'aspm-no-l0s' is set */ + /* Don't advertise L0s capability if 'aspm-no-l0s' */ + aspm_support = PCIE_LINK_STATE_L1; if (!of_property_read_bool(pcie->np, "aspm-no-l0s")) - aspm_support |= BIT(0); + aspm_support |= PCIE_LINK_STATE_L0S; tmp = readl(base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); u32p_replace_bits(&tmp, aspm_support, PCIE_RC_CFG_PRIV1_LINK_CAPABILITY_ASPM_SUPPORT_MASK); writel(tmp, base + PCIE_RC_CFG_PRIV1_LINK_CAPABILITY); - /* program additional inbound windows (RC_BAR4..RC_BAR10) */ - count = (pcie->soc_base == BCM2712) ? 7 : 0; - for (i = 0; i < count; i++) { - u64 bar_cpu, bar_size, bar_pci; - - ret = brcm_pcie_get_rc_bar_n(pcie, 1 + i, &bar_cpu, &bar_size, - &bar_pci); - if (ret) - break; - - tmp = lower_32_bits(bar_pci); - u32p_replace_bits(&tmp, brcm_pcie_encode_ibar_size(bar_size), - PCIE_MISC_RC_BAR_CONFIG_LO_SIZE_MASK); - writel(tmp, base + PCIE_MISC_RC_BAR4_CONFIG_LO + i * 8); - writel(upper_32_bits(bar_pci), - base + PCIE_MISC_RC_BAR4_CONFIG_HI + i * 8); - - tmp = upper_32_bits(bar_cpu) & - PCIE_MISC_UBUS_BAR_CONFIG_REMAP_HI_MASK; - writel(tmp, - base + PCIE_MISC_UBUS_BAR4_CONFIG_REMAP_HI + i * 8); - tmp = lower_32_bits(bar_cpu) & - PCIE_MISC_UBUS_BAR_CONFIG_REMAP_LO_MASK; - writel(tmp | PCIE_MISC_UBUS_BAR_CONFIG_REMAP_ENABLE, - base + PCIE_MISC_UBUS_BAR4_CONFIG_REMAP_LO + i * 8); - } - /* * For config space accesses on the RC, show the right class for * a PCIe-PCIe bridge (the default setting is to be EP mode). @@ -1727,32 +1600,169 @@ static void brcm_config_clkreq(struct brcm_pcie *pcie) } else { /* - * "safe" -- No power savings; refclk is driven by RC + * "safe" -- No power savings; refclk and CLKREQ# are driven by RC * unconditionally. */ if (strcmp(mode, "safe") != 0) dev_err(pcie->dev, err_msg); mode = "safe"; + clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_REFCLK_OVRD_OUT_MASK; + clkreq_cntl |= PCIE_MISC_HARD_PCIE_HARD_DEBUG_REFCLK_OVRD_ENABLE_MASK; + /* + * Un-advertise L1ss as configuring an EP to enter L1.x with CLKREQ# + * physically unconnected will result in a dead link. + */ + tmp = readl(pcie->base + PCIE_RC_CFG_PRIV1_ROOT_CAP); + u32p_replace_bits(&tmp, 2, PCIE_RC_CFG_PRIV1_ROOT_CAP_L1SS_MODE_MASK); + writel(tmp, pcie->base + PCIE_RC_CFG_PRIV1_ROOT_CAP); } writel(clkreq_cntl, pcie->base + HARD_DEBUG(pcie)); dev_info(pcie->dev, "clkreq-mode set to %s\n", mode); } +static void brcm_pcie_stats_trigger(struct brcm_pcie *pcie, u32 micros) +{ + u32 tmp; + + /* + * A 0->1 transition on CTRL_EN is required to clear counters and start capture. + * A microseconds count of 0 starts continuous gathering. + */ + tmp = readl(pcie->base + PCIE_RC_PL_STATS_CTRL); + u32p_replace_bits(&tmp, 0, PCIE_RC_PL_STATS_CTRL_EN_MASK); + writel(tmp, pcie->base + PCIE_RC_PL_STATS_CTRL); + + if (micros >= (1 << 28)) + micros = (1 << 28) - 1U; + u32p_replace_bits(&tmp, micros, PCIE_RC_PL_STATS_CTRL_LEN_MASK); + u32p_replace_bits(&tmp, 1, PCIE_RC_PL_STATS_CTRL_EN_MASK); + + writel(tmp, pcie->base + PCIE_RC_PL_STATS_CTRL); +} + +static void brcm_pcie_stats_capture(struct brcm_pcie *pcie) +{ + u32 tmp; + + /* Snapshot the counters - capture engine may still be running */ + pcie->tx_tlp = (u64)readl(pcie->base + PCIE_RC_PL_STATS_TXTLP_LO) + + ((u64)readl(pcie->base + PCIE_RC_PL_STATS_TXTLP_HI) << 32ULL); + pcie->rx_tlp = (u64)readl(pcie->base + PCIE_RC_PL_STATS_RXTLP_LO) + + ((u64)readl(pcie->base + PCIE_RC_PL_STATS_RXTLP_HI) << 32ULL); + pcie->tx_dllp = (u64)readl(pcie->base + PCIE_RC_PL_STATS_TXDLLP_LO) + + ((u64)readl(pcie->base + PCIE_RC_PL_STATS_TXDLLP_HI) << 32ULL); + pcie->rx_dllp = (u64)readl(pcie->base + PCIE_RC_PL_STATS_RXDLLP_LO) + + ((u64)readl(pcie->base + PCIE_RC_PL_STATS_RXDLLP_HI) << 32ULL); + + pcie->pl_rx_err = readl(pcie->base + PCIE_RC_PL_STATS_RXPL_ERR); + pcie->dl_rx_err = readl(pcie->base + PCIE_RC_PL_STATS_RXDL_ERR); + pcie->tl_rx_err = readl(pcie->base + PCIE_RC_PL_STATS_RXTL_ERR); + + tmp = readl(pcie->base + PCIE_RC_PL_LTSSM_STATS_3); + pcie->l0s_exit_time = FIELD_GET(PCIE_RC_PL_LTSSM_STATS_3_TIME_L0S_MASK, tmp); + pcie->recov_time = FIELD_GET(PCIE_RC_PL_LTSSM_STATS_3_TIME_RECOV_MASK, tmp); + + tmp = readl(pcie->base + PCIE_RC_PL_LTSSM_STATS_CNT); + pcie->l0s_fail_cnt = FIELD_GET(PCIE_RC_PL_LTSSM_STATS_CNT_L0S_FAIL_MASK, tmp); + pcie->recov_cnt = FIELD_GET(PCIE_RC_PL_LTSSM_STATS_CNT_RECOV_MASK, tmp); +} + +/* + * Dump the link state machine transitions for the first 100ms after fundamental reset release. + * Most link training completes in a far shorter time. + * + * The CPU-intensive nature of the capture means that this should only be used to + * diagnose fatal link startup failures. + */ +static void brcm_pcie_trace_link_start(struct brcm_pcie *pcie) +{ + struct device *dev = pcie->dev; + struct trace_entry *trace = pcie->ltssm_trace_buf; + int i = 0, j = 0; + u8 cur_state; + u32 ltssm_hist0, ltssm_hist1 = 0; + ktime_t start, timeout; + + start = ktime_get(); + timeout = ktime_add(start, ktime_set(0, NSEC_PER_MSEC * 100)); + /* + * The LTSSM history registers are implemented as an "open FIFO" where register data + * shuffles along with each push - or moves under one's feet, if you prefer. + * We can't atomically read more than one 32bit value (covering 4 entries), so poll + * quickly while guessing the position of the first value we haven't seen yet. + */ + do { + /* + * This delay appears to work around a HW bug where data can temporarily + * appear nibble-shifted. + */ + ndelay(10); + /* Snapshot the FIFO state. Lowest different "byte" is latest data. */ + ltssm_hist0 = readl(pcie->base + PCIE_RC_PL_LTSSM_HIST(3)); + if (ltssm_hist0 == ltssm_hist1) + continue; + ltssm_hist1 = ltssm_hist0; + + /* + * If the "fifo" has changed, we don't know by how much. + * Scan through byte-wise and look for states + */ + for (j = 24; j >= 0; j -= 8) { + cur_state = (ltssm_hist0 >> j) & 0xff; + /* Unassigned entry */ + if (cur_state == 0xff) + continue; + if (i > 0 && trace[i-1].st == cur_state) { + /* + * This is probably what we last saw. + * Next byte should be a new entry. + */ + j -= 8; + break; + } else if (i == 0 && brcm_pcie_decode_ltssm_state(cur_state)) { + /* Probably a new valid entry */ + break; + } + } + + for (; j >= 0 && i < TRACE_BUF_LEN; j -= 8) { + cur_state = (ltssm_hist0 >> j) & 0xff; + trace[i].st = cur_state; + trace[i].ts = ktime_sub(ktime_get(), start); + i++; + } + if (i == TRACE_BUF_LEN) + break; + } while (!ktime_after(ktime_get(), timeout)); + + dev_info(dev, "LTSSM trace captured %d events (max %u):\n", + i, TRACE_BUF_LEN); + for (i = 0; i < TRACE_BUF_LEN; i++) { + if (!trace[i].st) + break; + dev_info(dev, "%llu : %02x - %s\n", + ktime_to_us(trace[i].ts), + trace[i].st, + brcm_pcie_decode_ltssm_state(trace[i].st)); + } +} + static int brcm_pcie_start_link(struct brcm_pcie *pcie) { struct device *dev = pcie->dev; void __iomem *base = pcie->base; - u16 nlw, cls, lnksta; + u16 nlw, cls, lnksta, tmp16; bool ssc_good = false; - u32 tmp; - u16 tmp16; int ret, i; + u32 tmp; /* Limit the generation if specified */ if (pcie->gen) brcm_pcie_set_gen(pcie, pcie->gen); + brcm_pcie_stats_trigger(pcie, 0); + /* Unassert the fundamental reset */ if (pcie->tperst_clk_ms) { /* @@ -1776,6 +1786,8 @@ static int brcm_pcie_start_link(struct brcm_pcie *pcie) if (ret) return ret; + if (pcie->trace_ltssm) + brcm_pcie_trace_link_start(pcie); /* * Wait for 100ms after PERST# deassertion; see PCIe CEM specification * sections 2.2, PCIe r5.0, 6.6.1. @@ -1812,6 +1824,9 @@ static int brcm_pcie_start_link(struct brcm_pcie *pcie) pci_speed_string(pcie_link_speed[cls]), nlw, ssc_good ? "(SSC)" : "(!SSC)"); + /* Snapshot the boot-time stats */ + brcm_pcie_stats_capture(pcie); + /* * RootCtl bits are reset by perst_n, which undoes pci_enable_crs() * called prior to pci_add_new_bus() during probe. Re-enable here. @@ -1822,6 +1837,7 @@ static int brcm_pcie_start_link(struct brcm_pcie *pcie) u16p_replace_bits(&tmp16, 1, PCI_EXP_RTCTL_CRSSVE); writew(tmp16, base + BRCM_PCIE_CAP_REGS + PCI_EXP_RTCTL); } + return 0; } @@ -2142,6 +2158,7 @@ err_disable_clk: static void __brcm_pcie_remove(struct brcm_pcie *pcie) { + debugfs_remove_recursive(pcie->debugfs_dir); brcm_msi_remove(pcie); brcm_pcie_turn_off(pcie); if (brcm_phy_stop(pcie)) @@ -2193,13 +2210,6 @@ static const int pcie_offsets_bcm7712[] = { [PCIE_INTR2_CPU_BASE] = 0x4400, }; -static const int pcie_offsets_bcm2712[] = { - [EXT_CFG_INDEX] = 0x9000, - [EXT_CFG_DATA] = 0x9004, - [PCIE_HARD_DEBUG] = 0x4304, - [PCIE_INTR2_CPU_BASE] = 0x4400, -}; - static const struct pcie_cfg_data generic_cfg = { .offsets = pcie_offsets, .soc_base = GENERIC, @@ -2222,7 +2232,7 @@ static const struct pcie_cfg_data bcm2712_cfg = { .perst_set = brcm_pcie_perst_set_7278, .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_generic, .post_setup = brcm_pcie_post_setup_bcm2712, - .quirks = CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN, + .quirks = CFG_QUIRK_AVOID_BRIDGE_SHUTDOWN | CFG_QUIRK_NO_SSC, .num_inbound_wins = 10, }; @@ -2275,13 +2285,6 @@ static const struct pcie_cfg_data bcm7712_cfg = { .num_inbound_wins = 10, }; -static const struct pcie_cfg_data bcm2712_cfg = { - .offsets = pcie_offsets_bcm2712, - .perst_set = brcm_pcie_perst_set_2712, - .bridge_sw_init_set = brcm_pcie_bridge_sw_init_set_2712, - .soc_base = BCM2712, -}; - static const struct of_device_id brcm_pcie_match[] = { { .compatible = "brcm,bcm2711-pcie", .data = &bcm2711_cfg }, { .compatible = "brcm,bcm2712-pcie", .data = &bcm2712_cfg }, @@ -2293,7 +2296,6 @@ static const struct of_device_id brcm_pcie_match[] = { { .compatible = "brcm,bcm7435-pcie", .data = &bcm7435_cfg }, { .compatible = "brcm,bcm7445-pcie", .data = &generic_cfg }, { .compatible = "brcm,bcm7712-pcie", .data = &bcm7712_cfg }, - { .compatible = "brcm,bcm2712-pcie", .data = &bcm2712_cfg }, {}, }; @@ -2313,6 +2315,98 @@ static struct pci_ops brcm7425_pcie_ops = { .remove_bus = brcm_pcie_remove_bus, }; +static ssize_t debugfs_stats_trigger_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = filp->private_data; + struct brcm_pcie *pcie = m->private; + char kbuf[16] = {}; + unsigned long micros; + + if (count > sizeof(kbuf)) + return -EOVERFLOW; + + if (copy_from_user(kbuf, buf, count)) + return -EINVAL; + + if (kstrtol(kbuf, 0, µs) < 0) + return -EINVAL; + + if (micros >= (1 << 28)) + return -ERANGE; + + brcm_pcie_stats_trigger(pcie, micros); + return count; +} + +static int debugfs_stats_trigger_show(struct seq_file *s, void *unused) +{ + struct brcm_pcie *pcie = s->private; + u32 tmp; + + /* Return the state of the capture engine */ + tmp = readl(pcie->base + PCIE_RC_PL_STATS_CTRL); + tmp = FIELD_GET(PCIE_RC_PL_STATS_CTRL_EN_MASK, tmp); + seq_printf(s, "%u\n", tmp); + return 0; +} +DEFINE_SHOW_STORE_ATTRIBUTE(debugfs_stats_trigger); + +static ssize_t debugfs_stats_snapshot_write(struct file *filp, + const char __user *buf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = filp->private_data; + struct brcm_pcie *pcie = m->private; + + /* Any write triggers a snapshot of the stats register set */ + brcm_pcie_stats_capture(pcie); + return count; +} + +static int debugfs_stats_snapshot_show(struct seq_file *s, void *unused) +{ + struct brcm_pcie *pcie = s->private; + + seq_printf(s, "tx_tlp:\t\t%llu\n", pcie->tx_tlp); + seq_printf(s, "rx_tlp:\t\t%llu\n", pcie->rx_tlp); + seq_printf(s, "tx_dllp:\t%llu\n", pcie->tx_dllp); + seq_printf(s, "rx_dllp:\t%llu\n", pcie->rx_dllp); + seq_printf(s, "pl_rx_err:\t%u\n", pcie->pl_rx_err); + seq_printf(s, "dl_rx_err:\t%u\n", pcie->dl_rx_err); + seq_printf(s, "tl_rx_err:\t%u\n", pcie->tl_rx_err); + seq_printf(s, "l0s_exit_time:\t%u\n", pcie->l0s_exit_time); + seq_printf(s, "recov_time:\t%u\n", pcie->recov_time); + seq_printf(s, "l0s_fail_cnt\t%u\n", pcie->l0s_fail_cnt); + seq_printf(s, "recov_cnt:\t%u\n", pcie->recov_cnt); + + return 0; +} +DEFINE_SHOW_STORE_ATTRIBUTE(debugfs_stats_snapshot); + +static void brcm_pcie_init_debugfs(struct brcm_pcie *pcie) +{ + char *name; + + name = devm_kasprintf(pcie->dev, GFP_KERNEL, "%pOFP", pcie->dev->of_node); + if (!name) + return; + + pcie->debugfs_dir = debugfs_create_dir(name, NULL); + if (!pcie->debugfs_dir) + return; + + debugfs_create_file("stats_snapshot", 0644, pcie->debugfs_dir, pcie, + &debugfs_stats_snapshot_fops); + debugfs_create_file("stats_trigger", 0644, pcie->debugfs_dir, pcie, + &debugfs_stats_trigger_fops); +} + +static bool trace_ltssm; +module_param(trace_ltssm, bool, 0444); +MODULE_PARM_DESC(trace_ltssm, "Capture and dump link states during link training"); + static int brcm_pcie_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -2335,6 +2429,19 @@ static int brcm_pcie_probe(struct platform_device *pdev) pcie->dev = &pdev->dev; pcie->np = np; pcie->cfg = data; + pcie->trace_ltssm = trace_ltssm; + + brcm_pcie_init_debugfs(pcie); + + if (pcie->trace_ltssm) { + pcie->ltssm_trace_buf = devm_kzalloc(&pdev->dev, + sizeof(struct trace_entry) * TRACE_BUF_LEN, + GFP_KERNEL); + if (!pcie->ltssm_trace_buf) { + dev_err(&pdev->dev, "could not allocate trace buffer\n"); + return -ENOMEM; + } + } pcie->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(pcie->base)) @@ -2347,9 +2454,9 @@ static int brcm_pcie_probe(struct platform_device *pdev) ret = of_pci_get_max_link_speed(np); pcie->gen = (ret < 0) ? 0 : ret; - pcie->ssc = of_property_read_bool(np, "brcm,enable-ssc"); - pcie->l1ss = of_property_read_bool(np, "brcm,enable-l1ss"); - pcie->rcb_mps_mode = of_property_read_bool(np, "brcm,enable-mps-rcb"); + pcie->ssc = !(pcie->cfg->quirks & CFG_QUIRK_NO_SSC) && + of_property_read_bool(np, "brcm,enable-ssc"); + of_property_read_u32(np, "brcm,tperst-clk-ms", &pcie->tperst_clk_ms); pcie->rescal = devm_reset_control_get_optional_shared(&pdev->dev, "rescal"); @@ -2430,33 +2537,6 @@ static int brcm_pcie_probe(struct platform_device *pdev) dev_err(pcie->dev, "probe of internal MSI failed"); goto fail; } - } else if (pci_msi_enabled() && msi_np != pcie->np) { - /* Use RC_BAR1 for MIP access */ - u64 msi_pci_addr; - u64 msi_phys_addr; - - if (of_property_read_u64(msi_np, "brcm,msi-pci-addr", &msi_pci_addr)) { - dev_err(pcie->dev, "Unable to find MSI PCI address\n"); - ret = -EINVAL; - goto fail; - } - - if (of_property_read_u64(msi_np, "reg", &msi_phys_addr)) { - dev_err(pcie->dev, "Unable to find MSI physical address\n"); - ret = -EINVAL; - goto fail; - } - - writel(lower_32_bits(msi_pci_addr) | brcm_pcie_encode_ibar_size(0x1000), - pcie->base + PCIE_MISC_RC_BAR1_CONFIG_LO); - writel(upper_32_bits(msi_pci_addr), - pcie->base + PCIE_MISC_RC_BAR1_CONFIG_HI); - - writel(lower_32_bits(msi_phys_addr) | - PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_ACCESS_ENABLE_MASK, - pcie->base + PCIE_MISC_UBUS_BAR1_CONFIG_REMAP); - writel(upper_32_bits(msi_phys_addr), - pcie->base + PCIE_MISC_UBUS_BAR1_CONFIG_REMAP_HI); } bridge->ops = pcie->cfg->soc_base == BCM7425 ? |