diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index eb13294cb439..b314d44ded5e 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -53,7 +53,8 @@ i915-y += i915_drv.o \ intel_runtime_pm.o \ intel_sideband.o \ intel_uncore.o \ - intel_wakeref.o + intel_wakeref.o \ + vlv_suspend.o # core library code i915-y += \ diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c index 168d96b119e6..0e8b56d8fce2 100644 --- a/drivers/gpu/drm/i915/i915_drv.c +++ b/drivers/gpu/drm/i915/i915_drv.c @@ -81,71 +81,10 @@ #include "i915_vgpu.h" #include "intel_memory_region.h" #include "intel_pm.h" +#include "vlv_suspend.h" static struct drm_driver driver; -struct vlv_s0ix_state { - /* GAM */ - u32 wr_watermark; - u32 gfx_prio_ctrl; - u32 arb_mode; - u32 gfx_pend_tlb0; - u32 gfx_pend_tlb1; - u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM]; - u32 media_max_req_count; - u32 gfx_max_req_count; - u32 render_hwsp; - u32 ecochk; - u32 bsd_hwsp; - u32 blt_hwsp; - u32 tlb_rd_addr; - - /* MBC */ - u32 g3dctl; - u32 gsckgctl; - u32 mbctl; - - /* GCP */ - u32 ucgctl1; - u32 ucgctl3; - u32 rcgctl1; - u32 rcgctl2; - u32 rstctl; - u32 misccpctl; - - /* GPM */ - u32 gfxpause; - u32 rpdeuhwtc; - u32 rpdeuc; - u32 ecobus; - u32 pwrdwnupctl; - u32 rp_down_timeout; - u32 rp_deucsw; - u32 rcubmabdtmr; - u32 rcedata; - u32 spare2gh; - - /* Display 1 CZ domain */ - u32 gt_imr; - u32 gt_ier; - u32 pm_imr; - u32 pm_ier; - u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM]; - - /* GT SA CZ domain */ - u32 tilectl; - u32 gt_fifoctl; - u32 gtlc_wake_ctrl; - u32 gtlc_survive; - u32 pmwgicz; - - /* Display 2 CZ domain */ - u32 gu_ctl0; - u32 gu_ctl1; - u32 pcbr; - u32 clock_gate_dis2; -}; - static int i915_get_bridge_dev(struct drm_i915_private *dev_priv) { int domain = pci_domain_nr(dev_priv->drm.pdev->bus); @@ -437,29 +376,6 @@ static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv) } } -static int vlv_alloc_s0ix_state(struct drm_i915_private *i915) -{ - if (!IS_VALLEYVIEW(i915)) - return 0; - - /* we write all the values in the struct, so no need to zero it out */ - i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state), - GFP_KERNEL); - if (!i915->vlv_s0ix_state) - return -ENOMEM; - - return 0; -} - -static void vlv_free_s0ix_state(struct drm_i915_private *i915) -{ - if (!i915->vlv_s0ix_state) - return; - - kfree(i915->vlv_s0ix_state); - i915->vlv_s0ix_state = NULL; -} - static void sanitize_gpu(struct drm_i915_private *i915) { if (!INTEL_INFO(i915)->gpu_reset_clobbers_display) @@ -508,7 +424,7 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv) if (ret < 0) return ret; - ret = vlv_alloc_s0ix_state(dev_priv); + ret = vlv_suspend_init(dev_priv); if (ret < 0) goto err_workqueues; @@ -539,7 +455,7 @@ static int i915_driver_early_probe(struct drm_i915_private *dev_priv) err_gem: i915_gem_cleanup_early(dev_priv); intel_gt_driver_late_release(&dev_priv->gt); - vlv_free_s0ix_state(dev_priv); + vlv_suspend_cleanup(dev_priv); err_workqueues: i915_workqueues_cleanup(dev_priv); return ret; @@ -556,7 +472,7 @@ static void i915_driver_late_release(struct drm_i915_private *dev_priv) intel_power_domains_cleanup(dev_priv); i915_gem_cleanup_early(dev_priv); intel_gt_driver_late_release(&dev_priv->gt); - vlv_free_s0ix_state(dev_priv); + vlv_suspend_cleanup(dev_priv); i915_workqueues_cleanup(dev_priv); pm_qos_remove_request(&dev_priv->sb_qos); @@ -1674,10 +1590,6 @@ static void intel_suspend_encoders(struct drm_i915_private *dev_priv) drm_modeset_unlock_all(dev); } -static int vlv_resume_prepare(struct drm_i915_private *dev_priv, - bool rpm_resume); -static int vlv_suspend_complete(struct drm_i915_private *dev_priv); - static bool suspend_to_idle(struct drm_i915_private *dev_priv) { #if IS_ENABLED(CONFIG_ACPI_SLEEP) @@ -1764,7 +1676,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) struct drm_i915_private *dev_priv = to_i915(dev); struct pci_dev *pdev = dev_priv->drm.pdev; struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; - int ret = 0; + int ret; disable_rpm_wakeref_asserts(rpm); @@ -1777,9 +1689,7 @@ static int i915_drm_suspend_late(struct drm_device *dev, bool hibernation) intel_display_power_suspend_late(dev_priv); - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - ret = vlv_suspend_complete(dev_priv); - + ret = vlv_suspend_complete(dev_priv); if (ret) { drm_err(&dev_priv->drm, "Suspend complete failed: %d\n", ret); intel_power_domains_resume(dev_priv); @@ -1954,12 +1864,10 @@ static int i915_drm_resume_early(struct drm_device *dev) disable_rpm_wakeref_asserts(&dev_priv->runtime_pm); - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - ret = vlv_resume_prepare(dev_priv, false); + ret = vlv_resume_prepare(dev_priv, false); if (ret) drm_err(&dev_priv->drm, - "Resume prepare failed: %d, continuing anyway\n", - ret); + "Resume prepare failed: %d, continuing anyway\n", ret); intel_uncore_resume_early(&dev_priv->uncore); @@ -2126,390 +2034,11 @@ static int i915_pm_restore(struct device *kdev) return i915_pm_resume(kdev); } -/* - * Save all Gunit registers that may be lost after a D3 and a subsequent - * S0i[R123] transition. The list of registers needing a save/restore is - * defined in the VLV2_S0IXRegs document. This documents marks all Gunit - * registers in the following way: - * - Driver: saved/restored by the driver - * - Punit : saved/restored by the Punit firmware - * - No, w/o marking: no need to save/restore, since the register is R/O or - * used internally by the HW in a way that doesn't depend - * keeping the content across a suspend/resume. - * - Debug : used for debugging - * - * We save/restore all registers marked with 'Driver', with the following - * exceptions: - * - Registers out of use, including also registers marked with 'Debug'. - * These have no effect on the driver's operation, so we don't save/restore - * them to reduce the overhead. - * - Registers that are fully setup by an initialization function called from - * the resume path. For example many clock gating and RPS/RC6 registers. - * - Registers that provide the right functionality with their reset defaults. - * - * TODO: Except for registers that based on the above 3 criteria can be safely - * ignored, we save/restore all others, practically treating the HW context as - * a black-box for the driver. Further investigation is needed to reduce the - * saved/restored registers even further, by following the same 3 criteria. - */ -static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) -{ - struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state; - int i; - - if (!s) - return; - - /* GAM 0x4000-0x4770 */ - s->wr_watermark = I915_READ(GEN7_WR_WATERMARK); - s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL); - s->arb_mode = I915_READ(ARB_MODE); - s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0); - s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1); - - for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) - s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i)); - - s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); - s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT); - - s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7); - s->ecochk = I915_READ(GAM_ECOCHK); - s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7); - s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7); - - s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR); - - /* MBC 0x9024-0x91D0, 0x8500 */ - s->g3dctl = I915_READ(VLV_G3DCTL); - s->gsckgctl = I915_READ(VLV_GSCKGCTL); - s->mbctl = I915_READ(GEN6_MBCTL); - - /* GCP 0x9400-0x9424, 0x8100-0x810C */ - s->ucgctl1 = I915_READ(GEN6_UCGCTL1); - s->ucgctl3 = I915_READ(GEN6_UCGCTL3); - s->rcgctl1 = I915_READ(GEN6_RCGCTL1); - s->rcgctl2 = I915_READ(GEN6_RCGCTL2); - s->rstctl = I915_READ(GEN6_RSTCTL); - s->misccpctl = I915_READ(GEN7_MISCCPCTL); - - /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ - s->gfxpause = I915_READ(GEN6_GFXPAUSE); - s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC); - s->rpdeuc = I915_READ(GEN6_RPDEUC); - s->ecobus = I915_READ(ECOBUS); - s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL); - s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT); - s->rp_deucsw = I915_READ(GEN6_RPDEUCSW); - s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR); - s->rcedata = I915_READ(VLV_RCEDATA); - s->spare2gh = I915_READ(VLV_SPAREG2H); - - /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ - s->gt_imr = I915_READ(GTIMR); - s->gt_ier = I915_READ(GTIER); - s->pm_imr = I915_READ(GEN6_PMIMR); - s->pm_ier = I915_READ(GEN6_PMIER); - - for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) - s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i)); - - /* GT SA CZ domain, 0x100000-0x138124 */ - s->tilectl = I915_READ(TILECTL); - s->gt_fifoctl = I915_READ(GTFIFOCTL); - s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL); - s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG); - s->pmwgicz = I915_READ(VLV_PMWGICZ); - - /* Gunit-Display CZ domain, 0x182028-0x1821CF */ - s->gu_ctl0 = I915_READ(VLV_GU_CTL0); - s->gu_ctl1 = I915_READ(VLV_GU_CTL1); - s->pcbr = I915_READ(VLV_PCBR); - s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2); - - /* - * Not saving any of: - * DFT, 0x9800-0x9EC0 - * SARB, 0xB000-0xB1FC - * GAC, 0x5208-0x524C, 0x14000-0x14C000 - * PCI CFG - */ -} - -static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) -{ - struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state; - u32 val; - int i; - - if (!s) - return; - - /* GAM 0x4000-0x4770 */ - I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark); - I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); - I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16)); - I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); - I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); - - for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) - I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]); - - I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); - I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count); - - I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp); - I915_WRITE(GAM_ECOCHK, s->ecochk); - I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp); - I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp); - - I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr); - - /* MBC 0x9024-0x91D0, 0x8500 */ - I915_WRITE(VLV_G3DCTL, s->g3dctl); - I915_WRITE(VLV_GSCKGCTL, s->gsckgctl); - I915_WRITE(GEN6_MBCTL, s->mbctl); - - /* GCP 0x9400-0x9424, 0x8100-0x810C */ - I915_WRITE(GEN6_UCGCTL1, s->ucgctl1); - I915_WRITE(GEN6_UCGCTL3, s->ucgctl3); - I915_WRITE(GEN6_RCGCTL1, s->rcgctl1); - I915_WRITE(GEN6_RCGCTL2, s->rcgctl2); - I915_WRITE(GEN6_RSTCTL, s->rstctl); - I915_WRITE(GEN7_MISCCPCTL, s->misccpctl); - - /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ - I915_WRITE(GEN6_GFXPAUSE, s->gfxpause); - I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc); - I915_WRITE(GEN6_RPDEUC, s->rpdeuc); - I915_WRITE(ECOBUS, s->ecobus); - I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl); - I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout); - I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw); - I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr); - I915_WRITE(VLV_RCEDATA, s->rcedata); - I915_WRITE(VLV_SPAREG2H, s->spare2gh); - - /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ - I915_WRITE(GTIMR, s->gt_imr); - I915_WRITE(GTIER, s->gt_ier); - I915_WRITE(GEN6_PMIMR, s->pm_imr); - I915_WRITE(GEN6_PMIER, s->pm_ier); - - for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) - I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]); - - /* GT SA CZ domain, 0x100000-0x138124 */ - I915_WRITE(TILECTL, s->tilectl); - I915_WRITE(GTFIFOCTL, s->gt_fifoctl); - /* - * Preserve the GT allow wake and GFX force clock bit, they are not - * be restored, as they are used to control the s0ix suspend/resume - * sequence by the caller. - */ - val = I915_READ(VLV_GTLC_WAKE_CTRL); - val &= VLV_GTLC_ALLOWWAKEREQ; - val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ; - I915_WRITE(VLV_GTLC_WAKE_CTRL, val); - - val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); - val &= VLV_GFX_CLK_FORCE_ON_BIT; - val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT; - I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); - - I915_WRITE(VLV_PMWGICZ, s->pmwgicz); - - /* Gunit-Display CZ domain, 0x182028-0x1821CF */ - I915_WRITE(VLV_GU_CTL0, s->gu_ctl0); - I915_WRITE(VLV_GU_CTL1, s->gu_ctl1); - I915_WRITE(VLV_PCBR, s->pcbr); - I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); -} - -static int vlv_wait_for_pw_status(struct drm_i915_private *i915, - u32 mask, u32 val) -{ - i915_reg_t reg = VLV_GTLC_PW_STATUS; - u32 reg_value; - int ret; - - /* The HW does not like us polling for PW_STATUS frequently, so - * use the sleeping loop rather than risk the busy spin within - * intel_wait_for_register(). - * - * Transitioning between RC6 states should be at most 2ms (see - * valleyview_enable_rps) so use a 3ms timeout. - */ - ret = wait_for(((reg_value = - intel_uncore_read_notrace(&i915->uncore, reg)) & mask) - == val, 3); - - /* just trace the final value */ - trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); - - return ret; -} - -int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) -{ - u32 val; - int err; - - val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); - val &= ~VLV_GFX_CLK_FORCE_ON_BIT; - if (force_on) - val |= VLV_GFX_CLK_FORCE_ON_BIT; - I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); - - if (!force_on) - return 0; - - err = intel_wait_for_register(&dev_priv->uncore, - VLV_GTLC_SURVIVABILITY_REG, - VLV_GFX_CLK_STATUS_BIT, - VLV_GFX_CLK_STATUS_BIT, - 20); - if (err) - drm_err(&dev_priv->drm, - "timeout waiting for GFX clock force-on (%08x)\n", - I915_READ(VLV_GTLC_SURVIVABILITY_REG)); - - return err; -} - -static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow) -{ - u32 mask; - u32 val; - int err; - - val = I915_READ(VLV_GTLC_WAKE_CTRL); - val &= ~VLV_GTLC_ALLOWWAKEREQ; - if (allow) - val |= VLV_GTLC_ALLOWWAKEREQ; - I915_WRITE(VLV_GTLC_WAKE_CTRL, val); - POSTING_READ(VLV_GTLC_WAKE_CTRL); - - mask = VLV_GTLC_ALLOWWAKEACK; - val = allow ? mask : 0; - - err = vlv_wait_for_pw_status(dev_priv, mask, val); - if (err) - drm_err(&dev_priv->drm, "timeout disabling GT waking\n"); - - return err; -} - -static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, - bool wait_for_on) -{ - u32 mask; - u32 val; - - mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; - val = wait_for_on ? mask : 0; - - /* - * RC6 transitioning can be delayed up to 2 msec (see - * valleyview_enable_rps), use 3 msec for safety. - * - * This can fail to turn off the rc6 if the GPU is stuck after a failed - * reset and we are trying to force the machine to sleep. - */ - if (vlv_wait_for_pw_status(dev_priv, mask, val)) - drm_dbg(&dev_priv->drm, - "timeout waiting for GT wells to go %s\n", - onoff(wait_for_on)); -} - -static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv) -{ - if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) - return; - - drm_dbg(&dev_priv->drm, - "GT register access while GT waking disabled\n"); - I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); -} - -static int vlv_suspend_complete(struct drm_i915_private *dev_priv) -{ - u32 mask; - int err; - - /* - * Bspec defines the following GT well on flags as debug only, so - * don't treat them as hard failures. - */ - vlv_wait_for_gt_wells(dev_priv, false); - - mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; - drm_WARN_ON(&dev_priv->drm, - (I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask); - - vlv_check_no_gt_access(dev_priv); - - err = vlv_force_gfx_clock(dev_priv, true); - if (err) - goto err1; - - err = vlv_allow_gt_wake(dev_priv, false); - if (err) - goto err2; - - vlv_save_gunit_s0ix_state(dev_priv); - - err = vlv_force_gfx_clock(dev_priv, false); - if (err) - goto err2; - - return 0; - -err2: - /* For safety always re-enable waking and disable gfx clock forcing */ - vlv_allow_gt_wake(dev_priv, true); -err1: - vlv_force_gfx_clock(dev_priv, false); - - return err; -} - -static int vlv_resume_prepare(struct drm_i915_private *dev_priv, - bool rpm_resume) -{ - int err; - int ret; - - /* - * If any of the steps fail just try to continue, that's the best we - * can do at this point. Return the first error code (which will also - * leave RPM permanently disabled). - */ - ret = vlv_force_gfx_clock(dev_priv, true); - - vlv_restore_gunit_s0ix_state(dev_priv); - - err = vlv_allow_gt_wake(dev_priv, true); - if (!ret) - ret = err; - - err = vlv_force_gfx_clock(dev_priv, false); - if (!ret) - ret = err; - - vlv_check_no_gt_access(dev_priv); - - if (rpm_resume) - intel_init_clock_gating(dev_priv); - - return ret; -} - static int intel_runtime_suspend(struct device *kdev) { struct drm_i915_private *dev_priv = kdev_to_i915(kdev); struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; - int ret = 0; + int ret; if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv))) return -ENODEV; @@ -2532,9 +2061,7 @@ static int intel_runtime_suspend(struct device *kdev) intel_display_power_suspend(dev_priv); - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - ret = vlv_suspend_complete(dev_priv); - + ret = vlv_suspend_complete(dev_priv); if (ret) { drm_err(&dev_priv->drm, "Runtime suspend failed, disabling it (%d)\n", ret); @@ -2596,7 +2123,7 @@ static int intel_runtime_resume(struct device *kdev) { struct drm_i915_private *dev_priv = kdev_to_i915(kdev); struct intel_runtime_pm *rpm = &dev_priv->runtime_pm; - int ret = 0; + int ret; if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv))) return -ENODEV; @@ -2614,8 +2141,7 @@ static int intel_runtime_resume(struct device *kdev) intel_display_power_resume(dev_priv); - if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) - ret = vlv_resume_prepare(dev_priv, true); + ret = vlv_resume_prepare(dev_priv, true); intel_uncore_runtime_resume(&dev_priv->uncore); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index da509d9b8895..3330b538d379 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -1755,8 +1755,6 @@ void i915_driver_remove(struct drm_i915_private *i915); int i915_resume_switcheroo(struct drm_i915_private *i915); int i915_suspend_switcheroo(struct drm_i915_private *i915, pm_message_t state); -int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on); - static inline bool intel_gvt_active(struct drm_i915_private *dev_priv) { return dev_priv->gvt; diff --git a/drivers/gpu/drm/i915/vlv_suspend.c b/drivers/gpu/drm/i915/vlv_suspend.c new file mode 100644 index 000000000000..05047fe7765f --- /dev/null +++ b/drivers/gpu/drm/i915/vlv_suspend.c @@ -0,0 +1,484 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ + +#include + +#include + +#include "i915_drv.h" +#include "i915_reg.h" +#include "i915_trace.h" +#include "i915_utils.h" +#include "intel_pm.h" +#include "vlv_suspend.h" + +struct vlv_s0ix_state { + /* GAM */ + u32 wr_watermark; + u32 gfx_prio_ctrl; + u32 arb_mode; + u32 gfx_pend_tlb0; + u32 gfx_pend_tlb1; + u32 lra_limits[GEN7_LRA_LIMITS_REG_NUM]; + u32 media_max_req_count; + u32 gfx_max_req_count; + u32 render_hwsp; + u32 ecochk; + u32 bsd_hwsp; + u32 blt_hwsp; + u32 tlb_rd_addr; + + /* MBC */ + u32 g3dctl; + u32 gsckgctl; + u32 mbctl; + + /* GCP */ + u32 ucgctl1; + u32 ucgctl3; + u32 rcgctl1; + u32 rcgctl2; + u32 rstctl; + u32 misccpctl; + + /* GPM */ + u32 gfxpause; + u32 rpdeuhwtc; + u32 rpdeuc; + u32 ecobus; + u32 pwrdwnupctl; + u32 rp_down_timeout; + u32 rp_deucsw; + u32 rcubmabdtmr; + u32 rcedata; + u32 spare2gh; + + /* Display 1 CZ domain */ + u32 gt_imr; + u32 gt_ier; + u32 pm_imr; + u32 pm_ier; + u32 gt_scratch[GEN7_GT_SCRATCH_REG_NUM]; + + /* GT SA CZ domain */ + u32 tilectl; + u32 gt_fifoctl; + u32 gtlc_wake_ctrl; + u32 gtlc_survive; + u32 pmwgicz; + + /* Display 2 CZ domain */ + u32 gu_ctl0; + u32 gu_ctl1; + u32 pcbr; + u32 clock_gate_dis2; +}; + +/* + * Save all Gunit registers that may be lost after a D3 and a subsequent + * S0i[R123] transition. The list of registers needing a save/restore is + * defined in the VLV2_S0IXRegs document. This documents marks all Gunit + * registers in the following way: + * - Driver: saved/restored by the driver + * - Punit : saved/restored by the Punit firmware + * - No, w/o marking: no need to save/restore, since the register is R/O or + * used internally by the HW in a way that doesn't depend + * keeping the content across a suspend/resume. + * - Debug : used for debugging + * + * We save/restore all registers marked with 'Driver', with the following + * exceptions: + * - Registers out of use, including also registers marked with 'Debug'. + * These have no effect on the driver's operation, so we don't save/restore + * them to reduce the overhead. + * - Registers that are fully setup by an initialization function called from + * the resume path. For example many clock gating and RPS/RC6 registers. + * - Registers that provide the right functionality with their reset defaults. + * + * TODO: Except for registers that based on the above 3 criteria can be safely + * ignored, we save/restore all others, practically treating the HW context as + * a black-box for the driver. Further investigation is needed to reduce the + * saved/restored registers even further, by following the same 3 criteria. + */ +static void vlv_save_gunit_s0ix_state(struct drm_i915_private *dev_priv) +{ + struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state; + int i; + + if (!s) + return; + + /* GAM 0x4000-0x4770 */ + s->wr_watermark = I915_READ(GEN7_WR_WATERMARK); + s->gfx_prio_ctrl = I915_READ(GEN7_GFX_PRIO_CTRL); + s->arb_mode = I915_READ(ARB_MODE); + s->gfx_pend_tlb0 = I915_READ(GEN7_GFX_PEND_TLB0); + s->gfx_pend_tlb1 = I915_READ(GEN7_GFX_PEND_TLB1); + + for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) + s->lra_limits[i] = I915_READ(GEN7_LRA_LIMITS(i)); + + s->media_max_req_count = I915_READ(GEN7_MEDIA_MAX_REQ_COUNT); + s->gfx_max_req_count = I915_READ(GEN7_GFX_MAX_REQ_COUNT); + + s->render_hwsp = I915_READ(RENDER_HWS_PGA_GEN7); + s->ecochk = I915_READ(GAM_ECOCHK); + s->bsd_hwsp = I915_READ(BSD_HWS_PGA_GEN7); + s->blt_hwsp = I915_READ(BLT_HWS_PGA_GEN7); + + s->tlb_rd_addr = I915_READ(GEN7_TLB_RD_ADDR); + + /* MBC 0x9024-0x91D0, 0x8500 */ + s->g3dctl = I915_READ(VLV_G3DCTL); + s->gsckgctl = I915_READ(VLV_GSCKGCTL); + s->mbctl = I915_READ(GEN6_MBCTL); + + /* GCP 0x9400-0x9424, 0x8100-0x810C */ + s->ucgctl1 = I915_READ(GEN6_UCGCTL1); + s->ucgctl3 = I915_READ(GEN6_UCGCTL3); + s->rcgctl1 = I915_READ(GEN6_RCGCTL1); + s->rcgctl2 = I915_READ(GEN6_RCGCTL2); + s->rstctl = I915_READ(GEN6_RSTCTL); + s->misccpctl = I915_READ(GEN7_MISCCPCTL); + + /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ + s->gfxpause = I915_READ(GEN6_GFXPAUSE); + s->rpdeuhwtc = I915_READ(GEN6_RPDEUHWTC); + s->rpdeuc = I915_READ(GEN6_RPDEUC); + s->ecobus = I915_READ(ECOBUS); + s->pwrdwnupctl = I915_READ(VLV_PWRDWNUPCTL); + s->rp_down_timeout = I915_READ(GEN6_RP_DOWN_TIMEOUT); + s->rp_deucsw = I915_READ(GEN6_RPDEUCSW); + s->rcubmabdtmr = I915_READ(GEN6_RCUBMABDTMR); + s->rcedata = I915_READ(VLV_RCEDATA); + s->spare2gh = I915_READ(VLV_SPAREG2H); + + /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ + s->gt_imr = I915_READ(GTIMR); + s->gt_ier = I915_READ(GTIER); + s->pm_imr = I915_READ(GEN6_PMIMR); + s->pm_ier = I915_READ(GEN6_PMIER); + + for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) + s->gt_scratch[i] = I915_READ(GEN7_GT_SCRATCH(i)); + + /* GT SA CZ domain, 0x100000-0x138124 */ + s->tilectl = I915_READ(TILECTL); + s->gt_fifoctl = I915_READ(GTFIFOCTL); + s->gtlc_wake_ctrl = I915_READ(VLV_GTLC_WAKE_CTRL); + s->gtlc_survive = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + s->pmwgicz = I915_READ(VLV_PMWGICZ); + + /* Gunit-Display CZ domain, 0x182028-0x1821CF */ + s->gu_ctl0 = I915_READ(VLV_GU_CTL0); + s->gu_ctl1 = I915_READ(VLV_GU_CTL1); + s->pcbr = I915_READ(VLV_PCBR); + s->clock_gate_dis2 = I915_READ(VLV_GUNIT_CLOCK_GATE2); + + /* + * Not saving any of: + * DFT, 0x9800-0x9EC0 + * SARB, 0xB000-0xB1FC + * GAC, 0x5208-0x524C, 0x14000-0x14C000 + * PCI CFG + */ +} + +static void vlv_restore_gunit_s0ix_state(struct drm_i915_private *dev_priv) +{ + struct vlv_s0ix_state *s = dev_priv->vlv_s0ix_state; + u32 val; + int i; + + if (!s) + return; + + /* GAM 0x4000-0x4770 */ + I915_WRITE(GEN7_WR_WATERMARK, s->wr_watermark); + I915_WRITE(GEN7_GFX_PRIO_CTRL, s->gfx_prio_ctrl); + I915_WRITE(ARB_MODE, s->arb_mode | (0xffff << 16)); + I915_WRITE(GEN7_GFX_PEND_TLB0, s->gfx_pend_tlb0); + I915_WRITE(GEN7_GFX_PEND_TLB1, s->gfx_pend_tlb1); + + for (i = 0; i < ARRAY_SIZE(s->lra_limits); i++) + I915_WRITE(GEN7_LRA_LIMITS(i), s->lra_limits[i]); + + I915_WRITE(GEN7_MEDIA_MAX_REQ_COUNT, s->media_max_req_count); + I915_WRITE(GEN7_GFX_MAX_REQ_COUNT, s->gfx_max_req_count); + + I915_WRITE(RENDER_HWS_PGA_GEN7, s->render_hwsp); + I915_WRITE(GAM_ECOCHK, s->ecochk); + I915_WRITE(BSD_HWS_PGA_GEN7, s->bsd_hwsp); + I915_WRITE(BLT_HWS_PGA_GEN7, s->blt_hwsp); + + I915_WRITE(GEN7_TLB_RD_ADDR, s->tlb_rd_addr); + + /* MBC 0x9024-0x91D0, 0x8500 */ + I915_WRITE(VLV_G3DCTL, s->g3dctl); + I915_WRITE(VLV_GSCKGCTL, s->gsckgctl); + I915_WRITE(GEN6_MBCTL, s->mbctl); + + /* GCP 0x9400-0x9424, 0x8100-0x810C */ + I915_WRITE(GEN6_UCGCTL1, s->ucgctl1); + I915_WRITE(GEN6_UCGCTL3, s->ucgctl3); + I915_WRITE(GEN6_RCGCTL1, s->rcgctl1); + I915_WRITE(GEN6_RCGCTL2, s->rcgctl2); + I915_WRITE(GEN6_RSTCTL, s->rstctl); + I915_WRITE(GEN7_MISCCPCTL, s->misccpctl); + + /* GPM 0xA000-0xAA84, 0x8000-0x80FC */ + I915_WRITE(GEN6_GFXPAUSE, s->gfxpause); + I915_WRITE(GEN6_RPDEUHWTC, s->rpdeuhwtc); + I915_WRITE(GEN6_RPDEUC, s->rpdeuc); + I915_WRITE(ECOBUS, s->ecobus); + I915_WRITE(VLV_PWRDWNUPCTL, s->pwrdwnupctl); + I915_WRITE(GEN6_RP_DOWN_TIMEOUT,s->rp_down_timeout); + I915_WRITE(GEN6_RPDEUCSW, s->rp_deucsw); + I915_WRITE(GEN6_RCUBMABDTMR, s->rcubmabdtmr); + I915_WRITE(VLV_RCEDATA, s->rcedata); + I915_WRITE(VLV_SPAREG2H, s->spare2gh); + + /* Display CZ domain, 0x4400C-0x4402C, 0x4F000-0x4F11F */ + I915_WRITE(GTIMR, s->gt_imr); + I915_WRITE(GTIER, s->gt_ier); + I915_WRITE(GEN6_PMIMR, s->pm_imr); + I915_WRITE(GEN6_PMIER, s->pm_ier); + + for (i = 0; i < ARRAY_SIZE(s->gt_scratch); i++) + I915_WRITE(GEN7_GT_SCRATCH(i), s->gt_scratch[i]); + + /* GT SA CZ domain, 0x100000-0x138124 */ + I915_WRITE(TILECTL, s->tilectl); + I915_WRITE(GTFIFOCTL, s->gt_fifoctl); + /* + * Preserve the GT allow wake and GFX force clock bit, they are not + * be restored, as they are used to control the s0ix suspend/resume + * sequence by the caller. + */ + val = I915_READ(VLV_GTLC_WAKE_CTRL); + val &= VLV_GTLC_ALLOWWAKEREQ; + val |= s->gtlc_wake_ctrl & ~VLV_GTLC_ALLOWWAKEREQ; + I915_WRITE(VLV_GTLC_WAKE_CTRL, val); + + val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + val &= VLV_GFX_CLK_FORCE_ON_BIT; + val |= s->gtlc_survive & ~VLV_GFX_CLK_FORCE_ON_BIT; + I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); + + I915_WRITE(VLV_PMWGICZ, s->pmwgicz); + + /* Gunit-Display CZ domain, 0x182028-0x1821CF */ + I915_WRITE(VLV_GU_CTL0, s->gu_ctl0); + I915_WRITE(VLV_GU_CTL1, s->gu_ctl1); + I915_WRITE(VLV_PCBR, s->pcbr); + I915_WRITE(VLV_GUNIT_CLOCK_GATE2, s->clock_gate_dis2); +} + +static int vlv_wait_for_pw_status(struct drm_i915_private *i915, + u32 mask, u32 val) +{ + i915_reg_t reg = VLV_GTLC_PW_STATUS; + u32 reg_value; + int ret; + + /* The HW does not like us polling for PW_STATUS frequently, so + * use the sleeping loop rather than risk the busy spin within + * intel_wait_for_register(). + * + * Transitioning between RC6 states should be at most 2ms (see + * valleyview_enable_rps) so use a 3ms timeout. + */ + ret = wait_for(((reg_value = + intel_uncore_read_notrace(&i915->uncore, reg)) & mask) + == val, 3); + + /* just trace the final value */ + trace_i915_reg_rw(false, reg, reg_value, sizeof(reg_value), true); + + return ret; +} + +static int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on) +{ + u32 val; + int err; + + val = I915_READ(VLV_GTLC_SURVIVABILITY_REG); + val &= ~VLV_GFX_CLK_FORCE_ON_BIT; + if (force_on) + val |= VLV_GFX_CLK_FORCE_ON_BIT; + I915_WRITE(VLV_GTLC_SURVIVABILITY_REG, val); + + if (!force_on) + return 0; + + err = intel_wait_for_register(&dev_priv->uncore, + VLV_GTLC_SURVIVABILITY_REG, + VLV_GFX_CLK_STATUS_BIT, + VLV_GFX_CLK_STATUS_BIT, + 20); + if (err) + drm_err(&dev_priv->drm, + "timeout waiting for GFX clock force-on (%08x)\n", + I915_READ(VLV_GTLC_SURVIVABILITY_REG)); + + return err; +} + +static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow) +{ + u32 mask; + u32 val; + int err; + + val = I915_READ(VLV_GTLC_WAKE_CTRL); + val &= ~VLV_GTLC_ALLOWWAKEREQ; + if (allow) + val |= VLV_GTLC_ALLOWWAKEREQ; + I915_WRITE(VLV_GTLC_WAKE_CTRL, val); + POSTING_READ(VLV_GTLC_WAKE_CTRL); + + mask = VLV_GTLC_ALLOWWAKEACK; + val = allow ? mask : 0; + + err = vlv_wait_for_pw_status(dev_priv, mask, val); + if (err) + drm_err(&dev_priv->drm, "timeout disabling GT waking\n"); + + return err; +} + +static void vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv, + bool wait_for_on) +{ + u32 mask; + u32 val; + + mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK; + val = wait_for_on ? mask : 0; + + /* + * RC6 transitioning can be delayed up to 2 msec (see + * valleyview_enable_rps), use 3 msec for safety. + * + * This can fail to turn off the rc6 if the GPU is stuck after a failed + * reset and we are trying to force the machine to sleep. + */ + if (vlv_wait_for_pw_status(dev_priv, mask, val)) + drm_dbg(&dev_priv->drm, + "timeout waiting for GT wells to go %s\n", + onoff(wait_for_on)); +} + +static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv) +{ + if (!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEERR)) + return; + + drm_dbg(&dev_priv->drm, + "GT register access while GT waking disabled\n"); + I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR); +} + +int vlv_suspend_complete(struct drm_i915_private *dev_priv) +{ + u32 mask; + int err; + + if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) + return 0; + + /* + * Bspec defines the following GT well on flags as debug only, so + * don't treat them as hard failures. + */ + vlv_wait_for_gt_wells(dev_priv, false); + + mask = VLV_GTLC_RENDER_CTX_EXISTS | VLV_GTLC_MEDIA_CTX_EXISTS; + drm_WARN_ON(&dev_priv->drm, + (I915_READ(VLV_GTLC_WAKE_CTRL) & mask) != mask); + + vlv_check_no_gt_access(dev_priv); + + err = vlv_force_gfx_clock(dev_priv, true); + if (err) + goto err1; + + err = vlv_allow_gt_wake(dev_priv, false); + if (err) + goto err2; + + vlv_save_gunit_s0ix_state(dev_priv); + + err = vlv_force_gfx_clock(dev_priv, false); + if (err) + goto err2; + + return 0; + +err2: + /* For safety always re-enable waking and disable gfx clock forcing */ + vlv_allow_gt_wake(dev_priv, true); +err1: + vlv_force_gfx_clock(dev_priv, false); + + return err; +} + +int vlv_resume_prepare(struct drm_i915_private *dev_priv, bool rpm_resume) +{ + int err; + int ret; + + if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) + return 0; + + /* + * If any of the steps fail just try to continue, that's the best we + * can do at this point. Return the first error code (which will also + * leave RPM permanently disabled). + */ + ret = vlv_force_gfx_clock(dev_priv, true); + + vlv_restore_gunit_s0ix_state(dev_priv); + + err = vlv_allow_gt_wake(dev_priv, true); + if (!ret) + ret = err; + + err = vlv_force_gfx_clock(dev_priv, false); + if (!ret) + ret = err; + + vlv_check_no_gt_access(dev_priv); + + if (rpm_resume) + intel_init_clock_gating(dev_priv); + + return ret; +} + +int vlv_suspend_init(struct drm_i915_private *i915) +{ + if (!IS_VALLEYVIEW(i915)) + return 0; + + /* we write all the values in the struct, so no need to zero it out */ + i915->vlv_s0ix_state = kmalloc(sizeof(*i915->vlv_s0ix_state), + GFP_KERNEL); + if (!i915->vlv_s0ix_state) + return -ENOMEM; + + return 0; +} + +void vlv_suspend_cleanup(struct drm_i915_private *i915) +{ + if (!i915->vlv_s0ix_state) + return; + + kfree(i915->vlv_s0ix_state); + i915->vlv_s0ix_state = NULL; +} diff --git a/drivers/gpu/drm/i915/vlv_suspend.h b/drivers/gpu/drm/i915/vlv_suspend.h new file mode 100644 index 000000000000..895091cb1f62 --- /dev/null +++ b/drivers/gpu/drm/i915/vlv_suspend.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef __VLV_SUSPEND_H__ +#define __VLV_SUSPEND_H__ + +#include + +struct drm_i915_private; + +int vlv_suspend_init(struct drm_i915_private *i915); +void vlv_suspend_cleanup(struct drm_i915_private *i915); +int vlv_suspend_complete(struct drm_i915_private *i915); +int vlv_resume_prepare(struct drm_i915_private *i915, bool rpm_resume); + +#endif /* __VLV_SUSPEND_H__ */