From dad9abe93da8b2c41f374e326ff8bea12efefb0c Mon Sep 17 00:00:00 2001 From: CTCaer Date: Wed, 27 Aug 2025 15:45:58 +0300 Subject: [PATCH] minerva: update to v1.5 - "Perf" hack removal (match L4T mini Minerva) It's not a performance hack, it just kills low power modes. If wanted in L4T, use HP Mode in `ram_oc_opt`. - Simplify of burst regs config - Refactor of several bit defines and variables --- modules/hekate_libsys_minerva/mtc.h | 4 +- .../hekate_libsys_minerva/mtc_mc_emc_regs.h | 11 +- modules/hekate_libsys_minerva/sys_sdrammtc.c | 498 ++++++++---------- 3 files changed, 232 insertions(+), 281 deletions(-) diff --git a/modules/hekate_libsys_minerva/mtc.h b/modules/hekate_libsys_minerva/mtc.h index 860c7d7f..9d76fd4f 100644 --- a/modules/hekate_libsys_minerva/mtc.h +++ b/modules/hekate_libsys_minerva/mtc.h @@ -2,7 +2,7 @@ * Minerva Training Cell * DRAM Training for Tegra X1 SoC. Supports DDR2/3 and LPDDR3/4. * - * Copyright (c) 2018-2022 CTCaer + * Copyright (c) 2018-2025 CTCaer * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -63,7 +63,7 @@ #define NEEDS_TRAINING_SWAP_RANK BIT(8) #define NEEDS_TRAINING_IN_SELF_REFRESH BIT(9) -#define NEEDS_TRISTATE_TRAINING (NEEDS_TRAINING_CA | NEEDS_TRAINING_CA_VREF | \ +#define NEEDS_TRAINING (NEEDS_TRAINING_CA | NEEDS_TRAINING_CA_VREF | \ NEEDS_TRAINING_QUSE | NEEDS_TRAINING_WR | \ NEEDS_TRAINING_WR_VREF | NEEDS_TRAINING_RD | \ NEEDS_TRAINING_RD_VREF) diff --git a/modules/hekate_libsys_minerva/mtc_mc_emc_regs.h b/modules/hekate_libsys_minerva/mtc_mc_emc_regs.h index 569a9a17..83e85993 100644 --- a/modules/hekate_libsys_minerva/mtc_mc_emc_regs.h +++ b/modules/hekate_libsys_minerva/mtc_mc_emc_regs.h @@ -2,7 +2,7 @@ * Minerva Training Cell * DRAM Training for Tegra X1 SoC. Supports DDR2/3 and LPDDR3/4. * - * Copyright (c) 2018 CTCaer + * Copyright (c) 2018-2025 CTCaer * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -110,6 +110,13 @@ #define CLKCHANGE_COMPLETE_INT (1 << 4) #define EMC_DBG 0x8 +#define EMC_DBG_READ_MUX_ACTIVE BIT(0) +#define EMC_DBG_WRITE_MUX_ACTIVE BIT(1) +#define EMC_DBG_CFG_SWAP_ACTIVE_ONLY (0 << 26u) +#define EMC_DBG_CFG_SWAP_SWAP (1 << 26u) +#define EMC_DBG_CFG_SWAP_ASSEMBLY_ONLY (2 << 26u) +#define EMC_DBG_CFG_SWAP_MASK (3 << 26u) +#define EMC_DBG_WRITE_ACTIVE_ONLY BIT(30) #define EMC_CFG 0xC #define EMC_PIN 0x24 #define EMC_TIMING_CONTROL 0x28 @@ -628,4 +635,4 @@ #define EMC_PMACRO_OB_DDLL_LONG_DQ_BYTE6_SHIFT 0 #define EMC_PMACRO_OB_DDLL_LONG_DQ_BYTE7_SHIFT 16 -#endif \ No newline at end of file +#endif diff --git a/modules/hekate_libsys_minerva/sys_sdrammtc.c b/modules/hekate_libsys_minerva/sys_sdrammtc.c index f4475010..6dc7061f 100644 --- a/modules/hekate_libsys_minerva/sys_sdrammtc.c +++ b/modules/hekate_libsys_minerva/sys_sdrammtc.c @@ -2,7 +2,7 @@ * Minerva Training Cell * DRAM Training for Tegra X1 SoC. Supports LPDDR4. * - * Copyright (c) 2018-2022 CTCaer + * Copyright (c) 2018-2025 CTCaer * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, @@ -25,6 +25,9 @@ #include "types.h" #include +#define DVFS_CLOCK_CHANGE_VERSION "Minerva Training Cell v1.5_lpddr4" +#define EMC_PRELOCK_VERSION 2101 + #define EPRINTF(...) #define EPRINTFARGS(...) @@ -32,8 +35,6 @@ //#define OVERCLOCK_FREQ 1862400 //#define OVERCLOCK_VOLTAGE 1200000 // Default is 1100mV and in HOS 1125mV. -#define PERF_HACK - bool train_ram_patterns; /* @@ -1080,7 +1081,7 @@ static u32 _actual_osc_clocks(u32 in) static void _ccfifo_write(u32 addr, u32 data_val, u32 delay) //addr and delay are u16 { EMC(EMC_CCFIFO_DATA) = data_val; - EMC(EMC_CCFIFO_ADDR) = (addr & 0xffff) | ((delay & 0x7FFF) << 16) | (1 << 31); + EMC(EMC_CCFIFO_ADDR) = BIT(31) | (addr & 0xffff) | ((delay & 0x7FFF) << 16); } static bool _wait_emc_status(u32 reg_offset, u32 bit_mask, bool updated_state, u32 emc_channel) @@ -1208,11 +1209,11 @@ static u32 _pllm_clk_base_cfg(u32 rate_KHz, u32 clk_src_emc, bool new_src_is_PLL dividers = pllm_clk_config->pll_input_div | (pllm_clk_config->pll_feedback_div << 8) | ((pllm_clk_config->pll_post_div & 0x1F) << 20); if (new_src_is_PLLMB) { - CLOCK(CLK_RST_CONTROLLER_PLLMB_BASE) = dividers; + CLOCK(CLK_RST_CONTROLLER_PLLMB_BASE) = dividers; CLOCK(CLK_RST_CONTROLLER_PLLMB_BASE) |= PLLM_ENABLE; if ((clk_src_emc >> EMC_2X_CLK_SRC_SHIFT) == PLLM_UD) - clk_src_emc = (clk_src_emc & 0x1FFFFFFF) | (PLLMB_UD << EMC_2X_CLK_SRC_SHIFT); + clk_src_emc = (clk_src_emc & 0x1FFFFFFF) | (PLLMB_UD << EMC_2X_CLK_SRC_SHIFT); else if (!(clk_src_emc >> EMC_2X_CLK_SRC_SHIFT)) clk_src_emc |= (PLLMB_OUT0 << EMC_2X_CLK_SRC_SHIFT); @@ -1221,9 +1222,9 @@ static u32 _pllm_clk_base_cfg(u32 rate_KHz, u32 clk_src_emc, bool new_src_is_PLL } else { - CLOCK(CLK_RST_CONTROLLER_PLLM_BASE) = dividers; + CLOCK(CLK_RST_CONTROLLER_PLLM_BASE) = dividers; CLOCK(CLK_RST_CONTROLLER_PLLM_MISC2) |= PLLM_EN_LCKDET; - CLOCK(CLK_RST_CONTROLLER_PLLM_BASE) |= PLLM_ENABLE; + CLOCK(CLK_RST_CONTROLLER_PLLM_BASE) |= PLLM_ENABLE; if ((clk_src_emc >> EMC_2X_CLK_SRC_SHIFT) == PLLM_UD) clk_src_emc = (clk_src_emc & 0x1FFFFFFF) | (PLLM_UD << EMC_2X_CLK_SRC_SHIFT); @@ -1255,16 +1256,16 @@ static void _change_dll_src(emc_table_t *mtc_table_entry, u32 clk_src_emc) // Enable/Disable EMC DLL. if (mtc_table_entry->clk_out_enb_x_0_clk_enb_emc_dll) - CLOCK(CLK_RST_CONTROLLER_CLK_ENB_X_SET) = (1 << 14); + CLOCK(CLK_RST_CONTROLLER_CLK_ENB_X_SET) = BIT(14); else - CLOCK(CLK_RST_CONTROLLER_CLK_ENB_X_CLR) = (1 << 14); + CLOCK(CLK_RST_CONTROLLER_CLK_ENB_X_CLR) = BIT(14); // Commit clock write. (void)CLOCK(CLK_RST_CONTROLLER_CLK_OUT_ENB_X); _usleep(2); } -static u32 _digital_dll_prelock(emc_table_t *mtc_table_entry, u32 needs_tristate_training, u32 selected_clk_src_emc) +static u32 _digital_dll_prelock(emc_table_t *mtc_table_entry, bool in_training, u32 selected_clk_src_emc) { u32 dual_channel = (EMC(EMC_FBIO_CFG7) >> 1) & ((EMC(EMC_FBIO_CFG7) >> 2) & 1); @@ -1296,11 +1297,11 @@ static u32 _digital_dll_prelock(emc_table_t *mtc_table_entry, u32 needs_tristate while ((((EMC(EMC_DIG_DLL_STATUS) >> 17) & 1) ^ 1) | (((EMC(EMC_DIG_DLL_STATUS) >> 15) & 1) ^ 1)) ; - if (needs_tristate_training) + if (in_training) { - EMC(EMC_DBG) |= 2u; - EMC(EMC_CFG_DIG_DLL) &= 0xFFFFFFFE; //Disable CFG_DLL_EN: [PMC] Enable digital DLL. - EMC(EMC_DBG) &= 0xFFFFFFFD; + EMC(EMC_DBG) |= EMC_DBG_WRITE_MUX_ACTIVE; + EMC(EMC_CFG_DIG_DLL) &= 0xFFFFFFFE; // Disable CFG_DLL_EN. + EMC(EMC_DBG) &= ~EMC_DBG_WRITE_MUX_ACTIVE; while (EMC(EMC_CFG_DIG_DLL) & 1) ; @@ -1345,7 +1346,7 @@ static u32 _dvfs_power_ramp_down(bool flip_backward, emc_table_t *src_emc_table_ u32 pmacro_cmd_pad; u32 pmacro_rfu1; u32 pmacro_cfg5; - u32 pmacro_common_tx; + u32 pmacro_com_pad; u32 pmacro_dq_pad; u32 src_clk_per_pc = (100000 / src_clock_period) + 1; @@ -1353,18 +1354,18 @@ static u32 _dvfs_power_ramp_down(bool flip_backward, emc_table_t *src_emc_table_ if (flip_backward) { pmacro_cmd_pad = dst_emc_table_entry->burst_regs.emc_pmacro_cmd_pad_tx_ctrl; - pmacro_dq_pad = dst_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl; - pmacro_rfu1 = dst_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1; - pmacro_cfg5 = dst_emc_table_entry->burst_regs.emc_fbio_cfg5; - pmacro_common_tx = dst_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl; + pmacro_dq_pad = dst_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl; + pmacro_rfu1 = dst_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1; + pmacro_cfg5 = dst_emc_table_entry->burst_regs.emc_fbio_cfg5; + pmacro_com_pad = dst_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl; } else { pmacro_cmd_pad = src_emc_table_entry->burst_regs.emc_pmacro_cmd_pad_tx_ctrl; - pmacro_dq_pad = (dst_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0x101) | src_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl; - pmacro_rfu1 = src_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1; - pmacro_cfg5 = src_emc_table_entry->burst_regs.emc_fbio_cfg5; - pmacro_common_tx = src_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl; + pmacro_dq_pad = (dst_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0x101) | src_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl; + pmacro_rfu1 = src_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1; + pmacro_cfg5 = src_emc_table_entry->burst_regs.emc_fbio_cfg5; + pmacro_com_pad = src_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl; } u32 pmacro_cmd_pad_drvforceon = pmacro_cmd_pad | 0x4000000; @@ -1387,9 +1388,9 @@ static u32 _dvfs_power_ramp_down(bool flip_backward, emc_table_t *src_emc_table_ { pmacro_dq_pad = (pmacro_dq_pad & 0xFEFEFDFD) | 0x10200; pmacro_cmd_pad_drvforceon = (pmacro_cmd_pad & 0xFAFEFDFD) | 0x4010200; - _ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_drvforceon, (u32)src_clk_per_pc); + _ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_drvforceon, (u32)src_clk_per_pc); _ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad, 0); - _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFEEDFEED, 0); + _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFEEDFEED, 0); } ramp_down_wait += 300000; _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFE40FE40, (u32)src_clk_per_pc); @@ -1398,19 +1399,19 @@ static u32 _dvfs_power_ramp_down(bool flip_backward, emc_table_t *src_emc_table_ _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xF800F800, (u32)src_clk_per_pc); else { - _ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_drvforceon & 0xFEFEFDFD, (u32)src_clk_per_pc); + _ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_drvforceon & 0xFEFEFDFD, (u32)src_clk_per_pc); _ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad & 0xFEFEFDFD, 0); - _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xF800F800, 0); + _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xF800F800, 0); } } if (src_clock_period >= 1666) // Dvfs mid speed threshold. - _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & 0xFFFFFFF0, (u32)src_clk_per_pc); + _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad & 0xFFFFFFF0, (u32)src_clk_per_pc); else { ramp_down_wait += 400000; - _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & 0xFFFFFFFA, (u32)src_clk_per_pc); - _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & 0xFFFFFFF0, (u32)src_clk_per_pc); + _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad & 0xFFFFFFFA, (u32)src_clk_per_pc); + _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad & 0xFFFFFFF0, (u32)src_clk_per_pc); _ccfifo_write(EMC_INTSTATUS, 0, (u32)src_clk_per_pc); } @@ -1423,7 +1424,7 @@ static u32 _dvfs_power_ramp_up(bool flip_backward, emc_table_t *src_emc_table_en u32 pmacro_dq_pad; u32 pmacro_rfu1; u32 pmacro_cfg5; - u32 pmacro_common_tx; + u32 pmacro_com_pad; u32 pmacro_cmd_pad_data; u32 ramp_up_wait = 0; @@ -1432,49 +1433,49 @@ static u32 _dvfs_power_ramp_up(bool flip_backward, emc_table_t *src_emc_table_en if (flip_backward) { pmacro_cmd_pad = src_emc_table_entry->burst_regs.emc_pmacro_cmd_pad_tx_ctrl; - pmacro_dq_pad = src_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl; - pmacro_rfu1 = src_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1; - pmacro_cfg5 = src_emc_table_entry->burst_regs.emc_fbio_cfg5; - pmacro_common_tx = src_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl; + pmacro_dq_pad = src_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl; + pmacro_rfu1 = src_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1; + pmacro_cfg5 = src_emc_table_entry->burst_regs.emc_fbio_cfg5; + pmacro_com_pad = src_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl; } else if (needs_training & NEEDS_TRAINING_CA_COMBO) { pmacro_cmd_pad = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_cmd_pad_tx_ctrl; - pmacro_dq_pad = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_data_pad_tx_ctrl; - pmacro_rfu1 = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_brick_ctrl_rfu1; - pmacro_cfg5 = dst_emc_table_entry->shadow_regs_ca_train.emc_fbio_cfg5; - pmacro_common_tx = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_common_pad_tx_ctrl; + pmacro_dq_pad = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_data_pad_tx_ctrl; + pmacro_rfu1 = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_brick_ctrl_rfu1; + pmacro_cfg5 = dst_emc_table_entry->shadow_regs_ca_train.emc_fbio_cfg5; + pmacro_com_pad = dst_emc_table_entry->shadow_regs_ca_train.emc_pmacro_common_pad_tx_ctrl; } else if (needs_training & NEEDS_TRAINING_QUSE_COMBO) { pmacro_cmd_pad = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_cmd_pad_tx_ctrl; - pmacro_dq_pad = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_data_pad_tx_ctrl; - pmacro_rfu1 = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_brick_ctrl_rfu1; - pmacro_cfg5 = dst_emc_table_entry->shadow_regs_quse_train.emc_fbio_cfg5; - pmacro_common_tx = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_common_pad_tx_ctrl; + pmacro_dq_pad = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_data_pad_tx_ctrl; + pmacro_rfu1 = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_brick_ctrl_rfu1; + pmacro_cfg5 = dst_emc_table_entry->shadow_regs_quse_train.emc_fbio_cfg5; + pmacro_com_pad = dst_emc_table_entry->shadow_regs_quse_train.emc_pmacro_common_pad_tx_ctrl; } else if (needs_training & (NEEDS_TRAINING_WR_COMBO | NEEDS_TRAINING_RD_COMBO)) { pmacro_cmd_pad = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_cmd_pad_tx_ctrl; - pmacro_dq_pad = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_data_pad_tx_ctrl; - pmacro_rfu1 = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_brick_ctrl_rfu1; - pmacro_cfg5 = dst_emc_table_entry->shadow_regs_rdwr_train.emc_fbio_cfg5; - pmacro_common_tx = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_common_pad_tx_ctrl; + pmacro_dq_pad = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_data_pad_tx_ctrl; + pmacro_rfu1 = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_brick_ctrl_rfu1; + pmacro_cfg5 = dst_emc_table_entry->shadow_regs_rdwr_train.emc_fbio_cfg5; + pmacro_com_pad = dst_emc_table_entry->shadow_regs_rdwr_train.emc_pmacro_common_pad_tx_ctrl; } else { pmacro_cmd_pad = dst_emc_table_entry->burst_regs.emc_pmacro_cmd_pad_tx_ctrl; - pmacro_dq_pad = dst_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl; - pmacro_rfu1 = dst_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1; - pmacro_cfg5 = dst_emc_table_entry->burst_regs.emc_fbio_cfg5; - pmacro_common_tx = dst_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl; + pmacro_dq_pad = dst_emc_table_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl; + pmacro_rfu1 = dst_emc_table_entry->burst_regs.emc_pmacro_brick_ctrl_rfu1; + pmacro_cfg5 = dst_emc_table_entry->burst_regs.emc_fbio_cfg5; + pmacro_com_pad = dst_emc_table_entry->burst_regs.emc_pmacro_common_pad_tx_ctrl; } pmacro_cmd_pad_data = (pmacro_cmd_pad & 0xFAFEFDFD) | 0x4000000; if (dst_clock_period >= 1666) // Dvfs mid speed threshold. { - _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx | 8, 0); + _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad | 8, 0); _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 | 0x600, 0); _ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 & 0xFFFFFEFF, 12); @@ -1483,8 +1484,8 @@ static u32 _dvfs_power_ramp_up(bool flip_backward, emc_table_t *src_emc_table_en } else { - _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & 0xA, 0); - _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & 0xF, dst_clk_per_pc); + _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad & 0xA, 0); + _ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_com_pad & 0xF, dst_clk_per_pc); if (dst_clock_period < 1000) // Dvfs high speed threshold. { @@ -1492,9 +1493,9 @@ static u32 _dvfs_power_ramp_up(bool flip_backward, emc_table_t *src_emc_table_en _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFE40FE40, dst_clk_per_pc); else { - _ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, (pmacro_cmd_pad & 0xFAFEFDFD) | 0x4010200, dst_clk_per_pc); + _ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, (pmacro_cmd_pad & 0xFAFEFDFD) | 0x4010200, dst_clk_per_pc); _ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, (pmacro_dq_pad & 0xFEFEFDFD) | 0x10200, 0); - _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFE40FE40, 0); + _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFE40FE40, 0); } _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xFEEDFEED, dst_clk_per_pc); @@ -1504,9 +1505,9 @@ static u32 _dvfs_power_ramp_up(bool flip_backward, emc_table_t *src_emc_table_en else { pmacro_cmd_pad_data = pmacro_cmd_pad | 0x5010202; - _ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_data, dst_clk_per_pc); + _ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad_data, dst_clk_per_pc); _ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad | 0x1010202, 0); - _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1, 0); + _ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1, 0); } ramp_up_wait = 500000 + (dst_clock_period * 10); @@ -1537,7 +1538,7 @@ static u32 _minerva_update_clock_tree_delay(emc_table_t *src_emc_entry, emc_tabl u32 temp_ch1_0 = 0; u32 temp_ch1_1 = 0; - u32 upd_type_bits = 1 << update_type; + u32 upd_type_bits = BIT(update_type); u32 dst_rate_mhz = dst_emc_entry->rate_khz / 1000; u32 src_rate_mhz = src_emc_entry->rate_khz / 1000; @@ -1728,7 +1729,7 @@ calc_dev2: { _request_mmr_data(0x40130000, channel1_enabled); // Dev1 MRR 19. u32 mrr = EMC(EMC_MRR); - temp_ch0_0 = (mrr& 0xFF) << 8; + temp_ch0_0 = (mrr & 0xFF) << 8; temp_ch0_1 = mrr & 0xFF00; if (channel1_enabled) { @@ -1740,7 +1741,7 @@ calc_dev2: _request_mmr_data(0x40120000, channel1_enabled); // Dev1 MRR 18 mrr = EMC(EMC_MRR); temp_ch0_0 |= mrr & 0xFF; - temp_ch0_1 |= ((mrr & 0xFF00) >> 8); + temp_ch0_1 |= (mrr & 0xFF00) >> 8; if (channel1_enabled) { mrr = EMC_CH1(EMC_MRR); @@ -2204,14 +2205,14 @@ static bool _check_freq_changed(u32 dst_entry_rate_KHz, u32 dst_entry_clk_src_em static void _save_train_results(emc_table_t *mtc_table_entry, u32 needs_training, u32 dram_dev_num, bool channel1_enabled) { - bool needs_ca_training = !!(needs_training & NEEDS_TRAINING_CA); - bool needs_ca_vref_training = !!(needs_training & NEEDS_TRAINING_CA_VREF); - bool needs_quse_training = !!(needs_training & NEEDS_TRAINING_QUSE); + bool needs_ca_training = !!(needs_training & NEEDS_TRAINING_CA); + bool needs_ca_vref_training = !!(needs_training & NEEDS_TRAINING_CA_VREF); + bool needs_quse_training = !!(needs_training & NEEDS_TRAINING_QUSE); bool needs_quse_vref_training = !!(needs_training & NEEDS_TRAINING_QUSE_VREF); - bool needs_wr_training = !!(needs_training & NEEDS_TRAINING_WR); - bool needs_wr_vref_training = !!(needs_training & NEEDS_TRAINING_WR_VREF); - bool needs_rd_training = !!(needs_training & NEEDS_TRAINING_RD); - bool needs_rd_vref_training = !!(needs_training & NEEDS_TRAINING_RD_VREF); + bool needs_wr_training = !!(needs_training & NEEDS_TRAINING_WR); + bool needs_wr_vref_training = !!(needs_training & NEEDS_TRAINING_WR_VREF); + bool needs_rd_training = !!(needs_training & NEEDS_TRAINING_RD); + bool needs_rd_vref_training = !!(needs_training & NEEDS_TRAINING_RD_VREF); bool needs_training_in_self_refresh = !!(needs_training & NEEDS_TRAINING_IN_SELF_REFRESH); if (needs_ca_training) @@ -2596,22 +2597,22 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e */ bool compensate_trimmer_applicable = false; - bool needs_ca_combo_training = !!(needs_training & NEEDS_TRAINING_CA_COMBO); - bool needs_tristate_training = !!(needs_training & NEEDS_TRISTATE_TRAINING); + bool in_training = !!(needs_training & NEEDS_TRAINING); - bool needs_ca_training = !!(needs_training & NEEDS_TRAINING_CA); - bool needs_ca_vref_training = !!(needs_training & NEEDS_TRAINING_CA_VREF); - bool needs_quse_training = !!(needs_training & NEEDS_TRAINING_QUSE); + bool needs_ca_combo_training = !!(needs_training & NEEDS_TRAINING_CA_COMBO); + bool needs_ca_training = !!(needs_training & NEEDS_TRAINING_CA); + bool needs_ca_vref_training = !!(needs_training & NEEDS_TRAINING_CA_VREF); + bool needs_quse_training = !!(needs_training & NEEDS_TRAINING_QUSE); bool needs_quse_vref_training = !!(needs_training & NEEDS_TRAINING_QUSE_VREF); - bool needs_wr_training = !!(needs_training & NEEDS_TRAINING_WR); - bool needs_wr_vref_training = !!(needs_training & NEEDS_TRAINING_WR_VREF); - bool needs_rd_training = !!(needs_training & NEEDS_TRAINING_RD); - bool needs_rd_vref_training = !!(needs_training & NEEDS_TRAINING_RD_VREF); + bool needs_wr_training = !!(needs_training & NEEDS_TRAINING_WR); + bool needs_wr_vref_training = !!(needs_training & NEEDS_TRAINING_WR_VREF); + bool needs_rd_training = !!(needs_training & NEEDS_TRAINING_RD); + bool needs_rd_vref_training = !!(needs_training & NEEDS_TRAINING_RD_VREF); bool needs_swap_rank_training = !!(needs_training & NEEDS_TRAINING_SWAP_RANK); bool zcal_resistor_shared = (src_emc_entry->burst_regs.emc_zcal_wait_cnt >> 31) & 1; - bool enable_bg_regulator = (dst_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 & 1) ^ 1; - bool channel1_enabled = (src_emc_entry->burst_regs.emc_fbio_cfg7 >> 2) & 1; + bool enable_bg_regulator = (dst_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 & 1) ^ 1; + bool channel1_enabled = (src_emc_entry->burst_regs.emc_fbio_cfg7 >> 2) & 1; u32 dram_type = EMC(EMC_FBIO_CFG5) & 3; u32 dram_dev_num = (MC(MC_EMEM_ADR_CFG) & 1) + 1; @@ -2652,12 +2653,12 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 1.3 - Disable other power features. EPRINTF("Step 1.3"); - EMC(EMC_DBG) = emc_dbg_o | 2; - EMC(EMC_CFG) = emc_cfg; + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE; + EMC(EMC_CFG) = emc_cfg; EMC(EMC_SEL_DPD_CTRL) = emc_sel_dpd_ctrl; EMC(EMC_DBG) = emc_dbg_o; - if (!needs_tristate_training && dst_emc_entry->periodic_training) + if (!in_training && dst_emc_entry->periodic_training) { if (dram_dev_num == TWO_RANK) { @@ -2693,7 +2694,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e } EMC(EMC_INTSTATUS) = CLKCHANGE_COMPLETE_INT; - EMC(EMC_DBG) = emc_dbg_o | 2; + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE; EMC(EMC_CFG) = emc_cfg; EMC(EMC_SEL_DPD_CTRL) = emc_sel_dpd_ctrl; EMC(EMC_CFG_PIPE_CLK) = emc_cfg_pipe_clk_o | 1; // CLK_ALWAYS_ON. @@ -2704,7 +2705,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e if (bg_regulator_mode_change) { - EMC(EMC_DBG) = emc_dbg_o | 2; + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE; if (enable_bg_regulator) EMC(EMC_PMACRO_BG_BIAS_CTRL_0) = src_emc_entry->burst_regs.emc_pmacro_bg_bias_ctrl_0 & 0xFFFFFFFE; else @@ -2712,10 +2713,8 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e } // Check if we need to turn on VREF generator. - if ((!(src_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0x100) - && (dst_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0x100)) - || (!(src_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 1) - && (dst_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 1))) + if ((!(src_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0x100) && (dst_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0x100)) || + (!(src_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 1) && (dst_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 1))) { EMC(EMC_PMACRO_DATA_PAD_TX_CTRL) = (((dst_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 1) | (src_emc_entry->burst_regs.emc_pmacro_data_pad_tx_ctrl & 0xFFFFFFFE)) & 0xFFFFFEFF) @@ -2729,7 +2728,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 2 - Prelock the DLL. EPRINTF("Step 2"); if (dst_emc_entry->burst_regs.emc_cfg_dig_dll & 1) - _digital_dll_prelock(dst_emc_entry, needs_tristate_training, selected_clk_src_emc); // Prelock enabled for target frequency. + _digital_dll_prelock(dst_emc_entry, in_training, selected_clk_src_emc); // Prelock enabled for target frequency. else { _change_dll_src(dst_emc_entry, selected_clk_src_emc); @@ -2739,7 +2738,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 3 - Prepare autocal for the clock change. EPRINTF("Step 3"); EMC(EMC_AUTO_CAL_CONFIG) = (dst_emc_entry->emc_auto_cal_config & 0x7FFFF9FF) | 0x600; - EMC(EMC_DBG) = emc_dbg_o | 2; + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE; EMC(EMC_AUTO_CAL_CONFIG2) = dst_emc_entry->emc_auto_cal_config2; EMC(EMC_AUTO_CAL_CONFIG3) = dst_emc_entry->emc_auto_cal_config3; EMC(EMC_AUTO_CAL_CONFIG4) = dst_emc_entry->emc_auto_cal_config4; @@ -2775,16 +2774,16 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e u32 W2P_war = 0; u32 nRTP = 8; // <= 1066MHz. - if ( src_clock_period < 1000000 / 266 - && src_clock_period < 1000000 / 533 - && src_clock_period < 1000000 / 800 - && src_clock_period < 1000000 / 1066 ) + if ( src_clock_period < 1000000 / 266 // 266MHz - 3759.39 ps. + && src_clock_period < 1000000 / 533 // 533MHz - 1876.17 ps. + && src_clock_period < 1000000 / 800 // 800MHz - 1250.00 ps. + && src_clock_period < 1000000 / 1066) // 1066MHz - 938.09 ps. nRTP = 10; // 1067MHz < x <= 1333MHz. - if (src_clock_period < 1000000 / 1333) + if (src_clock_period < 1000000 / 1333) // 1333MHz - 750.19 ps. nRTP = 12; // 1333MHz < x <= 1600MHz. - if (src_clock_period < 1000000 / 1600) + if (src_clock_period < 1000000 / 1600) // 1600MHz - 625.00 ps. nRTP = 14; // 1600MHz < x <= 1866MHz. - if (src_clock_period < 1000000 / 1866) + if (src_clock_period < 1000000 / 1866) // 1866MHz - 535.91 ps. nRTP = 16; // > 1866MHz u32 tRPST = (src_emc_entry->emc_mrw >> 7) & 1; @@ -2832,15 +2831,15 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e } } - if ( src_emc_entry->burst_regs.emc_w2p != W2P_war - || src_emc_entry->burst_regs.emc_rp != RP_war - || src_emc_entry->burst_regs.emc_r2p != R2P_war - || src_emc_entry->burst_regs.emc_trpab != TRPab_war) + if (src_emc_entry->burst_regs.emc_w2p != W2P_war || + src_emc_entry->burst_regs.emc_rp != RP_war || + src_emc_entry->burst_regs.emc_r2p != R2P_war || + src_emc_entry->burst_regs.emc_trpab != TRPab_war) { - EMC(EMC_DBG) = emc_dbg_o | 2; - EMC(EMC_RP) = RP_war; - EMC(EMC_R2P) = R2P_war; - EMC(EMC_W2P) = W2P_war; + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE; + EMC(EMC_RP) = RP_war; + EMC(EMC_R2P) = R2P_war; + EMC(EMC_W2P) = W2P_war; EMC(EMC_TRPAB) = TRPab_war; EMC(EMC_DBG) = emc_dbg_o; (void)EMC(EMC_TRPAB); @@ -2879,90 +2878,66 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e mr13_catr_enable = mr13_flip_fspwr | 1; EMC(EMC_MRW3) = mr13_flip_fspwr; - EMC(EMC_MRW) = dst_emc_entry->emc_mrw; + EMC(EMC_MRW) = dst_emc_entry->emc_mrw; EMC(EMC_MRW2) = dst_emc_entry->emc_mrw2; // Step 8 - Program the shadow registers. EPRINTF("Step 8"); // Writing burst_regs. u32 reg_addr = 0; - u32 reg_val = 0; - u32 reg_check = false; + u32 reg_val = 0; burst_regs_table_t *dst_burst_regs = (burst_regs_table_t *)&dst_emc_entry->burst_regs; + u32 *burst_val_table = dst_burst_regs->burst_regs; - for (u32 i = 0; dst_emc_entry->num_burst > i; i++) + if (in_training) + { + if (needs_ca_combo_training) + burst_val_table = dst_burst_regs->shadow_regs_ca_train; + else if (needs_training & NEEDS_TRAINING_QUSE_COMBO) + burst_val_table = dst_burst_regs->shadow_regs_quse_train; + else if (needs_training & (NEEDS_TRAINING_WR_COMBO | NEEDS_TRAINING_RD_COMBO)) + burst_val_table = dst_burst_regs->shadow_regs_rdwr_train; + else + burst_val_table = NULL; + } + + for (u32 i = 0; burst_val_table && i < dst_emc_entry->num_burst; i++) { - reg_check = false; reg_addr = burst_regs_emc_addr_table[i]; - if (needs_tristate_training) - { - if (needs_ca_combo_training) - reg_val = dst_burst_regs->shadow_regs_ca_train[i]; - else if (needs_training & NEEDS_TRAINING_QUSE_COMBO) - reg_val = dst_burst_regs->shadow_regs_quse_train[i]; - else if (needs_training & (NEEDS_TRAINING_WR_COMBO | NEEDS_TRAINING_RD_COMBO)) - reg_val = dst_burst_regs->shadow_regs_rdwr_train[i]; - else - continue; - } - else - reg_val = dst_burst_regs->burst_regs[i]; + reg_val = burst_val_table[i]; - if ((reg_addr & 0xFFF7) != EMC_MRW6 - && (reg_addr - EMC_MRW7) & 0xFFFF7 - //&& (reg_addr & 0xEFF7) != 0x34B4 // EMC_MRW10. - && ((reg_addr & 0xEFFF) - 0x34B8) & 0xFFF7 // EMC_MRW11. - && reg_addr != EMC_TRAINING_CTRL - && reg_addr != EMC_MRW14 - && reg_addr != EMC_MRW15) - { - reg_check = true; - } - - if (reg_check && reg_addr == EMC_CFG) + switch (reg_addr) { + case EMC_CFG: reg_val &= 0xFFFFFFF; - - EMC(reg_addr) = reg_val; - continue; + break; + case EMC_ZCAL_INTERVAL: + reg_val = 0; + break; + case EMC_PMACRO_BRICK_CTRL_RFU1: + reg_val &= 0xF800F800; + break; + case EMC_PMACRO_CMD_PAD_TX_CTRL: + reg_val = (reg_val & 0xFAFEFDFD) | BIT(26); // CMD_CMD_TX_DRVFORCEON. + break; + case EMC_PMACRO_DATA_PAD_TX_CTRL: + reg_val &= 0xFEFEFDFD; + break; + case EMC_PMACRO_COMMON_PAD_TX_CTRL: + reg_val &= 0xFFFFFFF0; + break; + case EMC_PMACRO_AUTOCAL_CFG_COMMON: + reg_val |= BIT(16); // E_CAL_BYPASS_DVFS. + break; + case EMC_TRAINING_CTRL: + reg_val |= needs_swap_rank_training << 14; + break; } - if (reg_addr != EMC_CFG)// EMC_CFG - { - if (reg_addr != EMC_ZCAL_INTERVAL) - { - switch ( reg_addr ) - { - case EMC_PMACRO_AUTOCAL_CFG_COMMON: - reg_val |= 0x10000; - break; - case EMC_PMACRO_DATA_PAD_TX_CTRL: - reg_val &= 0xFEFEFDFD; - break; - case EMC_PMACRO_CMD_PAD_TX_CTRL: - reg_val = (reg_val & 0xFAFEFDFD) | 0x4000000; - break; - case EMC_PMACRO_BRICK_CTRL_RFU1: - reg_val &= 0xF800F800; - break; - case EMC_PMACRO_COMMON_PAD_TX_CTRL: - reg_val &= 0xFFFFFFF0; - break; - case EMC_TRAINING_CTRL: - reg_val |= needs_swap_rank_training << 14;// bit15 is TR_IN_SELF_REFRESH - break; - } - } - else - reg_val = 0; - } - else - reg_val &= 0xFFFFFFF; - EMC(reg_addr) = reg_val; } - if (needs_tristate_training) + if (in_training) EMC(EMC_MRW) = (src_emc_entry->run_clocks & 0xFF) | 0x170000; else EMC(EMC_MRW) = (dst_emc_entry->run_clocks & 0xFF) | 0x170000; @@ -2987,7 +2962,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e } // Writing training mod regs. - if (needs_tristate_training) + if (in_training) { for (u32 i = 0; dst_emc_entry->training_mod_num > i; i++) { @@ -3003,9 +2978,9 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e reg_addr = trim_regs_emc_addr_table[i]; if (reg_addr) { - if (((reg_addr & 0xFFFFFFF3) == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 - || (reg_addr & 0xFFFFFFF3) == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 - || (reg_addr & 0xFFFFFFFB) == EMC_DATA_BRLSHFT_0) + if (((reg_addr & 0xFFFFFFF0) == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || + (reg_addr & 0xFFFFFFF0) == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || + (reg_addr & 0xFFFFFFF8) == EMC_DATA_BRLSHFT_0) && compensate_trimmer_applicable) { EMC(reg_addr) = _minerva_apply_periodic_compensation_trimmer(dst_emc_entry, reg_addr); @@ -3016,45 +2991,23 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e } // Writing trim_regs_per_ch - reg_val = 0; for (u32 i = 0; dst_emc_entry->num_trim_per_ch > i; i++) { reg_addr = trim_perch_regs_emc01_addr_table[i]; - if (reg_addr && (channel1_enabled || reg_addr - 0x4000 > 0xFFF)) + if (reg_addr && (channel1_enabled || (reg_addr - 0x4000) > 0xFFF)) { - if (((reg_addr & 0xFFFFFFF3) == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 - || (reg_addr & 0xFFFFFFF3) == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 - || (reg_addr & 0xFFFFFFFB) == EMC_DATA_BRLSHFT_0) - && compensate_trimmer_applicable ) - { - reg_val = _minerva_apply_periodic_compensation_trimmer(dst_emc_entry, reg_addr & 0xFFF); - } - else if (((reg_addr & 0xFFFFFFFB) == 0x3660 - || (reg_addr & 0xFFFFFFDF) == 0x3648 - || (reg_addr & 0xFFFFFFF7) == 0x3644 - || reg_addr == 0x366C - || reg_addr == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 - || (reg_addr & 0xFFFFFFFB) == 0x3588) - && compensate_trimmer_applicable ) - { - reg_val = _minerva_apply_periodic_compensation_trimmer(dst_emc_entry, reg_addr & 0xFFF); - } - else if (((reg_addr & 0xFFFFFFF3) == 0x4660 - || (reg_addr & 0xFFFFFFF3) == 0x4640 - || (reg_addr & 0xFFFFFFFB) == 0x4588) + if (((reg_addr & 0xFFFFFFF8) == EMC0_DATA_BRLSHFT_0 || + (reg_addr & 0xFFFFFFF8) == EMC1_DATA_BRLSHFT_0) && compensate_trimmer_applicable) { - reg_val = _minerva_apply_periodic_compensation_trimmer(dst_emc_entry, reg_addr & 0xFFF); + EMC(reg_addr) = _minerva_apply_periodic_compensation_trimmer(dst_emc_entry, reg_addr & 0xFFF); } else - { - reg_val = trim_regs_table->trim_perch_regs[i]; - } - EMC(reg_addr) = reg_val; + EMC(reg_addr) = trim_regs_table->trim_perch_regs[i]; } } - if (needs_tristate_training) + if (in_training) { // Check delta wrt previous values (save value if margin exceeds what is set in table). if (needs_wr_training && dst_emc_entry->periodic_training) @@ -3078,13 +3031,13 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e EPRINTF("Step 9"); EMC(EMC_ZCAL_INTERVAL) = src_emc_entry->burst_regs.emc_zcal_interval & 0xFF000000; EMC(EMC_ZCAL_WAIT_CNT) = dst_emc_entry->burst_regs.emc_zcal_wait_cnt & 0xFFFFF800; - EMC(EMC_DBG) = emc_dbg_o | 0x40000002; + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_ACTIVE_ONLY | EMC_DBG_WRITE_MUX_ACTIVE; EMC(EMC_ZCAL_INTERVAL) = src_emc_entry->burst_regs.emc_zcal_interval & 0xFF000000; EMC(EMC_DBG) = emc_dbg_o; - if (needs_tristate_training) + if (in_training) { - EMC(EMC_DBG) = emc_dbg_o | 2; + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE; EMC(EMC_PMACRO_AUTOCAL_CFG_COMMON) = dst_emc_entry->burst_regs.emc_pmacro_autocal_cfg_common | 0x10000; if (needs_ca_combo_training) @@ -3095,7 +3048,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e if (channel1_enabled) _ccfifo_write(EMC_CFG_SYNC, 0, 0); - _ccfifo_write(EMC_DBG, (emc_dbg_o & 0xF3FFFFFF) | 0x4000000, 0); + _ccfifo_write(EMC_DBG, (emc_dbg_o & ~EMC_DBG_CFG_SWAP_MASK) | EMC_DBG_CFG_SWAP_SWAP, 0); } // Step 10 - Self refresh @@ -3105,11 +3058,11 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e if (!needs_ca_combo_training && (dst_clock_period <= 2000)) { _ccfifo_write(EMC_MRW3, mr13_flip_fspwr ^ 0x40, 0); - _ccfifo_write(EMC_MRW6, (src_emc_entry->burst_regs.emc_mrw6 & 0xC0C0) | (dst_emc_entry->burst_regs.emc_mrw6 & 0xFFFF3F3F), 0); + _ccfifo_write(EMC_MRW6, (src_emc_entry->burst_regs.emc_mrw6 & 0xC0C0) | (dst_emc_entry->burst_regs.emc_mrw6 & 0xFFFF3F3F), 0); _ccfifo_write(EMC_MRW14, (src_emc_entry->burst_regs.emc_mrw14 & 0x3838) | (dst_emc_entry->burst_regs.emc_mrw14 & 0xFFFF0707), 0); if (dram_dev_num == TWO_RANK) { - _ccfifo_write(EMC_MRW7, (src_emc_entry->burst_regs.emc_mrw7 & 0xC0C0) | (dst_emc_entry->burst_regs.emc_mrw7 & 0xFFFF3F3F), 0); + _ccfifo_write(EMC_MRW7, (src_emc_entry->burst_regs.emc_mrw7 & 0xC0C0) | (dst_emc_entry->burst_regs.emc_mrw7 & 0xFFFF3F3F), 0); _ccfifo_write(EMC_MRW15, (src_emc_entry->burst_regs.emc_mrw15 & 0x3838) | (dst_emc_entry->burst_regs.emc_mrw15 & 0xFFFF0707), 0); } @@ -3123,12 +3076,11 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e emc_dbg_val = emc_dbg_o; u32 tRP_src_timing = (src_emc_entry->dram_timings.t_rp * 1000) / src_clock_period; - bool in_self_refresh = false; u32 ref_delay = 0; - if (needs_tristate_training) + if (in_training) { - emc_dbg_val = (emc_dbg_o & 0xF3FFFFFF) | 0x44000000; + emc_dbg_val = (emc_dbg_o & ~EMC_DBG_CFG_SWAP_MASK) | EMC_DBG_WRITE_ACTIVE_ONLY | EMC_DBG_CFG_SWAP_SWAP; _ccfifo_write(EMC_DBG, emc_dbg_val, 0); } @@ -3159,15 +3111,15 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 11 - Ramp down. EPRINTF("Step 11"); _ccfifo_write(EMC_CFG_SYNC, 0, 0); - _ccfifo_write(EMC_DBG, emc_dbg_val | 0x40000002, 0); // WRITE_MUX_ACTIVE | WRITE_ACTIVE_ONLY + _ccfifo_write(EMC_DBG, emc_dbg_val | EMC_DBG_WRITE_ACTIVE_ONLY | EMC_DBG_WRITE_MUX_ACTIVE, 0); // WRITE_MUX_ACTIVE | WRITE_ACTIVE_ONLY ramp_down_wait = _dvfs_power_ramp_down(false, src_emc_entry, dst_emc_entry, src_clock_period); // Step 12 - Trigger clock change. EPRINTF("Step 12"); _ccfifo_write(EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 1, 0); - if (!needs_tristate_training) - _ccfifo_write(EMC_DBG, (emc_dbg_val & 0xBFFFFFFF) | 2, 0); + if (!in_training) + _ccfifo_write(EMC_DBG, (emc_dbg_val & ~EMC_DBG_WRITE_ACTIVE_ONLY) | EMC_DBG_WRITE_MUX_ACTIVE, 0); // Step 13 - Ramp up. EPRINTF("Step 13"); @@ -3214,7 +3166,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e if (dst_clock_period > 2000) _ccfifo_write(EMC_ZQ_CAL, 0x80000001, T_PDEX_timing); - if (!needs_tristate_training) + if (!in_training) _ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, T_PDEX_timing); emc_zq_cal = 0x80000002; @@ -3232,7 +3184,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e _ccfifo_write(EMC_ZQ_CAL, 0x80000002, T_PDEX_timing_final); _ccfifo_write(EMC_ZQ_CAL, 0x40000001, 0); - if (!needs_tristate_training) + if (!in_training) _ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, 0); emc_zq_cal = 0x40000002; @@ -3243,22 +3195,17 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e if (dst_clock_period > 2000) _ccfifo_write(EMC_ZQ_CAL, 1, T_PDEX_timing); - if (!needs_tristate_training) + if (!in_training) _ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, T_PDEX_timing); emc_zq_cal = 2; } // Disable self-refresh. - if (!needs_tristate_training) + if (!in_training) { -#ifdef PERF_HACK - // HACK: Setting ACTIVE_SELF_REF increases perf by 1-2%. - _ccfifo_write(EMC_SELF_REF, 0x100, 0); -#else _ccfifo_write(EMC_SELF_REF, 0, 0); -#endif - _ccfifo_write(EMC_REF, 0, 0); + _ccfifo_write(EMC_REF, 0, 0); } if (zq_latch_dvfs_wait_time < 0) @@ -3271,29 +3218,29 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 16 - LPDDR4 Conditional training kickoff. EPRINTF("Step 16"); - if (needs_tristate_training) + if (in_training) { _ccfifo_write(EMC_INTSTATUS, 0, 1020000 / dst_clock_period); u32 training_command = 0; if (needs_ca_training) - training_command |= (1 << 1); // CA: Initiates CA Training. + training_command |= BIT(1); // CA: Initiates CA Training. if (needs_ca_vref_training) - training_command |= (1 << 5); // CA_VREF: Initiates CA_VREF Training. + training_command |= BIT(5); // CA_VREF: Initiates CA_VREF Training. if (needs_quse_training) - training_command |= (1 << 4); // QUSE: Initiates QUSE Training. + training_command |= BIT(4); // QUSE: Initiates QUSE Training. if (needs_quse_vref_training) - training_command |= (1 << 8); // QUSE_VREF: Initiates DQS_VREF Training. + training_command |= BIT(8); // QUSE_VREF: Initiates DQS_VREF Training. if (needs_wr_training) - training_command |= (1 << 3); // WR: Initiates WR Training. + training_command |= BIT(3); // WR: Initiates WR Training. if (needs_wr_vref_training) - training_command |= (1 << 6); // WR_VREF: Initiates OB (write) DRAM_VREF Training. + training_command |= BIT(6); // WR_VREF: Initiates OB (write) DRAM_VREF Training. if (needs_rd_training) - training_command |= (1 << 2); // RD: Initiates RD Training. + training_command |= BIT(2); // RD: Initiates RD Training. if (needs_rd_vref_training) - training_command |= (1 << 7); // RD_VREF: Initiates IB_DQ_VREF Training. - training_command |= (1 << 31); // GO: Start the Training. + training_command |= BIT(7); // RD_VREF: Initiates IB_DQ_VREF Training. + training_command |= BIT(31); // GO: Start the Training. _ccfifo_write(EMC_TRAINING_CMD, training_command, 0); @@ -3317,12 +3264,12 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e _ccfifo_write(EMC_PIN, emc_pin_o & 0xFFFFFFF8, 0); _ccfifo_write(EMC_CFG_SYNC, 0, 0); - _ccfifo_write(EMC_DBG, emc_dbg_val | 0x40000002, 0); + _ccfifo_write(EMC_DBG, emc_dbg_val | EMC_DBG_WRITE_ACTIVE_ONLY | EMC_DBG_WRITE_MUX_ACTIVE, 0); _dvfs_power_ramp_down(true, src_emc_entry, dst_emc_entry, dst_clock_period); _ccfifo_write(EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 1, 0); - _ccfifo_write(EMC_DBG, (emc_dbg_val & 0xBFFFFFFF) | 2, 0); + _ccfifo_write(EMC_DBG, (emc_dbg_val & ~EMC_DBG_WRITE_ACTIVE_ONLY) | EMC_DBG_WRITE_MUX_ACTIVE, 0); _dvfs_power_ramp_up(true, src_emc_entry, dst_emc_entry, needs_training, src_clock_period); @@ -3357,17 +3304,17 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e if (!needs_ca_combo_training) _ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) ^ 0xC0000C0, 0); - _ccfifo_write(EMC_SELF_REF, 0, 0); // Was 0x100. + _ccfifo_write(EMC_SELF_REF, 0, 0); } // Step 19.2. EPRINTF("Step 19.2"); if (bg_regulator_mode_change) { - _ccfifo_write(EMC_DBG, emc_dbg_o | 2, 0); + _ccfifo_write(EMC_DBG, emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE, 0); u32 bg_regulator_switch_complete_wait_clks = 0; - if (needs_tristate_training) + if (in_training) { bg_regulator_switch_complete_wait_clks = 1250000 / src_clock_period; _ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0, @@ -3386,22 +3333,22 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 20 - Issue ref and optional QRST. EPRINTF("Step 20"); - if (needs_tristate_training) + if (in_training) _ccfifo_write(EMC_REF, 0, 0); // Step 21 - Restore ZCAL and ZCAL interval. EPRINTF("Step 21"); - _ccfifo_write(EMC_DBG, emc_dbg_o | 2, 0); + _ccfifo_write(EMC_DBG, emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE, 0); - if (needs_tristate_training) + if (in_training) _ccfifo_write(EMC_ZCAL_INTERVAL, src_emc_entry->burst_regs.emc_zcal_interval, 0); _ccfifo_write(EMC_CFG, dst_emc_entry->burst_regs.emc_cfg & 0xEFFFFFFF, 0); // Step 22 - Restore EMC_CFG_PIPE_CLK. EPRINTF("Step 22"); - //if (needs_tristate_training && dram_type == DRAM_TYPE_LPDDR4)//////////////// - if (needs_tristate_training) + //if (in_training && dram_type == DRAM_TYPE_LPDDR4)//////////////// + if (in_training) _ccfifo_write(EMC_SEL_DPD_CTRL, src_emc_entry->emc_sel_dpd_ctrl, 0); _ccfifo_write(EMC_DBG, emc_dbg_o, 0); @@ -3416,9 +3363,9 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e } // Step 23 - Clock Change. - EPRINTF("Step 23"); + EPRINTF("Step 23 - Clock Change"); // During training save current clock. - if (needs_tristate_training) + if (in_training) { u32 emc_clk_src = CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC); CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC_SAFE) = emc_clk_src; @@ -3442,11 +3389,11 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 24 - Save training results. EPRINTF("Step 24"); - if (needs_tristate_training) + if (in_training) { (void)MC(MC_EMEM_ADR_CFG); emc_dbg_val = EMC(EMC_DBG); - EMC(EMC_DBG) |= 1; + EMC(EMC_DBG) |= EMC_DBG_READ_MUX_ACTIVE; _save_train_results(dst_emc_entry, needs_training, dram_dev_num, channel1_enabled); @@ -3455,7 +3402,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 25 - Program MC updown regs. EPRINTF("Step 25"); - if ((dst_emc_entry->rate_khz > src_emc_entry->rate_khz) && !needs_tristate_training) + if (!in_training && (dst_emc_entry->rate_khz > src_emc_entry->rate_khz)) { for (u32 i = 0; dst_emc_entry->num_up_down > i; i++) MC(la_scale_regs_mc_addr_table[i]) = dst_emc_entry->la_scale_regs[i]; @@ -3467,17 +3414,14 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 26 - Restore ZCAL regs. EPRINTF("Step 26"); - if (!in_self_refresh) - { - EMC(EMC_DBG) = emc_dbg_o | 2; - EMC(EMC_ZCAL_WAIT_CNT) = dst_emc_entry->burst_regs.emc_zcal_wait_cnt; - EMC(EMC_ZCAL_INTERVAL) = dst_emc_entry->burst_regs.emc_zcal_interval; - EMC(EMC_DBG) = emc_dbg_o; - } + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE; + EMC(EMC_ZCAL_WAIT_CNT) = dst_emc_entry->burst_regs.emc_zcal_wait_cnt; + EMC(EMC_ZCAL_INTERVAL) = dst_emc_entry->burst_regs.emc_zcal_interval; + EMC(EMC_DBG) = emc_dbg_o; // Step 27 - Restore EMC_CFG, FDPD regs. EPRINTF("Step 27"); - EMC(EMC_DBG) = emc_dbg_o | 2; + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE; EMC(EMC_CFG) = dst_emc_entry->burst_regs.emc_cfg; EMC(EMC_DBG) = emc_dbg_o; EMC(EMC_FDPD_CTRL_CMD_NO_RAMP) = dst_emc_entry->emc_fdpd_ctrl_cmd_no_ramp; @@ -3485,13 +3429,13 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 28 - Training recover. EPRINTF("Step 28"); - if (needs_tristate_training) + if (in_training) { - EMC(EMC_DBG) = emc_dbg_o | 2; - EMC(EMC_CFG) = dst_emc_entry->burst_regs.emc_cfg; - EMC(EMC_SEL_DPD_CTRL) = dst_emc_entry->emc_sel_dpd_ctrl; - EMC(EMC_ZCAL_WAIT_CNT) = src_emc_entry->burst_regs.emc_zcal_wait_cnt; - EMC(EMC_ZCAL_INTERVAL) = src_emc_entry->burst_regs.emc_zcal_interval; + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE; + EMC(EMC_CFG) = dst_emc_entry->burst_regs.emc_cfg; + EMC(EMC_SEL_DPD_CTRL) = dst_emc_entry->emc_sel_dpd_ctrl; + EMC(EMC_ZCAL_WAIT_CNT) = src_emc_entry->burst_regs.emc_zcal_wait_cnt; + EMC(EMC_ZCAL_INTERVAL) = src_emc_entry->burst_regs.emc_zcal_interval; EMC(EMC_AUTO_CAL_CONFIG2) = src_emc_entry->emc_auto_cal_config2; EMC(EMC_AUTO_CAL_CONFIG3) = src_emc_entry->emc_auto_cal_config3; EMC(EMC_AUTO_CAL_CONFIG4) = src_emc_entry->emc_auto_cal_config4; @@ -3503,7 +3447,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e EMC(EMC_TR_DVFS) = dst_emc_entry->burst_regs.emc_tr_dvfs & 0xFFFFFFFE; } - EMC(EMC_DBG) = emc_dbg_o | 2; + EMC(EMC_DBG) = emc_dbg_o | EMC_DBG_WRITE_MUX_ACTIVE; EMC(EMC_PMACRO_AUTOCAL_CFG_COMMON) = dst_emc_entry->burst_regs.emc_pmacro_autocal_cfg_common; EMC(EMC_DBG) = emc_dbg_o; @@ -3516,7 +3460,7 @@ static u32 _minerva_set_clock(emc_table_t *src_emc_entry, emc_table_t *dst_emc_e // Step 30 - Re-enable autocal. EPRINTF("Step 30"); - if (needs_tristate_training) + if (in_training) EMC(EMC_AUTO_CAL_CONFIG) = src_emc_entry->emc_auto_cal_config; else { @@ -3579,12 +3523,12 @@ static void _minerva_train_patterns(emc_table_t *src_emc_entry, emc_table_t *dst bool dual_channel = (EMC(EMC_FBIO_CFG7) >> 1) & ((EMC(EMC_FBIO_CFG7) >> 2) & 1); - EMC(EMC_DBG) = (EMC(EMC_DBG) & 0xF3FFFFFF) | 0x8000000; + EMC(EMC_DBG) = (EMC(EMC_DBG) & ~EMC_DBG_CFG_SWAP_MASK) | EMC_DBG_CFG_SWAP_ASSEMBLY_ONLY; EMC(EMC_CFG_UPDATE) = (EMC(EMC_CFG_UPDATE) & 0xFFFFFFF9) | 4; _timing_update(dual_channel); EMC(EMC_CFG_UPDATE) &= 0xFFFFFFF9; - EMC(EMC_DBG) &= 0xF3FFFFFF; + EMC(EMC_DBG) &= ~EMC_DBG_CFG_SWAP_MASK; // EMC_DBG_CFG_SWAP_ACTIVE_ONLY. EMC(EMC_CFG_DIG_DLL) = (EMC(EMC_CFG_DIG_DLL) & 0xFFFFFF3E) | 0x80; _timing_update(dual_channel); @@ -3779,7 +3723,7 @@ static u32 _minerva_set_rate(mtc_config_t *mtc_cfg) u32 dst_clk_src_emc = dst_emc_entry->clk_src_emc; freq_changed = _check_freq_changed(dst_rate_khz, dst_clk_src_emc, src_rate_khz, src_clk_src_emc); - EPRINTFARGS("Requested freq change from %d to %d.", src_rate_khz, dst_rate_khz); + EPRINTFARGS("Requested op %d from %d to %d.", mtc_cfg->train_mode, src_rate_khz, dst_rate_khz); // Get current clock source. emc_clk_src = CLOCK(CLK_RST_CONTROLLER_CLK_SOURCE_EMC) >> EMC_2X_CLK_SRC_SHIFT; @@ -3891,10 +3835,10 @@ static void _minerva_get_table(mtc_config_t *mtc_cfg) } mtc_cfg->table_entries = EMC_TABLE_SIZE_R7 / EMC_TABLE_ENTRY_SIZE_R7; - mtc_cfg->rate_to = 0; - mtc_cfg->rate_from = 0; + mtc_cfg->rate_to = 0; + mtc_cfg->rate_from = 0; mtc_cfg->train_mode = 0; - mtc_cfg->prev_temp = 0; + mtc_cfg->prev_temp = 0; mtc_cfg->current_emc_table = NULL; // Important! @@ -3929,8 +3873,8 @@ void _minerva_init(mtc_config_t *mtc_cfg, bdkParams_t bp) // Change rates for OC RAM. if (mtc_cfg->rate_from == MAX_FREQ_T210) mtc_cfg->rate_from = OVERCLOCK_FREQ; - if (mtc_cfg->rate_to == MAX_FREQ_T210) - mtc_cfg->rate_to = OVERCLOCK_FREQ; + if (mtc_cfg->rate_to == MAX_FREQ_T210) + mtc_cfg->rate_to = OVERCLOCK_FREQ; #endif switch (mtc_cfg->train_mode) @@ -3961,8 +3905,8 @@ void _minerva_init(mtc_config_t *mtc_cfg, bdkParams_t bp) // Restore rates for OC RAM. if (mtc_cfg->rate_from == OVERCLOCK_FREQ) mtc_cfg->rate_from = MAX_FREQ_T210; - if (mtc_cfg->rate_to == OVERCLOCK_FREQ) - mtc_cfg->rate_to = MAX_FREQ_T210; + if (mtc_cfg->rate_to == OVERCLOCK_FREQ) + mtc_cfg->rate_to = MAX_FREQ_T210; #endif mtc_cfg->train_ram_patterns = train_ram_patterns;