OmniNX/Atmosphere
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c5edb031fa833012b4d3fa7934564d39aec27bd0
Atmosphere/fusee/fusee-secondary/src/mtc/mtc.c
Michael Scire c5edb031fa fusee: restore DRAM to 204MHz before starting main cpu on Mariko (closes #1600). 
Nintendo's Mariko tables result in trained frequency of 1599999 instead of 1600000.

PCV checks for rate == 1600000 exactly, when doing EMC init.

Thus EMC init does not succeed if we are trained to 1600000.

PCV has a fudge factor of 1000 used in SetEmcDvfsFreq, but this is not used in InitEmcDvfs.

This failure means that PCV cannot change rate back to 204MHz before sleep, and then after
wake extremely degraded performance is observed.

Restoring DRAM to 204MHz before boot causes EMC init to succeed/fixes performance degradation.
2021-08-30 07:19:28 -07:00

3750 lines
181 KiB
C
Raw Blame History

/*
* Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2018 CTCaer <ctcaer@gmail.com>
* Copyright (c) 2018-2020 Atmosphère-NX
*
* This program is free software; you can redistribute it and/or modify it
* under the terms and conditions of the GNU General Public License,
* version 2, as published by the Free Software Foundation.
*
* This program is distributed in the hope it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
* more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mtc.h"
#include "mtc_b01.h"
#include "mtc_tables.h"
#include "../car.h"
#include "../fuse.h"
#include "../timers.h"
#include "../../../../fusee/common/log.h"
/*
* Macros.
*/
#define max(x, y) ({ \
typeof(x) _max1 = (x); \
typeof(y) _max2 = (y); \
(void) (&_max1 == &_max2); \
_max1 > _max2 ? _max1 : _max2; })
#define max_t(type, x, y) ({ \
type __max1 = (x); \
type __max2 = (y); \
__max1 > __max2 ? __max1: __max2; })
#define TRIM_REG(chan, rank, reg, byte) \
((EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _MASK & \
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## \
rank ## _ ## reg ## _INDEX]) >> \
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte ## _SHIFT) \
+ \
(((EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \
byte ## _DATA_BRLSHFT_MASK & \
next_timing->trim_perch_regs[REG_EMC ## chan ## \
_EMC_DATA_BRLSHFT_ ## rank ## _INDEX]) >> \
EMC_DATA_BRLSHFT_ ## rank ## _RANK ## rank ## _BYTE ## \
byte ## _DATA_BRLSHFT_SHIFT) * 64)
#define CALC_TEMP(rank, reg, byte1, byte2, n) \
((new[n] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## \
reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _SHIFT) & \
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte1 ## _MASK) \
| \
((new[n + 1] << EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## \
reg ## _OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _SHIFT) & \
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK ## rank ## _ ## reg ## \
_OB_DDLL_LONG_DQ_RANK ## rank ## _BYTE ## byte2 ## _MASK) \
/*
* PTFV defines - basically just indexes into the per table PTFV array.
*/
#define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX 0
#define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1
#define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2
#define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX 3
#define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX 4
#define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX 5
#define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX 6
#define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX 7
#define PTFV_WRITE_SAMPLES_INDEX 8
#define PTFV_DVFS_SAMPLES_INDEX 9
#define PTFV_MOVAVG_WEIGHT_INDEX 10
#define PTFV_CONFIG_CTRL_INDEX 11
#define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0)
/*
* Do arithmetic in fixed point.
*/
#define MOVAVG_PRECISION_FACTOR 100
/*
* The division portion of the average operation.
*/
#define __AVERAGE_PTFV(dev) \
({ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \
next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
next_timing->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
/*
* The division portion of the average write operation.
*/
#define __AVERAGE_WRITE_PTFV(dev) \
({ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \
next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
next_timing->ptfv_list[PTFV_WRITE_SAMPLES_INDEX]; })
/*
* Convert val to fixed point and add it to the temporary average.
*/
#define __INCREMENT_PTFV(dev, val) \
({ next_timing->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \
((val) * MOVAVG_PRECISION_FACTOR); })
/*
* Convert a moving average back to integral form and return the value.
*/
#define __MOVAVG_AC(timing, dev) \
((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \
MOVAVG_PRECISION_FACTOR)
/* Weighted update. */
#define __WEIGHTED_UPDATE_PTFV(dev, nval) \
do { \
int w = PTFV_MOVAVG_WEIGHT_INDEX; \
int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX; \
\
next_timing->ptfv_list[dqs] = \
((nval * MOVAVG_PRECISION_FACTOR) + \
(next_timing->ptfv_list[dqs] * \
next_timing->ptfv_list[w])) / \
(next_timing->ptfv_list[w] + 1); \
} while (0)
/* Access a particular average. */
#define __MOVAVG(timing, dev) \
((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX])
static int g_active_timing_table_idx = -1;
static bool g_is_pllmb = false;
static bool g_fsp_for_next_freq = false;
static bool g_write_training_pattern = true;
#define DEFINE_REG(type, reg) (reg)
static const uint32_t burst_regs_per_ch_off[] = BURST_REGS_PER_CH_LIST;
static const uint32_t burst_regs_off[] = BURST_REGS_LIST;
static const uint32_t trim_regs_per_ch_off[] = TRIM_REGS_PER_CH_LIST;
static const uint32_t trim_regs_off[] = TRIM_REGS_LIST;
static const uint32_t vref_regs_per_ch_off[] = VREF_REGS_PER_CH_LIST;
static const uint32_t training_mod_regs_per_ch_off[] = TRAINING_MOD_REGS_PER_CH_LIST;
static const uint32_t burst_mc_regs_off[] = BURST_MC_REGS_LIST;
static const uint32_t la_scale_regs_off[] = BURST_UP_DOWN_REGS_LIST;
#undef DEFINE_REG
#define DEFINE_REG(type, reg) (type)
static const uint32_t burst_regs_per_ch_type[] = BURST_REGS_PER_CH_LIST;
static const uint32_t trim_regs_per_ch_type[] = TRIM_REGS_PER_CH_LIST;
static const uint32_t vref_regs_per_ch_type[] = VREF_REGS_PER_CH_LIST;
static const uint32_t training_mod_regs_per_ch_type[] = TRAINING_MOD_REGS_PER_CH_LIST;
#undef DEFINE_REG
static const uint32_t g_ram_pattern_dq[0x500] = {
0x18181818, 0x61616161, 0x85858585, 0x14141414, 0x51515151,
0x47474747, 0x1E1E1E1E, 0x79797979, 0xE5E5E5E5, 0x94949494,
0x51515151, 0x46464646, 0x19191919, 0x67676767, 0x9C9C9C9C,
0x71717171, 0xC5C5C5C5, 0x17171717, 0x5F5F5F5F, 0x7E7E7E7E,
0xFBFBFBFB, 0xEDEDEDED, 0xB4B4B4B4, 0xD2D2D2D2, 0x48484848,
0x21212121, 0x85858585, 0x16161616, 0x59595959, 0x66666666,
0x9A9A9A9A, 0x69696969, 0xA4A4A4A4, 0x93939393, 0x4F4F4F4F,
0x3F3F3F3F, 0xFCFCFCFC, 0xF3F3F3F3, 0xCDCDCDCD, 0x37373737,
0xDCDCDCDC, 0x70707070, 0xC3C3C3C3, 0x0F0F0F0F, 0x3E3E3E3E,
0xFAFAFAFA, 0xEBEBEBEB, 0xACACACAC, 0xB3B3B3B3, 0xCCCCCCCC,
0x31313131, 0xC5C5C5C5, 0x15151515, 0x57575757, 0x5F5F5F5F,
0x7F7F7F7F, 0xFDFDFDFD, 0xF4F4F4F4, 0xD0D0D0D0, 0x42424242,
0x08080808, 0x23232323, 0x8F8F8F8F, 0x3F3F3F3F, 0x18181818,
0x61616161, 0x85858585, 0x14141414, 0x51515151, 0x47474747,
0x1E1E1E1E, 0x79797979, 0xE5E5E5E5, 0x94949494, 0x51515151,
0x46464646, 0x19191919, 0x67676767, 0x9C9C9C9C, 0x71717171,
0xC5C5C5C5, 0x17171717, 0x5F5F5F5F, 0x7E7E7E7E, 0xFBFBFBFB,
0xEDEDEDED, 0xB4B4B4B4, 0xD2D2D2D2, 0x48484848, 0x21212121,
0x85858585, 0x16161616, 0x59595959, 0x66666666, 0x9A9A9A9A,
0x69696969, 0xA4A4A4A4, 0x93939393, 0x4F4F4F4F, 0x3F3F3F3F,
0xFCFCFCFC, 0xF3F3F3F3, 0xCDCDCDCD, 0x37373737, 0xDCDCDCDC,
0x70707070, 0xC3C3C3C3, 0x0F0F0F0F, 0x3E3E3E3E, 0xFAFAFAFA,
0xEBEBEBEB, 0xACACACAC, 0xB3B3B3B3, 0xCCCCCCCC, 0x31313131,
0xC5C5C5C5, 0x15151515, 0x57575757, 0x5F5F5F5F, 0x7F7F7F7F,
0xFDFDFDFD, 0xF4F4F4F4, 0xD0D0D0D0, 0x42424242, 0x08080808,
0x23232323, 0x8F8F8F8F, 0x3F3F3F3F, 0x06060606, 0x18181818,
0x21212121, 0x05050505, 0x14141414, 0x11111111, 0x07070707,
0x1E1E1E1E, 0x39393939, 0x25252525, 0x14141414, 0x11111111,
0x06060606, 0x19191919, 0x27272727, 0x1C1C1C1C, 0x31313131,
0x05050505, 0x17171717, 0x1F1F1F1F, 0x3E3E3E3E, 0x3B3B3B3B,
0x2D2D2D2D, 0x34343434, 0x12121212, 0x08080808, 0x21212121,
0x05050505, 0x16161616, 0x19191919, 0x26262626, 0x1A1A1A1A,
0x29292929, 0x24242424, 0x13131313, 0x0F0F0F0F, 0x3F3F3F3F,
0x3C3C3C3C, 0x33333333, 0x0D0D0D0D, 0x37373737, 0x1C1C1C1C,
0x30303030, 0x03030303, 0x0F0F0F0F, 0x3E3E3E3E, 0x3A3A3A3A,
0x2B2B2B2B, 0x2C2C2C2C, 0x33333333, 0x0C0C0C0C, 0x31313131,
0x05050505, 0x15151515, 0x17171717, 0x1F1F1F1F, 0x3F3F3F3F,
0x3D3D3D3D, 0x34343434, 0x10101010, 0x02020202, 0x08080808,
0x23232323, 0x0F0F0F0F, 0x06060606, 0x18181818, 0x21212121,
0x05050505, 0x14141414, 0x11111111, 0x07070707, 0x1E1E1E1E,
0x39393939, 0x25252525, 0x14141414, 0x11111111, 0x06060606,
0x19191919, 0x27272727, 0x1C1C1C1C, 0x31313131, 0x05050505,
0x17171717, 0x1F1F1F1F, 0x3E3E3E3E, 0x3B3B3B3B, 0x2D2D2D2D,
0x34343434, 0x12121212, 0x08080808, 0x21212121, 0x05050505,
0x16161616, 0x19191919, 0x26262626, 0x1A1A1A1A, 0x29292929,
0x24242424, 0x13131313, 0x0F0F0F0F, 0x3F3F3F3F, 0x3C3C3C3C,
0x33333333, 0x0D0D0D0D, 0x37373737, 0x1C1C1C1C, 0x30303030,
0x03030303, 0x0F0F0F0F, 0x3E3E3E3E, 0x3A3A3A3A, 0x2B2B2B2B,
0x2C2C2C2C, 0x33333333, 0x0C0C0C0C, 0x31313131, 0x05050505,
0x15151515, 0x17171717, 0x1F1F1F1F, 0x3F3F3F3F, 0x3D3D3D3D,
0x34343434, 0x10101010, 0x02020202, 0x08080808, 0x23232323,
0x0F0F0F0F,
0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000,
0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0xFFFFFFFF,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000,
0x00000000, 0xFFFFFFFF, 0x00000000, 0x00000000, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x00000000, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F, 0x3F3F3F3F,
0x3F3F3F3F, 0x00000000, 0x00000000, 0x00000000, 0x3F3F3F3F,
0x00000000,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF,
0x00000000, 0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000,
0xFFFFFFFF, 0x00000000, 0xFFFFFFFF, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F,
0x00000000, 0x3F3F3F3F, 0x00000000, 0x3F3F3F3F, 0x00000000,
0x3F3F3F3F,
0x80808080, 0x00000000, 0x80808080, 0x00000000, 0x80808080,
0x00000000, 0x80808080, 0x40404040, 0x00000000, 0x40404040,
0x00000000, 0x40404040, 0x00000000, 0x40404040, 0x20202020,
0x00000000, 0x20202020, 0x00000000, 0x20202020, 0x00000000,
0x20202020, 0x10101010, 0x00000000, 0x10101010, 0x00000000,
0x10101010, 0x00000000, 0x10101010, 0x08080808, 0x00000000,
0x08080808, 0x00000000, 0x08080808, 0x00000000, 0x08080808,
0x04040404, 0x00000000, 0x04040404, 0x00000000, 0x04040404,
0x00000000, 0x04040404, 0x02020202, 0x00000000, 0x02020202,
0x00000000, 0x02020202, 0x00000000, 0x02020202, 0x01010101,
0x00000000, 0x01010101, 0x00000000, 0x01010101, 0x00000000,
0x01010101, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x80808080,
0x00000000, 0x80808080, 0x00000000, 0x80808080, 0x00000000,
0x80808080, 0x40404040, 0x00000000, 0x40404040, 0x00000000,
0x40404040, 0x00000000, 0x40404040, 0x20202020, 0x00000000,
0x20202020, 0x00000000, 0x20202020, 0x00000000, 0x20202020,
0x10101010, 0x00000000, 0x10101010, 0x00000000, 0x10101010,
0x00000000, 0x10101010, 0x08080808, 0x00000000, 0x08080808,
0x00000000, 0x08080808, 0x00000000, 0x08080808, 0x04040404,
0x00000000, 0x04040404, 0x00000000, 0x04040404, 0x00000000,
0x04040404, 0x02020202, 0x00000000, 0x02020202, 0x00000000,
0x02020202, 0x00000000, 0x02020202, 0x01010101, 0x00000000,
0x01010101, 0x00000000, 0x01010101, 0x00000000, 0x01010101,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x20202020,
0x00000000, 0x20202020, 0x00000000, 0x20202020, 0x00000000,
0x20202020, 0x00000000, 0x20202020, 0x00000000, 0x10101010,
0x00000000, 0x10101010, 0x00000000, 0x10101010, 0x00000000,
0x10101010, 0x00000000, 0x10101010, 0x00000000, 0x08080808,
0x00000000, 0x08080808, 0x00000000, 0x08080808, 0x00000000,
0x08080808, 0x00000000, 0x08080808, 0x00000000, 0x04040404,
0x00000000, 0x04040404, 0x00000000, 0x04040404, 0x00000000,
0x04040404, 0x00000000, 0x04040404, 0x00000000, 0x02020202,
0x00000000, 0x02020202, 0x00000000, 0x02020202, 0x00000000,
0x02020202, 0x00000000, 0x02020202, 0x00000000, 0x01010101,
0x00000000, 0x01010101, 0x00000000, 0x01010101, 0x00000000,
0x01010101, 0x00000000, 0x01010101, 0x00000000, 0x00000000,
0x00000000, 0x00000000, 0x00000000, 0x20202020, 0x00000000,
0x20202020, 0x00000000, 0x20202020, 0x00000000, 0x20202020,
0x00000000, 0x20202020, 0x00000000, 0x10101010, 0x00000000,
0x10101010, 0x00000000, 0x10101010, 0x00000000, 0x10101010,
0x00000000, 0x10101010, 0x00000000, 0x08080808, 0x00000000,
0x08080808, 0x00000000, 0x08080808, 0x00000000, 0x08080808,
0x00000000, 0x08080808, 0x00000000, 0x04040404, 0x00000000,
0x04040404, 0x00000000, 0x04040404, 0x00000000, 0x04040404,
0x00000000, 0x04040404, 0x00000000, 0x02020202, 0x00000000,
0x02020202, 0x00000000, 0x02020202, 0x00000000, 0x02020202,
0x00000000, 0x02020202, 0x00000000, 0x01010101, 0x00000000,
0x01010101, 0x00000000, 0x01010101, 0x00000000, 0x01010101,
0x00000000, 0x01010101, 0x00000000, 0x00000000, 0x00000000,
0x00000000,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333,
0xAAAAAAAA, 0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA,
0x55555555, 0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555,
0xCCCCCCCC, 0x33333333, 0xAAAAAAAA, 0x55555555, 0xCCCCCCCC,
0x33333333
};
static const uint32_t g_ram_pattern_dmi[0x500] = {
0xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,
0x0, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,
0xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0x0, 0x0,
0xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0x0,
0xF, 0xF, 0xF, 0x0, 0xF, 0xF, 0xF, 0x0,
0x0, 0xF, 0xF, 0x0, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF,
0x0, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,
0xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,
0x0, 0xF, 0x0, 0xF, 0xF, 0x0, 0xF, 0xF,
0xF, 0xF, 0x0, 0xF, 0xF, 0x0, 0x0, 0x0,
0xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0x0,
0xF, 0xF, 0xF, 0x0, 0xF, 0xF, 0xF, 0x0,
0x0, 0xF, 0xF, 0x0, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF, 0xF,
0x0, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0, 0x0,
0xF, 0xF, 0x0, 0x0, 0x0, 0x0, 0xF, 0x0,
0xF, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0xF,
0x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,
0xF, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,
0xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF,
0xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0,
0xF, 0xF, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,
0x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0, 0x0,
0xF, 0xF, 0x0, 0x0, 0x0, 0x0, 0xF, 0x0,
0xF, 0x0, 0x0, 0x0, 0xF, 0xF, 0xF, 0xF,
0x0, 0x0, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,
0xF, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF, 0x0,
0xF, 0x0, 0xF, 0x0, 0x0, 0xF, 0xF, 0xF,
0xF, 0xF, 0x0, 0xF, 0x0, 0x0, 0x0, 0x0,
0xF, 0xF, 0xF, 0x0, 0x0, 0x0, 0xF, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0xF, 0x0, 0xF, 0x0, 0xF, 0x0, 0xF, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3,
0xA, 0x5, 0xC, 0x3, 0xA, 0x5, 0xC, 0x3
};
/* Determine the current SoC for Mariko specific code. */
static bool is_soc_mariko() {
return (fuse_get_soc_type() == 1);
}
/* Register read/write helpers. */
static inline void emc_write(uint32_t val, uint32_t offset) {
MAKE_EMC_REG(offset) = val;
}
static inline uint32_t emc_read(uint32_t offset) {
return MAKE_EMC_REG(offset);
}
static inline void emc0_write(uint32_t val, uint32_t offset) {
MAKE_EMC0_REG(offset) = val;
}
static inline uint32_t emc0_read(uint32_t offset) {
return MAKE_EMC0_REG(offset);
}
static inline void emc1_write(uint32_t val, uint32_t offset) {
MAKE_EMC1_REG(offset) = val;
}
static inline uint32_t emc1_read(uint32_t offset) {
return MAKE_EMC1_REG(offset);
}
static inline void emc_write_per_ch(uint32_t val, int type, uint32_t offset) {
switch (type) {
case REG_EMC:
emc_write(val, offset);
break;
case REG_EMC0:
emc0_write(val, offset);
break;
case REG_EMC1:
emc1_write(val, offset);
break;
}
}
static inline uint32_t emc_read_per_ch(int type, uint32_t offset) {
uint32_t val = 0;
switch (type) {
case REG_EMC:
val = emc_read(offset);
break;
case REG_EMC0:
val = emc0_read(offset);
break;
case REG_EMC1:
val = emc1_read(offset);
break;
}
return val;
}
static inline void mc_write(uint32_t val, uint32_t offset) {
MAKE_MC_REG(offset) = val;
}
static inline uint32_t mc_read(uint32_t offset) {
return MAKE_MC_REG(offset);
}
/* Configure clock change sequence FIFO */
static void ccfifo_write(uint32_t ccfifo_addr, uint32_t ccfifo_data, uint32_t ccfifo_stall_cnt) {
MAKE_EMC_REG(EMC_CCFIFO_DATA) = ccfifo_data;
MAKE_EMC_REG(EMC_CCFIFO_ADDR) = ((ccfifo_addr & 0xFFFF) | ((ccfifo_stall_cnt & 0x7FFF) << 16) | 0x80000000);
}
static void start_periodic_compensation() {
uint32_t mpc_req = 0x4B;
/* Write to EMC_MPC_0. */
emc_write(mpc_req, EMC_MPC);
/* Dummy read. */
mpc_req = emc_read(EMC_MPC);
}
static uint32_t actual_osc_clocks(uint32_t in) {
if (in < 0x40)
return in * 16;
else if (in < 0x80)
return 2048;
else if (in < 0xc0)
return 4096;
else
return 8192;
}
static void emc_set_shadow_bypass(int set) {
uint32_t emc_dbg = emc_read(EMC_DBG);
if (set)
emc_write(emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
else
emc_write(emc_dbg & ~EMC_DBG_WRITE_MUX_ACTIVE, EMC_DBG);
}
static uint32_t wait_for_update(uint32_t status_reg, uint32_t bit_mask, bool updated_state, int chan) {
for (int i = 0; i < EMC_STATUS_UPDATE_TIMEOUT; i++) {
if (chan == REG_EMC) {
if (((emc_read_per_ch(REG_EMC, status_reg) & bit_mask) != 0) == updated_state)
return 0;
} else {
if (((emc_read_per_ch(REG_EMC1, status_reg) & bit_mask) != 0) == updated_state)
return 0;
}
udelay(1);
}
/* Timeout. */
return 4;
}
static void emc_timing_update(bool dual_chan) {
/* Trigger the timing update event. */
emc_write(0x1, EMC_TIMING_CONTROL);
/* Wait for the update to finish. */
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_TIMING_UPDATE_STALLED, false, REG_EMC);
if (dual_chan)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_TIMING_UPDATE_STALLED, false, REG_EMC1);
}
static uint32_t get_dll_state(tegra_emc_timing_t *next_timing) {
bool next_dll_enabled = !(next_timing->emc_emrs & 0x1);
if (next_dll_enabled)
return DLL_ON;
else
return DLL_OFF;
}
static uint32_t div_o3(uint32_t a, uint32_t b) {
uint32_t result = a / b;
if ((b * result) < a)
return result + 1;
else
return result;
}
static uint32_t dvfs_power_ramp_down(bool flip_backward, tegra_emc_timing_t* current_timing, tegra_emc_timing_t* next_timing, uint32_t clk) {
uint32_t ramp_down_wait = 0;
uint32_t seq_wait = 0;
uint32_t pmacro_cmd_pad = 0;
uint32_t pmacro_dq_pad = 0;
uint32_t pmacro_cfg5 = 0;
uint32_t pmacro_rfu1 = 0;
uint32_t pmacro_common_tx = 0;
if (flip_backward) {
pmacro_cmd_pad = next_timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = next_timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = next_timing->burst_regs[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = next_timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
} else {
pmacro_cmd_pad = current_timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = ((next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & 0x101) | current_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]);
pmacro_rfu1 = current_timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = current_timing->burst_regs[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = current_timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
}
pmacro_cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, 0);
ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 | EMC_FBIO_CFG5_CMD_TX_DIS, 12);
ramp_down_wait = clk * 12;
seq_wait = (100000 / clk) + 1;
if (clk < (1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD)) {
if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
pmacro_cmd_pad &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
pmacro_cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC;
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, seq_wait);
ramp_down_wait += 100000;
pmacro_dq_pad &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
pmacro_dq_pad |= EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC;
ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad, 0);
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x01120112, 0);
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x01120112, seq_wait);
ramp_down_wait += 100000;
}
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x01bf01bf, seq_wait);
ramp_down_wait += 100000;
if (clk < (1000000 / IOBRICK_DCC_THRESHOLD)) {
pmacro_cmd_pad &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, seq_wait);
ramp_down_wait += 100000;
pmacro_dq_pad &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad, 0);
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x07ff07ff, 0);
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x07ff07ff, seq_wait);
ramp_down_wait += 100000;
}
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & ~0x07ff07ff, seq_wait + 19);
ramp_down_wait += (100000 + (20 * clk));
}
if (clk < (1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD)) {
ramp_down_wait += 100000;
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & ~0x5, seq_wait);
ramp_down_wait += 100000;
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & ~0xf, seq_wait);
ramp_down_wait += 100000;
ccfifo_write(0, 0, seq_wait);
ramp_down_wait += 100000;
} else {
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & ~0xf, seq_wait);
}
return ramp_down_wait;
}
static uint32_t dvfs_power_ramp_up(bool flip_backward, tegra_emc_timing_t* current_timing, tegra_emc_timing_t* next_timing, uint32_t training, uint32_t clk) {
uint32_t ramp_up_wait = 0;
uint32_t pmacro_cmd_pad = 0;
uint32_t pmacro_dq_pad = 0;
uint32_t pmacro_cfg5 = 0;
uint32_t pmacro_rfu1 = 0;
uint32_t pmacro_common_tx = 0;
if (flip_backward) {
pmacro_cmd_pad = current_timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = current_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = current_timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = current_timing->burst_regs[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = current_timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
} else if (training & 3) {
pmacro_cmd_pad = next_timing->shadow_regs_ca_train[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = next_timing->shadow_regs_ca_train[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = next_timing->shadow_regs_ca_train[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = next_timing->shadow_regs_ca_train[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = next_timing->shadow_regs_ca_train[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
} else if (training & 0xC) {
pmacro_cmd_pad = next_timing->shadow_regs_quse_train[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = next_timing->shadow_regs_quse_train[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = next_timing->shadow_regs_quse_train[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = next_timing->shadow_regs_quse_train[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = next_timing->shadow_regs_quse_train[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
} else if (training & 0xF0) {
pmacro_cmd_pad = next_timing->shadow_regs_rdwr_train[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = next_timing->shadow_regs_rdwr_train[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = next_timing->shadow_regs_rdwr_train[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = next_timing->shadow_regs_rdwr_train[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = next_timing->shadow_regs_rdwr_train[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
} else {
pmacro_cmd_pad = next_timing->burst_regs[EMC_PMACRO_CMD_PAD_TX_CTRL_INDEX];
pmacro_dq_pad = next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
pmacro_rfu1 = next_timing->burst_regs[EMC_PMACRO_BRICK_CTRL_RFU1_INDEX];
pmacro_cfg5 = next_timing->burst_regs[EMC_FBIO_CFG5_INDEX];
pmacro_common_tx = next_timing->burst_regs[EMC_PMACRO_COMMON_PAD_TX_CTRL_INDEX];
}
pmacro_cmd_pad |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & 0xa, 0);
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx & 0xf, (100000 / clk) + 1);
ramp_up_wait += 100000;
} else {
ccfifo_write(EMC_PMACRO_COMMON_PAD_TX_CTRL, pmacro_common_tx | 0x8, 0);
}
if (clk < 1000000 / DVFS_FGCG_HIGH_SPEED_THRESHOLD) {
if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
pmacro_cmd_pad |= (EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC);
pmacro_cmd_pad &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, (100000 / clk) + 1);
ramp_up_wait += 100000;
pmacro_dq_pad |= (EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC);
pmacro_dq_pad &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad, 0);
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xfe40fe40, 0);
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xfe40fe40, (100000 / clk) + 1);
ramp_up_wait += 100000;
}
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 & 0xfeedfeed, (100000 / clk) + 1);
ramp_up_wait += 100000;
if (clk < 1000000 / IOBRICK_DCC_THRESHOLD) {
pmacro_cmd_pad |= (EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, (100000 / clk) + 1);
ramp_up_wait += 100000;
pmacro_dq_pad |= (EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
ccfifo_write(EMC_PMACRO_DATA_PAD_TX_CTRL, pmacro_dq_pad, 0);
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1, 0);
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1, (100000 / clk) + 1);
ramp_up_wait += 100000;
}
ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, (100000 / clk) + 10);
ramp_up_wait += (100000 + (10 * clk));
} else if (clk < 1000000 / DVFS_FGCG_MID_SPEED_THRESHOLD) {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 | 0x06000600, (100000 / clk) + 1);
ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, (100000 / clk) + 10);
ramp_up_wait += (100000 + 10 * clk);
} else {
ccfifo_write(EMC_PMACRO_BRICK_CTRL_RFU1, pmacro_rfu1 | 0x00000600, 0);
ccfifo_write(EMC_FBIO_CFG5, pmacro_cfg5 & ~EMC_FBIO_CFG5_CMD_TX_DIS, 12);
ramp_up_wait += (12 * clk);
}
pmacro_cmd_pad &= ~EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
ccfifo_write(EMC_PMACRO_CMD_PAD_TX_CTRL, pmacro_cmd_pad, 5);
return ramp_up_wait;
}
static uint32_t apply_periodic_compensation_trimmer(tegra_emc_timing_t* next_timing, uint32_t offset) {
uint32_t temp = 0;
uint32_t next_timing_rate_mhz = next_timing->rate / 1000;
int tree_delta[4] = {0};
int tree_delta_taps[4] = {0};
int new[] = {
TRIM_REG(0, 0, 0, 0),
TRIM_REG(0, 0, 0, 1),
TRIM_REG(0, 0, 1, 2),
TRIM_REG(0, 0, 1, 3),
TRIM_REG(1, 0, 2, 4),
TRIM_REG(1, 0, 2, 5),
TRIM_REG(1, 0, 3, 6),
TRIM_REG(1, 0, 3, 7),
TRIM_REG(0, 1, 0, 0),
TRIM_REG(0, 1, 0, 1),
TRIM_REG(0, 1, 1, 2),
TRIM_REG(0, 1, 1, 3),
TRIM_REG(1, 1, 2, 4),
TRIM_REG(1, 1, 2, 5),
TRIM_REG(1, 1, 3, 6),
TRIM_REG(1, 1, 3, 7)
};
switch (offset) {
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
case EMC_DATA_BRLSHFT_0:
tree_delta[0] = 128 * (next_timing->current_dram_clktree_c0d0u0 - next_timing->trained_dram_clktree_c0d0u0);
tree_delta[1] = 128 * (next_timing->current_dram_clktree_c0d0u1 - next_timing->trained_dram_clktree_c0d0u1);
tree_delta[2] = 128 * (next_timing->current_dram_clktree_c1d0u0 - next_timing->trained_dram_clktree_c1d0u0);
tree_delta[3] = 128 * (next_timing->current_dram_clktree_c1d0u1 - next_timing->trained_dram_clktree_c1d0u1);
tree_delta_taps[0] = (tree_delta[0] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[1] = (tree_delta[1] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[2] = (tree_delta[2] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[3] = (tree_delta[3] * (int)next_timing_rate_mhz) / 1000000;
for (int i = 0; i < 4; i++) {
if ((tree_delta_taps[i] > next_timing->tree_margin) ||
(tree_delta_taps[i] <
(-1 * next_timing->tree_margin))) {
new[i * 2] = new[i * 2] + tree_delta_taps[i];
new[i * 2 + 1] = new[i * 2 + 1] +
tree_delta_taps[i];
}
}
if (offset == EMC_DATA_BRLSHFT_0) {
for (int i = 0; i < 8; i++)
new[i] = new[i] / 64;
} else {
for (int i = 0; i < 8; i++)
new[i] = new[i] % 64;
}
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
case EMC_DATA_BRLSHFT_1:
tree_delta[0] = 128 * (next_timing->current_dram_clktree_c0d1u0 - next_timing->trained_dram_clktree_c0d1u0);
tree_delta[1] = 128 * (next_timing->current_dram_clktree_c0d1u1 - next_timing->trained_dram_clktree_c0d1u1);
tree_delta[2] = 128 * (next_timing->current_dram_clktree_c1d1u0 - next_timing->trained_dram_clktree_c1d1u0);
tree_delta[3] = 128 * (next_timing->current_dram_clktree_c1d1u1 - next_timing->trained_dram_clktree_c1d1u1);
tree_delta_taps[0] = (tree_delta[0] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[1] = (tree_delta[1] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[2] = (tree_delta[2] * (int)next_timing_rate_mhz) / 1000000;
tree_delta_taps[3] = (tree_delta[3] * (int)next_timing_rate_mhz) / 1000000;
for (int i = 0; i < 4; i++) {
if ((tree_delta_taps[i] > next_timing->tree_margin) || (tree_delta_taps[i] < (-1 * next_timing->tree_margin))) {
new[8 + i * 2] = new[8 + i * 2] + tree_delta_taps[i];
new[8 + i * 2 + 1] = new[8 + i * 2 + 1] + tree_delta_taps[i];
}
}
if (offset == EMC_DATA_BRLSHFT_1) {
for (int i = 0; i < 8; i++)
new[i + 8] = new[i + 8] / 64;
} else {
for (int i = 0; i < 8; i++)
new[i + 8] = new[i + 8] % 64;
}
break;
}
switch (offset) {
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0:
temp = CALC_TEMP(0, 0, 0, 1, 0);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1:
temp = CALC_TEMP(0, 1, 2, 3, 2);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2:
temp = CALC_TEMP(0, 2, 4, 5, 4);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3:
temp = CALC_TEMP(0, 3, 6, 7, 6);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0:
temp = CALC_TEMP(1, 0, 0, 1, 8);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1:
temp = CALC_TEMP(1, 1, 2, 3, 10);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2:
temp = CALC_TEMP(1, 2, 4, 5, 12);
break;
case EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3:
temp = CALC_TEMP(1, 3, 6, 7, 14);
break;
case EMC_DATA_BRLSHFT_0:
temp = ((new[0] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE0_DATA_BRLSHFT_MASK) |
((new[1] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE1_DATA_BRLSHFT_MASK) |
((new[2] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE2_DATA_BRLSHFT_MASK) |
((new[3] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE3_DATA_BRLSHFT_MASK) |
((new[4] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE4_DATA_BRLSHFT_MASK) |
((new[5] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE5_DATA_BRLSHFT_MASK) |
((new[6] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE6_DATA_BRLSHFT_MASK) |
((new[7] <<
EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_0_RANK0_BYTE7_DATA_BRLSHFT_MASK);
break;
case EMC_DATA_BRLSHFT_1:
temp = ((new[8] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE0_DATA_BRLSHFT_MASK) |
((new[9] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE1_DATA_BRLSHFT_MASK) |
((new[10] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE2_DATA_BRLSHFT_MASK) |
((new[11] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE3_DATA_BRLSHFT_MASK) |
((new[12] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE4_DATA_BRLSHFT_MASK) |
((new[13] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE5_DATA_BRLSHFT_MASK) |
((new[14] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE6_DATA_BRLSHFT_MASK) |
((new[15] <<
EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_SHIFT) &
EMC_DATA_BRLSHFT_1_RANK1_BYTE7_DATA_BRLSHFT_MASK);
break;
default:
break;
}
return temp;
}
static uint32_t update_clock_tree_delay(tegra_emc_timing_t* current_timing, tegra_emc_timing_t* next_timing, uint32_t dram_dev_num, uint32_t channel_mode, int type) {
uint32_t mrr_req = 0, mrr_data = 0;
uint32_t temp0_0 = 0, temp0_1 = 0, temp1_0 = 0, temp1_1 = 0;
int tdel = 0, tmdel = 0, adel = 0;
uint32_t cval;
uint32_t current_timing_rate_mhz = (current_timing->rate / 1000);
uint32_t next_timing_rate_mhz = (next_timing->rate / 1000);
bool dvfs_pt1 = (type == DVFS_PT1);
bool training_pt1 = (type == TRAINING_PT1);
bool dvfs_update = (type == DVFS_UPDATE);
bool training_update = (type == TRAINING_UPDATE);
bool periodic_training_update = (type == PERIODIC_TRAINING_UPDATE);
/* Dev0 MSB. */
if (dvfs_pt1 || training_pt1 || periodic_training_update) {
mrr_req = ((2 << EMC_MRR_DEV_SEL_SHIFT) | (19 << EMC_MRR_MA_SHIFT));
emc_write(mrr_req, EMC_MRR);
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC);
if (channel_mode == DUAL_CHANNEL)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC1);
mrr_data = ((emc_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp0_0 = ((mrr_data & 0xff) << 8);
temp0_1 = (mrr_data & 0xff00);
if (channel_mode == DUAL_CHANNEL) {
mrr_data = ((emc1_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp1_0 = ((mrr_data & 0xff) << 8);
temp1_1 = (mrr_data & 0xff00);
}
/* Dev0 LSB. */
mrr_req = ((mrr_req & ~EMC_MRR_MA_MASK) | (18 << EMC_MRR_MA_SHIFT));
emc_write(mrr_req, EMC_MRR);
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC);
if (channel_mode == DUAL_CHANNEL)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC1);
mrr_data = ((emc_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp0_0 |= (mrr_data & 0xff);
temp0_1 |= ((mrr_data & 0xff00) >> 8);
if (channel_mode == DUAL_CHANNEL) {
mrr_data = ((emc1_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp1_0 |= (mrr_data & 0xff);
temp1_1 |= ((mrr_data & 0xff00) >> 8);
}
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp0_0));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C0D0U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D0U0);
else if (training_update)
__AVERAGE_WRITE_PTFV(C0D0U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D0U0, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = (next_timing->current_dram_clktree_c0d0u0 - __MOVAVG_AC(next_timing, C0D0U0));
tmdel = (tdel < 0) ? -1 * tdel : tdel;
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c0d0u0 = __MOVAVG_AC(next_timing, C0D0U0);
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) /
(current_timing_rate_mhz * 2 * temp0_1));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C0D0U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D0U1);
else if (training_update)
__AVERAGE_WRITE_PTFV(C0D0U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D0U1, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c0d0u1 - __MOVAVG_AC(next_timing, C0D0U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c0d0u1 = __MOVAVG_AC(next_timing, C0D0U1);
}
if (channel_mode == DUAL_CHANNEL) {
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp1_0));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C1D0U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D0U0);
else if (training_update)
__AVERAGE_WRITE_PTFV(C1D0U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D0U0, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c1d0u0 - __MOVAVG_AC(next_timing, C1D0U0);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c1d0u0 = __MOVAVG_AC(next_timing, C1D0U0);
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp1_1));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C1D0U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D0U1);
else if (training_update)
__AVERAGE_WRITE_PTFV(C1D0U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D0U1, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c1d0u1 - __MOVAVG_AC(next_timing, C1D0U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c1d0u1 = __MOVAVG_AC(next_timing, C1D0U1);
}
}
if (dram_dev_num != TWO_RANK)
return adel;
/* Dev1 MSB. */
if (dvfs_pt1 || training_pt1 || periodic_training_update) {
mrr_req = ((1 << EMC_MRR_DEV_SEL_SHIFT) | (19 << EMC_MRR_MA_SHIFT));
emc_write(mrr_req, EMC_MRR);
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC);
if (channel_mode == DUAL_CHANNEL)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC1);
mrr_data = ((emc_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp0_0 = ((mrr_data & 0xff) << 8);
temp0_1 = (mrr_data & 0xff00);
if (channel_mode == DUAL_CHANNEL) {
mrr_data = ((emc1_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp1_0 = ((mrr_data & 0xff) << 8);
temp1_1 = (mrr_data & 0xff00);
}
/* Dev1 LSB. */
mrr_req = ((mrr_req & ~EMC_MRR_MA_MASK) | (18 << EMC_MRR_MA_SHIFT));
emc_write(mrr_req, EMC_MRR);
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC);
if (channel_mode == DUAL_CHANNEL)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_MRR_DIVLD, true, REG_EMC1);
mrr_data = ((emc_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp0_0 |= (mrr_data & 0xff);
temp0_1 |= ((mrr_data & 0xff00) >> 8);
if (channel_mode == DUAL_CHANNEL) {
mrr_data = ((emc1_read(EMC_MRR) & EMC_MRR_DATA_MASK) << EMC_MRR_DATA_SHIFT);
temp1_0 |= (mrr_data & 0xff);
temp1_1 |= ((mrr_data & 0xff00) >> 8);
}
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp0_0));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C0D1U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D1U0);
else if (training_update)
__AVERAGE_WRITE_PTFV(C0D1U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D1U0, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c0d1u0 - __MOVAVG_AC(next_timing, C0D1U0);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c0d1u0 = __MOVAVG_AC(next_timing, C0D1U0);
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp0_1));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C0D1U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C0D1U1);
else if (training_update)
__AVERAGE_WRITE_PTFV(C0D1U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C0D1U1, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c0d1u1 - __MOVAVG_AC(next_timing, C0D1U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c0d1u1 = __MOVAVG_AC(next_timing, C0D1U1);
}
if (channel_mode == DUAL_CHANNEL) {
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp1_0));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C1D1U0, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D1U0);
else if (training_update)
__AVERAGE_WRITE_PTFV(C1D1U0);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D1U0, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c1d1u0 - __MOVAVG_AC(next_timing, C1D1U0);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c1d1u0 = __MOVAVG_AC(next_timing, C1D1U0);
}
cval = ((1000000 * actual_osc_clocks(current_timing->run_clocks)) / (current_timing_rate_mhz * 2 * temp1_1));
if (dvfs_pt1 || training_pt1)
__INCREMENT_PTFV(C1D1U1, cval);
else if (dvfs_update)
__AVERAGE_PTFV(C1D1U1);
else if (training_update)
__AVERAGE_WRITE_PTFV(C1D1U1);
else if (periodic_training_update)
__WEIGHTED_UPDATE_PTFV(C1D1U1, cval);
if (dvfs_update || training_update || periodic_training_update) {
tdel = next_timing->current_dram_clktree_c1d1u1 - __MOVAVG_AC(next_timing, C1D1U1);
tmdel = (tdel < 0) ? -1 * tdel : tdel;
if (tmdel > adel)
adel = tmdel;
if ((tmdel * 128 * next_timing_rate_mhz / 1000000) > next_timing->tree_margin)
next_timing->current_dram_clktree_c1d1u1 = __MOVAVG_AC(next_timing, C1D1U1);
}
}
if (training_update) {
next_timing->trained_dram_clktree_c0d0u0 = next_timing->current_dram_clktree_c0d0u0;
next_timing->trained_dram_clktree_c0d0u1 = next_timing->current_dram_clktree_c0d0u1;
next_timing->trained_dram_clktree_c0d1u0 = next_timing->current_dram_clktree_c0d1u0;
next_timing->trained_dram_clktree_c0d1u1 = next_timing->current_dram_clktree_c0d1u1;
next_timing->trained_dram_clktree_c1d0u0 = next_timing->current_dram_clktree_c1d0u0;
next_timing->trained_dram_clktree_c1d0u1 = next_timing->current_dram_clktree_c1d0u1;
next_timing->trained_dram_clktree_c1d1u0 = next_timing->current_dram_clktree_c1d1u0;
next_timing->trained_dram_clktree_c1d1u1 = next_timing->current_dram_clktree_c1d1u1;
}
return adel;
}
static void reset_dram_clktree_values(tegra_emc_timing_t *table) {
#define __RESET_CLKTREE(TBL, C, D, U) \
TBL->current_dram_clktree_c ## C ## d ## D ## u ## U = \
TBL->trained_dram_clktree_c ## C ## d ## D ## u ## U
__RESET_CLKTREE(table, 0, 0, 0);
__RESET_CLKTREE(table, 0, 0, 1);
__RESET_CLKTREE(table, 1, 0, 0);
__RESET_CLKTREE(table, 1, 0, 1);
__RESET_CLKTREE(table, 1, 1, 0);
__RESET_CLKTREE(table, 1, 1, 1);
}
static uint32_t periodic_compensation_handler(tegra_emc_timing_t *current_timing, tegra_emc_timing_t *next_timing, uint32_t dram_dev_num, uint32_t channel_mode, int type) {
#define __COPY_EMA(nt, lt, dev) \
({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) * \
(nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; })
uint32_t adel = 0;
uint32_t samples = next_timing->ptfv_list[PTFV_DVFS_SAMPLES_INDEX];
uint32_t samples_write = next_timing->ptfv_list[PTFV_WRITE_SAMPLES_INDEX];
uint32_t delay = 2 + (1000 * actual_osc_clocks(current_timing->run_clocks) / current_timing->rate);
if (!next_timing->periodic_training)
return 0;
if (type == DVFS_SEQUENCE) {
if (current_timing->periodic_training && (next_timing->ptfv_list[PTFV_CONFIG_CTRL_INDEX] & PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) {
/*
* If the previous frequency was using periodic
* calibration then we can reuse the previous
* frequencies EMA data.
*/
__COPY_EMA(next_timing, current_timing, C0D0U0);
__COPY_EMA(next_timing, current_timing, C0D0U1);
__COPY_EMA(next_timing, current_timing, C1D0U0);
__COPY_EMA(next_timing, current_timing, C1D0U1);
__COPY_EMA(next_timing, current_timing, C0D1U0);
__COPY_EMA(next_timing, current_timing, C0D1U1);
__COPY_EMA(next_timing, current_timing, C1D1U0);
__COPY_EMA(next_timing, current_timing, C1D1U1);
} else {
/* Reset the EMA.*/
__MOVAVG(next_timing, C0D0U0) = 0;
__MOVAVG(next_timing, C0D0U1) = 0;
__MOVAVG(next_timing, C1D0U0) = 0;
__MOVAVG(next_timing, C1D0U1) = 0;
__MOVAVG(next_timing, C0D1U0) = 0;
__MOVAVG(next_timing, C0D1U1) = 0;
__MOVAVG(next_timing, C1D1U0) = 0;
__MOVAVG(next_timing, C1D1U1) = 0;
for (int i = 0; i < samples; i++) {
start_periodic_compensation();
udelay(delay);
/* Generate next sample of data. */
adel = update_clock_tree_delay(current_timing, next_timing, dram_dev_num, channel_mode, DVFS_PT1);
}
}
adel = update_clock_tree_delay(current_timing, next_timing, dram_dev_num, channel_mode, DVFS_UPDATE);
} else if (type == WRITE_TRAINING_SEQUENCE) {
/* Reset the EMA.*/
__MOVAVG(next_timing, C0D0U0) = 0;
__MOVAVG(next_timing, C0D0U1) = 0;
__MOVAVG(next_timing, C1D0U0) = 0;
__MOVAVG(next_timing, C1D0U1) = 0;
__MOVAVG(next_timing, C0D1U0) = 0;
__MOVAVG(next_timing, C0D1U1) = 0;
__MOVAVG(next_timing, C1D1U0) = 0;
__MOVAVG(next_timing, C1D1U1) = 0;
for (int i = 0; i < samples_write; i++) {
start_periodic_compensation();
udelay(delay);
/* Generate next sample of data. */
update_clock_tree_delay(current_timing, next_timing, dram_dev_num, channel_mode, TRAINING_PT1);
}
adel = update_clock_tree_delay(current_timing, next_timing, dram_dev_num, channel_mode, TRAINING_UPDATE);
} else if (type == PERIODIC_TRAINING_SEQUENCE) {
start_periodic_compensation();
udelay(delay);
adel = update_clock_tree_delay(current_timing, next_timing, dram_dev_num, channel_mode, PERIODIC_TRAINING_UPDATE);
}
return adel;
}
static void set_over_temp_timing(tegra_emc_timing_t *next_timing, unsigned long state) {
#define REFRESH_X2 1
#define REFRESH_X4 2
#define REFRESH_SPEEDUP(val, speedup) \
(val = ((val) & 0xFFFF0000) | (((val) & 0xFFFF) >> (speedup)))
uint32_t ref = next_timing->burst_regs[EMC_REFRESH_INDEX];
uint32_t pre_ref = next_timing->burst_regs[EMC_PRE_REFRESH_REQ_CNT_INDEX];
uint32_t dsr_cntrl = next_timing->burst_regs[EMC_DYN_SELF_REF_CONTROL_INDEX];
switch (state) {
case TEGRA_DRAM_OVER_TEMP_NONE:
case TEGRA_DRAM_OVER_TEMP_THROTTLE:
break;
case TEGRA_DRAM_OVER_TEMP_REFRESH_X2:
REFRESH_SPEEDUP(ref, REFRESH_X2);
REFRESH_SPEEDUP(pre_ref, REFRESH_X2);
REFRESH_SPEEDUP(dsr_cntrl, REFRESH_X2);
break;
case TEGRA_DRAM_OVER_TEMP_REFRESH_X4:
REFRESH_SPEEDUP(ref, REFRESH_X4);
REFRESH_SPEEDUP(pre_ref, REFRESH_X4);
REFRESH_SPEEDUP(dsr_cntrl, REFRESH_X4);
break;
default:
return;
}
emc_write(ref, burst_regs_off[EMC_REFRESH_INDEX]);
emc_write(pre_ref, burst_regs_off[EMC_PRE_REFRESH_REQ_CNT_INDEX]);
emc_write(dsr_cntrl, burst_regs_off[EMC_DYN_SELF_REF_CONTROL_INDEX]);
}
static void change_dll_src(tegra_emc_timing_t* next_timing, uint32_t clk_src_emc_to) {
volatile tegra_car_t *car = car_get_regs();
uint32_t emc_2x_clk_src_to = (clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
uint32_t val = (((((next_timing->dll_clk_src & 0x1FFFFFFF) | (emc_2x_clk_src_to << EMC_CLK_EMC_2X_CLK_SRC_SHIFT)) & 0xFFFFFF00) | (clk_src_emc_to & 0xFF)) & 0xFFFFF3FF);
/* Clock source is PLLMB_UD */
if (emc_2x_clk_src_to == TEGRA_EMC_SRC_PLLMB_UD)
val |= 0x400;
else if (emc_2x_clk_src_to != TEGRA_EMC_SRC_PLLM_UD) /* Clock source is not PLLM_UD */
val |= 0x800;
/* Set EMC_DLL_CLK_SRC, DDLL_CLK_SEL and EMC_DLL_CLK_DIVISOR */
car->clk_source_emc_dll = val;
/* Clear and set CLK_ENB_EMC_DLL */
uint32_t clk_enb_emc_dll = ((car->clk_out_enb_x & 0xFFFFBFFF) | ((next_timing->clk_out_enb_x_0_clk_enb_emc_dll & 1) << 14));
car->clk_out_enb_x = clk_enb_emc_dll;
}
static uint32_t dll_prelock(tegra_emc_timing_t* next_timing, bool dvfs_with_training, uint32_t clk_src_emc_to) {
/* Check for dual channel LPDDR4 */
bool dual_channel_lpddr4_case = ((emc_read(EMC_FBIO_CFG7) & EMC_FBIO_CFG7_CH0_ENABLE) & (emc_read(EMC_FBIO_CFG7) & EMC_FBIO_CFG7_CH1_ENABLE));
uint32_t emc_dig_dll_status = 0;
uint32_t emc_cfg_dig_dll = (emc_read(EMC_CFG_DIG_DLL) & ~EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_MASK);
emc_cfg_dig_dll |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_LOCK_LIMIT_SHIFT);
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK;
emc_cfg_dig_dll |= (3 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
emc_cfg_dig_dll |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
/* Update EMC_CFG_DIG_DLL_0 */
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait until CFG_DLL_EN is cleared for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
/* Wait until CFG_DLL_EN is cleared for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
}
/* Manual configuration (superseded). */
/*
uint32_t emc_dll_cfg_0 = emc_read(EMC_DLL_CFG_0);
emc_dll_cfg_0 &= 0xDF00000F;
emc_dll_cfg_0 |= 0x1FA340AF;
emc_write(emc_dll_cfg_0, EMC_DLL_CFG_0);
*/
emc_write(next_timing->burst_regs[EMC_DLL_CFG_0_INDEX], EMC_DLL_CFG_0);
emc_write(next_timing->burst_regs[EMC_DLL_CFG_1_INDEX], EMC_DLL_CFG_1);
/* Manual configuration (superseded). */
/*
uint32_t ddllcal_ctrl_start_trim_val = 0;
if ((next_timing->rate >= 400000) && (next_timing->rate < 600000))
ddllcal_ctrl_start_trim_val = 150;
else if ((next_timing->rate >= 600000) && (next_timing->rate < 800000))
ddllcal_ctrl_start_trim_val = 100;
else if ((next_timing->rate >= 800000) && (next_timing->rate < 1000000))
ddllcal_ctrl_start_trim_val = 70;
else if ((next_timing->rate >= 1000000) && (next_timing->rate < 1200000))
ddllcal_ctrl_start_trim_val = 30;
else
ddllcal_ctrl_start_trim_val = 20;
uint32_t emc_dll_cfg_1 = emc_read(EMC_DLL_CFG_1);
emc_dll_cfg_1 &= EMC_DLL_CFG_1_DDLLCAL_CTRL_START_TRIM_MASK;
emc_dll_cfg_1 |= ddllcal_ctrl_start_trim_val;
emc_write(emc_dll_cfg_1, EMC_DLL_CFG_1);
*/
/* Configure the clock and reset controller for EMC DLL */
change_dll_src(next_timing, clk_src_emc_to);
/* Enable DLL */
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait until CFG_DLL_EN is set for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
/* Wait until CFG_DLL_EN is set for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
}
/* Wait until DLL_PRIV_UPDATED or DLL_LOCK have been cleared */
do {
emc_dig_dll_status = emc_read(EMC_DIG_DLL_STATUS);
} while (!(emc_dig_dll_status & EMC_DIG_DLL_STATUS_DLL_LOCK) || !(emc_dig_dll_status & EMC_DIG_DLL_STATUS_DLL_PRIV_UPDATED));
if (dvfs_with_training) {
/* Set WRITE_MUX to ACTIVE */
emc_set_shadow_bypass(ACTIVE);
/* Disable DLL */
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Set WRITE_MUX to ASSEMBLY */
emc_set_shadow_bypass(ASSEMBLY);
/* Wait until CFG_DLL_EN is cleared for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
/* Wait until CFG_DLL_EN is cleared for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
}
}
/* Return the DLL_OUT value */
return (emc_read(EMC_DIG_DLL_STATUS) & EMC_DIG_DLL_STATUS_DLL_OUT_MASK);
}
static void dll_disable(bool dual_channel_lpddr4_case) {
/* Disable DLL */
uint32_t emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait until CFG_DLL_EN is cleared for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
/* Wait until CFG_DLL_EN is cleared for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN);
}
}
static void dll_enable(bool dual_channel_lpddr4_case) {
/* Enable DLL */
uint32_t emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait until CFG_DLL_EN is set for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
/* Wait until CFG_DLL_EN is set for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
}
}
static void dll_enable_stall(bool dual_channel_lpddr4_case) {
/* Enable DLL */
uint32_t emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
emc_cfg_dig_dll |= EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
emc_cfg_dig_dll &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
emc_cfg_dig_dll = (emc_cfg_dig_dll & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait until CFG_DLL_EN is set for EMC */
do {
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
/* Wait until CFG_DLL_EN is set for EMC1 */
if (dual_channel_lpddr4_case) {
do {
emc_cfg_dig_dll = emc1_read(EMC_CFG_DIG_DLL);
} while (!(emc_cfg_dig_dll & EMC_CFG_DIG_DLL_CFG_DLL_EN));
}
}
static bool test_clk_ratio(uint32_t rate_to, uint32_t clk_src_emc_to, uint32_t rate_from, uint32_t clk_src_emc_from) {
volatile tegra_car_t *car = car_get_regs();
uint32_t emc_2x_clk_src = (car->clk_source_emc >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
uint32_t post_div = 0;
if ((emc_2x_clk_src == TEGRA_EMC_SRC_PLLM) || (emc_2x_clk_src == TEGRA_EMC_SRC_PLLM_UD)) {
post_div = ((car->pllm_base >> 0x14) & 0x1F);
} else if ((emc_2x_clk_src == TEGRA_EMC_SRC_PLLMB_UD) || (emc_2x_clk_src == TEGRA_EMC_SRC_PLLMB)) {
post_div = ((car->pllmb_base >> 0x14) & 0x1F);
}
/* Bad post divider value */
if (post_div > 0x05)
return false;
uint32_t emc_2x_clk_src_from = (clk_src_emc_from >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
uint32_t emc_2x_clk_src_to = (clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
uint8_t emc_2x_clk_div_from = (clk_src_emc_from & 0xFF);
uint8_t emc_2x_clk_div_to = (clk_src_emc_to & 0xFF);
if (emc_2x_clk_src_from <= TEGRA_EMC_SRC_PLLMB_UD)
emc_2x_clk_div_from = 0;
if (emc_2x_clk_src_to <= TEGRA_EMC_SRC_PLLMB_UD)
emc_2x_clk_div_to = 0;
/* Clock sources are different and one of them is CLK_M */
if ((emc_2x_clk_src_to != emc_2x_clk_src_from)
&& ((emc_2x_clk_src_to & 0xFFFFFFFB)
|| (emc_2x_clk_src_from & 0xFFFFFFFB)))
return true;
float val_to = (double)rate_to * ((double)((emc_2x_clk_div_to >> 1) + 1) + (double)(emc_2x_clk_div_to & 1) * 0.5) * (double)(post_div + 1);
float val_from = (double)rate_from * ((double)((emc_2x_clk_div_from >> 1) + 1) + (double)(emc_2x_clk_div_from & 1) * 0.5) * (double)(post_div + 1);
float ratio = (val_from / val_to);
if ((ratio > 1.01f) || (ratio < 0.99f))
return true;
return false;
}
static uint32_t set_pll(uint32_t rate_to, uint32_t rate_osc, uint32_t clk_src_emc_to, bool is_pllmb) {
volatile tegra_car_t *car = car_get_regs();
static const pll_cfg_t pll_vals[] = {
{0xB71B00, 0x2FAF0800, 0x42, 0x01, 0x00}, /* 800Mhz rate with 12Mhz oscillator (unsupported). */
{0xC65D40, 0x2FAF0800, 0x3D, 0x01, 0x00}, /* 800Mhz rate with 13Mhz oscillator (unsupported). */
{0x249F000, 0x11BD0400, 0x5D, 0x04, 0x02}, /* 297.6Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x17D78400, 0x7D, 0x04, 0x02}, /* 400Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x18519600, 0x55, 0x04, 0x01}, /* 408Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x1FC1E200, 0x6F, 0x04, 0x01}, /* 532.8Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x27AC4000, 0x68, 0x03, 0x01}, /* 665.6Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x2FAF0800, 0x7D, 0x03, 0x01}, /* 800Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x3780FC00, 0x61, 0x04, 0x00}, /* 931.2Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x3F83C400, 0x6F, 0x04, 0x00}, /* 1065.6Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x47868C00, 0x7D, 0x04, 0x00}, /* 1200Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x4F588000, 0x68, 0x03, 0x00}, /* 1331.2Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x56F9A000, 0x4C, 0x02, 0x00}, /* 1459.2Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x5F5E1000, 0x7D, 0x03, 0x00}, /* 1600Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x6F01F800, 0x61, 0x02, 0x00}, /* 1862.4Mhz rate with 38.4Mhz oscillator. */
{0x249F000, 0x7F078800, 0x6F, 0x02, 0x00}, /* 2131.2Mhz rate with 38.4Mhz oscillator. */
{0x00, 0x00, 0x00, 0x00, 0x00} /* Dummy entry. */
};
uint32_t rate_to_hz = (rate_to * 1000);
uint32_t rate_osc_hz = (rate_osc * 1000);
int freq_idx = 0;
for (int i = 0; pll_vals[i].osc_freq; i++) {
if ((rate_osc_hz == pll_vals[i].osc_freq) && (rate_to_hz == pll_vals[i].out_freq)) {
freq_idx = i;
break;
}
}
uint32_t res = clk_src_emc_to;
/* Failed to find the PLL values */
if (!pll_vals[freq_idx].osc_freq)
return res;
uint32_t feedback_div = pll_vals[freq_idx].feedback_div;
uint32_t input_div = pll_vals[freq_idx].input_div;
uint32_t post_div = pll_vals[freq_idx].post_div;
if (is_pllmb) {
/* Set PLLMB_DIVM, PLLMB_DIVN and PLLMB_DIVP */
car->pllmb_base = (input_div | (feedback_div << 0x08) | ((post_div & 0x1F) << 0x14));
/* Set PLLMB_ENABLE */
car->pllmb_base |= 0x40000000;
/* Clock source is PLLM_UD */
if ((clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT) == TEGRA_EMC_SRC_PLLM_UD)
res = ((clk_src_emc_to & 0x1FFFFFFF) | (TEGRA_EMC_SRC_PLLMB_UD << EMC_CLK_EMC_2X_CLK_SRC_SHIFT));
else if ((clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT) == TEGRA_EMC_SRC_PLLM) /* Clock source is PLLM_OUT0 */
res = (clk_src_emc_to | (TEGRA_EMC_SRC_PLLMB << EMC_CLK_EMC_2X_CLK_SRC_SHIFT));
while (!(car->pllmb_base & 0x8000000)) {
/* Wait for PLLMB_LOCK to be set */
}
} else {
/* Set PLLM_DIVM, PLLM_DIVN and PLLM_DIVP */
car->pllm_base = (input_div | (feedback_div << 0x08) | ((post_div & 0x1F) << 0x14));
/* Set PLLM_EN_LCKDET */
car->pllm_misc2 |= 0x10;
/* Set PLLM_ENABLE */
car->pllm_base |= 0x40000000;
/* Clock source is PLLM_UD */
if ((clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT) == TEGRA_EMC_SRC_PLLM_UD)
res = ((clk_src_emc_to & 0x1FFFFFFF) | (TEGRA_EMC_SRC_PLLM_UD << EMC_CLK_EMC_2X_CLK_SRC_SHIFT));
while (!(car->pllm_base & 0x8000000)) {
/* Wait for PLLM_LOCK to be set */
}
}
return res;
}
static void set_clock(tegra_emc_timing_t* current_timing, tegra_emc_timing_t* next_timing, uint32_t training, uint32_t next_clk_src) {
volatile tegra_car_t *car = car_get_regs();
/* Extract training values */
bool train_ca = (training & 0x01);
bool train_ca_vref = (training & 0x02);
bool train_quse = (training & 0x04);
bool train_quse_vref = (training & 0x08);
bool train_wr = (training & 0x10);
bool train_wr_vref = (training & 0x20);
bool train_rd = (training & 0x40);
bool train_rd_vref = (training & 0x80);
bool train_swap_rank = (training & 0x100);
bool train_self_refresh = (training & 0x200);
/* Check if we should do training. */
bool dvfs_with_training = (training & 0xF7);
bool skip_zqcal = false;
bool compensate_trimmer_applicable = false;
uint32_t zqcal_before_cc_cutoff = 2400; /* In picoseconds */
int zq_latch_dvfs_wait_time;
uint32_t mr13_catr_enable;
uint32_t mr13_flip_fspwr;
uint32_t mr13_flip_fspop;
int next_push, next_dq_e_ivref, next_dqs_e_ivref;
uint32_t zq_wait_long;
uint32_t zq_wait_short;
uint32_t tRTM;
uint32_t RP_war;
uint32_t R2P_war;
uint32_t TRPab_war;
int nRTP;
uint32_t deltaTWATM;
uint32_t W2P_war;
uint32_t tRPST;
uint32_t mrw_req;
uint32_t adel = 0;
uint32_t next_timing_rate_mhz = next_timing->rate / 1000;
/* Set some common values needed. */
int dram_type = emc_read(EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK >> EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;
int dram_dev_num = ((mc_read(MC_EMEM_ADR_CFG) & 1) + 1);
bool shared_zq_resistor = ((current_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] >> 31) & 1);
bool channel_mode = ((current_timing->burst_regs[EMC_FBIO_CFG7_INDEX] >> 2) & 1);
bool is_lpddr3 = (dram_type == DRAM_TYPE_LPDDR2) && ((next_timing->burst_regs[EMC_FBIO_CFG5_INDEX] >> 25) & 1);
bool opt_zcal_en_cc = ((next_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX] && !current_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX]) || (dram_type == DRAM_TYPE_LPDDR4));
bool opt_war_200024907 = (dram_type == DRAM_TYPE_LPDDR4);
bool opt_do_sw_qrst = false;
bool opt_cc_short_zcal = true;
bool opt_short_zcal = true;
bool save_restore_clkstop_pd = true;
uint32_t opt_dll_mode = (dram_type == DRAM_TYPE_DDR3) ? get_dll_state(next_timing) : DLL_OFF;
uint32_t opt_dvfs_mode = MAN_SR;
uint32_t emc_auto_cal_config = emc_read(EMC_AUTO_CAL_CONFIG);
/* In picoseconds. */
uint32_t source_clock_period = 1000000000 / current_timing->rate;
uint32_t destination_clock_period = 1000000000 / next_timing->rate;
uint32_t tFC_lpddr4 = 1000 * next_timing->dram_timings[T_FC_LPDDR4];
uint32_t tZQCAL_lpddr4 = 1000000;
int tZQCAL_lpddr4_fc_adj = (source_clock_period > zqcal_before_cc_cutoff) ? tZQCAL_lpddr4 / destination_clock_period : (tZQCAL_lpddr4 - tFC_lpddr4) / destination_clock_period;
g_fsp_for_next_freq = !g_fsp_for_next_freq;
uint32_t emc_dbg_o = emc_read(EMC_DBG);
uint32_t emc_pin_o = emc_read(EMC_PIN);
uint32_t emc_cfg_pipe_clk_o = emc_read(EMC_CFG_PIPE_CLK);
uint32_t emc_dbg = emc_dbg_o;
uint32_t emc_cfg = next_timing->burst_regs[EMC_CFG_INDEX];
uint32_t emc_sel_dpd_ctrl = next_timing->emc_sel_dpd_ctrl;
emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD | EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD);
emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN | EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN | EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN | EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN | EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN);
/* Step 1:
* Pre DVFS SW sequence.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 1\n");
/* Step 1.1: Disable DLL. */
uint32_t tmp = emc_read(EMC_CFG_DIG_DLL);
tmp &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
emc_write(tmp, EMC_CFG_DIG_DLL);
/* Request a timing update. */
emc_timing_update(channel_mode);
/* Wait for DLL to be disabled. */
wait_for_update(EMC_CFG_DIG_DLL, EMC_CFG_DIG_DLL_CFG_DLL_EN, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_CFG_DIG_DLL, EMC_CFG_DIG_DLL_CFG_DLL_EN, false, REG_EMC1);
/* Step 1.2: Disable AUTOCAL. */
emc_auto_cal_config = next_timing->emc_auto_cal_config;
uint32_t auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE;
emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START;
emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL;
emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL;
emc_auto_cal_config |= auto_cal_en;
emc_write(emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
/* Step 1.3: Disable other power features. */
emc_set_shadow_bypass(ACTIVE);
emc_write(emc_cfg, EMC_CFG);
emc_write(emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
emc_set_shadow_bypass(ASSEMBLY);
/* Skip this if dvfs_with_training is set. */
if (!dvfs_with_training && next_timing->periodic_training) {
if (dram_dev_num == TWO_RANK) {
/* Wait for DRAM to get out of power down. */
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, false, REG_EMC1);
} else {
wait_for_update(EMC_EMC_STATUS, 0x10, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, 0x10, false, REG_EMC1);
}
/* Wait for DRAM to get out of self refresh. */
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, false, REG_EMC1);
/* Reset all clock tree values. */
reset_dram_clktree_values(next_timing);
/* Do DVFS_SEQUENCE. */
adel = periodic_compensation_handler(current_timing, next_timing, dram_dev_num, channel_mode, DVFS_SEQUENCE);
/* Check if we should use compensate trimmer. */
compensate_trimmer_applicable = next_timing->periodic_training && ((adel * 128 * next_timing_rate_mhz) / 1000000) > next_timing->tree_margin;
}
emc_write(EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS);
emc_set_shadow_bypass(ACTIVE);
emc_write(emc_cfg, EMC_CFG);
emc_write(emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
emc_write(emc_cfg_pipe_clk_o | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON, EMC_CFG_PIPE_CLK);
emc_write(next_timing->emc_fdpd_ctrl_cmd_no_ramp & ~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE, EMC_FDPD_CTRL_CMD_NO_RAMP);
uint32_t bg_regulator_mode_change = ((next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^ (current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) || ((next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] &
EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^ (current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD));
uint32_t enable_bg_regulator = (next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0;
/* Check if we need to change BG the regulator. */
if (bg_regulator_mode_change) {
if (enable_bg_regulator)
emc_write(current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, EMC_PMACRO_BG_BIAS_CTRL_0);
else
emc_write(current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, EMC_PMACRO_BG_BIAS_CTRL_0);
}
/* Check if we need to turn on VREF generator. */
if ((((!current_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF)) &&
((next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF))) ||
((!(current_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF)) &&
((next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] &
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF))))
{
uint32_t pad_tx_ctrl = next_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
uint32_t last_pad_tx_ctrl = current_timing->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX];
next_dqs_e_ivref = pad_tx_ctrl & EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF;
next_dq_e_ivref = pad_tx_ctrl & EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF;
next_push = (last_pad_tx_ctrl & ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF & ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) | next_dq_e_ivref | next_dqs_e_ivref;
emc_write(next_push, EMC_PMACRO_DATA_PAD_TX_CTRL);
udelay(1);
} else if (bg_regulator_mode_change) {
udelay(1);
}
emc_set_shadow_bypass(ASSEMBLY);
/* Step 2:
* Prelock the DLL.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 2\n");
if (next_timing->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) {
dll_prelock(next_timing, dvfs_with_training, next_clk_src);
} else {
change_dll_src(next_timing, next_clk_src);
dll_disable(channel_mode);
}
/* Step 3:
* Prepare autocal for the clock change.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 3\n");
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2);
emc_write(next_timing->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3);
emc_write(next_timing->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4);
emc_write(next_timing->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5);
emc_write(next_timing->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6);
emc_write(next_timing->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7);
emc_write(next_timing->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8);
emc_set_shadow_bypass(ASSEMBLY);
emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START | auto_cal_en);
emc_write(emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
/* Step 4:
* Update EMC_CFG.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 4\n");
if ((source_clock_period > 50000) && (dram_type == DRAM_TYPE_LPDDR4))
ccfifo_write(EMC_SELF_REF, 1, 0);
else
emc_write(next_timing->emc_cfg_2, EMC_CFG_2);
/* Step 5:
* Prepare reference variables for ZQCAL regs.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 5\n");
uint32_t emc_zcal_interval = current_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX];
emc_zcal_interval &= 0xFF000000;
uint32_t emc_zcal_wait_cnt_old = current_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
uint32_t emc_zcal_wait_cnt_new = next_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX];
emc_zcal_wait_cnt_old &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
emc_zcal_wait_cnt_new &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK;
if (dram_type == DRAM_TYPE_LPDDR4)
zq_wait_long = max((uint32_t)1, div_o3(1000000, destination_clock_period));
else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
zq_wait_long = max(next_timing->min_mrs_wait, div_o3(360000, destination_clock_period)) + 4;
else if (dram_type == DRAM_TYPE_DDR3)
zq_wait_long = max((uint32_t)256, div_o3(320000, destination_clock_period) + 2);
else
zq_wait_long = 0;
if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3)
zq_wait_short = max(max(next_timing->min_mrs_wait, (uint32_t)6), div_o3(90000, destination_clock_period)) + 4;
else if (dram_type == DRAM_TYPE_DDR3)
zq_wait_short = max((uint32_t)64, div_o3(80000, destination_clock_period)) + 2;
else
zq_wait_short = 0;
/* TODO: Actually use the reference variables. */
(void)zq_wait_long;
(void)zq_wait_short;
/* Step 6:
* Training code.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 6\n");
if ((train_ca || train_ca_vref) && (dram_dev_num == TWO_RANK)) {
emc_write(0x107, EMC_PIN);
}
/* Step 7:
* Program FSP reference registers and send MRWs to new FSPWR.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 7\n");
/* Step 7.1: Bug 200024907 - Patch RP R2P */
if (opt_war_200024907) {
nRTP = 16;
if (source_clock_period >= 1000000/1866) /* 535.91 ps */
nRTP = 14;
if (source_clock_period >= 1000000/1600) /* 625.00 ps */
nRTP = 12;
if (source_clock_period >= 1000000/1333) /* 750.19 ps */
nRTP = 10;
if (source_clock_period >= 1000000/1066) /* 938.09 ps */
nRTP = 8;
deltaTWATM = max_t(uint32_t, div_o3(7500, source_clock_period), 8);
/*
* Originally there was a + .5 in the tRPST calculation.
* However since we can't do FP in the kernel and the tRTM
* computation was in a floating point ceiling function, adding
* one to tRTP should be ok. There is no other source of non
* integer values, so the result was always going to be
* something for the form: f_ceil(N + .5) = N + 1;
*/
tRPST = ((current_timing->emc_mrw & 0x80) >> 7);
tRTM = current_timing->dram_timings[RL] + div_o3(3600, source_clock_period) + max_t(uint32_t, div_o3(7500, source_clock_period), 8) + tRPST + 1 + nRTP;
if (current_timing->burst_regs[EMC_RP_INDEX] < tRTM) {
if (tRTM > (current_timing->burst_regs[EMC_R2P_INDEX] + current_timing->burst_regs[EMC_RP_INDEX])) {
R2P_war = tRTM - current_timing->burst_regs[EMC_RP_INDEX];
RP_war = current_timing->burst_regs[EMC_RP_INDEX];
TRPab_war = current_timing->burst_regs[EMC_TRPAB_INDEX];
if (R2P_war > 63) {
RP_war = R2P_war + current_timing->burst_regs[EMC_RP_INDEX] - 63;
if (TRPab_war < RP_war)
TRPab_war = RP_war;
R2P_war = 63;
}
} else {
R2P_war = current_timing-> burst_regs[EMC_R2P_INDEX];
RP_war = current_timing->burst_regs[EMC_RP_INDEX];
TRPab_war = current_timing->burst_regs[EMC_TRPAB_INDEX];
}
if (RP_war < deltaTWATM) {
W2P_war = current_timing->burst_regs[EMC_W2P_INDEX] + deltaTWATM - RP_war;
if (W2P_war > 63) {
RP_war = RP_war + W2P_war - 63;
if (TRPab_war < RP_war)
TRPab_war = RP_war;
W2P_war = 63;
}
} else {
W2P_war = current_timing->burst_regs[EMC_W2P_INDEX];
}
if ((current_timing->burst_regs[EMC_W2P_INDEX] != W2P_war)
|| (current_timing->burst_regs[EMC_R2P_INDEX] != R2P_war)
|| (current_timing->burst_regs[EMC_RP_INDEX] != RP_war)
|| (current_timing->burst_regs[EMC_TRPAB_INDEX] != TRPab_war))
{
emc_set_shadow_bypass(ACTIVE);
emc_write(RP_war, EMC_RP);
emc_write(R2P_war, EMC_R2P);
emc_write(W2P_war, EMC_W2P);
emc_write(TRPab_war, EMC_TRPAB);
emc_set_shadow_bypass(ASSEMBLY);
udelay(1);
}
}
}
if (!g_fsp_for_next_freq) {
mr13_flip_fspwr = (next_timing->emc_mrw3 & 0xffffff3f) | 0x80;
mr13_flip_fspop = (next_timing->emc_mrw3 & 0xffffff3f) | 0x00;
} else {
mr13_flip_fspwr = (next_timing->emc_mrw3 & 0xffffff3f) | 0x40;
mr13_flip_fspop = (next_timing->emc_mrw3 & 0xffffff3f) | 0xc0;
}
mr13_catr_enable = (mr13_flip_fspwr & 0xFFFFFFFE) | 0x01;
if (dram_dev_num == TWO_RANK) {
if (train_ca || train_ca_vref) {
if (train_swap_rank) {
mr13_flip_fspop = (mr13_flip_fspop & 0x3FFFFFFF) | 0x80000000;
mr13_catr_enable = (mr13_catr_enable & 0x3FFFFFFF)| 0x40000000;
} else {
mr13_flip_fspop = (mr13_flip_fspop & 0x3FFFFFFF) | 0x40000000;
mr13_catr_enable = (mr13_catr_enable & 0x3FFFFFFF) | 0x80000000;
}
} else {
if (train_swap_rank)
mr13_catr_enable = (mr13_catr_enable & 0x3FFFFFFF) | 0x40000000;
else
mr13_catr_enable = (mr13_catr_enable & 0x3FFFFFFF) | 0x80000000;
}
}
if (dram_type == DRAM_TYPE_LPDDR4) {
emc_write(mr13_flip_fspwr, EMC_MRW3);
emc_write(next_timing->emc_mrw, EMC_MRW);
emc_write(next_timing->emc_mrw2, EMC_MRW2);
}
/* Step 8:
* Program the shadow registers.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 8\n");
/* Set burst registers. */
for (int i = 0; i < next_timing->num_burst; i++) {
uint32_t var = 0;
uint32_t wval = 0;
if (!burst_regs_off[i])
continue;
var = burst_regs_off[i];
if (dvfs_with_training) {
if (train_ca || train_ca_vref)
wval = next_timing->shadow_regs_ca_train[i];
else if (train_quse || train_quse_vref)
wval = next_timing->shadow_regs_quse_train[i];
else if (train_wr || train_wr_vref || train_rd || train_rd_vref)
wval = next_timing->shadow_regs_rdwr_train[i];
}
else
wval = next_timing->burst_regs[i];
if (dram_type != DRAM_TYPE_LPDDR4 &&
(var == EMC_MRW6 || var == EMC_MRW7 ||
var == EMC_MRW8 || var == EMC_MRW9 ||
var == EMC_MRW10 || var == EMC_MRW11 ||
var == EMC_MRW12 || var == EMC_MRW13 ||
var == EMC_MRW14 || var == EMC_MRW15 ||
var == EMC_TRAINING_CTRL))
continue;
if (var == EMC_CFG) {
wval &= ~EMC_CFG_DRAM_ACPD;
wval &= ~EMC_CFG_DYN_SELF_REF;
if (dram_type == DRAM_TYPE_LPDDR4) {
wval &= ~EMC_CFG_DRAM_CLKSTOP_SR;
wval &= ~EMC_CFG_DRAM_CLKSTOP_PD;
}
} else if ((var == EMC_MRS_WAIT_CNT)
&& (dram_type == DRAM_TYPE_LPDDR2)
&& opt_zcal_en_cc && !opt_cc_short_zcal && opt_short_zcal) {
wval = (wval & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK <<
EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) |
((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) <<
EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
} else if ((var == EMC_ZCAL_WAIT_CNT)
&& (dram_type == DRAM_TYPE_DDR3)
&& opt_zcal_en_cc && !opt_cc_short_zcal && opt_short_zcal) {
wval = (wval & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK <<
EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) |
((zq_wait_long &
EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) <<
EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT);
} else if ((var == EMC_ZCAL_INTERVAL) && opt_zcal_en_cc) {
wval = 0; /* EMC_ZCAL_INTERVAL reset value. */
} else if (var == EMC_PMACRO_AUTOCAL_CFG_COMMON) {
wval |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS;
} else if (var == EMC_PMACRO_DATA_PAD_TX_CTRL) {
wval &=
~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC |
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC |
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC |
EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC);
} else if (var == EMC_PMACRO_CMD_PAD_TX_CTRL) {
wval |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON;
wval &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC |
EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC |
EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC |
EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC);
} else if (var == EMC_PMACRO_BRICK_CTRL_RFU1) {
wval &= 0xf800f800;
} else if (var == EMC_PMACRO_COMMON_PAD_TX_CTRL) {
wval &= 0xfffffff0;
} else if (var == EMC_TRAINING_CTRL) {
wval |= (train_swap_rank << 14); /* Training only. */
}
emc_write(wval, var);
}
/* Do EMC refresh adjustment here (disabled). */
set_over_temp_timing(next_timing, TEGRA_DRAM_OVER_TEMP_NONE);
if (dram_type == DRAM_TYPE_LPDDR4) {
/* Use the current timing when training. */
if (dvfs_with_training)
mrw_req = (23 << EMC_MRW_MRW_MA_SHIFT) | (current_timing->run_clocks & EMC_MRW_MRW_OP_MASK);
else
mrw_req = (23 << EMC_MRW_MRW_MA_SHIFT) | (next_timing->run_clocks & EMC_MRW_MRW_OP_MASK);
emc_write(mrw_req, EMC_MRW);
}
/* Per channel burst registers. */
for (int i = 0; i < next_timing->num_burst_per_ch; i++) {
if (!burst_regs_per_ch_off[i])
continue;
if (dram_type != DRAM_TYPE_LPDDR4 &&
(burst_regs_per_ch_off[i] == EMC_MRW6 ||
burst_regs_per_ch_off[i] == EMC_MRW7 ||
burst_regs_per_ch_off[i] == EMC_MRW8 ||
burst_regs_per_ch_off[i] == EMC_MRW9 ||
burst_regs_per_ch_off[i] == EMC_MRW10 ||
burst_regs_per_ch_off[i] == EMC_MRW11 ||
burst_regs_per_ch_off[i] == EMC_MRW12 ||
burst_regs_per_ch_off[i] == EMC_MRW13 ||
burst_regs_per_ch_off[i] == EMC_MRW14 ||
burst_regs_per_ch_off[i] == EMC_MRW15))
continue;
/* Filter out second channel if not in DUAL_CHANNEL mode. */
if ((channel_mode != DUAL_CHANNEL) && (burst_regs_per_ch_type[i] >= REG_EMC1))
continue;
emc_write_per_ch(next_timing->burst_reg_per_ch[i], burst_regs_per_ch_type[i], burst_regs_per_ch_off[i]);
}
/* Vref regs. */
for (int i = 0; i < next_timing->vref_num; i++) {
if (!vref_regs_per_ch_off[i])
continue;
if ((channel_mode != DUAL_CHANNEL) && (vref_regs_per_ch_type[i] >= REG_EMC1))
continue;
emc_write_per_ch(next_timing->vref_perch_regs[i], vref_regs_per_ch_type[i], vref_regs_per_ch_off[i]);
}
/* Training regs. */
if (dvfs_with_training) {
for (int i = 0; i < next_timing->training_mod_num; i++) {
if (!training_mod_regs_per_ch_off[i])
continue;
if ((channel_mode != DUAL_CHANNEL) && (training_mod_regs_per_ch_type[i] >= REG_EMC1))
continue;
emc_write_per_ch(next_timing->training_mod_regs[i], training_mod_regs_per_ch_type[i], training_mod_regs_per_ch_off[i]);
}
}
/* Trimmers. */
for (int i = 0; i < next_timing->num_trim; i++) {
uint32_t trim_reg;
if (!trim_regs_off[i])
continue;
trim_reg = trim_regs_off[i];
if (compensate_trimmer_applicable &&
(trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
trim_reg == EMC_DATA_BRLSHFT_0 ||
trim_reg == EMC_DATA_BRLSHFT_1)) {
uint32_t reg = apply_periodic_compensation_trimmer(next_timing, trim_reg);
emc_write(reg, trim_regs_off[i]);
} else {
emc_write(next_timing->trim_regs[i], trim_regs_off[i]);
}
}
/* Per channel trimmers. */
for (int i = 0; i < next_timing->num_trim_per_ch; i++) {
uint32_t trim_reg;
if (!trim_regs_per_ch_off[i])
continue;
if ((channel_mode != DUAL_CHANNEL) && (trim_regs_per_ch_type[i] >= REG_EMC1))
continue;
trim_reg = trim_regs_per_ch_off[i];
if (compensate_trimmer_applicable &&
(trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 ||
trim_reg == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 ||
trim_reg == EMC_DATA_BRLSHFT_0 ||
trim_reg == EMC_DATA_BRLSHFT_1)) {
uint32_t reg = apply_periodic_compensation_trimmer(next_timing, trim_reg);
emc_write_per_ch(reg, trim_regs_per_ch_type[i], trim_regs_per_ch_off[i]);
} else {
emc_write_per_ch(next_timing->trim_perch_regs[i], trim_regs_per_ch_type[i], trim_regs_per_ch_off[i]);
}
}
if (dvfs_with_training) {
if (train_wr && next_timing->periodic_training && (dram_type == DRAM_TYPE_LPDDR4)) {
periodic_compensation_handler(current_timing, next_timing, dram_dev_num, channel_mode, WRITE_TRAINING_SEQUENCE);
}
} else {
/* Write burst_mc_regs. */
for (int i = 0; i < next_timing->num_mc_regs; i++) {
mc_write(next_timing->burst_mc_regs[i], burst_mc_regs_off[i]);
}
}
/* Registers to be programmed on the faster clock. */
if (!dvfs_with_training && (next_timing->rate < current_timing->rate)) {
for (int i = 0; i < next_timing->num_up_down; i++) {
mc_write(next_timing->la_scale_regs[i], la_scale_regs_off[i]);
}
}
/* Step 9:
* LPDDR4 section A.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 9\n");
if (dram_type == DRAM_TYPE_LPDDR4) {
emc_write(emc_zcal_interval, EMC_ZCAL_INTERVAL);
emc_write(emc_zcal_wait_cnt_new, EMC_ZCAL_WAIT_CNT);
emc_write(emc_dbg_o | (EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY), EMC_DBG);
emc_write(emc_zcal_interval, EMC_ZCAL_INTERVAL);
emc_write(emc_dbg_o, EMC_DBG);
if (dvfs_with_training) {
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX] | EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS, EMC_PMACRO_AUTOCAL_CFG_COMMON);
if (train_ca || train_ca_vref)
emc_write(current_timing->burst_regs[EMC_FBIO_CFG5_INDEX] | EMC_FBIO_CFG5_CMD_BUS_RETURN_TO_ZERO, EMC_FBIO_CFG5);
emc_set_shadow_bypass(ASSEMBLY);
if (channel_mode)
ccfifo_write(EMC_CFG_SYNC, 0, 0);
/* Change CFG_SWAP. */
ccfifo_write(EMC_DBG, ((emc_dbg_o & 0xF3FFFFFF) | 0x4000000), 0);
}
}
/* Step 10:
* LPDDR4 and DDR3 common section.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 10\n");
if (opt_dvfs_mode == MAN_SR || dram_type == DRAM_TYPE_LPDDR4) {
if (dram_type == DRAM_TYPE_LPDDR4)
ccfifo_write(EMC_SELF_REF, 0x101, 0);
else
ccfifo_write(EMC_SELF_REF, 0x1, 0);
if (!(train_ca || train_ca_vref) && (dram_type == DRAM_TYPE_LPDDR4) && (source_clock_period <= zqcal_before_cc_cutoff)) {
ccfifo_write(EMC_MRW3, mr13_flip_fspwr ^ 0x40, 0);
ccfifo_write(EMC_MRW6, (next_timing->burst_regs[EMC_MRW6_INDEX] & 0xFFFF3F3F) | (current_timing->burst_regs[EMC_MRW6_INDEX] & 0x0000C0C0), 0);
ccfifo_write(EMC_MRW14, (next_timing->burst_regs[EMC_MRW14_INDEX] & 0xFFFF0707) | (current_timing->burst_regs[EMC_MRW14_INDEX] & 0x00003838), 0);
if (dram_dev_num == TWO_RANK) {
ccfifo_write(EMC_MRW7, (next_timing->burst_regs[EMC_MRW7_INDEX] & 0xFFFF3F3F) | (current_timing->burst_regs[EMC_MRW7_INDEX] & 0x0000C0C0), 0);
ccfifo_write(EMC_MRW15, (next_timing->burst_regs[EMC_MRW15_INDEX] & 0xFFFF0707) | (current_timing->burst_regs[EMC_MRW15_INDEX] & 0x00003838), 0);
}
if (opt_zcal_en_cc) {
if ((dram_dev_num == ONE_RANK) || shared_zq_resistor)
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
else
ccfifo_write(EMC_ZQ_CAL, EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
}
}
}
emc_dbg = emc_dbg_o;
if (dram_type == DRAM_TYPE_LPDDR4) {
if (dvfs_with_training) {
/* Change CFG_SWAP. */
emc_dbg = ((emc_dbg_o & 0xF3FFFFFF) | 0x4000000 | EMC_DBG_WRITE_ACTIVE_ONLY);
ccfifo_write(EMC_DBG, emc_dbg, 0);
}
if (train_ca || train_ca_vref) {
ccfifo_write(EMC_PMACRO_DATA_RX_TERM_MODE, current_timing->burst_regs[EMC_PMACRO_DATA_RX_TERM_MODE_INDEX] & 0xFFFFFCCC, 0);
if ((dram_dev_num == TWO_RANK) && train_swap_rank) {
ccfifo_write(EMC_MRW3, mr13_flip_fspop | 0x8, (1000 * current_timing->dram_timings[T_RP]) / source_clock_period);
ccfifo_write(EMC_MRW3, mr13_catr_enable | 0x8, 0);
} else {
ccfifo_write(EMC_MRW3, mr13_catr_enable | 0x8, (1000 * current_timing->dram_timings[T_RP]) / source_clock_period);
}
ccfifo_write(EMC_TR_CTRL_0, 0x15A, 0);
ccfifo_write(EMC_INTSTATUS, 0, 1000000 / source_clock_period);
} else {
ccfifo_write(EMC_MRW3, mr13_flip_fspop | 0x8, (1000 * current_timing->dram_timings[T_RP]) / source_clock_period);
ccfifo_write(EMC_INTSTATUS, 0, tFC_lpddr4 / source_clock_period);
}
}
bool ref_b4_sref_en = false;
bool cya_issue_pc_ref = false;
bool cya_allow_ref_cc = false;
if ((dram_type == DRAM_TYPE_LPDDR4) || (opt_dvfs_mode != MAN_SR)) {
uint32_t t = 30 + (cya_allow_ref_cc ? (4000 * current_timing->dram_timings[T_RFC]) + ((1000 * current_timing->dram_timings[T_RP]) / source_clock_period) : 0);
ccfifo_write(EMC_PIN, emc_pin_o & ~(EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE), t);
}
uint32_t ref_delay_mult = 1;
ref_delay_mult += ref_b4_sref_en ? 1 : 0;
ref_delay_mult += cya_allow_ref_cc ? 1 : 0;
ref_delay_mult += cya_issue_pc_ref ? 1 : 0;
uint32_t ref_delay = ref_delay_mult * ((1000 * current_timing->dram_timings[T_RP] / source_clock_period) + (1000 * current_timing->dram_timings[T_RFC] / source_clock_period)) + 20;
/* Step 11:
* Ramp down.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 11\n");
ccfifo_write(EMC_CFG_SYNC, 0, (dram_type == DRAM_TYPE_LPDDR4) ? 0 : ref_delay);
ccfifo_write(EMC_DBG, emc_dbg | (EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY), 0);
uint32_t ramp_down_wait = dvfs_power_ramp_down(false, current_timing, next_timing, source_clock_period);
/* Step 12:
* Trigger the clock change.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 12\n");
ccfifo_write(EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 1, 0);
if (!dvfs_with_training) {
ccfifo_write(EMC_DBG, (emc_dbg & ~EMC_DBG_WRITE_ACTIVE_ONLY) | EMC_DBG_WRITE_MUX_ACTIVE, 0);
}
/* Step 13:
* Ramp up.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 13\n");
uint32_t ramp_up_wait = dvfs_power_ramp_up(false, current_timing, next_timing, training, destination_clock_period);
ccfifo_write(EMC_DBG, emc_dbg, 0);
/* Step 14:
* Bringup CKE pins.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 14\n");
if ((dram_type == DRAM_TYPE_LPDDR4)) {
uint32_t r = emc_pin_o | EMC_PIN_PIN_CKE;
if (train_ca || train_ca_vref) {
r = emc_pin_o & ~(EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE);
if (dram_dev_num == TWO_RANK) {
if (train_swap_rank)
ccfifo_write(EMC_PIN, r | EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKE, 0);
else
ccfifo_write(EMC_PIN, r | EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB, 0);
}
else
ccfifo_write(EMC_PIN, r, 0);
} else if (dram_dev_num == TWO_RANK) {
ccfifo_write(EMC_PIN, r | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV, 0);
} else {
ccfifo_write(EMC_PIN, r & ~(EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV), 0);
}
}
/* Step 15:
* Calculate zqlatch wait time; has dependency on ramping times.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 15\n");
if (source_clock_period <= zqcal_before_cc_cutoff) {
int t = (int)(ramp_up_wait + ramp_down_wait) / (int)destination_clock_period;
zq_latch_dvfs_wait_time = (int)tZQCAL_lpddr4_fc_adj - t;
} else {
zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj - div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period);
}
if (!(train_ca || train_ca_vref) && (dram_type == DRAM_TYPE_LPDDR4) && opt_zcal_en_cc) {
if (dram_dev_num == ONE_RANK) {
if (source_clock_period > zqcal_before_cc_cutoff) {
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
}
if (!dvfs_with_training) {
ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
ccfifo_write(EMC_SELF_REF, 0x100, 0);
ccfifo_write(EMC_REF, 0, 0);
}
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, max_t(int, 0, zq_latch_dvfs_wait_time));
} else if (shared_zq_resistor) {
if (source_clock_period > zqcal_before_cc_cutoff) {
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
}
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, max_t(int, 0, zq_latch_dvfs_wait_time) + div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, 0);
if (!dvfs_with_training) {
ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, 0);
ccfifo_write(EMC_SELF_REF, 0x100, 0);
ccfifo_write(EMC_REF, 0, 0);
}
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, tZQCAL_lpddr4 / destination_clock_period);
} else {
if (source_clock_period > zqcal_before_cc_cutoff) {
ccfifo_write(EMC_ZQ_CAL, EMC_ZQ_CAL_ZQ_CAL_CMD, div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
}
if (!dvfs_with_training) {
ccfifo_write(EMC_MRW3, (mr13_flip_fspop & 0xF3FFFFF7) | 0xC000000, div_o3(1000 * next_timing->dram_timings[T_PDEX], destination_clock_period));
ccfifo_write(EMC_SELF_REF, 0x100, 0);
ccfifo_write(EMC_REF, 0, 0);
}
ccfifo_write(EMC_ZQ_CAL, EMC_ZQ_CAL_ZQ_LATCH_CMD, max_t(int, 0, zq_latch_dvfs_wait_time));
}
}
/* WAR: delay for zqlatch */
ccfifo_write(EMC_INTSTATUS, 0, 10);
/* Step 16:
* LPDDR4 Conditional Training Kickoff.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 16\n");
if (dvfs_with_training && (dram_type == DRAM_TYPE_LPDDR4)) {
ccfifo_write(EMC_INTSTATUS, 0, (1020000 / destination_clock_period));
uint32_t train_cmd = 0;
if (train_ca)
train_cmd |= (1 << 1); /* CA */
if (train_ca_vref)
train_cmd |= (1 << 5); /* CA_VREF */
if (train_quse)
train_cmd |= (1 << 4); /* QUSE */
if (train_quse_vref)
train_cmd |= (1 << 8); /* QUSE_VREF */
if (train_wr)
train_cmd |= (1 << 3); /* WR */
if (train_wr_vref)
train_cmd |= (1 << 6); /* WR_VREF */
if (train_rd)
train_cmd |= (1 << 2); /* RD */
if (train_rd_vref)
train_cmd |= (1 << 7); /* RD_VREF */
train_cmd |= (1 << 31); /* GO */
ccfifo_write(EMC_TRAINING_CMD, train_cmd, 0);
if (bg_regulator_mode_change) {
if (enable_bg_regulator)
ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0, current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, 0);
else
ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0, current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, 0);
}
ccfifo_write(EMC_SWITCH_BACK_CTRL, 1, 0);
if (!(train_ca || train_ca_vref) || train_swap_rank) {
ccfifo_write(EMC_MRW3, mr13_flip_fspop ^ 0xC0, 0);
ccfifo_write(EMC_INTSTATUS, 0, (1000000 / destination_clock_period));
}
ccfifo_write(EMC_PIN, emc_pin_o & ~(EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE), 0);
ccfifo_write(EMC_CFG_SYNC, 0, 0);
ccfifo_write(EMC_DBG, emc_dbg | (EMC_DBG_WRITE_ACTIVE_ONLY | EMC_DBG_WRITE_MUX_ACTIVE), 0);
dvfs_power_ramp_down(true, current_timing, next_timing, destination_clock_period);
ccfifo_write(EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 1, 0);
ccfifo_write(EMC_DBG, (emc_dbg & ~EMC_DBG_WRITE_ACTIVE_ONLY) | EMC_DBG_WRITE_MUX_ACTIVE, 0);
dvfs_power_ramp_up(true, current_timing, next_timing, training, source_clock_period);
ccfifo_write(EMC_DBG, emc_dbg, 0);
if (dram_dev_num == TWO_RANK)
ccfifo_write(EMC_PIN, emc_pin_o | (EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE), 0);
else
ccfifo_write(EMC_PIN, (emc_pin_o & ~(EMC_PIN_PIN_CKE_PER_DEV | EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE)) | EMC_PIN_PIN_CKE, 0);
if (train_ca || train_ca_vref) {
ccfifo_write(EMC_TR_CTRL_0, 0x4A, (200000 / source_clock_period));
ccfifo_write(EMC_TR_CTRL_0, 0x40, (1000000 / source_clock_period));
ccfifo_write(EMC_MRW3, mr13_catr_enable & 0xFFFFFFFE, 0);
ccfifo_write(EMC_INTSTATUS, 0, (1000000 / source_clock_period));
ccfifo_write(EMC_PMACRO_DATA_RX_TERM_MODE, current_timing->burst_regs[EMC_PMACRO_DATA_RX_TERM_MODE_INDEX], 0);
}
ccfifo_write(EMC_DBG, emc_dbg_o, 0);
if (opt_zcal_en_cc) {
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
ccfifo_write(EMC_ZQ_CAL, 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, (1000000 / source_clock_period));
if (dram_dev_num == TWO_RANK) {
if (shared_zq_resistor) {
if (!(train_ca || train_ca_vref) || train_swap_rank) {
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, (1000000 / source_clock_period));
if (!(train_ca || train_ca_vref))
ccfifo_write(EMC_MRW3, ((mr13_flip_fspop ^ 0xC0) & 0xF3FFFFF7) | 0xC000000, 0);
}
ccfifo_write(EMC_SELF_REF, 0x100, 0);
skip_zqcal = true;
} else {
if ((train_ca || train_ca_vref) && !train_swap_rank) {
ccfifo_write(EMC_SELF_REF, 0x100, 0);
skip_zqcal = true;
} else {
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
ccfifo_write(EMC_ZQ_CAL, 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_LATCH_CMD, (1000000 / source_clock_period));
}
}
}
}
if (!skip_zqcal) {
if (!(train_ca || train_ca_vref))
ccfifo_write(EMC_MRW3, ((mr13_flip_fspop ^ 0xC0) & 0xF3FFFFF7) | 0xC000000, 0);
ccfifo_write(EMC_SELF_REF, 0x100, 0);
}
}
if (!skip_zqcal) {
/* Step 17:
* MANSR exit self refresh.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 17\n");
if ((opt_dvfs_mode == MAN_SR) && (dram_type != DRAM_TYPE_LPDDR4))
ccfifo_write(EMC_SELF_REF, 0, 0);
/* Step 18:
* Send MRWs to LPDDR3/DDR3.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 18\n");
if (dram_type == DRAM_TYPE_LPDDR2) {
ccfifo_write(EMC_MRW2, next_timing->emc_mrw2, 0);
ccfifo_write(EMC_MRW, next_timing->emc_mrw, 0);
if (is_lpddr3) {
ccfifo_write(EMC_MRW4, next_timing->emc_mrw4, 0);
}
} else if (dram_type == DRAM_TYPE_DDR3) {
if (opt_dll_mode == DLL_ON) {
ccfifo_write(EMC_EMRS, next_timing->emc_emrs & ~EMC_EMRS_USE_EMRS_LONG_CNT, 0);
}
ccfifo_write(EMC_EMRS2, next_timing->emc_emrs2 & ~EMC_EMRS2_USE_EMRS2_LONG_CNT, 0);
ccfifo_write(EMC_MRS, next_timing->emc_mrs | EMC_EMRS_USE_EMRS_LONG_CNT, 0);
}
/* Step 19:
* ZQCAL for LPDDR3/DDR3
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 19\n");
if (opt_zcal_en_cc) {
if (dram_type == DRAM_TYPE_LPDDR2) {
uint32_t r;
uint32_t zq_op = opt_cc_short_zcal ? 0x56 : 0xAB;
uint32_t zcal_wait_time_ps = opt_cc_short_zcal ? 90000 : 360000;
uint32_t zcal_wait_time_clocks = div_o3(zcal_wait_time_ps, destination_clock_period);
r = zcal_wait_time_clocks << EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT | zcal_wait_time_clocks << EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT;
ccfifo_write(EMC_MRS_WAIT_CNT2, r, 0);
ccfifo_write(EMC_MRW, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | EMC_MRW_USE_MRW_EXT_CNT | 10 << EMC_MRW_MRW_MA_SHIFT | zq_op << EMC_MRW_MRW_OP_SHIFT, 0);
if (dram_dev_num == TWO_RANK) {
r = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | EMC_MRW_USE_MRW_EXT_CNT | 10 << EMC_MRW_MRW_MA_SHIFT | zq_op << EMC_MRW_MRW_OP_SHIFT;
ccfifo_write(EMC_MRW, r, 0);
}
} else if (dram_type == DRAM_TYPE_DDR3) {
uint32_t zq_op = opt_cc_short_zcal ? 0 : EMC_ZQ_CAL_LONG;
ccfifo_write(EMC_ZQ_CAL, zq_op | 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
if (dram_dev_num == TWO_RANK) {
ccfifo_write(EMC_ZQ_CAL, zq_op | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | EMC_ZQ_CAL_ZQ_CAL_CMD, 0);
}
}
}
}
if (bg_regulator_mode_change) {
emc_set_shadow_bypass(ACTIVE);
uint32_t bg_regulator_switch_complete_wait_clks = ramp_up_wait > 1250000 ? 0 : (1250000 - ramp_up_wait) / destination_clock_period;
if (dvfs_with_training) {
bg_regulator_switch_complete_wait_clks = (1250000 / source_clock_period);
ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0, current_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], bg_regulator_switch_complete_wait_clks);
} else {
ccfifo_write(EMC_PMACRO_BG_BIAS_CTRL_0, next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], bg_regulator_switch_complete_wait_clks);
}
emc_set_shadow_bypass(ASSEMBLY);
}
/* Step 20:
* Issue ref and optional QRST.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 20\n");
if (dvfs_with_training || (dram_type != DRAM_TYPE_LPDDR4))
ccfifo_write(EMC_REF, 0, 0);
if (opt_do_sw_qrst) {
ccfifo_write(EMC_ISSUE_QRST, 1, 0);
ccfifo_write(EMC_ISSUE_QRST, 0, 2);
}
/* Step 21:
* Restore ZCAL and ZCAL interval.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 21\n");
if (save_restore_clkstop_pd || opt_zcal_en_cc) {
emc_set_shadow_bypass(ACTIVE);
if (opt_zcal_en_cc) {
if (dvfs_with_training) {
ccfifo_write(EMC_ZCAL_INTERVAL, current_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX], 0);
} else if (dram_type != DRAM_TYPE_LPDDR4) {
ccfifo_write(EMC_ZCAL_INTERVAL, next_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX], 0);
}
}
if (save_restore_clkstop_pd) {
ccfifo_write(EMC_CFG, next_timing->burst_regs[EMC_CFG_INDEX] & ~EMC_CFG_DYN_SELF_REF, 0);
}
if (dvfs_with_training && (dram_type == DRAM_TYPE_LPDDR4)) {
ccfifo_write(EMC_SEL_DPD_CTRL, current_timing->emc_sel_dpd_ctrl, 0);
}
emc_set_shadow_bypass(ASSEMBLY);
}
/* Step 22:
* Restore EMC_CFG_PIPE_CLK.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 22\n");
ccfifo_write(EMC_CFG_PIPE_CLK, emc_cfg_pipe_clk_o, 0);
if (bg_regulator_mode_change) {
if (enable_bg_regulator) {
emc_write(next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, EMC_PMACRO_BG_BIAS_CTRL_0);
} else {
emc_write(next_timing->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, EMC_PMACRO_BG_BIAS_CTRL_0);
}
}
/* Step 23:
* Do clock change.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 23\n");
if (dvfs_with_training) {
car->clk_source_emc_safe = car->clk_source_emc;
change_dll_src(current_timing, car->clk_source_emc);
}
uint32_t cfg_dig_dll_tmp = emc_read(EMC_CFG_DIG_DLL);
cfg_dig_dll_tmp |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC;
cfg_dig_dll_tmp &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK;
cfg_dig_dll_tmp &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK;
cfg_dig_dll_tmp &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN;
cfg_dig_dll_tmp = (cfg_dig_dll_tmp & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT);
emc_write(cfg_dig_dll_tmp, EMC_CFG_DIG_DLL);
car->clk_source_emc = next_clk_src;
wait_for_update(EMC_INTSTATUS, EMC_INTSTATUS_CLKCHANGE_COMPLETE, true, REG_EMC);
/* Step 24:
* Save training results.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 24\n");
if (dvfs_with_training) {
uint32_t emc_dbg_tmp = emc_read(EMC_DBG);
emc_write(emc_dbg_tmp | 1, EMC_DBG); /* Set READ_MUX to ASSEMBLY. */
/* Save CA results. */
if (train_ca) {
next_timing->trim_perch_regs[REG_EMC0_EMC_CMD_BRLSHFT_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_CMD_BRLSHFT_0);
next_timing->trim_perch_regs[REG_EMC1_EMC_CMD_BRLSHFT_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_CMD_BRLSHFT_1): 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_4);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1,EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_5) : 0;
if (train_self_refresh) {
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD0_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD1_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD2_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2_INDEX] = emc_read(EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_CMD3_2);
}
}
/* Save CA_VREF results. */
if (train_ca_vref) {
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW10_INDEX] = (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_CA_VREF) & 0xFFFF) | 0x880C0000;
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW10_INDEX] = (channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_CA_VREF) & 0xFFFF : 0) | 0x880C0000;
if (dram_dev_num == TWO_RANK) {
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW11_INDEX] = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_CA_VREF) >> 16) & 0xFF) | (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_CA_VREF) >> 24 << 8) | (0x480C0000 & 0xFFFFFF00);
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW11_INDEX] = (((channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_CA_VREF) : 0) >> 16) & 0xFF) | ((channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_CA_VREF) : 0) >> 24 << 8) | (0x480C0000 & 0xFFFFFF00);
} else {
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW11_INDEX] = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_CA_VREF) >> 16) & 0xFF) | (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_CA_VREF) >> 24 << 8) | (0xC80C0000 & 0xFFFFFF00);
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW11_INDEX] = (((channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_CA_VREF) : 0) >> 16) & 0xFF) | ((channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_CA_VREF) : 0) >> 24 << 8) | (0xC80C0000 & 0xFFFFFF00);
}
}
/* Save QUSE results. */
if (train_quse || train_rd) {
next_timing->trim_perch_regs[REG_EMC0_EMC_QUSE_BRLSHFT_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_QUSE_BRLSHFT_0);
next_timing->trim_perch_regs[REG_EMC1_EMC_QUSE_BRLSHFT_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_QUSE_BRLSHFT_1) : 0;
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_QUSE_DDLL_RANK0_0);
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK0_1_INDEX]= emc_read_per_ch(REG_EMC0, EMC_PMACRO_QUSE_DDLL_RANK0_1);
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK0_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_QUSE_DDLL_RANK0_2) : 0;
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK0_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_QUSE_DDLL_RANK0_3) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_perch_regs[REG_EMC0_EMC_QUSE_BRLSHFT_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_QUSE_BRLSHFT_2);
next_timing->trim_perch_regs[REG_EMC1_EMC_QUSE_BRLSHFT_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_QUSE_BRLSHFT_3) : 0;
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_QUSE_DDLL_RANK1_0);
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_QUSE_DDLL_RANK1_1);
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK1_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_QUSE_DDLL_RANK1_2) : 0;
next_timing->trim_regs[EMC_PMACRO_QUSE_DDLL_RANK1_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_QUSE_DDLL_RANK1_3) : 0;
}
}
/* Save QUSE_VREF results. */
if (train_quse_vref) {
if (dram_dev_num == TWO_RANK) {
uint32_t emc0_opt_dqs_array[4] = {0};
uint32_t emc1_opt_dqs_array[4] = {0};
uint32_t emc1_training_opt_dqs_ib_vref_rank0_val = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_DQS_IB_VREF_RANK0) : 0;
uint32_t emc1_training_opt_dqs_ib_vref_rank1_val = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_DQS_IB_VREF_RANK1) : 0;
for (int i = 0; i < 4; i++) {
emc0_opt_dqs_array[i] = (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQS_IB_VREF_RANK0) >> (8 * i)) & 0xFF;
emc1_opt_dqs_array[i] = (emc1_training_opt_dqs_ib_vref_rank0_val >> (8 * i)) & 0xFF;
}
uint32_t ib_vref_dqs_0 = 0;
uint32_t ib_vref_dqs_1 = 0;
for (int i = 0; i < 4; i++)
{
ib_vref_dqs_0 |= (emc0_opt_dqs_array[i] + ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQS_IB_VREF_RANK1) >> (8 * i)) & 0xFF)) >> 1 << (8 * i);
ib_vref_dqs_1 |= (emc1_opt_dqs_array[i] + ((emc1_training_opt_dqs_ib_vref_rank1_val >> (8 * i)) & 0xFF)) >> 1 << (8 * i);
}
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQS_0_INDEX] = ib_vref_dqs_0;
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQS_1_INDEX] = ib_vref_dqs_1;
}
else
{
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQS_0_INDEX] = emc_read(EMC_PMACRO_IB_VREF_DQS_0);
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQS_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_VREF_DQS_1) : 0;
}
}
/* Save RD results. */
if (train_rd) {
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2_INDEX]= channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK0_3) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_LONG_DQS_RANK1_3) : 0;
}
if (train_self_refresh) {
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE0_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE1_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE2_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE3_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE4_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE5_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE6_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK0_BYTE7_2) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE0_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE1_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE2_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_0);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_1);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE3_2);
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE4_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE5_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE6_2) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_0) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_1) : 0;
next_timing->trim_regs[EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_IB_DDLL_SHORT_DQ_RANK1_BYTE7_2) : 0;
}
}
/* Save RD_VREF results. */
if (train_rd_vref) {
uint8_t ib_vref_dq_byte0_icr = (emc_read(EMC_PMACRO_IB_VREF_DQ_0) & 0x7F) + (next_timing->save_restore_mod_regs[0] & 0x7F);
if (next_timing->save_restore_mod_regs[0] & 0x80000000)
ib_vref_dq_byte0_icr = (emc_read(EMC_PMACRO_IB_VREF_DQ_0) & 0x7F) - (next_timing->save_restore_mod_regs[0] & 0x7F);
uint8_t ib_vref_dq_byte1_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 8) & 0x7F) + (next_timing->save_restore_mod_regs[1] & 0x7F);
if (next_timing->save_restore_mod_regs[1] & 0x80000000)
ib_vref_dq_byte1_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 8) & 0x7F) - (next_timing->save_restore_mod_regs[1] & 0x7F);
uint8_t ib_vref_dq_byte2_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 16) & 0x7F) + (next_timing->save_restore_mod_regs[2] & 0x7F);
if (next_timing->save_restore_mod_regs[2] & 0x80000000)
ib_vref_dq_byte2_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 16) & 0x7F) - (next_timing->save_restore_mod_regs[2] & 0x7F);
uint8_t ib_vref_dq_byte3_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 24) & 0x7F) + (next_timing->save_restore_mod_regs[3] & 0x7F);
if (next_timing->save_restore_mod_regs[3] & 0x80000000)
ib_vref_dq_byte3_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_0) >> 24) & 0x7F) - (next_timing->save_restore_mod_regs[3] & 0x7F);
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQ_0_INDEX] = ((ib_vref_dq_byte0_icr & 0x7F) | (ib_vref_dq_byte1_icr & 0x7F) << 8) | ((ib_vref_dq_byte2_icr & 0x7F) << 16) | ((ib_vref_dq_byte3_icr & 0x7F) << 24);
uint8_t ib_vref_dq_byte4_icr = (emc_read(EMC_PMACRO_IB_VREF_DQ_1) & 0x7F) + (next_timing->save_restore_mod_regs[4] & 0x7F);
if (next_timing->save_restore_mod_regs[4] & 0x80000000)
ib_vref_dq_byte4_icr = (emc_read(EMC_PMACRO_IB_VREF_DQ_1) & 0x7F) - (next_timing->save_restore_mod_regs[4] & 0x7F);
uint8_t ib_vref_dq_byte5_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 8) & 0x7F) + (next_timing->save_restore_mod_regs[5] & 0x7F);
if (next_timing->save_restore_mod_regs[5] & 0x80000000)
ib_vref_dq_byte5_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 8) & 0x7F) - (next_timing->save_restore_mod_regs[5] & 0x7F);
uint8_t ib_vref_dq_byte6_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 16) & 0x7F) + (next_timing->save_restore_mod_regs[6] & 0x7F);
if (next_timing->save_restore_mod_regs[6] & 0x80000000)
ib_vref_dq_byte6_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 16) & 0x7F) - (next_timing->save_restore_mod_regs[6] & 0x7F);
uint8_t ib_vref_dq_byte7_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 24) & 0x7F) + (next_timing->save_restore_mod_regs[7] & 0x7F);
if (next_timing->save_restore_mod_regs[7] & 0x80000000)
ib_vref_dq_byte7_icr = ((emc_read(EMC_PMACRO_IB_VREF_DQ_1) >> 24) & 0x7F) - (next_timing->save_restore_mod_regs[7] & 0x7F);
next_timing->trim_regs[EMC_PMACRO_IB_VREF_DQ_1_INDEX] = ((ib_vref_dq_byte4_icr & 0x7F) | (ib_vref_dq_byte5_icr & 0x7F) << 8) | ((ib_vref_dq_byte6_icr & 0x7F) << 16) | ((ib_vref_dq_byte7_icr & 0x7F) << 24);
}
}
/* Save WR results. */
if (train_wr) {
next_timing->trim_perch_regs[REG_EMC0_EMC_DATA_BRLSHFT_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_DATA_BRLSHFT_0);
next_timing->trim_perch_regs[REG_EMC1_EMC_DATA_BRLSHFT_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_DATA_BRLSHFT_0) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_perch_regs[REG_EMC0_EMC_DATA_BRLSHFT_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_DATA_BRLSHFT_1);
next_timing->trim_perch_regs[REG_EMC1_EMC_DATA_BRLSHFT_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_DATA_BRLSHFT_1) : 0;
}
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3) : 0;
}
if (train_self_refresh) {
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE0_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE1_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE2_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE3_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE4_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE5_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE6_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK0_BYTE7_2) : 0;
if (dram_dev_num == TWO_RANK) {
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE0_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE1_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE2_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_0);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_1);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2_INDEX] = emc_read_per_ch(REG_EMC0, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE3_2);
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE4_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE5_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE6_2) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_0) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_1) : 0;
next_timing->trim_regs[EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2_INDEX] = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_PMACRO_OB_DDLL_SHORT_DQ_RANK1_BYTE7_2) : 0;
}
}
/* Save WR_VREF results. */
if (train_wr_vref) {
uint32_t emc1_ranks_sub_partitions = channel_mode ? emc_read_per_ch(REG_EMC1, EMC_TRAINING_OPT_DQ_OB_VREF) : 0;
uint8_t emc0_ib_vref_dq_byte8_modded_plus = next_timing->save_restore_mod_regs[8] + emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF);
if (next_timing->save_restore_mod_regs[8] & 0x80000000)
emc0_ib_vref_dq_byte8_modded_plus = emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) - next_timing->save_restore_mod_regs[8];
uint8_t emc0_mrw12_op_sp1 = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) & 0xFFFF) >> 8) + next_timing->save_restore_mod_regs[9];
if (next_timing->save_restore_mod_regs[9] & 0x80000000)
emc0_mrw12_op_sp1 = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) & 0xFFFF) >> 8) - next_timing->save_restore_mod_regs[9];
uint8_t emc0_mrw13_op_sp0 = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) >> 16) & 0xFF) + next_timing->save_restore_mod_regs[8];
if (next_timing->save_restore_mod_regs[8] & 0x80000000)
emc0_mrw13_op_sp0 = ((emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) >> 16) & 0xFF) - next_timing->save_restore_mod_regs[8];
uint8_t emc0_ib_vref_dq_byte9_modded_a_plus = next_timing->save_restore_mod_regs[9] + (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) >> 24);
if (next_timing->save_restore_mod_regs[9] & 0x80000000)
emc0_ib_vref_dq_byte9_modded_a_plus = (emc_read_per_ch(REG_EMC0, EMC_TRAINING_OPT_DQ_OB_VREF) >> 24) - (uint8_t)next_timing->save_restore_mod_regs[9];
uint8_t emc0_ib_vref_dq_byte10_modded_plus = emc1_ranks_sub_partitions + next_timing->save_restore_mod_regs[10];
if (next_timing->save_restore_mod_regs[10] & 0x80000000)
emc0_ib_vref_dq_byte10_modded_plus = emc1_ranks_sub_partitions - next_timing->save_restore_mod_regs[10];
uint8_t emc0_ib_vref_dq_byte11_modded_plus = ((emc1_ranks_sub_partitions & 0xFFFF) >> 8) + next_timing->save_restore_mod_regs[11];
if (next_timing->save_restore_mod_regs[11] & 0x80000000)
emc0_ib_vref_dq_byte11_modded_plus = ((emc1_ranks_sub_partitions & 0xFFFF) >> 8) - next_timing->save_restore_mod_regs[11];
uint8_t emc1_mrw13_op_sp0 = ((emc1_ranks_sub_partitions >> 16) & 0xFF) + next_timing->save_restore_mod_regs[10];
if (next_timing->save_restore_mod_regs[10] & 0x80000000)
emc1_mrw13_op_sp0 = ((emc1_ranks_sub_partitions >> 16) & 0xFF) - next_timing->save_restore_mod_regs[10];
uint8_t emc1_mrw13_op_sp1 = (emc1_ranks_sub_partitions >> 24) + next_timing->save_restore_mod_regs[11];
if (next_timing->save_restore_mod_regs[11] & 0x80000000)
emc1_mrw13_op_sp1 = (emc1_ranks_sub_partitions >> 24) - next_timing->save_restore_mod_regs[11];
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW12_INDEX] = (uint8_t)emc0_ib_vref_dq_byte10_modded_plus | 0x880E0000 | (emc0_ib_vref_dq_byte11_modded_plus << 8);
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW12_INDEX] = emc0_ib_vref_dq_byte8_modded_plus | 0x880E0000 | (emc0_mrw12_op_sp1 << 8);
if (dram_dev_num == TWO_RANK) {
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW13_INDEX] = emc0_ib_vref_dq_byte9_modded_a_plus << 8 | emc0_mrw13_op_sp0 | 0x480E0000;
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW13_INDEX] = (emc1_mrw13_op_sp1 << 8) | emc1_mrw13_op_sp0 | 0x480E0000;
} else {
next_timing->burst_reg_per_ch[REG_EMC0_EMC_MRW13_INDEX] = emc0_ib_vref_dq_byte9_modded_a_plus << 8 | emc0_mrw13_op_sp0 | 0xC80E0000;
next_timing->burst_reg_per_ch[REG_EMC1_EMC_MRW13_INDEX] = (emc1_mrw13_op_sp1 << 8) | emc1_mrw13_op_sp0 | 0xC80E0000;
}
}
}
emc_write(emc_dbg_tmp, EMC_DBG);
}
/* Step 25:
* Program MC updown registers.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 25\n");
if ((next_timing->rate > current_timing->rate) && !dvfs_with_training) {
for (int i = 0; i < next_timing->num_up_down; i++) {
mc_write(next_timing->la_scale_regs[i], la_scale_regs_off[i]);
}
/* Request a timing update. */
emc_timing_update(channel_mode);
}
/* Step 26:
* Restore ZCAL registers.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 26\n");
if (dram_type == DRAM_TYPE_LPDDR4) {
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], EMC_ZCAL_WAIT_CNT);
emc_write(next_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX], EMC_ZCAL_INTERVAL);
emc_set_shadow_bypass(ASSEMBLY);
}
if ((dram_type != DRAM_TYPE_LPDDR4)
&& opt_zcal_en_cc
&& !opt_short_zcal
&& opt_cc_short_zcal)
{
udelay(2);
emc_set_shadow_bypass(ACTIVE);
if (dram_type == DRAM_TYPE_LPDDR2) {
emc_write(next_timing->burst_regs[EMC_MRS_WAIT_CNT_INDEX], EMC_MRS_WAIT_CNT);
} else if (dram_type == DRAM_TYPE_DDR3) {
emc_write(next_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], EMC_ZCAL_WAIT_CNT);
}
emc_set_shadow_bypass(ASSEMBLY);
}
/* Step 27:
* Restore EMC_CFG, FDPD registers.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 27\n");
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->burst_regs[EMC_CFG_INDEX], EMC_CFG);
emc_set_shadow_bypass(ASSEMBLY);
emc_write(next_timing->emc_fdpd_ctrl_cmd_no_ramp, EMC_FDPD_CTRL_CMD_NO_RAMP);
emc_write(next_timing->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
/* Step 28:
* Training recover.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 28\n");
if (dvfs_with_training && (dram_type == DRAM_TYPE_LPDDR4)) {
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->burst_regs[EMC_CFG_INDEX], EMC_CFG);
emc_write(next_timing->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL);
emc_write(current_timing->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], EMC_ZCAL_WAIT_CNT);
emc_write(current_timing->burst_regs[EMC_ZCAL_INTERVAL_INDEX], EMC_ZCAL_INTERVAL);
emc_write(current_timing->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2);
emc_write(current_timing->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3);
emc_write(current_timing->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4);
emc_write(current_timing->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5);
emc_write(current_timing->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6);
emc_write(current_timing->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7);
emc_write(current_timing->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8);
emc_set_shadow_bypass(ASSEMBLY);
emc_write(next_timing->burst_regs[EMC_TR_DVFS_INDEX] & ~EMC_TR_DVFS_TRAINING_DVFS, EMC_TR_DVFS);
}
emc_set_shadow_bypass(ACTIVE);
emc_write(next_timing->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX], EMC_PMACRO_AUTOCAL_CFG_COMMON);
emc_set_shadow_bypass(ASSEMBLY);
/* Step 29:
* Power fix WAR.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 29\n");
emc_write(EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 |
EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7,
EMC_PMACRO_CFG_PM_GLOBAL_0);
emc_write(EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR, EMC_PMACRO_TRAINING_CTRL_0);
emc_write(EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR, EMC_PMACRO_TRAINING_CTRL_1);
emc_write(0, EMC_PMACRO_CFG_PM_GLOBAL_0);
/* Step 30:
* Re-enable autocal.
*/
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - step 30\n");
if (dvfs_with_training) {
emc_auto_cal_config = current_timing->emc_auto_cal_config;
/* Restore FSP to account for switch back. Only needed in training. */
g_fsp_for_next_freq = !g_fsp_for_next_freq;
} else {
emc_auto_cal_config = next_timing->emc_auto_cal_config;
if (next_timing->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) {
dll_enable_stall(channel_mode);
}
}
emc_write(emc_auto_cal_config, EMC_AUTO_CAL_CONFIG);
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Clock set - done!\n");
}
static void do_periodic_emc_compensation(tegra_emc_timing_t* current_timing) {
uint32_t reg_count = 10;
uint32_t reg_list[] = {
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2,
EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3,
EMC_DATA_BRLSHFT_0,
EMC_DATA_BRLSHFT_1
};
if (current_timing->periodic_training) {
int dram_dev_num = ((mc_read(MC_EMEM_ADR_CFG) & 1) + 1);
bool channel_mode = ((current_timing->burst_regs[EMC_FBIO_CFG7_INDEX] >> 2) & 1);
uint32_t emc_cfg_o = emc_read(EMC_CFG);
uint32_t emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD | EMC_CFG_DRAM_CLKSTOP_PD | EMC_CFG_DRAM_CLKSTOP_PD);
/*
* 1. Power optimizations should be off.
*/
emc_write(emc_cfg, EMC_CFG);
/* Does emc_timing_update() for above changes. */
dll_disable(channel_mode);
if (dram_dev_num == TWO_RANK) {
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, false, REG_EMC1);
} else {
wait_for_update(EMC_EMC_STATUS, 0x10, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, 0x10, false, REG_EMC1);
}
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, false, REG_EMC1);
wait_for_update(EMC_EMC_STATUS, 0x01, false, REG_EMC);
if (channel_mode)
wait_for_update(EMC_EMC_STATUS, 0x01, false, REG_EMC1);
uint32_t emc_cfg_update = emc_read(EMC_CFG_UPDATE);
emc_write((emc_cfg_update & ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK) | (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT), EMC_CFG_UPDATE);
/*
* 2. osc kick off - this assumes training and dvfs have set
* correct MR23.
*/
start_periodic_compensation();
/*
* 3. Let dram capture its clock tree delays.
*/
udelay((actual_osc_clocks(current_timing->run_clocks) * 1000) / current_timing->rate + 1);
/*
* 4. Check delta wrt previous values (save value if margin
* exceeds what is set in table).
*/
uint32_t del = periodic_compensation_handler(current_timing, current_timing, dram_dev_num, channel_mode, PERIODIC_TRAINING_SEQUENCE);
/*
* 5. Apply compensation w.r.t. trained values (if clock tree
* has drifted more than the set margin).
*/
if (current_timing->tree_margin < ((del * 128 * (current_timing->rate / 1000)) / 1000000)) {
for (int i = 0; i < reg_count; i++) {
uint32_t tmp = apply_periodic_compensation_trimmer(current_timing, reg_list[i]);
emc_write(tmp, reg_list[i]);
}
}
emc_write(emc_cfg_o, EMC_CFG);
/*
* 6. Timing update actually applies the new trimmers.
*/
emc_timing_update(channel_mode);
/* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */
emc_write(emc_cfg_update, EMC_CFG_UPDATE);
/* 6.2. Restore the DLL. */
dll_enable(channel_mode);
}
}
static void train_set_clock(tegra_emc_timing_t* current_timing, tegra_emc_timing_t* next_timing, bool update_clk, uint32_t next_clk_src) {
/* Check for dual channel LPDDR4 */
bool dual_channel_lpddr4_case = ((emc_read(EMC_FBIO_CFG7) & EMC_FBIO_CFG7_CH0_ENABLE) & (emc_read(EMC_FBIO_CFG7) & EMC_FBIO_CFG7_CH1_ENABLE));
/* Get the DRAM type */
uint32_t dram_type = (next_timing->burst_regs[EMC_FBIO_CFG5_INDEX] & 0x03);
if (g_write_training_pattern) {
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Writing training patterns...\n");
/* Write the training data into pattern RAM */
for (int i = 0; i < 0x100; i++) {
uint32_t training_pattern_val = g_ram_pattern_dq[i + (next_timing->training_pattern * 0x100)];
/* Write the DQ data into pattern RAM */
emc_write(training_pattern_val, EMC_TRAINING_PATRAM_DQ);
training_pattern_val = g_ram_pattern_dmi[i + (next_timing->training_pattern * 0x100)];
/* Write the DMI data into pattern RAM */
emc_write(training_pattern_val & 0x0F, EMC_TRAINING_PATRAM_DMI);
/* Enable writing into pattern RAM and select the offset */
emc_write(0x80000000 + i, EMC_TRAINING_PATRAM_CTRL);
}
}
/* Only write the training pattern once */
g_write_training_pattern = false;
if (next_timing->needs_training && !next_timing->trained) {
uint32_t training_flags = next_timing->needs_training;
/* Read from MC_EMEM_ADR_CFG */
uint32_t dram_dev_num = mc_read(MC_EMEM_ADR_CFG);
int training_params_idx = 0;
int training_params[8] = {};
if (training_flags & 0x03) {
training_params_idx = 1;
training_params[0] = (training_flags & 0x203);
if (dram_dev_num & 0x01) {
training_params_idx = 2;
training_params[1] = (training_flags & 0x303);
}
}
if ((dram_dev_num & 0x01) && (training_flags & 0x0C)) {
training_params[training_params_idx] = (training_flags & 0x20C);
training_params[training_params_idx + 1] = (training_flags & 0x204);
training_params_idx += 2;
} else if (training_flags & 0x0C) {
training_params[training_params_idx++] = (training_flags & 0x20C);
}
if (training_flags & 0xF0)
training_params[training_params_idx++] = (training_flags & 0x2F0);
for (int i = 0; i < training_params_idx; i++) {
/* Adjust the clock */
set_clock(current_timing, next_timing, training_params[i], next_clk_src);
/* Change CFG_SWAP to ASSEMBLY_ONLY */
uint32_t emc_dbg = emc_read(EMC_DBG);
emc_dbg = ((emc_dbg & 0xF3FFFFFF) | 0x8000000);
emc_write(emc_dbg, EMC_DBG);
/* Change UPDATE_AUTO_CAL_IN_UPDATE to ALWAYS */
uint32_t emc_cfg_update = emc_read(EMC_CFG_UPDATE);
emc_cfg_update = ((emc_cfg_update & 0xFFFFFFF9) | 0x04);
emc_write(emc_cfg_update, EMC_CFG_UPDATE);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Change UPDATE_AUTO_CAL_IN_UPDATE to NEVER */
emc_cfg_update = emc_read(EMC_CFG_UPDATE);
emc_cfg_update &= 0xFFFFFFF9;
emc_write(emc_cfg_update, EMC_CFG_UPDATE);
/* Change CFG_SWAP to ACTIVE_ONLY */
emc_dbg = emc_read(EMC_DBG);
emc_dbg &= 0xF3FFFFFF;
emc_write(emc_dbg, EMC_DBG);
/* Disable DLL and change CFG_DLL_MODE to RUN_PERIODIC */
uint32_t emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
emc_cfg_dig_dll = ((emc_cfg_dig_dll & 0xFFFFFF3E) | 0x80);
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Disable or enable DLL */
emc_cfg_dig_dll = emc_read(EMC_CFG_DIG_DLL);
if (next_timing->burst_regs[EMC_CFG_DIG_DLL_INDEX] == 0x01)
emc_cfg_dig_dll |= 0x01;
else
emc_cfg_dig_dll &= 0xFFFFFFFE;
/* Change CFG_DLL_MODE to RUN_PERIODIC */
emc_cfg_dig_dll = ((emc_cfg_dig_dll & 0xFFFFFF3F) | 0x80);
emc_write(emc_cfg_dig_dll, EMC_CFG_DIG_DLL);
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
/* Wait for DLL_LOCK to be set */
uint32_t emc_dig_dll_status = 0;
do {
emc_dig_dll_status = emc_read(EMC_DIG_DLL_STATUS);
} while (!(emc_dig_dll_status & EMC_DIG_DLL_STATUS_DLL_LOCK));
/* Check if DRAM is LPDDR4 */
if (dram_type == DRAM_TYPE_LPDDR4) {
emc_write(current_timing->burst_regs[EMC_RP_INDEX], EMC_RP);
emc_write(current_timing->burst_regs[EMC_R2P_INDEX], EMC_R2P);
emc_write(current_timing->burst_regs[EMC_W2P_INDEX], EMC_W2P);
emc_write(current_timing->burst_regs[EMC_TRPAB_INDEX], EMC_TRPAB);
}
/* Request a timing update event */
emc_timing_update(dual_channel_lpddr4_case);
}
/* We've finished training */
next_timing->trained = 1;
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Memory is trained!\n");
}
/* Change the clock if requested */
if (update_clk)
set_clock(current_timing, next_timing, 0, next_clk_src);
}
static int train_one(int z_val, uint32_t next_rate, uint32_t current_rate, tegra_emc_timing_t* timing_tables, int timing_tables_count, TrainMode mode) {
int current_timing_table_idx = -1;
int next_timing_table_idx = -1;
tegra_emc_timing_t* current_timing_table;
tegra_emc_timing_t* next_timing_table;
uint32_t next_clk_src = 0;
/* Too many table entries */
if (timing_tables_count > 0x384)
return 4;
/* Locate the right tables for the transition */
for (int i = 0; i < timing_tables_count; i++) {
uint32_t rate = timing_tables[i].rate;
if (rate == current_rate)
current_timing_table_idx = i;
else if (rate == next_rate)
next_timing_table_idx = i;
}
/* Failed to find the tables. */
if ((current_timing_table_idx < 0) || (next_timing_table_idx < 0))
return 4;
current_timing_table = (tegra_emc_timing_t*)&timing_tables[current_timing_table_idx];
next_timing_table = (tegra_emc_timing_t*)&timing_tables[next_timing_table_idx];
uint32_t clk_src_emc_from = current_timing_table->clk_src_emc;
uint32_t clk_src_emc_to = next_timing_table->clk_src_emc;
uint32_t rate_from = current_timing_table->rate;
uint32_t rate_to = next_timing_table->rate;
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Changing rate from %d to %d!\n", rate_from, rate_to);
bool diff_freq = test_clk_ratio(rate_to, clk_src_emc_to, rate_from, clk_src_emc_from);
if (diff_freq) {
uint32_t emc_2x_clk_src = (clk_src_emc_from >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
if (emc_2x_clk_src & 0x03) {
if ((emc_2x_clk_src - TEGRA_EMC_SRC_PLLMB_UD) <= 1) {
g_is_pllmb = false;
}
} else {
g_is_pllmb = !g_is_pllmb;
}
/* Configure the PLL values */
next_clk_src = set_pll(rate_to, 0x9600, clk_src_emc_to, g_is_pllmb);
} else {
uint32_t emc_2x_clk_src = (clk_src_emc_to >> EMC_CLK_EMC_2X_CLK_SRC_SHIFT);
if ((emc_2x_clk_src != TEGRA_EMC_SRC_PLLMB) && emc_2x_clk_src) {
if (((emc_2x_clk_src - TEGRA_EMC_SRC_PLLM_UD) <= TEGRA_EMC_SRC_PLLC) && g_is_pllmb) {
next_clk_src = ((clk_src_emc_to & 0x1FFFFFFF) | (TEGRA_EMC_SRC_PLLMB_UD << EMC_CLK_EMC_2X_CLK_SRC_SHIFT));
}
} else if (g_is_pllmb) {
next_clk_src = ((clk_src_emc_to & 0x1FFFFFFF) | (TEGRA_EMC_SRC_PLLMB << EMC_CLK_EMC_2X_CLK_SRC_SHIFT));
} else {
next_clk_src = clk_src_emc_to;
}
}
if (mode == OP_SWITCH) {
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Train mode is OP_SWITCH!\n");
set_clock(current_timing_table, next_timing_table, false, next_clk_src);
g_active_timing_table_idx = next_timing_table_idx;
if (next_timing_table->periodic_training) {
do_periodic_emc_compensation(next_timing_table);
}
} else if (mode == OP_TRAIN) {
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Train mode is OP_TRAIN!\n");
train_set_clock(current_timing_table, next_timing_table, false, next_clk_src);
g_active_timing_table_idx = next_timing_table_idx;
if (diff_freq) {
g_is_pllmb = !g_is_pllmb;
}
} else if (mode == OP_TRAIN_SWITCH) {
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Train mode is OP_TRAIN_SWITCH!\n");
train_set_clock(current_timing_table, next_timing_table, true, next_clk_src);
g_active_timing_table_idx = next_timing_table_idx;
if (next_timing_table->periodic_training) {
do_periodic_emc_compensation(next_timing_table);
}
} else
return 4;
return 0;
}
static void train_dram_erista(void) {
volatile tegra_car_t *car = car_get_regs();
tegra_emc_timing_t *timing_tables;
uint32_t dram_id = fuse_get_dram_id();
/* Find the right timing table set. */
if (dram_id == 0x01)
timing_tables = (tegra_emc_timing_t *)nx_abca2_dram_1;
else if ((dram_id == 0x00) || (dram_id == 0x02) || (dram_id == 0x03) || (dram_id == 0x04))
timing_tables = (tegra_emc_timing_t *)nx_abca2_dram_0234;
else
fatal_error("[MTC]: Missing tables for DRAM id %d!\n", dram_id);
/* Locate the right timing table. */
int boot_index = 0;
for (boot_index = 0; boot_index < MTC_TABLES_MAX_ENTRIES; boot_index++) {
print(SCREEN_LOG_LEVEL_DEBUG, "%d (%d kHz): %s\n", boot_index, timing_tables[boot_index].rate, timing_tables[boot_index].dvfs_ver);
if (car->clk_source_emc == timing_tables[boot_index].clk_src_emc)
break;
}
if (boot_index >= MTC_TABLES_MAX_ENTRIES) {
fatal_error("[MTC]: Failed to find timing table!\n");
}
/* Switch to 800Mhz. */
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Switching from %dMhz to 800Mhz\n", timing_tables[boot_index].rate / 1000);
train_one(0, 800000, timing_tables[boot_index].rate, timing_tables, MTC_TABLES_MAX_ENTRIES, OP_TRAIN_SWITCH);
/* Switch to 1600Mhz. */
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Switching from %dMhz to 1600Mhz\n", timing_tables[g_active_timing_table_idx].rate / 1000);
train_one(0, 1600000, timing_tables[g_active_timing_table_idx].rate, timing_tables, MTC_TABLES_MAX_ENTRIES, OP_TRAIN_SWITCH);
/* Wait a while. */
mdelay(100);
/* Do periodic compensation. */
do_periodic_emc_compensation((tegra_emc_timing_t*)&timing_tables[g_active_timing_table_idx]);
print(SCREEN_LOG_LEVEL_DEBUG, "[MTC]: Done!\n");
}
void train_dram(void) {
if (is_soc_mariko()) {
train_dram_mariko();
} else {
train_dram_erista();
}
}
Reference in New Issue View Git Blame Copy Permalink