drivers/scsi/ufs/ufshcd.c

   1 /*
   2  * Universal Flash Storage Host controller driver Core
   3  *
   4  * This code is based on drivers/scsi/ufs/ufshcd.c
   5  * Copyright (C) 2011-2013 Samsung India Software Operations
   6  * Copyright (c) 2013-2016, The Linux Foundation. All rights reserved.
   7  *
   8  * Authors:
   9  *      Santosh Yaraganavi <santosh.sy@samsung.com>
  10  *      Vinayak Holikatti <h.vinayak@samsung.com>
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  * See the COPYING file in the top-level directory or visit
  17  * <http://www.gnu.org/licenses/gpl-2.0.html>
  18  *
  19  * This program is distributed in the hope that it will be useful,
  20  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22  * GNU General Public License for more details.
  23  *
  24  * This program is provided "AS IS" and "WITH ALL FAULTS" and
  25  * without warranty of any kind. You are solely responsible for
  26  * determining the appropriateness of using and distributing
  27  * the program and assume all risks associated with your exercise
  28  * of rights with respect to the program, including but not limited
  29  * to infringement of third party rights, the risks and costs of
  30  * program errors, damage to or loss of data, programs or equipment,
  31  * and unavailability or interruption of operations. Under no
  32  * circumstances will the contributor of this Program be liable for
  33  * any damages of any kind arising from your use or distribution of
  34  * this program.
  35  *
  36  * The Linux Foundation chooses to take subject only to the GPLv2
  37  * license terms, and distributes only under these terms.
  38  */
  39
  40 #include <linux/async.h>
  41 #include <linux/devfreq.h>
  42 #include <linux/nls.h>
  43 #include <linux/of.h>
  44 #include <linux/bitfield.h>
  45 #include "ufshcd.h"
  46 #include "ufs_quirks.h"
  47 #include "unipro.h"
  48 #include "ufs-sysfs.h"
  49
  50 #define CREATE_TRACE_POINTS
  51 #include <trace/events/ufs.h>
  52
  53 #define UFSHCD_REQ_SENSE_SIZE   18
  54
  55 #define UFSHCD_ENABLE_INTRS     (UTP_TRANSFER_REQ_COMPL |\
  56                                  UTP_TASK_REQ_COMPL |\
  57                                  UFSHCD_ERROR_MASK)
  58 /* UIC command timeout, unit: ms */
  59 #define UIC_CMD_TIMEOUT 500
  60
  61 /* NOP OUT retries waiting for NOP IN response */
  62 #define NOP_OUT_RETRIES    10
  63 /* Timeout after 30 msecs if NOP OUT hangs without response */
  64 #define NOP_OUT_TIMEOUT    30 /* msecs */
  65
  66 /* Query request retries */
  67 #define QUERY_REQ_RETRIES 3
  68 /* Query request timeout */
  69 #define QUERY_REQ_TIMEOUT 1500 /* 1.5 seconds */
  70
  71 /* Task management command timeout */
  72 #define TM_CMD_TIMEOUT  100 /* msecs */
  73
  74 /* maximum number of retries for a general UIC command  */
  75 #define UFS_UIC_COMMAND_RETRIES 3
  76
  77 /* maximum number of link-startup retries */
  78 #define DME_LINKSTARTUP_RETRIES 3
  79
  80 /* Maximum retries for Hibern8 enter */
  81 #define UIC_HIBERN8_ENTER_RETRIES 3
  82
  83 /* maximum number of reset retries before giving up */
  84 #define MAX_HOST_RESET_RETRIES 5
  85
  86 /* Expose the flag value from utp_upiu_query.value */
  87 #define MASK_QUERY_UPIU_FLAG_LOC 0xFF
  88
  89 /* Interrupt aggregation default timeout, unit: 40us */
  90 #define INT_AGGR_DEF_TO 0x02
  91
  92 #define ufshcd_toggle_vreg(_dev, _vreg, _on)                            \
  93         ({                                                              \
  94                 int _ret;                                               \
  95                 if (_on)                                                \
  96                         _ret = ufshcd_enable_vreg(_dev, _vreg);         \
  97                 else                                                    \
  98                         _ret = ufshcd_disable_vreg(_dev, _vreg);        \
  99                 _ret;                                                   \
 100         })
 101
 102 #define ufshcd_hex_dump(prefix_str, buf, len) do {                       \
 103         size_t __len = (len);                                            \
 104         print_hex_dump(KERN_ERR, prefix_str,                             \
 105                        __len > 4 ? DUMP_PREFIX_OFFSET : DUMP_PREFIX_NONE,\
 106                        16, 4, buf, __len, false);                        \
 107 } while (0)
 108
 109 int ufshcd_dump_regs(struct ufs_hba *hba, size_t offset, size_t len,
 110                      const char *prefix)
 111 {
 112         u32 *regs;
 113         size_t pos;
 114
 115         if (offset % 4 != 0 || len % 4 != 0) /* keep readl happy */
 116                 return -EINVAL;
 117
 118         regs = kzalloc(len, GFP_KERNEL);
 119         if (!regs)
 120                 return -ENOMEM;
 121
 122         for (pos = 0; pos < len; pos += 4) {
 123                 if (offset == 0 &&
 124                     pos >= REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER &&
 125                     pos <= REG_UIC_ERROR_CODE_DME)
 126                         continue;
 127                 regs[pos / 4] = ufshcd_readl(hba, offset + pos);
 128         }
 129
 130         ufshcd_hex_dump(prefix, regs, len);
 131         kfree(regs);
 132
 133         return 0;
 134 }
 135 EXPORT_SYMBOL_GPL(ufshcd_dump_regs);
 136
 137 enum {
 138         UFSHCD_MAX_CHANNEL      = 0,
 139         UFSHCD_MAX_ID           = 1,
 140         UFSHCD_CMD_PER_LUN      = 32,
 141         UFSHCD_CAN_QUEUE        = 32,
 142 };
 143
 144 /* UFSHCD states */
 145 enum {
 146         UFSHCD_STATE_RESET,
 147         UFSHCD_STATE_ERROR,
 148         UFSHCD_STATE_OPERATIONAL,
 149         UFSHCD_STATE_EH_SCHEDULED,
 150 };
 151
 152 /* UFSHCD error handling flags */
 153 enum {
 154         UFSHCD_EH_IN_PROGRESS = (1 << 0),
 155 };
 156
 157 /* UFSHCD UIC layer error flags */
 158 enum {
 159         UFSHCD_UIC_DL_PA_INIT_ERROR = (1 << 0), /* Data link layer error */
 160         UFSHCD_UIC_DL_NAC_RECEIVED_ERROR = (1 << 1), /* Data link layer error */
 161         UFSHCD_UIC_DL_TCx_REPLAY_ERROR = (1 << 2), /* Data link layer error */
 162         UFSHCD_UIC_NL_ERROR = (1 << 3), /* Network layer error */
 163         UFSHCD_UIC_TL_ERROR = (1 << 4), /* Transport Layer error */
 164         UFSHCD_UIC_DME_ERROR = (1 << 5), /* DME error */
 165 };
 166
 167 #define ufshcd_set_eh_in_progress(h) \
 168         ((h)->eh_flags |= UFSHCD_EH_IN_PROGRESS)
 169 #define ufshcd_eh_in_progress(h) \
 170         ((h)->eh_flags & UFSHCD_EH_IN_PROGRESS)
 171 #define ufshcd_clear_eh_in_progress(h) \
 172         ((h)->eh_flags &= ~UFSHCD_EH_IN_PROGRESS)
 173
 174 #define ufshcd_set_ufs_dev_active(h) \
 175         ((h)->curr_dev_pwr_mode = UFS_ACTIVE_PWR_MODE)
 176 #define ufshcd_set_ufs_dev_sleep(h) \
 177         ((h)->curr_dev_pwr_mode = UFS_SLEEP_PWR_MODE)
 178 #define ufshcd_set_ufs_dev_poweroff(h) \
 179         ((h)->curr_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE)
 180 #define ufshcd_is_ufs_dev_active(h) \
 181         ((h)->curr_dev_pwr_mode == UFS_ACTIVE_PWR_MODE)
 182 #define ufshcd_is_ufs_dev_sleep(h) \
 183         ((h)->curr_dev_pwr_mode == UFS_SLEEP_PWR_MODE)
 184 #define ufshcd_is_ufs_dev_poweroff(h) \
 185         ((h)->curr_dev_pwr_mode == UFS_POWERDOWN_PWR_MODE)
 186
 187 struct ufs_pm_lvl_states ufs_pm_lvl_states[] = {
 188         {UFS_ACTIVE_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 189         {UFS_ACTIVE_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 190         {UFS_SLEEP_PWR_MODE, UIC_LINK_ACTIVE_STATE},
 191         {UFS_SLEEP_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 192         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_HIBERN8_STATE},
 193         {UFS_POWERDOWN_PWR_MODE, UIC_LINK_OFF_STATE},
 194 };
 195
 196 static inline enum ufs_dev_pwr_mode
 197 ufs_get_pm_lvl_to_dev_pwr_mode(enum ufs_pm_level lvl)
 198 {
 199         return ufs_pm_lvl_states[lvl].dev_state;
 200 }
 201
 202 static inline enum uic_link_state
 203 ufs_get_pm_lvl_to_link_pwr_state(enum ufs_pm_level lvl)
 204 {
 205         return ufs_pm_lvl_states[lvl].link_state;
 206 }
 207
 208 static inline enum ufs_pm_level
 209 ufs_get_desired_pm_lvl_for_dev_link_state(enum ufs_dev_pwr_mode dev_state,
 210                                         enum uic_link_state link_state)
 211 {
 212         enum ufs_pm_level lvl;
 213
 214         for (lvl = UFS_PM_LVL_0; lvl < UFS_PM_LVL_MAX; lvl++) {
 215                 if ((ufs_pm_lvl_states[lvl].dev_state == dev_state) &&
 216                         (ufs_pm_lvl_states[lvl].link_state == link_state))
 217                         return lvl;
 218         }
 219
 220         /* if no match found, return the level 0 */
 221         return UFS_PM_LVL_0;
 222 }
 223
 224 static struct ufs_dev_fix ufs_fixups[] = {
 225         /* UFS cards deviations table */
 226         UFS_FIX(UFS_VENDOR_MICRON, UFS_ANY_MODEL,
 227                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
 228         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
 229                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
 230         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ),
 231         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
 232                 UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS),
 233         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
 234                 UFS_DEVICE_NO_FASTAUTO),
 235         UFS_FIX(UFS_VENDOR_SAMSUNG, UFS_ANY_MODEL,
 236                 UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE),
 237         UFS_FIX(UFS_VENDOR_TOSHIBA, UFS_ANY_MODEL,
 238                 UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM),
 239         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9C8KBADG",
 240                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
 241         UFS_FIX(UFS_VENDOR_TOSHIBA, "THGLF2G9D8KBADG",
 242                 UFS_DEVICE_QUIRK_PA_TACTIVATE),
 243         UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL, UFS_DEVICE_NO_VCCQ),
 244         UFS_FIX(UFS_VENDOR_SKHYNIX, UFS_ANY_MODEL,
 245                 UFS_DEVICE_QUIRK_HOST_PA_SAVECONFIGTIME),
 246         UFS_FIX(UFS_VENDOR_SKHYNIX, "hB8aL1" /*H28U62301AMR*/,
 247                 UFS_DEVICE_QUIRK_HOST_VS_DEBUGSAVECONFIGTIME),
 248
 249         END_FIX
 250 };
 251
 252 static void ufshcd_tmc_handler(struct ufs_hba *hba);
 253 static void ufshcd_async_scan(void *data, async_cookie_t cookie);
 254 static int ufshcd_reset_and_restore(struct ufs_hba *hba);
 255 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd);
 256 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag);
 257 static void ufshcd_hba_exit(struct ufs_hba *hba);
 258 static int ufshcd_probe_hba(struct ufs_hba *hba);
 259 static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
 260                                  bool skip_ref_clk);
 261 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on);
 262 static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused);
 263 static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba);
 264 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba);
 265 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba);
 266 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba);
 267 static void ufshcd_resume_clkscaling(struct ufs_hba *hba);
 268 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 269 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba);
 270 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up);
 271 static irqreturn_t ufshcd_intr(int irq, void *__hba);
 272 static int ufshcd_change_power_mode(struct ufs_hba *hba,
 273                              struct ufs_pa_layer_attr *pwr_mode);
 274 static inline bool ufshcd_valid_tag(struct ufs_hba *hba, int tag)
 275 {
 276         return tag >= 0 && tag < hba->nutrs;
 277 }
 278
 279 static inline int ufshcd_enable_irq(struct ufs_hba *hba)
 280 {
 281         int ret = 0;
 282
 283         if (!hba->is_irq_enabled) {
 284                 ret = request_irq(hba->irq, ufshcd_intr, IRQF_SHARED, UFSHCD,
 285                                 hba);
 286                 if (ret)
 287                         dev_err(hba->dev, "%s: request_irq failed, ret=%d\n",
 288                                 __func__, ret);
 289                 hba->is_irq_enabled = true;
 290         }
 291
 292         return ret;
 293 }
 294
 295 static inline void ufshcd_disable_irq(struct ufs_hba *hba)
 296 {
 297         if (hba->is_irq_enabled) {
 298                 free_irq(hba->irq, hba);
 299                 hba->is_irq_enabled = false;
 300         }
 301 }
 302
 303 static void ufshcd_scsi_unblock_requests(struct ufs_hba *hba)
 304 {
 305         if (atomic_dec_and_test(&hba->scsi_block_reqs_cnt))
 306                 scsi_unblock_requests(hba->host);
 307 }
 308
 309 static void ufshcd_scsi_block_requests(struct ufs_hba *hba)
 310 {
 311         if (atomic_inc_return(&hba->scsi_block_reqs_cnt) == 1)
 312                 scsi_block_requests(hba->host);
 313 }
 314
 315 /* replace non-printable or non-ASCII characters with spaces */
 316 static inline void ufshcd_remove_non_printable(char *val)
 317 {
 318         if (!val)
 319                 return;
 320
 321         if (*val < 0x20 || *val > 0x7e)
 322                 *val = ' ';
 323 }
 324
 325 static void ufshcd_add_cmd_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 326                 const char *str)
 327 {
 328         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
 329
 330         trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->sc.cdb);
 331 }
 332
 333 static void ufshcd_add_query_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 334                 const char *str)
 335 {
 336         struct utp_upiu_req *rq = hba->lrb[tag].ucd_req_ptr;
 337
 338         trace_ufshcd_upiu(dev_name(hba->dev), str, &rq->header, &rq->qr);
 339 }
 340
 341 static void ufshcd_add_tm_upiu_trace(struct ufs_hba *hba, unsigned int tag,
 342                 const char *str)
 343 {
 344         struct utp_task_req_desc *descp;
 345         struct utp_upiu_task_req *task_req;
 346         int off = (int)tag - hba->nutrs;
 347
 348         descp = &hba->utmrdl_base_addr[off];
 349         task_req = (struct utp_upiu_task_req *)descp->task_req_upiu;
 350         trace_ufshcd_upiu(dev_name(hba->dev), str, &task_req->header,
 351                         &task_req->input_param1);
 352 }
 353
 354 static void ufshcd_add_command_trace(struct ufs_hba *hba,
 355                 unsigned int tag, const char *str)
 356 {
 357         sector_t lba = -1;
 358         u8 opcode = 0;
 359         u32 intr, doorbell;
 360         struct ufshcd_lrb *lrbp = &hba->lrb[tag];
 361         struct scsi_cmnd *cmd = lrbp->cmd;
 362         int transfer_len = -1;
 363
 364         if (!trace_ufshcd_command_enabled()) {
 365                 /* trace UPIU W/O tracing command */
 366                 if (cmd)
 367                         ufshcd_add_cmd_upiu_trace(hba, tag, str);
 368                 return;
 369         }
 370
 371         if (cmd) { /* data phase exists */
 372                 /* trace UPIU also */
 373                 ufshcd_add_cmd_upiu_trace(hba, tag, str);
 374                 opcode = cmd->cmnd[0];
 375                 if ((opcode == READ_10) || (opcode == WRITE_10)) {
 376                         /*
 377                          * Currently we only fully trace read(10) and write(10)
 378                          * commands
 379                          */
 380                         if (cmd->request && cmd->request->bio)
 381                                 lba = cmd->request->bio->bi_iter.bi_sector;
 382                         transfer_len = be32_to_cpu(
 383                                 lrbp->ucd_req_ptr->sc.exp_data_transfer_len);
 384                 }
 385         }
 386
 387         intr = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
 388         doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
 389         trace_ufshcd_command(dev_name(hba->dev), str, tag,
 390                                 doorbell, transfer_len, intr, lba, opcode);
 391 }
 392
 393 static void ufshcd_print_clk_freqs(struct ufs_hba *hba)
 394 {
 395         struct ufs_clk_info *clki;
 396         struct list_head *head = &hba->clk_list_head;
 397
 398         if (list_empty(head))
 399                 return;
 400
 401         list_for_each_entry(clki, head, list) {
 402                 if (!IS_ERR_OR_NULL(clki->clk) && clki->min_freq &&
 403                                 clki->max_freq)
 404                         dev_err(hba->dev, "clk: %s, rate: %u\n",
 405                                         clki->name, clki->curr_freq);
 406         }
 407 }
 408
 409 static void ufshcd_print_uic_err_hist(struct ufs_hba *hba,
 410                 struct ufs_uic_err_reg_hist *err_hist, char *err_name)
 411 {
 412         int i;
 413
 414         for (i = 0; i < UIC_ERR_REG_HIST_LENGTH; i++) {
 415                 int p = (i + err_hist->pos - 1) % UIC_ERR_REG_HIST_LENGTH;
 416
 417                 if (err_hist->reg[p] == 0)
 418                         continue;
 419                 dev_err(hba->dev, "%s[%d] = 0x%x at %lld us\n", err_name, i,
 420                         err_hist->reg[p], ktime_to_us(err_hist->tstamp[p]));
 421         }
 422 }
 423
 424 static void ufshcd_print_host_regs(struct ufs_hba *hba)
 425 {
 426         ufshcd_dump_regs(hba, 0, UFSHCI_REG_SPACE_SIZE, "host_regs: ");
 427         dev_err(hba->dev, "hba->ufs_version = 0x%x, hba->capabilities = 0x%x\n",
 428                 hba->ufs_version, hba->capabilities);
 429         dev_err(hba->dev,
 430                 "hba->outstanding_reqs = 0x%x, hba->outstanding_tasks = 0x%x\n",
 431                 (u32)hba->outstanding_reqs, (u32)hba->outstanding_tasks);
 432         dev_err(hba->dev,
 433                 "last_hibern8_exit_tstamp at %lld us, hibern8_exit_cnt = %d\n",
 434                 ktime_to_us(hba->ufs_stats.last_hibern8_exit_tstamp),
 435                 hba->ufs_stats.hibern8_exit_cnt);
 436
 437         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.pa_err, "pa_err");
 438         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dl_err, "dl_err");
 439         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.nl_err, "nl_err");
 440         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.tl_err, "tl_err");
 441         ufshcd_print_uic_err_hist(hba, &hba->ufs_stats.dme_err, "dme_err");
 442
 443         ufshcd_print_clk_freqs(hba);
 444
 445         if (hba->vops && hba->vops->dbg_register_dump)
 446                 hba->vops->dbg_register_dump(hba);
 447 }
 448
 449 static
 450 void ufshcd_print_trs(struct ufs_hba *hba, unsigned long bitmap, bool pr_prdt)
 451 {
 452         struct ufshcd_lrb *lrbp;
 453         int prdt_length;
 454         int tag;
 455
 456         for_each_set_bit(tag, &bitmap, hba->nutrs) {
 457                 lrbp = &hba->lrb[tag];
 458
 459                 dev_err(hba->dev, "UPIU[%d] - issue time %lld us\n",
 460                                 tag, ktime_to_us(lrbp->issue_time_stamp));
 461                 dev_err(hba->dev, "UPIU[%d] - complete time %lld us\n",
 462                                 tag, ktime_to_us(lrbp->compl_time_stamp));
 463                 dev_err(hba->dev,
 464                         "UPIU[%d] - Transfer Request Descriptor phys@0x%llx\n",
 465                         tag, (u64)lrbp->utrd_dma_addr);
 466
 467                 ufshcd_hex_dump("UPIU TRD: ", lrbp->utr_descriptor_ptr,
 468                                 sizeof(struct utp_transfer_req_desc));
 469                 dev_err(hba->dev, "UPIU[%d] - Request UPIU phys@0x%llx\n", tag,
 470                         (u64)lrbp->ucd_req_dma_addr);
 471                 ufshcd_hex_dump("UPIU REQ: ", lrbp->ucd_req_ptr,
 472                                 sizeof(struct utp_upiu_req));
 473                 dev_err(hba->dev, "UPIU[%d] - Response UPIU phys@0x%llx\n", tag,
 474                         (u64)lrbp->ucd_rsp_dma_addr);
 475                 ufshcd_hex_dump("UPIU RSP: ", lrbp->ucd_rsp_ptr,
 476                                 sizeof(struct utp_upiu_rsp));
 477
 478                 prdt_length = le16_to_cpu(
 479                         lrbp->utr_descriptor_ptr->prd_table_length);
 480                 dev_err(hba->dev,
 481                         "UPIU[%d] - PRDT - %d entries  phys@0x%llx\n",
 482                         tag, prdt_length,
 483                         (u64)lrbp->ucd_prdt_dma_addr);
 484
 485                 if (pr_prdt)
 486                         ufshcd_hex_dump("UPIU PRDT: ", lrbp->ucd_prdt_ptr,
 487                                 sizeof(struct ufshcd_sg_entry) * prdt_length);
 488         }
 489 }
 490
 491 static void ufshcd_print_tmrs(struct ufs_hba *hba, unsigned long bitmap)
 492 {
 493         struct utp_task_req_desc *tmrdp;
 494         int tag;
 495
 496         for_each_set_bit(tag, &bitmap, hba->nutmrs) {
 497                 tmrdp = &hba->utmrdl_base_addr[tag];
 498                 dev_err(hba->dev, "TM[%d] - Task Management Header\n", tag);
 499                 ufshcd_hex_dump("TM TRD: ", &tmrdp->header,
 500                                 sizeof(struct request_desc_header));
 501                 dev_err(hba->dev, "TM[%d] - Task Management Request UPIU\n",
 502                                 tag);
 503                 ufshcd_hex_dump("TM REQ: ", tmrdp->task_req_upiu,
 504                                 sizeof(struct utp_upiu_req));
 505                 dev_err(hba->dev, "TM[%d] - Task Management Response UPIU\n",
 506                                 tag);
 507                 ufshcd_hex_dump("TM RSP: ", tmrdp->task_rsp_upiu,
 508                                 sizeof(struct utp_task_req_desc));
 509         }
 510 }
 511
 512 static void ufshcd_print_host_state(struct ufs_hba *hba)
 513 {
 514         dev_err(hba->dev, "UFS Host state=%d\n", hba->ufshcd_state);
 515         dev_err(hba->dev, "lrb in use=0x%lx, outstanding reqs=0x%lx tasks=0x%lx\n",
 516                 hba->lrb_in_use, hba->outstanding_reqs, hba->outstanding_tasks);
 517         dev_err(hba->dev, "saved_err=0x%x, saved_uic_err=0x%x\n",
 518                 hba->saved_err, hba->saved_uic_err);
 519         dev_err(hba->dev, "Device power mode=%d, UIC link state=%d\n",
 520                 hba->curr_dev_pwr_mode, hba->uic_link_state);
 521         dev_err(hba->dev, "PM in progress=%d, sys. suspended=%d\n",
 522                 hba->pm_op_in_progress, hba->is_sys_suspended);
 523         dev_err(hba->dev, "Auto BKOPS=%d, Host self-block=%d\n",
 524                 hba->auto_bkops_enabled, hba->host->host_self_blocked);
 525         dev_err(hba->dev, "Clk gate=%d\n", hba->clk_gating.state);
 526         dev_err(hba->dev, "error handling flags=0x%x, req. abort count=%d\n",
 527                 hba->eh_flags, hba->req_abort_count);
 528         dev_err(hba->dev, "Host capabilities=0x%x, caps=0x%x\n",
 529                 hba->capabilities, hba->caps);
 530         dev_err(hba->dev, "quirks=0x%x, dev. quirks=0x%x\n", hba->quirks,
 531                 hba->dev_quirks);
 532 }
 533
 534 /**
 535  * ufshcd_print_pwr_info - print power params as saved in hba
 536  * power info
 537  * @hba: per-adapter instance
 538  */
 539 static void ufshcd_print_pwr_info(struct ufs_hba *hba)
 540 {
 541         static const char * const names[] = {
 542                 "INVALID MODE",
 543                 "FAST MODE",
 544                 "SLOW_MODE",
 545                 "INVALID MODE",
 546                 "FASTAUTO_MODE",
 547                 "SLOWAUTO_MODE",
 548                 "INVALID MODE",
 549         };
 550
 551         dev_err(hba->dev, "%s:[RX, TX]: gear=[%d, %d], lane[%d, %d], pwr[%s, %s], rate = %d\n",
 552                  __func__,
 553                  hba->pwr_info.gear_rx, hba->pwr_info.gear_tx,
 554                  hba->pwr_info.lane_rx, hba->pwr_info.lane_tx,
 555                  names[hba->pwr_info.pwr_rx],
 556                  names[hba->pwr_info.pwr_tx],
 557                  hba->pwr_info.hs_rate);
 558 }
 559
 560 /*
 561  * ufshcd_wait_for_register - wait for register value to change
 562  * @hba - per-adapter interface
 563  * @reg - mmio register offset
 564  * @mask - mask to apply to read register value
 565  * @val - wait condition
 566  * @interval_us - polling interval in microsecs
 567  * @timeout_ms - timeout in millisecs
 568  * @can_sleep - perform sleep or just spin
 569  *
 570  * Returns -ETIMEDOUT on error, zero on success
 571  */
 572 int ufshcd_wait_for_register(struct ufs_hba *hba, u32 reg, u32 mask,
 573                                 u32 val, unsigned long interval_us,
 574                                 unsigned long timeout_ms, bool can_sleep)
 575 {
 576         int err = 0;
 577         unsigned long timeout = jiffies + msecs_to_jiffies(timeout_ms);
 578
 579         /* ignore bits that we don't intend to wait on */
 580         val = val & mask;
 581
 582         while ((ufshcd_readl(hba, reg) & mask) != val) {
 583                 if (can_sleep)
 584                         usleep_range(interval_us, interval_us + 50);
 585                 else
 586                         udelay(interval_us);
 587                 if (time_after(jiffies, timeout)) {
 588                         if ((ufshcd_readl(hba, reg) & mask) != val)
 589                                 err = -ETIMEDOUT;
 590                         break;
 591                 }
 592         }
 593
 594         return err;
 595 }
 596
 597 /**
 598  * ufshcd_get_intr_mask - Get the interrupt bit mask
 599  * @hba: Pointer to adapter instance
 600  *
 601  * Returns interrupt bit mask per version
 602  */
 603 static inline u32 ufshcd_get_intr_mask(struct ufs_hba *hba)
 604 {
 605         u32 intr_mask = 0;
 606
 607         switch (hba->ufs_version) {
 608         case UFSHCI_VERSION_10:
 609                 intr_mask = INTERRUPT_MASK_ALL_VER_10;
 610                 break;
 611         case UFSHCI_VERSION_11:
 612         case UFSHCI_VERSION_20:
 613                 intr_mask = INTERRUPT_MASK_ALL_VER_11;
 614                 break;
 615         case UFSHCI_VERSION_21:
 616         default:
 617                 intr_mask = INTERRUPT_MASK_ALL_VER_21;
 618                 break;
 619         }
 620
 621         return intr_mask;
 622 }
 623
 624 /**
 625  * ufshcd_get_ufs_version - Get the UFS version supported by the HBA
 626  * @hba: Pointer to adapter instance
 627  *
 628  * Returns UFSHCI version supported by the controller
 629  */
 630 static inline u32 ufshcd_get_ufs_version(struct ufs_hba *hba)
 631 {
 632         if (hba->quirks & UFSHCD_QUIRK_BROKEN_UFS_HCI_VERSION)
 633                 return ufshcd_vops_get_ufs_hci_version(hba);
 634
 635         return ufshcd_readl(hba, REG_UFS_VERSION);
 636 }
 637
 638 /**
 639  * ufshcd_is_device_present - Check if any device connected to
 640  *                            the host controller
 641  * @hba: pointer to adapter instance
 642  *
 643  * Returns true if device present, false if no device detected
 644  */
 645 static inline bool ufshcd_is_device_present(struct ufs_hba *hba)
 646 {
 647         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) &
 648                                                 DEVICE_PRESENT) ? true : false;
 649 }
 650
 651 /**
 652  * ufshcd_get_tr_ocs - Get the UTRD Overall Command Status
 653  * @lrbp: pointer to local command reference block
 654  *
 655  * This function is used to get the OCS field from UTRD
 656  * Returns the OCS field in the UTRD
 657  */
 658 static inline int ufshcd_get_tr_ocs(struct ufshcd_lrb *lrbp)
 659 {
 660         return le32_to_cpu(lrbp->utr_descriptor_ptr->header.dword_2) & MASK_OCS;
 661 }
 662
 663 /**
 664  * ufshcd_get_tmr_ocs - Get the UTMRD Overall Command Status
 665  * @task_req_descp: pointer to utp_task_req_desc structure
 666  *
 667  * This function is used to get the OCS field from UTMRD
 668  * Returns the OCS field in the UTMRD
 669  */
 670 static inline int
 671 ufshcd_get_tmr_ocs(struct utp_task_req_desc *task_req_descp)
 672 {
 673         return le32_to_cpu(task_req_descp->header.dword_2) & MASK_OCS;
 674 }
 675
 676 /**
 677  * ufshcd_get_tm_free_slot - get a free slot for task management request
 678  * @hba: per adapter instance
 679  * @free_slot: pointer to variable with available slot value
 680  *
 681  * Get a free tag and lock it until ufshcd_put_tm_slot() is called.
 682  * Returns 0 if free slot is not available, else return 1 with tag value
 683  * in @free_slot.
 684  */
 685 static bool ufshcd_get_tm_free_slot(struct ufs_hba *hba, int *free_slot)
 686 {
 687         int tag;
 688         bool ret = false;
 689
 690         if (!free_slot)
 691                 goto out;
 692
 693         do {
 694                 tag = find_first_zero_bit(&hba->tm_slots_in_use, hba->nutmrs);
 695                 if (tag >= hba->nutmrs)
 696                         goto out;
 697         } while (test_and_set_bit_lock(tag, &hba->tm_slots_in_use));
 698
 699         *free_slot = tag;
 700         ret = true;
 701 out:
 702         return ret;
 703 }
 704
 705 static inline void ufshcd_put_tm_slot(struct ufs_hba *hba, int slot)
 706 {
 707         clear_bit_unlock(slot, &hba->tm_slots_in_use);
 708 }
 709
 710 /**
 711  * ufshcd_utrl_clear - Clear a bit in UTRLCLR register
 712  * @hba: per adapter instance
 713  * @pos: position of the bit to be cleared
 714  */
 715 static inline void ufshcd_utrl_clear(struct ufs_hba *hba, u32 pos)
 716 {
 717         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 718                 ufshcd_writel(hba, (1 << pos), REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 719         else
 720                 ufshcd_writel(hba, ~(1 << pos),
 721                                 REG_UTP_TRANSFER_REQ_LIST_CLEAR);
 722 }
 723
 724 /**
 725  * ufshcd_utmrl_clear - Clear a bit in UTRMLCLR register
 726  * @hba: per adapter instance
 727  * @pos: position of the bit to be cleared
 728  */
 729 static inline void ufshcd_utmrl_clear(struct ufs_hba *hba, u32 pos)
 730 {
 731         if (hba->quirks & UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR)
 732                 ufshcd_writel(hba, (1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 733         else
 734                 ufshcd_writel(hba, ~(1 << pos), REG_UTP_TASK_REQ_LIST_CLEAR);
 735 }
 736
 737 /**
 738  * ufshcd_outstanding_req_clear - Clear a bit in outstanding request field
 739  * @hba: per adapter instance
 740  * @tag: position of the bit to be cleared
 741  */
 742 static inline void ufshcd_outstanding_req_clear(struct ufs_hba *hba, int tag)
 743 {
 744         __clear_bit(tag, &hba->outstanding_reqs);
 745 }
 746
 747 /**
 748  * ufshcd_get_lists_status - Check UCRDY, UTRLRDY and UTMRLRDY
 749  * @reg: Register value of host controller status
 750  *
 751  * Returns integer, 0 on Success and positive value if failed
 752  */
 753 static inline int ufshcd_get_lists_status(u32 reg)
 754 {
 755         return !((reg & UFSHCD_STATUS_READY) == UFSHCD_STATUS_READY);
 756 }
 757
 758 /**
 759  * ufshcd_get_uic_cmd_result - Get the UIC command result
 760  * @hba: Pointer to adapter instance
 761  *
 762  * This function gets the result of UIC command completion
 763  * Returns 0 on success, non zero value on error
 764  */
 765 static inline int ufshcd_get_uic_cmd_result(struct ufs_hba *hba)
 766 {
 767         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_2) &
 768                MASK_UIC_COMMAND_RESULT;
 769 }
 770
 771 /**
 772  * ufshcd_get_dme_attr_val - Get the value of attribute returned by UIC command
 773  * @hba: Pointer to adapter instance
 774  *
 775  * This function gets UIC command argument3
 776  * Returns 0 on success, non zero value on error
 777  */
 778 static inline u32 ufshcd_get_dme_attr_val(struct ufs_hba *hba)
 779 {
 780         return ufshcd_readl(hba, REG_UIC_COMMAND_ARG_3);
 781 }
 782
 783 /**
 784  * ufshcd_get_req_rsp - returns the TR response transaction type
 785  * @ucd_rsp_ptr: pointer to response UPIU
 786  */
 787 static inline int
 788 ufshcd_get_req_rsp(struct utp_upiu_rsp *ucd_rsp_ptr)
 789 {
 790         return be32_to_cpu(ucd_rsp_ptr->header.dword_0) >> 24;
 791 }
 792
 793 /**
 794  * ufshcd_get_rsp_upiu_result - Get the result from response UPIU
 795  * @ucd_rsp_ptr: pointer to response UPIU
 796  *
 797  * This function gets the response status and scsi_status from response UPIU
 798  * Returns the response result code.
 799  */
 800 static inline int
 801 ufshcd_get_rsp_upiu_result(struct utp_upiu_rsp *ucd_rsp_ptr)
 802 {
 803         return be32_to_cpu(ucd_rsp_ptr->header.dword_1) & MASK_RSP_UPIU_RESULT;
 804 }
 805
 806 /*
 807  * ufshcd_get_rsp_upiu_data_seg_len - Get the data segment length
 808  *                              from response UPIU
 809  * @ucd_rsp_ptr: pointer to response UPIU
 810  *
 811  * Return the data segment length.
 812  */
 813 static inline unsigned int
 814 ufshcd_get_rsp_upiu_data_seg_len(struct utp_upiu_rsp *ucd_rsp_ptr)
 815 {
 816         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 817                 MASK_RSP_UPIU_DATA_SEG_LEN;
 818 }
 819
 820 /**
 821  * ufshcd_is_exception_event - Check if the device raised an exception event
 822  * @ucd_rsp_ptr: pointer to response UPIU
 823  *
 824  * The function checks if the device raised an exception event indicated in
 825  * the Device Information field of response UPIU.
 826  *
 827  * Returns true if exception is raised, false otherwise.
 828  */
 829 static inline bool ufshcd_is_exception_event(struct utp_upiu_rsp *ucd_rsp_ptr)
 830 {
 831         return be32_to_cpu(ucd_rsp_ptr->header.dword_2) &
 832                         MASK_RSP_EXCEPTION_EVENT ? true : false;
 833 }
 834
 835 /**
 836  * ufshcd_reset_intr_aggr - Reset interrupt aggregation values.
 837  * @hba: per adapter instance
 838  */
 839 static inline void
 840 ufshcd_reset_intr_aggr(struct ufs_hba *hba)
 841 {
 842         ufshcd_writel(hba, INT_AGGR_ENABLE |
 843                       INT_AGGR_COUNTER_AND_TIMER_RESET,
 844                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 845 }
 846
 847 /**
 848  * ufshcd_config_intr_aggr - Configure interrupt aggregation values.
 849  * @hba: per adapter instance
 850  * @cnt: Interrupt aggregation counter threshold
 851  * @tmout: Interrupt aggregation timeout value
 852  */
 853 static inline void
 854 ufshcd_config_intr_aggr(struct ufs_hba *hba, u8 cnt, u8 tmout)
 855 {
 856         ufshcd_writel(hba, INT_AGGR_ENABLE | INT_AGGR_PARAM_WRITE |
 857                       INT_AGGR_COUNTER_THLD_VAL(cnt) |
 858                       INT_AGGR_TIMEOUT_VAL(tmout),
 859                       REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 860 }
 861
 862 /**
 863  * ufshcd_disable_intr_aggr - Disables interrupt aggregation.
 864  * @hba: per adapter instance
 865  */
 866 static inline void ufshcd_disable_intr_aggr(struct ufs_hba *hba)
 867 {
 868         ufshcd_writel(hba, 0, REG_UTP_TRANSFER_REQ_INT_AGG_CONTROL);
 869 }
 870
 871 /**
 872  * ufshcd_enable_run_stop_reg - Enable run-stop registers,
 873  *                      When run-stop registers are set to 1, it indicates the
 874  *                      host controller that it can process the requests
 875  * @hba: per adapter instance
 876  */
 877 static void ufshcd_enable_run_stop_reg(struct ufs_hba *hba)
 878 {
 879         ufshcd_writel(hba, UTP_TASK_REQ_LIST_RUN_STOP_BIT,
 880                       REG_UTP_TASK_REQ_LIST_RUN_STOP);
 881         ufshcd_writel(hba, UTP_TRANSFER_REQ_LIST_RUN_STOP_BIT,
 882                       REG_UTP_TRANSFER_REQ_LIST_RUN_STOP);
 883 }
 884
 885 /**
 886  * ufshcd_hba_start - Start controller initialization sequence
 887  * @hba: per adapter instance
 888  */
 889 static inline void ufshcd_hba_start(struct ufs_hba *hba)
 890 {
 891         ufshcd_writel(hba, CONTROLLER_ENABLE, REG_CONTROLLER_ENABLE);
 892 }
 893
 894 /**
 895  * ufshcd_is_hba_active - Get controller state
 896  * @hba: per adapter instance
 897  *
 898  * Returns false if controller is active, true otherwise
 899  */
 900 static inline bool ufshcd_is_hba_active(struct ufs_hba *hba)
 901 {
 902         return (ufshcd_readl(hba, REG_CONTROLLER_ENABLE) & CONTROLLER_ENABLE)
 903                 ? false : true;
 904 }
 905
 906 u32 ufshcd_get_local_unipro_ver(struct ufs_hba *hba)
 907 {
 908         /* HCI version 1.0 and 1.1 supports UniPro 1.41 */
 909         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
 910             (hba->ufs_version == UFSHCI_VERSION_11))
 911                 return UFS_UNIPRO_VER_1_41;
 912         else
 913                 return UFS_UNIPRO_VER_1_6;
 914 }
 915 EXPORT_SYMBOL(ufshcd_get_local_unipro_ver);
 916
 917 static bool ufshcd_is_unipro_pa_params_tuning_req(struct ufs_hba *hba)
 918 {
 919         /*
 920          * If both host and device support UniPro ver1.6 or later, PA layer
 921          * parameters tuning happens during link startup itself.
 922          *
 923          * We can manually tune PA layer parameters if either host or device
 924          * doesn't support UniPro ver 1.6 or later. But to keep manual tuning
 925          * logic simple, we will only do manual tuning if local unipro version
 926          * doesn't support ver1.6 or later.
 927          */
 928         if (ufshcd_get_local_unipro_ver(hba) < UFS_UNIPRO_VER_1_6)
 929                 return true;
 930         else
 931                 return false;
 932 }
 933
 934 static int ufshcd_scale_clks(struct ufs_hba *hba, bool scale_up)
 935 {
 936         int ret = 0;
 937         struct ufs_clk_info *clki;
 938         struct list_head *head = &hba->clk_list_head;
 939         ktime_t start = ktime_get();
 940         bool clk_state_changed = false;
 941
 942         if (list_empty(head))
 943                 goto out;
 944
 945         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, PRE_CHANGE);
 946         if (ret)
 947                 return ret;
 948
 949         list_for_each_entry(clki, head, list) {
 950                 if (!IS_ERR_OR_NULL(clki->clk)) {
 951                         if (scale_up && clki->max_freq) {
 952                                 if (clki->curr_freq == clki->max_freq)
 953                                         continue;
 954
 955                                 clk_state_changed = true;
 956                                 ret = clk_set_rate(clki->clk, clki->max_freq);
 957                                 if (ret) {
 958                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
 959                                                 __func__, clki->name,
 960                                                 clki->max_freq, ret);
 961                                         break;
 962                                 }
 963                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
 964                                                 "scaled up", clki->name,
 965                                                 clki->curr_freq,
 966                                                 clki->max_freq);
 967
 968                                 clki->curr_freq = clki->max_freq;
 969
 970                         } else if (!scale_up && clki->min_freq) {
 971                                 if (clki->curr_freq == clki->min_freq)
 972                                         continue;
 973
 974                                 clk_state_changed = true;
 975                                 ret = clk_set_rate(clki->clk, clki->min_freq);
 976                                 if (ret) {
 977                                         dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
 978                                                 __func__, clki->name,
 979                                                 clki->min_freq, ret);
 980                                         break;
 981                                 }
 982                                 trace_ufshcd_clk_scaling(dev_name(hba->dev),
 983                                                 "scaled down", clki->name,
 984                                                 clki->curr_freq,
 985                                                 clki->min_freq);
 986                                 clki->curr_freq = clki->min_freq;
 987                         }
 988                 }
 989                 dev_dbg(hba->dev, "%s: clk: %s, rate: %lu\n", __func__,
 990                                 clki->name, clk_get_rate(clki->clk));
 991         }
 992
 993         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
 994
 995 out:
 996         if (clk_state_changed)
 997                 trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
 998                         (scale_up ? "up" : "down"),
 999                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1000         return ret;
1001 }
1002
1003 /**
1004  * ufshcd_is_devfreq_scaling_required - check if scaling is required or not
1005  * @hba: per adapter instance
1006  * @scale_up: True if scaling up and false if scaling down
1007  *
1008  * Returns true if scaling is required, false otherwise.
1009  */
1010 static bool ufshcd_is_devfreq_scaling_required(struct ufs_hba *hba,
1011                                                bool scale_up)
1012 {
1013         struct ufs_clk_info *clki;
1014         struct list_head *head = &hba->clk_list_head;
1015
1016         if (list_empty(head))
1017                 return false;
1018
1019         list_for_each_entry(clki, head, list) {
1020                 if (!IS_ERR_OR_NULL(clki->clk)) {
1021                         if (scale_up && clki->max_freq) {
1022                                 if (clki->curr_freq == clki->max_freq)
1023                                         continue;
1024                                 return true;
1025                         } else if (!scale_up && clki->min_freq) {
1026                                 if (clki->curr_freq == clki->min_freq)
1027                                         continue;
1028                                 return true;
1029                         }
1030                 }
1031         }
1032
1033         return false;
1034 }
1035
1036 static int ufshcd_wait_for_doorbell_clr(struct ufs_hba *hba,
1037                                         u64 wait_timeout_us)
1038 {
1039         unsigned long flags;
1040         int ret = 0;
1041         u32 tm_doorbell;
1042         u32 tr_doorbell;
1043         bool timeout = false, do_last_check = false;
1044         ktime_t start;
1045
1046         ufshcd_hold(hba, false);
1047         spin_lock_irqsave(hba->host->host_lock, flags);
1048         /*
1049          * Wait for all the outstanding tasks/transfer requests.
1050          * Verify by checking the doorbell registers are clear.
1051          */
1052         start = ktime_get();
1053         do {
1054                 if (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL) {
1055                         ret = -EBUSY;
1056                         goto out;
1057                 }
1058
1059                 tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
1060                 tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1061                 if (!tm_doorbell && !tr_doorbell) {
1062                         timeout = false;
1063                         break;
1064                 } else if (do_last_check) {
1065                         break;
1066                 }
1067
1068                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1069                 schedule();
1070                 if (ktime_to_us(ktime_sub(ktime_get(), start)) >
1071                     wait_timeout_us) {
1072                         timeout = true;
1073                         /*
1074                          * We might have scheduled out for long time so make
1075                          * sure to check if doorbells are cleared by this time
1076                          * or not.
1077                          */
1078                         do_last_check = true;
1079                 }
1080                 spin_lock_irqsave(hba->host->host_lock, flags);
1081         } while (tm_doorbell || tr_doorbell);
1082
1083         if (timeout) {
1084                 dev_err(hba->dev,
1085                         "%s: timedout waiting for doorbell to clear (tm=0x%x, tr=0x%x)\n",
1086                         __func__, tm_doorbell, tr_doorbell);
1087                 ret = -EBUSY;
1088         }
1089 out:
1090         spin_unlock_irqrestore(hba->host->host_lock, flags);
1091         ufshcd_release(hba);
1092         return ret;
1093 }
1094
1095 /**
1096  * ufshcd_scale_gear - scale up/down UFS gear
1097  * @hba: per adapter instance
1098  * @scale_up: True for scaling up gear and false for scaling down
1099  *
1100  * Returns 0 for success,
1101  * Returns -EBUSY if scaling can't happen at this time
1102  * Returns non-zero for any other errors
1103  */
1104 static int ufshcd_scale_gear(struct ufs_hba *hba, bool scale_up)
1105 {
1106         #define UFS_MIN_GEAR_TO_SCALE_DOWN      UFS_HS_G1
1107         int ret = 0;
1108         struct ufs_pa_layer_attr new_pwr_info;
1109
1110         if (scale_up) {
1111                 memcpy(&new_pwr_info, &hba->clk_scaling.saved_pwr_info.info,
1112                        sizeof(struct ufs_pa_layer_attr));
1113         } else {
1114                 memcpy(&new_pwr_info, &hba->pwr_info,
1115                        sizeof(struct ufs_pa_layer_attr));
1116
1117                 if (hba->pwr_info.gear_tx > UFS_MIN_GEAR_TO_SCALE_DOWN
1118                     || hba->pwr_info.gear_rx > UFS_MIN_GEAR_TO_SCALE_DOWN) {
1119                         /* save the current power mode */
1120                         memcpy(&hba->clk_scaling.saved_pwr_info.info,
1121                                 &hba->pwr_info,
1122                                 sizeof(struct ufs_pa_layer_attr));
1123
1124                         /* scale down gear */
1125                         new_pwr_info.gear_tx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1126                         new_pwr_info.gear_rx = UFS_MIN_GEAR_TO_SCALE_DOWN;
1127                 }
1128         }
1129
1130         /* check if the power mode needs to be changed or not? */
1131         ret = ufshcd_change_power_mode(hba, &new_pwr_info);
1132
1133         if (ret)
1134                 dev_err(hba->dev, "%s: failed err %d, old gear: (tx %d rx %d), new gear: (tx %d rx %d)",
1135                         __func__, ret,
1136                         hba->pwr_info.gear_tx, hba->pwr_info.gear_rx,
1137                         new_pwr_info.gear_tx, new_pwr_info.gear_rx);
1138
1139         return ret;
1140 }
1141
1142 static int ufshcd_clock_scaling_prepare(struct ufs_hba *hba)
1143 {
1144         #define DOORBELL_CLR_TOUT_US            (1000 * 1000) /* 1 sec */
1145         int ret = 0;
1146         /*
1147          * make sure that there are no outstanding requests when
1148          * clock scaling is in progress
1149          */
1150         ufshcd_scsi_block_requests(hba);
1151         down_write(&hba->clk_scaling_lock);
1152         if (ufshcd_wait_for_doorbell_clr(hba, DOORBELL_CLR_TOUT_US)) {
1153                 ret = -EBUSY;
1154                 up_write(&hba->clk_scaling_lock);
1155                 ufshcd_scsi_unblock_requests(hba);
1156         }
1157
1158         return ret;
1159 }
1160
1161 static void ufshcd_clock_scaling_unprepare(struct ufs_hba *hba)
1162 {
1163         up_write(&hba->clk_scaling_lock);
1164         ufshcd_scsi_unblock_requests(hba);
1165 }
1166
1167 /**
1168  * ufshcd_devfreq_scale - scale up/down UFS clocks and gear
1169  * @hba: per adapter instance
1170  * @scale_up: True for scaling up and false for scalin down
1171  *
1172  * Returns 0 for success,
1173  * Returns -EBUSY if scaling can't happen at this time
1174  * Returns non-zero for any other errors
1175  */
1176 static int ufshcd_devfreq_scale(struct ufs_hba *hba, bool scale_up)
1177 {
1178         int ret = 0;
1179
1180         /* let's not get into low power until clock scaling is completed */
1181         ufshcd_hold(hba, false);
1182
1183         ret = ufshcd_clock_scaling_prepare(hba);
1184         if (ret)
1185                 return ret;
1186
1187         /* scale down the gear before scaling down clocks */
1188         if (!scale_up) {
1189                 ret = ufshcd_scale_gear(hba, false);
1190                 if (ret)
1191                         goto out;
1192         }
1193
1194         ret = ufshcd_scale_clks(hba, scale_up);
1195         if (ret) {
1196                 if (!scale_up)
1197                         ufshcd_scale_gear(hba, true);
1198                 goto out;
1199         }
1200
1201         /* scale up the gear after scaling up clocks */
1202         if (scale_up) {
1203                 ret = ufshcd_scale_gear(hba, true);
1204                 if (ret) {
1205                         ufshcd_scale_clks(hba, false);
1206                         goto out;
1207                 }
1208         }
1209
1210         ret = ufshcd_vops_clk_scale_notify(hba, scale_up, POST_CHANGE);
1211
1212 out:
1213         ufshcd_clock_scaling_unprepare(hba);
1214         ufshcd_release(hba);
1215         return ret;
1216 }
1217
1218 static void ufshcd_clk_scaling_suspend_work(struct work_struct *work)
1219 {
1220         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1221                                            clk_scaling.suspend_work);
1222         unsigned long irq_flags;
1223
1224         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1225         if (hba->clk_scaling.active_reqs || hba->clk_scaling.is_suspended) {
1226                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1227                 return;
1228         }
1229         hba->clk_scaling.is_suspended = true;
1230         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1231
1232         __ufshcd_suspend_clkscaling(hba);
1233 }
1234
1235 static void ufshcd_clk_scaling_resume_work(struct work_struct *work)
1236 {
1237         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1238                                            clk_scaling.resume_work);
1239         unsigned long irq_flags;
1240
1241         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1242         if (!hba->clk_scaling.is_suspended) {
1243                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1244                 return;
1245         }
1246         hba->clk_scaling.is_suspended = false;
1247         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1248
1249         devfreq_resume_device(hba->devfreq);
1250 }
1251
1252 static int ufshcd_devfreq_target(struct device *dev,
1253                                 unsigned long *freq, u32 flags)
1254 {
1255         int ret = 0;
1256         struct ufs_hba *hba = dev_get_drvdata(dev);
1257         ktime_t start;
1258         bool scale_up, sched_clk_scaling_suspend_work = false;
1259         struct list_head *clk_list = &hba->clk_list_head;
1260         struct ufs_clk_info *clki;
1261         unsigned long irq_flags;
1262
1263         if (!ufshcd_is_clkscaling_supported(hba))
1264                 return -EINVAL;
1265
1266         spin_lock_irqsave(hba->host->host_lock, irq_flags);
1267         if (ufshcd_eh_in_progress(hba)) {
1268                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1269                 return 0;
1270         }
1271
1272         if (!hba->clk_scaling.active_reqs)
1273                 sched_clk_scaling_suspend_work = true;
1274
1275         if (list_empty(clk_list)) {
1276                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1277                 goto out;
1278         }
1279
1280         clki = list_first_entry(&hba->clk_list_head, struct ufs_clk_info, list);
1281         scale_up = (*freq == clki->max_freq) ? true : false;
1282         if (!ufshcd_is_devfreq_scaling_required(hba, scale_up)) {
1283                 spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1284                 ret = 0;
1285                 goto out; /* no state change required */
1286         }
1287         spin_unlock_irqrestore(hba->host->host_lock, irq_flags);
1288
1289         pm_runtime_get_noresume(hba->dev);
1290         if (!pm_runtime_active(hba->dev)) {
1291                 pm_runtime_put_noidle(hba->dev);
1292                 ret = -EAGAIN;
1293                 goto out;
1294         }
1295         start = ktime_get();
1296         ret = ufshcd_devfreq_scale(hba, scale_up);
1297         pm_runtime_put(hba->dev);
1298
1299         trace_ufshcd_profile_clk_scaling(dev_name(hba->dev),
1300                 (scale_up ? "up" : "down"),
1301                 ktime_to_us(ktime_sub(ktime_get(), start)), ret);
1302
1303 out:
1304         if (sched_clk_scaling_suspend_work)
1305                 queue_work(hba->clk_scaling.workq,
1306                            &hba->clk_scaling.suspend_work);
1307
1308         return ret;
1309 }
1310
1311
1312 static int ufshcd_devfreq_get_dev_status(struct device *dev,
1313                 struct devfreq_dev_status *stat)
1314 {
1315         struct ufs_hba *hba = dev_get_drvdata(dev);
1316         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1317         unsigned long flags;
1318
1319         if (!ufshcd_is_clkscaling_supported(hba))
1320                 return -EINVAL;
1321
1322         memset(stat, 0, sizeof(*stat));
1323
1324         spin_lock_irqsave(hba->host->host_lock, flags);
1325         if (!scaling->window_start_t)
1326                 goto start_window;
1327
1328         if (scaling->is_busy_started)
1329                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1330                                         scaling->busy_start_t));
1331
1332         stat->total_time = jiffies_to_usecs((long)jiffies -
1333                                 (long)scaling->window_start_t);
1334         stat->busy_time = scaling->tot_busy_t;
1335 start_window:
1336         scaling->window_start_t = jiffies;
1337         scaling->tot_busy_t = 0;
1338
1339         if (hba->outstanding_reqs) {
1340                 scaling->busy_start_t = ktime_get();
1341                 scaling->is_busy_started = true;
1342         } else {
1343                 scaling->busy_start_t = 0;
1344                 scaling->is_busy_started = false;
1345         }
1346         spin_unlock_irqrestore(hba->host->host_lock, flags);
1347         return 0;
1348 }
1349
1350 static struct devfreq_dev_profile ufs_devfreq_profile = {
1351         .polling_ms     = 100,
1352         .target         = ufshcd_devfreq_target,
1353         .get_dev_status = ufshcd_devfreq_get_dev_status,
1354 };
1355
1356 static int ufshcd_devfreq_init(struct ufs_hba *hba)
1357 {
1358         struct list_head *clk_list = &hba->clk_list_head;
1359         struct ufs_clk_info *clki;
1360         struct devfreq *devfreq;
1361         int ret;
1362
1363         /* Skip devfreq if we don't have any clocks in the list */
1364         if (list_empty(clk_list))
1365                 return 0;
1366
1367         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1368         dev_pm_opp_add(hba->dev, clki->min_freq, 0);
1369         dev_pm_opp_add(hba->dev, clki->max_freq, 0);
1370
1371         devfreq = devfreq_add_device(hba->dev,
1372                         &ufs_devfreq_profile,
1373                         DEVFREQ_GOV_SIMPLE_ONDEMAND,
1374                         NULL);
1375         if (IS_ERR(devfreq)) {
1376                 ret = PTR_ERR(devfreq);
1377                 dev_err(hba->dev, "Unable to register with devfreq %d\n", ret);
1378
1379                 dev_pm_opp_remove(hba->dev, clki->min_freq);
1380                 dev_pm_opp_remove(hba->dev, clki->max_freq);
1381                 return ret;
1382         }
1383
1384         hba->devfreq = devfreq;
1385
1386         return 0;
1387 }
1388
1389 static void ufshcd_devfreq_remove(struct ufs_hba *hba)
1390 {
1391         struct list_head *clk_list = &hba->clk_list_head;
1392         struct ufs_clk_info *clki;
1393
1394         if (!hba->devfreq)
1395                 return;
1396
1397         devfreq_remove_device(hba->devfreq);
1398         hba->devfreq = NULL;
1399
1400         clki = list_first_entry(clk_list, struct ufs_clk_info, list);
1401         dev_pm_opp_remove(hba->dev, clki->min_freq);
1402         dev_pm_opp_remove(hba->dev, clki->max_freq);
1403 }
1404
1405 static void __ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1406 {
1407         unsigned long flags;
1408
1409         devfreq_suspend_device(hba->devfreq);
1410         spin_lock_irqsave(hba->host->host_lock, flags);
1411         hba->clk_scaling.window_start_t = 0;
1412         spin_unlock_irqrestore(hba->host->host_lock, flags);
1413 }
1414
1415 static void ufshcd_suspend_clkscaling(struct ufs_hba *hba)
1416 {
1417         unsigned long flags;
1418         bool suspend = false;
1419
1420         if (!ufshcd_is_clkscaling_supported(hba))
1421                 return;
1422
1423         spin_lock_irqsave(hba->host->host_lock, flags);
1424         if (!hba->clk_scaling.is_suspended) {
1425                 suspend = true;
1426                 hba->clk_scaling.is_suspended = true;
1427         }
1428         spin_unlock_irqrestore(hba->host->host_lock, flags);
1429
1430         if (suspend)
1431                 __ufshcd_suspend_clkscaling(hba);
1432 }
1433
1434 static void ufshcd_resume_clkscaling(struct ufs_hba *hba)
1435 {
1436         unsigned long flags;
1437         bool resume = false;
1438
1439         if (!ufshcd_is_clkscaling_supported(hba))
1440                 return;
1441
1442         spin_lock_irqsave(hba->host->host_lock, flags);
1443         if (hba->clk_scaling.is_suspended) {
1444                 resume = true;
1445                 hba->clk_scaling.is_suspended = false;
1446         }
1447         spin_unlock_irqrestore(hba->host->host_lock, flags);
1448
1449         if (resume)
1450                 devfreq_resume_device(hba->devfreq);
1451 }
1452
1453 static ssize_t ufshcd_clkscale_enable_show(struct device *dev,
1454                 struct device_attribute *attr, char *buf)
1455 {
1456         struct ufs_hba *hba = dev_get_drvdata(dev);
1457
1458         return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_scaling.is_allowed);
1459 }
1460
1461 static ssize_t ufshcd_clkscale_enable_store(struct device *dev,
1462                 struct device_attribute *attr, const char *buf, size_t count)
1463 {
1464         struct ufs_hba *hba = dev_get_drvdata(dev);
1465         u32 value;
1466         int err;
1467
1468         if (kstrtou32(buf, 0, &value))
1469                 return -EINVAL;
1470
1471         value = !!value;
1472         if (value == hba->clk_scaling.is_allowed)
1473                 goto out;
1474
1475         pm_runtime_get_sync(hba->dev);
1476         ufshcd_hold(hba, false);
1477
1478         cancel_work_sync(&hba->clk_scaling.suspend_work);
1479         cancel_work_sync(&hba->clk_scaling.resume_work);
1480
1481         hba->clk_scaling.is_allowed = value;
1482
1483         if (value) {
1484                 ufshcd_resume_clkscaling(hba);
1485         } else {
1486                 ufshcd_suspend_clkscaling(hba);
1487                 err = ufshcd_devfreq_scale(hba, true);
1488                 if (err)
1489                         dev_err(hba->dev, "%s: failed to scale clocks up %d\n",
1490                                         __func__, err);
1491         }
1492
1493         ufshcd_release(hba);
1494         pm_runtime_put_sync(hba->dev);
1495 out:
1496         return count;
1497 }
1498
1499 static void ufshcd_clkscaling_init_sysfs(struct ufs_hba *hba)
1500 {
1501         hba->clk_scaling.enable_attr.show = ufshcd_clkscale_enable_show;
1502         hba->clk_scaling.enable_attr.store = ufshcd_clkscale_enable_store;
1503         sysfs_attr_init(&hba->clk_scaling.enable_attr.attr);
1504         hba->clk_scaling.enable_attr.attr.name = "clkscale_enable";
1505         hba->clk_scaling.enable_attr.attr.mode = 0644;
1506         if (device_create_file(hba->dev, &hba->clk_scaling.enable_attr))
1507                 dev_err(hba->dev, "Failed to create sysfs for clkscale_enable\n");
1508 }
1509
1510 static void ufshcd_ungate_work(struct work_struct *work)
1511 {
1512         int ret;
1513         unsigned long flags;
1514         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1515                         clk_gating.ungate_work);
1516
1517         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1518
1519         spin_lock_irqsave(hba->host->host_lock, flags);
1520         if (hba->clk_gating.state == CLKS_ON) {
1521                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1522                 goto unblock_reqs;
1523         }
1524
1525         spin_unlock_irqrestore(hba->host->host_lock, flags);
1526         ufshcd_setup_clocks(hba, true);
1527
1528         /* Exit from hibern8 */
1529         if (ufshcd_can_hibern8_during_gating(hba)) {
1530                 /* Prevent gating in this path */
1531                 hba->clk_gating.is_suspended = true;
1532                 if (ufshcd_is_link_hibern8(hba)) {
1533                         ret = ufshcd_uic_hibern8_exit(hba);
1534                         if (ret)
1535                                 dev_err(hba->dev, "%s: hibern8 exit failed %d\n",
1536                                         __func__, ret);
1537                         else
1538                                 ufshcd_set_link_active(hba);
1539                 }
1540                 hba->clk_gating.is_suspended = false;
1541         }
1542 unblock_reqs:
1543         ufshcd_scsi_unblock_requests(hba);
1544 }
1545
1546 /**
1547  * ufshcd_hold - Enable clocks that were gated earlier due to ufshcd_release.
1548  * Also, exit from hibern8 mode and set the link as active.
1549  * @hba: per adapter instance
1550  * @async: This indicates whether caller should ungate clocks asynchronously.
1551  */
1552 int ufshcd_hold(struct ufs_hba *hba, bool async)
1553 {
1554         int rc = 0;
1555         bool flush_result;
1556         unsigned long flags;
1557
1558         if (!ufshcd_is_clkgating_allowed(hba))
1559                 goto out;
1560         spin_lock_irqsave(hba->host->host_lock, flags);
1561         hba->clk_gating.active_reqs++;
1562
1563         if (ufshcd_eh_in_progress(hba)) {
1564                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1565                 return 0;
1566         }
1567
1568 start:
1569         switch (hba->clk_gating.state) {
1570         case CLKS_ON:
1571                 /*
1572                  * Wait for the ungate work to complete if in progress.
1573                  * Though the clocks may be in ON state, the link could
1574                  * still be in hibner8 state if hibern8 is allowed
1575                  * during clock gating.
1576                  * Make sure we exit hibern8 state also in addition to
1577                  * clocks being ON.
1578                  */
1579                 if (ufshcd_can_hibern8_during_gating(hba) &&
1580                     ufshcd_is_link_hibern8(hba)) {
1581                         if (async) {
1582                                 rc = -EAGAIN;
1583                                 hba->clk_gating.active_reqs--;
1584                                 break;
1585                         }
1586                         spin_unlock_irqrestore(hba->host->host_lock, flags);
1587                         flush_result = flush_work(&hba->clk_gating.ungate_work);
1588                         if (hba->clk_gating.is_suspended && !flush_result)
1589                                 goto out;
1590                         spin_lock_irqsave(hba->host->host_lock, flags);
1591                         goto start;
1592                 }
1593                 break;
1594         case REQ_CLKS_OFF:
1595                 if (cancel_delayed_work(&hba->clk_gating.gate_work)) {
1596                         hba->clk_gating.state = CLKS_ON;
1597                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1598                                                 hba->clk_gating.state);
1599                         break;
1600                 }
1601                 /*
1602                  * If we are here, it means gating work is either done or
1603                  * currently running. Hence, fall through to cancel gating
1604                  * work and to enable clocks.
1605                  */
1606         case CLKS_OFF:
1607                 hba->clk_gating.state = REQ_CLKS_ON;
1608                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1609                                         hba->clk_gating.state);
1610                 if (queue_work(hba->clk_gating.clk_gating_workq,
1611                                &hba->clk_gating.ungate_work))
1612                         ufshcd_scsi_block_requests(hba);
1613                 /*
1614                  * fall through to check if we should wait for this
1615                  * work to be done or not.
1616                  */
1617         case REQ_CLKS_ON:
1618                 if (async) {
1619                         rc = -EAGAIN;
1620                         hba->clk_gating.active_reqs--;
1621                         break;
1622                 }
1623
1624                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1625                 flush_work(&hba->clk_gating.ungate_work);
1626                 /* Make sure state is CLKS_ON before returning */
1627                 spin_lock_irqsave(hba->host->host_lock, flags);
1628                 goto start;
1629         default:
1630                 dev_err(hba->dev, "%s: clk gating is in invalid state %d\n",
1631                                 __func__, hba->clk_gating.state);
1632                 break;
1633         }
1634         spin_unlock_irqrestore(hba->host->host_lock, flags);
1635 out:
1636         return rc;
1637 }
1638 EXPORT_SYMBOL_GPL(ufshcd_hold);
1639
1640 static void ufshcd_gate_work(struct work_struct *work)
1641 {
1642         struct ufs_hba *hba = container_of(work, struct ufs_hba,
1643                         clk_gating.gate_work.work);
1644         unsigned long flags;
1645
1646         spin_lock_irqsave(hba->host->host_lock, flags);
1647         /*
1648          * In case you are here to cancel this work the gating state
1649          * would be marked as REQ_CLKS_ON. In this case save time by
1650          * skipping the gating work and exit after changing the clock
1651          * state to CLKS_ON.
1652          */
1653         if (hba->clk_gating.is_suspended ||
1654                 (hba->clk_gating.state == REQ_CLKS_ON)) {
1655                 hba->clk_gating.state = CLKS_ON;
1656                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1657                                         hba->clk_gating.state);
1658                 goto rel_lock;
1659         }
1660
1661         if (hba->clk_gating.active_reqs
1662                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1663                 || hba->lrb_in_use || hba->outstanding_tasks
1664                 || hba->active_uic_cmd || hba->uic_async_done)
1665                 goto rel_lock;
1666
1667         spin_unlock_irqrestore(hba->host->host_lock, flags);
1668
1669         /* put the link into hibern8 mode before turning off clocks */
1670         if (ufshcd_can_hibern8_during_gating(hba)) {
1671                 if (ufshcd_uic_hibern8_enter(hba)) {
1672                         hba->clk_gating.state = CLKS_ON;
1673                         trace_ufshcd_clk_gating(dev_name(hba->dev),
1674                                                 hba->clk_gating.state);
1675                         goto out;
1676                 }
1677                 ufshcd_set_link_hibern8(hba);
1678         }
1679
1680         if (!ufshcd_is_link_active(hba))
1681                 ufshcd_setup_clocks(hba, false);
1682         else
1683                 /* If link is active, device ref_clk can't be switched off */
1684                 __ufshcd_setup_clocks(hba, false, true);
1685
1686         /*
1687          * In case you are here to cancel this work the gating state
1688          * would be marked as REQ_CLKS_ON. In this case keep the state
1689          * as REQ_CLKS_ON which would anyway imply that clocks are off
1690          * and a request to turn them on is pending. By doing this way,
1691          * we keep the state machine in tact and this would ultimately
1692          * prevent from doing cancel work multiple times when there are
1693          * new requests arriving before the current cancel work is done.
1694          */
1695         spin_lock_irqsave(hba->host->host_lock, flags);
1696         if (hba->clk_gating.state == REQ_CLKS_OFF) {
1697                 hba->clk_gating.state = CLKS_OFF;
1698                 trace_ufshcd_clk_gating(dev_name(hba->dev),
1699                                         hba->clk_gating.state);
1700         }
1701 rel_lock:
1702         spin_unlock_irqrestore(hba->host->host_lock, flags);
1703 out:
1704         return;
1705 }
1706
1707 /* host lock must be held before calling this variant */
1708 static void __ufshcd_release(struct ufs_hba *hba)
1709 {
1710         if (!ufshcd_is_clkgating_allowed(hba))
1711                 return;
1712
1713         hba->clk_gating.active_reqs--;
1714
1715         if (hba->clk_gating.active_reqs || hba->clk_gating.is_suspended
1716                 || hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL
1717                 || hba->lrb_in_use || hba->outstanding_tasks
1718                 || hba->active_uic_cmd || hba->uic_async_done
1719                 || ufshcd_eh_in_progress(hba))
1720                 return;
1721
1722         hba->clk_gating.state = REQ_CLKS_OFF;
1723         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
1724         queue_delayed_work(hba->clk_gating.clk_gating_workq,
1725                            &hba->clk_gating.gate_work,
1726                            msecs_to_jiffies(hba->clk_gating.delay_ms));
1727 }
1728
1729 void ufshcd_release(struct ufs_hba *hba)
1730 {
1731         unsigned long flags;
1732
1733         spin_lock_irqsave(hba->host->host_lock, flags);
1734         __ufshcd_release(hba);
1735         spin_unlock_irqrestore(hba->host->host_lock, flags);
1736 }
1737 EXPORT_SYMBOL_GPL(ufshcd_release);
1738
1739 static ssize_t ufshcd_clkgate_delay_show(struct device *dev,
1740                 struct device_attribute *attr, char *buf)
1741 {
1742         struct ufs_hba *hba = dev_get_drvdata(dev);
1743
1744         return snprintf(buf, PAGE_SIZE, "%lu\n", hba->clk_gating.delay_ms);
1745 }
1746
1747 static ssize_t ufshcd_clkgate_delay_store(struct device *dev,
1748                 struct device_attribute *attr, const char *buf, size_t count)
1749 {
1750         struct ufs_hba *hba = dev_get_drvdata(dev);
1751         unsigned long flags, value;
1752
1753         if (kstrtoul(buf, 0, &value))
1754                 return -EINVAL;
1755
1756         spin_lock_irqsave(hba->host->host_lock, flags);
1757         hba->clk_gating.delay_ms = value;
1758         spin_unlock_irqrestore(hba->host->host_lock, flags);
1759         return count;
1760 }
1761
1762 static ssize_t ufshcd_clkgate_enable_show(struct device *dev,
1763                 struct device_attribute *attr, char *buf)
1764 {
1765         struct ufs_hba *hba = dev_get_drvdata(dev);
1766
1767         return snprintf(buf, PAGE_SIZE, "%d\n", hba->clk_gating.is_enabled);
1768 }
1769
1770 static ssize_t ufshcd_clkgate_enable_store(struct device *dev,
1771                 struct device_attribute *attr, const char *buf, size_t count)
1772 {
1773         struct ufs_hba *hba = dev_get_drvdata(dev);
1774         unsigned long flags;
1775         u32 value;
1776
1777         if (kstrtou32(buf, 0, &value))
1778                 return -EINVAL;
1779
1780         value = !!value;
1781         if (value == hba->clk_gating.is_enabled)
1782                 goto out;
1783
1784         if (value) {
1785                 ufshcd_release(hba);
1786         } else {
1787                 spin_lock_irqsave(hba->host->host_lock, flags);
1788                 hba->clk_gating.active_reqs++;
1789                 spin_unlock_irqrestore(hba->host->host_lock, flags);
1790         }
1791
1792         hba->clk_gating.is_enabled = value;
1793 out:
1794         return count;
1795 }
1796
1797 static void ufshcd_init_clk_scaling(struct ufs_hba *hba)
1798 {
1799         char wq_name[sizeof("ufs_clkscaling_00")];
1800
1801         if (!ufshcd_is_clkscaling_supported(hba))
1802                 return;
1803
1804         INIT_WORK(&hba->clk_scaling.suspend_work,
1805                   ufshcd_clk_scaling_suspend_work);
1806         INIT_WORK(&hba->clk_scaling.resume_work,
1807                   ufshcd_clk_scaling_resume_work);
1808
1809         snprintf(wq_name, sizeof(wq_name), "ufs_clkscaling_%d",
1810                  hba->host->host_no);
1811         hba->clk_scaling.workq = create_singlethread_workqueue(wq_name);
1812
1813         ufshcd_clkscaling_init_sysfs(hba);
1814 }
1815
1816 static void ufshcd_exit_clk_scaling(struct ufs_hba *hba)
1817 {
1818         if (!ufshcd_is_clkscaling_supported(hba))
1819                 return;
1820
1821         destroy_workqueue(hba->clk_scaling.workq);
1822         ufshcd_devfreq_remove(hba);
1823 }
1824
1825 static void ufshcd_init_clk_gating(struct ufs_hba *hba)
1826 {
1827         char wq_name[sizeof("ufs_clk_gating_00")];
1828
1829         if (!ufshcd_is_clkgating_allowed(hba))
1830                 return;
1831
1832         hba->clk_gating.delay_ms = 150;
1833         INIT_DELAYED_WORK(&hba->clk_gating.gate_work, ufshcd_gate_work);
1834         INIT_WORK(&hba->clk_gating.ungate_work, ufshcd_ungate_work);
1835
1836         snprintf(wq_name, ARRAY_SIZE(wq_name), "ufs_clk_gating_%d",
1837                  hba->host->host_no);
1838         hba->clk_gating.clk_gating_workq = alloc_ordered_workqueue(wq_name,
1839                                                            WQ_MEM_RECLAIM);
1840
1841         hba->clk_gating.is_enabled = true;
1842
1843         hba->clk_gating.delay_attr.show = ufshcd_clkgate_delay_show;
1844         hba->clk_gating.delay_attr.store = ufshcd_clkgate_delay_store;
1845         sysfs_attr_init(&hba->clk_gating.delay_attr.attr);
1846         hba->clk_gating.delay_attr.attr.name = "clkgate_delay_ms";
1847         hba->clk_gating.delay_attr.attr.mode = 0644;
1848         if (device_create_file(hba->dev, &hba->clk_gating.delay_attr))
1849                 dev_err(hba->dev, "Failed to create sysfs for clkgate_delay\n");
1850
1851         hba->clk_gating.enable_attr.show = ufshcd_clkgate_enable_show;
1852         hba->clk_gating.enable_attr.store = ufshcd_clkgate_enable_store;
1853         sysfs_attr_init(&hba->clk_gating.enable_attr.attr);
1854         hba->clk_gating.enable_attr.attr.name = "clkgate_enable";
1855         hba->clk_gating.enable_attr.attr.mode = 0644;
1856         if (device_create_file(hba->dev, &hba->clk_gating.enable_attr))
1857                 dev_err(hba->dev, "Failed to create sysfs for clkgate_enable\n");
1858 }
1859
1860 static void ufshcd_exit_clk_gating(struct ufs_hba *hba)
1861 {
1862         if (!ufshcd_is_clkgating_allowed(hba))
1863                 return;
1864         device_remove_file(hba->dev, &hba->clk_gating.delay_attr);
1865         device_remove_file(hba->dev, &hba->clk_gating.enable_attr);
1866         cancel_work_sync(&hba->clk_gating.ungate_work);
1867         cancel_delayed_work_sync(&hba->clk_gating.gate_work);
1868         destroy_workqueue(hba->clk_gating.clk_gating_workq);
1869 }
1870
1871 /* Must be called with host lock acquired */
1872 static void ufshcd_clk_scaling_start_busy(struct ufs_hba *hba)
1873 {
1874         bool queue_resume_work = false;
1875
1876         if (!ufshcd_is_clkscaling_supported(hba))
1877                 return;
1878
1879         if (!hba->clk_scaling.active_reqs++)
1880                 queue_resume_work = true;
1881
1882         if (!hba->clk_scaling.is_allowed || hba->pm_op_in_progress)
1883                 return;
1884
1885         if (queue_resume_work)
1886                 queue_work(hba->clk_scaling.workq,
1887                            &hba->clk_scaling.resume_work);
1888
1889         if (!hba->clk_scaling.window_start_t) {
1890                 hba->clk_scaling.window_start_t = jiffies;
1891                 hba->clk_scaling.tot_busy_t = 0;
1892                 hba->clk_scaling.is_busy_started = false;
1893         }
1894
1895         if (!hba->clk_scaling.is_busy_started) {
1896                 hba->clk_scaling.busy_start_t = ktime_get();
1897                 hba->clk_scaling.is_busy_started = true;
1898         }
1899 }
1900
1901 static void ufshcd_clk_scaling_update_busy(struct ufs_hba *hba)
1902 {
1903         struct ufs_clk_scaling *scaling = &hba->clk_scaling;
1904
1905         if (!ufshcd_is_clkscaling_supported(hba))
1906                 return;
1907
1908         if (!hba->outstanding_reqs && scaling->is_busy_started) {
1909                 scaling->tot_busy_t += ktime_to_us(ktime_sub(ktime_get(),
1910                                         scaling->busy_start_t));
1911                 scaling->busy_start_t = 0;
1912                 scaling->is_busy_started = false;
1913         }
1914 }
1915 /**
1916  * ufshcd_send_command - Send SCSI or device management commands
1917  * @hba: per adapter instance
1918  * @task_tag: Task tag of the command
1919  */
1920 static inline
1921 void ufshcd_send_command(struct ufs_hba *hba, unsigned int task_tag)
1922 {
1923         hba->lrb[task_tag].issue_time_stamp = ktime_get();
1924         hba->lrb[task_tag].compl_time_stamp = ktime_set(0, 0);
1925         ufshcd_add_command_trace(hba, task_tag, "send");
1926         ufshcd_clk_scaling_start_busy(hba);
1927         __set_bit(task_tag, &hba->outstanding_reqs);
1928         ufshcd_writel(hba, 1 << task_tag, REG_UTP_TRANSFER_REQ_DOOR_BELL);
1929         /* Make sure that doorbell is committed immediately */
1930         wmb();
1931 }
1932
1933 /**
1934  * ufshcd_copy_sense_data - Copy sense data in case of check condition
1935  * @lrbp: pointer to local reference block
1936  */
1937 static inline void ufshcd_copy_sense_data(struct ufshcd_lrb *lrbp)
1938 {
1939         int len;
1940         if (lrbp->sense_buffer &&
1941             ufshcd_get_rsp_upiu_data_seg_len(lrbp->ucd_rsp_ptr)) {
1942                 int len_to_copy;
1943
1944                 len = be16_to_cpu(lrbp->ucd_rsp_ptr->sr.sense_data_len);
1945                 len_to_copy = min_t(int, RESPONSE_UPIU_SENSE_DATA_LENGTH, len);
1946
1947                 memcpy(lrbp->sense_buffer,
1948                         lrbp->ucd_rsp_ptr->sr.sense_data,
1949                         min_t(int, len_to_copy, UFSHCD_REQ_SENSE_SIZE));
1950         }
1951 }
1952
1953 /**
1954  * ufshcd_copy_query_response() - Copy the Query Response and the data
1955  * descriptor
1956  * @hba: per adapter instance
1957  * @lrbp: pointer to local reference block
1958  */
1959 static
1960 int ufshcd_copy_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
1961 {
1962         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
1963
1964         memcpy(&query_res->upiu_res, &lrbp->ucd_rsp_ptr->qr, QUERY_OSF_SIZE);
1965
1966         /* Get the descriptor */
1967         if (hba->dev_cmd.query.descriptor &&
1968             lrbp->ucd_rsp_ptr->qr.opcode == UPIU_QUERY_OPCODE_READ_DESC) {
1969                 u8 *descp = (u8 *)lrbp->ucd_rsp_ptr +
1970                                 GENERAL_UPIU_REQUEST_SIZE;
1971                 u16 resp_len;
1972                 u16 buf_len;
1973
1974                 /* data segment length */
1975                 resp_len = be32_to_cpu(lrbp->ucd_rsp_ptr->header.dword_2) &
1976                                                 MASK_QUERY_DATA_SEG_LEN;
1977                 buf_len = be16_to_cpu(
1978                                 hba->dev_cmd.query.request.upiu_req.length);
1979                 if (likely(buf_len >= resp_len)) {
1980                         memcpy(hba->dev_cmd.query.descriptor, descp, resp_len);
1981                 } else {
1982                         dev_warn(hba->dev,
1983                                 "%s: Response size is bigger than buffer",
1984                                 __func__);
1985                         return -EINVAL;
1986                 }
1987         }
1988
1989         return 0;
1990 }
1991
1992 /**
1993  * ufshcd_hba_capabilities - Read controller capabilities
1994  * @hba: per adapter instance
1995  */
1996 static inline void ufshcd_hba_capabilities(struct ufs_hba *hba)
1997 {
1998         hba->capabilities = ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES);
1999
2000         /* nutrs and nutmrs are 0 based values */
2001         hba->nutrs = (hba->capabilities & MASK_TRANSFER_REQUESTS_SLOTS) + 1;
2002         hba->nutmrs =
2003         ((hba->capabilities & MASK_TASK_MANAGEMENT_REQUEST_SLOTS) >> 16) + 1;
2004 }
2005
2006 /**
2007  * ufshcd_ready_for_uic_cmd - Check if controller is ready
2008  *                            to accept UIC commands
2009  * @hba: per adapter instance
2010  * Return true on success, else false
2011  */
2012 static inline bool ufshcd_ready_for_uic_cmd(struct ufs_hba *hba)
2013 {
2014         if (ufshcd_readl(hba, REG_CONTROLLER_STATUS) & UIC_COMMAND_READY)
2015                 return true;
2016         else
2017                 return false;
2018 }
2019
2020 /**
2021  * ufshcd_get_upmcrs - Get the power mode change request status
2022  * @hba: Pointer to adapter instance
2023  *
2024  * This function gets the UPMCRS field of HCS register
2025  * Returns value of UPMCRS field
2026  */
2027 static inline u8 ufshcd_get_upmcrs(struct ufs_hba *hba)
2028 {
2029         return (ufshcd_readl(hba, REG_CONTROLLER_STATUS) >> 8) & 0x7;
2030 }
2031
2032 /**
2033  * ufshcd_dispatch_uic_cmd - Dispatch UIC commands to unipro layers
2034  * @hba: per adapter instance
2035  * @uic_cmd: UIC command
2036  *
2037  * Mutex must be held.
2038  */
2039 static inline void
2040 ufshcd_dispatch_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2041 {
2042         WARN_ON(hba->active_uic_cmd);
2043
2044         hba->active_uic_cmd = uic_cmd;
2045
2046         /* Write Args */
2047         ufshcd_writel(hba, uic_cmd->argument1, REG_UIC_COMMAND_ARG_1);
2048         ufshcd_writel(hba, uic_cmd->argument2, REG_UIC_COMMAND_ARG_2);
2049         ufshcd_writel(hba, uic_cmd->argument3, REG_UIC_COMMAND_ARG_3);
2050
2051         /* Write UIC Cmd */
2052         ufshcd_writel(hba, uic_cmd->command & COMMAND_OPCODE_MASK,
2053                       REG_UIC_COMMAND);
2054 }
2055
2056 /**
2057  * ufshcd_wait_for_uic_cmd - Wait complectioin of UIC command
2058  * @hba: per adapter instance
2059  * @uic_cmd: UIC command
2060  *
2061  * Must be called with mutex held.
2062  * Returns 0 only if success.
2063  */
2064 static int
2065 ufshcd_wait_for_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2066 {
2067         int ret;
2068         unsigned long flags;
2069
2070         if (wait_for_completion_timeout(&uic_cmd->done,
2071                                         msecs_to_jiffies(UIC_CMD_TIMEOUT)))
2072                 ret = uic_cmd->argument2 & MASK_UIC_COMMAND_RESULT;
2073         else
2074                 ret = -ETIMEDOUT;
2075
2076         spin_lock_irqsave(hba->host->host_lock, flags);
2077         hba->active_uic_cmd = NULL;
2078         spin_unlock_irqrestore(hba->host->host_lock, flags);
2079
2080         return ret;
2081 }
2082
2083 /**
2084  * __ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2085  * @hba: per adapter instance
2086  * @uic_cmd: UIC command
2087  * @completion: initialize the completion only if this is set to true
2088  *
2089  * Identical to ufshcd_send_uic_cmd() expect mutex. Must be called
2090  * with mutex held and host_lock locked.
2091  * Returns 0 only if success.
2092  */
2093 static int
2094 __ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd,
2095                       bool completion)
2096 {
2097         if (!ufshcd_ready_for_uic_cmd(hba)) {
2098                 dev_err(hba->dev,
2099                         "Controller not ready to accept UIC commands\n");
2100                 return -EIO;
2101         }
2102
2103         if (completion)
2104                 init_completion(&uic_cmd->done);
2105
2106         ufshcd_dispatch_uic_cmd(hba, uic_cmd);
2107
2108         return 0;
2109 }
2110
2111 /**
2112  * ufshcd_send_uic_cmd - Send UIC commands and retrieve the result
2113  * @hba: per adapter instance
2114  * @uic_cmd: UIC command
2115  *
2116  * Returns 0 only if success.
2117  */
2118 static int
2119 ufshcd_send_uic_cmd(struct ufs_hba *hba, struct uic_command *uic_cmd)
2120 {
2121         int ret;
2122         unsigned long flags;
2123
2124         ufshcd_hold(hba, false);
2125         mutex_lock(&hba->uic_cmd_mutex);
2126         ufshcd_add_delay_before_dme_cmd(hba);
2127
2128         spin_lock_irqsave(hba->host->host_lock, flags);
2129         ret = __ufshcd_send_uic_cmd(hba, uic_cmd, true);
2130         spin_unlock_irqrestore(hba->host->host_lock, flags);
2131         if (!ret)
2132                 ret = ufshcd_wait_for_uic_cmd(hba, uic_cmd);
2133
2134         mutex_unlock(&hba->uic_cmd_mutex);
2135
2136         ufshcd_release(hba);
2137         return ret;
2138 }
2139
2140 /**
2141  * ufshcd_map_sg - Map scatter-gather list to prdt
2142  * @hba: per adapter instance
2143  * @lrbp: pointer to local reference block
2144  *
2145  * Returns 0 in case of success, non-zero value in case of failure
2146  */
2147 static int ufshcd_map_sg(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2148 {
2149         struct ufshcd_sg_entry *prd_table;
2150         struct scatterlist *sg;
2151         struct scsi_cmnd *cmd;
2152         int sg_segments;
2153         int i;
2154
2155         cmd = lrbp->cmd;
2156         sg_segments = scsi_dma_map(cmd);
2157         if (sg_segments < 0)
2158                 return sg_segments;
2159
2160         if (sg_segments) {
2161                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN)
2162                         lrbp->utr_descriptor_ptr->prd_table_length =
2163                                 cpu_to_le16((u16)(sg_segments *
2164                                         sizeof(struct ufshcd_sg_entry)));
2165                 else
2166                         lrbp->utr_descriptor_ptr->prd_table_length =
2167                                 cpu_to_le16((u16) (sg_segments));
2168
2169                 prd_table = (struct ufshcd_sg_entry *)lrbp->ucd_prdt_ptr;
2170
2171                 scsi_for_each_sg(cmd, sg, sg_segments, i) {
2172                         prd_table[i].size  =
2173                                 cpu_to_le32(((u32) sg_dma_len(sg))-1);
2174                         prd_table[i].base_addr =
2175                                 cpu_to_le32(lower_32_bits(sg->dma_address));
2176                         prd_table[i].upper_addr =
2177                                 cpu_to_le32(upper_32_bits(sg->dma_address));
2178                         prd_table[i].reserved = 0;
2179                 }
2180         } else {
2181                 lrbp->utr_descriptor_ptr->prd_table_length = 0;
2182         }
2183
2184         return 0;
2185 }
2186
2187 /**
2188  * ufshcd_enable_intr - enable interrupts
2189  * @hba: per adapter instance
2190  * @intrs: interrupt bits
2191  */
2192 static void ufshcd_enable_intr(struct ufs_hba *hba, u32 intrs)
2193 {
2194         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2195
2196         if (hba->ufs_version == UFSHCI_VERSION_10) {
2197                 u32 rw;
2198                 rw = set & INTERRUPT_MASK_RW_VER_10;
2199                 set = rw | ((set ^ intrs) & intrs);
2200         } else {
2201                 set |= intrs;
2202         }
2203
2204         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2205 }
2206
2207 /**
2208  * ufshcd_disable_intr - disable interrupts
2209  * @hba: per adapter instance
2210  * @intrs: interrupt bits
2211  */
2212 static void ufshcd_disable_intr(struct ufs_hba *hba, u32 intrs)
2213 {
2214         u32 set = ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
2215
2216         if (hba->ufs_version == UFSHCI_VERSION_10) {
2217                 u32 rw;
2218                 rw = (set & INTERRUPT_MASK_RW_VER_10) &
2219                         ~(intrs & INTERRUPT_MASK_RW_VER_10);
2220                 set = rw | ((set & intrs) & ~INTERRUPT_MASK_RW_VER_10);
2221
2222         } else {
2223                 set &= ~intrs;
2224         }
2225
2226         ufshcd_writel(hba, set, REG_INTERRUPT_ENABLE);
2227 }
2228
2229 /**
2230  * ufshcd_prepare_req_desc_hdr() - Fills the requests header
2231  * descriptor according to request
2232  * @lrbp: pointer to local reference block
2233  * @upiu_flags: flags required in the header
2234  * @cmd_dir: requests data direction
2235  */
2236 static void ufshcd_prepare_req_desc_hdr(struct ufshcd_lrb *lrbp,
2237                         u32 *upiu_flags, enum dma_data_direction cmd_dir)
2238 {
2239         struct utp_transfer_req_desc *req_desc = lrbp->utr_descriptor_ptr;
2240         u32 data_direction;
2241         u32 dword_0;
2242
2243         if (cmd_dir == DMA_FROM_DEVICE) {
2244                 data_direction = UTP_DEVICE_TO_HOST;
2245                 *upiu_flags = UPIU_CMD_FLAGS_READ;
2246         } else if (cmd_dir == DMA_TO_DEVICE) {
2247                 data_direction = UTP_HOST_TO_DEVICE;
2248                 *upiu_flags = UPIU_CMD_FLAGS_WRITE;
2249         } else {
2250                 data_direction = UTP_NO_DATA_TRANSFER;
2251                 *upiu_flags = UPIU_CMD_FLAGS_NONE;
2252         }
2253
2254         dword_0 = data_direction | (lrbp->command_type
2255                                 << UPIU_COMMAND_TYPE_OFFSET);
2256         if (lrbp->intr_cmd)
2257                 dword_0 |= UTP_REQ_DESC_INT_CMD;
2258
2259         /* Transfer request descriptor header fields */
2260         req_desc->header.dword_0 = cpu_to_le32(dword_0);
2261         /* dword_1 is reserved, hence it is set to 0 */
2262         req_desc->header.dword_1 = 0;
2263         /*
2264          * assigning invalid value for command status. Controller
2265          * updates OCS on command completion, with the command
2266          * status
2267          */
2268         req_desc->header.dword_2 =
2269                 cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
2270         /* dword_3 is reserved, hence it is set to 0 */
2271         req_desc->header.dword_3 = 0;
2272
2273         req_desc->prd_table_length = 0;
2274 }
2275
2276 /**
2277  * ufshcd_prepare_utp_scsi_cmd_upiu() - fills the utp_transfer_req_desc,
2278  * for scsi commands
2279  * @lrbp: local reference block pointer
2280  * @upiu_flags: flags
2281  */
2282 static
2283 void ufshcd_prepare_utp_scsi_cmd_upiu(struct ufshcd_lrb *lrbp, u32 upiu_flags)
2284 {
2285         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2286         unsigned short cdb_len;
2287
2288         /* command descriptor fields */
2289         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2290                                 UPIU_TRANSACTION_COMMAND, upiu_flags,
2291                                 lrbp->lun, lrbp->task_tag);
2292         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2293                                 UPIU_COMMAND_SET_TYPE_SCSI, 0, 0, 0);
2294
2295         /* Total EHS length and Data segment length will be zero */
2296         ucd_req_ptr->header.dword_2 = 0;
2297
2298         ucd_req_ptr->sc.exp_data_transfer_len =
2299                 cpu_to_be32(lrbp->cmd->sdb.length);
2300
2301         cdb_len = min_t(unsigned short, lrbp->cmd->cmd_len, MAX_CDB_SIZE);
2302         memset(ucd_req_ptr->sc.cdb, 0, MAX_CDB_SIZE);
2303         memcpy(ucd_req_ptr->sc.cdb, lrbp->cmd->cmnd, cdb_len);
2304
2305         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2306 }
2307
2308 /**
2309  * ufshcd_prepare_utp_query_req_upiu() - fills the utp_transfer_req_desc,
2310  * for query requsts
2311  * @hba: UFS hba
2312  * @lrbp: local reference block pointer
2313  * @upiu_flags: flags
2314  */
2315 static void ufshcd_prepare_utp_query_req_upiu(struct ufs_hba *hba,
2316                                 struct ufshcd_lrb *lrbp, u32 upiu_flags)
2317 {
2318         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2319         struct ufs_query *query = &hba->dev_cmd.query;
2320         u16 len = be16_to_cpu(query->request.upiu_req.length);
2321         u8 *descp = (u8 *)lrbp->ucd_req_ptr + GENERAL_UPIU_REQUEST_SIZE;
2322
2323         /* Query request header */
2324         ucd_req_ptr->header.dword_0 = UPIU_HEADER_DWORD(
2325                         UPIU_TRANSACTION_QUERY_REQ, upiu_flags,
2326                         lrbp->lun, lrbp->task_tag);
2327         ucd_req_ptr->header.dword_1 = UPIU_HEADER_DWORD(
2328                         0, query->request.query_func, 0, 0);
2329
2330         /* Data segment length only need for WRITE_DESC */
2331         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2332                 ucd_req_ptr->header.dword_2 =
2333                         UPIU_HEADER_DWORD(0, 0, (len >> 8), (u8)len);
2334         else
2335                 ucd_req_ptr->header.dword_2 = 0;
2336
2337         /* Copy the Query Request buffer as is */
2338         memcpy(&ucd_req_ptr->qr, &query->request.upiu_req,
2339                         QUERY_OSF_SIZE);
2340
2341         /* Copy the Descriptor */
2342         if (query->request.upiu_req.opcode == UPIU_QUERY_OPCODE_WRITE_DESC)
2343                 memcpy(descp, query->descriptor, len);
2344
2345         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2346 }
2347
2348 static inline void ufshcd_prepare_utp_nop_upiu(struct ufshcd_lrb *lrbp)
2349 {
2350         struct utp_upiu_req *ucd_req_ptr = lrbp->ucd_req_ptr;
2351
2352         memset(ucd_req_ptr, 0, sizeof(struct utp_upiu_req));
2353
2354         /* command descriptor fields */
2355         ucd_req_ptr->header.dword_0 =
2356                 UPIU_HEADER_DWORD(
2357                         UPIU_TRANSACTION_NOP_OUT, 0, 0, lrbp->task_tag);
2358         /* clear rest of the fields of basic header */
2359         ucd_req_ptr->header.dword_1 = 0;
2360         ucd_req_ptr->header.dword_2 = 0;
2361
2362         memset(lrbp->ucd_rsp_ptr, 0, sizeof(struct utp_upiu_rsp));
2363 }
2364
2365 /**
2366  * ufshcd_comp_devman_upiu - UFS Protocol Information Unit(UPIU)
2367  *                           for Device Management Purposes
2368  * @hba: per adapter instance
2369  * @lrbp: pointer to local reference block
2370  */
2371 static int ufshcd_comp_devman_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2372 {
2373         u32 upiu_flags;
2374         int ret = 0;
2375
2376         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2377             (hba->ufs_version == UFSHCI_VERSION_11))
2378                 lrbp->command_type = UTP_CMD_TYPE_DEV_MANAGE;
2379         else
2380                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2381
2382         ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags, DMA_NONE);
2383         if (hba->dev_cmd.type == DEV_CMD_TYPE_QUERY)
2384                 ufshcd_prepare_utp_query_req_upiu(hba, lrbp, upiu_flags);
2385         else if (hba->dev_cmd.type == DEV_CMD_TYPE_NOP)
2386                 ufshcd_prepare_utp_nop_upiu(lrbp);
2387         else
2388                 ret = -EINVAL;
2389
2390         return ret;
2391 }
2392
2393 /**
2394  * ufshcd_comp_scsi_upiu - UFS Protocol Information Unit(UPIU)
2395  *                         for SCSI Purposes
2396  * @hba: per adapter instance
2397  * @lrbp: pointer to local reference block
2398  */
2399 static int ufshcd_comp_scsi_upiu(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2400 {
2401         u32 upiu_flags;
2402         int ret = 0;
2403
2404         if ((hba->ufs_version == UFSHCI_VERSION_10) ||
2405             (hba->ufs_version == UFSHCI_VERSION_11))
2406                 lrbp->command_type = UTP_CMD_TYPE_SCSI;
2407         else
2408                 lrbp->command_type = UTP_CMD_TYPE_UFS_STORAGE;
2409
2410         if (likely(lrbp->cmd)) {
2411                 ufshcd_prepare_req_desc_hdr(lrbp, &upiu_flags,
2412                                                 lrbp->cmd->sc_data_direction);
2413                 ufshcd_prepare_utp_scsi_cmd_upiu(lrbp, upiu_flags);
2414         } else {
2415                 ret = -EINVAL;
2416         }
2417
2418         return ret;
2419 }
2420
2421 /**
2422  * ufshcd_upiu_wlun_to_scsi_wlun - maps UPIU W-LUN id to SCSI W-LUN ID
2423  * @upiu_wlun_id: UPIU W-LUN id
2424  *
2425  * Returns SCSI W-LUN id
2426  */
2427 static inline u16 ufshcd_upiu_wlun_to_scsi_wlun(u8 upiu_wlun_id)
2428 {
2429         return (upiu_wlun_id & ~UFS_UPIU_WLUN_ID) | SCSI_W_LUN_BASE;
2430 }
2431
2432 /**
2433  * ufshcd_queuecommand - main entry point for SCSI requests
2434  * @host: SCSI host pointer
2435  * @cmd: command from SCSI Midlayer
2436  *
2437  * Returns 0 for success, non-zero in case of failure
2438  */
2439 static int ufshcd_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmd)
2440 {
2441         struct ufshcd_lrb *lrbp;
2442         struct ufs_hba *hba;
2443         unsigned long flags;
2444         int tag;
2445         int err = 0;
2446
2447         hba = shost_priv(host);
2448
2449         tag = cmd->request->tag;
2450         if (!ufshcd_valid_tag(hba, tag)) {
2451                 dev_err(hba->dev,
2452                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
2453                         __func__, tag, cmd, cmd->request);
2454                 BUG();
2455         }
2456
2457         if (!down_read_trylock(&hba->clk_scaling_lock))
2458                 return SCSI_MLQUEUE_HOST_BUSY;
2459
2460         spin_lock_irqsave(hba->host->host_lock, flags);
2461         switch (hba->ufshcd_state) {
2462         case UFSHCD_STATE_OPERATIONAL:
2463                 break;
2464         case UFSHCD_STATE_EH_SCHEDULED:
2465         case UFSHCD_STATE_RESET:
2466                 err = SCSI_MLQUEUE_HOST_BUSY;
2467                 goto out_unlock;
2468         case UFSHCD_STATE_ERROR:
2469                 set_host_byte(cmd, DID_ERROR);
2470                 cmd->scsi_done(cmd);
2471                 goto out_unlock;
2472         default:
2473                 dev_WARN_ONCE(hba->dev, 1, "%s: invalid state %d\n",
2474                                 __func__, hba->ufshcd_state);
2475                 set_host_byte(cmd, DID_BAD_TARGET);
2476                 cmd->scsi_done(cmd);
2477                 goto out_unlock;
2478         }
2479
2480         /* if error handling is in progress, don't issue commands */
2481         if (ufshcd_eh_in_progress(hba)) {
2482                 set_host_byte(cmd, DID_ERROR);
2483                 cmd->scsi_done(cmd);
2484                 goto out_unlock;
2485         }
2486         spin_unlock_irqrestore(hba->host->host_lock, flags);
2487
2488         hba->req_abort_count = 0;
2489
2490         /* acquire the tag to make sure device cmds don't use it */
2491         if (test_and_set_bit_lock(tag, &hba->lrb_in_use)) {
2492                 /*
2493                  * Dev manage command in progress, requeue the command.
2494                  * Requeuing the command helps in cases where the request *may*
2495                  * find different tag instead of waiting for dev manage command
2496                  * completion.
2497                  */
2498                 err = SCSI_MLQUEUE_HOST_BUSY;
2499                 goto out;
2500         }
2501
2502         err = ufshcd_hold(hba, true);
2503         if (err) {
2504                 err = SCSI_MLQUEUE_HOST_BUSY;
2505                 clear_bit_unlock(tag, &hba->lrb_in_use);
2506                 goto out;
2507         }
2508         WARN_ON(hba->clk_gating.state != CLKS_ON);
2509
2510         lrbp = &hba->lrb[tag];
2511
2512         WARN_ON(lrbp->cmd);
2513         lrbp->cmd = cmd;
2514         lrbp->sense_bufflen = UFSHCD_REQ_SENSE_SIZE;
2515         lrbp->sense_buffer = cmd->sense_buffer;
2516         lrbp->task_tag = tag;
2517         lrbp->lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
2518         lrbp->intr_cmd = !ufshcd_is_intr_aggr_allowed(hba) ? true : false;
2519         lrbp->req_abort_skip = false;
2520
2521         ufshcd_comp_scsi_upiu(hba, lrbp);
2522
2523         err = ufshcd_map_sg(hba, lrbp);
2524         if (err) {
2525                 ufshcd_release(hba);
2526                 lrbp->cmd = NULL;
2527                 clear_bit_unlock(tag, &hba->lrb_in_use);
2528                 goto out;
2529         }
2530         /* Make sure descriptors are ready before ringing the doorbell */
2531         wmb();
2532
2533         /* issue command to the controller */
2534         spin_lock_irqsave(hba->host->host_lock, flags);
2535         ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
2536         ufshcd_send_command(hba, tag);
2537 out_unlock:
2538         spin_unlock_irqrestore(hba->host->host_lock, flags);
2539 out:
2540         up_read(&hba->clk_scaling_lock);
2541         return err;
2542 }
2543
2544 static int ufshcd_compose_dev_cmd(struct ufs_hba *hba,
2545                 struct ufshcd_lrb *lrbp, enum dev_cmd_type cmd_type, int tag)
2546 {
2547         lrbp->cmd = NULL;
2548         lrbp->sense_bufflen = 0;
2549         lrbp->sense_buffer = NULL;
2550         lrbp->task_tag = tag;
2551         lrbp->lun = 0; /* device management cmd is not specific to any LUN */
2552         lrbp->intr_cmd = true; /* No interrupt aggregation */
2553         hba->dev_cmd.type = cmd_type;
2554
2555         return ufshcd_comp_devman_upiu(hba, lrbp);
2556 }
2557
2558 static int
2559 ufshcd_clear_cmd(struct ufs_hba *hba, int tag)
2560 {
2561         int err = 0;
2562         unsigned long flags;
2563         u32 mask = 1 << tag;
2564
2565         /* clear outstanding transaction before retry */
2566         spin_lock_irqsave(hba->host->host_lock, flags);
2567         ufshcd_utrl_clear(hba, tag);
2568         spin_unlock_irqrestore(hba->host->host_lock, flags);
2569
2570         /*
2571          * wait for for h/w to clear corresponding bit in door-bell.
2572          * max. wait is 1 sec.
2573          */
2574         err = ufshcd_wait_for_register(hba,
2575                         REG_UTP_TRANSFER_REQ_DOOR_BELL,
2576                         mask, ~mask, 1000, 1000, true);
2577
2578         return err;
2579 }
2580
2581 static int
2582 ufshcd_check_query_response(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2583 {
2584         struct ufs_query_res *query_res = &hba->dev_cmd.query.response;
2585
2586         /* Get the UPIU response */
2587         query_res->response = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr) >>
2588                                 UPIU_RSP_CODE_OFFSET;
2589         return query_res->response;
2590 }
2591
2592 /**
2593  * ufshcd_dev_cmd_completion() - handles device management command responses
2594  * @hba: per adapter instance
2595  * @lrbp: pointer to local reference block
2596  */
2597 static int
2598 ufshcd_dev_cmd_completion(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
2599 {
2600         int resp;
2601         int err = 0;
2602
2603         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
2604         resp = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
2605
2606         switch (resp) {
2607         case UPIU_TRANSACTION_NOP_IN:
2608                 if (hba->dev_cmd.type != DEV_CMD_TYPE_NOP) {
2609                         err = -EINVAL;
2610                         dev_err(hba->dev, "%s: unexpected response %x\n",
2611                                         __func__, resp);
2612                 }
2613                 break;
2614         case UPIU_TRANSACTION_QUERY_RSP:
2615                 err = ufshcd_check_query_response(hba, lrbp);
2616                 if (!err)
2617                         err = ufshcd_copy_query_response(hba, lrbp);
2618                 break;
2619         case UPIU_TRANSACTION_REJECT_UPIU:
2620                 /* TODO: handle Reject UPIU Response */
2621                 err = -EPERM;
2622                 dev_err(hba->dev, "%s: Reject UPIU not fully implemented\n",
2623                                 __func__);
2624                 break;
2625         default:
2626                 err = -EINVAL;
2627                 dev_err(hba->dev, "%s: Invalid device management cmd response: %x\n",
2628                                 __func__, resp);
2629                 break;
2630         }
2631
2632         return err;
2633 }
2634
2635 static int ufshcd_wait_for_dev_cmd(struct ufs_hba *hba,
2636                 struct ufshcd_lrb *lrbp, int max_timeout)
2637 {
2638         int err = 0;
2639         unsigned long time_left;
2640         unsigned long flags;
2641
2642         time_left = wait_for_completion_timeout(hba->dev_cmd.complete,
2643                         msecs_to_jiffies(max_timeout));
2644
2645         /* Make sure descriptors are ready before ringing the doorbell */
2646         wmb();
2647         spin_lock_irqsave(hba->host->host_lock, flags);
2648         hba->dev_cmd.complete = NULL;
2649         if (likely(time_left)) {
2650                 err = ufshcd_get_tr_ocs(lrbp);
2651                 if (!err)
2652                         err = ufshcd_dev_cmd_completion(hba, lrbp);
2653         }
2654         spin_unlock_irqrestore(hba->host->host_lock, flags);
2655
2656         if (!time_left) {
2657                 err = -ETIMEDOUT;
2658                 dev_dbg(hba->dev, "%s: dev_cmd request timedout, tag %d\n",
2659                         __func__, lrbp->task_tag);
2660                 if (!ufshcd_clear_cmd(hba, lrbp->task_tag))
2661                         /* successfully cleared the command, retry if needed */
2662                         err = -EAGAIN;
2663                 /*
2664                  * in case of an error, after clearing the doorbell,
2665                  * we also need to clear the outstanding_request
2666                  * field in hba
2667                  */
2668                 ufshcd_outstanding_req_clear(hba, lrbp->task_tag);
2669         }
2670
2671         return err;
2672 }
2673
2674 /**
2675  * ufshcd_get_dev_cmd_tag - Get device management command tag
2676  * @hba: per-adapter instance
2677  * @tag_out: pointer to variable with available slot value
2678  *
2679  * Get a free slot and lock it until device management command
2680  * completes.
2681  *
2682  * Returns false if free slot is unavailable for locking, else
2683  * return true with tag value in @tag.
2684  */
2685 static bool ufshcd_get_dev_cmd_tag(struct ufs_hba *hba, int *tag_out)
2686 {
2687         int tag;
2688         bool ret = false;
2689         unsigned long tmp;
2690
2691         if (!tag_out)
2692                 goto out;
2693
2694         do {
2695                 tmp = ~hba->lrb_in_use;
2696                 tag = find_last_bit(&tmp, hba->nutrs);
2697                 if (tag >= hba->nutrs)
2698                         goto out;
2699         } while (test_and_set_bit_lock(tag, &hba->lrb_in_use));
2700
2701         *tag_out = tag;
2702         ret = true;
2703 out:
2704         return ret;
2705 }
2706
2707 static inline void ufshcd_put_dev_cmd_tag(struct ufs_hba *hba, int tag)
2708 {
2709         clear_bit_unlock(tag, &hba->lrb_in_use);
2710 }
2711
2712 /**
2713  * ufshcd_exec_dev_cmd - API for sending device management requests
2714  * @hba: UFS hba
2715  * @cmd_type: specifies the type (NOP, Query...)
2716  * @timeout: time in seconds
2717  *
2718  * NOTE: Since there is only one available tag for device management commands,
2719  * it is expected you hold the hba->dev_cmd.lock mutex.
2720  */
2721 static int ufshcd_exec_dev_cmd(struct ufs_hba *hba,
2722                 enum dev_cmd_type cmd_type, int timeout)
2723 {
2724         struct ufshcd_lrb *lrbp;
2725         int err;
2726         int tag;
2727         struct completion wait;
2728         unsigned long flags;
2729
2730         down_read(&hba->clk_scaling_lock);
2731
2732         /*
2733          * Get free slot, sleep if slots are unavailable.
2734          * Even though we use wait_event() which sleeps indefinitely,
2735          * the maximum wait time is bounded by SCSI request timeout.
2736          */
2737         wait_event(hba->dev_cmd.tag_wq, ufshcd_get_dev_cmd_tag(hba, &tag));
2738
2739         init_completion(&wait);
2740         lrbp = &hba->lrb[tag];
2741         WARN_ON(lrbp->cmd);
2742         err = ufshcd_compose_dev_cmd(hba, lrbp, cmd_type, tag);
2743         if (unlikely(err))
2744                 goto out_put_tag;
2745
2746         hba->dev_cmd.complete = &wait;
2747
2748         ufshcd_add_query_upiu_trace(hba, tag, "query_send");
2749         /* Make sure descriptors are ready before ringing the doorbell */
2750         wmb();
2751         spin_lock_irqsave(hba->host->host_lock, flags);
2752         ufshcd_vops_setup_xfer_req(hba, tag, (lrbp->cmd ? true : false));
2753         ufshcd_send_command(hba, tag);
2754         spin_unlock_irqrestore(hba->host->host_lock, flags);
2755
2756         err = ufshcd_wait_for_dev_cmd(hba, lrbp, timeout);
2757
2758         ufshcd_add_query_upiu_trace(hba, tag,
2759                         err ? "query_complete_err" : "query_complete");
2760
2761 out_put_tag:
2762         ufshcd_put_dev_cmd_tag(hba, tag);
2763         wake_up(&hba->dev_cmd.tag_wq);
2764         up_read(&hba->clk_scaling_lock);
2765         return err;
2766 }
2767
2768 /**
2769  * ufshcd_init_query() - init the query response and request parameters
2770  * @hba: per-adapter instance
2771  * @request: address of the request pointer to be initialized
2772  * @response: address of the response pointer to be initialized
2773  * @opcode: operation to perform
2774  * @idn: flag idn to access
2775  * @index: LU number to access
2776  * @selector: query/flag/descriptor further identification
2777  */
2778 static inline void ufshcd_init_query(struct ufs_hba *hba,
2779                 struct ufs_query_req **request, struct ufs_query_res **response,
2780                 enum query_opcode opcode, u8 idn, u8 index, u8 selector)
2781 {
2782         *request = &hba->dev_cmd.query.request;
2783         *response = &hba->dev_cmd.query.response;
2784         memset(*request, 0, sizeof(struct ufs_query_req));
2785         memset(*response, 0, sizeof(struct ufs_query_res));
2786         (*request)->upiu_req.opcode = opcode;
2787         (*request)->upiu_req.idn = idn;
2788         (*request)->upiu_req.index = index;
2789         (*request)->upiu_req.selector = selector;
2790 }
2791
2792 static int ufshcd_query_flag_retry(struct ufs_hba *hba,
2793         enum query_opcode opcode, enum flag_idn idn, bool *flag_res)
2794 {
2795         int ret;
2796         int retries;
2797
2798         for (retries = 0; retries < QUERY_REQ_RETRIES; retries++) {
2799                 ret = ufshcd_query_flag(hba, opcode, idn, flag_res);
2800                 if (ret)
2801                         dev_dbg(hba->dev,
2802                                 "%s: failed with error %d, retries %d\n",
2803                                 __func__, ret, retries);
2804                 else
2805                         break;
2806         }
2807
2808         if (ret)
2809                 dev_err(hba->dev,
2810                         "%s: query attribute, opcode %d, idn %d, failed with error %d after %d retires\n",
2811                         __func__, opcode, idn, ret, retries);
2812         return ret;
2813 }
2814
2815 /**
2816  * ufshcd_query_flag() - API function for sending flag query requests
2817  * @hba: per-adapter instance
2818  * @opcode: flag query to perform
2819  * @idn: flag idn to access
2820  * @flag_res: the flag value after the query request completes
2821  *
2822  * Returns 0 for success, non-zero in case of failure
2823  */
2824 int ufshcd_query_flag(struct ufs_hba *hba, enum query_opcode opcode,
2825                         enum flag_idn idn, bool *flag_res)
2826 {
2827         struct ufs_query_req *request = NULL;
2828         struct ufs_query_res *response = NULL;
2829         int err, index = 0, selector = 0;
2830         int timeout = QUERY_REQ_TIMEOUT;
2831
2832         BUG_ON(!hba);
2833
2834         ufshcd_hold(hba, false);
2835         mutex_lock(&hba->dev_cmd.lock);
2836         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2837                         selector);
2838
2839         switch (opcode) {
2840         case UPIU_QUERY_OPCODE_SET_FLAG:
2841         case UPIU_QUERY_OPCODE_CLEAR_FLAG:
2842         case UPIU_QUERY_OPCODE_TOGGLE_FLAG:
2843                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2844                 break;
2845         case UPIU_QUERY_OPCODE_READ_FLAG:
2846                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2847                 if (!flag_res) {
2848                         /* No dummy reads */
2849                         dev_err(hba->dev, "%s: Invalid argument for read request\n",
2850                                         __func__);
2851                         err = -EINVAL;
2852                         goto out_unlock;
2853                 }
2854                 break;
2855         default:
2856                 dev_err(hba->dev,
2857                         "%s: Expected query flag opcode but got = %d\n",
2858                         __func__, opcode);
2859                 err = -EINVAL;
2860                 goto out_unlock;
2861         }
2862
2863         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, timeout);
2864
2865         if (err) {
2866                 dev_err(hba->dev,
2867                         "%s: Sending flag query for idn %d failed, err = %d\n",
2868                         __func__, idn, err);
2869                 goto out_unlock;
2870         }
2871
2872         if (flag_res)
2873                 *flag_res = (be32_to_cpu(response->upiu_res.value) &
2874                                 MASK_QUERY_UPIU_FLAG_LOC) & 0x1;
2875
2876 out_unlock:
2877         mutex_unlock(&hba->dev_cmd.lock);
2878         ufshcd_release(hba);
2879         return err;
2880 }
2881
2882 /**
2883  * ufshcd_query_attr - API function for sending attribute requests
2884  * @hba: per-adapter instance
2885  * @opcode: attribute opcode
2886  * @idn: attribute idn to access
2887  * @index: index field
2888  * @selector: selector field
2889  * @attr_val: the attribute value after the query request completes
2890  *
2891  * Returns 0 for success, non-zero in case of failure
2892 */
2893 int ufshcd_query_attr(struct ufs_hba *hba, enum query_opcode opcode,
2894                       enum attr_idn idn, u8 index, u8 selector, u32 *attr_val)
2895 {
2896         struct ufs_query_req *request = NULL;
2897         struct ufs_query_res *response = NULL;
2898         int err;
2899
2900         BUG_ON(!hba);
2901
2902         ufshcd_hold(hba, false);
2903         if (!attr_val) {
2904                 dev_err(hba->dev, "%s: attribute value required for opcode 0x%x\n",
2905                                 __func__, opcode);
2906                 err = -EINVAL;
2907                 goto out;
2908         }
2909
2910         mutex_lock(&hba->dev_cmd.lock);
2911         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
2912                         selector);
2913
2914         switch (opcode) {
2915         case UPIU_QUERY_OPCODE_WRITE_ATTR:
2916                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
2917                 request->upiu_req.value = cpu_to_be32(*attr_val);
2918                 break;
2919         case UPIU_QUERY_OPCODE_READ_ATTR:
2920                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
2921                 break;
2922         default:
2923                 dev_err(hba->dev, "%s: Expected query attr opcode but got = 0x%.2x\n",
2924                                 __func__, opcode);
2925                 err = -EINVAL;
2926                 goto out_unlock;
2927         }
2928
2929         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
2930
2931         if (err) {
2932                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
2933                                 __func__, opcode, idn, index, err);
2934                 goto out_unlock;
2935         }
2936
2937         *attr_val = be32_to_cpu(response->upiu_res.value);
2938
2939 out_unlock:
2940         mutex_unlock(&hba->dev_cmd.lock);
2941 out:
2942         ufshcd_release(hba);
2943         return err;
2944 }
2945
2946 /**
2947  * ufshcd_query_attr_retry() - API function for sending query
2948  * attribute with retries
2949  * @hba: per-adapter instance
2950  * @opcode: attribute opcode
2951  * @idn: attribute idn to access
2952  * @index: index field
2953  * @selector: selector field
2954  * @attr_val: the attribute value after the query request
2955  * completes
2956  *
2957  * Returns 0 for success, non-zero in case of failure
2958 */
2959 static int ufshcd_query_attr_retry(struct ufs_hba *hba,
2960         enum query_opcode opcode, enum attr_idn idn, u8 index, u8 selector,
2961         u32 *attr_val)
2962 {
2963         int ret = 0;
2964         u32 retries;
2965
2966          for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
2967                 ret = ufshcd_query_attr(hba, opcode, idn, index,
2968                                                 selector, attr_val);
2969                 if (ret)
2970                         dev_dbg(hba->dev, "%s: failed with error %d, retries %d\n",
2971                                 __func__, ret, retries);
2972                 else
2973                         break;
2974         }
2975
2976         if (ret)
2977                 dev_err(hba->dev,
2978                         "%s: query attribute, idn %d, failed with error %d after %d retires\n",
2979                         __func__, idn, ret, QUERY_REQ_RETRIES);
2980         return ret;
2981 }
2982
2983 static int __ufshcd_query_descriptor(struct ufs_hba *hba,
2984                         enum query_opcode opcode, enum desc_idn idn, u8 index,
2985                         u8 selector, u8 *desc_buf, int *buf_len)
2986 {
2987         struct ufs_query_req *request = NULL;
2988         struct ufs_query_res *response = NULL;
2989         int err;
2990
2991         BUG_ON(!hba);
2992
2993         ufshcd_hold(hba, false);
2994         if (!desc_buf) {
2995                 dev_err(hba->dev, "%s: descriptor buffer required for opcode 0x%x\n",
2996                                 __func__, opcode);
2997                 err = -EINVAL;
2998                 goto out;
2999         }
3000
3001         if (*buf_len < QUERY_DESC_MIN_SIZE || *buf_len > QUERY_DESC_MAX_SIZE) {
3002                 dev_err(hba->dev, "%s: descriptor buffer size (%d) is out of range\n",
3003                                 __func__, *buf_len);
3004                 err = -EINVAL;
3005                 goto out;
3006         }
3007
3008         mutex_lock(&hba->dev_cmd.lock);
3009         ufshcd_init_query(hba, &request, &response, opcode, idn, index,
3010                         selector);
3011         hba->dev_cmd.query.descriptor = desc_buf;
3012         request->upiu_req.length = cpu_to_be16(*buf_len);
3013
3014         switch (opcode) {
3015         case UPIU_QUERY_OPCODE_WRITE_DESC:
3016                 request->query_func = UPIU_QUERY_FUNC_STANDARD_WRITE_REQUEST;
3017                 break;
3018         case UPIU_QUERY_OPCODE_READ_DESC:
3019                 request->query_func = UPIU_QUERY_FUNC_STANDARD_READ_REQUEST;
3020                 break;
3021         default:
3022                 dev_err(hba->dev,
3023                                 "%s: Expected query descriptor opcode but got = 0x%.2x\n",
3024                                 __func__, opcode);
3025                 err = -EINVAL;
3026                 goto out_unlock;
3027         }
3028
3029         err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_QUERY, QUERY_REQ_TIMEOUT);
3030
3031         if (err) {
3032                 dev_err(hba->dev, "%s: opcode 0x%.2x for idn %d failed, index %d, err = %d\n",
3033                                 __func__, opcode, idn, index, err);
3034                 goto out_unlock;
3035         }
3036
3037         *buf_len = be16_to_cpu(response->upiu_res.length);
3038
3039 out_unlock:
3040         hba->dev_cmd.query.descriptor = NULL;
3041         mutex_unlock(&hba->dev_cmd.lock);
3042 out:
3043         ufshcd_release(hba);
3044         return err;
3045 }
3046
3047 /**
3048  * ufshcd_query_descriptor_retry - API function for sending descriptor requests
3049  * @hba: per-adapter instance
3050  * @opcode: attribute opcode
3051  * @idn: attribute idn to access
3052  * @index: index field
3053  * @selector: selector field
3054  * @desc_buf: the buffer that contains the descriptor
3055  * @buf_len: length parameter passed to the device
3056  *
3057  * Returns 0 for success, non-zero in case of failure.
3058  * The buf_len parameter will contain, on return, the length parameter
3059  * received on the response.
3060  */
3061 int ufshcd_query_descriptor_retry(struct ufs_hba *hba,
3062                                   enum query_opcode opcode,
3063                                   enum desc_idn idn, u8 index,
3064                                   u8 selector,
3065                                   u8 *desc_buf, int *buf_len)
3066 {
3067         int err;
3068         int retries;
3069
3070         for (retries = QUERY_REQ_RETRIES; retries > 0; retries--) {
3071                 err = __ufshcd_query_descriptor(hba, opcode, idn, index,
3072                                                 selector, desc_buf, buf_len);
3073                 if (!err || err == -EINVAL)
3074                         break;
3075         }
3076
3077         return err;
3078 }
3079
3080 /**
3081  * ufshcd_read_desc_length - read the specified descriptor length from header
3082  * @hba: Pointer to adapter instance
3083  * @desc_id: descriptor idn value
3084  * @desc_index: descriptor index
3085  * @desc_length: pointer to variable to read the length of descriptor
3086  *
3087  * Return 0 in case of success, non-zero otherwise
3088  */
3089 static int ufshcd_read_desc_length(struct ufs_hba *hba,
3090         enum desc_idn desc_id,
3091         int desc_index,
3092         int *desc_length)
3093 {
3094         int ret;
3095         u8 header[QUERY_DESC_HDR_SIZE];
3096         int header_len = QUERY_DESC_HDR_SIZE;
3097
3098         if (desc_id >= QUERY_DESC_IDN_MAX)
3099                 return -EINVAL;
3100
3101         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3102                                         desc_id, desc_index, 0, header,
3103                                         &header_len);
3104
3105         if (ret) {
3106                 dev_err(hba->dev, "%s: Failed to get descriptor header id %d",
3107                         __func__, desc_id);
3108                 return ret;
3109         } else if (desc_id != header[QUERY_DESC_DESC_TYPE_OFFSET]) {
3110                 dev_warn(hba->dev, "%s: descriptor header id %d and desc_id %d mismatch",
3111                         __func__, header[QUERY_DESC_DESC_TYPE_OFFSET],
3112                         desc_id);
3113                 ret = -EINVAL;
3114         }
3115
3116         *desc_length = header[QUERY_DESC_LENGTH_OFFSET];
3117         return ret;
3118
3119 }
3120
3121 /**
3122  * ufshcd_map_desc_id_to_length - map descriptor IDN to its length
3123  * @hba: Pointer to adapter instance
3124  * @desc_id: descriptor idn value
3125  * @desc_len: mapped desc length (out)
3126  *
3127  * Return 0 in case of success, non-zero otherwise
3128  */
3129 int ufshcd_map_desc_id_to_length(struct ufs_hba *hba,
3130         enum desc_idn desc_id, int *desc_len)
3131 {
3132         switch (desc_id) {
3133         case QUERY_DESC_IDN_DEVICE:
3134                 *desc_len = hba->desc_size.dev_desc;
3135                 break;
3136         case QUERY_DESC_IDN_POWER:
3137                 *desc_len = hba->desc_size.pwr_desc;
3138                 break;
3139         case QUERY_DESC_IDN_GEOMETRY:
3140                 *desc_len = hba->desc_size.geom_desc;
3141                 break;
3142         case QUERY_DESC_IDN_CONFIGURATION:
3143                 *desc_len = hba->desc_size.conf_desc;
3144                 break;
3145         case QUERY_DESC_IDN_UNIT:
3146                 *desc_len = hba->desc_size.unit_desc;
3147                 break;
3148         case QUERY_DESC_IDN_INTERCONNECT:
3149                 *desc_len = hba->desc_size.interc_desc;
3150                 break;
3151         case QUERY_DESC_IDN_STRING:
3152                 *desc_len = QUERY_DESC_MAX_SIZE;
3153                 break;
3154         case QUERY_DESC_IDN_HEALTH:
3155                 *desc_len = hba->desc_size.hlth_desc;
3156                 break;
3157         case QUERY_DESC_IDN_RFU_0:
3158         case QUERY_DESC_IDN_RFU_1:
3159                 *desc_len = 0;
3160                 break;
3161         default:
3162                 *desc_len = 0;
3163                 return -EINVAL;
3164         }
3165         return 0;
3166 }
3167 EXPORT_SYMBOL(ufshcd_map_desc_id_to_length);
3168
3169 /**
3170  * ufshcd_read_desc_param - read the specified descriptor parameter
3171  * @hba: Pointer to adapter instance
3172  * @desc_id: descriptor idn value
3173  * @desc_index: descriptor index
3174  * @param_offset: offset of the parameter to read
3175  * @param_read_buf: pointer to buffer where parameter would be read
3176  * @param_size: sizeof(param_read_buf)
3177  *
3178  * Return 0 in case of success, non-zero otherwise
3179  */
3180 int ufshcd_read_desc_param(struct ufs_hba *hba,
3181                            enum desc_idn desc_id,
3182                            int desc_index,
3183                            u8 param_offset,
3184                            u8 *param_read_buf,
3185                            u8 param_size)
3186 {
3187         int ret;
3188         u8 *desc_buf;
3189         int buff_len;
3190         bool is_kmalloc = true;
3191
3192         /* Safety check */
3193         if (desc_id >= QUERY_DESC_IDN_MAX || !param_size)
3194                 return -EINVAL;
3195
3196         /* Get the max length of descriptor from structure filled up at probe
3197          * time.
3198          */
3199         ret = ufshcd_map_desc_id_to_length(hba, desc_id, &buff_len);
3200
3201         /* Sanity checks */
3202         if (ret || !buff_len) {
3203                 dev_err(hba->dev, "%s: Failed to get full descriptor length",
3204                         __func__);
3205                 return ret;
3206         }
3207
3208         /* Check whether we need temp memory */
3209         if (param_offset != 0 || param_size < buff_len) {
3210                 desc_buf = kmalloc(buff_len, GFP_KERNEL);
3211                 if (!desc_buf)
3212                         return -ENOMEM;
3213         } else {
3214                 desc_buf = param_read_buf;
3215                 is_kmalloc = false;
3216         }
3217
3218         /* Request for full descriptor */
3219         ret = ufshcd_query_descriptor_retry(hba, UPIU_QUERY_OPCODE_READ_DESC,
3220                                         desc_id, desc_index, 0,
3221                                         desc_buf, &buff_len);
3222
3223         if (ret) {
3224                 dev_err(hba->dev, "%s: Failed reading descriptor. desc_id %d, desc_index %d, param_offset %d, ret %d",
3225                         __func__, desc_id, desc_index, param_offset, ret);
3226                 goto out;
3227         }
3228
3229         /* Sanity check */
3230         if (desc_buf[QUERY_DESC_DESC_TYPE_OFFSET] != desc_id) {
3231                 dev_err(hba->dev, "%s: invalid desc_id %d in descriptor header",
3232                         __func__, desc_buf[QUERY_DESC_DESC_TYPE_OFFSET]);
3233                 ret = -EINVAL;
3234                 goto out;
3235         }
3236
3237         /* Check wherher we will not copy more data, than available */
3238         if (is_kmalloc && param_size > buff_len)
3239                 param_size = buff_len;
3240
3241         if (is_kmalloc)
3242                 memcpy(param_read_buf, &desc_buf[param_offset], param_size);
3243 out:
3244         if (is_kmalloc)
3245                 kfree(desc_buf);
3246         return ret;
3247 }
3248
3249 static inline int ufshcd_read_desc(struct ufs_hba *hba,
3250                                    enum desc_idn desc_id,
3251                                    int desc_index,
3252                                    u8 *buf,
3253                                    u32 size)
3254 {
3255         return ufshcd_read_desc_param(hba, desc_id, desc_index, 0, buf, size);
3256 }
3257
3258 static inline int ufshcd_read_power_desc(struct ufs_hba *hba,
3259                                          u8 *buf,
3260                                          u32 size)
3261 {
3262         return ufshcd_read_desc(hba, QUERY_DESC_IDN_POWER, 0, buf, size);
3263 }
3264
3265 static int ufshcd_read_device_desc(struct ufs_hba *hba, u8 *buf, u32 size)
3266 {
3267         return ufshcd_read_desc(hba, QUERY_DESC_IDN_DEVICE, 0, buf, size);
3268 }
3269
3270 /**
3271  * ufshcd_read_string_desc - read string descriptor
3272  * @hba: pointer to adapter instance
3273  * @desc_index: descriptor index
3274  * @buf: pointer to buffer where descriptor would be read
3275  * @size: size of buf
3276  * @ascii: if true convert from unicode to ascii characters
3277  *
3278  * Return 0 in case of success, non-zero otherwise
3279  */
3280 int ufshcd_read_string_desc(struct ufs_hba *hba, int desc_index,
3281                             u8 *buf, u32 size, bool ascii)
3282 {
3283         int err = 0;
3284
3285         err = ufshcd_read_desc(hba,
3286                                 QUERY_DESC_IDN_STRING, desc_index, buf, size);
3287
3288         if (err) {
3289                 dev_err(hba->dev, "%s: reading String Desc failed after %d retries. err = %d\n",
3290                         __func__, QUERY_REQ_RETRIES, err);
3291                 goto out;
3292         }
3293
3294         if (ascii) {
3295                 int desc_len;
3296                 int ascii_len;
3297                 int i;
3298                 char *buff_ascii;
3299
3300                 desc_len = buf[0];
3301                 /* remove header and divide by 2 to move from UTF16 to UTF8 */
3302                 ascii_len = (desc_len - QUERY_DESC_HDR_SIZE) / 2 + 1;
3303                 if (size < ascii_len + QUERY_DESC_HDR_SIZE) {
3304                         dev_err(hba->dev, "%s: buffer allocated size is too small\n",
3305                                         __func__);
3306                         err = -ENOMEM;
3307                         goto out;
3308                 }
3309
3310                 buff_ascii = kmalloc(ascii_len, GFP_KERNEL);
3311                 if (!buff_ascii) {
3312                         err = -ENOMEM;
3313                         goto out;
3314                 }
3315
3316                 /*
3317                  * the descriptor contains string in UTF16 format
3318                  * we need to convert to utf-8 so it can be displayed
3319                  */
3320                 utf16s_to_utf8s((wchar_t *)&buf[QUERY_DESC_HDR_SIZE],
3321                                 desc_len - QUERY_DESC_HDR_SIZE,
3322                                 UTF16_BIG_ENDIAN, buff_ascii, ascii_len);
3323
3324                 /* replace non-printable or non-ASCII characters with spaces */
3325                 for (i = 0; i < ascii_len; i++)
3326                         ufshcd_remove_non_printable(&buff_ascii[i]);
3327
3328                 memset(buf + QUERY_DESC_HDR_SIZE, 0,
3329                                 size - QUERY_DESC_HDR_SIZE);
3330                 memcpy(buf + QUERY_DESC_HDR_SIZE, buff_ascii, ascii_len);
3331                 buf[QUERY_DESC_LENGTH_OFFSET] = ascii_len + QUERY_DESC_HDR_SIZE;
3332                 kfree(buff_ascii);
3333         }
3334 out:
3335         return err;
3336 }
3337
3338 /**
3339  * ufshcd_read_unit_desc_param - read the specified unit descriptor parameter
3340  * @hba: Pointer to adapter instance
3341  * @lun: lun id
3342  * @param_offset: offset of the parameter to read
3343  * @param_read_buf: pointer to buffer where parameter would be read
3344  * @param_size: sizeof(param_read_buf)
3345  *
3346  * Return 0 in case of success, non-zero otherwise
3347  */
3348 static inline int ufshcd_read_unit_desc_param(struct ufs_hba *hba,
3349                                               int lun,
3350                                               enum unit_desc_param param_offset,
3351                                               u8 *param_read_buf,
3352                                               u32 param_size)
3353 {
3354         /*
3355          * Unit descriptors are only available for general purpose LUs (LUN id
3356          * from 0 to 7) and RPMB Well known LU.
3357          */
3358         if (!ufs_is_valid_unit_desc_lun(lun))
3359                 return -EOPNOTSUPP;
3360
3361         return ufshcd_read_desc_param(hba, QUERY_DESC_IDN_UNIT, lun,
3362                                       param_offset, param_read_buf, param_size);
3363 }
3364
3365 /**
3366  * ufshcd_memory_alloc - allocate memory for host memory space data structures
3367  * @hba: per adapter instance
3368  *
3369  * 1. Allocate DMA memory for Command Descriptor array
3370  *      Each command descriptor consist of Command UPIU, Response UPIU and PRDT
3371  * 2. Allocate DMA memory for UTP Transfer Request Descriptor List (UTRDL).
3372  * 3. Allocate DMA memory for UTP Task Management Request Descriptor List
3373  *      (UTMRDL)
3374  * 4. Allocate memory for local reference block(lrb).
3375  *
3376  * Returns 0 for success, non-zero in case of failure
3377  */
3378 static int ufshcd_memory_alloc(struct ufs_hba *hba)
3379 {
3380         size_t utmrdl_size, utrdl_size, ucdl_size;
3381
3382         /* Allocate memory for UTP command descriptors */
3383         ucdl_size = (sizeof(struct utp_transfer_cmd_desc) * hba->nutrs);
3384         hba->ucdl_base_addr = dmam_alloc_coherent(hba->dev,
3385                                                   ucdl_size,
3386                                                   &hba->ucdl_dma_addr,
3387                                                   GFP_KERNEL);
3388
3389         /*
3390          * UFSHCI requires UTP command descriptor to be 128 byte aligned.
3391          * make sure hba->ucdl_dma_addr is aligned to PAGE_SIZE
3392          * if hba->ucdl_dma_addr is aligned to PAGE_SIZE, then it will
3393          * be aligned to 128 bytes as well
3394          */
3395         if (!hba->ucdl_base_addr ||
3396             WARN_ON(hba->ucdl_dma_addr & (PAGE_SIZE - 1))) {
3397                 dev_err(hba->dev,
3398                         "Command Descriptor Memory allocation failed\n");
3399                 goto out;
3400         }
3401
3402         /*
3403          * Allocate memory for UTP Transfer descriptors
3404          * UFSHCI requires 1024 byte alignment of UTRD
3405          */
3406         utrdl_size = (sizeof(struct utp_transfer_req_desc) * hba->nutrs);
3407         hba->utrdl_base_addr = dmam_alloc_coherent(hba->dev,
3408                                                    utrdl_size,
3409                                                    &hba->utrdl_dma_addr,
3410                                                    GFP_KERNEL);
3411         if (!hba->utrdl_base_addr ||
3412             WARN_ON(hba->utrdl_dma_addr & (PAGE_SIZE - 1))) {
3413                 dev_err(hba->dev,
3414                         "Transfer Descriptor Memory allocation failed\n");
3415                 goto out;
3416         }
3417
3418         /*
3419          * Allocate memory for UTP Task Management descriptors
3420          * UFSHCI requires 1024 byte alignment of UTMRD
3421          */
3422         utmrdl_size = sizeof(struct utp_task_req_desc) * hba->nutmrs;
3423         hba->utmrdl_base_addr = dmam_alloc_coherent(hba->dev,
3424                                                     utmrdl_size,
3425                                                     &hba->utmrdl_dma_addr,
3426                                                     GFP_KERNEL);
3427         if (!hba->utmrdl_base_addr ||
3428             WARN_ON(hba->utmrdl_dma_addr & (PAGE_SIZE - 1))) {
3429                 dev_err(hba->dev,
3430                 "Task Management Descriptor Memory allocation failed\n");
3431                 goto out;
3432         }
3433
3434         /* Allocate memory for local reference block */
3435         hba->lrb = devm_kcalloc(hba->dev,
3436                                 hba->nutrs, sizeof(struct ufshcd_lrb),
3437                                 GFP_KERNEL);
3438         if (!hba->lrb) {
3439                 dev_err(hba->dev, "LRB Memory allocation failed\n");
3440                 goto out;
3441         }
3442         return 0;
3443 out:
3444         return -ENOMEM;
3445 }
3446
3447 /**
3448  * ufshcd_host_memory_configure - configure local reference block with
3449  *                              memory offsets
3450  * @hba: per adapter instance
3451  *
3452  * Configure Host memory space
3453  * 1. Update Corresponding UTRD.UCDBA and UTRD.UCDBAU with UCD DMA
3454  * address.
3455  * 2. Update each UTRD with Response UPIU offset, Response UPIU length
3456  * and PRDT offset.
3457  * 3. Save the corresponding addresses of UTRD, UCD.CMD, UCD.RSP and UCD.PRDT
3458  * into local reference block.
3459  */
3460 static void ufshcd_host_memory_configure(struct ufs_hba *hba)
3461 {
3462         struct utp_transfer_cmd_desc *cmd_descp;
3463         struct utp_transfer_req_desc *utrdlp;
3464         dma_addr_t cmd_desc_dma_addr;
3465         dma_addr_t cmd_desc_element_addr;
3466         u16 response_offset;
3467         u16 prdt_offset;
3468         int cmd_desc_size;
3469         int i;
3470
3471         utrdlp = hba->utrdl_base_addr;
3472         cmd_descp = hba->ucdl_base_addr;
3473
3474         response_offset =
3475                 offsetof(struct utp_transfer_cmd_desc, response_upiu);
3476         prdt_offset =
3477                 offsetof(struct utp_transfer_cmd_desc, prd_table);
3478
3479         cmd_desc_size = sizeof(struct utp_transfer_cmd_desc);
3480         cmd_desc_dma_addr = hba->ucdl_dma_addr;
3481
3482         for (i = 0; i < hba->nutrs; i++) {
3483                 /* Configure UTRD with command descriptor base address */
3484                 cmd_desc_element_addr =
3485                                 (cmd_desc_dma_addr + (cmd_desc_size * i));
3486                 utrdlp[i].command_desc_base_addr_lo =
3487                                 cpu_to_le32(lower_32_bits(cmd_desc_element_addr));
3488                 utrdlp[i].command_desc_base_addr_hi =
3489                                 cpu_to_le32(upper_32_bits(cmd_desc_element_addr));
3490
3491                 /* Response upiu and prdt offset should be in double words */
3492                 if (hba->quirks & UFSHCD_QUIRK_PRDT_BYTE_GRAN) {
3493                         utrdlp[i].response_upiu_offset =
3494                                 cpu_to_le16(response_offset);
3495                         utrdlp[i].prd_table_offset =
3496                                 cpu_to_le16(prdt_offset);
3497                         utrdlp[i].response_upiu_length =
3498                                 cpu_to_le16(ALIGNED_UPIU_SIZE);
3499                 } else {
3500                         utrdlp[i].response_upiu_offset =
3501                                 cpu_to_le16((response_offset >> 2));
3502                         utrdlp[i].prd_table_offset =
3503                                 cpu_to_le16((prdt_offset >> 2));
3504                         utrdlp[i].response_upiu_length =
3505                                 cpu_to_le16(ALIGNED_UPIU_SIZE >> 2);
3506                 }
3507
3508                 hba->lrb[i].utr_descriptor_ptr = (utrdlp + i);
3509                 hba->lrb[i].utrd_dma_addr = hba->utrdl_dma_addr +
3510                                 (i * sizeof(struct utp_transfer_req_desc));
3511                 hba->lrb[i].ucd_req_ptr =
3512                         (struct utp_upiu_req *)(cmd_descp + i);
3513                 hba->lrb[i].ucd_req_dma_addr = cmd_desc_element_addr;
3514                 hba->lrb[i].ucd_rsp_ptr =
3515                         (struct utp_upiu_rsp *)cmd_descp[i].response_upiu;
3516                 hba->lrb[i].ucd_rsp_dma_addr = cmd_desc_element_addr +
3517                                 response_offset;
3518                 hba->lrb[i].ucd_prdt_ptr =
3519                         (struct ufshcd_sg_entry *)cmd_descp[i].prd_table;
3520                 hba->lrb[i].ucd_prdt_dma_addr = cmd_desc_element_addr +
3521                                 prdt_offset;
3522         }
3523 }
3524
3525 /**
3526  * ufshcd_dme_link_startup - Notify Unipro to perform link startup
3527  * @hba: per adapter instance
3528  *
3529  * UIC_CMD_DME_LINK_STARTUP command must be issued to Unipro layer,
3530  * in order to initialize the Unipro link startup procedure.
3531  * Once the Unipro links are up, the device connected to the controller
3532  * is detected.
3533  *
3534  * Returns 0 on success, non-zero value on failure
3535  */
3536 static int ufshcd_dme_link_startup(struct ufs_hba *hba)
3537 {
3538         struct uic_command uic_cmd = {0};
3539         int ret;
3540
3541         uic_cmd.command = UIC_CMD_DME_LINK_STARTUP;
3542
3543         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3544         if (ret)
3545                 dev_dbg(hba->dev,
3546                         "dme-link-startup: error code %d\n", ret);
3547         return ret;
3548 }
3549 /**
3550  * ufshcd_dme_reset - UIC command for DME_RESET
3551  * @hba: per adapter instance
3552  *
3553  * DME_RESET command is issued in order to reset UniPro stack.
3554  * This function now deal with cold reset.
3555  *
3556  * Returns 0 on success, non-zero value on failure
3557  */
3558 static int ufshcd_dme_reset(struct ufs_hba *hba)
3559 {
3560         struct uic_command uic_cmd = {0};
3561         int ret;
3562
3563         uic_cmd.command = UIC_CMD_DME_RESET;
3564
3565         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3566         if (ret)
3567                 dev_err(hba->dev,
3568                         "dme-reset: error code %d\n", ret);
3569
3570         return ret;
3571 }
3572
3573 /**
3574  * ufshcd_dme_enable - UIC command for DME_ENABLE
3575  * @hba: per adapter instance
3576  *
3577  * DME_ENABLE command is issued in order to enable UniPro stack.
3578  *
3579  * Returns 0 on success, non-zero value on failure
3580  */
3581 static int ufshcd_dme_enable(struct ufs_hba *hba)
3582 {
3583         struct uic_command uic_cmd = {0};
3584         int ret;
3585
3586         uic_cmd.command = UIC_CMD_DME_ENABLE;
3587
3588         ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3589         if (ret)
3590                 dev_err(hba->dev,
3591                         "dme-enable: error code %d\n", ret);
3592
3593         return ret;
3594 }
3595
3596 static inline void ufshcd_add_delay_before_dme_cmd(struct ufs_hba *hba)
3597 {
3598         #define MIN_DELAY_BEFORE_DME_CMDS_US    1000
3599         unsigned long min_sleep_time_us;
3600
3601         if (!(hba->quirks & UFSHCD_QUIRK_DELAY_BEFORE_DME_CMDS))
3602                 return;
3603
3604         /*
3605          * last_dme_cmd_tstamp will be 0 only for 1st call to
3606          * this function
3607          */
3608         if (unlikely(!ktime_to_us(hba->last_dme_cmd_tstamp))) {
3609                 min_sleep_time_us = MIN_DELAY_BEFORE_DME_CMDS_US;
3610         } else {
3611                 unsigned long delta =
3612                         (unsigned long) ktime_to_us(
3613                                 ktime_sub(ktime_get(),
3614                                 hba->last_dme_cmd_tstamp));
3615
3616                 if (delta < MIN_DELAY_BEFORE_DME_CMDS_US)
3617                         min_sleep_time_us =
3618                                 MIN_DELAY_BEFORE_DME_CMDS_US - delta;
3619                 else
3620                         return; /* no more delay required */
3621         }
3622
3623         /* allow sleep for extra 50us if needed */
3624         usleep_range(min_sleep_time_us, min_sleep_time_us + 50);
3625 }
3626
3627 /**
3628  * ufshcd_dme_set_attr - UIC command for DME_SET, DME_PEER_SET
3629  * @hba: per adapter instance
3630  * @attr_sel: uic command argument1
3631  * @attr_set: attribute set type as uic command argument2
3632  * @mib_val: setting value as uic command argument3
3633  * @peer: indicate whether peer or local
3634  *
3635  * Returns 0 on success, non-zero value on failure
3636  */
3637 int ufshcd_dme_set_attr(struct ufs_hba *hba, u32 attr_sel,
3638                         u8 attr_set, u32 mib_val, u8 peer)
3639 {
3640         struct uic_command uic_cmd = {0};
3641         static const char *const action[] = {
3642                 "dme-set",
3643                 "dme-peer-set"
3644         };
3645         const char *set = action[!!peer];
3646         int ret;
3647         int retries = UFS_UIC_COMMAND_RETRIES;
3648
3649         uic_cmd.command = peer ?
3650                 UIC_CMD_DME_PEER_SET : UIC_CMD_DME_SET;
3651         uic_cmd.argument1 = attr_sel;
3652         uic_cmd.argument2 = UIC_ARG_ATTR_TYPE(attr_set);
3653         uic_cmd.argument3 = mib_val;
3654
3655         do {
3656                 /* for peer attributes we retry upon failure */
3657                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3658                 if (ret)
3659                         dev_dbg(hba->dev, "%s: attr-id 0x%x val 0x%x error code %d\n",
3660                                 set, UIC_GET_ATTR_ID(attr_sel), mib_val, ret);
3661         } while (ret && peer && --retries);
3662
3663         if (ret)
3664                 dev_err(hba->dev, "%s: attr-id 0x%x val 0x%x failed %d retries\n",
3665                         set, UIC_GET_ATTR_ID(attr_sel), mib_val,
3666                         UFS_UIC_COMMAND_RETRIES - retries);
3667
3668         return ret;
3669 }
3670 EXPORT_SYMBOL_GPL(ufshcd_dme_set_attr);
3671
3672 /**
3673  * ufshcd_dme_get_attr - UIC command for DME_GET, DME_PEER_GET
3674  * @hba: per adapter instance
3675  * @attr_sel: uic command argument1
3676  * @mib_val: the value of the attribute as returned by the UIC command
3677  * @peer: indicate whether peer or local
3678  *
3679  * Returns 0 on success, non-zero value on failure
3680  */
3681 int ufshcd_dme_get_attr(struct ufs_hba *hba, u32 attr_sel,
3682                         u32 *mib_val, u8 peer)
3683 {
3684         struct uic_command uic_cmd = {0};
3685         static const char *const action[] = {
3686                 "dme-get",
3687                 "dme-peer-get"
3688         };
3689         const char *get = action[!!peer];
3690         int ret;
3691         int retries = UFS_UIC_COMMAND_RETRIES;
3692         struct ufs_pa_layer_attr orig_pwr_info;
3693         struct ufs_pa_layer_attr temp_pwr_info;
3694         bool pwr_mode_change = false;
3695
3696         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)) {
3697                 orig_pwr_info = hba->pwr_info;
3698                 temp_pwr_info = orig_pwr_info;
3699
3700                 if (orig_pwr_info.pwr_tx == FAST_MODE ||
3701                     orig_pwr_info.pwr_rx == FAST_MODE) {
3702                         temp_pwr_info.pwr_tx = FASTAUTO_MODE;
3703                         temp_pwr_info.pwr_rx = FASTAUTO_MODE;
3704                         pwr_mode_change = true;
3705                 } else if (orig_pwr_info.pwr_tx == SLOW_MODE ||
3706                     orig_pwr_info.pwr_rx == SLOW_MODE) {
3707                         temp_pwr_info.pwr_tx = SLOWAUTO_MODE;
3708                         temp_pwr_info.pwr_rx = SLOWAUTO_MODE;
3709                         pwr_mode_change = true;
3710                 }
3711                 if (pwr_mode_change) {
3712                         ret = ufshcd_change_power_mode(hba, &temp_pwr_info);
3713                         if (ret)
3714                                 goto out;
3715                 }
3716         }
3717
3718         uic_cmd.command = peer ?
3719                 UIC_CMD_DME_PEER_GET : UIC_CMD_DME_GET;
3720         uic_cmd.argument1 = attr_sel;
3721
3722         do {
3723                 /* for peer attributes we retry upon failure */
3724                 ret = ufshcd_send_uic_cmd(hba, &uic_cmd);
3725                 if (ret)
3726                         dev_dbg(hba->dev, "%s: attr-id 0x%x error code %d\n",
3727                                 get, UIC_GET_ATTR_ID(attr_sel), ret);
3728         } while (ret && peer && --retries);
3729
3730         if (ret)
3731                 dev_err(hba->dev, "%s: attr-id 0x%x failed %d retries\n",
3732                         get, UIC_GET_ATTR_ID(attr_sel),
3733                         UFS_UIC_COMMAND_RETRIES - retries);
3734
3735         if (mib_val && !ret)
3736                 *mib_val = uic_cmd.argument3;
3737
3738         if (peer && (hba->quirks & UFSHCD_QUIRK_DME_PEER_ACCESS_AUTO_MODE)
3739             && pwr_mode_change)
3740                 ufshcd_change_power_mode(hba, &orig_pwr_info);
3741 out:
3742         return ret;
3743 }
3744 EXPORT_SYMBOL_GPL(ufshcd_dme_get_attr);
3745
3746 /**
3747  * ufshcd_uic_pwr_ctrl - executes UIC commands (which affects the link power
3748  * state) and waits for it to take effect.
3749  *
3750  * @hba: per adapter instance
3751  * @cmd: UIC command to execute
3752  *
3753  * DME operations like DME_SET(PA_PWRMODE), DME_HIBERNATE_ENTER &
3754  * DME_HIBERNATE_EXIT commands take some time to take its effect on both host
3755  * and device UniPro link and hence it's final completion would be indicated by
3756  * dedicated status bits in Interrupt Status register (UPMS, UHES, UHXS) in
3757  * addition to normal UIC command completion Status (UCCS). This function only
3758  * returns after the relevant status bits indicate the completion.
3759  *
3760  * Returns 0 on success, non-zero value on failure
3761  */
3762 static int ufshcd_uic_pwr_ctrl(struct ufs_hba *hba, struct uic_command *cmd)
3763 {
3764         struct completion uic_async_done;
3765         unsigned long flags;
3766         u8 status;
3767         int ret;
3768         bool reenable_intr = false;
3769
3770         mutex_lock(&hba->uic_cmd_mutex);
3771         init_completion(&uic_async_done);
3772         ufshcd_add_delay_before_dme_cmd(hba);
3773
3774         spin_lock_irqsave(hba->host->host_lock, flags);
3775         hba->uic_async_done = &uic_async_done;
3776         if (ufshcd_readl(hba, REG_INTERRUPT_ENABLE) & UIC_COMMAND_COMPL) {
3777                 ufshcd_disable_intr(hba, UIC_COMMAND_COMPL);
3778                 /*
3779                  * Make sure UIC command completion interrupt is disabled before
3780                  * issuing UIC command.
3781                  */
3782                 wmb();
3783                 reenable_intr = true;
3784         }
3785         ret = __ufshcd_send_uic_cmd(hba, cmd, false);
3786         spin_unlock_irqrestore(hba->host->host_lock, flags);
3787         if (ret) {
3788                 dev_err(hba->dev,
3789                         "pwr ctrl cmd 0x%x with mode 0x%x uic error %d\n",
3790                         cmd->command, cmd->argument3, ret);
3791                 goto out;
3792         }
3793
3794         if (!wait_for_completion_timeout(hba->uic_async_done,
3795                                          msecs_to_jiffies(UIC_CMD_TIMEOUT))) {
3796                 dev_err(hba->dev,
3797                         "pwr ctrl cmd 0x%x with mode 0x%x completion timeout\n",
3798                         cmd->command, cmd->argument3);
3799                 ret = -ETIMEDOUT;
3800                 goto out;
3801         }
3802
3803         status = ufshcd_get_upmcrs(hba);
3804         if (status != PWR_LOCAL) {
3805                 dev_err(hba->dev,
3806                         "pwr ctrl cmd 0x%x failed, host upmcrs:0x%x\n",
3807                         cmd->command, status);
3808                 ret = (status != PWR_OK) ? status : -1;
3809         }
3810 out:
3811         if (ret) {
3812                 ufshcd_print_host_state(hba);
3813                 ufshcd_print_pwr_info(hba);
3814                 ufshcd_print_host_regs(hba);
3815         }
3816
3817         spin_lock_irqsave(hba->host->host_lock, flags);
3818         hba->active_uic_cmd = NULL;
3819         hba->uic_async_done = NULL;
3820         if (reenable_intr)
3821                 ufshcd_enable_intr(hba, UIC_COMMAND_COMPL);
3822         spin_unlock_irqrestore(hba->host->host_lock, flags);
3823         mutex_unlock(&hba->uic_cmd_mutex);
3824
3825         return ret;
3826 }
3827
3828 /**
3829  * ufshcd_uic_change_pwr_mode - Perform the UIC power mode chage
3830  *                              using DME_SET primitives.
3831  * @hba: per adapter instance
3832  * @mode: powr mode value
3833  *
3834  * Returns 0 on success, non-zero value on failure
3835  */
3836 static int ufshcd_uic_change_pwr_mode(struct ufs_hba *hba, u8 mode)
3837 {
3838         struct uic_command uic_cmd = {0};
3839         int ret;
3840
3841         if (hba->quirks & UFSHCD_QUIRK_BROKEN_PA_RXHSUNTERMCAP) {
3842                 ret = ufshcd_dme_set(hba,
3843                                 UIC_ARG_MIB_SEL(PA_RXHSUNTERMCAP, 0), 1);
3844                 if (ret) {
3845                         dev_err(hba->dev, "%s: failed to enable PA_RXHSUNTERMCAP ret %d\n",
3846                                                 __func__, ret);
3847                         goto out;
3848                 }
3849         }
3850
3851         uic_cmd.command = UIC_CMD_DME_SET;
3852         uic_cmd.argument1 = UIC_ARG_MIB(PA_PWRMODE);
3853         uic_cmd.argument3 = mode;
3854         ufshcd_hold(hba, false);
3855         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3856         ufshcd_release(hba);
3857
3858 out:
3859         return ret;
3860 }
3861
3862 static int ufshcd_link_recovery(struct ufs_hba *hba)
3863 {
3864         int ret;
3865         unsigned long flags;
3866
3867         spin_lock_irqsave(hba->host->host_lock, flags);
3868         hba->ufshcd_state = UFSHCD_STATE_RESET;
3869         ufshcd_set_eh_in_progress(hba);
3870         spin_unlock_irqrestore(hba->host->host_lock, flags);
3871
3872         ret = ufshcd_host_reset_and_restore(hba);
3873
3874         spin_lock_irqsave(hba->host->host_lock, flags);
3875         if (ret)
3876                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
3877         ufshcd_clear_eh_in_progress(hba);
3878         spin_unlock_irqrestore(hba->host->host_lock, flags);
3879
3880         if (ret)
3881                 dev_err(hba->dev, "%s: link recovery failed, err %d",
3882                         __func__, ret);
3883
3884         return ret;
3885 }
3886
3887 static int __ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
3888 {
3889         int ret;
3890         struct uic_command uic_cmd = {0};
3891         ktime_t start = ktime_get();
3892
3893         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER, PRE_CHANGE);
3894
3895         uic_cmd.command = UIC_CMD_DME_HIBER_ENTER;
3896         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3897         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "enter",
3898                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3899
3900         if (ret) {
3901                 int err;
3902
3903                 dev_err(hba->dev, "%s: hibern8 enter failed. ret = %d\n",
3904                         __func__, ret);
3905
3906                 /*
3907                  * If link recovery fails then return error code returned from
3908                  * ufshcd_link_recovery().
3909                  * If link recovery succeeds then return -EAGAIN to attempt
3910                  * hibern8 enter retry again.
3911                  */
3912                 err = ufshcd_link_recovery(hba);
3913                 if (err) {
3914                         dev_err(hba->dev, "%s: link recovery failed", __func__);
3915                         ret = err;
3916                 } else {
3917                         ret = -EAGAIN;
3918                 }
3919         } else
3920                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_ENTER,
3921                                                                 POST_CHANGE);
3922
3923         return ret;
3924 }
3925
3926 static int ufshcd_uic_hibern8_enter(struct ufs_hba *hba)
3927 {
3928         int ret = 0, retries;
3929
3930         for (retries = UIC_HIBERN8_ENTER_RETRIES; retries > 0; retries--) {
3931                 ret = __ufshcd_uic_hibern8_enter(hba);
3932                 if (!ret)
3933                         goto out;
3934         }
3935 out:
3936         return ret;
3937 }
3938
3939 static int ufshcd_uic_hibern8_exit(struct ufs_hba *hba)
3940 {
3941         struct uic_command uic_cmd = {0};
3942         int ret;
3943         ktime_t start = ktime_get();
3944
3945         ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT, PRE_CHANGE);
3946
3947         uic_cmd.command = UIC_CMD_DME_HIBER_EXIT;
3948         ret = ufshcd_uic_pwr_ctrl(hba, &uic_cmd);
3949         trace_ufshcd_profile_hibern8(dev_name(hba->dev), "exit",
3950                              ktime_to_us(ktime_sub(ktime_get(), start)), ret);
3951
3952         if (ret) {
3953                 dev_err(hba->dev, "%s: hibern8 exit failed. ret = %d\n",
3954                         __func__, ret);
3955                 ret = ufshcd_link_recovery(hba);
3956         } else {
3957                 ufshcd_vops_hibern8_notify(hba, UIC_CMD_DME_HIBER_EXIT,
3958                                                                 POST_CHANGE);
3959                 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_get();
3960                 hba->ufs_stats.hibern8_exit_cnt++;
3961         }
3962
3963         return ret;
3964 }
3965
3966 static void ufshcd_auto_hibern8_enable(struct ufs_hba *hba)
3967 {
3968         unsigned long flags;
3969
3970         if (!(hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) || !hba->ahit)
3971                 return;
3972
3973         spin_lock_irqsave(hba->host->host_lock, flags);
3974         ufshcd_writel(hba, hba->ahit, REG_AUTO_HIBERNATE_IDLE_TIMER);
3975         spin_unlock_irqrestore(hba->host->host_lock, flags);
3976 }
3977
3978  /**
3979  * ufshcd_init_pwr_info - setting the POR (power on reset)
3980  * values in hba power info
3981  * @hba: per-adapter instance
3982  */
3983 static void ufshcd_init_pwr_info(struct ufs_hba *hba)
3984 {
3985         hba->pwr_info.gear_rx = UFS_PWM_G1;
3986         hba->pwr_info.gear_tx = UFS_PWM_G1;
3987         hba->pwr_info.lane_rx = 1;
3988         hba->pwr_info.lane_tx = 1;
3989         hba->pwr_info.pwr_rx = SLOWAUTO_MODE;
3990         hba->pwr_info.pwr_tx = SLOWAUTO_MODE;
3991         hba->pwr_info.hs_rate = 0;
3992 }
3993
3994 /**
3995  * ufshcd_get_max_pwr_mode - reads the max power mode negotiated with device
3996  * @hba: per-adapter instance
3997  */
3998 static int ufshcd_get_max_pwr_mode(struct ufs_hba *hba)
3999 {
4000         struct ufs_pa_layer_attr *pwr_info = &hba->max_pwr_info.info;
4001
4002         if (hba->max_pwr_info.is_valid)
4003                 return 0;
4004
4005         pwr_info->pwr_tx = FAST_MODE;
4006         pwr_info->pwr_rx = FAST_MODE;
4007         pwr_info->hs_rate = PA_HS_MODE_B;
4008
4009         /* Get the connected lane count */
4010         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDRXDATALANES),
4011                         &pwr_info->lane_rx);
4012         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4013                         &pwr_info->lane_tx);
4014
4015         if (!pwr_info->lane_rx || !pwr_info->lane_tx) {
4016                 dev_err(hba->dev, "%s: invalid connected lanes value. rx=%d, tx=%d\n",
4017                                 __func__,
4018                                 pwr_info->lane_rx,
4019                                 pwr_info->lane_tx);
4020                 return -EINVAL;
4021         }
4022
4023         /*
4024          * First, get the maximum gears of HS speed.
4025          * If a zero value, it means there is no HSGEAR capability.
4026          * Then, get the maximum gears of PWM speed.
4027          */
4028         ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR), &pwr_info->gear_rx);
4029         if (!pwr_info->gear_rx) {
4030                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4031                                 &pwr_info->gear_rx);
4032                 if (!pwr_info->gear_rx) {
4033                         dev_err(hba->dev, "%s: invalid max pwm rx gear read = %d\n",
4034                                 __func__, pwr_info->gear_rx);
4035                         return -EINVAL;
4036                 }
4037                 pwr_info->pwr_rx = SLOW_MODE;
4038         }
4039
4040         ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXHSGEAR),
4041                         &pwr_info->gear_tx);
4042         if (!pwr_info->gear_tx) {
4043                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_MAXRXPWMGEAR),
4044                                 &pwr_info->gear_tx);
4045                 if (!pwr_info->gear_tx) {
4046                         dev_err(hba->dev, "%s: invalid max pwm tx gear read = %d\n",
4047                                 __func__, pwr_info->gear_tx);
4048                         return -EINVAL;
4049                 }
4050                 pwr_info->pwr_tx = SLOW_MODE;
4051         }
4052
4053         hba->max_pwr_info.is_valid = true;
4054         return 0;
4055 }
4056
4057 static int ufshcd_change_power_mode(struct ufs_hba *hba,
4058                              struct ufs_pa_layer_attr *pwr_mode)
4059 {
4060         int ret;
4061
4062         /* if already configured to the requested pwr_mode */
4063         if (pwr_mode->gear_rx == hba->pwr_info.gear_rx &&
4064             pwr_mode->gear_tx == hba->pwr_info.gear_tx &&
4065             pwr_mode->lane_rx == hba->pwr_info.lane_rx &&
4066             pwr_mode->lane_tx == hba->pwr_info.lane_tx &&
4067             pwr_mode->pwr_rx == hba->pwr_info.pwr_rx &&
4068             pwr_mode->pwr_tx == hba->pwr_info.pwr_tx &&
4069             pwr_mode->hs_rate == hba->pwr_info.hs_rate) {
4070                 dev_dbg(hba->dev, "%s: power already configured\n", __func__);
4071                 return 0;
4072         }
4073
4074         /*
4075          * Configure attributes for power mode change with below.
4076          * - PA_RXGEAR, PA_ACTIVERXDATALANES, PA_RXTERMINATION,
4077          * - PA_TXGEAR, PA_ACTIVETXDATALANES, PA_TXTERMINATION,
4078          * - PA_HSSERIES
4079          */
4080         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXGEAR), pwr_mode->gear_rx);
4081         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVERXDATALANES),
4082                         pwr_mode->lane_rx);
4083         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4084                         pwr_mode->pwr_rx == FAST_MODE)
4085                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), TRUE);
4086         else
4087                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), FALSE);
4088
4089         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXGEAR), pwr_mode->gear_tx);
4090         ufshcd_dme_set(hba, UIC_ARG_MIB(PA_ACTIVETXDATALANES),
4091                         pwr_mode->lane_tx);
4092         if (pwr_mode->pwr_tx == FASTAUTO_MODE ||
4093                         pwr_mode->pwr_tx == FAST_MODE)
4094                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), TRUE);
4095         else
4096                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), FALSE);
4097
4098         if (pwr_mode->pwr_rx == FASTAUTO_MODE ||
4099             pwr_mode->pwr_tx == FASTAUTO_MODE ||
4100             pwr_mode->pwr_rx == FAST_MODE ||
4101             pwr_mode->pwr_tx == FAST_MODE)
4102                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HSSERIES),
4103                                                 pwr_mode->hs_rate);
4104
4105         ret = ufshcd_uic_change_pwr_mode(hba, pwr_mode->pwr_rx << 4
4106                         | pwr_mode->pwr_tx);
4107
4108         if (ret) {
4109                 dev_err(hba->dev,
4110                         "%s: power mode change failed %d\n", __func__, ret);
4111         } else {
4112                 ufshcd_vops_pwr_change_notify(hba, POST_CHANGE, NULL,
4113                                                                 pwr_mode);
4114
4115                 memcpy(&hba->pwr_info, pwr_mode,
4116                         sizeof(struct ufs_pa_layer_attr));
4117         }
4118
4119         return ret;
4120 }
4121
4122 /**
4123  * ufshcd_config_pwr_mode - configure a new power mode
4124  * @hba: per-adapter instance
4125  * @desired_pwr_mode: desired power configuration
4126  */
4127 int ufshcd_config_pwr_mode(struct ufs_hba *hba,
4128                 struct ufs_pa_layer_attr *desired_pwr_mode)
4129 {
4130         struct ufs_pa_layer_attr final_params = { 0 };
4131         int ret;
4132
4133         ret = ufshcd_vops_pwr_change_notify(hba, PRE_CHANGE,
4134                                         desired_pwr_mode, &final_params);
4135
4136         if (ret)
4137                 memcpy(&final_params, desired_pwr_mode, sizeof(final_params));
4138
4139         ret = ufshcd_change_power_mode(hba, &final_params);
4140         if (!ret)
4141                 ufshcd_print_pwr_info(hba);
4142
4143         return ret;
4144 }
4145 EXPORT_SYMBOL_GPL(ufshcd_config_pwr_mode);
4146
4147 /**
4148  * ufshcd_complete_dev_init() - checks device readiness
4149  * @hba: per-adapter instance
4150  *
4151  * Set fDeviceInit flag and poll until device toggles it.
4152  */
4153 static int ufshcd_complete_dev_init(struct ufs_hba *hba)
4154 {
4155         int i;
4156         int err;
4157         bool flag_res = 1;
4158
4159         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
4160                 QUERY_FLAG_IDN_FDEVICEINIT, NULL);
4161         if (err) {
4162                 dev_err(hba->dev,
4163                         "%s setting fDeviceInit flag failed with error %d\n",
4164                         __func__, err);
4165                 goto out;
4166         }
4167
4168         /* poll for max. 1000 iterations for fDeviceInit flag to clear */
4169         for (i = 0; i < 1000 && !err && flag_res; i++)
4170                 err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
4171                         QUERY_FLAG_IDN_FDEVICEINIT, &flag_res);
4172
4173         if (err)
4174                 dev_err(hba->dev,
4175                         "%s reading fDeviceInit flag failed with error %d\n",
4176                         __func__, err);
4177         else if (flag_res)
4178                 dev_err(hba->dev,
4179                         "%s fDeviceInit was not cleared by the device\n",
4180                         __func__);
4181
4182 out:
4183         return err;
4184 }
4185
4186 /**
4187  * ufshcd_make_hba_operational - Make UFS controller operational
4188  * @hba: per adapter instance
4189  *
4190  * To bring UFS host controller to operational state,
4191  * 1. Enable required interrupts
4192  * 2. Configure interrupt aggregation
4193  * 3. Program UTRL and UTMRL base address
4194  * 4. Configure run-stop-registers
4195  *
4196  * Returns 0 on success, non-zero value on failure
4197  */
4198 static int ufshcd_make_hba_operational(struct ufs_hba *hba)
4199 {
4200         int err = 0;
4201         u32 reg;
4202
4203         /* Enable required interrupts */
4204         ufshcd_enable_intr(hba, UFSHCD_ENABLE_INTRS);
4205
4206         /* Configure interrupt aggregation */
4207         if (ufshcd_is_intr_aggr_allowed(hba))
4208                 ufshcd_config_intr_aggr(hba, hba->nutrs - 1, INT_AGGR_DEF_TO);
4209         else
4210                 ufshcd_disable_intr_aggr(hba);
4211
4212         /* Configure UTRL and UTMRL base address registers */
4213         ufshcd_writel(hba, lower_32_bits(hba->utrdl_dma_addr),
4214                         REG_UTP_TRANSFER_REQ_LIST_BASE_L);
4215         ufshcd_writel(hba, upper_32_bits(hba->utrdl_dma_addr),
4216                         REG_UTP_TRANSFER_REQ_LIST_BASE_H);
4217         ufshcd_writel(hba, lower_32_bits(hba->utmrdl_dma_addr),
4218                         REG_UTP_TASK_REQ_LIST_BASE_L);
4219         ufshcd_writel(hba, upper_32_bits(hba->utmrdl_dma_addr),
4220                         REG_UTP_TASK_REQ_LIST_BASE_H);
4221
4222         /*
4223          * Make sure base address and interrupt setup are updated before
4224          * enabling the run/stop registers below.
4225          */
4226         wmb();
4227
4228         /*
4229          * UCRDY, UTMRLDY and UTRLRDY bits must be 1
4230          */
4231         reg = ufshcd_readl(hba, REG_CONTROLLER_STATUS);
4232         if (!(ufshcd_get_lists_status(reg))) {
4233                 ufshcd_enable_run_stop_reg(hba);
4234         } else {
4235                 dev_err(hba->dev,
4236                         "Host controller not ready to process requests");
4237                 err = -EIO;
4238                 goto out;
4239         }
4240
4241 out:
4242         return err;
4243 }
4244
4245 /**
4246  * ufshcd_hba_stop - Send controller to reset state
4247  * @hba: per adapter instance
4248  * @can_sleep: perform sleep or just spin
4249  */
4250 static inline void ufshcd_hba_stop(struct ufs_hba *hba, bool can_sleep)
4251 {
4252         int err;
4253
4254         ufshcd_writel(hba, CONTROLLER_DISABLE,  REG_CONTROLLER_ENABLE);
4255         err = ufshcd_wait_for_register(hba, REG_CONTROLLER_ENABLE,
4256                                         CONTROLLER_ENABLE, CONTROLLER_DISABLE,
4257                                         10, 1, can_sleep);
4258         if (err)
4259                 dev_err(hba->dev, "%s: Controller disable failed\n", __func__);
4260 }
4261
4262 /**
4263  * ufshcd_hba_execute_hce - initialize the controller
4264  * @hba: per adapter instance
4265  *
4266  * The controller resets itself and controller firmware initialization
4267  * sequence kicks off. When controller is ready it will set
4268  * the Host Controller Enable bit to 1.
4269  *
4270  * Returns 0 on success, non-zero value on failure
4271  */
4272 static int ufshcd_hba_execute_hce(struct ufs_hba *hba)
4273 {
4274         int retry;
4275
4276         /*
4277          * msleep of 1 and 5 used in this function might result in msleep(20),
4278          * but it was necessary to send the UFS FPGA to reset mode during
4279          * development and testing of this driver. msleep can be changed to
4280          * mdelay and retry count can be reduced based on the controller.
4281          */
4282         if (!ufshcd_is_hba_active(hba))
4283                 /* change controller state to "reset state" */
4284                 ufshcd_hba_stop(hba, true);
4285
4286         /* UniPro link is disabled at this point */
4287         ufshcd_set_link_off(hba);
4288
4289         ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4290
4291         /* start controller initialization sequence */
4292         ufshcd_hba_start(hba);
4293
4294         /*
4295          * To initialize a UFS host controller HCE bit must be set to 1.
4296          * During initialization the HCE bit value changes from 1->0->1.
4297          * When the host controller completes initialization sequence
4298          * it sets the value of HCE bit to 1. The same HCE bit is read back
4299          * to check if the controller has completed initialization sequence.
4300          * So without this delay the value HCE = 1, set in the previous
4301          * instruction might be read back.
4302          * This delay can be changed based on the controller.
4303          */
4304         msleep(1);
4305
4306         /* wait for the host controller to complete initialization */
4307         retry = 10;
4308         while (ufshcd_is_hba_active(hba)) {
4309                 if (retry) {
4310                         retry--;
4311                 } else {
4312                         dev_err(hba->dev,
4313                                 "Controller enable failed\n");
4314                         return -EIO;
4315                 }
4316                 msleep(5);
4317         }
4318
4319         /* enable UIC related interrupts */
4320         ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4321
4322         ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4323
4324         return 0;
4325 }
4326
4327 static int ufshcd_hba_enable(struct ufs_hba *hba)
4328 {
4329         int ret;
4330
4331         if (hba->quirks & UFSHCI_QUIRK_BROKEN_HCE) {
4332                 ufshcd_set_link_off(hba);
4333                 ufshcd_vops_hce_enable_notify(hba, PRE_CHANGE);
4334
4335                 /* enable UIC related interrupts */
4336                 ufshcd_enable_intr(hba, UFSHCD_UIC_MASK);
4337                 ret = ufshcd_dme_reset(hba);
4338                 if (!ret) {
4339                         ret = ufshcd_dme_enable(hba);
4340                         if (!ret)
4341                                 ufshcd_vops_hce_enable_notify(hba, POST_CHANGE);
4342                         if (ret)
4343                                 dev_err(hba->dev,
4344                                         "Host controller enable failed with non-hce\n");
4345                 }
4346         } else {
4347                 ret = ufshcd_hba_execute_hce(hba);
4348         }
4349
4350         return ret;
4351 }
4352 static int ufshcd_disable_tx_lcc(struct ufs_hba *hba, bool peer)
4353 {
4354         int tx_lanes, i, err = 0;
4355
4356         if (!peer)
4357                 ufshcd_dme_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4358                                &tx_lanes);
4359         else
4360                 ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES),
4361                                     &tx_lanes);
4362         for (i = 0; i < tx_lanes; i++) {
4363                 if (!peer)
4364                         err = ufshcd_dme_set(hba,
4365                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4366                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4367                                         0);
4368                 else
4369                         err = ufshcd_dme_peer_set(hba,
4370                                 UIC_ARG_MIB_SEL(TX_LCC_ENABLE,
4371                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(i)),
4372                                         0);
4373                 if (err) {
4374                         dev_err(hba->dev, "%s: TX LCC Disable failed, peer = %d, lane = %d, err = %d",
4375                                 __func__, peer, i, err);
4376                         break;
4377                 }
4378         }
4379
4380         return err;
4381 }
4382
4383 static inline int ufshcd_disable_device_tx_lcc(struct ufs_hba *hba)
4384 {
4385         return ufshcd_disable_tx_lcc(hba, true);
4386 }
4387
4388 /**
4389  * ufshcd_link_startup - Initialize unipro link startup
4390  * @hba: per adapter instance
4391  *
4392  * Returns 0 for success, non-zero in case of failure
4393  */
4394 static int ufshcd_link_startup(struct ufs_hba *hba)
4395 {
4396         int ret;
4397         int retries = DME_LINKSTARTUP_RETRIES;
4398         bool link_startup_again = false;
4399
4400         /*
4401          * If UFS device isn't active then we will have to issue link startup
4402          * 2 times to make sure the device state move to active.
4403          */
4404         if (!ufshcd_is_ufs_dev_active(hba))
4405                 link_startup_again = true;
4406
4407 link_startup:
4408         do {
4409                 ufshcd_vops_link_startup_notify(hba, PRE_CHANGE);
4410
4411                 ret = ufshcd_dme_link_startup(hba);
4412
4413                 /* check if device is detected by inter-connect layer */
4414                 if (!ret && !ufshcd_is_device_present(hba)) {
4415                         dev_err(hba->dev, "%s: Device not present\n", __func__);
4416                         ret = -ENXIO;
4417                         goto out;
4418                 }
4419
4420                 /*
4421                  * DME link lost indication is only received when link is up,
4422                  * but we can't be sure if the link is up until link startup
4423                  * succeeds. So reset the local Uni-Pro and try again.
4424                  */
4425                 if (ret && ufshcd_hba_enable(hba))
4426                         goto out;
4427         } while (ret && retries--);
4428
4429         if (ret)
4430                 /* failed to get the link up... retire */
4431                 goto out;
4432
4433         if (link_startup_again) {
4434                 link_startup_again = false;
4435                 retries = DME_LINKSTARTUP_RETRIES;
4436                 goto link_startup;
4437         }
4438
4439         /* Mark that link is up in PWM-G1, 1-lane, SLOW-AUTO mode */
4440         ufshcd_init_pwr_info(hba);
4441         ufshcd_print_pwr_info(hba);
4442
4443         if (hba->quirks & UFSHCD_QUIRK_BROKEN_LCC) {
4444                 ret = ufshcd_disable_device_tx_lcc(hba);
4445                 if (ret)
4446                         goto out;
4447         }
4448
4449         /* Include any host controller configuration via UIC commands */
4450         ret = ufshcd_vops_link_startup_notify(hba, POST_CHANGE);
4451         if (ret)
4452                 goto out;
4453
4454         ret = ufshcd_make_hba_operational(hba);
4455 out:
4456         if (ret) {
4457                 dev_err(hba->dev, "link startup failed %d\n", ret);
4458                 ufshcd_print_host_state(hba);
4459                 ufshcd_print_pwr_info(hba);
4460                 ufshcd_print_host_regs(hba);
4461         }
4462         return ret;
4463 }
4464
4465 /**
4466  * ufshcd_verify_dev_init() - Verify device initialization
4467  * @hba: per-adapter instance
4468  *
4469  * Send NOP OUT UPIU and wait for NOP IN response to check whether the
4470  * device Transport Protocol (UTP) layer is ready after a reset.
4471  * If the UTP layer at the device side is not initialized, it may
4472  * not respond with NOP IN UPIU within timeout of %NOP_OUT_TIMEOUT
4473  * and we retry sending NOP OUT for %NOP_OUT_RETRIES iterations.
4474  */
4475 static int ufshcd_verify_dev_init(struct ufs_hba *hba)
4476 {
4477         int err = 0;
4478         int retries;
4479
4480         ufshcd_hold(hba, false);
4481         mutex_lock(&hba->dev_cmd.lock);
4482         for (retries = NOP_OUT_RETRIES; retries > 0; retries--) {
4483                 err = ufshcd_exec_dev_cmd(hba, DEV_CMD_TYPE_NOP,
4484                                                NOP_OUT_TIMEOUT);
4485
4486                 if (!err || err == -ETIMEDOUT)
4487                         break;
4488
4489                 dev_dbg(hba->dev, "%s: error %d retrying\n", __func__, err);
4490         }
4491         mutex_unlock(&hba->dev_cmd.lock);
4492         ufshcd_release(hba);
4493
4494         if (err)
4495                 dev_err(hba->dev, "%s: NOP OUT failed %d\n", __func__, err);
4496         return err;
4497 }
4498
4499 /**
4500  * ufshcd_set_queue_depth - set lun queue depth
4501  * @sdev: pointer to SCSI device
4502  *
4503  * Read bLUQueueDepth value and activate scsi tagged command
4504  * queueing. For WLUN, queue depth is set to 1. For best-effort
4505  * cases (bLUQueueDepth = 0) the queue depth is set to a maximum
4506  * value that host can queue.
4507  */
4508 static void ufshcd_set_queue_depth(struct scsi_device *sdev)
4509 {
4510         int ret = 0;
4511         u8 lun_qdepth;
4512         struct ufs_hba *hba;
4513
4514         hba = shost_priv(sdev->host);
4515
4516         lun_qdepth = hba->nutrs;
4517         ret = ufshcd_read_unit_desc_param(hba,
4518                                           ufshcd_scsi_to_upiu_lun(sdev->lun),
4519                                           UNIT_DESC_PARAM_LU_Q_DEPTH,
4520                                           &lun_qdepth,
4521                                           sizeof(lun_qdepth));
4522
4523         /* Some WLUN doesn't support unit descriptor */
4524         if (ret == -EOPNOTSUPP)
4525                 lun_qdepth = 1;
4526         else if (!lun_qdepth)
4527                 /* eventually, we can figure out the real queue depth */
4528                 lun_qdepth = hba->nutrs;
4529         else
4530                 lun_qdepth = min_t(int, lun_qdepth, hba->nutrs);
4531
4532         dev_dbg(hba->dev, "%s: activate tcq with queue depth %d\n",
4533                         __func__, lun_qdepth);
4534         scsi_change_queue_depth(sdev, lun_qdepth);
4535 }
4536
4537 /*
4538  * ufshcd_get_lu_wp - returns the "b_lu_write_protect" from UNIT DESCRIPTOR
4539  * @hba: per-adapter instance
4540  * @lun: UFS device lun id
4541  * @b_lu_write_protect: pointer to buffer to hold the LU's write protect info
4542  *
4543  * Returns 0 in case of success and b_lu_write_protect status would be returned
4544  * @b_lu_write_protect parameter.
4545  * Returns -ENOTSUPP if reading b_lu_write_protect is not supported.
4546  * Returns -EINVAL in case of invalid parameters passed to this function.
4547  */
4548 static int ufshcd_get_lu_wp(struct ufs_hba *hba,
4549                             u8 lun,
4550                             u8 *b_lu_write_protect)
4551 {
4552         int ret;
4553
4554         if (!b_lu_write_protect)
4555                 ret = -EINVAL;
4556         /*
4557          * According to UFS device spec, RPMB LU can't be write
4558          * protected so skip reading bLUWriteProtect parameter for
4559          * it. For other W-LUs, UNIT DESCRIPTOR is not available.
4560          */
4561         else if (lun >= UFS_UPIU_MAX_GENERAL_LUN)
4562                 ret = -ENOTSUPP;
4563         else
4564                 ret = ufshcd_read_unit_desc_param(hba,
4565                                           lun,
4566                                           UNIT_DESC_PARAM_LU_WR_PROTECT,
4567                                           b_lu_write_protect,
4568                                           sizeof(*b_lu_write_protect));
4569         return ret;
4570 }
4571
4572 /**
4573  * ufshcd_get_lu_power_on_wp_status - get LU's power on write protect
4574  * status
4575  * @hba: per-adapter instance
4576  * @sdev: pointer to SCSI device
4577  *
4578  */
4579 static inline void ufshcd_get_lu_power_on_wp_status(struct ufs_hba *hba,
4580                                                     struct scsi_device *sdev)
4581 {
4582         if (hba->dev_info.f_power_on_wp_en &&
4583             !hba->dev_info.is_lu_power_on_wp) {
4584                 u8 b_lu_write_protect;
4585
4586                 if (!ufshcd_get_lu_wp(hba, ufshcd_scsi_to_upiu_lun(sdev->lun),
4587                                       &b_lu_write_protect) &&
4588                     (b_lu_write_protect == UFS_LU_POWER_ON_WP))
4589                         hba->dev_info.is_lu_power_on_wp = true;
4590         }
4591 }
4592
4593 /**
4594  * ufshcd_slave_alloc - handle initial SCSI device configurations
4595  * @sdev: pointer to SCSI device
4596  *
4597  * Returns success
4598  */
4599 static int ufshcd_slave_alloc(struct scsi_device *sdev)
4600 {
4601         struct ufs_hba *hba;
4602
4603         hba = shost_priv(sdev->host);
4604
4605         /* Mode sense(6) is not supported by UFS, so use Mode sense(10) */
4606         sdev->use_10_for_ms = 1;
4607
4608         /* allow SCSI layer to restart the device in case of errors */
4609         sdev->allow_restart = 1;
4610
4611         /* REPORT SUPPORTED OPERATION CODES is not supported */
4612         sdev->no_report_opcodes = 1;
4613
4614         /* WRITE_SAME command is not supported */
4615         sdev->no_write_same = 1;
4616
4617         ufshcd_set_queue_depth(sdev);
4618
4619         ufshcd_get_lu_power_on_wp_status(hba, sdev);
4620
4621         return 0;
4622 }
4623
4624 /**
4625  * ufshcd_change_queue_depth - change queue depth
4626  * @sdev: pointer to SCSI device
4627  * @depth: required depth to set
4628  *
4629  * Change queue depth and make sure the max. limits are not crossed.
4630  */
4631 static int ufshcd_change_queue_depth(struct scsi_device *sdev, int depth)
4632 {
4633         struct ufs_hba *hba = shost_priv(sdev->host);
4634
4635         if (depth > hba->nutrs)
4636                 depth = hba->nutrs;
4637         return scsi_change_queue_depth(sdev, depth);
4638 }
4639
4640 /**
4641  * ufshcd_slave_configure - adjust SCSI device configurations
4642  * @sdev: pointer to SCSI device
4643  */
4644 static int ufshcd_slave_configure(struct scsi_device *sdev)
4645 {
4646         struct request_queue *q = sdev->request_queue;
4647
4648         blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1);
4649         blk_queue_max_segment_size(q, PRDT_DATA_BYTE_COUNT_MAX);
4650
4651         return 0;
4652 }
4653
4654 /**
4655  * ufshcd_slave_destroy - remove SCSI device configurations
4656  * @sdev: pointer to SCSI device
4657  */
4658 static void ufshcd_slave_destroy(struct scsi_device *sdev)
4659 {
4660         struct ufs_hba *hba;
4661
4662         hba = shost_priv(sdev->host);
4663         /* Drop the reference as it won't be needed anymore */
4664         if (ufshcd_scsi_to_upiu_lun(sdev->lun) == UFS_UPIU_UFS_DEVICE_WLUN) {
4665                 unsigned long flags;
4666
4667                 spin_lock_irqsave(hba->host->host_lock, flags);
4668                 hba->sdev_ufs_device = NULL;
4669                 spin_unlock_irqrestore(hba->host->host_lock, flags);
4670         }
4671 }
4672
4673 /**
4674  * ufshcd_task_req_compl - handle task management request completion
4675  * @hba: per adapter instance
4676  * @index: index of the completed request
4677  * @resp: task management service response
4678  *
4679  * Returns non-zero value on error, zero on success
4680  */
4681 static int ufshcd_task_req_compl(struct ufs_hba *hba, u32 index, u8 *resp)
4682 {
4683         struct utp_task_req_desc *task_req_descp;
4684         struct utp_upiu_task_rsp *task_rsp_upiup;
4685         unsigned long flags;
4686         int ocs_value;
4687         int task_result;
4688
4689         spin_lock_irqsave(hba->host->host_lock, flags);
4690
4691         /* Clear completed tasks from outstanding_tasks */
4692         __clear_bit(index, &hba->outstanding_tasks);
4693
4694         task_req_descp = hba->utmrdl_base_addr;
4695         ocs_value = ufshcd_get_tmr_ocs(&task_req_descp[index]);
4696
4697         if (ocs_value == OCS_SUCCESS) {
4698                 task_rsp_upiup = (struct utp_upiu_task_rsp *)
4699                                 task_req_descp[index].task_rsp_upiu;
4700                 task_result = be32_to_cpu(task_rsp_upiup->output_param1);
4701                 task_result = task_result & MASK_TM_SERVICE_RESP;
4702                 if (resp)
4703                         *resp = (u8)task_result;
4704         } else {
4705                 dev_err(hba->dev, "%s: failed, ocs = 0x%x\n",
4706                                 __func__, ocs_value);
4707         }
4708         spin_unlock_irqrestore(hba->host->host_lock, flags);
4709
4710         return ocs_value;
4711 }
4712
4713 /**
4714  * ufshcd_scsi_cmd_status - Update SCSI command result based on SCSI status
4715  * @lrbp: pointer to local reference block of completed command
4716  * @scsi_status: SCSI command status
4717  *
4718  * Returns value base on SCSI command status
4719  */
4720 static inline int
4721 ufshcd_scsi_cmd_status(struct ufshcd_lrb *lrbp, int scsi_status)
4722 {
4723         int result = 0;
4724
4725         switch (scsi_status) {
4726         case SAM_STAT_CHECK_CONDITION:
4727                 ufshcd_copy_sense_data(lrbp);
4728         case SAM_STAT_GOOD:
4729                 result |= DID_OK << 16 |
4730                           COMMAND_COMPLETE << 8 |
4731                           scsi_status;
4732                 break;
4733         case SAM_STAT_TASK_SET_FULL:
4734         case SAM_STAT_BUSY:
4735         case SAM_STAT_TASK_ABORTED:
4736                 ufshcd_copy_sense_data(lrbp);
4737                 result |= scsi_status;
4738                 break;
4739         default:
4740                 result |= DID_ERROR << 16;
4741                 break;
4742         } /* end of switch */
4743
4744         return result;
4745 }
4746
4747 /**
4748  * ufshcd_transfer_rsp_status - Get overall status of the response
4749  * @hba: per adapter instance
4750  * @lrbp: pointer to local reference block of completed command
4751  *
4752  * Returns result of the command to notify SCSI midlayer
4753  */
4754 static inline int
4755 ufshcd_transfer_rsp_status(struct ufs_hba *hba, struct ufshcd_lrb *lrbp)
4756 {
4757         int result = 0;
4758         int scsi_status;
4759         int ocs;
4760
4761         /* overall command status of utrd */
4762         ocs = ufshcd_get_tr_ocs(lrbp);
4763
4764         switch (ocs) {
4765         case OCS_SUCCESS:
4766                 result = ufshcd_get_req_rsp(lrbp->ucd_rsp_ptr);
4767                 hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
4768                 switch (result) {
4769                 case UPIU_TRANSACTION_RESPONSE:
4770                         /*
4771                          * get the response UPIU result to extract
4772                          * the SCSI command status
4773                          */
4774                         result = ufshcd_get_rsp_upiu_result(lrbp->ucd_rsp_ptr);
4775
4776                         /*
4777                          * get the result based on SCSI status response
4778                          * to notify the SCSI midlayer of the command status
4779                          */
4780                         scsi_status = result & MASK_SCSI_STATUS;
4781                         result = ufshcd_scsi_cmd_status(lrbp, scsi_status);
4782
4783                         /*
4784                          * Currently we are only supporting BKOPs exception
4785                          * events hence we can ignore BKOPs exception event
4786                          * during power management callbacks. BKOPs exception
4787                          * event is not expected to be raised in runtime suspend
4788                          * callback as it allows the urgent bkops.
4789                          * During system suspend, we are anyway forcefully
4790                          * disabling the bkops and if urgent bkops is needed
4791                          * it will be enabled on system resume. Long term
4792                          * solution could be to abort the system suspend if
4793                          * UFS device needs urgent BKOPs.
4794                          */
4795                         if (!hba->pm_op_in_progress &&
4796                             ufshcd_is_exception_event(lrbp->ucd_rsp_ptr))
4797                                 schedule_work(&hba->eeh_work);
4798                         break;
4799                 case UPIU_TRANSACTION_REJECT_UPIU:
4800                         /* TODO: handle Reject UPIU Response */
4801                         result = DID_ERROR << 16;
4802                         dev_err(hba->dev,
4803                                 "Reject UPIU not fully implemented\n");
4804                         break;
4805                 default:
4806                         result = DID_ERROR << 16;
4807                         dev_err(hba->dev,
4808                                 "Unexpected request response code = %x\n",
4809                                 result);
4810                         break;
4811                 }
4812                 break;
4813         case OCS_ABORTED:
4814                 result |= DID_ABORT << 16;
4815                 break;
4816         case OCS_INVALID_COMMAND_STATUS:
4817                 result |= DID_REQUEUE << 16;
4818                 break;
4819         case OCS_INVALID_CMD_TABLE_ATTR:
4820         case OCS_INVALID_PRDT_ATTR:
4821         case OCS_MISMATCH_DATA_BUF_SIZE:
4822         case OCS_MISMATCH_RESP_UPIU_SIZE:
4823         case OCS_PEER_COMM_FAILURE:
4824         case OCS_FATAL_ERROR:
4825         default:
4826                 result |= DID_ERROR << 16;
4827                 dev_err(hba->dev,
4828                                 "OCS error from controller = %x for tag %d\n",
4829                                 ocs, lrbp->task_tag);
4830                 ufshcd_print_host_regs(hba);
4831                 ufshcd_print_host_state(hba);
4832                 break;
4833         } /* end of switch */
4834
4835         if ((host_byte(result) != DID_OK) && !hba->silence_err_logs)
4836                 ufshcd_print_trs(hba, 1 << lrbp->task_tag, true);
4837         return result;
4838 }
4839
4840 /**
4841  * ufshcd_uic_cmd_compl - handle completion of uic command
4842  * @hba: per adapter instance
4843  * @intr_status: interrupt status generated by the controller
4844  */
4845 static void ufshcd_uic_cmd_compl(struct ufs_hba *hba, u32 intr_status)
4846 {
4847         if ((intr_status & UIC_COMMAND_COMPL) && hba->active_uic_cmd) {
4848                 hba->active_uic_cmd->argument2 |=
4849                         ufshcd_get_uic_cmd_result(hba);
4850                 hba->active_uic_cmd->argument3 =
4851                         ufshcd_get_dme_attr_val(hba);
4852                 complete(&hba->active_uic_cmd->done);
4853         }
4854
4855         if ((intr_status & UFSHCD_UIC_PWR_MASK) && hba->uic_async_done)
4856                 complete(hba->uic_async_done);
4857 }
4858
4859 /**
4860  * __ufshcd_transfer_req_compl - handle SCSI and query command completion
4861  * @hba: per adapter instance
4862  * @completed_reqs: requests to complete
4863  */
4864 static void __ufshcd_transfer_req_compl(struct ufs_hba *hba,
4865                                         unsigned long completed_reqs)
4866 {
4867         struct ufshcd_lrb *lrbp;
4868         struct scsi_cmnd *cmd;
4869         int result;
4870         int index;
4871
4872         for_each_set_bit(index, &completed_reqs, hba->nutrs) {
4873                 lrbp = &hba->lrb[index];
4874                 cmd = lrbp->cmd;
4875                 if (cmd) {
4876                         ufshcd_add_command_trace(hba, index, "complete");
4877                         result = ufshcd_transfer_rsp_status(hba, lrbp);
4878                         scsi_dma_unmap(cmd);
4879                         cmd->result = result;
4880                         /* Mark completed command as NULL in LRB */
4881                         lrbp->cmd = NULL;
4882                         clear_bit_unlock(index, &hba->lrb_in_use);
4883                         /* Do not touch lrbp after scsi done */
4884                         cmd->scsi_done(cmd);
4885                         __ufshcd_release(hba);
4886                 } else if (lrbp->command_type == UTP_CMD_TYPE_DEV_MANAGE ||
4887                         lrbp->command_type == UTP_CMD_TYPE_UFS_STORAGE) {
4888                         if (hba->dev_cmd.complete) {
4889                                 ufshcd_add_command_trace(hba, index,
4890                                                 "dev_complete");
4891                                 complete(hba->dev_cmd.complete);
4892                         }
4893                 }
4894                 if (ufshcd_is_clkscaling_supported(hba))
4895                         hba->clk_scaling.active_reqs--;
4896
4897                 lrbp->compl_time_stamp = ktime_get();
4898         }
4899
4900         /* clear corresponding bits of completed commands */
4901         hba->outstanding_reqs ^= completed_reqs;
4902
4903         ufshcd_clk_scaling_update_busy(hba);
4904
4905         /* we might have free'd some tags above */
4906         wake_up(&hba->dev_cmd.tag_wq);
4907 }
4908
4909 /**
4910  * ufshcd_transfer_req_compl - handle SCSI and query command completion
4911  * @hba: per adapter instance
4912  */
4913 static void ufshcd_transfer_req_compl(struct ufs_hba *hba)
4914 {
4915         unsigned long completed_reqs;
4916         u32 tr_doorbell;
4917
4918         /* Resetting interrupt aggregation counters first and reading the
4919          * DOOR_BELL afterward allows us to handle all the completed requests.
4920          * In order to prevent other interrupts starvation the DB is read once
4921          * after reset. The down side of this solution is the possibility of
4922          * false interrupt if device completes another request after resetting
4923          * aggregation and before reading the DB.
4924          */
4925         if (ufshcd_is_intr_aggr_allowed(hba) &&
4926             !(hba->quirks & UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR))
4927                 ufshcd_reset_intr_aggr(hba);
4928
4929         tr_doorbell = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
4930         completed_reqs = tr_doorbell ^ hba->outstanding_reqs;
4931
4932         __ufshcd_transfer_req_compl(hba, completed_reqs);
4933 }
4934
4935 /**
4936  * ufshcd_disable_ee - disable exception event
4937  * @hba: per-adapter instance
4938  * @mask: exception event to disable
4939  *
4940  * Disables exception event in the device so that the EVENT_ALERT
4941  * bit is not set.
4942  *
4943  * Returns zero on success, non-zero error value on failure.
4944  */
4945 static int ufshcd_disable_ee(struct ufs_hba *hba, u16 mask)
4946 {
4947         int err = 0;
4948         u32 val;
4949
4950         if (!(hba->ee_ctrl_mask & mask))
4951                 goto out;
4952
4953         val = hba->ee_ctrl_mask & ~mask;
4954         val &= MASK_EE_STATUS;
4955         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
4956                         QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4957         if (!err)
4958                 hba->ee_ctrl_mask &= ~mask;
4959 out:
4960         return err;
4961 }
4962
4963 /**
4964  * ufshcd_enable_ee - enable exception event
4965  * @hba: per-adapter instance
4966  * @mask: exception event to enable
4967  *
4968  * Enable corresponding exception event in the device to allow
4969  * device to alert host in critical scenarios.
4970  *
4971  * Returns zero on success, non-zero error value on failure.
4972  */
4973 static int ufshcd_enable_ee(struct ufs_hba *hba, u16 mask)
4974 {
4975         int err = 0;
4976         u32 val;
4977
4978         if (hba->ee_ctrl_mask & mask)
4979                 goto out;
4980
4981         val = hba->ee_ctrl_mask | mask;
4982         val &= MASK_EE_STATUS;
4983         err = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
4984                         QUERY_ATTR_IDN_EE_CONTROL, 0, 0, &val);
4985         if (!err)
4986                 hba->ee_ctrl_mask |= mask;
4987 out:
4988         return err;
4989 }
4990
4991 /**
4992  * ufshcd_enable_auto_bkops - Allow device managed BKOPS
4993  * @hba: per-adapter instance
4994  *
4995  * Allow device to manage background operations on its own. Enabling
4996  * this might lead to inconsistent latencies during normal data transfers
4997  * as the device is allowed to manage its own way of handling background
4998  * operations.
4999  *
5000  * Returns zero on success, non-zero on failure.
5001  */
5002 static int ufshcd_enable_auto_bkops(struct ufs_hba *hba)
5003 {
5004         int err = 0;
5005
5006         if (hba->auto_bkops_enabled)
5007                 goto out;
5008
5009         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_SET_FLAG,
5010                         QUERY_FLAG_IDN_BKOPS_EN, NULL);
5011         if (err) {
5012                 dev_err(hba->dev, "%s: failed to enable bkops %d\n",
5013                                 __func__, err);
5014                 goto out;
5015         }
5016
5017         hba->auto_bkops_enabled = true;
5018         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Enabled");
5019
5020         /* No need of URGENT_BKOPS exception from the device */
5021         err = ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5022         if (err)
5023                 dev_err(hba->dev, "%s: failed to disable exception event %d\n",
5024                                 __func__, err);
5025 out:
5026         return err;
5027 }
5028
5029 /**
5030  * ufshcd_disable_auto_bkops - block device in doing background operations
5031  * @hba: per-adapter instance
5032  *
5033  * Disabling background operations improves command response latency but
5034  * has drawback of device moving into critical state where the device is
5035  * not-operable. Make sure to call ufshcd_enable_auto_bkops() whenever the
5036  * host is idle so that BKOPS are managed effectively without any negative
5037  * impacts.
5038  *
5039  * Returns zero on success, non-zero on failure.
5040  */
5041 static int ufshcd_disable_auto_bkops(struct ufs_hba *hba)
5042 {
5043         int err = 0;
5044
5045         if (!hba->auto_bkops_enabled)
5046                 goto out;
5047
5048         /*
5049          * If host assisted BKOPs is to be enabled, make sure
5050          * urgent bkops exception is allowed.
5051          */
5052         err = ufshcd_enable_ee(hba, MASK_EE_URGENT_BKOPS);
5053         if (err) {
5054                 dev_err(hba->dev, "%s: failed to enable exception event %d\n",
5055                                 __func__, err);
5056                 goto out;
5057         }
5058
5059         err = ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_CLEAR_FLAG,
5060                         QUERY_FLAG_IDN_BKOPS_EN, NULL);
5061         if (err) {
5062                 dev_err(hba->dev, "%s: failed to disable bkops %d\n",
5063                                 __func__, err);
5064                 ufshcd_disable_ee(hba, MASK_EE_URGENT_BKOPS);
5065                 goto out;
5066         }
5067
5068         hba->auto_bkops_enabled = false;
5069         trace_ufshcd_auto_bkops_state(dev_name(hba->dev), "Disabled");
5070         hba->is_urgent_bkops_lvl_checked = false;
5071 out:
5072         return err;
5073 }
5074
5075 /**
5076  * ufshcd_force_reset_auto_bkops - force reset auto bkops state
5077  * @hba: per adapter instance
5078  *
5079  * After a device reset the device may toggle the BKOPS_EN flag
5080  * to default value. The s/w tracking variables should be updated
5081  * as well. This function would change the auto-bkops state based on
5082  * UFSHCD_CAP_KEEP_AUTO_BKOPS_ENABLED_EXCEPT_SUSPEND.
5083  */
5084 static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba)
5085 {
5086         if (ufshcd_keep_autobkops_enabled_except_suspend(hba)) {
5087                 hba->auto_bkops_enabled = false;
5088                 hba->ee_ctrl_mask |= MASK_EE_URGENT_BKOPS;
5089                 ufshcd_enable_auto_bkops(hba);
5090         } else {
5091                 hba->auto_bkops_enabled = true;
5092                 hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS;
5093                 ufshcd_disable_auto_bkops(hba);
5094         }
5095         hba->is_urgent_bkops_lvl_checked = false;
5096 }
5097
5098 static inline int ufshcd_get_bkops_status(struct ufs_hba *hba, u32 *status)
5099 {
5100         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5101                         QUERY_ATTR_IDN_BKOPS_STATUS, 0, 0, status);
5102 }
5103
5104 /**
5105  * ufshcd_bkops_ctrl - control the auto bkops based on current bkops status
5106  * @hba: per-adapter instance
5107  * @status: bkops_status value
5108  *
5109  * Read the bkops_status from the UFS device and Enable fBackgroundOpsEn
5110  * flag in the device to permit background operations if the device
5111  * bkops_status is greater than or equal to "status" argument passed to
5112  * this function, disable otherwise.
5113  *
5114  * Returns 0 for success, non-zero in case of failure.
5115  *
5116  * NOTE: Caller of this function can check the "hba->auto_bkops_enabled" flag
5117  * to know whether auto bkops is enabled or disabled after this function
5118  * returns control to it.
5119  */
5120 static int ufshcd_bkops_ctrl(struct ufs_hba *hba,
5121                              enum bkops_status status)
5122 {
5123         int err;
5124         u32 curr_status = 0;
5125
5126         err = ufshcd_get_bkops_status(hba, &curr_status);
5127         if (err) {
5128                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5129                                 __func__, err);
5130                 goto out;
5131         } else if (curr_status > BKOPS_STATUS_MAX) {
5132                 dev_err(hba->dev, "%s: invalid BKOPS status %d\n",
5133                                 __func__, curr_status);
5134                 err = -EINVAL;
5135                 goto out;
5136         }
5137
5138         if (curr_status >= status)
5139                 err = ufshcd_enable_auto_bkops(hba);
5140         else
5141                 err = ufshcd_disable_auto_bkops(hba);
5142 out:
5143         return err;
5144 }
5145
5146 /**
5147  * ufshcd_urgent_bkops - handle urgent bkops exception event
5148  * @hba: per-adapter instance
5149  *
5150  * Enable fBackgroundOpsEn flag in the device to permit background
5151  * operations.
5152  *
5153  * If BKOPs is enabled, this function returns 0, 1 if the bkops in not enabled
5154  * and negative error value for any other failure.
5155  */
5156 static int ufshcd_urgent_bkops(struct ufs_hba *hba)
5157 {
5158         return ufshcd_bkops_ctrl(hba, hba->urgent_bkops_lvl);
5159 }
5160
5161 static inline int ufshcd_get_ee_status(struct ufs_hba *hba, u32 *status)
5162 {
5163         return ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_READ_ATTR,
5164                         QUERY_ATTR_IDN_EE_STATUS, 0, 0, status);
5165 }
5166
5167 static void ufshcd_bkops_exception_event_handler(struct ufs_hba *hba)
5168 {
5169         int err;
5170         u32 curr_status = 0;
5171
5172         if (hba->is_urgent_bkops_lvl_checked)
5173                 goto enable_auto_bkops;
5174
5175         err = ufshcd_get_bkops_status(hba, &curr_status);
5176         if (err) {
5177                 dev_err(hba->dev, "%s: failed to get BKOPS status %d\n",
5178                                 __func__, err);
5179                 goto out;
5180         }
5181
5182         /*
5183          * We are seeing that some devices are raising the urgent bkops
5184          * exception events even when BKOPS status doesn't indicate performace
5185          * impacted or critical. Handle these device by determining their urgent
5186          * bkops status at runtime.
5187          */
5188         if (curr_status < BKOPS_STATUS_PERF_IMPACT) {
5189                 dev_err(hba->dev, "%s: device raised urgent BKOPS exception for bkops status %d\n",
5190                                 __func__, curr_status);
5191                 /* update the current status as the urgent bkops level */
5192                 hba->urgent_bkops_lvl = curr_status;
5193                 hba->is_urgent_bkops_lvl_checked = true;
5194         }
5195
5196 enable_auto_bkops:
5197         err = ufshcd_enable_auto_bkops(hba);
5198 out:
5199         if (err < 0)
5200                 dev_err(hba->dev, "%s: failed to handle urgent bkops %d\n",
5201                                 __func__, err);
5202 }
5203
5204 /**
5205  * ufshcd_exception_event_handler - handle exceptions raised by device
5206  * @work: pointer to work data
5207  *
5208  * Read bExceptionEventStatus attribute from the device and handle the
5209  * exception event accordingly.
5210  */
5211 static void ufshcd_exception_event_handler(struct work_struct *work)
5212 {
5213         struct ufs_hba *hba;
5214         int err;
5215         u32 status = 0;
5216         hba = container_of(work, struct ufs_hba, eeh_work);
5217
5218         pm_runtime_get_sync(hba->dev);
5219         scsi_block_requests(hba->host);
5220         err = ufshcd_get_ee_status(hba, &status);
5221         if (err) {
5222                 dev_err(hba->dev, "%s: failed to get exception status %d\n",
5223                                 __func__, err);
5224                 goto out;
5225         }
5226
5227         status &= hba->ee_ctrl_mask;
5228
5229         if (status & MASK_EE_URGENT_BKOPS)
5230                 ufshcd_bkops_exception_event_handler(hba);
5231
5232 out:
5233         scsi_unblock_requests(hba->host);
5234         pm_runtime_put_sync(hba->dev);
5235         return;
5236 }
5237
5238 /* Complete requests that have door-bell cleared */
5239 static void ufshcd_complete_requests(struct ufs_hba *hba)
5240 {
5241         ufshcd_transfer_req_compl(hba);
5242         ufshcd_tmc_handler(hba);
5243 }
5244
5245 /**
5246  * ufshcd_quirk_dl_nac_errors - This function checks if error handling is
5247  *                              to recover from the DL NAC errors or not.
5248  * @hba: per-adapter instance
5249  *
5250  * Returns true if error handling is required, false otherwise
5251  */
5252 static bool ufshcd_quirk_dl_nac_errors(struct ufs_hba *hba)
5253 {
5254         unsigned long flags;
5255         bool err_handling = true;
5256
5257         spin_lock_irqsave(hba->host->host_lock, flags);
5258         /*
5259          * UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS only workaround the
5260          * device fatal error and/or DL NAC & REPLAY timeout errors.
5261          */
5262         if (hba->saved_err & (CONTROLLER_FATAL_ERROR | SYSTEM_BUS_FATAL_ERROR))
5263                 goto out;
5264
5265         if ((hba->saved_err & DEVICE_FATAL_ERROR) ||
5266             ((hba->saved_err & UIC_ERROR) &&
5267              (hba->saved_uic_err & UFSHCD_UIC_DL_TCx_REPLAY_ERROR)))
5268                 goto out;
5269
5270         if ((hba->saved_err & UIC_ERROR) &&
5271             (hba->saved_uic_err & UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)) {
5272                 int err;
5273                 /*
5274                  * wait for 50ms to see if we can get any other errors or not.
5275                  */
5276                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5277                 msleep(50);
5278                 spin_lock_irqsave(hba->host->host_lock, flags);
5279
5280                 /*
5281                  * now check if we have got any other severe errors other than
5282                  * DL NAC error?
5283                  */
5284                 if ((hba->saved_err & INT_FATAL_ERRORS) ||
5285                     ((hba->saved_err & UIC_ERROR) &&
5286                     (hba->saved_uic_err & ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)))
5287                         goto out;
5288
5289                 /*
5290                  * As DL NAC is the only error received so far, send out NOP
5291                  * command to confirm if link is still active or not.
5292                  *   - If we don't get any response then do error recovery.
5293                  *   - If we get response then clear the DL NAC error bit.
5294                  */
5295
5296                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5297                 err = ufshcd_verify_dev_init(hba);
5298                 spin_lock_irqsave(hba->host->host_lock, flags);
5299
5300                 if (err)
5301                         goto out;
5302
5303                 /* Link seems to be alive hence ignore the DL NAC errors */
5304                 if (hba->saved_uic_err == UFSHCD_UIC_DL_NAC_RECEIVED_ERROR)
5305                         hba->saved_err &= ~UIC_ERROR;
5306                 /* clear NAC error */
5307                 hba->saved_uic_err &= ~UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5308                 if (!hba->saved_uic_err) {
5309                         err_handling = false;
5310                         goto out;
5311                 }
5312         }
5313 out:
5314         spin_unlock_irqrestore(hba->host->host_lock, flags);
5315         return err_handling;
5316 }
5317
5318 /**
5319  * ufshcd_err_handler - handle UFS errors that require s/w attention
5320  * @work: pointer to work structure
5321  */
5322 static void ufshcd_err_handler(struct work_struct *work)
5323 {
5324         struct ufs_hba *hba;
5325         unsigned long flags;
5326         u32 err_xfer = 0;
5327         u32 err_tm = 0;
5328         int err = 0;
5329         int tag;
5330         bool needs_reset = false;
5331
5332         hba = container_of(work, struct ufs_hba, eh_work);
5333
5334         pm_runtime_get_sync(hba->dev);
5335         ufshcd_hold(hba, false);
5336
5337         spin_lock_irqsave(hba->host->host_lock, flags);
5338         if (hba->ufshcd_state == UFSHCD_STATE_RESET)
5339                 goto out;
5340
5341         hba->ufshcd_state = UFSHCD_STATE_RESET;
5342         ufshcd_set_eh_in_progress(hba);
5343
5344         /* Complete requests that have door-bell cleared by h/w */
5345         ufshcd_complete_requests(hba);
5346
5347         if (hba->dev_quirks & UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5348                 bool ret;
5349
5350                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5351                 /* release the lock as ufshcd_quirk_dl_nac_errors() may sleep */
5352                 ret = ufshcd_quirk_dl_nac_errors(hba);
5353                 spin_lock_irqsave(hba->host->host_lock, flags);
5354                 if (!ret)
5355                         goto skip_err_handling;
5356         }
5357         if ((hba->saved_err & INT_FATAL_ERRORS) ||
5358             ((hba->saved_err & UIC_ERROR) &&
5359             (hba->saved_uic_err & (UFSHCD_UIC_DL_PA_INIT_ERROR |
5360                                    UFSHCD_UIC_DL_NAC_RECEIVED_ERROR |
5361                                    UFSHCD_UIC_DL_TCx_REPLAY_ERROR))))
5362                 needs_reset = true;
5363
5364         /*
5365          * if host reset is required then skip clearing the pending
5366          * transfers forcefully because they will get cleared during
5367          * host reset and restore
5368          */
5369         if (needs_reset)
5370                 goto skip_pending_xfer_clear;
5371
5372         /* release lock as clear command might sleep */
5373         spin_unlock_irqrestore(hba->host->host_lock, flags);
5374         /* Clear pending transfer requests */
5375         for_each_set_bit(tag, &hba->outstanding_reqs, hba->nutrs) {
5376                 if (ufshcd_clear_cmd(hba, tag)) {
5377                         err_xfer = true;
5378                         goto lock_skip_pending_xfer_clear;
5379                 }
5380         }
5381
5382         /* Clear pending task management requests */
5383         for_each_set_bit(tag, &hba->outstanding_tasks, hba->nutmrs) {
5384                 if (ufshcd_clear_tm_cmd(hba, tag)) {
5385                         err_tm = true;
5386                         goto lock_skip_pending_xfer_clear;
5387                 }
5388         }
5389
5390 lock_skip_pending_xfer_clear:
5391         spin_lock_irqsave(hba->host->host_lock, flags);
5392
5393         /* Complete the requests that are cleared by s/w */
5394         ufshcd_complete_requests(hba);
5395
5396         if (err_xfer || err_tm)
5397                 needs_reset = true;
5398
5399 skip_pending_xfer_clear:
5400         /* Fatal errors need reset */
5401         if (needs_reset) {
5402                 unsigned long max_doorbells = (1UL << hba->nutrs) - 1;
5403
5404                 /*
5405                  * ufshcd_reset_and_restore() does the link reinitialization
5406                  * which will need atleast one empty doorbell slot to send the
5407                  * device management commands (NOP and query commands).
5408                  * If there is no slot empty at this moment then free up last
5409                  * slot forcefully.
5410                  */
5411                 if (hba->outstanding_reqs == max_doorbells)
5412                         __ufshcd_transfer_req_compl(hba,
5413                                                     (1UL << (hba->nutrs - 1)));
5414
5415                 spin_unlock_irqrestore(hba->host->host_lock, flags);
5416                 err = ufshcd_reset_and_restore(hba);
5417                 spin_lock_irqsave(hba->host->host_lock, flags);
5418                 if (err) {
5419                         dev_err(hba->dev, "%s: reset and restore failed\n",
5420                                         __func__);
5421                         hba->ufshcd_state = UFSHCD_STATE_ERROR;
5422                 }
5423                 /*
5424                  * Inform scsi mid-layer that we did reset and allow to handle
5425                  * Unit Attention properly.
5426                  */
5427                 scsi_report_bus_reset(hba->host, 0);
5428                 hba->saved_err = 0;
5429                 hba->saved_uic_err = 0;
5430         }
5431
5432 skip_err_handling:
5433         if (!needs_reset) {
5434                 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
5435                 if (hba->saved_err || hba->saved_uic_err)
5436                         dev_err_ratelimited(hba->dev, "%s: exit: saved_err 0x%x saved_uic_err 0x%x",
5437                             __func__, hba->saved_err, hba->saved_uic_err);
5438         }
5439
5440         ufshcd_clear_eh_in_progress(hba);
5441
5442 out:
5443         spin_unlock_irqrestore(hba->host->host_lock, flags);
5444         ufshcd_scsi_unblock_requests(hba);
5445         ufshcd_release(hba);
5446         pm_runtime_put_sync(hba->dev);
5447 }
5448
5449 static void ufshcd_update_uic_reg_hist(struct ufs_uic_err_reg_hist *reg_hist,
5450                 u32 reg)
5451 {
5452         reg_hist->reg[reg_hist->pos] = reg;
5453         reg_hist->tstamp[reg_hist->pos] = ktime_get();
5454         reg_hist->pos = (reg_hist->pos + 1) % UIC_ERR_REG_HIST_LENGTH;
5455 }
5456
5457 /**
5458  * ufshcd_update_uic_error - check and set fatal UIC error flags.
5459  * @hba: per-adapter instance
5460  */
5461 static void ufshcd_update_uic_error(struct ufs_hba *hba)
5462 {
5463         u32 reg;
5464
5465         /* PHY layer lane error */
5466         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_PHY_ADAPTER_LAYER);
5467         /* Ignore LINERESET indication, as this is not an error */
5468         if ((reg & UIC_PHY_ADAPTER_LAYER_ERROR) &&
5469                         (reg & UIC_PHY_ADAPTER_LAYER_LANE_ERR_MASK)) {
5470                 /*
5471                  * To know whether this error is fatal or not, DB timeout
5472                  * must be checked but this error is handled separately.
5473                  */
5474                 dev_dbg(hba->dev, "%s: UIC Lane error reported\n", __func__);
5475                 ufshcd_update_uic_reg_hist(&hba->ufs_stats.pa_err, reg);
5476         }
5477
5478         /* PA_INIT_ERROR is fatal and needs UIC reset */
5479         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DATA_LINK_LAYER);
5480         if (reg)
5481                 ufshcd_update_uic_reg_hist(&hba->ufs_stats.dl_err, reg);
5482
5483         if (reg & UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
5484                 hba->uic_error |= UFSHCD_UIC_DL_PA_INIT_ERROR;
5485         else if (hba->dev_quirks &
5486                    UFS_DEVICE_QUIRK_RECOVERY_FROM_DL_NAC_ERRORS) {
5487                 if (reg & UIC_DATA_LINK_LAYER_ERROR_NAC_RECEIVED)
5488                         hba->uic_error |=
5489                                 UFSHCD_UIC_DL_NAC_RECEIVED_ERROR;
5490                 else if (reg & UIC_DATA_LINK_LAYER_ERROR_TCx_REPLAY_TIMEOUT)
5491                         hba->uic_error |= UFSHCD_UIC_DL_TCx_REPLAY_ERROR;
5492         }
5493
5494         /* UIC NL/TL/DME errors needs software retry */
5495         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_NETWORK_LAYER);
5496         if (reg) {
5497                 ufshcd_update_uic_reg_hist(&hba->ufs_stats.nl_err, reg);
5498                 hba->uic_error |= UFSHCD_UIC_NL_ERROR;
5499         }
5500
5501         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_TRANSPORT_LAYER);
5502         if (reg) {
5503                 ufshcd_update_uic_reg_hist(&hba->ufs_stats.tl_err, reg);
5504                 hba->uic_error |= UFSHCD_UIC_TL_ERROR;
5505         }
5506
5507         reg = ufshcd_readl(hba, REG_UIC_ERROR_CODE_DME);
5508         if (reg) {
5509                 ufshcd_update_uic_reg_hist(&hba->ufs_stats.dme_err, reg);
5510                 hba->uic_error |= UFSHCD_UIC_DME_ERROR;
5511         }
5512
5513         dev_dbg(hba->dev, "%s: UIC error flags = 0x%08x\n",
5514                         __func__, hba->uic_error);
5515 }
5516
5517 /**
5518  * ufshcd_check_errors - Check for errors that need s/w attention
5519  * @hba: per-adapter instance
5520  */
5521 static void ufshcd_check_errors(struct ufs_hba *hba)
5522 {
5523         bool queue_eh_work = false;
5524
5525         if (hba->errors & INT_FATAL_ERRORS)
5526                 queue_eh_work = true;
5527
5528         if (hba->errors & UIC_ERROR) {
5529                 hba->uic_error = 0;
5530                 ufshcd_update_uic_error(hba);
5531                 if (hba->uic_error)
5532                         queue_eh_work = true;
5533         }
5534
5535         if (queue_eh_work) {
5536                 /*
5537                  * update the transfer error masks to sticky bits, let's do this
5538                  * irrespective of current ufshcd_state.
5539                  */
5540                 hba->saved_err |= hba->errors;
5541                 hba->saved_uic_err |= hba->uic_error;
5542
5543                 /* handle fatal errors only when link is functional */
5544                 if (hba->ufshcd_state == UFSHCD_STATE_OPERATIONAL) {
5545                         /* block commands from scsi mid-layer */
5546                         ufshcd_scsi_block_requests(hba);
5547
5548                         hba->ufshcd_state = UFSHCD_STATE_EH_SCHEDULED;
5549
5550                         /* dump controller state before resetting */
5551                         if (hba->saved_err & (INT_FATAL_ERRORS | UIC_ERROR)) {
5552                                 bool pr_prdt = !!(hba->saved_err &
5553                                                 SYSTEM_BUS_FATAL_ERROR);
5554
5555                                 dev_err(hba->dev, "%s: saved_err 0x%x saved_uic_err 0x%x\n",
5556                                         __func__, hba->saved_err,
5557                                         hba->saved_uic_err);
5558
5559                                 ufshcd_print_host_regs(hba);
5560                                 ufshcd_print_pwr_info(hba);
5561                                 ufshcd_print_tmrs(hba, hba->outstanding_tasks);
5562                                 ufshcd_print_trs(hba, hba->outstanding_reqs,
5563                                                         pr_prdt);
5564                         }
5565                         schedule_work(&hba->eh_work);
5566                 }
5567         }
5568         /*
5569          * if (!queue_eh_work) -
5570          * Other errors are either non-fatal where host recovers
5571          * itself without s/w intervention or errors that will be
5572          * handled by the SCSI core layer.
5573          */
5574 }
5575
5576 /**
5577  * ufshcd_tmc_handler - handle task management function completion
5578  * @hba: per adapter instance
5579  */
5580 static void ufshcd_tmc_handler(struct ufs_hba *hba)
5581 {
5582         u32 tm_doorbell;
5583
5584         tm_doorbell = ufshcd_readl(hba, REG_UTP_TASK_REQ_DOOR_BELL);
5585         hba->tm_condition = tm_doorbell ^ hba->outstanding_tasks;
5586         wake_up(&hba->tm_wq);
5587 }
5588
5589 /**
5590  * ufshcd_sl_intr - Interrupt service routine
5591  * @hba: per adapter instance
5592  * @intr_status: contains interrupts generated by the controller
5593  */
5594 static void ufshcd_sl_intr(struct ufs_hba *hba, u32 intr_status)
5595 {
5596         hba->errors = UFSHCD_ERROR_MASK & intr_status;
5597         if (hba->errors)
5598                 ufshcd_check_errors(hba);
5599
5600         if (intr_status & UFSHCD_UIC_MASK)
5601                 ufshcd_uic_cmd_compl(hba, intr_status);
5602
5603         if (intr_status & UTP_TASK_REQ_COMPL)
5604                 ufshcd_tmc_handler(hba);
5605
5606         if (intr_status & UTP_TRANSFER_REQ_COMPL)
5607                 ufshcd_transfer_req_compl(hba);
5608 }
5609
5610 /**
5611  * ufshcd_intr - Main interrupt service routine
5612  * @irq: irq number
5613  * @__hba: pointer to adapter instance
5614  *
5615  * Returns IRQ_HANDLED - If interrupt is valid
5616  *              IRQ_NONE - If invalid interrupt
5617  */
5618 static irqreturn_t ufshcd_intr(int irq, void *__hba)
5619 {
5620         u32 intr_status, enabled_intr_status = 0;
5621         irqreturn_t retval = IRQ_NONE;
5622         struct ufs_hba *hba = __hba;
5623         int retries = hba->nutrs;
5624
5625         spin_lock(hba->host->host_lock);
5626         intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
5627
5628         /*
5629          * There could be max of hba->nutrs reqs in flight and in worst case
5630          * if the reqs get finished 1 by 1 after the interrupt status is
5631          * read, make sure we handle them by checking the interrupt status
5632          * again in a loop until we process all of the reqs before returning.
5633          */
5634         while (intr_status && retries--) {
5635                 enabled_intr_status =
5636                         intr_status & ufshcd_readl(hba, REG_INTERRUPT_ENABLE);
5637                 if (intr_status)
5638                         ufshcd_writel(hba, intr_status, REG_INTERRUPT_STATUS);
5639                 if (enabled_intr_status) {
5640                         ufshcd_sl_intr(hba, enabled_intr_status);
5641                         retval = IRQ_HANDLED;
5642                 }
5643
5644                 intr_status = ufshcd_readl(hba, REG_INTERRUPT_STATUS);
5645         }
5646
5647         spin_unlock(hba->host->host_lock);
5648         return retval;
5649 }
5650
5651 static int ufshcd_clear_tm_cmd(struct ufs_hba *hba, int tag)
5652 {
5653         int err = 0;
5654         u32 mask = 1 << tag;
5655         unsigned long flags;
5656
5657         if (!test_bit(tag, &hba->outstanding_tasks))
5658                 goto out;
5659
5660         spin_lock_irqsave(hba->host->host_lock, flags);
5661         ufshcd_utmrl_clear(hba, tag);
5662         spin_unlock_irqrestore(hba->host->host_lock, flags);
5663
5664         /* poll for max. 1 sec to clear door bell register by h/w */
5665         err = ufshcd_wait_for_register(hba,
5666                         REG_UTP_TASK_REQ_DOOR_BELL,
5667                         mask, 0, 1000, 1000, true);
5668 out:
5669         return err;
5670 }
5671
5672 /**
5673  * ufshcd_issue_tm_cmd - issues task management commands to controller
5674  * @hba: per adapter instance
5675  * @lun_id: LUN ID to which TM command is sent
5676  * @task_id: task ID to which the TM command is applicable
5677  * @tm_function: task management function opcode
5678  * @tm_response: task management service response return value
5679  *
5680  * Returns non-zero value on error, zero on success.
5681  */
5682 static int ufshcd_issue_tm_cmd(struct ufs_hba *hba, int lun_id, int task_id,
5683                 u8 tm_function, u8 *tm_response)
5684 {
5685         struct utp_task_req_desc *task_req_descp;
5686         struct utp_upiu_task_req *task_req_upiup;
5687         struct Scsi_Host *host;
5688         unsigned long flags;
5689         int free_slot;
5690         int err;
5691         int task_tag;
5692
5693         host = hba->host;
5694
5695         /*
5696          * Get free slot, sleep if slots are unavailable.
5697          * Even though we use wait_event() which sleeps indefinitely,
5698          * the maximum wait time is bounded by %TM_CMD_TIMEOUT.
5699          */
5700         wait_event(hba->tm_tag_wq, ufshcd_get_tm_free_slot(hba, &free_slot));
5701         ufshcd_hold(hba, false);
5702
5703         spin_lock_irqsave(host->host_lock, flags);
5704         task_req_descp = hba->utmrdl_base_addr;
5705         task_req_descp += free_slot;
5706
5707         /* Configure task request descriptor */
5708         task_req_descp->header.dword_0 = cpu_to_le32(UTP_REQ_DESC_INT_CMD);
5709         task_req_descp->header.dword_2 =
5710                         cpu_to_le32(OCS_INVALID_COMMAND_STATUS);
5711
5712         /* Configure task request UPIU */
5713         task_req_upiup =
5714                 (struct utp_upiu_task_req *) task_req_descp->task_req_upiu;
5715         task_tag = hba->nutrs + free_slot;
5716         task_req_upiup->header.dword_0 =
5717                 UPIU_HEADER_DWORD(UPIU_TRANSACTION_TASK_REQ, 0,
5718                                               lun_id, task_tag);
5719         task_req_upiup->header.dword_1 =
5720                 UPIU_HEADER_DWORD(0, tm_function, 0, 0);
5721         /*
5722          * The host shall provide the same value for LUN field in the basic
5723          * header and for Input Parameter.
5724          */
5725         task_req_upiup->input_param1 = cpu_to_be32(lun_id);
5726         task_req_upiup->input_param2 = cpu_to_be32(task_id);
5727
5728         ufshcd_vops_setup_task_mgmt(hba, free_slot, tm_function);
5729
5730         /* send command to the controller */
5731         __set_bit(free_slot, &hba->outstanding_tasks);
5732
5733         /* Make sure descriptors are ready before ringing the task doorbell */
5734         wmb();
5735
5736         ufshcd_writel(hba, 1 << free_slot, REG_UTP_TASK_REQ_DOOR_BELL);
5737         /* Make sure that doorbell is committed immediately */
5738         wmb();
5739
5740         spin_unlock_irqrestore(host->host_lock, flags);
5741
5742         ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_send");
5743
5744         /* wait until the task management command is completed */
5745         err = wait_event_timeout(hba->tm_wq,
5746                         test_bit(free_slot, &hba->tm_condition),
5747                         msecs_to_jiffies(TM_CMD_TIMEOUT));
5748         if (!err) {
5749                 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete_err");
5750                 dev_err(hba->dev, "%s: task management cmd 0x%.2x timed-out\n",
5751                                 __func__, tm_function);
5752                 if (ufshcd_clear_tm_cmd(hba, free_slot))
5753                         dev_WARN(hba->dev, "%s: unable clear tm cmd (slot %d) after timeout\n",
5754                                         __func__, free_slot);
5755                 err = -ETIMEDOUT;
5756         } else {
5757                 err = ufshcd_task_req_compl(hba, free_slot, tm_response);
5758                 ufshcd_add_tm_upiu_trace(hba, task_tag, "tm_complete");
5759         }
5760
5761         clear_bit(free_slot, &hba->tm_condition);
5762         ufshcd_put_tm_slot(hba, free_slot);
5763         wake_up(&hba->tm_tag_wq);
5764
5765         ufshcd_release(hba);
5766         return err;
5767 }
5768
5769 /**
5770  * ufshcd_eh_device_reset_handler - device reset handler registered to
5771  *                                    scsi layer.
5772  * @cmd: SCSI command pointer
5773  *
5774  * Returns SUCCESS/FAILED
5775  */
5776 static int ufshcd_eh_device_reset_handler(struct scsi_cmnd *cmd)
5777 {
5778         struct Scsi_Host *host;
5779         struct ufs_hba *hba;
5780         u32 pos;
5781         int err;
5782         u8 resp = 0xF, lun;
5783         unsigned long flags;
5784
5785         host = cmd->device->host;
5786         hba = shost_priv(host);
5787
5788         lun = ufshcd_scsi_to_upiu_lun(cmd->device->lun);
5789         err = ufshcd_issue_tm_cmd(hba, lun, 0, UFS_LOGICAL_RESET, &resp);
5790         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
5791                 if (!err)
5792                         err = resp;
5793                 goto out;
5794         }
5795
5796         /* clear the commands that were pending for corresponding LUN */
5797         for_each_set_bit(pos, &hba->outstanding_reqs, hba->nutrs) {
5798                 if (hba->lrb[pos].lun == lun) {
5799                         err = ufshcd_clear_cmd(hba, pos);
5800                         if (err)
5801                                 break;
5802                 }
5803         }
5804         spin_lock_irqsave(host->host_lock, flags);
5805         ufshcd_transfer_req_compl(hba);
5806         spin_unlock_irqrestore(host->host_lock, flags);
5807
5808 out:
5809         hba->req_abort_count = 0;
5810         if (!err) {
5811                 err = SUCCESS;
5812         } else {
5813                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
5814                 err = FAILED;
5815         }
5816         return err;
5817 }
5818
5819 static void ufshcd_set_req_abort_skip(struct ufs_hba *hba, unsigned long bitmap)
5820 {
5821         struct ufshcd_lrb *lrbp;
5822         int tag;
5823
5824         for_each_set_bit(tag, &bitmap, hba->nutrs) {
5825                 lrbp = &hba->lrb[tag];
5826                 lrbp->req_abort_skip = true;
5827         }
5828 }
5829
5830 /**
5831  * ufshcd_abort - abort a specific command
5832  * @cmd: SCSI command pointer
5833  *
5834  * Abort the pending command in device by sending UFS_ABORT_TASK task management
5835  * command, and in host controller by clearing the door-bell register. There can
5836  * be race between controller sending the command to the device while abort is
5837  * issued. To avoid that, first issue UFS_QUERY_TASK to check if the command is
5838  * really issued and then try to abort it.
5839  *
5840  * Returns SUCCESS/FAILED
5841  */
5842 static int ufshcd_abort(struct scsi_cmnd *cmd)
5843 {
5844         struct Scsi_Host *host;
5845         struct ufs_hba *hba;
5846         unsigned long flags;
5847         unsigned int tag;
5848         int err = 0;
5849         int poll_cnt;
5850         u8 resp = 0xF;
5851         struct ufshcd_lrb *lrbp;
5852         u32 reg;
5853
5854         host = cmd->device->host;
5855         hba = shost_priv(host);
5856         tag = cmd->request->tag;
5857         lrbp = &hba->lrb[tag];
5858         if (!ufshcd_valid_tag(hba, tag)) {
5859                 dev_err(hba->dev,
5860                         "%s: invalid command tag %d: cmd=0x%p, cmd->request=0x%p",
5861                         __func__, tag, cmd, cmd->request);
5862                 BUG();
5863         }
5864
5865         /*
5866          * Task abort to the device W-LUN is illegal. When this command
5867          * will fail, due to spec violation, scsi err handling next step
5868          * will be to send LU reset which, again, is a spec violation.
5869          * To avoid these unnecessary/illegal step we skip to the last error
5870          * handling stage: reset and restore.
5871          */
5872         if (lrbp->lun == UFS_UPIU_UFS_DEVICE_WLUN)
5873                 return ufshcd_eh_host_reset_handler(cmd);
5874
5875         ufshcd_hold(hba, false);
5876         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5877         /* If command is already aborted/completed, return SUCCESS */
5878         if (!(test_bit(tag, &hba->outstanding_reqs))) {
5879                 dev_err(hba->dev,
5880                         "%s: cmd at tag %d already completed, outstanding=0x%lx, doorbell=0x%x\n",
5881                         __func__, tag, hba->outstanding_reqs, reg);
5882                 goto out;
5883         }
5884
5885         if (!(reg & (1 << tag))) {
5886                 dev_err(hba->dev,
5887                 "%s: cmd was completed, but without a notifying intr, tag = %d",
5888                 __func__, tag);
5889         }
5890
5891         /* Print Transfer Request of aborted task */
5892         dev_err(hba->dev, "%s: Device abort task at tag %d\n", __func__, tag);
5893
5894         /*
5895          * Print detailed info about aborted request.
5896          * As more than one request might get aborted at the same time,
5897          * print full information only for the first aborted request in order
5898          * to reduce repeated printouts. For other aborted requests only print
5899          * basic details.
5900          */
5901         scsi_print_command(hba->lrb[tag].cmd);
5902         if (!hba->req_abort_count) {
5903                 ufshcd_print_host_regs(hba);
5904                 ufshcd_print_host_state(hba);
5905                 ufshcd_print_pwr_info(hba);
5906                 ufshcd_print_trs(hba, 1 << tag, true);
5907         } else {
5908                 ufshcd_print_trs(hba, 1 << tag, false);
5909         }
5910         hba->req_abort_count++;
5911
5912         /* Skip task abort in case previous aborts failed and report failure */
5913         if (lrbp->req_abort_skip) {
5914                 err = -EIO;
5915                 goto out;
5916         }
5917
5918         for (poll_cnt = 100; poll_cnt; poll_cnt--) {
5919                 err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
5920                                 UFS_QUERY_TASK, &resp);
5921                 if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_SUCCEEDED) {
5922                         /* cmd pending in the device */
5923                         dev_err(hba->dev, "%s: cmd pending in the device. tag = %d\n",
5924                                 __func__, tag);
5925                         break;
5926                 } else if (!err && resp == UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
5927                         /*
5928                          * cmd not pending in the device, check if it is
5929                          * in transition.
5930                          */
5931                         dev_err(hba->dev, "%s: cmd at tag %d not pending in the device.\n",
5932                                 __func__, tag);
5933                         reg = ufshcd_readl(hba, REG_UTP_TRANSFER_REQ_DOOR_BELL);
5934                         if (reg & (1 << tag)) {
5935                                 /* sleep for max. 200us to stabilize */
5936                                 usleep_range(100, 200);
5937                                 continue;
5938                         }
5939                         /* command completed already */
5940                         dev_err(hba->dev, "%s: cmd at tag %d successfully cleared from DB.\n",
5941                                 __func__, tag);
5942                         goto cleanup;
5943                 } else {
5944                         dev_err(hba->dev,
5945                                 "%s: no response from device. tag = %d, err %d\n",
5946                                 __func__, tag, err);
5947                         if (!err)
5948                                 err = resp; /* service response error */
5949                         goto out;
5950                 }
5951         }
5952
5953         if (!poll_cnt) {
5954                 err = -EBUSY;
5955                 goto out;
5956         }
5957
5958         err = ufshcd_issue_tm_cmd(hba, lrbp->lun, lrbp->task_tag,
5959                         UFS_ABORT_TASK, &resp);
5960         if (err || resp != UPIU_TASK_MANAGEMENT_FUNC_COMPL) {
5961                 if (!err) {
5962                         err = resp; /* service response error */
5963                         dev_err(hba->dev, "%s: issued. tag = %d, err %d\n",
5964                                 __func__, tag, err);
5965                 }
5966                 goto out;
5967         }
5968
5969         err = ufshcd_clear_cmd(hba, tag);
5970         if (err) {
5971                 dev_err(hba->dev, "%s: Failed clearing cmd at tag %d, err %d\n",
5972                         __func__, tag, err);
5973                 goto out;
5974         }
5975
5976 cleanup:
5977         scsi_dma_unmap(cmd);
5978
5979         spin_lock_irqsave(host->host_lock, flags);
5980         ufshcd_outstanding_req_clear(hba, tag);
5981         hba->lrb[tag].cmd = NULL;
5982         spin_unlock_irqrestore(host->host_lock, flags);
5983
5984         clear_bit_unlock(tag, &hba->lrb_in_use);
5985         wake_up(&hba->dev_cmd.tag_wq);
5986
5987 out:
5988         if (!err) {
5989                 err = SUCCESS;
5990         } else {
5991                 dev_err(hba->dev, "%s: failed with err %d\n", __func__, err);
5992                 ufshcd_set_req_abort_skip(hba, hba->outstanding_reqs);
5993                 err = FAILED;
5994         }
5995
5996         /*
5997          * This ufshcd_release() corresponds to the original scsi cmd that got
5998          * aborted here (as we won't get any IRQ for it).
5999          */
6000         ufshcd_release(hba);
6001         return err;
6002 }
6003
6004 /**
6005  * ufshcd_host_reset_and_restore - reset and restore host controller
6006  * @hba: per-adapter instance
6007  *
6008  * Note that host controller reset may issue DME_RESET to
6009  * local and remote (device) Uni-Pro stack and the attributes
6010  * are reset to default state.
6011  *
6012  * Returns zero on success, non-zero on failure
6013  */
6014 static int ufshcd_host_reset_and_restore(struct ufs_hba *hba)
6015 {
6016         int err;
6017         unsigned long flags;
6018
6019         /*
6020          * Stop the host controller and complete the requests
6021          * cleared by h/w
6022          */
6023         spin_lock_irqsave(hba->host->host_lock, flags);
6024         ufshcd_hba_stop(hba, false);
6025         hba->silence_err_logs = true;
6026         ufshcd_complete_requests(hba);
6027         hba->silence_err_logs = false;
6028         spin_unlock_irqrestore(hba->host->host_lock, flags);
6029
6030         /* scale up clocks to max frequency before full reinitialization */
6031         ufshcd_scale_clks(hba, true);
6032
6033         err = ufshcd_hba_enable(hba);
6034         if (err)
6035                 goto out;
6036
6037         /* Establish the link again and restore the device */
6038         err = ufshcd_probe_hba(hba);
6039
6040         if (!err && (hba->ufshcd_state != UFSHCD_STATE_OPERATIONAL))
6041                 err = -EIO;
6042 out:
6043         if (err)
6044                 dev_err(hba->dev, "%s: Host init failed %d\n", __func__, err);
6045
6046         return err;
6047 }
6048
6049 /**
6050  * ufshcd_reset_and_restore - reset and re-initialize host/device
6051  * @hba: per-adapter instance
6052  *
6053  * Reset and recover device, host and re-establish link. This
6054  * is helpful to recover the communication in fatal error conditions.
6055  *
6056  * Returns zero on success, non-zero on failure
6057  */
6058 static int ufshcd_reset_and_restore(struct ufs_hba *hba)
6059 {
6060         int err = 0;
6061         int retries = MAX_HOST_RESET_RETRIES;
6062
6063         do {
6064                 err = ufshcd_host_reset_and_restore(hba);
6065         } while (err && --retries);
6066
6067         return err;
6068 }
6069
6070 /**
6071  * ufshcd_eh_host_reset_handler - host reset handler registered to scsi layer
6072  * @cmd: SCSI command pointer
6073  *
6074  * Returns SUCCESS/FAILED
6075  */
6076 static int ufshcd_eh_host_reset_handler(struct scsi_cmnd *cmd)
6077 {
6078         int err;
6079         unsigned long flags;
6080         struct ufs_hba *hba;
6081
6082         hba = shost_priv(cmd->device->host);
6083
6084         ufshcd_hold(hba, false);
6085         /*
6086          * Check if there is any race with fatal error handling.
6087          * If so, wait for it to complete. Even though fatal error
6088          * handling does reset and restore in some cases, don't assume
6089          * anything out of it. We are just avoiding race here.
6090          */
6091         do {
6092                 spin_lock_irqsave(hba->host->host_lock, flags);
6093                 if (!(work_pending(&hba->eh_work) ||
6094                             hba->ufshcd_state == UFSHCD_STATE_RESET ||
6095                             hba->ufshcd_state == UFSHCD_STATE_EH_SCHEDULED))
6096                         break;
6097                 spin_unlock_irqrestore(hba->host->host_lock, flags);
6098                 dev_dbg(hba->dev, "%s: reset in progress\n", __func__);
6099                 flush_work(&hba->eh_work);
6100         } while (1);
6101
6102         hba->ufshcd_state = UFSHCD_STATE_RESET;
6103         ufshcd_set_eh_in_progress(hba);
6104         spin_unlock_irqrestore(hba->host->host_lock, flags);
6105
6106         err = ufshcd_reset_and_restore(hba);
6107
6108         spin_lock_irqsave(hba->host->host_lock, flags);
6109         if (!err) {
6110                 err = SUCCESS;
6111                 hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6112         } else {
6113                 err = FAILED;
6114                 hba->ufshcd_state = UFSHCD_STATE_ERROR;
6115         }
6116         ufshcd_clear_eh_in_progress(hba);
6117         spin_unlock_irqrestore(hba->host->host_lock, flags);
6118
6119         ufshcd_release(hba);
6120         return err;
6121 }
6122
6123 /**
6124  * ufshcd_get_max_icc_level - calculate the ICC level
6125  * @sup_curr_uA: max. current supported by the regulator
6126  * @start_scan: row at the desc table to start scan from
6127  * @buff: power descriptor buffer
6128  *
6129  * Returns calculated max ICC level for specific regulator
6130  */
6131 static u32 ufshcd_get_max_icc_level(int sup_curr_uA, u32 start_scan, char *buff)
6132 {
6133         int i;
6134         int curr_uA;
6135         u16 data;
6136         u16 unit;
6137
6138         for (i = start_scan; i >= 0; i--) {
6139                 data = be16_to_cpup((__be16 *)&buff[2 * i]);
6140                 unit = (data & ATTR_ICC_LVL_UNIT_MASK) >>
6141                                                 ATTR_ICC_LVL_UNIT_OFFSET;
6142                 curr_uA = data & ATTR_ICC_LVL_VALUE_MASK;
6143                 switch (unit) {
6144                 case UFSHCD_NANO_AMP:
6145                         curr_uA = curr_uA / 1000;
6146                         break;
6147                 case UFSHCD_MILI_AMP:
6148                         curr_uA = curr_uA * 1000;
6149                         break;
6150                 case UFSHCD_AMP:
6151                         curr_uA = curr_uA * 1000 * 1000;
6152                         break;
6153                 case UFSHCD_MICRO_AMP:
6154                 default:
6155                         break;
6156                 }
6157                 if (sup_curr_uA >= curr_uA)
6158                         break;
6159         }
6160         if (i < 0) {
6161                 i = 0;
6162                 pr_err("%s: Couldn't find valid icc_level = %d", __func__, i);
6163         }
6164
6165         return (u32)i;
6166 }
6167
6168 /**
6169  * ufshcd_calc_icc_level - calculate the max ICC level
6170  * In case regulators are not initialized we'll return 0
6171  * @hba: per-adapter instance
6172  * @desc_buf: power descriptor buffer to extract ICC levels from.
6173  * @len: length of desc_buff
6174  *
6175  * Returns calculated ICC level
6176  */
6177 static u32 ufshcd_find_max_sup_active_icc_level(struct ufs_hba *hba,
6178                                                         u8 *desc_buf, int len)
6179 {
6180         u32 icc_level = 0;
6181
6182         if (!hba->vreg_info.vcc || !hba->vreg_info.vccq ||
6183                                                 !hba->vreg_info.vccq2) {
6184                 dev_err(hba->dev,
6185                         "%s: Regulator capability was not set, actvIccLevel=%d",
6186                                                         __func__, icc_level);
6187                 goto out;
6188         }
6189
6190         if (hba->vreg_info.vcc && hba->vreg_info.vcc->max_uA)
6191                 icc_level = ufshcd_get_max_icc_level(
6192                                 hba->vreg_info.vcc->max_uA,
6193                                 POWER_DESC_MAX_ACTV_ICC_LVLS - 1,
6194                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCC_0]);
6195
6196         if (hba->vreg_info.vccq && hba->vreg_info.vccq->max_uA)
6197                 icc_level = ufshcd_get_max_icc_level(
6198                                 hba->vreg_info.vccq->max_uA,
6199                                 icc_level,
6200                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ_0]);
6201
6202         if (hba->vreg_info.vccq2 && hba->vreg_info.vccq2->max_uA)
6203                 icc_level = ufshcd_get_max_icc_level(
6204                                 hba->vreg_info.vccq2->max_uA,
6205                                 icc_level,
6206                                 &desc_buf[PWR_DESC_ACTIVE_LVLS_VCCQ2_0]);
6207 out:
6208         return icc_level;
6209 }
6210
6211 static void ufshcd_init_icc_levels(struct ufs_hba *hba)
6212 {
6213         int ret;
6214         int buff_len = hba->desc_size.pwr_desc;
6215         u8 *desc_buf;
6216
6217         desc_buf = kmalloc(buff_len, GFP_KERNEL);
6218         if (!desc_buf)
6219                 return;
6220
6221         ret = ufshcd_read_power_desc(hba, desc_buf, buff_len);
6222         if (ret) {
6223                 dev_err(hba->dev,
6224                         "%s: Failed reading power descriptor.len = %d ret = %d",
6225                         __func__, buff_len, ret);
6226                 goto out;
6227         }
6228
6229         hba->init_prefetch_data.icc_level =
6230                         ufshcd_find_max_sup_active_icc_level(hba,
6231                         desc_buf, buff_len);
6232         dev_dbg(hba->dev, "%s: setting icc_level 0x%x",
6233                         __func__, hba->init_prefetch_data.icc_level);
6234
6235         ret = ufshcd_query_attr_retry(hba, UPIU_QUERY_OPCODE_WRITE_ATTR,
6236                 QUERY_ATTR_IDN_ACTIVE_ICC_LVL, 0, 0,
6237                 &hba->init_prefetch_data.icc_level);
6238
6239         if (ret)
6240                 dev_err(hba->dev,
6241                         "%s: Failed configuring bActiveICCLevel = %d ret = %d",
6242                         __func__, hba->init_prefetch_data.icc_level , ret);
6243
6244 out:
6245         kfree(desc_buf);
6246 }
6247
6248 /**
6249  * ufshcd_scsi_add_wlus - Adds required W-LUs
6250  * @hba: per-adapter instance
6251  *
6252  * UFS device specification requires the UFS devices to support 4 well known
6253  * logical units:
6254  *      "REPORT_LUNS" (address: 01h)
6255  *      "UFS Device" (address: 50h)
6256  *      "RPMB" (address: 44h)
6257  *      "BOOT" (address: 30h)
6258  * UFS device's power management needs to be controlled by "POWER CONDITION"
6259  * field of SSU (START STOP UNIT) command. But this "power condition" field
6260  * will take effect only when its sent to "UFS device" well known logical unit
6261  * hence we require the scsi_device instance to represent this logical unit in
6262  * order for the UFS host driver to send the SSU command for power management.
6263  *
6264  * We also require the scsi_device instance for "RPMB" (Replay Protected Memory
6265  * Block) LU so user space process can control this LU. User space may also
6266  * want to have access to BOOT LU.
6267  *
6268  * This function adds scsi device instances for each of all well known LUs
6269  * (except "REPORT LUNS" LU).
6270  *
6271  * Returns zero on success (all required W-LUs are added successfully),
6272  * non-zero error value on failure (if failed to add any of the required W-LU).
6273  */
6274 static int ufshcd_scsi_add_wlus(struct ufs_hba *hba)
6275 {
6276         int ret = 0;
6277         struct scsi_device *sdev_rpmb;
6278         struct scsi_device *sdev_boot;
6279
6280         hba->sdev_ufs_device = __scsi_add_device(hba->host, 0, 0,
6281                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN), NULL);
6282         if (IS_ERR(hba->sdev_ufs_device)) {
6283                 ret = PTR_ERR(hba->sdev_ufs_device);
6284                 hba->sdev_ufs_device = NULL;
6285                 goto out;
6286         }
6287         scsi_device_put(hba->sdev_ufs_device);
6288
6289         sdev_rpmb = __scsi_add_device(hba->host, 0, 0,
6290                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_RPMB_WLUN), NULL);
6291         if (IS_ERR(sdev_rpmb)) {
6292                 ret = PTR_ERR(sdev_rpmb);
6293                 goto remove_sdev_ufs_device;
6294         }
6295         scsi_device_put(sdev_rpmb);
6296
6297         sdev_boot = __scsi_add_device(hba->host, 0, 0,
6298                 ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_BOOT_WLUN), NULL);
6299         if (IS_ERR(sdev_boot))
6300                 dev_err(hba->dev, "%s: BOOT WLUN not found\n", __func__);
6301         else
6302                 scsi_device_put(sdev_boot);
6303         goto out;
6304
6305 remove_sdev_ufs_device:
6306         scsi_remove_device(hba->sdev_ufs_device);
6307 out:
6308         return ret;
6309 }
6310
6311 static int ufs_get_device_desc(struct ufs_hba *hba,
6312                                struct ufs_dev_desc *dev_desc)
6313 {
6314         int err;
6315         size_t buff_len;
6316         u8 model_index;
6317         u8 *desc_buf;
6318
6319         buff_len = max_t(size_t, hba->desc_size.dev_desc,
6320                          QUERY_DESC_MAX_SIZE + 1);
6321         desc_buf = kmalloc(buff_len, GFP_KERNEL);
6322         if (!desc_buf) {
6323                 err = -ENOMEM;
6324                 goto out;
6325         }
6326
6327         err = ufshcd_read_device_desc(hba, desc_buf, hba->desc_size.dev_desc);
6328         if (err) {
6329                 dev_err(hba->dev, "%s: Failed reading Device Desc. err = %d\n",
6330                         __func__, err);
6331                 goto out;
6332         }
6333
6334         /*
6335          * getting vendor (manufacturerID) and Bank Index in big endian
6336          * format
6337          */
6338         dev_desc->wmanufacturerid = desc_buf[DEVICE_DESC_PARAM_MANF_ID] << 8 |
6339                                      desc_buf[DEVICE_DESC_PARAM_MANF_ID + 1];
6340
6341         model_index = desc_buf[DEVICE_DESC_PARAM_PRDCT_NAME];
6342
6343         /* Zero-pad entire buffer for string termination. */
6344         memset(desc_buf, 0, buff_len);
6345
6346         err = ufshcd_read_string_desc(hba, model_index, desc_buf,
6347                                       QUERY_DESC_MAX_SIZE, true/*ASCII*/);
6348         if (err) {
6349                 dev_err(hba->dev, "%s: Failed reading Product Name. err = %d\n",
6350                         __func__, err);
6351                 goto out;
6352         }
6353
6354         desc_buf[QUERY_DESC_MAX_SIZE] = '\0';
6355         strlcpy(dev_desc->model, (desc_buf + QUERY_DESC_HDR_SIZE),
6356                 min_t(u8, desc_buf[QUERY_DESC_LENGTH_OFFSET],
6357                       MAX_MODEL_LEN));
6358
6359         /* Null terminate the model string */
6360         dev_desc->model[MAX_MODEL_LEN] = '\0';
6361
6362 out:
6363         kfree(desc_buf);
6364         return err;
6365 }
6366
6367 static void ufs_fixup_device_setup(struct ufs_hba *hba,
6368                                    struct ufs_dev_desc *dev_desc)
6369 {
6370         struct ufs_dev_fix *f;
6371
6372         for (f = ufs_fixups; f->quirk; f++) {
6373                 if ((f->card.wmanufacturerid == dev_desc->wmanufacturerid ||
6374                      f->card.wmanufacturerid == UFS_ANY_VENDOR) &&
6375                     (STR_PRFX_EQUAL(f->card.model, dev_desc->model) ||
6376                      !strcmp(f->card.model, UFS_ANY_MODEL)))
6377                         hba->dev_quirks |= f->quirk;
6378         }
6379 }
6380
6381 /**
6382  * ufshcd_tune_pa_tactivate - Tunes PA_TActivate of local UniPro
6383  * @hba: per-adapter instance
6384  *
6385  * PA_TActivate parameter can be tuned manually if UniPro version is less than
6386  * 1.61. PA_TActivate needs to be greater than or equal to peerM-PHY's
6387  * RX_MIN_ACTIVATETIME_CAPABILITY attribute. This optimal value can help reduce
6388  * the hibern8 exit latency.
6389  *
6390  * Returns zero on success, non-zero error value on failure.
6391  */
6392 static int ufshcd_tune_pa_tactivate(struct ufs_hba *hba)
6393 {
6394         int ret = 0;
6395         u32 peer_rx_min_activatetime = 0, tuned_pa_tactivate;
6396
6397         ret = ufshcd_dme_peer_get(hba,
6398                                   UIC_ARG_MIB_SEL(
6399                                         RX_MIN_ACTIVATETIME_CAPABILITY,
6400                                         UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6401                                   &peer_rx_min_activatetime);
6402         if (ret)
6403                 goto out;
6404
6405         /* make sure proper unit conversion is applied */
6406         tuned_pa_tactivate =
6407                 ((peer_rx_min_activatetime * RX_MIN_ACTIVATETIME_UNIT_US)
6408                  / PA_TACTIVATE_TIME_UNIT_US);
6409         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6410                              tuned_pa_tactivate);
6411
6412 out:
6413         return ret;
6414 }
6415
6416 /**
6417  * ufshcd_tune_pa_hibern8time - Tunes PA_Hibern8Time of local UniPro
6418  * @hba: per-adapter instance
6419  *
6420  * PA_Hibern8Time parameter can be tuned manually if UniPro version is less than
6421  * 1.61. PA_Hibern8Time needs to be maximum of local M-PHY's
6422  * TX_HIBERN8TIME_CAPABILITY & peer M-PHY's RX_HIBERN8TIME_CAPABILITY.
6423  * This optimal value can help reduce the hibern8 exit latency.
6424  *
6425  * Returns zero on success, non-zero error value on failure.
6426  */
6427 static int ufshcd_tune_pa_hibern8time(struct ufs_hba *hba)
6428 {
6429         int ret = 0;
6430         u32 local_tx_hibern8_time_cap = 0, peer_rx_hibern8_time_cap = 0;
6431         u32 max_hibern8_time, tuned_pa_hibern8time;
6432
6433         ret = ufshcd_dme_get(hba,
6434                              UIC_ARG_MIB_SEL(TX_HIBERN8TIME_CAPABILITY,
6435                                         UIC_ARG_MPHY_TX_GEN_SEL_INDEX(0)),
6436                                   &local_tx_hibern8_time_cap);
6437         if (ret)
6438                 goto out;
6439
6440         ret = ufshcd_dme_peer_get(hba,
6441                                   UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAPABILITY,
6442                                         UIC_ARG_MPHY_RX_GEN_SEL_INDEX(0)),
6443                                   &peer_rx_hibern8_time_cap);
6444         if (ret)
6445                 goto out;
6446
6447         max_hibern8_time = max(local_tx_hibern8_time_cap,
6448                                peer_rx_hibern8_time_cap);
6449         /* make sure proper unit conversion is applied */
6450         tuned_pa_hibern8time = ((max_hibern8_time * HIBERN8TIME_UNIT_US)
6451                                 / PA_HIBERN8_TIME_UNIT_US);
6452         ret = ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
6453                              tuned_pa_hibern8time);
6454 out:
6455         return ret;
6456 }
6457
6458 /**
6459  * ufshcd_quirk_tune_host_pa_tactivate - Ensures that host PA_TACTIVATE is
6460  * less than device PA_TACTIVATE time.
6461  * @hba: per-adapter instance
6462  *
6463  * Some UFS devices require host PA_TACTIVATE to be lower than device
6464  * PA_TACTIVATE, we need to enable UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE quirk
6465  * for such devices.
6466  *
6467  * Returns zero on success, non-zero error value on failure.
6468  */
6469 static int ufshcd_quirk_tune_host_pa_tactivate(struct ufs_hba *hba)
6470 {
6471         int ret = 0;
6472         u32 granularity, peer_granularity;
6473         u32 pa_tactivate, peer_pa_tactivate;
6474         u32 pa_tactivate_us, peer_pa_tactivate_us;
6475         u8 gran_to_us_table[] = {1, 4, 8, 16, 32, 100};
6476
6477         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6478                                   &granularity);
6479         if (ret)
6480                 goto out;
6481
6482         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
6483                                   &peer_granularity);
6484         if (ret)
6485                 goto out;
6486
6487         if ((granularity < PA_GRANULARITY_MIN_VAL) ||
6488             (granularity > PA_GRANULARITY_MAX_VAL)) {
6489                 dev_err(hba->dev, "%s: invalid host PA_GRANULARITY %d",
6490                         __func__, granularity);
6491                 return -EINVAL;
6492         }
6493
6494         if ((peer_granularity < PA_GRANULARITY_MIN_VAL) ||
6495             (peer_granularity > PA_GRANULARITY_MAX_VAL)) {
6496                 dev_err(hba->dev, "%s: invalid device PA_GRANULARITY %d",
6497                         __func__, peer_granularity);
6498                 return -EINVAL;
6499         }
6500
6501         ret = ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE), &pa_tactivate);
6502         if (ret)
6503                 goto out;
6504
6505         ret = ufshcd_dme_peer_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
6506                                   &peer_pa_tactivate);
6507         if (ret)
6508                 goto out;
6509
6510         pa_tactivate_us = pa_tactivate * gran_to_us_table[granularity - 1];
6511         peer_pa_tactivate_us = peer_pa_tactivate *
6512                              gran_to_us_table[peer_granularity - 1];
6513
6514         if (pa_tactivate_us > peer_pa_tactivate_us) {
6515                 u32 new_peer_pa_tactivate;
6516
6517                 new_peer_pa_tactivate = pa_tactivate_us /
6518                                       gran_to_us_table[peer_granularity - 1];
6519                 new_peer_pa_tactivate++;
6520                 ret = ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
6521                                           new_peer_pa_tactivate);
6522         }
6523
6524 out:
6525         return ret;
6526 }
6527
6528 static void ufshcd_tune_unipro_params(struct ufs_hba *hba)
6529 {
6530         if (ufshcd_is_unipro_pa_params_tuning_req(hba)) {
6531                 ufshcd_tune_pa_tactivate(hba);
6532                 ufshcd_tune_pa_hibern8time(hba);
6533         }
6534
6535         if (hba->dev_quirks & UFS_DEVICE_QUIRK_PA_TACTIVATE)
6536                 /* set 1ms timeout for PA_TACTIVATE */
6537                 ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE), 10);
6538
6539         if (hba->dev_quirks & UFS_DEVICE_QUIRK_HOST_PA_TACTIVATE)
6540                 ufshcd_quirk_tune_host_pa_tactivate(hba);
6541
6542         ufshcd_vops_apply_dev_quirks(hba);
6543 }
6544
6545 static void ufshcd_clear_dbg_ufs_stats(struct ufs_hba *hba)
6546 {
6547         int err_reg_hist_size = sizeof(struct ufs_uic_err_reg_hist);
6548
6549         hba->ufs_stats.hibern8_exit_cnt = 0;
6550         hba->ufs_stats.last_hibern8_exit_tstamp = ktime_set(0, 0);
6551
6552         memset(&hba->ufs_stats.pa_err, 0, err_reg_hist_size);
6553         memset(&hba->ufs_stats.dl_err, 0, err_reg_hist_size);
6554         memset(&hba->ufs_stats.nl_err, 0, err_reg_hist_size);
6555         memset(&hba->ufs_stats.tl_err, 0, err_reg_hist_size);
6556         memset(&hba->ufs_stats.dme_err, 0, err_reg_hist_size);
6557
6558         hba->req_abort_count = 0;
6559 }
6560
6561 static void ufshcd_init_desc_sizes(struct ufs_hba *hba)
6562 {
6563         int err;
6564
6565         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_DEVICE, 0,
6566                 &hba->desc_size.dev_desc);
6567         if (err)
6568                 hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
6569
6570         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_POWER, 0,
6571                 &hba->desc_size.pwr_desc);
6572         if (err)
6573                 hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
6574
6575         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_INTERCONNECT, 0,
6576                 &hba->desc_size.interc_desc);
6577         if (err)
6578                 hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
6579
6580         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_CONFIGURATION, 0,
6581                 &hba->desc_size.conf_desc);
6582         if (err)
6583                 hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
6584
6585         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_UNIT, 0,
6586                 &hba->desc_size.unit_desc);
6587         if (err)
6588                 hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
6589
6590         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_GEOMETRY, 0,
6591                 &hba->desc_size.geom_desc);
6592         if (err)
6593                 hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
6594         err = ufshcd_read_desc_length(hba, QUERY_DESC_IDN_HEALTH, 0,
6595                 &hba->desc_size.hlth_desc);
6596         if (err)
6597                 hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
6598 }
6599
6600 static void ufshcd_def_desc_sizes(struct ufs_hba *hba)
6601 {
6602         hba->desc_size.dev_desc = QUERY_DESC_DEVICE_DEF_SIZE;
6603         hba->desc_size.pwr_desc = QUERY_DESC_POWER_DEF_SIZE;
6604         hba->desc_size.interc_desc = QUERY_DESC_INTERCONNECT_DEF_SIZE;
6605         hba->desc_size.conf_desc = QUERY_DESC_CONFIGURATION_DEF_SIZE;
6606         hba->desc_size.unit_desc = QUERY_DESC_UNIT_DEF_SIZE;
6607         hba->desc_size.geom_desc = QUERY_DESC_GEOMETRY_DEF_SIZE;
6608         hba->desc_size.hlth_desc = QUERY_DESC_HEALTH_DEF_SIZE;
6609 }
6610
6611 /**
6612  * ufshcd_probe_hba - probe hba to detect device and initialize
6613  * @hba: per-adapter instance
6614  *
6615  * Execute link-startup and verify device initialization
6616  */
6617 static int ufshcd_probe_hba(struct ufs_hba *hba)
6618 {
6619         struct ufs_dev_desc card = {0};
6620         int ret;
6621         ktime_t start = ktime_get();
6622
6623         ret = ufshcd_link_startup(hba);
6624         if (ret)
6625                 goto out;
6626
6627         /* set the default level for urgent bkops */
6628         hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT;
6629         hba->is_urgent_bkops_lvl_checked = false;
6630
6631         /* Debug counters initialization */
6632         ufshcd_clear_dbg_ufs_stats(hba);
6633
6634         /* UniPro link is active now */
6635         ufshcd_set_link_active(hba);
6636
6637         /* Enable Auto-Hibernate if configured */
6638         ufshcd_auto_hibern8_enable(hba);
6639
6640         ret = ufshcd_verify_dev_init(hba);
6641         if (ret)
6642                 goto out;
6643
6644         ret = ufshcd_complete_dev_init(hba);
6645         if (ret)
6646                 goto out;
6647
6648         /* Init check for device descriptor sizes */
6649         ufshcd_init_desc_sizes(hba);
6650
6651         ret = ufs_get_device_desc(hba, &card);
6652         if (ret) {
6653                 dev_err(hba->dev, "%s: Failed getting device info. err = %d\n",
6654                         __func__, ret);
6655                 goto out;
6656         }
6657
6658         ufs_fixup_device_setup(hba, &card);
6659         ufshcd_tune_unipro_params(hba);
6660
6661         ret = ufshcd_set_vccq_rail_unused(hba,
6662                 (hba->dev_quirks & UFS_DEVICE_NO_VCCQ) ? true : false);
6663         if (ret)
6664                 goto out;
6665
6666         /* UFS device is also active now */
6667         ufshcd_set_ufs_dev_active(hba);
6668         ufshcd_force_reset_auto_bkops(hba);
6669         hba->wlun_dev_clr_ua = true;
6670
6671         if (ufshcd_get_max_pwr_mode(hba)) {
6672                 dev_err(hba->dev,
6673                         "%s: Failed getting max supported power mode\n",
6674                         __func__);
6675         } else {
6676                 ret = ufshcd_config_pwr_mode(hba, &hba->max_pwr_info.info);
6677                 if (ret) {
6678                         dev_err(hba->dev, "%s: Failed setting power mode, err = %d\n",
6679                                         __func__, ret);
6680                         goto out;
6681                 }
6682         }
6683
6684         /* set the state as operational after switching to desired gear */
6685         hba->ufshcd_state = UFSHCD_STATE_OPERATIONAL;
6686
6687         /*
6688          * If we are in error handling context or in power management callbacks
6689          * context, no need to scan the host
6690          */
6691         if (!ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
6692                 bool flag;
6693
6694                 /* clear any previous UFS device information */
6695                 memset(&hba->dev_info, 0, sizeof(hba->dev_info));
6696                 if (!ufshcd_query_flag_retry(hba, UPIU_QUERY_OPCODE_READ_FLAG,
6697                                 QUERY_FLAG_IDN_PWR_ON_WPE, &flag))
6698                         hba->dev_info.f_power_on_wp_en = flag;
6699
6700                 if (!hba->is_init_prefetch)
6701                         ufshcd_init_icc_levels(hba);
6702
6703                 /* Add required well known logical units to scsi mid layer */
6704                 ret = ufshcd_scsi_add_wlus(hba);
6705                 if (ret)
6706                         goto out;
6707
6708                 /* Initialize devfreq after UFS device is detected */
6709                 if (ufshcd_is_clkscaling_supported(hba)) {
6710                         memcpy(&hba->clk_scaling.saved_pwr_info.info,
6711                                 &hba->pwr_info,
6712                                 sizeof(struct ufs_pa_layer_attr));
6713                         hba->clk_scaling.saved_pwr_info.is_valid = true;
6714                         if (!hba->devfreq) {
6715                                 ret = ufshcd_devfreq_init(hba);
6716                                 if (ret)
6717                                         goto out;
6718                         }
6719                         hba->clk_scaling.is_allowed = true;
6720                 }
6721
6722                 scsi_scan_host(hba->host);
6723                 pm_runtime_put_sync(hba->dev);
6724         }
6725
6726         if (!hba->is_init_prefetch)
6727                 hba->is_init_prefetch = true;
6728
6729 out:
6730         /*
6731          * If we failed to initialize the device or the device is not
6732          * present, turn off the power/clocks etc.
6733          */
6734         if (ret && !ufshcd_eh_in_progress(hba) && !hba->pm_op_in_progress) {
6735                 pm_runtime_put_sync(hba->dev);
6736                 ufshcd_exit_clk_scaling(hba);
6737                 ufshcd_hba_exit(hba);
6738         }
6739
6740         trace_ufshcd_init(dev_name(hba->dev), ret,
6741                 ktime_to_us(ktime_sub(ktime_get(), start)),
6742                 hba->curr_dev_pwr_mode, hba->uic_link_state);
6743         return ret;
6744 }
6745
6746 /**
6747  * ufshcd_async_scan - asynchronous execution for probing hba
6748  * @data: data pointer to pass to this function
6749  * @cookie: cookie data
6750  */
6751 static void ufshcd_async_scan(void *data, async_cookie_t cookie)
6752 {
6753         struct ufs_hba *hba = (struct ufs_hba *)data;
6754
6755         ufshcd_probe_hba(hba);
6756 }
6757
6758 static enum blk_eh_timer_return ufshcd_eh_timed_out(struct scsi_cmnd *scmd)
6759 {
6760         unsigned long flags;
6761         struct Scsi_Host *host;
6762         struct ufs_hba *hba;
6763         int index;
6764         bool found = false;
6765
6766         if (!scmd || !scmd->device || !scmd->device->host)
6767                 return BLK_EH_DONE;
6768
6769         host = scmd->device->host;
6770         hba = shost_priv(host);
6771         if (!hba)
6772                 return BLK_EH_DONE;
6773
6774         spin_lock_irqsave(host->host_lock, flags);
6775
6776         for_each_set_bit(index, &hba->outstanding_reqs, hba->nutrs) {
6777                 if (hba->lrb[index].cmd == scmd) {
6778                         found = true;
6779                         break;
6780                 }
6781         }
6782
6783         spin_unlock_irqrestore(host->host_lock, flags);
6784
6785         /*
6786          * Bypass SCSI error handling and reset the block layer timer if this
6787          * SCSI command was not actually dispatched to UFS driver, otherwise
6788          * let SCSI layer handle the error as usual.
6789          */
6790         return found ? BLK_EH_DONE : BLK_EH_RESET_TIMER;
6791 }
6792
6793 static const struct attribute_group *ufshcd_driver_groups[] = {
6794         &ufs_sysfs_unit_descriptor_group,
6795         &ufs_sysfs_lun_attributes_group,
6796         NULL,
6797 };
6798
6799 static struct scsi_host_template ufshcd_driver_template = {
6800         .module                 = THIS_MODULE,
6801         .name                   = UFSHCD,
6802         .proc_name              = UFSHCD,
6803         .queuecommand           = ufshcd_queuecommand,
6804         .slave_alloc            = ufshcd_slave_alloc,
6805         .slave_configure        = ufshcd_slave_configure,
6806         .slave_destroy          = ufshcd_slave_destroy,
6807         .change_queue_depth     = ufshcd_change_queue_depth,
6808         .eh_abort_handler       = ufshcd_abort,
6809         .eh_device_reset_handler = ufshcd_eh_device_reset_handler,
6810         .eh_host_reset_handler   = ufshcd_eh_host_reset_handler,
6811         .eh_timed_out           = ufshcd_eh_timed_out,
6812         .this_id                = -1,
6813         .sg_tablesize           = SG_ALL,
6814         .cmd_per_lun            = UFSHCD_CMD_PER_LUN,
6815         .can_queue              = UFSHCD_CAN_QUEUE,
6816         .max_host_blocked       = 1,
6817         .track_queue_depth      = 1,
6818         .sdev_groups            = ufshcd_driver_groups,
6819 };
6820
6821 static int ufshcd_config_vreg_load(struct device *dev, struct ufs_vreg *vreg,
6822                                    int ua)
6823 {
6824         int ret;
6825
6826         if (!vreg)
6827                 return 0;
6828
6829         /*
6830          * "set_load" operation shall be required on those regulators
6831          * which specifically configured current limitation. Otherwise
6832          * zero max_uA may cause unexpected behavior when regulator is
6833          * enabled or set as high power mode.
6834          */
6835         if (!vreg->max_uA)
6836                 return 0;
6837
6838         ret = regulator_set_load(vreg->reg, ua);
6839         if (ret < 0) {
6840                 dev_err(dev, "%s: %s set load (ua=%d) failed, err=%d\n",
6841                                 __func__, vreg->name, ua, ret);
6842         }
6843
6844         return ret;
6845 }
6846
6847 static inline int ufshcd_config_vreg_lpm(struct ufs_hba *hba,
6848                                          struct ufs_vreg *vreg)
6849 {
6850         if (!vreg)
6851                 return 0;
6852         else if (vreg->unused)
6853                 return 0;
6854         else
6855                 return ufshcd_config_vreg_load(hba->dev, vreg,
6856                                                UFS_VREG_LPM_LOAD_UA);
6857 }
6858
6859 static inline int ufshcd_config_vreg_hpm(struct ufs_hba *hba,
6860                                          struct ufs_vreg *vreg)
6861 {
6862         if (!vreg)
6863                 return 0;
6864         else if (vreg->unused)
6865                 return 0;
6866         else
6867                 return ufshcd_config_vreg_load(hba->dev, vreg, vreg->max_uA);
6868 }
6869
6870 static int ufshcd_config_vreg(struct device *dev,
6871                 struct ufs_vreg *vreg, bool on)
6872 {
6873         int ret = 0;
6874         struct regulator *reg;
6875         const char *name;
6876         int min_uV, uA_load;
6877
6878         BUG_ON(!vreg);
6879
6880         reg = vreg->reg;
6881         name = vreg->name;
6882
6883         if (regulator_count_voltages(reg) > 0) {
6884                 if (vreg->min_uV && vreg->max_uV) {
6885                         min_uV = on ? vreg->min_uV : 0;
6886                         ret = regulator_set_voltage(reg, min_uV, vreg->max_uV);
6887                         if (ret) {
6888                                 dev_err(dev,
6889                                         "%s: %s set voltage failed, err=%d\n",
6890                                         __func__, name, ret);
6891                                 goto out;
6892                         }
6893                 }
6894
6895                 uA_load = on ? vreg->max_uA : 0;
6896                 ret = ufshcd_config_vreg_load(dev, vreg, uA_load);
6897                 if (ret)
6898                         goto out;
6899         }
6900 out:
6901         return ret;
6902 }
6903
6904 static int ufshcd_enable_vreg(struct device *dev, struct ufs_vreg *vreg)
6905 {
6906         int ret = 0;
6907
6908         if (!vreg)
6909                 goto out;
6910         else if (vreg->enabled || vreg->unused)
6911                 goto out;
6912
6913         ret = ufshcd_config_vreg(dev, vreg, true);
6914         if (!ret)
6915                 ret = regulator_enable(vreg->reg);
6916
6917         if (!ret)
6918                 vreg->enabled = true;
6919         else
6920                 dev_err(dev, "%s: %s enable failed, err=%d\n",
6921                                 __func__, vreg->name, ret);
6922 out:
6923         return ret;
6924 }
6925
6926 static int ufshcd_disable_vreg(struct device *dev, struct ufs_vreg *vreg)
6927 {
6928         int ret = 0;
6929
6930         if (!vreg)
6931                 goto out;
6932         else if (!vreg->enabled || vreg->unused)
6933                 goto out;
6934
6935         ret = regulator_disable(vreg->reg);
6936
6937         if (!ret) {
6938                 /* ignore errors on applying disable config */
6939                 ufshcd_config_vreg(dev, vreg, false);
6940                 vreg->enabled = false;
6941         } else {
6942                 dev_err(dev, "%s: %s disable failed, err=%d\n",
6943                                 __func__, vreg->name, ret);
6944         }
6945 out:
6946         return ret;
6947 }
6948
6949 static int ufshcd_setup_vreg(struct ufs_hba *hba, bool on)
6950 {
6951         int ret = 0;
6952         struct device *dev = hba->dev;
6953         struct ufs_vreg_info *info = &hba->vreg_info;
6954
6955         if (!info)
6956                 goto out;
6957
6958         ret = ufshcd_toggle_vreg(dev, info->vcc, on);
6959         if (ret)
6960                 goto out;
6961
6962         ret = ufshcd_toggle_vreg(dev, info->vccq, on);
6963         if (ret)
6964                 goto out;
6965
6966         ret = ufshcd_toggle_vreg(dev, info->vccq2, on);
6967         if (ret)
6968                 goto out;
6969
6970 out:
6971         if (ret) {
6972                 ufshcd_toggle_vreg(dev, info->vccq2, false);
6973                 ufshcd_toggle_vreg(dev, info->vccq, false);
6974                 ufshcd_toggle_vreg(dev, info->vcc, false);
6975         }
6976         return ret;
6977 }
6978
6979 static int ufshcd_setup_hba_vreg(struct ufs_hba *hba, bool on)
6980 {
6981         struct ufs_vreg_info *info = &hba->vreg_info;
6982
6983         if (info)
6984                 return ufshcd_toggle_vreg(hba->dev, info->vdd_hba, on);
6985
6986         return 0;
6987 }
6988
6989 static int ufshcd_get_vreg(struct device *dev, struct ufs_vreg *vreg)
6990 {
6991         int ret = 0;
6992
6993         if (!vreg)
6994                 goto out;
6995
6996         vreg->reg = devm_regulator_get(dev, vreg->name);
6997         if (IS_ERR(vreg->reg)) {
6998                 ret = PTR_ERR(vreg->reg);
6999                 dev_err(dev, "%s: %s get failed, err=%d\n",
7000                                 __func__, vreg->name, ret);
7001         }
7002 out:
7003         return ret;
7004 }
7005
7006 static int ufshcd_init_vreg(struct ufs_hba *hba)
7007 {
7008         int ret = 0;
7009         struct device *dev = hba->dev;
7010         struct ufs_vreg_info *info = &hba->vreg_info;
7011
7012         if (!info)
7013                 goto out;
7014
7015         ret = ufshcd_get_vreg(dev, info->vcc);
7016         if (ret)
7017                 goto out;
7018
7019         ret = ufshcd_get_vreg(dev, info->vccq);
7020         if (ret)
7021                 goto out;
7022
7023         ret = ufshcd_get_vreg(dev, info->vccq2);
7024 out:
7025         return ret;
7026 }
7027
7028 static int ufshcd_init_hba_vreg(struct ufs_hba *hba)
7029 {
7030         struct ufs_vreg_info *info = &hba->vreg_info;
7031
7032         if (info)
7033                 return ufshcd_get_vreg(hba->dev, info->vdd_hba);
7034
7035         return 0;
7036 }
7037
7038 static int ufshcd_set_vccq_rail_unused(struct ufs_hba *hba, bool unused)
7039 {
7040         int ret = 0;
7041         struct ufs_vreg_info *info = &hba->vreg_info;
7042
7043         if (!info)
7044                 goto out;
7045         else if (!info->vccq)
7046                 goto out;
7047
7048         if (unused) {
7049                 /* shut off the rail here */
7050                 ret = ufshcd_toggle_vreg(hba->dev, info->vccq, false);
7051                 /*
7052                  * Mark this rail as no longer used, so it doesn't get enabled
7053                  * later by mistake
7054                  */
7055                 if (!ret)
7056                         info->vccq->unused = true;
7057         } else {
7058                 /*
7059                  * rail should have been already enabled hence just make sure
7060                  * that unused flag is cleared.
7061                  */
7062                 info->vccq->unused = false;
7063         }
7064 out:
7065         return ret;
7066 }
7067
7068 static int __ufshcd_setup_clocks(struct ufs_hba *hba, bool on,
7069                                         bool skip_ref_clk)
7070 {
7071         int ret = 0;
7072         struct ufs_clk_info *clki;
7073         struct list_head *head = &hba->clk_list_head;
7074         unsigned long flags;
7075         ktime_t start = ktime_get();
7076         bool clk_state_changed = false;
7077
7078         if (list_empty(head))
7079                 goto out;
7080
7081         /*
7082          * vendor specific setup_clocks ops may depend on clocks managed by
7083          * this standard driver hence call the vendor specific setup_clocks
7084          * before disabling the clocks managed here.
7085          */
7086         if (!on) {
7087                 ret = ufshcd_vops_setup_clocks(hba, on, PRE_CHANGE);
7088                 if (ret)
7089                         return ret;
7090         }
7091
7092         list_for_each_entry(clki, head, list) {
7093                 if (!IS_ERR_OR_NULL(clki->clk)) {
7094                         if (skip_ref_clk && !strcmp(clki->name, "ref_clk"))
7095                                 continue;
7096
7097                         clk_state_changed = on ^ clki->enabled;
7098                         if (on && !clki->enabled) {
7099                                 ret = clk_prepare_enable(clki->clk);
7100                                 if (ret) {
7101                                         dev_err(hba->dev, "%s: %s prepare enable failed, %d\n",
7102                                                 __func__, clki->name, ret);
7103                                         goto out;
7104                                 }
7105                         } else if (!on && clki->enabled) {
7106                                 clk_disable_unprepare(clki->clk);
7107                         }
7108                         clki->enabled = on;
7109                         dev_dbg(hba->dev, "%s: clk: %s %sabled\n", __func__,
7110                                         clki->name, on ? "en" : "dis");
7111                 }
7112         }
7113
7114         /*
7115          * vendor specific setup_clocks ops may depend on clocks managed by
7116          * this standard driver hence call the vendor specific setup_clocks
7117          * after enabling the clocks managed here.
7118          */
7119         if (on) {
7120                 ret = ufshcd_vops_setup_clocks(hba, on, POST_CHANGE);
7121                 if (ret)
7122                         return ret;
7123         }
7124
7125 out:
7126         if (ret) {
7127                 list_for_each_entry(clki, head, list) {
7128                         if (!IS_ERR_OR_NULL(clki->clk) && clki->enabled)
7129                                 clk_disable_unprepare(clki->clk);
7130                 }
7131         } else if (!ret && on) {
7132                 spin_lock_irqsave(hba->host->host_lock, flags);
7133                 hba->clk_gating.state = CLKS_ON;
7134                 trace_ufshcd_clk_gating(dev_name(hba->dev),
7135                                         hba->clk_gating.state);
7136                 spin_unlock_irqrestore(hba->host->host_lock, flags);
7137         }
7138
7139         if (clk_state_changed)
7140                 trace_ufshcd_profile_clk_gating(dev_name(hba->dev),
7141                         (on ? "on" : "off"),
7142                         ktime_to_us(ktime_sub(ktime_get(), start)), ret);
7143         return ret;
7144 }
7145
7146 static int ufshcd_setup_clocks(struct ufs_hba *hba, bool on)
7147 {
7148         return  __ufshcd_setup_clocks(hba, on, false);
7149 }
7150
7151 static int ufshcd_init_clocks(struct ufs_hba *hba)
7152 {
7153         int ret = 0;
7154         struct ufs_clk_info *clki;
7155         struct device *dev = hba->dev;
7156         struct list_head *head = &hba->clk_list_head;
7157
7158         if (list_empty(head))
7159                 goto out;
7160
7161         list_for_each_entry(clki, head, list) {
7162                 if (!clki->name)
7163                         continue;
7164
7165                 clki->clk = devm_clk_get(dev, clki->name);
7166                 if (IS_ERR(clki->clk)) {
7167                         ret = PTR_ERR(clki->clk);
7168                         dev_err(dev, "%s: %s clk get failed, %d\n",
7169                                         __func__, clki->name, ret);
7170                         goto out;
7171                 }
7172
7173                 if (clki->max_freq) {
7174                         ret = clk_set_rate(clki->clk, clki->max_freq);
7175                         if (ret) {
7176                                 dev_err(hba->dev, "%s: %s clk set rate(%dHz) failed, %d\n",
7177                                         __func__, clki->name,
7178                                         clki->max_freq, ret);
7179                                 goto out;
7180                         }
7181                         clki->curr_freq = clki->max_freq;
7182                 }
7183                 dev_dbg(dev, "%s: clk: %s, rate: %lu\n", __func__,
7184                                 clki->name, clk_get_rate(clki->clk));
7185         }
7186 out:
7187         return ret;
7188 }
7189
7190 static int ufshcd_variant_hba_init(struct ufs_hba *hba)
7191 {
7192         int err = 0;
7193
7194         if (!hba->vops)
7195                 goto out;
7196
7197         err = ufshcd_vops_init(hba);
7198         if (err)
7199                 goto out;
7200
7201         err = ufshcd_vops_setup_regulators(hba, true);
7202         if (err)
7203                 goto out_exit;
7204
7205         goto out;
7206
7207 out_exit:
7208         ufshcd_vops_exit(hba);
7209 out:
7210         if (err)
7211                 dev_err(hba->dev, "%s: variant %s init failed err %d\n",
7212                         __func__, ufshcd_get_var_name(hba), err);
7213         return err;
7214 }
7215
7216 static void ufshcd_variant_hba_exit(struct ufs_hba *hba)
7217 {
7218         if (!hba->vops)
7219                 return;
7220
7221         ufshcd_vops_setup_regulators(hba, false);
7222
7223         ufshcd_vops_exit(hba);
7224 }
7225
7226 static int ufshcd_hba_init(struct ufs_hba *hba)
7227 {
7228         int err;
7229
7230         /*
7231          * Handle host controller power separately from the UFS device power
7232          * rails as it will help controlling the UFS host controller power
7233          * collapse easily which is different than UFS device power collapse.
7234          * Also, enable the host controller power before we go ahead with rest
7235          * of the initialization here.
7236          */
7237         err = ufshcd_init_hba_vreg(hba);
7238         if (err)
7239                 goto out;
7240
7241         err = ufshcd_setup_hba_vreg(hba, true);
7242         if (err)
7243                 goto out;
7244
7245         err = ufshcd_init_clocks(hba);
7246         if (err)
7247                 goto out_disable_hba_vreg;
7248
7249         err = ufshcd_setup_clocks(hba, true);
7250         if (err)
7251                 goto out_disable_hba_vreg;
7252
7253         err = ufshcd_init_vreg(hba);
7254         if (err)
7255                 goto out_disable_clks;
7256
7257         err = ufshcd_setup_vreg(hba, true);
7258         if (err)
7259                 goto out_disable_clks;
7260
7261         err = ufshcd_variant_hba_init(hba);
7262         if (err)
7263                 goto out_disable_vreg;
7264
7265         hba->is_powered = true;
7266         goto out;
7267
7268 out_disable_vreg:
7269         ufshcd_setup_vreg(hba, false);
7270 out_disable_clks:
7271         ufshcd_setup_clocks(hba, false);
7272 out_disable_hba_vreg:
7273         ufshcd_setup_hba_vreg(hba, false);
7274 out:
7275         return err;
7276 }
7277
7278 static void ufshcd_hba_exit(struct ufs_hba *hba)
7279 {
7280         if (hba->is_powered) {
7281                 ufshcd_variant_hba_exit(hba);
7282                 ufshcd_setup_vreg(hba, false);
7283                 ufshcd_suspend_clkscaling(hba);
7284                 if (ufshcd_is_clkscaling_supported(hba))
7285                         if (hba->devfreq)
7286                                 ufshcd_suspend_clkscaling(hba);
7287                 ufshcd_setup_clocks(hba, false);
7288                 ufshcd_setup_hba_vreg(hba, false);
7289                 hba->is_powered = false;
7290         }
7291 }
7292
7293 static int
7294 ufshcd_send_request_sense(struct ufs_hba *hba, struct scsi_device *sdp)
7295 {
7296         unsigned char cmd[6] = {REQUEST_SENSE,
7297                                 0,
7298                                 0,
7299                                 0,
7300                                 UFSHCD_REQ_SENSE_SIZE,
7301                                 0};
7302         char *buffer;
7303         int ret;
7304
7305         buffer = kzalloc(UFSHCD_REQ_SENSE_SIZE, GFP_KERNEL);
7306         if (!buffer) {
7307                 ret = -ENOMEM;
7308                 goto out;
7309         }
7310
7311         ret = scsi_execute(sdp, cmd, DMA_FROM_DEVICE, buffer,
7312                         UFSHCD_REQ_SENSE_SIZE, NULL, NULL,
7313                         msecs_to_jiffies(1000), 3, 0, RQF_PM, NULL);
7314         if (ret)
7315                 pr_err("%s: failed with err %d\n", __func__, ret);
7316
7317         kfree(buffer);
7318 out:
7319         return ret;
7320 }
7321
7322 /**
7323  * ufshcd_set_dev_pwr_mode - sends START STOP UNIT command to set device
7324  *                           power mode
7325  * @hba: per adapter instance
7326  * @pwr_mode: device power mode to set
7327  *
7328  * Returns 0 if requested power mode is set successfully
7329  * Returns non-zero if failed to set the requested power mode
7330  */
7331 static int ufshcd_set_dev_pwr_mode(struct ufs_hba *hba,
7332                                      enum ufs_dev_pwr_mode pwr_mode)
7333 {
7334         unsigned char cmd[6] = { START_STOP };
7335         struct scsi_sense_hdr sshdr;
7336         struct scsi_device *sdp;
7337         unsigned long flags;
7338         int ret;
7339
7340         spin_lock_irqsave(hba->host->host_lock, flags);
7341         sdp = hba->sdev_ufs_device;
7342         if (sdp) {
7343                 ret = scsi_device_get(sdp);
7344                 if (!ret && !scsi_device_online(sdp)) {
7345                         ret = -ENODEV;
7346                         scsi_device_put(sdp);
7347                 }
7348         } else {
7349                 ret = -ENODEV;
7350         }
7351         spin_unlock_irqrestore(hba->host->host_lock, flags);
7352
7353         if (ret)
7354                 return ret;
7355
7356         /*
7357          * If scsi commands fail, the scsi mid-layer schedules scsi error-
7358          * handling, which would wait for host to be resumed. Since we know
7359          * we are functional while we are here, skip host resume in error
7360          * handling context.
7361          */
7362         hba->host->eh_noresume = 1;
7363         if (hba->wlun_dev_clr_ua) {
7364                 ret = ufshcd_send_request_sense(hba, sdp);
7365                 if (ret)
7366                         goto out;
7367                 /* Unit attention condition is cleared now */
7368                 hba->wlun_dev_clr_ua = false;
7369         }
7370
7371         cmd[4] = pwr_mode << 4;
7372
7373         /*
7374          * Current function would be generally called from the power management
7375          * callbacks hence set the RQF_PM flag so that it doesn't resume the
7376          * already suspended childs.
7377          */
7378         ret = scsi_execute(sdp, cmd, DMA_NONE, NULL, 0, NULL, &sshdr,
7379                         START_STOP_TIMEOUT, 0, 0, RQF_PM, NULL);
7380         if (ret) {
7381                 sdev_printk(KERN_WARNING, sdp,
7382                             "START_STOP failed for power mode: %d, result %x\n",
7383                             pwr_mode, ret);
7384                 if (driver_byte(ret) == DRIVER_SENSE)
7385                         scsi_print_sense_hdr(sdp, NULL, &sshdr);
7386         }
7387
7388         if (!ret)
7389                 hba->curr_dev_pwr_mode = pwr_mode;
7390 out:
7391         scsi_device_put(sdp);
7392         hba->host->eh_noresume = 0;
7393         return ret;
7394 }
7395
7396 static int ufshcd_link_state_transition(struct ufs_hba *hba,
7397                                         enum uic_link_state req_link_state,
7398                                         int check_for_bkops)
7399 {
7400         int ret = 0;
7401
7402         if (req_link_state == hba->uic_link_state)
7403                 return 0;
7404
7405         if (req_link_state == UIC_LINK_HIBERN8_STATE) {
7406                 ret = ufshcd_uic_hibern8_enter(hba);
7407                 if (!ret)
7408                         ufshcd_set_link_hibern8(hba);
7409                 else
7410                         goto out;
7411         }
7412         /*
7413          * If autobkops is enabled, link can't be turned off because
7414          * turning off the link would also turn off the device.
7415          */
7416         else if ((req_link_state == UIC_LINK_OFF_STATE) &&
7417                    (!check_for_bkops || (check_for_bkops &&
7418                     !hba->auto_bkops_enabled))) {
7419                 /*
7420                  * Let's make sure that link is in low power mode, we are doing
7421                  * this currently by putting the link in Hibern8. Otherway to
7422                  * put the link in low power mode is to send the DME end point
7423                  * to device and then send the DME reset command to local
7424                  * unipro. But putting the link in hibern8 is much faster.
7425                  */
7426                 ret = ufshcd_uic_hibern8_enter(hba);
7427                 if (ret)
7428                         goto out;
7429                 /*
7430                  * Change controller state to "reset state" which
7431                  * should also put the link in off/reset state
7432                  */
7433                 ufshcd_hba_stop(hba, true);
7434                 /*
7435                  * TODO: Check if we need any delay to make sure that
7436                  * controller is reset
7437                  */
7438                 ufshcd_set_link_off(hba);
7439         }
7440
7441 out:
7442         return ret;
7443 }
7444
7445 static void ufshcd_vreg_set_lpm(struct ufs_hba *hba)
7446 {
7447         /*
7448          * It seems some UFS devices may keep drawing more than sleep current
7449          * (atleast for 500us) from UFS rails (especially from VCCQ rail).
7450          * To avoid this situation, add 2ms delay before putting these UFS
7451          * rails in LPM mode.
7452          */
7453         if (!ufshcd_is_link_active(hba) &&
7454             hba->dev_quirks & UFS_DEVICE_QUIRK_DELAY_BEFORE_LPM)
7455                 usleep_range(2000, 2100);
7456
7457         /*
7458          * If UFS device is either in UFS_Sleep turn off VCC rail to save some
7459          * power.
7460          *
7461          * If UFS device and link is in OFF state, all power supplies (VCC,
7462          * VCCQ, VCCQ2) can be turned off if power on write protect is not
7463          * required. If UFS link is inactive (Hibern8 or OFF state) and device
7464          * is in sleep state, put VCCQ & VCCQ2 rails in LPM mode.
7465          *
7466          * Ignore the error returned by ufshcd_toggle_vreg() as device is anyway
7467          * in low power state which would save some power.
7468          */
7469         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7470             !hba->dev_info.is_lu_power_on_wp) {
7471                 ufshcd_setup_vreg(hba, false);
7472         } else if (!ufshcd_is_ufs_dev_active(hba)) {
7473                 ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7474                 if (!ufshcd_is_link_active(hba)) {
7475                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7476                         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq2);
7477                 }
7478         }
7479 }
7480
7481 static int ufshcd_vreg_set_hpm(struct ufs_hba *hba)
7482 {
7483         int ret = 0;
7484
7485         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba) &&
7486             !hba->dev_info.is_lu_power_on_wp) {
7487                 ret = ufshcd_setup_vreg(hba, true);
7488         } else if (!ufshcd_is_ufs_dev_active(hba)) {
7489                 if (!ret && !ufshcd_is_link_active(hba)) {
7490                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq);
7491                         if (ret)
7492                                 goto vcc_disable;
7493                         ret = ufshcd_config_vreg_hpm(hba, hba->vreg_info.vccq2);
7494                         if (ret)
7495                                 goto vccq_lpm;
7496                 }
7497                 ret = ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, true);
7498         }
7499         goto out;
7500
7501 vccq_lpm:
7502         ufshcd_config_vreg_lpm(hba, hba->vreg_info.vccq);
7503 vcc_disable:
7504         ufshcd_toggle_vreg(hba->dev, hba->vreg_info.vcc, false);
7505 out:
7506         return ret;
7507 }
7508
7509 static void ufshcd_hba_vreg_set_lpm(struct ufs_hba *hba)
7510 {
7511         if (ufshcd_is_link_off(hba))
7512                 ufshcd_setup_hba_vreg(hba, false);
7513 }
7514
7515 static void ufshcd_hba_vreg_set_hpm(struct ufs_hba *hba)
7516 {
7517         if (ufshcd_is_link_off(hba))
7518                 ufshcd_setup_hba_vreg(hba, true);
7519 }
7520
7521 /**
7522  * ufshcd_suspend - helper function for suspend operations
7523  * @hba: per adapter instance
7524  * @pm_op: desired low power operation type
7525  *
7526  * This function will try to put the UFS device and link into low power
7527  * mode based on the "rpm_lvl" (Runtime PM level) or "spm_lvl"
7528  * (System PM level).
7529  *
7530  * If this function is called during shutdown, it will make sure that
7531  * both UFS device and UFS link is powered off.
7532  *
7533  * NOTE: UFS device & link must be active before we enter in this function.
7534  *
7535  * Returns 0 for success and non-zero for failure
7536  */
7537 static int ufshcd_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7538 {
7539         int ret = 0;
7540         enum ufs_pm_level pm_lvl;
7541         enum ufs_dev_pwr_mode req_dev_pwr_mode;
7542         enum uic_link_state req_link_state;
7543
7544         hba->pm_op_in_progress = 1;
7545         if (!ufshcd_is_shutdown_pm(pm_op)) {
7546                 pm_lvl = ufshcd_is_runtime_pm(pm_op) ?
7547                          hba->rpm_lvl : hba->spm_lvl;
7548                 req_dev_pwr_mode = ufs_get_pm_lvl_to_dev_pwr_mode(pm_lvl);
7549                 req_link_state = ufs_get_pm_lvl_to_link_pwr_state(pm_lvl);
7550         } else {
7551                 req_dev_pwr_mode = UFS_POWERDOWN_PWR_MODE;
7552                 req_link_state = UIC_LINK_OFF_STATE;
7553         }
7554
7555         /*
7556          * If we can't transition into any of the low power modes
7557          * just gate the clocks.
7558          */
7559         ufshcd_hold(hba, false);
7560         hba->clk_gating.is_suspended = true;
7561
7562         if (hba->clk_scaling.is_allowed) {
7563                 cancel_work_sync(&hba->clk_scaling.suspend_work);
7564                 cancel_work_sync(&hba->clk_scaling.resume_work);
7565                 ufshcd_suspend_clkscaling(hba);
7566         }
7567
7568         if (req_dev_pwr_mode == UFS_ACTIVE_PWR_MODE &&
7569                         req_link_state == UIC_LINK_ACTIVE_STATE) {
7570                 goto disable_clks;
7571         }
7572
7573         if ((req_dev_pwr_mode == hba->curr_dev_pwr_mode) &&
7574             (req_link_state == hba->uic_link_state))
7575                 goto enable_gating;
7576
7577         /* UFS device & link must be active before we enter in this function */
7578         if (!ufshcd_is_ufs_dev_active(hba) || !ufshcd_is_link_active(hba)) {
7579                 ret = -EINVAL;
7580                 goto enable_gating;
7581         }
7582
7583         if (ufshcd_is_runtime_pm(pm_op)) {
7584                 if (ufshcd_can_autobkops_during_suspend(hba)) {
7585                         /*
7586                          * The device is idle with no requests in the queue,
7587                          * allow background operations if bkops status shows
7588                          * that performance might be impacted.
7589                          */
7590                         ret = ufshcd_urgent_bkops(hba);
7591                         if (ret)
7592                                 goto enable_gating;
7593                 } else {
7594                         /* make sure that auto bkops is disabled */
7595                         ufshcd_disable_auto_bkops(hba);
7596                 }
7597         }
7598
7599         if ((req_dev_pwr_mode != hba->curr_dev_pwr_mode) &&
7600              ((ufshcd_is_runtime_pm(pm_op) && !hba->auto_bkops_enabled) ||
7601                !ufshcd_is_runtime_pm(pm_op))) {
7602                 /* ensure that bkops is disabled */
7603                 ufshcd_disable_auto_bkops(hba);
7604                 ret = ufshcd_set_dev_pwr_mode(hba, req_dev_pwr_mode);
7605                 if (ret)
7606                         goto enable_gating;
7607         }
7608
7609         ret = ufshcd_link_state_transition(hba, req_link_state, 1);
7610         if (ret)
7611                 goto set_dev_active;
7612
7613         ufshcd_vreg_set_lpm(hba);
7614
7615 disable_clks:
7616         /*
7617          * Call vendor specific suspend callback. As these callbacks may access
7618          * vendor specific host controller register space call them before the
7619          * host clocks are ON.
7620          */
7621         ret = ufshcd_vops_suspend(hba, pm_op);
7622         if (ret)
7623                 goto set_link_active;
7624
7625         if (!ufshcd_is_link_active(hba))
7626                 ufshcd_setup_clocks(hba, false);
7627         else
7628                 /* If link is active, device ref_clk can't be switched off */
7629                 __ufshcd_setup_clocks(hba, false, true);
7630
7631         hba->clk_gating.state = CLKS_OFF;
7632         trace_ufshcd_clk_gating(dev_name(hba->dev), hba->clk_gating.state);
7633         /*
7634          * Disable the host irq as host controller as there won't be any
7635          * host controller transaction expected till resume.
7636          */
7637         ufshcd_disable_irq(hba);
7638         /* Put the host controller in low power mode if possible */
7639         ufshcd_hba_vreg_set_lpm(hba);
7640         goto out;
7641
7642 set_link_active:
7643         if (hba->clk_scaling.is_allowed)
7644                 ufshcd_resume_clkscaling(hba);
7645         ufshcd_vreg_set_hpm(hba);
7646         if (ufshcd_is_link_hibern8(hba) && !ufshcd_uic_hibern8_exit(hba))
7647                 ufshcd_set_link_active(hba);
7648         else if (ufshcd_is_link_off(hba))
7649                 ufshcd_host_reset_and_restore(hba);
7650 set_dev_active:
7651         if (!ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE))
7652                 ufshcd_disable_auto_bkops(hba);
7653 enable_gating:
7654         if (hba->clk_scaling.is_allowed)
7655                 ufshcd_resume_clkscaling(hba);
7656         hba->clk_gating.is_suspended = false;
7657         ufshcd_release(hba);
7658 out:
7659         hba->pm_op_in_progress = 0;
7660         return ret;
7661 }
7662
7663 /**
7664  * ufshcd_resume - helper function for resume operations
7665  * @hba: per adapter instance
7666  * @pm_op: runtime PM or system PM
7667  *
7668  * This function basically brings the UFS device, UniPro link and controller
7669  * to active state.
7670  *
7671  * Returns 0 for success and non-zero for failure
7672  */
7673 static int ufshcd_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
7674 {
7675         int ret;
7676         enum uic_link_state old_link_state;
7677
7678         hba->pm_op_in_progress = 1;
7679         old_link_state = hba->uic_link_state;
7680
7681         ufshcd_hba_vreg_set_hpm(hba);
7682         /* Make sure clocks are enabled before accessing controller */
7683         ret = ufshcd_setup_clocks(hba, true);
7684         if (ret)
7685                 goto out;
7686
7687         /* enable the host irq as host controller would be active soon */
7688         ret = ufshcd_enable_irq(hba);
7689         if (ret)
7690                 goto disable_irq_and_vops_clks;
7691
7692         ret = ufshcd_vreg_set_hpm(hba);
7693         if (ret)
7694                 goto disable_irq_and_vops_clks;
7695
7696         /*
7697          * Call vendor specific resume callback. As these callbacks may access
7698          * vendor specific host controller register space call them when the
7699          * host clocks are ON.
7700          */
7701         ret = ufshcd_vops_resume(hba, pm_op);
7702         if (ret)
7703                 goto disable_vreg;
7704
7705         if (ufshcd_is_link_hibern8(hba)) {
7706                 ret = ufshcd_uic_hibern8_exit(hba);
7707                 if (!ret)
7708                         ufshcd_set_link_active(hba);
7709                 else
7710                         goto vendor_suspend;
7711         } else if (ufshcd_is_link_off(hba)) {
7712                 ret = ufshcd_host_reset_and_restore(hba);
7713                 /*
7714                  * ufshcd_host_reset_and_restore() should have already
7715                  * set the link state as active
7716                  */
7717                 if (ret || !ufshcd_is_link_active(hba))
7718                         goto vendor_suspend;
7719         }
7720
7721         if (!ufshcd_is_ufs_dev_active(hba)) {
7722                 ret = ufshcd_set_dev_pwr_mode(hba, UFS_ACTIVE_PWR_MODE);
7723                 if (ret)
7724                         goto set_old_link_state;
7725         }
7726
7727         if (ufshcd_keep_autobkops_enabled_except_suspend(hba))
7728                 ufshcd_enable_auto_bkops(hba);
7729         else
7730                 /*
7731                  * If BKOPs operations are urgently needed at this moment then
7732                  * keep auto-bkops enabled or else disable it.
7733                  */
7734                 ufshcd_urgent_bkops(hba);
7735
7736         hba->clk_gating.is_suspended = false;
7737
7738         if (hba->clk_scaling.is_allowed)
7739                 ufshcd_resume_clkscaling(hba);
7740
7741         /* Schedule clock gating in case of no access to UFS device yet */
7742         ufshcd_release(hba);
7743
7744         /* Enable Auto-Hibernate if configured */
7745         ufshcd_auto_hibern8_enable(hba);
7746
7747         goto out;
7748
7749 set_old_link_state:
7750         ufshcd_link_state_transition(hba, old_link_state, 0);
7751 vendor_suspend:
7752         ufshcd_vops_suspend(hba, pm_op);
7753 disable_vreg:
7754         ufshcd_vreg_set_lpm(hba);
7755 disable_irq_and_vops_clks:
7756         ufshcd_disable_irq(hba);
7757         if (hba->clk_scaling.is_allowed)
7758                 ufshcd_suspend_clkscaling(hba);
7759         ufshcd_setup_clocks(hba, false);
7760 out:
7761         hba->pm_op_in_progress = 0;
7762         return ret;
7763 }
7764
7765 /**
7766  * ufshcd_system_suspend - system suspend routine
7767  * @hba: per adapter instance
7768  *
7769  * Check the description of ufshcd_suspend() function for more details.
7770  *
7771  * Returns 0 for success and non-zero for failure
7772  */
7773 int ufshcd_system_suspend(struct ufs_hba *hba)
7774 {
7775         int ret = 0;
7776         ktime_t start = ktime_get();
7777
7778         if (!hba || !hba->is_powered)
7779                 return 0;
7780
7781         if ((ufs_get_pm_lvl_to_dev_pwr_mode(hba->spm_lvl) ==
7782              hba->curr_dev_pwr_mode) &&
7783             (ufs_get_pm_lvl_to_link_pwr_state(hba->spm_lvl) ==
7784              hba->uic_link_state))
7785                 goto out;
7786
7787         if (pm_runtime_suspended(hba->dev)) {
7788                 /*
7789                  * UFS device and/or UFS link low power states during runtime
7790                  * suspend seems to be different than what is expected during
7791                  * system suspend. Hence runtime resume the devic & link and
7792                  * let the system suspend low power states to take effect.
7793                  * TODO: If resume takes longer time, we might have optimize
7794                  * it in future by not resuming everything if possible.
7795                  */
7796                 ret = ufshcd_runtime_resume(hba);
7797                 if (ret)
7798                         goto out;
7799         }
7800
7801         ret = ufshcd_suspend(hba, UFS_SYSTEM_PM);
7802 out:
7803         trace_ufshcd_system_suspend(dev_name(hba->dev), ret,
7804                 ktime_to_us(ktime_sub(ktime_get(), start)),
7805                 hba->curr_dev_pwr_mode, hba->uic_link_state);
7806         if (!ret)
7807                 hba->is_sys_suspended = true;
7808         return ret;
7809 }
7810 EXPORT_SYMBOL(ufshcd_system_suspend);
7811
7812 /**
7813  * ufshcd_system_resume - system resume routine
7814  * @hba: per adapter instance
7815  *
7816  * Returns 0 for success and non-zero for failure
7817  */
7818
7819 int ufshcd_system_resume(struct ufs_hba *hba)
7820 {
7821         int ret = 0;
7822         ktime_t start = ktime_get();
7823
7824         if (!hba)
7825                 return -EINVAL;
7826
7827         if (!hba->is_powered || pm_runtime_suspended(hba->dev))
7828                 /*
7829                  * Let the runtime resume take care of resuming
7830                  * if runtime suspended.
7831                  */
7832                 goto out;
7833         else
7834                 ret = ufshcd_resume(hba, UFS_SYSTEM_PM);
7835 out:
7836         trace_ufshcd_system_resume(dev_name(hba->dev), ret,
7837                 ktime_to_us(ktime_sub(ktime_get(), start)),
7838                 hba->curr_dev_pwr_mode, hba->uic_link_state);
7839         if (!ret)
7840                 hba->is_sys_suspended = false;
7841         return ret;
7842 }
7843 EXPORT_SYMBOL(ufshcd_system_resume);
7844
7845 /**
7846  * ufshcd_runtime_suspend - runtime suspend routine
7847  * @hba: per adapter instance
7848  *
7849  * Check the description of ufshcd_suspend() function for more details.
7850  *
7851  * Returns 0 for success and non-zero for failure
7852  */
7853 int ufshcd_runtime_suspend(struct ufs_hba *hba)
7854 {
7855         int ret = 0;
7856         ktime_t start = ktime_get();
7857
7858         if (!hba)
7859                 return -EINVAL;
7860
7861         if (!hba->is_powered)
7862                 goto out;
7863         else
7864                 ret = ufshcd_suspend(hba, UFS_RUNTIME_PM);
7865 out:
7866         trace_ufshcd_runtime_suspend(dev_name(hba->dev), ret,
7867                 ktime_to_us(ktime_sub(ktime_get(), start)),
7868                 hba->curr_dev_pwr_mode, hba->uic_link_state);
7869         return ret;
7870 }
7871 EXPORT_SYMBOL(ufshcd_runtime_suspend);
7872
7873 /**
7874  * ufshcd_runtime_resume - runtime resume routine
7875  * @hba: per adapter instance
7876  *
7877  * This function basically brings the UFS device, UniPro link and controller
7878  * to active state. Following operations are done in this function:
7879  *
7880  * 1. Turn on all the controller related clocks
7881  * 2. Bring the UniPro link out of Hibernate state
7882  * 3. If UFS device is in sleep state, turn ON VCC rail and bring the UFS device
7883  *    to active state.
7884  * 4. If auto-bkops is enabled on the device, disable it.
7885  *
7886  * So following would be the possible power state after this function return
7887  * successfully:
7888  *      S1: UFS device in Active state with VCC rail ON
7889  *          UniPro link in Active state
7890  *          All the UFS/UniPro controller clocks are ON
7891  *
7892  * Returns 0 for success and non-zero for failure
7893  */
7894 int ufshcd_runtime_resume(struct ufs_hba *hba)
7895 {
7896         int ret = 0;
7897         ktime_t start = ktime_get();
7898
7899         if (!hba)
7900                 return -EINVAL;
7901
7902         if (!hba->is_powered)
7903                 goto out;
7904         else
7905                 ret = ufshcd_resume(hba, UFS_RUNTIME_PM);
7906 out:
7907         trace_ufshcd_runtime_resume(dev_name(hba->dev), ret,
7908                 ktime_to_us(ktime_sub(ktime_get(), start)),
7909                 hba->curr_dev_pwr_mode, hba->uic_link_state);
7910         return ret;
7911 }
7912 EXPORT_SYMBOL(ufshcd_runtime_resume);
7913
7914 int ufshcd_runtime_idle(struct ufs_hba *hba)
7915 {
7916         return 0;
7917 }
7918 EXPORT_SYMBOL(ufshcd_runtime_idle);
7919
7920 /**
7921  * ufshcd_shutdown - shutdown routine
7922  * @hba: per adapter instance
7923  *
7924  * This function would power off both UFS device and UFS link.
7925  *
7926  * Returns 0 always to allow force shutdown even in case of errors.
7927  */
7928 int ufshcd_shutdown(struct ufs_hba *hba)
7929 {
7930         int ret = 0;
7931
7932         if (!hba->is_powered)
7933                 goto out;
7934
7935         if (ufshcd_is_ufs_dev_poweroff(hba) && ufshcd_is_link_off(hba))
7936                 goto out;
7937
7938         pm_runtime_get_sync(hba->dev);
7939
7940         ret = ufshcd_suspend(hba, UFS_SHUTDOWN_PM);
7941 out:
7942         if (ret)
7943                 dev_err(hba->dev, "%s failed, err %d\n", __func__, ret);
7944         /* allow force shutdown even in case of errors */
7945         return 0;
7946 }
7947 EXPORT_SYMBOL(ufshcd_shutdown);
7948
7949 /**
7950  * ufshcd_remove - de-allocate SCSI host and host memory space
7951  *              data structure memory
7952  * @hba: per adapter instance
7953  */
7954 void ufshcd_remove(struct ufs_hba *hba)
7955 {
7956         ufs_sysfs_remove_nodes(hba->dev);
7957         scsi_remove_host(hba->host);
7958         /* disable interrupts */
7959         ufshcd_disable_intr(hba, hba->intr_mask);
7960         ufshcd_hba_stop(hba, true);
7961
7962         ufshcd_exit_clk_scaling(hba);
7963         ufshcd_exit_clk_gating(hba);
7964         if (ufshcd_is_clkscaling_supported(hba))
7965                 device_remove_file(hba->dev, &hba->clk_scaling.enable_attr);
7966         ufshcd_hba_exit(hba);
7967 }
7968 EXPORT_SYMBOL_GPL(ufshcd_remove);
7969
7970 /**
7971  * ufshcd_dealloc_host - deallocate Host Bus Adapter (HBA)
7972  * @hba: pointer to Host Bus Adapter (HBA)
7973  */
7974 void ufshcd_dealloc_host(struct ufs_hba *hba)
7975 {
7976         scsi_host_put(hba->host);
7977 }
7978 EXPORT_SYMBOL_GPL(ufshcd_dealloc_host);
7979
7980 /**
7981  * ufshcd_set_dma_mask - Set dma mask based on the controller
7982  *                       addressing capability
7983  * @hba: per adapter instance
7984  *
7985  * Returns 0 for success, non-zero for failure
7986  */
7987 static int ufshcd_set_dma_mask(struct ufs_hba *hba)
7988 {
7989         if (hba->capabilities & MASK_64_ADDRESSING_SUPPORT) {
7990                 if (!dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(64)))
7991                         return 0;
7992         }
7993         return dma_set_mask_and_coherent(hba->dev, DMA_BIT_MASK(32));
7994 }
7995
7996 /**
7997  * ufshcd_alloc_host - allocate Host Bus Adapter (HBA)
7998  * @dev: pointer to device handle
7999  * @hba_handle: driver private handle
8000  * Returns 0 on success, non-zero value on failure
8001  */
8002 int ufshcd_alloc_host(struct device *dev, struct ufs_hba **hba_handle)
8003 {
8004         struct Scsi_Host *host;
8005         struct ufs_hba *hba;
8006         int err = 0;
8007
8008         if (!dev) {
8009                 dev_err(dev,
8010                 "Invalid memory reference for dev is NULL\n");
8011                 err = -ENODEV;
8012                 goto out_error;
8013         }
8014
8015         host = scsi_host_alloc(&ufshcd_driver_template,
8016                                 sizeof(struct ufs_hba));
8017         if (!host) {
8018                 dev_err(dev, "scsi_host_alloc failed\n");
8019                 err = -ENOMEM;
8020                 goto out_error;
8021         }
8022
8023         /*
8024          * Do not use blk-mq at this time because blk-mq does not support
8025          * runtime pm.
8026          */
8027         host->use_blk_mq = false;
8028
8029         hba = shost_priv(host);
8030         hba->host = host;
8031         hba->dev = dev;
8032         *hba_handle = hba;
8033
8034         INIT_LIST_HEAD(&hba->clk_list_head);
8035
8036 out_error:
8037         return err;
8038 }
8039 EXPORT_SYMBOL(ufshcd_alloc_host);
8040
8041 /**
8042  * ufshcd_init - Driver initialization routine
8043  * @hba: per-adapter instance
8044  * @mmio_base: base register address
8045  * @irq: Interrupt line of device
8046  * Returns 0 on success, non-zero value on failure
8047  */
8048 int ufshcd_init(struct ufs_hba *hba, void __iomem *mmio_base, unsigned int irq)
8049 {
8050         int err;
8051         struct Scsi_Host *host = hba->host;
8052         struct device *dev = hba->dev;
8053
8054         /*
8055          * dev_set_drvdata() must be called before any callbacks are registered
8056          * that use dev_get_drvdata() (frequency scaling, clock scaling, hwmon,
8057          * sysfs).
8058          */
8059         dev_set_drvdata(dev, hba);
8060
8061         if (!mmio_base) {
8062                 dev_err(hba->dev,
8063                 "Invalid memory reference for mmio_base is NULL\n");
8064                 err = -ENODEV;
8065                 goto out_error;
8066         }
8067
8068         hba->mmio_base = mmio_base;
8069         hba->irq = irq;
8070
8071         /* Set descriptor lengths to specification defaults */
8072         ufshcd_def_desc_sizes(hba);
8073
8074         err = ufshcd_hba_init(hba);
8075         if (err)
8076                 goto out_error;
8077
8078         /* Read capabilities registers */
8079         ufshcd_hba_capabilities(hba);
8080
8081         /* Get UFS version supported by the controller */
8082         hba->ufs_version = ufshcd_get_ufs_version(hba);
8083
8084         if ((hba->ufs_version != UFSHCI_VERSION_10) &&
8085             (hba->ufs_version != UFSHCI_VERSION_11) &&
8086             (hba->ufs_version != UFSHCI_VERSION_20) &&
8087             (hba->ufs_version != UFSHCI_VERSION_21))
8088                 dev_err(hba->dev, "invalid UFS version 0x%x\n",
8089                         hba->ufs_version);
8090
8091         /* Get Interrupt bit mask per version */
8092         hba->intr_mask = ufshcd_get_intr_mask(hba);
8093
8094         err = ufshcd_set_dma_mask(hba);
8095         if (err) {
8096                 dev_err(hba->dev, "set dma mask failed\n");
8097                 goto out_disable;
8098         }
8099
8100         /* Allocate memory for host memory space */
8101         err = ufshcd_memory_alloc(hba);
8102         if (err) {
8103                 dev_err(hba->dev, "Memory allocation failed\n");
8104                 goto out_disable;
8105         }
8106
8107         /* Configure LRB */
8108         ufshcd_host_memory_configure(hba);
8109
8110         host->can_queue = hba->nutrs;
8111         host->cmd_per_lun = hba->nutrs;
8112         host->max_id = UFSHCD_MAX_ID;
8113         host->max_lun = UFS_MAX_LUNS;
8114         host->max_channel = UFSHCD_MAX_CHANNEL;
8115         host->unique_id = host->host_no;
8116         host->max_cmd_len = MAX_CDB_SIZE;
8117
8118         hba->max_pwr_info.is_valid = false;
8119
8120         /* Initailize wait queue for task management */
8121         init_waitqueue_head(&hba->tm_wq);
8122         init_waitqueue_head(&hba->tm_tag_wq);
8123
8124         /* Initialize work queues */
8125         INIT_WORK(&hba->eh_work, ufshcd_err_handler);
8126         INIT_WORK(&hba->eeh_work, ufshcd_exception_event_handler);
8127
8128         /* Initialize UIC command mutex */
8129         mutex_init(&hba->uic_cmd_mutex);
8130
8131         /* Initialize mutex for device management commands */
8132         mutex_init(&hba->dev_cmd.lock);
8133
8134         init_rwsem(&hba->clk_scaling_lock);
8135
8136         /* Initialize device management tag acquire wait queue */
8137         init_waitqueue_head(&hba->dev_cmd.tag_wq);
8138
8139         ufshcd_init_clk_gating(hba);
8140
8141         ufshcd_init_clk_scaling(hba);
8142
8143         /*
8144          * In order to avoid any spurious interrupt immediately after
8145          * registering UFS controller interrupt handler, clear any pending UFS
8146          * interrupt status and disable all the UFS interrupts.
8147          */
8148         ufshcd_writel(hba, ufshcd_readl(hba, REG_INTERRUPT_STATUS),
8149                       REG_INTERRUPT_STATUS);
8150         ufshcd_writel(hba, 0, REG_INTERRUPT_ENABLE);
8151         /*
8152          * Make sure that UFS interrupts are disabled and any pending interrupt
8153          * status is cleared before registering UFS interrupt handler.
8154          */
8155         mb();
8156
8157         /* IRQ registration */
8158         err = devm_request_irq(dev, irq, ufshcd_intr, IRQF_SHARED, UFSHCD, hba);
8159         if (err) {
8160                 dev_err(hba->dev, "request irq failed\n");
8161                 goto exit_gating;
8162         } else {
8163                 hba->is_irq_enabled = true;
8164         }
8165
8166         err = scsi_add_host(host, hba->dev);
8167         if (err) {
8168                 dev_err(hba->dev, "scsi_add_host failed\n");
8169                 goto exit_gating;
8170         }
8171
8172         /* Host controller enable */
8173         err = ufshcd_hba_enable(hba);
8174         if (err) {
8175                 dev_err(hba->dev, "Host controller enable failed\n");
8176                 ufshcd_print_host_regs(hba);
8177                 ufshcd_print_host_state(hba);
8178                 goto out_remove_scsi_host;
8179         }
8180
8181         /*
8182          * Set the default power management level for runtime and system PM.
8183          * Default power saving mode is to keep UFS link in Hibern8 state
8184          * and UFS device in sleep state.
8185          */
8186         hba->rpm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8187                                                 UFS_SLEEP_PWR_MODE,
8188                                                 UIC_LINK_HIBERN8_STATE);
8189         hba->spm_lvl = ufs_get_desired_pm_lvl_for_dev_link_state(
8190                                                 UFS_SLEEP_PWR_MODE,
8191                                                 UIC_LINK_HIBERN8_STATE);
8192
8193         /* Set the default auto-hiberate idle timer value to 150 ms */
8194         if (hba->capabilities & MASK_AUTO_HIBERN8_SUPPORT) {
8195                 hba->ahit = FIELD_PREP(UFSHCI_AHIBERN8_TIMER_MASK, 150) |
8196                             FIELD_PREP(UFSHCI_AHIBERN8_SCALE_MASK, 3);
8197         }
8198
8199         /* Hold auto suspend until async scan completes */
8200         pm_runtime_get_sync(dev);
8201         atomic_set(&hba->scsi_block_reqs_cnt, 0);
8202         /*
8203          * We are assuming that device wasn't put in sleep/power-down
8204          * state exclusively during the boot stage before kernel.
8205          * This assumption helps avoid doing link startup twice during
8206          * ufshcd_probe_hba().
8207          */
8208         ufshcd_set_ufs_dev_active(hba);
8209
8210         async_schedule(ufshcd_async_scan, hba);
8211         ufs_sysfs_add_nodes(hba->dev);
8212
8213         return 0;
8214
8215 out_remove_scsi_host:
8216         scsi_remove_host(hba->host);
8217 exit_gating:
8218         ufshcd_exit_clk_scaling(hba);
8219         ufshcd_exit_clk_gating(hba);
8220 out_disable:
8221         hba->is_irq_enabled = false;
8222         ufshcd_hba_exit(hba);
8223 out_error:
8224         return err;
8225 }
8226 EXPORT_SYMBOL_GPL(ufshcd_init);
8227
8228 MODULE_AUTHOR("Santosh Yaragnavi <santosh.sy@samsung.com>");
8229 MODULE_AUTHOR("Vinayak Holikatti <h.vinayak@samsung.com>");
8230 MODULE_DESCRIPTION("Generic UFS host controller driver Core");
8231 MODULE_LICENSE("GPL");
8232 MODULE_VERSION(UFSHCD_DRIVER_VERSION);