|
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index] [PATCH v3 4/4] scsi: use percpu counters for iostat counters in struct scsi_device
iorequest_cnt and iodone_cnt are updated on every command dispatch and
completion, often from different CPUs on high queue depth workloads.
Using adjacent atomic_t fields causes cache line contention between the
submission and completion paths.
Extend the same treatment to ioerr_cnt and iotmo_cnt so all four iostat
counters in struct scsi_device use struct percpu_counter.
Suggested-by: John Garry <john.g.garry@xxxxxxxxxx>
Signed-off-by: Sumit Saxena <sumit.saxena@xxxxxxxxxxxx>
---
drivers/scsi/scsi_error.c | 4 ++--
drivers/scsi/scsi_lib.c | 10 +++++-----
drivers/scsi/scsi_scan.c | 8 ++++++++
drivers/scsi/scsi_sysfs.c | 23 ++++++++++++++---------
drivers/scsi/sd.c | 2 +-
include/scsi/scsi_device.h | 9 +++++----
6 files changed, 35 insertions(+), 21 deletions(-)
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index 147127fb4db9..b1aa7da2ba7c 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -349,7 +349,7 @@ enum blk_eh_timer_return scsi_timeout(struct request *req)
trace_scsi_dispatch_cmd_timeout(scmd);
scsi_log_completion(scmd, TIMEOUT_ERROR);
- atomic_inc(&scmd->device->iotmo_cnt);
+ percpu_counter_inc(&scmd->device->iotmo_cnt);
if (host->eh_deadline != -1 && !host->last_reset)
host->last_reset = jiffies;
@@ -370,7 +370,7 @@ enum blk_eh_timer_return scsi_timeout(struct request *req)
*/
if (test_and_set_bit(SCMD_STATE_COMPLETE, &scmd->state))
return BLK_EH_DONE;
- atomic_inc(&scmd->device->iodone_cnt);
+ percpu_counter_inc(&scmd->device->iodone_cnt);
if (scsi_abort_command(scmd) != SUCCESS) {
set_host_byte(scmd, DID_TIME_OUT);
scsi_eh_scmd_add(scmd);
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 6e8c7a42603e..979fdace33ac 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -1554,9 +1554,9 @@ static void scsi_complete(struct request *rq)
INIT_LIST_HEAD(&cmd->eh_entry);
- atomic_inc(&cmd->device->iodone_cnt);
+ percpu_counter_inc(&cmd->device->iodone_cnt);
if (cmd->result)
- atomic_inc(&cmd->device->ioerr_cnt);
+ percpu_counter_inc(&cmd->device->ioerr_cnt);
disposition = scsi_decide_disposition(cmd);
if (disposition != SUCCESS && scsi_cmd_runtime_exceeced(cmd))
@@ -1592,7 +1592,7 @@ static enum scsi_qc_status scsi_dispatch_cmd(struct
scsi_cmnd *cmd)
struct Scsi_Host *host = cmd->device->host;
int rtn = 0;
- atomic_inc(&cmd->device->iorequest_cnt);
+ percpu_counter_inc(&cmd->device->iorequest_cnt);
/* check if the device is still usable */
if (unlikely(cmd->device->sdev_state == SDEV_DEL)) {
@@ -1614,7 +1614,7 @@ static enum scsi_qc_status scsi_dispatch_cmd(struct
scsi_cmnd *cmd)
*/
SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
"queuecommand : device blocked\n"));
- atomic_dec(&cmd->device->iorequest_cnt);
+ percpu_counter_dec(&cmd->device->iorequest_cnt);
return SCSI_MLQUEUE_DEVICE_BUSY;
}
@@ -1647,7 +1647,7 @@ static enum scsi_qc_status scsi_dispatch_cmd(struct
scsi_cmnd *cmd)
trace_scsi_dispatch_cmd_start(cmd);
rtn = host->hostt->queuecommand(host, cmd);
if (rtn) {
- atomic_dec(&cmd->device->iorequest_cnt);
+ percpu_counter_dec(&cmd->device->iorequest_cnt);
trace_scsi_dispatch_cmd_error(cmd, rtn);
if (rtn != SCSI_MLQUEUE_DEVICE_BUSY &&
rtn != SCSI_MLQUEUE_TARGET_BUSY)
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index 121a14d5fdb8..bc885c72f01e 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -350,6 +350,14 @@ static struct scsi_device *scsi_alloc_sdev(struct
scsi_target *starget,
scsi_sysfs_device_initialize(sdev);
+ if (percpu_counter_init(&sdev->iorequest_cnt, 0, GFP_KERNEL) ||
+ percpu_counter_init(&sdev->iodone_cnt, 0, GFP_KERNEL) ||
+ percpu_counter_init(&sdev->ioerr_cnt, 0, GFP_KERNEL) ||
+ percpu_counter_init(&sdev->iotmo_cnt, 0, GFP_KERNEL)) {
+ ret = -ENOMEM;
+ goto out_device_destroy;
+ }
+
if (scsi_device_is_pseudo_dev(sdev))
return sdev;
diff --git a/drivers/scsi/scsi_sysfs.c b/drivers/scsi/scsi_sysfs.c
index dfc3559e7e04..f652edd16497 100644
--- a/drivers/scsi/scsi_sysfs.c
+++ b/drivers/scsi/scsi_sysfs.c
@@ -516,6 +516,10 @@ static void scsi_device_dev_release(struct device *dev)
if (vpd_pgb7)
kfree_rcu(vpd_pgb7, rcu);
kfree(sdev->inquiry);
+ percpu_counter_destroy(&sdev->iotmo_cnt);
+ percpu_counter_destroy(&sdev->ioerr_cnt);
+ percpu_counter_destroy(&sdev->iodone_cnt);
+ percpu_counter_destroy(&sdev->iorequest_cnt);
kfree(sdev);
if (parent)
@@ -936,26 +940,27 @@ static ssize_t
show_iostat_counterbits(struct device *dev, struct device_attribute *attr,
char *buf)
{
- return snprintf(buf, 20, "%d\n", (int)sizeof(atomic_t) * 8);
+ /* iostat counters are per-CPU sums (s64). Report width for tools. */
+ return sysfs_emit(buf, "%zu\n", sizeof(s64) * 8);
}
static DEVICE_ATTR(iocounterbits, S_IRUGO, show_iostat_counterbits, NULL);
-#define show_sdev_iostat(field)
\
+#define show_sdev_iostat_percpu(field) \
static ssize_t \
show_iostat_##field(struct device *dev, struct device_attribute *attr, \
char *buf) \
{ \
struct scsi_device *sdev = to_scsi_device(dev); \
- unsigned long long count = atomic_read(&sdev->field); \
- return snprintf(buf, 20, "0x%llx\n", count); \
+ unsigned long long count = percpu_counter_sum(&sdev->field); \
+ return sysfs_emit(buf, "0x%llx\n", count); \
} \
-static DEVICE_ATTR(field, S_IRUGO, show_iostat_##field, NULL)
+static DEVICE_ATTR(field, 0444, show_iostat_##field, NULL)
-show_sdev_iostat(iorequest_cnt);
-show_sdev_iostat(iodone_cnt);
-show_sdev_iostat(ioerr_cnt);
-show_sdev_iostat(iotmo_cnt);
+show_sdev_iostat_percpu(iorequest_cnt);
+show_sdev_iostat_percpu(iodone_cnt);
+show_sdev_iostat_percpu(ioerr_cnt);
+show_sdev_iostat_percpu(iotmo_cnt);
static ssize_t
sdev_show_modalias(struct device *dev, struct device_attribute *attr, char
*buf)
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index adc3fa55ca2c..b7ce01de17b3 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -4043,7 +4043,7 @@ static int sd_probe(struct scsi_device *sdp)
sdkp->index = index;
sdkp->max_retries = SD_MAX_RETRIES;
atomic_set(&sdkp->openers, 0);
- atomic_set(&sdkp->device->ioerr_cnt, 0);
+ percpu_counter_set(&sdkp->device->ioerr_cnt, 0);
if (!sdp->request_queue->rq_timeout) {
if (sdp->type != TYPE_MOD)
diff --git a/include/scsi/scsi_device.h b/include/scsi/scsi_device.h
index 029f5115b2ea..4be36bf2a475 100644
--- a/include/scsi/scsi_device.h
+++ b/include/scsi/scsi_device.h
@@ -9,6 +9,7 @@
#include <scsi/scsi.h>
#include <scsi/scsi_common.h>
#include <linux/atomic.h>
+#include <linux/percpu_counter.h>
#include <linux/sbitmap.h>
struct bsg_device;
@@ -272,10 +273,10 @@ struct scsi_device {
unsigned int max_device_blocked; /* what device_blocked counts down
from */
#define SCSI_DEFAULT_DEVICE_BLOCKED 3
- atomic_t iorequest_cnt;
- atomic_t iodone_cnt;
- atomic_t ioerr_cnt;
- atomic_t iotmo_cnt;
+ struct percpu_counter iorequest_cnt;
+ struct percpu_counter iodone_cnt;
+ struct percpu_counter ioerr_cnt;
+ struct percpu_counter iotmo_cnt;
struct device sdev_gendev,
sdev_dev;
--
2.43.7
|
 |
Lists.xenproject.org is hosted with RackSpace, monitoring our |