cloud · container · openshift · power · ppc64le

openshift on ppc64le: check whether or not the storage of a node is good enough for etcd

The image used in this test is hosted at quay.io/rpsene/ocp-perf-tools-ppc64le:v1 and the source code at https://github.com/rpsene/images/tree/ppc64le/etcd-perf.

[root@ocp-46-20210209-094616-b5ca3a632c-bastion-0 ~]# oc get nodes

NAME       STATUS     ROLES    AGE     VERSION
master-0   Ready      master   7h3m    v1.19.0+1833054
master-1   Ready      master   7h3m    v1.19.0+1833054
master-2   Ready      master   7h2m    v1.19.0+1833054
worker-0   Ready      worker   6h48m   v1.19.0+1833054
worker-1   NotReady   worker   6h49m   v1.19.0+1833054

[root@ocp-46-20210209-094616-b5ca3a632c-bastion-0 ~]# oc debug node/master-0

Creating debug namespace/openshift-debug-node-nmtw9 ...
Starting pod/master-0-debug ...
To use host binaries, run `chroot /host`
Pod IP: 192.168.24.196
If you don't see a command prompt, try pressing enter.
sh-4.4# chroot /host

sh-4.4# podman run --volume /var/lib/etcd:/var/lib/etcd:Z quay.io/rpsene/ocp-perf-tools-ppc64le:v1
---------------------------------------------------------------- Running fio ---------------------------------------------------------------------------
{
  "fio version" : "fio-3.19",
  "timestamp" : 1612904175,
  "timestamp_ms" : 1612904175145,
  "time" : "Tue Feb  9 20:56:15 2021",
  "global options" : {
    "rw" : "write",
    "ioengine" : "sync",
    "fdatasync" : "1",
    "directory" : "/var/lib/etcd",
    "size" : "22m",
    "bs" : "2300"
  },
  "jobs" : [
    {
      "jobname" : "etcd_perf",
      "groupid" : 0,
      "error" : 0,
      "eta" : 0,
      "elapsed" : 13,
      "job options" : {
        "name" : "etcd_perf"
      },
      "read" : {
        "io_bytes" : 0,
        "io_kbytes" : 0,
        "bw_bytes" : 0,
        "bw" : 0,
        "iops" : 0.000000,
        "runtime" : 0,
        "total_ios" : 0,
        "short_ios" : 10029,
        "drop_ios" : 0,
        "slat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "clat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "lat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "bw_min" : 0,
        "bw_max" : 0,
        "bw_agg" : 0.000000,
        "bw_mean" : 0.000000,
        "bw_dev" : 0.000000,
        "bw_samples" : 0,
        "iops_min" : 0,
        "iops_max" : 0,
        "iops_mean" : 0.000000,
        "iops_stddev" : 0.000000,
        "iops_samples" : 0
      },
      "write" : {
        "io_bytes" : 23066700,
        "io_kbytes" : 22526,
        "bw_bytes" : 1886846,
        "bw" : 1842,
        "iops" : 820.368098,
        "runtime" : 12225,
        "total_ios" : 10029,
        "short_ios" : 0,
        "drop_ios" : 0,
        "slat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "clat_ns" : {
          "min" : 2812,
          "max" : 8914877,
          "mean" : 374324.464054,
          "stddev" : 578924.082239,
          "N" : 10029,
          "percentile" : {
            "1.000000" : 3152,
            "5.000000" : 3280,
            "10.000000" : 3376,
            "20.000000" : 3632,
            "30.000000" : 4832,
            "40.000000" : 6688,
            "50.000000" : 119296,
            "60.000000" : 129536,
            "70.000000" : 144384,
            "80.000000" : 1204224,
            "90.000000" : 1318912,
            "95.000000" : 1368064,
            "99.000000" : 1482752,
            "99.500000" : 1794048,
            "99.900000" : 3457024,
            "99.950000" : 4423680,
            "99.990000" : 8847360
          }
        },
        "lat_ns" : {
          "min" : 3509,
          "max" : 8916031,
          "mean" : 375200.876359,
          "stddev" : 578895.727515,
          "N" : 10029
        },
        "bw_min" : 1644,
        "bw_max" : 4363,
        "bw_agg" : 98.801122,
        "bw_mean" : 1819.916667,
        "bw_dev" : 541.973440,
        "bw_samples" : 24,
        "iops_min" : 732,
        "iops_max" : 1942,
        "iops_mean" : 810.333333,
        "iops_stddev" : 241.177306,
        "iops_samples" : 24
      },
      "trim" : {
        "io_bytes" : 0,
        "io_kbytes" : 0,
        "bw_bytes" : 0,
        "bw" : 0,
        "iops" : 0.000000,
        "runtime" : 0,
        "total_ios" : 0,
        "short_ios" : 0,
        "drop_ios" : 0,
        "slat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "clat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "lat_ns" : {
          "min" : 0,
          "max" : 0,
          "mean" : 0.000000,
          "stddev" : 0.000000,
          "N" : 0
        },
        "bw_min" : 0,
        "bw_max" : 0,
        "bw_agg" : 0.000000,
        "bw_mean" : 0.000000,
        "bw_dev" : 0.000000,
        "bw_samples" : 0,
        "iops_min" : 0,
        "iops_max" : 0,
        "iops_mean" : 0.000000,
        "iops_stddev" : 0.000000,
        "iops_samples" : 0
      },
      "sync" : {
        "total_ios" : 0,
        "lat_ns" : {
          "min" : 158939,
          "max" : 16302863,
          "mean" : 837095.533154,
          "stddev" : 703984.449588,
          "N" : 10029,
          "percentile" : {
            "1.000000" : 189440,
            "5.000000" : 211968,
            "10.000000" : 220160,
            "20.000000" : 236544,
            "30.000000" : 268288,
            "40.000000" : 284672,
            "50.000000" : 305152,
            "60.000000" : 1482752,
            "70.000000" : 1515520,
            "80.000000" : 1531904,
            "90.000000" : 1548288,
            "95.000000" : 1564672,
            "99.000000" : 1908736,
            "99.500000" : 2342912,
            "99.900000" : 4046848,
            "99.950000" : 5931008,
            "99.990000" : 12910592
          }
        }
      },
      "job_runtime" : 12224,
      "usr_cpu" : 0.785340,
      "sys_cpu" : 1.775196,
      "ctx" : 21058,
      "majf" : 0,
      "minf" : 11,
      "iodepth_level" : {
        "1" : 200.000000,
        "2" : 0.000000,
        "4" : 0.000000,
        "8" : 0.000000,
        "16" : 0.000000,
        "32" : 0.000000,
        ">=64" : 0.000000
      },
      "iodepth_submit" : {
        "0" : 0.000000,
        "4" : 100.000000,
        "8" : 0.000000,
        "16" : 0.000000,
        "32" : 0.000000,
        "64" : 0.000000,
        ">=64" : 0.000000
      },
      "iodepth_complete" : {
        "0" : 0.000000,
        "4" : 100.000000,
        "8" : 0.000000,
        "16" : 0.000000,
        "32" : 0.000000,
        "64" : 0.000000,
        ">=64" : 0.000000
      },
      "latency_ns" : {
        "2" : 0.000000,
        "4" : 0.000000,
        "10" : 0.000000,
        "20" : 0.000000,
        "50" : 0.000000,
        "100" : 0.000000,
        "250" : 0.000000,
        "500" : 0.000000,
        "750" : 0.000000,
        "1000" : 0.000000
      },
      "latency_us" : {
        "2" : 0.000000,
        "4" : 24.917739,
        "10" : 17.698674,
        "20" : 1.056935,
        "50" : 0.169508,
        "100" : 0.049855,
        "250" : 30.581314,
        "500" : 0.628178,
        "750" : 0.159537,
        "1000" : 0.179480
      },
      "latency_ms" : {
        "2" : 24.259647,
        "4" : 0.209393,
        "10" : 0.089740,
        "20" : 0.000000,
        "50" : 0.000000,
        "100" : 0.000000,
        "250" : 0.000000,
        "500" : 0.000000,
        "750" : 0.000000,
        "1000" : 0.000000,
        "2000" : 0.000000,
        ">=2000" : 0.000000
      },
      "latency_depth" : 1,
      "latency_target" : 0,
      "latency_percentile" : 100.000000,
      "latency_window" : 0
    }
  ],
  "disk_util" : [
    {
      "name" : "dm-0",
      "read_ios" : 5598,
      "write_ios" : 10145,
      "read_merges" : 0,
      "write_merges" : 0,
      "read_ticks" : 3820,
      "write_ticks" : 8320,
      "in_queue" : 12140,
      "util" : 99.214788,
      "aggr_read_ios" : 5632,
      "aggr_write_ios" : 10206,
      "aggr_read_merges" : 0,
      "aggr_write_merge" : 1,
      "aggr_read_ticks" : 3585,
      "aggr_write_ticks" : 8404,
      "aggr_in_queue" : 10,
      "aggr_util" : 99.180261
    },
    {
      "name" : "sde",
      "read_ios" : 5632,
      "write_ios" : 10206,
      "read_merges" : 0,
      "write_merges" : 1,
      "read_ticks" : 3585,
      "write_ticks" : 8404,
      "in_queue" : 10,
      "util" : 99.180261
    }
  ]
}
--------------------------------------------------------------------------------------------------------------------------------------------------------
99th percentile of fsync is 1908736 ns
99th percentile of the fsync is within the recommended threshold - 10 ms, the disk can be used to host etcd

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out /  Change )

Google photo

You are commenting using your Google account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s