Choose timezone
Your profile timezone:
Powered by Indico
v3.3.3
Meeting room:
https://l.infn.it/mlinfn-room
Dati grezzissimi e Puramente raw, ma sono i seguenti test:
#Rand RW mix 75%-25% BS 4K
time fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=test-4k --filename=test-4k --bs=4k --iodepth=64 --size=5G --readwrite=randrw --rwmixread=75
#Seq BS 4k Read
time fio --name=seqread --rw=read --direct=1 --ioengine=libaio --bs=4k --size=5G --iodepth=64
#Seq BS 4k Write
time fio --name=seqwrite --rw=write --direct=1 --ioengine=libaio --bs=4k --size=5G --iodepth=64
#Seq BS 4M Write
time fio --name=seqwrite --rw=write --direct=1 --ioengine=libaio --bs=4m --size=5G --iodepth=64
#Seq BS 4M Read
time fio --name=seqread --rw=read --direct=1 --ioengine=libaio --bs=4m --size=5G --iodepth=64
Eseguiti su bare metal (Macchinone1) VM cloud@cnaf (usa NVMe) , VM infncloud (usa cynder)
Stefano
# Rand RW mix 75%-25% BS 4M
# MACCHINONE1
[root@nova-ml-206-07-13 fiotest]# df -h /var/lib/nova/LSD
Filesystem Size Used Avail Use% Mounted on
/dev/md127 24T 1.6T 21T 7% /var/lib/nova/LSD
[root@nova-ml-206-07-13 fiotest]# pwd
/var/lib/nova/LSD/fiotest
[root@nova-ml-206-07-13 fiotest]# # Rand RW mix 75%-25% BS 4M
[root@nova-ml-206-07-13 fiotest]# time fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=test-4k --filename=test-4k --bs=4k --iodep
th=64 --size=5G --readwrite=randrw --rwmixread=75
test-4k: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.19
Starting 1 process
test-4k: Laying out IO file (1 file / 5120MiB)
Jobs: 1 (f=1): [m(1)][100.0%][r=759MiB/s,w=252MiB/s][r=194k,w=64.5k IOPS][eta 00m:00s]
test-4k: (groupid=0, jobs=1): err= 0: pid=3032921: Fri Jan 13 15:28:13 2023
read: IOPS=204k, BW=795MiB/s (834MB/s)(3837MiB/4827msec)
bw ( KiB/s): min=770520, max=867379, per=100.00%, avg=816741.89, stdev=35977.87, samples=9
iops : min=192630, max=216844, avg=204185.33, stdev=8994.26, samples=9
write: IOPS=68.0k, BW=266MiB/s (279MB/s)(1283MiB/4827msec); 0 zone resets
bw ( KiB/s): min=258192, max=288742, per=100.00%, avg=272993.56, stdev=12412.38, samples=9
iops : min=64548, max=72185, avg=68248.33, stdev=3103.02, samples=9
cpu : usr=17.99%, sys=56.94%, ctx=17304, majf=0, minf=12
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=982350,328370,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=795MiB/s (834MB/s), 795MiB/s-795MiB/s (834MB/s-834MB/s), io=3837MiB (4024MB), run=4827-4827msec
WRITE: bw=266MiB/s (279MB/s), 266MiB/s-266MiB/s (279MB/s-279MB/s), io=1283MiB (1345MB), run=4827-4827msec
Disk stats (read/write):
md127: ios=950515/317662, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=491175/164186, aggrmerge=0/0, aggrticks=69183/2668, aggrin_queue=718
52, aggrutil=97.01%
nvme0n1: ios=491167/164194, merge=0/0, ticks=69196/2667, in_queue=71864, util=96.99%
nvme1n1: ios=491183/164179, merge=0/0, ticks=69170/2670, in_queue=71840, util=97.01%
real 0m10.677s
user 0m3.794s
sys 0m7.027s
# VM cloud@cnaf, usa NVMe
# Rand RW mix 75%-25% BS 4M
[root@sdpt4 fiotest]# time fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=test-4k --filename=test-4k --bs=4k --iodepth=64 --size
=5G --readwrite=randrw --rwmixread=75
test-4k: (g=0): rw=randrw, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.19
Starting 1 process
test-4k: Laying out IO file (1 file / 5120MiB)
Jobs: 1 (f=1): [m(1)][100.0%][r=462MiB/s,w=156MiB/s][r=118k,w=39.8k IOPS][eta 00m:00s]
test-4k: (groupid=0, jobs=1): err= 0: pid=65309: Fri Jan 13 15:29:12 2023
read: IOPS=121k, BW=473MiB/s (496MB/s)(3837MiB/8108msec)
bw ( KiB/s): min=472160, max=496656, per=100.00%, avg=486631.93, stdev=6760.03, samples=15
iops : min=118040, max=124164, avg=121658.07, stdev=1689.82, samples=15
write: IOPS=40.5k, BW=158MiB/s (166MB/s)(1283MiB/8108msec); 0 zone resets
bw ( KiB/s): min=159488, max=164976, per=100.00%, avg=162547.13, stdev=1624.68, samples=15
iops : min=39872, max=41244, avg=40636.73, stdev=406.18, samples=15
cpu : usr=8.93%, sys=49.70%, ctx=80947, majf=0, minf=8
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=982350,328370,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=473MiB/s (496MB/s), 473MiB/s-473MiB/s (496MB/s-496MB/s), io=3837MiB (4024MB), run=8108-8108msec
WRITE: bw=158MiB/s (166MB/s), 158MiB/s-158MiB/s (166MB/s-166MB/s), io=1283MiB (1345MB), run=8108-8108msec
Disk stats (read/write):
vda: ios=960105/320925, merge=0/0, ticks=225186/196538, in_queue=421725, util=98.89%
real 0m14.560s
user 0m1.803s
sys 0m8.950s
# VM INFNCLOUD, usa cynder
root@vnode-0:~/fiotest#
root@vnode-0:~/fiotest# time fio --randrepeat=1 --ioengine=libaio --direct=1 --gtod_reduce=1 --name=test-4m --filename=test-4m --bs=4m --iodepth=64 --size=5G --readwrite=randrw --rwmixread=75
test-4m: (g=0): rw=randrw, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64
fio-3.16
Starting 1 process
test-4m: Laying out IO file (1 file / 5120MiB)
Jobs: 1 (f=0): [f(1)][100.0%][r=621MiB/s,w=200MiB/s][r=155,w=50 IOPS][eta 00m:00s]
test-4m: (groupid=0, jobs=1): err= 0: pid=1665091: Fri Jan 13 11:36:22 2023
read: IOPS=199, BW=800MiB/s (839MB/s)(3768MiB/4712msec)
bw ( KiB/s): min=417792, max=925696, per=99.26%, avg=812828.44, stdev=163828.62, samples=9
iops : min= 102, max= 226, avg=198.44, stdev=40.00, samples=9
write: IOPS=71, BW=287MiB/s (301MB/s)(1352MiB/4712msec); 0 zone resets
bw ( KiB/s): min=131072, max=344064, per=100.00%, avg=294912.00, stdev=68661.47, samples=9
iops : min= 32, max= 84, avg=72.00, stdev=16.76, samples=9
cpu : usr=1.68%, sys=4.29%, ctx=1168, majf=0, minf=8
IO depths : 1=0.1%, 2=0.2%, 4=0.3%, 8=0.6%, 16=1.2%, 32=2.5%, >=64=95.1%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=942,338,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=800MiB/s (839MB/s), 800MiB/s-800MiB/s (839MB/s-839MB/s), io=3768MiB (3951MB), run=4712-4712msec
WRITE: bw=287MiB/s (301MB/s), 287MiB/s-287MiB/s (301MB/s-301MB/s), io=1352MiB (1418MB), run=4712-4712msec
Disk stats (read/write):
vda: ios=3797/1353, merge=0/0, ticks=725011/270806, in_queue=985480, util=93.93%
real 0m4.637s
user 0m0.235s
sys 0m0.599s
#Seq BS 4k Read
#Macchinone1
[root@nova-ml-206-07-13 fiotest]# time fio --name=seqread --rw=read --direct=1 --ioengine=libaio --bs=4k --size=5G --iodepth=64
seqread: (g=0): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.19
Starting 1 process
seqread: Laying out IO file (1 file / 5120MiB)
Jobs: 1 (f=1): [R(1)][100.0%][r=1148MiB/s][r=294k IOPS][eta 00m:00s]
seqread: (groupid=0, jobs=1): err= 0: pid=3024752: Fri Jan 13 14:22:55 2023
read: IOPS=291k, BW=1138MiB/s (1194MB/s)(5120MiB/4498msec)
slat (nsec): min=1324, max=160722, avg=2514.26, stdev=1399.02
clat (usec): min=22, max=12696, avg=216.64, stdev=132.29
lat (usec): min=23, max=12698, avg=219.24, stdev=132.31
clat percentiles (usec):
| 1.00th=[ 117], 5.00th=[ 141], 10.00th=[ 151], 20.00th=[ 165],
| 30.00th=[ 176], 40.00th=[ 190], 50.00th=[ 206], 60.00th=[ 225],
| 70.00th=[ 241], 80.00th=[ 260], 90.00th=[ 281], 95.00th=[ 302],
| 99.00th=[ 375], 99.50th=[ 457], 99.90th=[ 1680], 99.95th=[ 2671],
| 99.99th=[ 4178]
bw ( MiB/s): min= 1056, max= 1188, per=99.70%, avg=1134.86, stdev=51.39, samples=8
iops : min=270463, max=304197, avg=290524.88, stdev=13155.67, samples=8
lat (usec) : 50=0.01%, 100=0.39%, 250=74.83%, 500=24.39%, 750=0.18%
lat (usec) : 1000=0.06%
lat (msec) : 2=0.07%, 4=0.07%, 10=0.01%, 20=0.01%
cpu : usr=19.03%, sys=57.55%, ctx=8671, majf=0, minf=195
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=1310720,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=1138MiB/s (1194MB/s), 1138MiB/s-1138MiB/s (1194MB/s-1194MB/s), io=5120MiB (5369MB), run=4498-4498msec
Disk stats (read/write):
md127: ios=1304553/30, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=655360/10, aggrmerge=0/0, aggrticks=98643/1, aggrin_queue=98644, aggrut
il=97.25%
nvme0n1: ios=655360/15, merge=0/0, ticks=94058/1, in_queue=94060, util=97.04%
nvme1n1: ios=655360/6, merge=0/0, ticks=103228/1, in_queue=103229, util=97.25%
real 0m5.435s
user 0m2.726s
sys 0m3.126s
#VM INFNCLOUD
root@vnode-0:~/fiotest# time fio --name=seqread --rw=read --direct=1 --ioengine=libaio --bs=4k --size=5G --iodepth=64
seqread: (g=0): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.16
Starting 1 process
Jobs: 1 (f=1): [R(1)][100.0%][r=30.5MiB/s][r=7815 IOPS][eta 00m:00s]
seqread: (groupid=0, jobs=1): err= 0: pid=1665489: Fri Jan 13 12:00:54 2023
read: IOPS=11.9k, BW=46.6MiB/s (48.8MB/s)(5120MiB/109970msec)
slat (nsec): min=1611, max=1007.1k, avg=4170.11, stdev=3465.51
clat (usec): min=178, max=331677, avg=5364.93, stdev=15192.82
lat (usec): min=183, max=331681, avg=5369.21, stdev=15193.24
clat percentiles (usec):
| 1.00th=[ 988], 5.00th=[ 1205], 10.00th=[ 1270], 20.00th=[ 1385],
| 30.00th=[ 1467], 40.00th=[ 1549], 50.00th=[ 1647], 60.00th=[ 1778],
| 70.00th=[ 1975], 80.00th=[ 2409], 90.00th=[ 11207], 95.00th=[ 22414],
| 99.00th=[ 64750], 99.50th=[102237], 99.90th=[217056], 99.95th=[246416],
| 99.99th=[295699]
bw ( KiB/s): min= 5408, max=97008, per=100.00%, avg=47680.72, stdev=23726.10, samples=219
iops : min= 1352, max=24252, avg=11920.16, stdev=5931.53, samples=219
lat (usec) : 250=0.01%, 500=0.12%, 750=0.34%, 1000=0.59%
lat (msec) : 2=69.85%, 4=11.58%, 10=6.08%, 20=5.22%, 50=4.74%
lat (msec) : 100=0.94%, 250=0.47%, 500=0.05%
cpu : usr=1.43%, sys=6.48%, ctx=386390, majf=0, minf=76
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=1310720,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=46.6MiB/s (48.8MB/s), 46.6MiB/s-46.6MiB/s (48.8MB/s-48.8MB/s), io=5120MiB (5369MB), run=109970-109970msec
Disk stats (read/write):
vda: ios=1309952/315, merge=0/46, ticks=7011150/6665, in_queue=4818708, util=99.88%
real 1m36.187s
user 0m1.681s
sys 0m8.742s
#VM Cloud@CNAF
[root@sdpt4 fiotest]# time fio --name=seqread --rw=read --direct=1 --ioengine=libaio --bs=4k --size=5G --iodepth=64
seqread: (g=0): rw=read, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.19
Starting 1 process
seqread: Laying out IO file (1 file / 5120MiB)
Jobs: 1 (f=1): [R(1)][100.0%][r=790MiB/s][r=202k IOPS][eta 00m:00s]
seqread: (groupid=0, jobs=1): err= 0: pid=60483: Fri Jan 13 14:23:16 2023
read: IOPS=201k, BW=787MiB/s (825MB/s)(5120MiB/6505msec)
slat (nsec): min=1617, max=582981, avg=4198.36, stdev=3104.66
clat (usec): min=91, max=1488, avg=313.02, stdev=50.74
lat (usec): min=99, max=1491, avg=317.32, stdev=50.92
clat percentiles (usec):
| 1.00th=[ 217], 5.00th=[ 241], 10.00th=[ 255], 20.00th=[ 273],
| 30.00th=[ 285], 40.00th=[ 297], 50.00th=[ 310], 60.00th=[ 322],
| 70.00th=[ 334], 80.00th=[ 351], 90.00th=[ 375], 95.00th=[ 396],
| 99.00th=[ 437], 99.50th=[ 453], 99.90th=[ 506], 99.95th=[ 562],
| 99.99th=[ 1352]
bw ( KiB/s): min=784008, max=830947, per=100.00%, avg=806100.75, stdev=15541.28, samples=12
iops : min=196002, max=207736, avg=201525.25, stdev=3885.03, samples=12
lat (usec) : 100=0.01%, 250=7.89%, 500=92.00%, 750=0.08%, 1000=0.01%
lat (msec) : 2=0.03%
cpu : usr=12.55%, sys=71.08%, ctx=232, majf=0, minf=74
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=1310720,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=787MiB/s (825MB/s), 787MiB/s-787MiB/s (825MB/s-825MB/s), io=5120MiB (5369MB), run=6505-6505msec
Disk stats (read/write):
vda: ios=1283275/0, merge=0/0, ticks=229905/0, in_queue=229905, util=98.61%
real 0m8.506s
user 0m0.863s
sys 0m5.816s
#Seq BS 4k Write
#Macchinone1
[root@nova-ml-206-07-13 fiotest]# time fio --name=seqwrite --rw=write --direct=1 --ioengine=libaio --bs=4k --size=5G --iodepth=64
seqwrite: (g=0): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.19
Starting 1 process
Jobs: 1 (f=1): [W(1)][100.0%][w=1037MiB/s][w=265k IOPS][eta 00m:00s]
seqwrite: (groupid=0, jobs=1): err= 0: pid=3033206: Fri Jan 13 15:30:46 2023
write: IOPS=279k, BW=1089MiB/s (1142MB/s)(5120MiB/4703msec); 0 zone resets
slat (nsec): min=1494, max=181856, avg=2811.42, stdev=1136.01
clat (usec): min=14, max=508, avg=226.44, stdev=21.02
lat (usec): min=17, max=511, avg=229.34, stdev=21.27
clat percentiles (usec):
| 1.00th=[ 206], 5.00th=[ 208], 10.00th=[ 212], 20.00th=[ 215],
| 30.00th=[ 217], 40.00th=[ 219], 50.00th=[ 221], 60.00th=[ 223],
| 70.00th=[ 225], 80.00th=[ 231], 90.00th=[ 258], 95.00th=[ 265],
| 99.00th=[ 322], 99.50th=[ 330], 99.90th=[ 383], 99.95th=[ 404],
| 99.99th=[ 429]
bw ( MiB/s): min= 1008, max= 1128, per=100.00%, avg=1089.53, stdev=41.61, samples=9
iops : min=258158, max=288908, avg=278919.78, stdev=10652.28, samples=9
lat (usec) : 20=0.01%, 50=0.01%, 100=0.01%, 250=87.57%, 500=12.43%
lat (usec) : 750=0.01%
cpu : usr=17.99%, sys=60.36%, ctx=26, majf=0, minf=14
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=0,1310720,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
WRITE: bw=1089MiB/s (1142MB/s), 1089MiB/s-1089MiB/s (1142MB/s-1142MB/s), io=5120MiB (5369MB), run=4703-4703msec
Disk stats (read/write):
md127: ios=0/1297397, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=0/655363, aggrmerge=0/1, aggrticks=0/7847, aggrin_queue=7847, aggrutil=9
7.45%
nvme0n1: ios=0/655362, merge=0/0, ticks=0/7872, in_queue=7872, util=97.45%
nvme1n1: ios=0/655364, merge=0/2, ticks=0/7822, in_queue=7823, util=97.45%
real 0m5.937s
user 0m2.837s
sys 0m3.508s
#VM CLOUD@CNAF
[root@sdpt4 fiotest]# time fio --name=seqwrite --rw=write --direct=1 --ioengine=libaio --bs=4k --size=5G --iodepth=64
seqwrite: (g=0): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.19
Starting 1 process
Jobs: 1 (f=1): [W(1)][100.0%][w=675MiB/s][w=173k IOPS][eta 00m:00s]
seqwrite: (groupid=0, jobs=1): err= 0: pid=65468: Fri Jan 13 15:31:22 2023
write: IOPS=170k, BW=665MiB/s (697MB/s)(5120MiB/7701msec); 0 zone resets
slat (nsec): min=1561, max=161959, avg=3447.89, stdev=5496.02
clat (usec): min=78, max=2849, avg=372.11, stdev=57.39
lat (usec): min=82, max=2851, avg=375.66, stdev=56.81
clat percentiles (usec):
| 1.00th=[ 247], 5.00th=[ 289], 10.00th=[ 314], 20.00th=[ 343],
| 30.00th=[ 351], 40.00th=[ 359], 50.00th=[ 367], 60.00th=[ 375],
| 70.00th=[ 388], 80.00th=[ 408], 90.00th=[ 437], 95.00th=[ 461],
| 99.00th=[ 515], 99.50th=[ 545], 99.90th=[ 644], 99.95th=[ 1004],
| 99.99th=[ 1516]
bw ( KiB/s): min=646792, max=701208, per=100.00%, avg=681922.73, stdev=15810.97, samples=15
iops : min=161698, max=175302, avg=170480.67, stdev=3952.74, samples=15
lat (usec) : 100=0.01%, 250=1.12%, 500=97.41%, 750=1.39%, 1000=0.02%
lat (msec) : 2=0.05%, 4=0.01%
cpu : usr=11.21%, sys=52.84%, ctx=60634, majf=0, minf=11
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=0,1310720,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
WRITE: bw=665MiB/s (697MB/s), 665MiB/s-665MiB/s (697MB/s-697MB/s), io=5120MiB (5369MB), run=7701-7701msec
Disk stats (read/write):
vda: ios=0/1303978, merge=0/10, ticks=0/302109, in_queue=302110, util=98.85%
real 0m8.941s
user 0m1.063s
sys 0m4.811s
#VM INFNCLOUD
root@vnode-0:~/fiotest# time fio --name=seqwrite --rw=write --direct=1 --ioengine=libaio --bs=4k --size=5G --iodepth=64
seqwrite: (g=0): rw=write, bs=(R) 4096B-4096B, (W) 4096B-4096B, (T) 4096B-4096B, ioengine=libaio, iodepth=64
fio-3.16
Starting 1 process
Jobs: 1 (f=1): [W(1)][100.0%][w=6780KiB/s][w=1695 IOPS][eta 00m:00s]
seqwrite: (groupid=0, jobs=1): err= 0: pid=1667865: Fri Jan 13 15:43:09 2023
write: IOPS=1983, BW=7935KiB/s (8125kB/s)(5120MiB/660765msec); 0 zone resets
slat (nsec): min=1958, max=140256k, avg=4897.67, stdev=220416.70
clat (usec): min=439, max=617747, avg=32258.40, stdev=47047.81
lat (usec): min=448, max=617752, avg=32263.43, stdev=47047.78
clat percentiles (usec):
| 1.00th=[ 1188], 5.00th=[ 6194], 10.00th=[ 8979], 20.00th=[ 11207],
| 30.00th=[ 12780], 40.00th=[ 13960], 50.00th=[ 15401], 60.00th=[ 17433],
| 70.00th=[ 21365], 80.00th=[ 26608], 90.00th=[100140], 95.00th=[149947],
| 99.00th=[223347], 99.50th=[250610], 99.90th=[308282], 99.95th=[337642],
| 99.99th=[463471]
bw ( KiB/s): min= 1040, max=18856, per=99.98%, avg=7932.17, stdev=2722.68, samples=1321
iops : min= 260, max= 4714, avg=1983.02, stdev=680.67, samples=1321
lat (usec) : 500=0.01%, 750=0.15%, 1000=0.36%
lat (msec) : 2=2.88%, 4=1.20%, 10=10.74%, 20=51.80%, 50=19.47%
lat (msec) : 100=3.35%, 250=9.55%, 500=0.49%, 750=0.01%
cpu : usr=0.29%, sys=0.96%, ctx=47845, majf=0, minf=13
IO depths : 1=0.1%, 2=0.1%, 4=0.1%, 8=0.1%, 16=0.1%, 32=0.1%, >=64=100.0%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=0,1310720,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
WRITE: bw=7935KiB/s (8125kB/s), 7935KiB/s-7935KiB/s (8125kB/s-8125kB/s), io=5120MiB (5369MB), run=660765-660765msec
Disk stats (read/write):
vda: ios=0/1311800, merge=0/559, ticks=0/42058399, in_queue=39643076, util=99.83%
real 11m1.322s --> !!!!!!!!!!
user 0m2.941s
sys 0m8.053s
#Seq BS 4M Write
#Macchinone1
[root@nova-ml-206-07-13 fiotest]# time fio --name=seqwrite --rw=write --direct=1 --ioengine=libaio --bs=4m --size=5G --iodepth=64
seqwrite: (g=0): rw=write, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64
fio-3.19
Starting 1 process
Jobs: 1 (f=1)
seqwrite: (groupid=0, jobs=1): err= 0: pid=3029195: Fri Jan 13 14:59:47 2023
write: IOPS=1662, BW=6649MiB/s (6972MB/s)(5120MiB/770msec); 0 zone resets
slat (usec): min=284, max=649, avg=477.91, stdev=58.80
clat (usec): min=29199, max=72981, avg=37258.95, stdev=6504.82
lat (usec): min=29499, max=73412, avg=37737.05, stdev=6505.18
clat percentiles (usec):
| 1.00th=[30016], 5.00th=[30016], 10.00th=[30278], 20.00th=[31065],
| 30.00th=[36963], 40.00th=[36963], 50.00th=[36963], 60.00th=[37487],
| 70.00th=[37487], 80.00th=[37487], 90.00th=[42206], 95.00th=[52167],
| 99.00th=[65274], 99.50th=[67634], 99.90th=[70779], 99.95th=[72877],
| 99.99th=[72877]
bw ( MiB/s): min= 6788, max= 6788, per=100.00%, avg=6788.35, stdev= 0.00, samples=1
iops : min= 1697, max= 1697, avg=1697.00, stdev= 0.00, samples=1
lat (msec) : 50=93.67%, 100=6.33%
cpu : usr=43.82%, sys=25.49%, ctx=753, majf=0, minf=14
IO depths : 1=0.1%, 2=0.2%, 4=0.3%, 8=0.6%, 16=1.2%, 32=2.5%, >=64=95.1%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=0,1280,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
WRITE: bw=6649MiB/s (6972MB/s), 6649MiB/s-6649MiB/s (6972MB/s-6972MB/s), io=5120MiB (5369MB), run=770-770msec
Disk stats (read/write):
md127: ios=0/40960, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=0/20480, aggrmerge=0/0, aggrticks=0/650124, aggrin_queue=650123, aggrutil=
74.47%
nvme0n1: ios=0/20480, merge=0/0, ticks=0/641668, in_queue=641667, util=73.02%
nvme1n1: ios=0/20480, merge=0/0, ticks=0/658580, in_queue=658580, util=74.47%
real 0m2.212s
user 0m2.322s
sys 0m1.001s
#VM Cloud@CNAF
[root@sdpt4 fiotest]# time fio --name=seqwrite --rw=write --direct=1 --ioengine=libaio --bs=4m --size=5G --iodepth=64
seqwrite: (g=0): rw=write, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64
fio-3.19
Starting 1 process
seqwrite: (groupid=0, jobs=1): err= 0: pid=63284: Fri Jan 13 15:00:46 2023
write: IOPS=1693, BW=6772MiB/s (7101MB/s)(5120MiB/756msec); 0 zone resets
slat (usec): min=104, max=2034, avg=395.02, stdev=146.96
clat (msec): min=19, max=115, avg=36.84, stdev= 9.54
lat (msec): min=19, max=115, avg=37.24, stdev= 9.55
clat percentiles (msec):
| 1.00th=[ 22], 5.00th=[ 23], 10.00th=[ 27], 20.00th=[ 36],
| 30.00th=[ 37], 40.00th=[ 37], 50.00th=[ 37], 60.00th=[ 38],
| 70.00th=[ 38], 80.00th=[ 38], 90.00th=[ 39], 95.00th=[ 47],
| 99.00th=[ 91], 99.50th=[ 91], 99.90th=[ 92], 99.95th=[ 115],
| 99.99th=[ 115]
bw ( MiB/s): min= 6764, max= 6764, per=99.88%, avg=6764.29, stdev= 0.00, samples=1
iops : min= 1691, max= 1691, avg=1691.00, stdev= 0.00, samples=1
lat (msec) : 20=0.31%, 50=95.08%, 100=4.53%, 250=0.08%
cpu : usr=53.91%, sys=7.95%, ctx=2443, majf=0, minf=12
IO depths : 1=0.1%, 2=0.2%, 4=0.3%, 8=0.6%, 16=1.2%, 32=2.5%, >=64=95.1%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=0,1280,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
WRITE: bw=6772MiB/s (7101MB/s), 6772MiB/s-6772MiB/s (7101MB/s-7101MB/s), io=5120MiB (5369MB), run=756-756msec
Disk stats (read/write):
vda: ios=0/3795, merge=0/7, ticks=0/130025, in_queue=130024, util=74.90%
real 0m2.277s
user 0m0.642s
sys 0m1.189s
#VM INFNCLOUD
root@vnode-0:~/fiotest# time fio --name=seqwrite --rw=write --direct=1 --ioengine=libaio --bs=4m --size=5G --iodepth=64
seqwrite: (g=0): rw=write, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64
fio-3.16
Starting 1 process
Jobs: 1 (f=1): [W(1)][100.0%][w=525MiB/s][w=131 IOPS][eta 00m:00s]
seqwrite: (groupid=0, jobs=1): err= 0: pid=1667485: Fri Jan 13 15:01:27 2023
write: IOPS=188, BW=754MiB/s (791MB/s)(5120MiB/6786msec); 0 zone resets
slat (usec): min=262, max=236504, avg=915.24, stdev=7494.12
clat (msec): min=65, max=776, avg=337.22, stdev=109.68
lat (msec): min=122, max=776, avg=338.14, stdev=109.48
clat percentiles (msec):
| 1.00th=[ 163], 5.00th=[ 207], 10.00th=[ 224], 20.00th=[ 247],
| 30.00th=[ 266], 40.00th=[ 288], 50.00th=[ 317], 60.00th=[ 342],
| 70.00th=[ 376], 80.00th=[ 430], 90.00th=[ 485], 95.00th=[ 558],
| 99.00th=[ 667], 99.50th=[ 735], 99.90th=[ 776], 99.95th=[ 776],
| 99.99th=[ 776]
bw ( KiB/s): min=540672, max=974848, per=99.26%, avg=766897.23, stdev=137873.19, samples=13
iops : min= 132, max= 238, avg=187.23, stdev=33.66, samples=13
lat (msec) : 100=0.08%, 250=22.42%, 500=69.22%, 750=8.05%, 1000=0.23%
cpu : usr=6.90%, sys=2.92%, ctx=537, majf=0, minf=13
IO depths : 1=0.1%, 2=0.2%, 4=0.3%, 8=0.6%, 16=1.2%, 32=2.5%, >=64=95.1%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=0,1280,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
WRITE: bw=754MiB/s (791MB/s), 754MiB/s-754MiB/s (791MB/s-791MB/s), io=5120MiB (5369MB), run=6786-6786msec
Disk stats (read/write):
vda: ios=0/5156, merge=0/1221, ticks=0/1576360, in_queue=1566168, util=95.40%
real 0m7.551s
user 0m0.629s
sys 0m0.682s
#Seq BS 4M Read
#Macchinone1
[root@nova-ml-206-07-13 fiotest]# time fio --name=seqread --rw=read --direct=1 --ioengine=libaio --bs=4m --size=5G --iodepth=64
seqread: (g=0): rw=read, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64
fio-3.19
Starting 1 process
seqread: (groupid=0, jobs=1): err= 0: pid=3029843: Fri Jan 13 15:04:15 2023
read: IOPS=1596, BW=6384MiB/s (6694MB/s)(5120MiB/802msec)
slat (usec): min=108, max=1418, avg=225.40, stdev=253.63
clat (usec): min=11199, max=83367, avg=37781.93, stdev=8450.98
lat (usec): min=11378, max=84411, avg=38007.44, stdev=8515.95
clat percentiles (usec):
| 1.00th=[11863], 5.00th=[24511], 10.00th=[31327], 20.00th=[34866],
| 30.00th=[35914], 40.00th=[36963], 50.00th=[37487], 60.00th=[38536],
| 70.00th=[39060], 80.00th=[40633], 90.00th=[43254], 95.00th=[50594],
| 99.00th=[70779], 99.50th=[78119], 99.90th=[82314], 99.95th=[83362],
| 99.99th=[83362]
bw ( MiB/s): min= 6213, max= 6213, per=97.33%, avg=6213.29, stdev= 0.00, samples=1
iops : min= 1553, max= 1553, avg=1553.00, stdev= 0.00, samples=1
lat (msec) : 20=3.91%, 50=90.62%, 100=5.47%
cpu : usr=0.00%, sys=33.46%, ctx=899, majf=0, minf=61460
IO depths : 1=0.1%, 2=0.2%, 4=0.3%, 8=0.6%, 16=1.2%, 32=2.5%, >=64=95.1%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=1280,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=6384MiB/s (6694MB/s), 6384MiB/s-6384MiB/s (6694MB/s-6694MB/s), io=5120MiB (5369MB), run=802-802msec
Disk stats (read/write):
md127: ios=34208/0, merge=0/0, ticks=0/0, in_queue=0, util=0.00%, aggrios=20480/0, aggrmerge=0/0, aggrticks=601248/0, aggrin_queue=601247, aggrutil=
83.63%
nvme0n1: ios=20480/0, merge=0/0, ticks=601154/0, in_queue=601154, util=83.63%
nvme1n1: ios=20480/0, merge=0/0, ticks=601342/0, in_queue=601341, util=83.63%
real 0m1.932s
user 0m2.021s
sys 0m0.964s
# VM CLOUD@CNAF
[root@sdpt4 fiotest]# time fio --name=seqread --rw=read --direct=1 --ioengine=libaio --bs=4m --size=5G --iodepth=64
seqread: (g=0): rw=read, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64
fio-3.19
Starting 1 process
seqread: (groupid=0, jobs=1): err= 0: pid=63603: Fri Jan 13 15:05:08 2023
read: IOPS=1655, BW=6624MiB/s (6945MB/s)(5120MiB/773msec)
slat (usec): min=21, max=12519, avg=110.38, stdev=471.60
clat (msec): min=3, max=528, avg=37.08, stdev=34.16
lat (msec): min=4, max=528, avg=37.19, stdev=34.15
clat percentiles (msec):
| 1.00th=[ 11], 5.00th=[ 25], 10.00th=[ 28], 20.00th=[ 31],
| 30.00th=[ 32], 40.00th=[ 34], 50.00th=[ 35], 60.00th=[ 36],
| 70.00th=[ 38], 80.00th=[ 40], 90.00th=[ 43], 95.00th=[ 46],
| 99.00th=[ 66], 99.50th=[ 71], 99.90th=[ 527], 99.95th=[ 527],
| 99.99th=[ 527]
bw ( MiB/s): min= 6477, max= 6477, per=97.80%, avg=6477.66, stdev= 0.00, samples=1
iops : min= 1619, max= 1619, avg=1619.00, stdev= 0.00, samples=1
lat (msec) : 4=0.08%, 10=0.78%, 20=1.80%, 50=94.53%, 100=2.34%
lat (msec) : 750=0.47%
cpu : usr=0.52%, sys=13.86%, ctx=878, majf=0, minf=653
IO depths : 1=0.1%, 2=0.2%, 4=0.3%, 8=0.6%, 16=1.2%, 32=2.5%, >=64=95.1%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=1280,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=6624MiB/s (6945MB/s), 6624MiB/s-6624MiB/s (6945MB/s-6945MB/s), io=5120MiB (5369MB), run=773-773msec
Disk stats (read/write):
vda: ios=4218/0, merge=5/0, ticks=141059/0, in_queue=141060, util=86.65%
real 0m1.994s
user 0m0.196s
sys 0m0.910s
#VM INFNCLOUD
root@vnode-0:~/fiotest# time fio --name=seqread --rw=read --direct=1 --ioengine=libaio --bs=4m --size=5G --iodepth=64
seqread: (g=0): rw=read, bs=(R) 4096KiB-4096KiB, (W) 4096KiB-4096KiB, (T) 4096KiB-4096KiB, ioengine=libaio, iodepth=64
fio-3.16
Starting 1 process
Jobs: 1 (f=1): [R(1)][100.0%][r=629MiB/s][r=157 IOPS][eta 00m:00s]
seqread: (groupid=0, jobs=1): err= 0: pid=1667513: Fri Jan 13 15:05:44 2023
read: IOPS=210, BW=842MiB/s (882MB/s)(5120MiB/6084msec)
slat (usec): min=93, max=14181, avg=272.25, stdev=659.35
clat (msec): min=71, max=936, avg=300.69, stdev=114.30
lat (msec): min=72, max=936, avg=300.97, stdev=114.20
clat percentiles (msec):
| 1.00th=[ 144], 5.00th=[ 184], 10.00th=[ 201], 20.00th=[ 226],
| 30.00th=[ 243], 40.00th=[ 257], 50.00th=[ 271], 60.00th=[ 288],
| 70.00th=[ 313], 80.00th=[ 359], 90.00th=[ 439], 95.00th=[ 535],
| 99.00th=[ 768], 99.50th=[ 785], 99.90th=[ 927], 99.95th=[ 936],
| 99.99th=[ 936]
bw ( KiB/s): min=286720, max=997429, per=96.39%, avg=830639.08, stdev=187245.83, samples=12
iops : min= 70, max= 243, avg=202.75, stdev=45.67, samples=12
lat (msec) : 100=0.23%, 250=35.94%, 500=57.50%, 750=5.23%, 1000=1.09%
cpu : usr=0.25%, sys=5.15%, ctx=794, majf=0, minf=65547
IO depths : 1=0.1%, 2=0.2%, 4=0.3%, 8=0.6%, 16=1.2%, 32=2.5%, >=64=95.1%
submit : 0=0.0%, 4=100.0%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.0%, >=64=0.0%
complete : 0=0.0%, 4=99.9%, 8=0.0%, 16=0.0%, 32=0.0%, 64=0.1%, >=64=0.0%
issued rwts: total=1280,0,0,0 short=0,0,0,0 dropped=0,0,0,0
latency : target=0, window=0, percentile=100.00%, depth=64
Run status group 0 (all jobs):
READ: bw=842MiB/s (882MB/s), 842MiB/s-842MiB/s (882MB/s-882MB/s), io=5120MiB (5369MB), run=6084-6084msec
Disk stats (read/write):
vda: ios=5136/2, merge=0/5, ticks=1352505/565, in_queue=1342676, util=98.06%
real 0m6.641s
user 0m0.129s
sys 0m0.643s