{"id":351612,"date":"2024-05-20T21:51:27","date_gmt":"2024-05-20T21:51:27","guid":{"rendered":"http:\/\/savepearlharbor.com\/?p=351612"},"modified":"-0001-11-30T00:00:00","modified_gmt":"2024-05-20T21:51:27","slug":"","status":"publish","type":"post","link":"https:\/\/savepearlharbor.com\/?p=351612","title":{"rendered":"<span>\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u0441\u043e\u0444\u0442 nvme \u0440\u0435\u0439\u0434\u043e\u0432 \u043d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 mdadm, LVM, ZFS \u0438 LVM+vdo<\/span>"},"content":{"rendered":"<div><!--[--><!--]--><\/div>\n<div id=\"post-content-body\">\n<div>\n<div class=\"article-formatted-body article-formatted-body article-formatted-body_version-2\">\n<div xmlns=\"http:\/\/www.w3.org\/1999\/xhtml\">\n<p>UPD: 13.08.2023 &#8212; \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d\u044b \u0442\u0435\u0441\u0442\u044b \u043f\u043e mdadm, lvm, zfs<\/p>\n<p>\u0423 \u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438 \u0435\u0441\u0442\u044c \u0434\u0432\u0435 \u0446\u0435\u043b\u0438:<\/p>\n<ol>\n<li>\n<p>\u041f\u0440\u043e\u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u0442\u0440\u0451\u0445 \u0441\u0438\u0441\u0442\u0435\u043c \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u0435\u043d\u0438\u044f \u0444\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0445 \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432 \u0432 \u043e\u0434\u043d\u043e \u043b\u043e\u0433\u0438\u0447\u0435\u0441\u043a\u043e\u0435, \u043a\u0430\u043a \u0434\u043b\u044f \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u043e\u0433\u043e \u0434\u043e\u0441\u0442\u0443\u043f\u0430 \u043a \u043d\u0435\u043c\u0443, \u0442\u0430\u043a \u0438 \u0434\u043b\u044f \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u044f \u0432 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u043c\u0430\u0441\u0441\u0438\u0432\u0430 \u0434\u043b\u044f \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438.<\/p>\n<\/li>\n<li>\n<p>\u0421\u043e\u0437\u0434\u0430\u043d\u0438\u0435 \u0431\u044d\u043a\u044d\u043d\u0434\u0430 \u0434\u043b\u044f \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430 \u0432 \u0432\u0438\u0434\u0435 \u043e\u0434\u043d\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u043b\u0435\u0440\u043d\u043e\u0433\u043e \u0445\u0440\u0430\u043d\u0438\u043b\u0438\u0449\u0430.<\/p>\n<\/li>\n<\/ol>\n<p>\u0418\u043d\u044b\u043c\u0438 \u0441\u043b\u043e\u0432\u0430\u043c\u0438 \u0432\u044b\u0432\u043e\u0434\u044b \u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438 \u043d\u0435 \u043f\u0440\u0438\u043c\u0435\u043d\u0438\u043c\u044b, \u043a\u043e\u0433\u0434\u0430 \u0412\u0430\u043c \u043d\u0443\u0436\u043d\u043e:<\/p>\n<ol>\n<li>\n<p>\u0421\u0438\u043d\u0445\u0440\u043e\u043d\u043d\u043e\u0435 \u043e\u0442\u043a\u0430\u0437\u043e\u0443\u0441\u0442\u043e\u0439\u0447\u0438\u0432\u043e\u0435 \u0440\u0435\u0448\u0435\u043d\u0438\u0435<\/p>\n<\/li>\n<li>\n<p>\u041d\u0430\u0434\u0435\u0436\u043d\u043e\u0441\u0442\u044c >>> \u0441\u043a\u043e\u0440\u043e\u0441\u0442\u044c<\/p>\n<\/li>\n<li>\n<p>\u0414\u043e\u043b\u0433\u043e\u0432\u0440\u0435\u043c\u0435\u043d\u043d\u043e\u0435 \u0440\u0435\u0448\u0435\u043d\u0438\u0435, \u043a\u043e\u0442\u043e\u0440\u043e\u0435 \u043c\u043e\u0436\u043d\u043e \u043f\u043e\u0441\u0442\u0430\u0432\u0438\u0442\u044c \u0438 \u0437\u0430\u0431\u044b\u0442\u044c<\/p>\n<\/li>\n<\/ol>\n<p>\u0412 \u0441\u0432\u044f\u0437\u0438 \u0441\u043e \u0437\u043d\u0430\u0447\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u043c \u0443\u0434\u0435\u0448\u0435\u0432\u043b\u0435\u043d\u0438\u0435\u043c NVMe \u0434\u0438\u0441\u043a\u043e\u0432 \u043d\u0430 \u0440\u0430\u0437\u043b\u0438\u0447\u043d\u044b\u0445 \u0440\u0435\u0441\u0443\u0440\u0441\u0430\u0445 (\u0442\u0430\u043a\u0438\u0445 \u043a\u0430\u043a \u0430\u0432\u0438\u0442\u043e) \u043c\u043e\u0436\u043d\u043e \u043d\u0430\u0439\u0442\u0438 nvme \u0434\u0438\u0441\u043a\u0438 \u0434\u0435\u0448\u0435\u0432\u043b\u0435 \u0438\u0445 \u0430\u043d\u0430\u043b\u043e\u0433\u043e\u0432 \u043d\u0430 SATA \u0438\u043b\u0438 SAS \u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u0435 \u0441 \u0442\u0430\u043a\u0438\u043c \u0436\u0435 \u043e\u0431\u044a\u0451\u043c\u043e\u043c, \u043d\u043e \u043e\u0441\u0442\u0430\u0451\u0442\u0441\u044f \u0432\u043e\u043f\u0440\u043e\u0441 \u0432 \u0438\u0445 \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u0435\u043d\u0438\u0435 \u0432 \u043e\u0434\u0438\u043d \u043b\u043e\u0433\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0434\u0438\u0441\u043a. <\/p>\n<p>\u041a \u0441\u043e\u0436\u0430\u043b\u0435\u043d\u0438\u044e, \u0442\u0435\u043a\u0443\u0449\u0438\u0435 \u0440\u0435\u0439\u0434 \u043a\u043e\u043d\u0442\u0440\u043e\u043b\u043b\u0435\u0440\u044b \u0434\u043b\u044f NVMe \u0434\u0438\u0441\u043a\u043e\u0432 \u043d\u0435 \u043e\u0442\u043b\u0438\u0447\u0430\u044e\u0442\u0441\u044f \u043e\u0441\u043e\u0431\u043e\u0439 \u0434\u0435\u0448\u0435\u0432\u0438\u0437\u043d\u043e\u0439 \u0438\\\u0438\u043b\u0438 \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e\u043c \u0434\u043e\u0441\u0442\u0443\u043f\u043d\u044b\u0445 \u0440\u0430\u0437\u044a\u0451\u043c\u043e\u0432 \u0434\u043b\u044f \u0438\u0445 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u044f, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u043e\u0434\u043d\u0438\u043c \u0438\u0437 \u0432\u0430\u0440\u0438\u0430\u043d\u0442\u043e\u0432 \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435 \u043f\u0440\u043e\u0433\u0440\u0430\u043c\u043c\u043d\u043e\u0433\u043e \u0440\u0435\u0439\u0434\u0430. <\/p>\n<p> \u0414\u043b\u044f \u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u0432\u044b\u0431\u0440\u0430\u043d\u044b \u043f\u0440\u043e\u0433\u0440\u0430\u043c\u043c\u043d\u044b\u0435 \u0440\u0435\u0439\u0434\u044b:<\/p>\n<ul>\n<li>\n<p>mdadm (UPD: 13.08.2023, \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d\u044b \u0442\u0435\u0441\u0442\u044b \u0441 \u0437\u0430\u043a\u0440\u044b\u0442\u0438\u0435\u043c write hole &#8212; ppl &amp; log)<\/p>\n<\/li>\n<li>\n<p>lvm (\u0438 \u043d\u0430\u0434\u0441\u0442\u0440\u043e\u0439\u043a\u0430 VDO) &#8212; (UPD: 13.08.2023 \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d \u0442\u0435\u0441\u0442 \u0441 alloc=anywhere)<\/p>\n<\/li>\n<li>\n<p>ZFS (UPD: 13.08.2023 \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d\u044b \u0442\u0435\u0441\u0442\u044b \u0441 zpool blocksize = 128k)<\/p>\n<\/li>\n<\/ul>\n<p> \u041f\u0440\u043e \u043a\u0430\u0436\u0443\u044e \u0438\u0437 \u0432\u044b\u0431\u0440\u0430\u043d\u043d\u044b\u0445 \u0441\u0438\u0441\u0442\u0435\u043c \u043c\u043e\u0436\u043d\u043e \u043f\u043e\u0434\u0440\u043e\u0431\u043d\u0435\u0435 \u043f\u043e\u0447\u0438\u0442\u0430\u0442\u044c:<\/p>\n<ul>\n<li>\n<p><a href=\"https:\/\/habr.com\/ru\/articles\/504692\/\" rel=\"noopener noreferrer nofollow\">\u0422\u0443\u0442 \u043f\u0440\u043e ZFS<\/a> <\/p>\n<\/li>\n<li>\n<p><a href=\"https:\/\/www.opennet.ru\/man.shtml?topic=mdadm&amp;category=8&amp;russian=2\" rel=\"noopener noreferrer nofollow\">\u0422\u0443\u0442 \u043f\u0440\u043e mdadm<\/a> <\/p>\n<\/li>\n<li>\n<p><a href=\"http:\/\/xgu.ru\/wiki\/LVM\" rel=\"noopener noreferrer nofollow\">\u0422\u0443\u0442 \u043f\u0440\u043e LVM<\/a> \u0438 <a href=\"https:\/\/access.redhat.com\/documentation\/ru-ru\/red_hat_enterprise_linux\/8\/html\/deduplicating_and_compressing_logical_volumes_on_rhel\/introduction-to-vdo-on-lvm_deduplicating-and-compressing-logical-volumes-on-rhel\" rel=\"noopener noreferrer nofollow\">\u0442\u0443\u0442 \u043f\u0440\u043e \u043d\u0430\u0434\u0441\u0442\u0440\u043e\u0439\u043a\u0443 VDO<\/a><\/p>\n<\/li>\n<\/ul>\n<p>\u0414\u043b\u044f \u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u043e\u0433\u043e \u043c\u0430\u0441\u0441\u0438\u0432\u0430 \u0431\u044b\u043b\u0438 \u0432\u044b\u0431\u0440\u0430\u043d\u044b \u0442\u0435\u0441\u0442\u044b \u043e\u043f\u0438\u0441\u0430\u043d\u043d\u044b\u0435 <a href=\"https:\/\/habr.com\/ru\/companies\/flant\/articles\/664150\/\" rel=\"noopener noreferrer nofollow\">\u0442\u0443\u0442<\/a> \u0438 <a href=\"https:\/\/github.com\/vitalif\/vitastor\/blob\/master\/docs\/performance\/understanding.en.md#%D0%BA%D0%BE%D0%BC%D0%B0%D0%BD%D0%B4%D1%8B-fio\" rel=\"noopener noreferrer nofollow\">\u0442\u0443\u0442<\/a>.<\/p>\n<p>\u0414\u043b\u044f \u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u043c\u0430\u0441\u0441\u0438\u0432\u0430 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u0431\u0443\u0434\u0435\u0442 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c\u0441\u044f HCIbench, \u0430 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b \u0432\u0437\u044f\u0442\u044b \u0438\u0437 <a href=\"https:\/\/cloud.redhat.com\/blog\/software-design-pattern-benchmarking\" rel=\"noopener noreferrer nofollow\">\u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0435\u0439 \u0441\u0442\u0430\u0442\u044c\u0438<\/a>.<\/p>\n<h2>\u0422\u0435\u0441\u0442\u043e\u0432\u044b\u0439 \u0441\u0442\u0435\u043d\u0434<\/h2>\n<p><strong>\u0421\u0435\u0440\u0432\u0435\u0440 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438:<\/strong><\/p>\n<ul>\n<li>\n<p>Motherboard: Supermicro H11SSL-i<\/p>\n<\/li>\n<li>\n<p>CPU: EPYC 7302<\/p>\n<\/li>\n<li>\n<p>RAM: 4x64GB Micron 2933MHz<\/p>\n<\/li>\n<li>\n<p>\u0421\u0435\u0442\u044c: 40GbE ConnectX-3 Pro, 10GbE\/25GbE ConnectX-4 LN EN<\/p>\n<\/li>\n<li>\n<p>\u041e\u0421: ESXi 7U3 build 20036586<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u0421\u0435\u0440\u0432\u0435\u0440 \u0445\u0440\u0430\u043d\u0435\u043d\u0438\u044f<\/strong> (\u0432 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0435\u043c \u0433\u0438\u0431\u0440\u0438\u0434\u043d\u044b\u0439 \u0441\u0435\u0440\u0432\u0435\u0440 \u0441 PCIe Passthrough \u0434\u043b\u044f \u0442\u0435\u0441\u0442\u043e\u0432 \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u0445\u0440\u0430\u043d\u0438\u043b\u0438\u0449\u0430):<\/p>\n<ul>\n<li>\n<p>Motherboard: Tyan S8030 (ver 1GbE)<\/p>\n<\/li>\n<li>\n<p>CPU: EPYC 7302<\/p>\n<\/li>\n<li>\n<p>RAM: 4x64GB Micron 2933MHz<\/p>\n<\/li>\n<li>\n<p>\u0421\u0435\u0442\u044c: 40GbE ConnectX-3 Pro x2<\/p>\n<\/li>\n<li>\n<p>\u0414\u0438\u0441\u043a\u0438 1: 6 x PM9A3 1.92TB, \u0444\u043e\u0440\u043c\u0430\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u044b \u0432 512n<\/p>\n<\/li>\n<li>\n<p>\u0414\u0438\u0441\u043a\u0438 2: 1 x PM1725 3.2\u0422\u0411 (\u043d\u0435 \u0443\u0447\u0430\u0441\u0442\u0432\u0443\u0435\u0442 \u0432 \u043a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u0438 \u0440\u0435\u0439\u0434\u0430, \u043d\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442\u0441\u044f \u0434\u043b\u044f \u0441\u0440\u0430\u0432\u043d\u0435\u043d\u0438\u044f)<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u0421\u043e\u0444\u0442 \u0434\u043b\u044f \u0442\u0435\u0441\u0442\u043e\u0432 \u043d\u0430 \u0444\u0438\u0437\u0438\u043a\u0435:<\/strong><\/p>\n<ul>\n<li>\n<p>Debian 12, \u044f\u0434\u0440\u043e 6.1.0-10<\/p>\n<\/li>\n<li>\n<p>FIO: 3.33-3<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u0421\u043e\u0444\u0442 \u0434\u043b\u044f \u0442\u0435\u0441\u0442\u043e\u0432 \u043f\u043e\u0432\u0435\u0440\u0445 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0438\u0430\u0446\u0438\u0438 (to-be)<\/strong><\/p>\n<ul>\n<li>\n<p>\u041e\u0421: ESXi 7U3 build<\/p>\n<\/li>\n<li>\n<p>\u0412\u041c: Ubuntu 22.04.03, <a href=\"https:\/\/9to5linux.com\/ubuntu-22-04-lts-is-now-powered-by-linux-kernel-6-2-from-ubuntu-23-04\" rel=\"noopener noreferrer nofollow\">\u044f\u0434\u0440\u043e 6.2<\/a><\/p>\n<\/li>\n<li>\n<p>FIO: 3.28-1<\/p>\n<\/li>\n<\/ul>\n<details class=\"spoiler\">\n<summary>\u0421\u043f\u0438\u0441\u043e\u043a \u0442\u0435\u0441\u0442\u043e\u0432 FIO \u043f\u0440\u0438\u0432\u0435\u0434\u0451\u043d \u043d\u0438\u0436\u0435:<\/summary>\n<div class=\"spoiler__content\">\n<p><code>fio -name=seq_mbs_write_T1Q1N1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4M -iodepth=32 -rw=write -runtime=60 -filename=\/dev\/nvme1n1 &amp;&amp; sleep 10 &amp;&amp; fio -name=seq_mbs_read_T1Q1N1 -ioengine=libaio -direct=1 -invalidate=1  -bs=4M -iodepth=32 -rw=read -runtime=60 -filename=\/dev\/nvme1n1 &amp;&amp; sleep 10 &amp;&amp; fio -name=rand_iops_write_T1Q128N1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=128 -numjobs=4 -group_reporting -rw=randwrite -runtime=60 -randrepeat=0 -filename=\/dev\/nvme1n1 &amp;&amp; sleep 10 &amp;&amp; fio -name=rand_iops_read_T1Q128N1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=128 -numjobs=4 -group_reporting -rw=randread -runtime=60 -randrepeat=0 -filename=\/dev\/nvme1n1 &amp;&amp; sleep 10 &amp;&amp; fio -name=rand_iops_read_T1Q1N1 -randrepeat=0 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=1 -rw=randread -runtime=60 -filename=\/dev\/nvme1n1 &amp;&amp; sleep 10 &amp;&amp; fio -name=rand_iops_write_T1Q1N1 -randrepeat=0 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=1 -fsync=1 -rw=randwrite -runtime=60 -filename=\/dev\/nvme1n1<\/code><\/p>\n<\/div>\n<\/details>\n<p>\u0420\u0430\u0441\u0441\u043c\u0430\u0442\u0440\u0438\u0432\u0430\u0442\u044c\u0441\u044f \u0431\u0443\u0434\u0435\u0442 2 \u0441\u0446\u0435\u043d\u0430\u0440\u0438\u044f:<\/p>\n<ol>\n<li>\n<p>Raid 0 &#8212; (\u0432\u0441\u0435 \u0436\u0435 \u0434\u0435\u043b\u0430\u044e\u0442 \u0431\u044d\u043a\u0430\u043f\u044b) \u043a\u0430\u043a \u043a\u0440\u0430\u0439\u043d\u0438\u0439 \u0441\u0446\u0435\u043d\u0430\u0440\u0438\u0439 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0433\u043e \u0440\u0435\u0448\u0435\u043d\u0438\u044f, \u043a\u043e\u0433\u0434\u0430 \u0437\u0430 \u0441\u043e\u0445\u0440\u0430\u043d\u043d\u043e\u0441\u0442\u044c \u0434\u0430\u043d\u043d\u044b\u0445 (\u043d\u0435 \u0434\u043e\u0441\u0442\u0443\u043f\u043d\u043e\u0441\u0442\u044c) \u043e\u0442\u0432\u0435\u0447\u0430\u0435\u0442 \u0438\u043b\u0438 \u0440\u0435\u043f\u043b\u0438\u043a\u0430\u0446\u0438\u044f \u0438\u043b\u0438 \u0431\u044d\u043a\u0430\u043f\u044b.<\/p>\n<\/li>\n<li>\n<p>Raid 5 &#8212; \u043a\u0430\u043a \u0445\u043e\u0440\u043e\u0448\u0430\u044f \u0441\u0445\u0435\u043c\u0430 \u0434\u043b\u044f 6 \u0434\u0438\u0441\u043a\u043e\u0432<\/p>\n<\/li>\n<\/ol>\n<p>\u0412 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u0434\u043e\u0441\u0442\u0443\u043f\u0430 \u0431\u0443\u0434\u0435\u0442 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c\u0441\u044f iSER \u0438 NVMe-oF<\/p>\n<p>\u041d\u0438\u0436\u0435 \u043f\u0440\u0438\u0432\u0435\u0434\u0435\u043d\u0430 \u0441\u0432\u043e\u0434\u043d\u0430\u044f \u0442\u0430\u0431\u043b\u0438\u0446\u0430 \u043f\u043e \u0434\u0438\u0441\u043a\u0430\u043c \u0438 \u0433\u0440\u0430\u0444\u0438\u043a \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438: <\/p>\n<details class=\"spoiler\">\n<summary>\u0421\u0432\u043e\u0434\u043d\u0430\u044f \u0442\u0430\u0431\u043b\u0438\u0446\u0430 \u043f\u043e \u0434\u0438\u0441\u043a\u0430\u043c \u0432 \u0441\u0438\u0441\u0442\u0435\u043c\u0435<\/summary>\n<div class=\"spoiler__content\">\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">(MB\/s) NVMe#<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"left\">NVMe0 (PM1725)<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"left\">NVMe1 (PM9A3)<\/p>\n<\/td>\n<td>\n<p align=\"left\">NVMe2 (PM9A3)<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">1 473.50<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">2 810.50<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 809.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">5 068.50<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">5 882.50<\/p>\n<\/td>\n<td>\n<p align=\"right\">5 891.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">327.00<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">2 792.00<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 787.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">3 957.00<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">4 620.00<\/p>\n<\/td>\n<td>\n<p align=\"right\">4 619.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">163.50<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">233.00<\/p>\n<\/td>\n<td>\n<p align=\"right\">233.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">44.80<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">90.30<\/p>\n<\/td>\n<td>\n<p align=\"right\">90.40<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">(MB\/s) NVMe#<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"left\">NVMe3 (PM9A3)<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"left\">NVMe4 (PM9A3)<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"left\">NVMe5 (PM9A3)<\/p>\n<\/td>\n<td>\n<p align=\"left\">NVMe6 (PM9A3)<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">2 807.50<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">2 811.00<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">2 809.50<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 810.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">5 884.00<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">5 915.00<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">5 904.50<\/p>\n<\/td>\n<td>\n<p align=\"right\">5 885.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">2 793.00<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">2 789.00<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">2 779.00<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 796.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">4 615.00<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">4 537.50<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">4 526.50<\/p>\n<\/td>\n<td>\n<p align=\"right\">4 619.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">231.50<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">230.00<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">231.00<\/p>\n<\/td>\n<td>\n<p align=\"right\">232.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">90.65<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">90.15<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">90.35<\/p>\n<\/td>\n<td>\n<p align=\"right\">90.70<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/details>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/ccf\/bc3\/ec2\/ccfbc3ec23c5b6940c0d462196f8973d.png\" alt=\"\u0421\u0432\u043e\u0434\u043d\u044b\u0439 \u0433\u0440\u0430\u0444\u0438\u043a \u043f\u043e \u0434\u0438\u0441\u043a\u0430\u043c\" title=\"\u0421\u0432\u043e\u0434\u043d\u044b\u0439 \u0433\u0440\u0430\u0444\u0438\u043a \u043f\u043e \u0434\u0438\u0441\u043a\u0430\u043c\" width=\"2694\" height=\"1255\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/ccf\/bc3\/ec2\/ccfbc3ec23c5b6940c0d462196f8973d.png\"\/><\/p>\n<div><figcaption>\u0421\u0432\u043e\u0434\u043d\u044b\u0439 \u0433\u0440\u0430\u0444\u0438\u043a \u043f\u043e \u0434\u0438\u0441\u043a\u0430\u043c<\/figcaption><\/div>\n<\/figure>\n<p><a href=\"https:\/\/drive.google.com\/file\/d\/1BZWyqOeO_TfrG7vdsnGrhb2TV-mBqzXZ\/view?usp=sharing\" rel=\"noopener noreferrer nofollow\">\u0421\u0441\u044b\u043b\u043a\u0430 \u043d\u0430 excel \u0444\u0430\u0439\u043b \u0441\u043e \u0432\u0441\u0435\u043c\u0438 \u0434\u0430\u043d\u043d\u044b\u043c\u0438<\/a>.<\/p>\n<h2>MDADM<\/h2>\n<p>\u0421\u043e\u0437\u0434\u0430\u043d\u0438\u0435 mdadm \u0434\u043e\u0441\u0442\u0430\u0442\u043e\u0447\u043d\u043e \u0442\u0438\u043f\u0438\u0447\u043d\u043e:<br \/> mdadm &#8212;create &#8212;verbose \/dev\/md0 &#8212;level=0 &#8212;raid-devices=6 \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 \/dev\/nvme6n1<\/p>\n<p>\u0414\u043b\u044f 5-\u043e\u0433\u043e \u0440\u0435\u0439\u0434\u0430  (\u041f\u0435\u0440\u0435\u0434 \u043d\u0430\u0447\u0430\u043b\u043e\u043c \u0442\u0435\u0441\u0442\u043e\u0432 \u0431\u044b\u043b\u043e \u0432\u044b\u0436\u0434\u0430\u043d\u043e 3 \u0447\u0430\u0441\u0430, \u043f\u043e\u043a\u0430 \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u043b\u0441\u044f \u0440\u0435\u0439\u0434)<br \/>mdadm &#8212;create &#8212;verbose \/dev\/md5 &#8212;level=5 &#8212;raid-devices=6 \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 \/dev\/nvme6n1<\/p>\n<p>\u0414\u043e\u043f\u043e\u043b\u043d\u0438\u0442<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\"><strong>RAID 0<\/strong>\u00a0<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"center\">mdadm-local-raid0<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">11 980.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">20 275.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">11 827.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 817 002.93 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">11 622.40 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 771 891.56 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">172.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">42 160.66 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">99.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">24 269.23 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\"><strong>RAID 5<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"center\">mdadm-local-raid5<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">1 398.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">20 172.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">260.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">63 857.62 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">10 444.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 503 104.94 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">35.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">8 609.11 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">90.35 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">22 084.20 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\"><strong>RAID 5<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"left\">mdadm-local-raid5-ppl<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">629.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">19 558.40 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">239.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">58 477.84 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">10 444.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 502 086.56 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">36.55 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">8 927.77 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">107.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">26 229.00 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\"><strong>RAID 5<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"left\">mdadm-local-raid5-log<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">725.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">20 172.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">208.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">50 794.78 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">10 547.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 516 402.47 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">33.10 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">8 106.66 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">90.15 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">22 037.52 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<h2>LVM<\/h2>\n<p>LVM &#8212; \u0434\u043b\u044f \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e\u0442\u0441\u044f \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b<br \/> lvcreate -i6 -I64 &#8212;type striped  -l 100%VG -n lvm_stripe stripe<br \/> lvcreate -i5 -I64 &#8212;type raid5 -l 100%VG -n lvm_raid5 raid<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\"><strong>RAID0<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"center\">lvm-local-raid0<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">10 905.60 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">20 480.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">14 080.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">3 359 597.24 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">14 080.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">3 358 893.54 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">221.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">53 959.59 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">81.05 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">19 793.28 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\"><strong>RAID5 <\/strong><\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"center\">lvm-local-raid5 alloc default<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">1427.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">18 995.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">202.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">49 445.24 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">9816.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 407 383.56 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">31.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">49 445.24 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">87.25 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">21 266.52 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\"><strong>RAID5<\/strong> <\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"center\">lvm-local-raid5-alloc-normal<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">1 352.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">19 302.40 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">203.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">49 445.235 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">9 858.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 407 383.56 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">31.40 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">7 672.28 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">87.10 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">21 266.52 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u00a0<strong>RAID5<\/strong> <\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"left\">lvm-local-raid5-alloc-anywhere<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">1 340.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">19 046.4 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">202.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">49 372.28 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">9 826.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 400 206.15 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">31.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">7 620.88 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">87.05 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">21 248.33 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<h2>LVM+VDO<\/h2>\n<p>\u0422\u0430\u043a \u043a\u0430\u043a \u043d\u0430 Debian vdo \u043f\u0440\u043e\u0441\u0442\u043e \u043d\u0435\u0442, \u0434\u0430\u0436\u0435 \u0432 \u0432\u0438\u0434\u0435 \u043f\u0430\u043a\u0435\u0442\u0430 \u043e\u043d \u0434\u043e\u0431\u0430\u0432\u043b\u044f\u043b\u0441\u044f \u043a\u0430\u043a \u043c\u043e\u0434\u0443\u043b\u044c \u043a \u044f\u0434\u0440\u0443 \u0441 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435\u043c<a href=\"https:\/\/github.com\/dm-vdo\/vdo\/issues\/64\" rel=\"noopener noreferrer nofollow\"> \u044d\u0442\u043e\u0433\u043e \u0440\u0443\u043a\u043e\u0432\u043e\u0434\u0441\u0442\u0432\u0430<\/a>), \u0442\u0435\u0441\u0442\u044b \u043f\u0440\u043e\u0432\u043e\u0434\u0438\u043b\u0438\u0441\u044c \u0441 \u0432\u043a\u043b\u044e\u0447\u0451\u043d\u043d\u044b\u043c \u0434\u0435\u0434\u0443\u043f\u043e\u043c, \u0442\u0430\u043a \u043a\u0430\u043a \u044d\u0442\u043e \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440 \u043f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e.<br \/>\u041f\u0440\u0438 \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u0438 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043b\u0438\u0441\u044c \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b, \u0442\u0430\u043a \u043a\u0430\u043a \u043f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e VDO \u043d\u0435 \u043f\u043e\u0434\u0434\u0435\u0440\u0436\u0438\u0432\u0430\u0435\u0442 raid \u043a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u044e, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u043f\u0443\u043b \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u043b\u0441\u044f \u0447\u0435\u0440\u0435\u0437 lvconvert.<\/p>\n<pre><code>lvcreate -i6 -I64 --type striped -l 100%VG -n stripe-vdo lvm lvconvert --type vdo-pool --virtualsize 8TB \/dev\/lvm\/stripe-vdo<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u00a0<strong>RAID0<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"center\">lvm-vdo-local-raid0<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">735.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">2 279.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">415.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">101 344.40 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">901.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">220 089.32 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">6.14 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">1 534.41 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">39.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">9 520.67 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<pre><code>lvcreate -i6 -I64 --type raid5 -l 100%VG -n lvm_raid5 raid5 lvconvert --type vdo-pool --virtualsize 8TB \/dev\/lvm\/raid5-vdo<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p>\u00a0<strong>RAID5<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">lvm-vdo-local-raid5<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p>\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">230.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">2 406.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">79.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">19 388.92 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">1 536.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">374 751.79 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">1.59 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">398.57 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">34.25 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">8 329.27 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<h2>ZFS<\/h2>\n<p>\u041f\u043e\u0441\u043b\u0435 \u043a\u0430\u0436\u0434\u043e\u0439 \u0433\u0440\u0443\u043f\u043f\u044b \u0442\u0435\u0441\u0442\u043e\u0432 \u0441\u0435\u0440\u0432\u0435\u0440 \u043f\u0435\u0440\u0435\u0437\u0430\u043f\u0443\u0441\u043a\u0430\u043b\u0441\u044f, \u0442.\u0435., \u043a \u043f\u0440\u0438\u043c\u0435\u0440\u0443, \u043f\u043e\u0441\u043b\u0435 \u0442\u0435\u0441\u0442\u043e\u0432 stripe 16k \u0438 \u043f\u0435\u0440\u0435\u0434 \u0442\u0435\u0441\u0442\u0430\u043c\u0438 \u043d\u0430 8k, \u0447\u0442\u043e\u0431\u044b \u043e\u0434\u043d\u043e\u0437\u043d\u0430\u0447\u043d\u043e \u043e\u0447\u0438\u0441\u0442\u0438\u0442\u044c ARC.<br \/>\u041a\u043e\u0433\u0434\u0430 \u0432\u044b\u0439\u0434\u0435\u0442 ZFS 3.0 \u0441 \u043f\u0430\u0442\u0447\u0435\u043c O_DIRECT \u0434\u043b\u044f nvme, \u044f \u0440\u0430\u0441\u0448\u0438\u0440\u044e \u0435\u0451, \u0441\u0435\u0439\u0447\u0430\u0441 \u0436\u0435 ZFS \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<p><strong>v1<\/strong> &#8212; \u0431\u043b\u043e\u043a <strong>16k<\/strong> (\u043d\u043e\u0432\u044b\u0435 \u0434\u0435\u0444\u043e\u043b\u0442\u043d\u044b\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b), \u043a\u043e\u043c\u043f\u0440\u0435\u0441\u0441\u0438\u044f lz4 (\u043d\u0435 \u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u0432\u043b\u0438\u044f\u043d\u0438\u044f \u043d\u0430 IOPS, \u043f\u043e \u043c\u043d\u043e\u0436\u0435\u0441\u0442\u0432\u0435\u043d\u043d\u044b\u043c \u0437\u0430\u044f\u0432\u043b\u0435\u043d\u0438\u044f\u043c \u0438 \u043f\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u043e\u043c \u0442\u0435\u0441\u0442\u043e\u0432), \u0434\u0435\u0434\u0443\u043f \u0432\u044b\u043a\u043b\u044e\u0447\u0435\u043d:<\/p>\n<pre><code>zpool create -o ashift=12 -O compression=lz4 -O atime=off -O recordsize=16k nvme \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 \/dev\/nvme6n1 zfs create -s -V 10T -o volblocksize=16k -o compression=lz4 nvme\/iser<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u00a0<strong>Stripe<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"196\" width=\"196\">\n<p align=\"center\">zfs-local-stripe-16k<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"196\" width=\"196\">\n<p align=\"right\">2 826.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"196\" width=\"196\">\n<p align=\"right\">5 955.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"196\" width=\"196\">\n<p align=\"right\">176.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">43 120.67 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"196\" width=\"196\">\n<p align=\"right\">827.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">201 500.96 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"196\" width=\"196\">\n<p align=\"right\">22.85 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">5 580.61 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"196\" width=\"196\">\n<p align=\"right\">66.35 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">16 209.50 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u00a0<strong>RaidZ<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"197\" width=\"197\">\n<p align=\"center\">zfs-local-raidZ-16k<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"197\" width=\"197\">\n<p align=\"right\">1 701.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"197\" width=\"197\">\n<p align=\"right\">3 967.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"197\" width=\"197\">\n<p align=\"right\">154.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">37 638.80 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"197\" width=\"197\">\n<p align=\"right\">427.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">104 231.22 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"197\" width=\"197\">\n<p align=\"right\">11.30 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 757.17 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"197\" width=\"197\">\n<p align=\"right\">28.75 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">7 017.52 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<hr\/>\n<p><strong>2<\/strong> &#8212; \u0431\u043b\u043e\u043a <strong>8\u043a<\/strong> (\u0441\u0442\u0430\u0440\u044b\u0435 \u0434\u0435\u0444\u043e\u043b\u0442\u043d\u044b\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b)<\/p>\n<pre><code>zpool create -o ashift=12 -O compression=lz4 -O atime=off -O recordsize=8k nvme \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 \/dev\/nvme6n1 zfs create -s -V 10T -o volblocksize=8k -o compression=lz4 nvme\/iser<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u00a0<strong>Stripe<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"184\" width=\"184\">\n<p align=\"center\">zfs-local-stripe-8k<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"184\" width=\"184\">\n<p align=\"right\">1 513.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"184\" width=\"184\">\n<p align=\"right\">3 352.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"184\" width=\"184\">\n<p align=\"right\">155.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">37 860.44 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"184\" width=\"184\">\n<p align=\"right\">710.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">172 631.13 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"184\" width=\"184\">\n<p align=\"right\">21.75 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">5 311.53 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"184\" width=\"184\">\n<p align=\"right\">65.55 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">16 005.12 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p>\u00a0<strong>RaidZ<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"center\">zfs-local-raidZ-8k<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"right\">1 035.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"right\">2 877.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"right\">153.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">37 383.61 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"right\">427.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">104 453.05 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"right\">10.90 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 657.64 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"right\">23.45 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">5 716.16 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<hr\/>\n<p> <strong>v3 &#8212;<\/strong> \u0431\u043b\u043e\u043a <strong>64\u043a<\/strong> (\u043a\u0430\u043a \u043f\u0440\u043e\u0441\u0442\u043e \u0431\u043e\u043b\u044c\u0448\u043e\u0439 \u0431\u043b\u043e\u043a)<\/p>\n<pre><code>zpool create -o ashift=12 -O compression=lz4 -O atime=off -O recordsize=64k nvme \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 \/dev\/nvme6n1 zfs create -s -V 10T -o volblocksize=64k -o compression=lz4 nvme\/iser<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p><strong>Stripe<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"center\">zfs-local-stripe-64k<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">4 778.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">5 121.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">183.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">44 785.15 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">1 062.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">259 321.58 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">22.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\"> 5 559.02 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">65.85 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">16 085.35 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u00a0<strong>RaidZ<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"center\">zfs-local-raidZ-64k<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">3 758.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">4 269.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">156.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">38 111.73 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">1 086.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">265 232.02 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">17.90 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">4 372.47 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">73.15 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">17 855.23 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p><strong>v4<\/strong> &#8212; \u0412\u0441\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b \u043f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e, zpool recordsize = <strong>128k<\/strong>, zvol blocksize=<strong>8k<\/strong><\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\"><strong>Stripe<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"left\">zfs-local-stripe-default<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">1 568.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">3 494.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">166.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">40 598.80 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">741.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">180 330.46 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">17.40 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">4 241.05 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">45.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">11 033.33 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u00a0<strong>RaidZ<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"left\">zfs-local-raidZ-default<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">986.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">3 538.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">145.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">35 350.86 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">715.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">173 890.22 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">17.30 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">4 221.97 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">68.75 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">16 777.54 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p><strong>v5<\/strong> &#8212; zpool recordsize = <strong>128k<\/strong>, zvol blocksize=<strong>16k<\/strong><\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\"><strong>Stripe<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"left\">zfs-local-stripe-128k-16k<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">3 037.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">5 812.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">172.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">42 164.89 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">926.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">225 542.80 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">22.55 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">5 498.39 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"312\" width=\"312\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">62.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">15 186.53 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u00a0<strong>RaidZ<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"left\">zfs-local-raidZ-128k-16k<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">1 693.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">5 858.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">138.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">33 766.93 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">828.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">201 839.62 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">17.55 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">4 286.75 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"313\" width=\"313\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">66.60 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">16 262.31 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p>\u0422\u0430\u043a\u0436\u0435 \u0434\u043b\u044f ZFS \u0434\u043e\u043f\u043e\u043b\u043d\u0438\u0442\u0435\u043b\u044c\u043d\u043e \u0431\u044b\u043b \u0441\u0434\u0435\u043b\u0430\u043d \u0442\u0435\u0441\u0442 \u043d\u0430 64 \u043f\u043e\u0442\u043e\u043a\u0430 \u0432 \u0432\u0438\u0434\u0435 64 \u0440\u0430\u0437\u043d\u044b\u0445 \u0444\u0430\u0439\u043b\u043e\u0432 \u043d\u0430 \u043f\u0443\u043b\u0435:<\/p>\n<details class=\"spoiler\">\n<summary>recordsize=8k<\/summary>\n<div class=\"spoiler__content\">\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td>\n<p align=\"left\">\u00a0<strong>Stripe<\/strong><\/p>\n<\/td>\n<td>\n<p align=\"left\">zfs64-local-stripe-8k<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">1 242.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">3 123.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">275.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">1 436.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">196.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">1 439.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td>\n<p align=\"left\">\u00a0<strong>RaidZ<\/strong><\/p>\n<\/td>\n<td>\n<p align=\"left\">zfs64-local-raidZ-8k<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">981.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">3 756.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">187.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">1 192.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">138.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">1 209.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/details>\n<details class=\"spoiler\">\n<summary>recordsize=16k<\/summary>\n<div class=\"spoiler__content\">\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"320\" width=\"320\">\n<p align=\"left\">\u00a0<strong>Stripe<\/strong><\/p>\n<\/td>\n<td>\n<p align=\"center\">zfs64-local-stripe-16k<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"320\" width=\"320\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 346.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"320\" width=\"320\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">4 508.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"320\" width=\"320\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">296.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"320\" width=\"320\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">1 248.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"320\" width=\"320\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">201.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"320\" width=\"320\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">1 228.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td>\n<p><strong>RaidZ<\/strong>\u00a0<\/p>\n<\/td>\n<td>\n<p align=\"center\">zfs64-local-raidZ-16k<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">1 579.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">5 044.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">186.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">1 000.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">138.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">1 003.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/details>\n<details class=\"spoiler\">\n<summary>recordsize=64k<\/summary>\n<div class=\"spoiler__content\">\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td>\n<p align=\"left\">\u00a0<strong>Stripe<\/strong><\/p>\n<\/td>\n<td>\n<p align=\"center\">zfs64-local-stripe-64k<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">5 252.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">6 260.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">241.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">506.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">151.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">562.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td>\n<p align=\"left\">\u00a0<strong>RaidZ<\/strong><\/p>\n<\/td>\n<td>\n<p align=\"center\">zfs64-local-raidZ-64k<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\"> 3 658.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">5 737.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">177.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">460.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">128.00 MB\/s<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">504.50 MB\/s<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/details>\n<p> \u0421\u0432\u043e\u0434\u043d\u044b\u0435 \u0433\u0440\u0430\u0444\u0438\u043a\u0438 \u043f\u043e \u0442\u0435\u0441\u0442\u0430\u043c:<\/p>\n<details class=\"spoiler\">\n<summary>\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/summary>\n<div class=\"spoiler__content\">\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/7b7\/ab1\/6fd\/7b7ab16fdf96905bb76d83ecc60adb54.png\" alt=\"\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32\" title=\"\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32\" width=\"4065\" height=\"1205\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/7b7\/ab1\/6fd\/7b7ab16fdf96905bb76d83ecc60adb54.png\"\/><\/p>\n<div><figcaption>\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/figcaption><\/div>\n<\/figure>\n<\/div>\n<\/details>\n<details class=\"spoiler\">\n<summary>\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/summary>\n<div class=\"spoiler__content\">\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/390\/6bf\/abe\/3906bfabe6cb95850788ec742836d155.png\" alt=\"\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32\" title=\"\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32\" width=\"4533\" height=\"1246\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/390\/6bf\/abe\/3906bfabe6cb95850788ec742836d155.png\"\/><\/p>\n<div><figcaption>\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/figcaption><\/div>\n<\/figure>\n<\/div>\n<\/details>\n<details class=\"spoiler\">\n<summary>\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/summary>\n<div class=\"spoiler__content\">\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/abf\/759\/ef3\/abf759ef3f59205219a902fd8a68df13.png\" alt=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16\" title=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16\" width=\"4071\" height=\"1246\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/abf\/759\/ef3\/abf759ef3f59205219a902fd8a68df13.png\"\/><\/p>\n<div><figcaption>\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/figcaption><\/div>\n<\/figure>\n<\/div>\n<\/details>\n<details class=\"spoiler\">\n<summary>\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/summary>\n<div class=\"spoiler__content\">\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/890\/6fe\/abd\/8906feabd90c558b4480a808091bf4ea.png\" alt=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16\" title=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16\" width=\"4472\" height=\"1246\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/890\/6fe\/abd\/8906feabd90c558b4480a808091bf4ea.png\"\/><\/p>\n<div><figcaption>\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/figcaption><\/div>\n<\/figure>\n<\/div>\n<\/details>\n<details class=\"spoiler\">\n<summary>\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/summary>\n<div class=\"spoiler__content\">\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/6ca\/7dc\/9c6\/6ca7dc9c63c5452b491a13a371c3450e.png\" alt=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1\" title=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1\" width=\"4041\" height=\"1246\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/6ca\/7dc\/9c6\/6ca7dc9c63c5452b491a13a371c3450e.png\"\/><\/p>\n<div><figcaption>\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/figcaption><\/div>\n<\/figure>\n<p>\u0438 \u0431\u0435\u0437 ZFS64<\/p>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/ebe\/3fd\/3d3\/ebe3fd3d302bb25084122a7e9d6709a1.png\" alt=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1 \u0431\u0435\u0437 zfs64\" title=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1 \u0431\u0435\u0437 zfs64\" width=\"4041\" height=\"1245\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/ebe\/3fd\/3d3\/ebe3fd3d302bb25084122a7e9d6709a1.png\"\/><\/p>\n<div><figcaption>\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1 \u0431\u0435\u0437 zfs64<\/figcaption><\/div>\n<\/figure>\n<p>\u044c\u0432\u0444<\/p>\n<\/div>\n<\/details>\n<details class=\"spoiler\">\n<summary>\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/summary>\n<div class=\"spoiler__content\">\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/95a\/eb2\/81e\/95aeb281ef5c97bd1e72c63156338a34.png\" alt=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1\" title=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1\" width=\"4461\" height=\"1246\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/95a\/eb2\/81e\/95aeb281ef5c97bd1e72c63156338a34.png\"\/><\/p>\n<div><figcaption>\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/figcaption><\/div>\n<\/figure>\n<p>\u0418 \u0431\u0435\u0437 ZFS64<\/p>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/9f6\/d28\/4af\/9f6d284afa011092a67375d70ae8379d.png\" alt=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1 \u0431\u0435\u0437 ZFS64\" title=\"\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1 \u0431\u0435\u0437 ZFS64\" width=\"3842\" height=\"1246\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/9f6\/d28\/4af\/9f6d284afa011092a67375d70ae8379d.png\"\/><\/p>\n<div><figcaption>\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1 \u0431\u0435\u0437 ZFS64<\/figcaption><\/div>\n<\/figure>\n<\/div>\n<\/details>\n<h2>\u0412\u044b\u0432\u043e\u0434<\/h2>\n<p>mdadm \u0438 stripe LVM \u0438\u043c\u0435\u044e\u0442 +- \u043e\u0434\u0438\u043d\u0430\u043a\u043e\u0432\u0443\u044e \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c, \u043f\u0440\u0438 \u044d\u0442\u043e\u043c vdo \u0441 \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430\u043c\u0438 \u043f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e \u0432 \u0440\u0435\u0436\u0438\u043c\u0435 stripe \u0434\u0435\u043c\u043e\u043d\u0441\u0442\u0440\u0438\u0440\u0443\u0435\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c (\u0434\u043b\u044f \u0441\u043b\u0443\u0447\u0430\u0439\u043d\u044b\u0445 \u043e\u043f\u0435\u0440\u0430\u0446\u0438\u0439) \u043d\u0430 \u0443\u0440\u043e\u0432\u043d\u0435 ZFS.<\/p>\n<p>mdadm \u0441 \u0437\u0430\u043a\u0440\u044b\u0442\u044b\u043c write-hole \u0434\u043e\u0441\u0442\u0430\u0442\u043e\u0447\u043d\u043e \u0441\u0438\u043b\u044c\u043d\u043e \u0440\u0435\u0436\u0435\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u043a\u0440\u0443\u043f\u043d\u043e\u0439 \u0437\u0430\u043f\u0438\u0441\u0438, \u043d\u043e \u043f\u043e\u0447\u0442\u0438 \u043d\u0435 \u0443\u043c\u0435\u043d\u044c\u0448\u0430\u0435\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u043c\u0435\u043b\u043a\u043e\u0431\u043b\u043e\u0447\u043d\u043e\u0439 \u0437\u0430\u043f\u0438\u0441\u0438. <\/p>\n<p>ZFS \u043d\u0430 64 \u043f\u043e\u0442\u043e\u043a\u043e\u0432 \u0432\u044b\u0440\u044b\u0432\u0430\u0435\u0442\u0441\u044f \u0432\u043f\u0435\u0440\u0451\u0434 \u0432 \u0442\u0435\u0441\u0442\u0430\u0445 qd=1, \u0442\u0430\u043a \u043a\u0430\u043a \u043e\u043d \u0438\u0433\u043d\u043e\u0440\u0438\u0440\u0443\u0435\u0442\u0441\u044f fsync \u0438 \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e \u044d\u0442\u043e 64 \u043e\u043f\u0435\u0440\u0430\u0446\u0438\u0438 \u0434\u043b\u044f \u0434\u0430\u043d\u043d\u043e\u0433\u043e \u0442\u0435\u0441\u0442\u0430. \u0412 \u0447\u0442\u0435\u043d\u0438\u0438 \u043d\u0430 64 \u043e\u0434\u043d\u0438\u0445 \u043f\u043e\u0442\u043e\u043a\u0430 \u0436\u0435 \u043f\u0440\u0435\u043a\u0440\u0430\u0441\u043d\u043e\u0435 \u0440\u0430\u0431\u043e\u0442\u0430\u0435\u0442 ARC.<\/p>\n<p>\u0412 \u0441\u0442\u0430\u043d\u0434\u0430\u0440\u0442\u043d\u043e\u043c \u0441\u043e\u0441\u0442\u043e\u044f\u043d\u0438\u0438 &#8212; ZFS \u0434\u0435\u043c\u043e\u043d\u0441\u0442\u0440\u0438\u0440\u0443\u0435\u0442 \u0442\u043e, \u0447\u0442\u043e \u043e\u0436\u0438\u0434\u0430\u0435\u0448\u044c \u043e\u0442 CoW \u0441\u0438\u0441\u0442\u0435\u043c\u044b \u043e\u0440\u0438\u0435\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u043e\u0439 \u043d\u0430 HDD. \u041d\u0435 \u043e\u0441\u043e\u0431\u043e \u0431\u044b\u0441\u0442\u0440\u0443\u044e \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u043d\u0430 NVMe. \u041e\u0441\u043e\u0431\u0435\u043d\u043d\u043e \u0433\u0440\u0443\u0441\u0442\u043d\u043e \u0441\u043e \u0441\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0439 \u0437\u0430\u043f\u0438\u0441\u044c\u044e \u0441 \u0431\u043e\u043b\u044c\u0448\u043e\u0439 \u043e\u0447\u0435\u0440\u0435\u0434\u044c\u044e. <\/p>\n<p>\u0415\u0441\u043b\u0438 \u0441\u0442\u0440\u043e\u0438\u0442\u044c \u0442\u0430\u0431\u043b\u0438\u0446\u0443 \u0431\u0435\u0440\u044f mdadm \u0437\u0430 100% \u043f\u043e\u043b\u0443\u0447\u0438\u0442\u0441\u044f \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0430\u044f \u043a\u0430\u0440\u0442\u0438\u043d\u0430:<\/p>\n<details class=\"spoiler\">\n<summary>Raid0\/Stripe<\/summary>\n<div class=\"spoiler__content\">\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"left\"><strong>Raid0\/Stripe<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"126\" width=\"126\">\n<p align=\"right\">mdadm-raid0<\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">lvm-raid0<\/p>\n<\/td>\n<td data-colwidth=\"133\" width=\"133\">\n<p align=\"right\">lvm-vdo-raid0<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"126\" width=\"126\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">91.03%<\/p>\n<\/td>\n<td data-colwidth=\"133\" width=\"133\">\n<p align=\"right\">6.13%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"126\" width=\"126\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">101.01%<\/p>\n<\/td>\n<td data-colwidth=\"133\" width=\"133\">\n<p align=\"right\">11.24%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"126\" width=\"126\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">119.05%<\/p>\n<\/td>\n<td data-colwidth=\"133\" width=\"133\">\n<p align=\"right\">3.51%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"126\" width=\"126\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">121.15%<\/p>\n<\/td>\n<td data-colwidth=\"133\" width=\"133\">\n<p align=\"right\">7.76%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"126\" width=\"126\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">128.12%<\/p>\n<\/td>\n<td data-colwidth=\"133\" width=\"133\">\n<p align=\"right\">3.56%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"192\" width=\"192\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"126\" width=\"126\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">81.46%<\/p>\n<\/td>\n<td data-colwidth=\"133\" width=\"133\">\n<p align=\"right\">39.20%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\"><strong>Raid0\/Stripe<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"86\" width=\"86\">\n<p align=\"right\">zfs-default<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">zfs-8k<\/p>\n<\/td>\n<td data-colwidth=\"105\" width=\"105\">\n<p align=\"right\">zfs-128k-16k<\/p>\n<\/td>\n<td data-colwidth=\"78\" width=\"78\">\n<p align=\"right\">zfs-16k<\/p>\n<\/td>\n<td data-colwidth=\"72\" width=\"72\">\n<p align=\"right\">zfs-64k<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"86\" width=\"86\">\n<p align=\"right\">13.09%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">12.63%<\/p>\n<\/td>\n<td data-colwidth=\"105\" width=\"105\">\n<p align=\"right\">25.35%<\/p>\n<\/td>\n<td data-colwidth=\"78\" width=\"78\">\n<p align=\"right\">23.59%<\/p>\n<\/td>\n<td data-colwidth=\"72\" width=\"72\">\n<p align=\"right\">39.88%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"86\" width=\"86\">\n<p align=\"right\">17.23%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">16.53%<\/p>\n<\/td>\n<td data-colwidth=\"105\" width=\"105\">\n<p align=\"right\">28.67%<\/p>\n<\/td>\n<td data-colwidth=\"78\" width=\"78\">\n<p align=\"right\">29.37%<\/p>\n<\/td>\n<td data-colwidth=\"72\" width=\"72\">\n<p align=\"right\">25.26%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"86\" width=\"86\">\n<p align=\"right\">1.42%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">1.31%<\/p>\n<\/td>\n<td data-colwidth=\"105\" width=\"105\">\n<p align=\"right\">1.47%<\/p>\n<\/td>\n<td data-colwidth=\"78\" width=\"78\">\n<p align=\"right\">1.49%<\/p>\n<\/td>\n<td data-colwidth=\"72\" width=\"72\">\n<p align=\"right\">1.55%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"86\" width=\"86\">\n<p align=\"right\">6.32%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">6.11%<\/p>\n<\/td>\n<td data-colwidth=\"105\" width=\"105\">\n<p align=\"right\">7.90%<\/p>\n<\/td>\n<td data-colwidth=\"78\" width=\"78\">\n<p align=\"right\">7.12%<\/p>\n<\/td>\n<td data-colwidth=\"72\" width=\"72\">\n<p align=\"right\">9.14%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"86\" width=\"86\">\n<p align=\"right\">10.09%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">12.61%<\/p>\n<\/td>\n<td data-colwidth=\"105\" width=\"105\">\n<p align=\"right\">13.07%<\/p>\n<\/td>\n<td data-colwidth=\"78\" width=\"78\">\n<p align=\"right\">13.25%<\/p>\n<\/td>\n<td data-colwidth=\"72\" width=\"72\">\n<p align=\"right\">13.22%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"86\" width=\"86\">\n<p align=\"right\">45.43%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">65.88%<\/p>\n<\/td>\n<td data-colwidth=\"105\" width=\"105\">\n<p align=\"right\">62.51%<\/p>\n<\/td>\n<td data-colwidth=\"78\" width=\"78\">\n<p align=\"right\">66.68%<\/p>\n<\/td>\n<td data-colwidth=\"72\" width=\"72\">\n<p align=\"right\">66.18%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\"><strong>Raid0\/Stripe<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"130\" width=\"130\">\n<p align=\"right\">zfs64-8k<\/p>\n<\/td>\n<td data-colwidth=\"147\" width=\"147\">\n<p align=\"right\">zfs64-16k<\/p>\n<\/td>\n<td>\n<p align=\"right\">zfs64-64k<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"130\" width=\"130\">\n<p align=\"right\">10.37%<\/p>\n<\/td>\n<td data-colwidth=\"147\" width=\"147\">\n<p align=\"right\">19.58%<\/p>\n<\/td>\n<td>\n<p align=\"right\">43.84%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"130\" width=\"130\">\n<p align=\"right\">15.40%<\/p>\n<\/td>\n<td data-colwidth=\"147\" width=\"147\">\n<p align=\"right\">22.23%<\/p>\n<\/td>\n<td>\n<p align=\"right\">30.88%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"130\" width=\"130\">\n<p align=\"right\">2.33%<\/p>\n<\/td>\n<td data-colwidth=\"147\" width=\"147\">\n<p align=\"right\">2.51%<\/p>\n<\/td>\n<td>\n<p align=\"right\">2.04%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"130\" width=\"130\">\n<p align=\"right\">12.36%<\/p>\n<\/td>\n<td data-colwidth=\"147\" width=\"147\">\n<p align=\"right\">10.74%<\/p>\n<\/td>\n<td>\n<p align=\"right\">4.35%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"130\" width=\"130\">\n<p align=\"right\">113.91%<\/p>\n<\/td>\n<td data-colwidth=\"147\" width=\"147\">\n<p align=\"right\">116.52%<\/p>\n<\/td>\n<td>\n<p align=\"right\">87.54%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"156\" width=\"156\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"130\" width=\"130\">\n<p align=\"right\">1446.23%<\/p>\n<\/td>\n<td data-colwidth=\"147\" width=\"147\">\n<p align=\"right\">1234.67%<\/p>\n<\/td>\n<td>\n<p align=\"right\">564.82%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p>\u041e\u0442\u0434\u0435\u043b\u044c\u043d\u043e \u0441\u0440\u0430\u0432\u043d\u0435\u043d\u0438\u0435 ZFS 16k \u0441\\\u0431\u0435\u0437 \u0434\u0435\u0434\u0443\u043f\u0430:<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td><\/td>\n<td>\n<p align=\"right\">zfs-local-stripe-16k<\/p>\n<\/td>\n<td>\n<p align=\"right\">zfs-local-stripe-16k-dedup<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f   \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">27.16%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435   \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">102.38%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">53.57%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">97.94%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">97.16%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td>\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">105.05%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/details>\n<details class=\"spoiler\">\n<summary>Raid5\/RaidZ   <\/summary>\n<div class=\"spoiler__content\">\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"left\"><strong>Raid5\/RaidZ<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"118\" width=\"118\">\n<p align=\"left\">mdadm-raid5<\/p>\n<\/td>\n<td data-colwidth=\"143\" width=\"143\">\n<p align=\"left\">mdadm-raid5-ppl<\/p>\n<\/td>\n<td>\n<p align=\"left\">mdadm-raid5-log<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"118\" width=\"118\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"143\" width=\"143\">\n<p align=\"right\">44.98%<\/p>\n<\/td>\n<td>\n<p align=\"right\">51.84%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"118\" width=\"118\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"143\" width=\"143\">\n<p align=\"right\">96.95%<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"118\" width=\"118\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"143\" width=\"143\">\n<p align=\"right\">92.12%<\/p>\n<\/td>\n<td>\n<p align=\"right\">80.00%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"118\" width=\"118\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"143\" width=\"143\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.98%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"118\" width=\"118\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"143\" width=\"143\">\n<p align=\"right\">103.84%<\/p>\n<\/td>\n<td>\n<p align=\"right\">94.03%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"118\" width=\"118\">\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"143\" width=\"143\">\n<p align=\"right\">118.43%<\/p>\n<\/td>\n<td>\n<p align=\"right\">99.78%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\n<p align=\"left\"><strong>Raid5\/RaidZ<\/strong>   <\/p>\n<\/td>\n<td data-colwidth=\"114\" width=\"114\">\n<p align=\"right\">lvm-raid5<\/p>\n<\/td>\n<td data-colwidth=\"140\" width=\"140\">\n<p align=\"right\">lvm-raid5-alloc-normal<\/p>\n<\/td>\n<td data-colwidth=\"137\" width=\"137\">\n<p align=\"right\">lvm-vdo-raid5<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"114\" width=\"114\">\n<p align=\"right\">76.02%<\/p>\n<\/td>\n<td data-colwidth=\"140\" width=\"140\">\n<p align=\"right\">77.20%<\/p>\n<\/td>\n<td data-colwidth=\"137\" width=\"137\">\n<p align=\"right\">15.12%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"114\" width=\"114\">\n<p align=\"right\">59.52%<\/p>\n<\/td>\n<td data-colwidth=\"140\" width=\"140\">\n<p align=\"right\">56.98%<\/p>\n<\/td>\n<td data-colwidth=\"137\" width=\"137\">\n<p align=\"right\">11.96%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"114\" width=\"114\">\n<p align=\"right\">90.87%<\/p>\n<\/td>\n<td data-colwidth=\"140\" width=\"140\">\n<p align=\"right\">92.09%<\/p>\n<\/td>\n<td data-colwidth=\"137\" width=\"137\">\n<p align=\"right\">32.25%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"114\" width=\"114\">\n<p align=\"right\">88.71%<\/p>\n<\/td>\n<td data-colwidth=\"140\" width=\"140\">\n<p align=\"right\">88.28%<\/p>\n<\/td>\n<td data-colwidth=\"137\" width=\"137\">\n<p align=\"right\">14.56%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"114\" width=\"114\">\n<p align=\"right\">56.87%<\/p>\n<\/td>\n<td data-colwidth=\"140\" width=\"140\">\n<p align=\"right\">74.31%<\/p>\n<\/td>\n<td data-colwidth=\"137\" width=\"137\">\n<p align=\"right\">5.13%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"114\" width=\"114\">\n<p align=\"right\">87.18%<\/p>\n<\/td>\n<td data-colwidth=\"140\" width=\"140\">\n<p align=\"right\">91.69%<\/p>\n<\/td>\n<td data-colwidth=\"137\" width=\"137\">\n<p align=\"right\">55.24%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"160\" width=\"160\">\n<p><strong>Raid5\/RaidZ<\/strong>   <\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"left\">zfs-raidz-default<\/p>\n<\/td>\n<td data-colwidth=\"88\" width=\"88\">\n<p align=\"left\">zfs-raidZ-8k<\/p>\n<\/td>\n<td data-colwidth=\"89\" width=\"89\">\n<p align=\"left\">zfs-128k-16k<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"left\">zfs-raidZ-16k   <\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"left\">zfs-raidZ-64k   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"160\" width=\"160\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">70.54%<\/p>\n<\/td>\n<td data-colwidth=\"88\" width=\"88\">\n<p align=\"right\">67.95%<\/p>\n<\/td>\n<td data-colwidth=\"89\" width=\"89\">\n<p align=\"right\">121.09%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">111.61%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">246.59%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"160\" width=\"160\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">17.54%<\/p>\n<\/td>\n<td data-colwidth=\"88\" width=\"88\">\n<p align=\"right\">14.29%<\/p>\n<\/td>\n<td data-colwidth=\"89\" width=\"89\">\n<p align=\"right\">29.04%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">19.71%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">21.21%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"160\" width=\"160\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">55.77%<\/p>\n<\/td>\n<td data-colwidth=\"88\" width=\"88\">\n<p align=\"right\">62.07%<\/p>\n<\/td>\n<td data-colwidth=\"89\" width=\"89\">\n<p align=\"right\">53.08%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">62.68%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">63.29%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"160\" width=\"160\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">6.85%<\/p>\n<\/td>\n<td data-colwidth=\"88\" width=\"88\">\n<p align=\"right\">4.05%<\/p>\n<\/td>\n<td data-colwidth=\"89\" width=\"89\">\n<p align=\"right\">7.93%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">4.05%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">10.30%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"160\" width=\"160\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">49.15%<\/p>\n<\/td>\n<td data-colwidth=\"88\" width=\"88\">\n<p align=\"right\">35.22%<\/p>\n<\/td>\n<td data-colwidth=\"89\" width=\"89\">\n<p align=\"right\">49.86%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">36.51%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">57.84%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"160\" width=\"160\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">76.09%<\/p>\n<\/td>\n<td data-colwidth=\"88\" width=\"88\">\n<p align=\"right\">37.82%<\/p>\n<\/td>\n<td data-colwidth=\"89\" width=\"89\">\n<p align=\"right\">73.71%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">46.37%<\/p>\n<\/td>\n<td data-colwidth=\"79\" width=\"79\">\n<p align=\"right\">117.98%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\"><strong>Raid5\/RaidZ<\/strong>   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">zfs64-raidZ-8k<\/p>\n<\/td>\n<td data-colwidth=\"132\" width=\"132\">\n<p align=\"right\">zfs64-raidZ-16k<\/p>\n<\/td>\n<td>\n<p align=\"right\">zfs64-raidZ-64k<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">64.37%<\/p>\n<\/td>\n<td data-colwidth=\"132\" width=\"132\">\n<p align=\"right\">103.61%<\/p>\n<\/td>\n<td>\n<p align=\"right\">240.06%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">18.66%<\/p>\n<\/td>\n<td data-colwidth=\"132\" width=\"132\">\n<p align=\"right\">25.06%<\/p>\n<\/td>\n<td>\n<p align=\"right\">28.51%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">76.06%<\/p>\n<\/td>\n<td data-colwidth=\"132\" width=\"132\">\n<p align=\"right\">75.46%<\/p>\n<\/td>\n<td>\n<p align=\"right\">71.81%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">11.31%<\/p>\n<\/td>\n<td data-colwidth=\"132\" width=\"132\">\n<p align=\"right\">9.48%<\/p>\n<\/td>\n<td>\n<p align=\"right\">4.37%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">445.88%<\/p>\n<\/td>\n<td data-colwidth=\"132\" width=\"132\">\n<p align=\"right\">445.88%<\/p>\n<\/td>\n<td>\n<p align=\"right\">413.57%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"186\" width=\"186\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">1950.00%<\/p>\n<\/td>\n<td data-colwidth=\"132\" width=\"132\">\n<p align=\"right\">1618.55%<\/p>\n<\/td>\n<td>\n<p align=\"right\">813.71%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/details>\n<p>\u0414\u043b\u044f \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0438\u0445 \u0442\u0435\u0441\u0442\u043e\u0432 \u0441 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435\u043c iSER \u0438 NVMe-oF \u0431\u0443\u0434\u0443\u0442 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c\u0441\u044f 3 \u043e\u0441\u043d\u043e\u0432\u043d\u044b\u0445 \u0432\u0430\u0440\u0438\u0430\u043d\u0442\u0430:<\/p>\n<ul>\n<li>\n<p>mdadm stripe<\/p>\n<\/li>\n<li>\n<p>lvm stripe<\/p>\n<\/li>\n<li>\n<p>ZFS stripe \u0441 recordsize=128k, blocksize=16k \u0438 \u043a\u043e\u043c\u043f\u0440\u0435\u0441\u0441\u0438\u0435\u0439<\/p>\n<\/li>\n<li>\n<p>ZFS stripe \u0441 recordsize=128k, blocksize=16k \u0438 \u043a\u043e\u043c\u043f\u0440\u0435\u0441\u0441\u0438\u0435\u0439 + \u0434\u0435\u0434\u0443\u043f\u043e\u043c<\/p>\n<\/li>\n<\/ul>\n<p>P.S. \u0415\u0441\u043b\u0438 \u0412\u044b \u0441\u0447\u0438\u0442\u0430\u0435\u0442\u0435, \u0447\u0442\u043e \u044d\u0442\u0438 \u0442\u0435\u0441\u0442\u044b \u043d\u0435\u043f\u043e\u043b\u043d\u044b\u0435, \u0432 \u043d\u0438\u0445 \u0447\u0442\u043e-\u0442\u043e \u0443\u043f\u0443\u0449\u0435\u043d\u043e \u0438\u043b\u0438 \u043e\u043d\u0438 \u043d\u0435\u043a\u043e\u0440\u0440\u0435\u043a\u0442\u043d\u043e &#8212; \u044f \u043e\u0442\u043a\u0440\u044b\u0442 \u043a \u043b\u044e\u0431\u044b\u043c \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u044f\u043c \u043f\u043e \u0438\u0445 \u0434\u043e\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u044e.<\/p>\n<details class=\"spoiler\">\n<summary>\u0414\u043b\u044f \u0442\u0435\u0441\u0442\u0430 zfs64 \u0431\u044b\u043b \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u0439 \u0444\u043e\u0440\u043c\u0430\u0442 \u043a\u043e\u043c\u0430\u043d\u0434\u044b:<\/summary>\n<div class=\"spoiler__content\">\n<p>fio -name=test1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4M -iodepth=32 -rw=write -runtime=60 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt &amp;&amp; sleep 30 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4M -iodepth=32 -rw=read -runtime=60 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt &amp;&amp; sleep 30 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -invaqdlidate=1 -bs=4k -iodepth=128 -numjobs=16 -group_reporting -rw=randwrite -runtime=60 -randrepeat=0 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt &amp;&amp; sleep 30 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=128 -numjobs=16 -group_reporting -rw=randread -runtime=60 -randrepeat=0 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt &amp;&amp; sleep 30 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -randrepeat=0 -rw=randwrite -bs=4k -numjobs=1 -iodepth=1 -fsync=1 -runtime=60 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt &amp;&amp; sleep 30 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -randrepeat=0 -rw=randread -bs=4k -numjobs=1 -iodepth=1 -fsync=1 -runtime=60 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;###&#187; >> zfs64-.txt &amp;&amp; sleep 60 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4M -iodepth=32 -rw=write -runtime=60 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt &amp;&amp; sleep 30 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4M -iodepth=32 -rw=read -runtime=60 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt &amp;&amp; sleep 30 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=128 -numjobs=16 -group_reporting -rw=randwrite -runtime=60 -randrepeat=0 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt &amp;&amp; sleep 30 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=128 -numjobs=16 -group_reporting -rw=randread -runtime=60 -randrepeat=0 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt &amp;&amp; sleep 30 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -randrepeat=0 -rw=randwrite -bs=4k -numjobs=1 -iodepth=1 -fsync=1 -runtime=60 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt &amp;&amp; sleep 30 &amp;&amp; fio -name=test1 -ioengine=libaio -direct=1 -randrepeat=0 -rw=randread -bs=4k -numjobs=1 -iodepth=1 -fsync=1 -runtime=60 -filesize=20G >> zfs64-.txt &amp;&amp; echo &#171;***&#187; >> zfs64-.txt<\/p>\n<\/div>\n<\/details>\n<\/div>\n<\/div>\n<\/div>\n<p><!----><!----><\/div>\n<p><!----><!----><br \/> \u0441\u0441\u044b\u043b\u043a\u0430 \u043d\u0430 \u043e\u0440\u0438\u0433\u0438\u043d\u0430\u043b \u0441\u0442\u0430\u0442\u044c\u0438 <a href=\"https:\/\/habr.com\/ru\/articles\/753322\/\"> https:\/\/habr.com\/ru\/articles\/753322\/<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<div><!--[--><!--]--><\/div>\n<div id=\"post-content-body\">\n<div>\n<div class=\"article-formatted-body article-formatted-body article-formatted-body_version-2\">\n<div xmlns=\"http:\/\/www.w3.org\/1999\/xhtml\">\n<p>UPD: 13.08.2023 &#8212; \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d\u044b \u0442\u0435\u0441\u0442\u044b \u043f\u043e mdadm, lvm, zfs<\/p>\n<p>\u0423 \u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438 \u0435\u0441\u0442\u044c \u0434\u0432\u0435 \u0446\u0435\u043b\u0438:<\/p>\n<ol>\n<li>\n<p>\u041f\u0440\u043e\u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u0442\u0440\u0451\u0445 \u0441\u0438\u0441\u0442\u0435\u043c \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u0435\u043d\u0438\u044f \u0444\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0445 \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432 \u0432 \u043e\u0434\u043d\u043e \u043b\u043e\u0433\u0438\u0447\u0435\u0441\u043a\u043e\u0435, \u043a\u0430\u043a \u0434\u043b\u044f \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u043e\u0433\u043e \u0434\u043e\u0441\u0442\u0443\u043f\u0430 \u043a \u043d\u0435\u043c\u0443, \u0442\u0430\u043a \u0438 \u0434\u043b\u044f \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u044f \u0432 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u043c\u0430\u0441\u0441\u0438\u0432\u0430 \u0434\u043b\u044f \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438.<\/p>\n<\/li>\n<li>\n<p>\u0421\u043e\u0437\u0434\u0430\u043d\u0438\u0435 \u0431\u044d\u043a\u044d\u043d\u0434\u0430 \u0434\u043b\u044f \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430 \u0432 \u0432\u0438\u0434\u0435 \u043e\u0434\u043d\u043e\u043a\u043e\u043d\u0442\u0440\u043e\u043b\u043b\u0435\u0440\u043d\u043e\u0433\u043e \u0445\u0440\u0430\u043d\u0438\u043b\u0438\u0449\u0430.<\/p>\n<\/li>\n<\/ol>\n<p>\u0418\u043d\u044b\u043c\u0438 \u0441\u043b\u043e\u0432\u0430\u043c\u0438 \u0432\u044b\u0432\u043e\u0434\u044b \u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438 \u043d\u0435 \u043f\u0440\u0438\u043c\u0435\u043d\u0438\u043c\u044b, \u043a\u043e\u0433\u0434\u0430 \u0412\u0430\u043c \u043d\u0443\u0436\u043d\u043e:<\/p>\n<ol>\n<li>\n<p>\u0421\u0438\u043d\u0445\u0440\u043e\u043d\u043d\u043e\u0435 \u043e\u0442\u043a\u0430\u0437\u043e\u0443\u0441\u0442\u043e\u0439\u0447\u0438\u0432\u043e\u0435 \u0440\u0435\u0448\u0435\u043d\u0438\u0435<\/p>\n<\/li>\n<li>\n<p>\u041d\u0430\u0434\u0435\u0436\u043d\u043e\u0441\u0442\u044c >>> \u0441\u043a\u043e\u0440\u043e\u0441\u0442\u044c<\/p>\n<\/li>\n<li>\n<p>\u0414\u043e\u043b\u0433\u043e\u0432\u0440\u0435\u043c\u0435\u043d\u043d\u043e\u0435 \u0440\u0435\u0448\u0435\u043d\u0438\u0435, \u043a\u043e\u0442\u043e\u0440\u043e\u0435 \u043c\u043e\u0436\u043d\u043e \u043f\u043e\u0441\u0442\u0430\u0432\u0438\u0442\u044c \u0438 \u0437\u0430\u0431\u044b\u0442\u044c<\/p>\n<\/li>\n<\/ol>\n<p>\u0412 \u0441\u0432\u044f\u0437\u0438 \u0441\u043e \u0437\u043d\u0430\u0447\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u043c \u0443\u0434\u0435\u0448\u0435\u0432\u043b\u0435\u043d\u0438\u0435\u043c NVMe \u0434\u0438\u0441\u043a\u043e\u0432 \u043d\u0430 \u0440\u0430\u0437\u043b\u0438\u0447\u043d\u044b\u0445 \u0440\u0435\u0441\u0443\u0440\u0441\u0430\u0445 (\u0442\u0430\u043a\u0438\u0445 \u043a\u0430\u043a \u0430\u0432\u0438\u0442\u043e) \u043c\u043e\u0436\u043d\u043e \u043d\u0430\u0439\u0442\u0438 nvme \u0434\u0438\u0441\u043a\u0438 \u0434\u0435\u0448\u0435\u0432\u043b\u0435 \u0438\u0445 \u0430\u043d\u0430\u043b\u043e\u0433\u043e\u0432 \u043d\u0430 SATA \u0438\u043b\u0438 SAS \u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441\u0435 \u0441 \u0442\u0430\u043a\u0438\u043c \u0436\u0435 \u043e\u0431\u044a\u0451\u043c\u043e\u043c, \u043d\u043e \u043e\u0441\u0442\u0430\u0451\u0442\u0441\u044f \u0432\u043e\u043f\u0440\u043e\u0441 \u0432 \u0438\u0445 \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u0435\u043d\u0438\u0435 \u0432 \u043e\u0434\u0438\u043d \u043b\u043e\u0433\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0434\u0438\u0441\u043a. <\/p>\n<p>\u041a \u0441\u043e\u0436\u0430\u043b\u0435\u043d\u0438\u044e, \u0442\u0435\u043a\u0443\u0449\u0438\u0435 \u0440\u0435\u0439\u0434 \u043a\u043e\u043d\u0442\u0440\u043e\u043b\u043b\u0435\u0440\u044b \u0434\u043b\u044f NVMe \u0434\u0438\u0441\u043a\u043e\u0432 \u043d\u0435 \u043e\u0442\u043b\u0438\u0447\u0430\u044e\u0442\u0441\u044f \u043e\u0441\u043e\u0431\u043e\u0439 \u0434\u0435\u0448\u0435\u0432\u0438\u0437\u043d\u043e\u0439 \u0438\\\u0438\u043b\u0438 \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u043e\u043c \u0434\u043e\u0441\u0442\u0443\u043f\u043d\u044b\u0445 \u0440\u0430\u0437\u044a\u0451\u043c\u043e\u0432 \u0434\u043b\u044f \u0438\u0445 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u044f, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u043e\u0434\u043d\u0438\u043c \u0438\u0437 \u0432\u0430\u0440\u0438\u0430\u043d\u0442\u043e\u0432 \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435 \u043f\u0440\u043e\u0433\u0440\u0430\u043c\u043c\u043d\u043e\u0433\u043e \u0440\u0435\u0439\u0434\u0430. <\/p>\n<p> \u0414\u043b\u044f \u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u0432\u044b\u0431\u0440\u0430\u043d\u044b \u043f\u0440\u043e\u0433\u0440\u0430\u043c\u043c\u043d\u044b\u0435 \u0440\u0435\u0439\u0434\u044b:<\/p>\n<ul>\n<li>\n<p>mdadm (UPD: 13.08.2023, \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d\u044b \u0442\u0435\u0441\u0442\u044b \u0441 \u0437\u0430\u043a\u0440\u044b\u0442\u0438\u0435\u043c write hole &#8212; ppl &amp; log)<\/p>\n<\/li>\n<li>\n<p>lvm (\u0438 \u043d\u0430\u0434\u0441\u0442\u0440\u043e\u0439\u043a\u0430 VDO) &#8212; (UPD: 13.08.2023 \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d \u0442\u0435\u0441\u0442 \u0441 alloc=anywhere)<\/p>\n<\/li>\n<li>\n<p>ZFS (UPD: 13.08.2023 \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d\u044b \u0442\u0435\u0441\u0442\u044b \u0441 zpool blocksize = 128k)<\/p>\n<\/li>\n<\/ul>\n<p> \u041f\u0440\u043e \u043a\u0430\u0436\u0443\u044e \u0438\u0437 \u0432\u044b\u0431\u0440\u0430\u043d\u043d\u044b\u0445 \u0441\u0438\u0441\u0442\u0435\u043c \u043c\u043e\u0436\u043d\u043e \u043f\u043e\u0434\u0440\u043e\u0431\u043d\u0435\u0435 \u043f\u043e\u0447\u0438\u0442\u0430\u0442\u044c:<\/p>\n<ul>\n<li>\n<p><a href=\"https:\/\/habr.com\/ru\/articles\/504692\/\" rel=\"noopener noreferrer nofollow\">\u0422\u0443\u0442 \u043f\u0440\u043e ZFS<\/a> <\/p>\n<\/li>\n<li>\n<p><a href=\"https:\/\/www.opennet.ru\/man.shtml?topic=mdadm&amp;category=8&amp;russian=2\" rel=\"noopener noreferrer nofollow\">\u0422\u0443\u0442 \u043f\u0440\u043e mdadm<\/a> <\/p>\n<\/li>\n<li>\n<p><a href=\"http:\/\/xgu.ru\/wiki\/LVM\" rel=\"noopener noreferrer nofollow\">\u0422\u0443\u0442 \u043f\u0440\u043e LVM<\/a> \u0438 <a href=\"https:\/\/access.redhat.com\/documentation\/ru-ru\/red_hat_enterprise_linux\/8\/html\/deduplicating_and_compressing_logical_volumes_on_rhel\/introduction-to-vdo-on-lvm_deduplicating-and-compressing-logical-volumes-on-rhel\" rel=\"noopener noreferrer nofollow\">\u0442\u0443\u0442 \u043f\u0440\u043e \u043d\u0430\u0434\u0441\u0442\u0440\u043e\u0439\u043a\u0443 VDO<\/a><\/p>\n<\/li>\n<\/ul>\n<p>\u0414\u043b\u044f \u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u043e\u0433\u043e \u043c\u0430\u0441\u0441\u0438\u0432\u0430 \u0431\u044b\u043b\u0438 \u0432\u044b\u0431\u0440\u0430\u043d\u044b \u0442\u0435\u0441\u0442\u044b \u043e\u043f\u0438\u0441\u0430\u043d\u043d\u044b\u0435 <a href=\"https:\/\/habr.com\/ru\/companies\/flant\/articles\/664150\/\" rel=\"noopener noreferrer nofollow\">\u0442\u0443\u0442<\/a> \u0438 <a href=\"https:\/\/github.com\/vitalif\/vitastor\/blob\/master\/docs\/performance\/understanding.en.md#%D0%BA%D0%BE%D0%BC%D0%B0%D0%BD%D0%B4%D1%8B-fio\" rel=\"noopener noreferrer nofollow\">\u0442\u0443\u0442<\/a>.<\/p>\n<p>\u0414\u043b\u044f \u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u043c\u0430\u0441\u0441\u0438\u0432\u0430 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u0431\u0443\u0434\u0435\u0442 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c\u0441\u044f HCIbench, \u0430 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b \u0432\u0437\u044f\u0442\u044b \u0438\u0437 <a href=\"https:\/\/cloud.redhat.com\/blog\/software-design-pattern-benchmarking\" rel=\"noopener noreferrer nofollow\">\u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0435\u0439 \u0441\u0442\u0430\u0442\u044c\u0438<\/a>.<\/p>\n<h2>\u0422\u0435\u0441\u0442\u043e\u0432\u044b\u0439 \u0441\u0442\u0435\u043d\u0434<\/h2>\n<p><strong>\u0421\u0435\u0440\u0432\u0435\u0440 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438:<\/strong><\/p>\n<ul>\n<li>\n<p>Motherboard: Supermicro H11SSL-i<\/p>\n<\/li>\n<li>\n<p>CPU: EPYC 7302<\/p>\n<\/li>\n<li>\n<p>RAM: 4x64GB Micron 2933MHz<\/p>\n<\/li>\n<li>\n<p>\u0421\u0435\u0442\u044c: 40GbE ConnectX-3 Pro, 10GbE\/25GbE ConnectX-4 LN EN<\/p>\n<\/li>\n<li>\n<p>\u041e\u0421: ESXi 7U3 build 20036586<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u0421\u0435\u0440\u0432\u0435\u0440 \u0445\u0440\u0430\u043d\u0435\u043d\u0438\u044f<\/strong> (\u0432 \u0434\u0430\u043b\u044c\u043d\u0435\u0439\u0448\u0435\u043c \u0433\u0438\u0431\u0440\u0438\u0434\u043d\u044b\u0439 \u0441\u0435\u0440\u0432\u0435\u0440 \u0441 PCIe Passthrough \u0434\u043b\u044f \u0442\u0435\u0441\u0442\u043e\u0432 \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u0445\u0440\u0430\u043d\u0438\u043b\u0438\u0449\u0430):<\/p>\n<ul>\n<li>\n<p>Motherboard: Tyan S8030 (ver 1GbE)<\/p>\n<\/li>\n<li>\n<p>CPU: EPYC 7302<\/p>\n<\/li>\n<li>\n<p>RAM: 4x64GB Micron 2933MHz<\/p>\n<\/li>\n<li>\n<p>\u0421\u0435\u0442\u044c: 40GbE ConnectX-3 Pro x2<\/p>\n<\/li>\n<li>\n<p>\u0414\u0438\u0441\u043a\u0438 1: 6 x PM9A3 1.92TB, \u0444\u043e\u0440\u043c\u0430\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u044b \u0432 512n<\/p>\n<\/li>\n<li>\n<p>\u0414\u0438\u0441\u043a\u0438 2: 1 x PM1725 3.2\u0422\u0411 (\u043d\u0435 \u0443\u0447\u0430\u0441\u0442\u0432\u0443\u0435\u0442 \u0432 \u043a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u0438 \u0440\u0435\u0439\u0434\u0430, \u043d\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442\u0441\u044f \u0434\u043b\u044f \u0441\u0440\u0430\u0432\u043d\u0435\u043d\u0438\u044f)<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u0421\u043e\u0444\u0442 \u0434\u043b\u044f \u0442\u0435\u0441\u0442\u043e\u0432 \u043d\u0430 \u0444\u0438\u0437\u0438\u043a\u0435:<\/strong><\/p>\n<ul>\n<li>\n<p>Debian 12, \u044f\u0434\u0440\u043e 6.1.0-10<\/p>\n<\/li>\n<li>\n<p>FIO: 3.33-3<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u0421\u043e\u0444\u0442 \u0434\u043b\u044f \u0442\u0435\u0441\u0442\u043e\u0432 \u043f\u043e\u0432\u0435\u0440\u0445 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0438\u0430\u0446\u0438\u0438 (to-be)<\/strong><\/p>\n<ul>\n<li>\n<p>\u041e\u0421: ESXi 7U3 build<\/p>\n<\/li>\n<li>\n<p>\u0412\u041c: Ubuntu 22.04.03, <a href=\"https:\/\/9to5linux.com\/ubuntu-22-04-lts-is-now-powered-by-linux-kernel-6-2-from-ubuntu-23-04\" rel=\"noopener noreferrer nofollow\">\u044f\u0434\u0440\u043e 6.2<\/a><\/p>\n<\/li>\n<li>\n<p>FIO: 3.28-1<\/p>\n<\/li>\n<\/ul>\n<details class=\"spoiler\">\n<summary>\u0421\u043f\u0438\u0441\u043e\u043a \u0442\u0435\u0441\u0442\u043e\u0432 FIO \u043f\u0440\u0438\u0432\u0435\u0434\u0451\u043d \u043d\u0438\u0436\u0435:<\/summary>\n<div class=\"spoiler__content\">\n<p><code>fio -name=seq_mbs_write_T1Q1N1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4M -iodepth=32 -rw=write -runtime=60 -filename=\/dev\/nvme1n1 &amp;&amp; sleep 10 &amp;&amp; fio -name=seq_mbs_read_T1Q1N1 -ioengine=libaio -direct=1 -invalidate=1  -bs=4M -iodepth=32 -rw=read -runtime=60 -filename=\/dev\/nvme1n1 &amp;&amp; sleep 10 &amp;&amp; fio -name=rand_iops_write_T1Q128N1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=128 -numjobs=4 -group_reporting -rw=randwrite -runtime=60 -randrepeat=0 -filename=\/dev\/nvme1n1 &amp;&amp; sleep 10 &amp;&amp; fio -name=rand_iops_read_T1Q128N1 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=128 -numjobs=4 -group_reporting -rw=randread -runtime=60 -randrepeat=0 -filename=\/dev\/nvme1n1 &amp;&amp; sleep 10 &amp;&amp; fio -name=rand_iops_read_T1Q1N1 -randrepeat=0 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=1 -rw=randread -runtime=60 -filename=\/dev\/nvme1n1 &amp;&amp; sleep 10 &amp;&amp; fio -name=rand_iops_write_T1Q1N1 -randrepeat=0 -ioengine=libaio -direct=1 -invalidate=1 -bs=4k -iodepth=1 -fsync=1 -rw=randwrite -runtime=60 -filename=\/dev\/nvme1n1<\/code><\/p>\n<\/div>\n<\/details>\n<p>\u0420\u0430\u0441\u0441\u043c\u0430\u0442\u0440\u0438\u0432\u0430\u0442\u044c\u0441\u044f \u0431\u0443\u0434\u0435\u0442 2 \u0441\u0446\u0435\u043d\u0430\u0440\u0438\u044f:<\/p>\n<ol>\n<li>\n<p>Raid 0 &#8212; (\u0432\u0441\u0435 \u0436\u0435 \u0434\u0435\u043b\u0430\u044e\u0442 \u0431\u044d\u043a\u0430\u043f\u044b) \u043a\u0430\u043a \u043a\u0440\u0430\u0439\u043d\u0438\u0439 \u0441\u0446\u0435\u043d\u0430\u0440\u0438\u0439 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0433\u043e \u0440\u0435\u0448\u0435\u043d\u0438\u044f, \u043a\u043e\u0433\u0434\u0430 \u0437\u0430 \u0441\u043e\u0445\u0440\u0430\u043d\u043d\u043e\u0441\u0442\u044c \u0434\u0430\u043d\u043d\u044b\u0445 (\u043d\u0435 \u0434\u043e\u0441\u0442\u0443\u043f\u043d\u043e\u0441\u0442\u044c) \u043e\u0442\u0432\u0435\u0447\u0430\u0435\u0442 \u0438\u043b\u0438 \u0440\u0435\u043f\u043b\u0438\u043a\u0430\u0446\u0438\u044f \u0438\u043b\u0438 \u0431\u044d\u043a\u0430\u043f\u044b.<\/p>\n<\/li>\n<li>\n<p>Raid 5 &#8212; \u043a\u0430\u043a \u0445\u043e\u0440\u043e\u0448\u0430\u044f \u0441\u0445\u0435\u043c\u0430 \u0434\u043b\u044f 6 \u0434\u0438\u0441\u043a\u043e\u0432<\/p>\n<\/li>\n<\/ol>\n<p>\u0412 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u0434\u043e\u0441\u0442\u0443\u043f\u0430 \u0431\u0443\u0434\u0435\u0442 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c\u0441\u044f iSER \u0438 NVMe-oF<\/p>\n<p>\u041d\u0438\u0436\u0435 \u043f\u0440\u0438\u0432\u0435\u0434\u0435\u043d\u0430 \u0441\u0432\u043e\u0434\u043d\u0430\u044f \u0442\u0430\u0431\u043b\u0438\u0446\u0430 \u043f\u043e \u0434\u0438\u0441\u043a\u0430\u043c \u0438 \u0433\u0440\u0430\u0444\u0438\u043a \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438: <\/p>\n<details class=\"spoiler\">\n<summary>\u0421\u0432\u043e\u0434\u043d\u0430\u044f \u0442\u0430\u0431\u043b\u0438\u0446\u0430 \u043f\u043e \u0434\u0438\u0441\u043a\u0430\u043c \u0432 \u0441\u0438\u0441\u0442\u0435\u043c\u0435<\/summary>\n<div class=\"spoiler__content\">\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">(MB\/s) NVMe#<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"left\">NVMe0 (PM1725)<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"left\">NVMe1 (PM9A3)<\/p>\n<\/td>\n<td>\n<p align=\"left\">NVMe2 (PM9A3)<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">1 473.50<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">2 810.50<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 809.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">5 068.50<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">5 882.50<\/p>\n<\/td>\n<td>\n<p align=\"right\">5 891.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">327.00<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">2 792.00<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 787.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">3 957.00<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">4 620.00<\/p>\n<\/td>\n<td>\n<p align=\"right\">4 619.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">163.50<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">233.00<\/p>\n<\/td>\n<td>\n<p align=\"right\">233.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"94\" width=\"94\">\n<p align=\"right\">44.80<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">90.30<\/p>\n<\/td>\n<td>\n<p align=\"right\">90.40<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">(MB\/s) NVMe#<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"left\">NVMe3 (PM9A3)<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"left\">NVMe4 (PM9A3)<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"left\">NVMe5 (PM9A3)<\/p>\n<\/td>\n<td>\n<p align=\"left\">NVMe6 (PM9A3)<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">2 807.50<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">2 811.00<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">2 809.50<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 810.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">5 884.00<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">5 915.00<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">5 904.50<\/p>\n<\/td>\n<td>\n<p align=\"right\">5 885.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">2 793.00<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">2 789.00<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">2 779.00<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 796.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">4 615.00<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">4 537.50<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">4 526.50<\/p>\n<\/td>\n<td>\n<p align=\"right\">4 619.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">231.50<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">230.00<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">231.00<\/p>\n<\/td>\n<td>\n<p align=\"right\">232.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"239\" width=\"239\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"82\" width=\"82\">\n<p align=\"right\">90.65<\/p>\n<\/td>\n<td data-colwidth=\"90\" width=\"90\">\n<p align=\"right\">90.15<\/p>\n<\/td>\n<td data-colwidth=\"85\" width=\"85\">\n<p align=\"right\">90.35<\/p>\n<\/td>\n<td>\n<p align=\"right\">90.70<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/details>\n<figure class=\"full-width\">\n<div><figcaption>\u0421\u0432\u043e\u0434\u043d\u044b\u0439 \u0433\u0440\u0430\u0444\u0438\u043a \u043f\u043e \u0434\u0438\u0441\u043a\u0430\u043c<\/figcaption><\/div>\n<\/figure>\n<p><a href=\"https:\/\/drive.google.com\/file\/d\/1BZWyqOeO_TfrG7vdsnGrhb2TV-mBqzXZ\/view?usp=sharing\" rel=\"noopener noreferrer nofollow\">\u0421\u0441\u044b\u043b\u043a\u0430 \u043d\u0430 excel \u0444\u0430\u0439\u043b \u0441\u043e \u0432\u0441\u0435\u043c\u0438 \u0434\u0430\u043d\u043d\u044b\u043c\u0438<\/a>.<\/p>\n<h2>MDADM<\/h2>\n<p>\u0421\u043e\u0437\u0434\u0430\u043d\u0438\u0435 mdadm \u0434\u043e\u0441\u0442\u0430\u0442\u043e\u0447\u043d\u043e \u0442\u0438\u043f\u0438\u0447\u043d\u043e:<br \/> mdadm &#8212;create &#8212;verbose \/dev\/md0 &#8212;level=0 &#8212;raid-devices=6 \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 \/dev\/nvme6n1<\/p>\n<p>\u0414\u043b\u044f 5-\u043e\u0433\u043e \u0440\u0435\u0439\u0434\u0430  (\u041f\u0435\u0440\u0435\u0434 \u043d\u0430\u0447\u0430\u043b\u043e\u043c \u0442\u0435\u0441\u0442\u043e\u0432 \u0431\u044b\u043b\u043e \u0432\u044b\u0436\u0434\u0430\u043d\u043e 3 \u0447\u0430\u0441\u0430, \u043f\u043e\u043a\u0430 \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u043b\u0441\u044f \u0440\u0435\u0439\u0434)<br \/>mdadm &#8212;create &#8212;verbose \/dev\/md5 &#8212;level=5 &#8212;raid-devices=6 \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 \/dev\/nvme6n1<\/p>\n<p>\u0414\u043e\u043f\u043e\u043b\u043d\u0438\u0442<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\"><strong>RAID 0<\/strong>\u00a0<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"center\">mdadm-local-raid0<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">11 980.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">20 275.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">11 827.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 817 002.93 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">11 622.40 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 771 891.56 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">172.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">42 160.66 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"325\" width=\"325\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"180\" width=\"180\">\n<p align=\"right\">99.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">24 269.23 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\"><strong>RAID 5<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"center\">mdadm-local-raid5<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">1 398.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">20 172.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">260.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">63 857.62 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">10 444.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 503 104.94 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">35.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">8 609.11 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">90.35 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">22 084.20 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\"><strong>RAID 5<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"left\">mdadm-local-raid5-ppl<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">629.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">19 558.40 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">239.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">58 477.84 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">10 444.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 502 086.56 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">36.55 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">8 927.77 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">107.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">26 229.00 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\"><strong>RAID 5<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"left\">mdadm-local-raid5-log<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">725.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">20 172.80 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">208.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">50 794.78 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">10 547.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 516 402.47 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">33.10 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">8 106.66 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"326\" width=\"326\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"176\" width=\"176\">\n<p align=\"right\">90.15 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">22 037.52 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<h2>LVM<\/h2>\n<p>LVM &#8212; \u0434\u043b\u044f \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e\u0442\u0441\u044f \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b<br \/> lvcreate -i6 -I64 &#8212;type striped  -l 100%VG -n lvm_stripe stripe<br \/> lvcreate -i5 -I64 &#8212;type raid5 -l 100%VG -n lvm_raid5 raid<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\"><strong>RAID0<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"center\">lvm-local-raid0<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">10 905.60 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">20 480.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">14 080.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">3 359 597.24 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">14 080.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">3 358 893.54 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">221.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">53 959.59 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"307\" width=\"307\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"199\" width=\"199\">\n<p align=\"right\">81.05 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">19 793.28 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\"><strong>RAID5 <\/strong><\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"center\">lvm-local-raid5 alloc default<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">1427.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">18 995.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">202.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">49 445.24 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">9816.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 407 383.56 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">31.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">49 445.24 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"290\" width=\"290\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"219\" width=\"219\">\n<p align=\"right\">87.25 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">21 266.52 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\"><strong>RAID5<\/strong> <\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"center\">lvm-local-raid5-alloc-normal<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">1 352.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">19 302.40 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">203.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">49 445.235 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">9 858.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 407 383.56 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">31.40 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">7 672.28 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"291\" width=\"291\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"216\" width=\"216\">\n<p align=\"right\">87.10 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">21 266.52 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u00a0<strong>RAID5<\/strong> <\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"left\">lvm-local-raid5-alloc-anywhere<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">1 340.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">19 046.4 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">202.00 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">49 372.28 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">9 826.50 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">2 400 206.15 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">31.20 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">7 620.88 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"289\" width=\"289\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"217\" width=\"217\">\n<p align=\"right\">87.05 MB\/s<\/p>\n<\/td>\n<td>\n<p align=\"right\">21 248.33 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<h2>LVM+VDO<\/h2>\n<p>\u0422\u0430\u043a \u043a\u0430\u043a \u043d\u0430 Debian vdo \u043f\u0440\u043e\u0441\u0442\u043e \u043d\u0435\u0442, \u0434\u0430\u0436\u0435 \u0432 \u0432\u0438\u0434\u0435 \u043f\u0430\u043a\u0435\u0442\u0430 \u043e\u043d \u0434\u043e\u0431\u0430\u0432\u043b\u044f\u043b\u0441\u044f \u043a\u0430\u043a \u043c\u043e\u0434\u0443\u043b\u044c \u043a \u044f\u0434\u0440\u0443 \u0441 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435\u043c<a href=\"https:\/\/github.com\/dm-vdo\/vdo\/issues\/64\" rel=\"noopener noreferrer nofollow\"> \u044d\u0442\u043e\u0433\u043e \u0440\u0443\u043a\u043e\u0432\u043e\u0434\u0441\u0442\u0432\u0430<\/a>), \u0442\u0435\u0441\u0442\u044b \u043f\u0440\u043e\u0432\u043e\u0434\u0438\u043b\u0438\u0441\u044c \u0441 \u0432\u043a\u043b\u044e\u0447\u0451\u043d\u043d\u044b\u043c \u0434\u0435\u0434\u0443\u043f\u043e\u043c, \u0442\u0430\u043a \u043a\u0430\u043a \u044d\u0442\u043e \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440 \u043f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e.<br \/>\u041f\u0440\u0438 \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u0438 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043b\u0438\u0441\u044c \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b, \u0442\u0430\u043a \u043a\u0430\u043a \u043f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e VDO \u043d\u0435 \u043f\u043e\u0434\u0434\u0435\u0440\u0436\u0438\u0432\u0430\u0435\u0442 raid \u043a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0430\u0446\u0438\u044e, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u043f\u0443\u043b \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u043b\u0441\u044f \u0447\u0435\u0440\u0435\u0437 lvconvert.<\/p>\n<pre><code>lvcreate -i6 -I64 --type striped -l 100%VG -n stripe-vdo lvm lvconvert --type vdo-pool --virtualsize 8TB \/dev\/lvm\/stripe-vdo<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u00a0<strong>RAID0<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"center\">lvm-vdo-local-raid0<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">735.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">2 279.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">415.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">101 344.40 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">901.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">220 089.32 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">6.14 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">1 534.41 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"302\" width=\"302\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"190\" width=\"190\">\n<p align=\"right\">39.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">9 520.67 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<pre><code>lvcreate -i6 -I64 --type raid5 -l 100%VG -n lvm_raid5 raid5 lvconvert --type vdo-pool --virtualsize 8TB \/dev\/lvm\/raid5-vdo<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p>\u00a0<strong>RAID5<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">lvm-vdo-local-raid5<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p>\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">230.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">2 406.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">&#8212;<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">79.50 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">19 388.92 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">1 536.00 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">374 751.79 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">1.59 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">398.57 IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"317\" width=\"317\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"193\" width=\"193\">\n<p align=\"right\">34.25 MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">8 329.27 IOPS<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<h2>ZFS<\/h2>\n<p>\u041f\u043e\u0441\u043b\u0435 \u043a\u0430\u0436\u0434\u043e\u0439 \u0433\u0440\u0443\u043f\u043f\u044b \u0442\u0435\u0441\u0442\u043e\u0432 \u0441\u0435\u0440\u0432\u0435\u0440 \u043f\u0435\u0440\u0435\u0437\u0430\u043f\u0443\u0441\u043a\u0430\u043b\u0441\u044f, \u0442.\u0435., \u043a \u043f\u0440\u0438\u043c\u0435\u0440\u0443, \u043f\u043e\u0441\u043b\u0435 \u0442\u0435\u0441\u0442\u043e\u0432 stripe 16k \u0438 \u043f\u0435\u0440\u0435\u0434 \u0442\u0435\u0441\u0442\u0430\u043c\u0438 \u043d\u0430 8k, \u0447\u0442\u043e\u0431\u044b \u043e\u0434\u043d\u043e\u0437\u043d\u0430\u0447\u043d\u043e \u043e\u0447\u0438\u0441\u0442\u0438\u0442\u044c ARC.<br \/>\u041a\u043e\u0433\u0434\u0430 \u0432\u044b\u0439\u0434\u0435\u0442 ZFS 3.0 \u0441 \u043f\u0430\u0442\u0447\u0435\u043c O_DIRECT \u0434\u043b\u044f nvme, \u044f \u0440\u0430\u0441\u0448\u0438\u0440\u044e \u0435\u0451, \u0441\u0435\u0439\u0447\u0430\u0441 \u0436\u0435 ZFS \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<p><strong>v1<\/strong> &#8212; \u0431\u043b\u043e\u043a <strong>16k<\/strong> (\u043d\u043e\u0432\u044b\u0435 \u0434\u0435\u0444\u043e\u043b\u0442\u043d\u044b\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b), \u043a\u043e\u043c\u043f\u0440\u0435\u0441\u0441\u0438\u044f lz4 (\u043d\u0435 \u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u0432\u043b\u0438\u044f\u043d\u0438\u044f \u043d\u0430 IOPS, \u043f\u043e \u043c\u043d\u043e\u0436\u0435\u0441\u0442\u0432\u0435\u043d\u043d\u044b\u043c \u0437\u0430\u044f\u0432\u043b\u0435\u043d\u0438\u044f\u043c \u0438 \u043f\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u043e\u043c \u0442\u0435\u0441\u0442\u043e\u0432), \u0434\u0435\u0434\u0443\u043f \u0432\u044b\u043a\u043b\u044e\u0447\u0435\u043d:<\/p>\n<pre><code>zpool create -o ashift=12 -O compression=lz4 -O atime=off -O recordsize=16k nvme \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 \/dev\/nvme6n1 zfs create -s -V 10T -o volblocksize=16k -o compression=lz4 nvme\/iser<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u00a0<strong>Stripe<\/strong><\/p>\n<\/td>\n<td data-colwidth=\"196\" width=\"196\">\n<p align=\"center\">zfs-local-stripe-16k<\/p>\n<\/td>\n<td>\n<p align=\"left\">\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"309\" width=\"309\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[],"tags":[],"class_list":["post-351612","post","type-post","status-publish","format-standard","hentry"],"_links":{"self":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts\/351612","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=351612"}],"version-history":[{"count":0,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts\/351612\/revisions"}],"wp:attachment":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=351612"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=351612"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=351612"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}