{"id":355695,"date":"2024-05-20T23:14:09","date_gmt":"2024-05-20T23:14:09","guid":{"rendered":"http:\/\/savepearlharbor.com\/?p=355695"},"modified":"-0001-11-30T00:00:00","modified_gmt":"-0001-11-29T21:00:00","slug":"","status":"publish","type":"post","link":"https:\/\/savepearlharbor.com\/?p=355695","title":{"rendered":"<span>\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u0445\u0440\u0430\u043d\u0438\u043b\u0438\u0449\u0430 \u043d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 \u0441\u043e\u0444\u0442 \u0440\u0435\u0439\u0434\u043e\u0432 (mdadm, LVM \u0438 ZFS) \u043f\u0440\u0438 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0438 iSER \u0438 NVMe-oF<\/span>"},"content":{"rendered":"<div><!--[--><!--]--><\/div>\n<div id=\"post-content-body\">\n<div>\n<div class=\"article-formatted-body article-formatted-body article-formatted-body_version-2\">\n<div xmlns=\"http:\/\/www.w3.org\/1999\/xhtml\">\n<p>\u041f\u0435\u0440\u0432\u0430\u044f \u0441\u0442\u0430\u0442\u044c\u044f \u0431\u044b\u043b\u0430 \u043f\u043e\u0441\u0432\u044f\u0449\u0435\u043d\u0430 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438 \u0441\u043e \u0441\u0442\u043e\u0440\u043e\u043d\u044b \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u043e\u0439 \u0441\u0438\u0441\u0442\u0435\u043c\u044b (<a href=\"https:\/\/habr.com\/ru\/articles\/753322\/\" rel=\"noopener noreferrer nofollow\">https:\/\/habr.com\/ru\/articles\/753322\/<\/a>). <\/p>\n<p>\u0421\u043c\u044b\u0441\u043b \u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438 \u043e\u0441\u0442\u0430\u0451\u0442\u0441\u044f \u0441\u0445\u043e\u0436\u0435\u0439, \u043f\u043e\u043a\u0430\u0437\u0430\u0442\u044c \u043c\u0430\u043a\u0441\u0438\u043c\u0430\u043b\u044c\u043d\u0443\u044e \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c, \u043a\u043e\u0433\u0434\u0430 \u0432\u043e\u043f\u0440\u043e\u0441 \u0441\u043e\u0445\u0440\u0430\u043d\u043d\u043e\u0441\u0442\u0438 \u0434\u0430\u043d\u043d\u044b\u0445 \u0440\u0435\u0448\u0430\u0435\u0442\u0441\u044f \u0440\u0435\u043f\u043b\u0438\u043a\u0430\u043c\u0438 \u0438\u043b\u0438 \u0431\u044d\u043a\u0430\u043f\u0430\u043c\u0438.<\/p>\n<p>\u0426\u0435\u043b\u044c \u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438 \u043f\u0440\u043e\u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u0442\u0440\u0451\u0445 \u0441\u0438\u0441\u0442\u0435\u043c \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u0435\u043d\u0438\u044f \u0444\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0445 \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432 \u0432 \u043e\u0434\u043d\u0443 \u043b\u043e\u0433\u0438\u0447\u0435\u0441\u043a\u0443\u044e \u0433\u0440\u0443\u043f\u043f\u0443 \u043f\u0440\u0438 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0438 iSER \u0438 NVMe-oF.<\/p>\n<p>\u0412 \u0440\u0430\u043c\u043a\u0430\u0445 \u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438 \u0431\u0443\u0434\u0443\u0442 \u0441\u0440\u0430\u0432\u043d\u0438\u0432\u0430\u0442\u044c\u0441\u044f \u0442\u0440\u0438 \u0441\u0438\u0441\u0442\u0435\u043c\u044b, \u043f\u0440\u043e\u0434\u0435\u043c\u043e\u0441\u0442\u0440\u0438\u0440\u0443\u044e\u0449\u0438\u0435 \u0441\u0430\u043c\u044b\u0435 \u0432\u044b\u0441\u043e\u043a\u0438\u0435 \u043f\u043e\u043a\u0430\u0437\u0430\u0442\u0435\u043b\u0438 \u043f\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0430\u043c \u0442\u0435\u0441\u0442\u043e\u0432 \u0438\u0437 \u043f\u0435\u0440\u0432\u043e\u0439 \u0447\u0430\u0441\u0442\u0438.<\/p>\n<ol>\n<li>\n<p>mdadm raid0<\/p>\n<\/li>\n<li>\n<p>LVM stripe<\/p>\n<\/li>\n<li>\n<p>ZFS stripe (default lz4)<\/p>\n<\/li>\n<\/ol>\n<p>\u0410 \u0442\u0430\u043a\u0436\u0435 \u0441\u0430\u043c\u043e\u0435 \u0444\u0443\u043d\u043a\u0446\u0438\u043e\u043d\u0430\u043b\u044c\u043d\u043e\u0435 \u0440\u0435\u0448\u0435\u043d\u0438\u0435 \u0438\u0437 \u0434\u043e\u0441\u0442\u0443\u043f\u043d\u044b\u0445 \u0431\u0435\u0441\u043f\u043b\u0430\u0442\u043d\u044b\u0445 \u0431\u044d\u043a\u044d\u043d\u0434\u043e\u0432:<\/p>\n<ol start=\"4\">\n<li>\n<p>ZFS stripe \u0441 \u043a\u043e\u043c\u043f\u0440\u0435\u0441\u0441\u0438\u0435\u0439 \u0438 \u0434\u0435\u0434\u0443\u043f\u043b\u0438\u043a\u0430\u0446\u0438\u0435\u0439<\/p>\n<\/li>\n<\/ol>\n<p>\u041f\u0440\u0438 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u0438 \u0447\u0435\u0440\u0435\u0437 iSER \u0438 NVMe-oF.<\/p>\n<p>\u041e\u0433\u043b\u0430\u0432\u043b\u0435\u043d\u0438\u0435:<\/p>\n<ol>\n<li>\n<p><a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#1\" rel=\"noopener noreferrer nofollow\">\u0422\u0435\u0441\u0442\u043e\u0432\u044b\u0439 \u0441\u0442\u0435\u043d\u0434<\/a><\/p>\n<\/li>\n<li>\n<p><a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#2\" rel=\"noopener noreferrer nofollow\">iSER<\/a><\/p>\n<p>2.1 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#2.1\" rel=\"noopener noreferrer nofollow\">MDADM<\/a><\/p>\n<p>2.2 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#2.2\" rel=\"noopener noreferrer nofollow\">LVM<\/a><\/p>\n<p>2.3 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#2.3\" rel=\"noopener noreferrer nofollow\">ZFS<\/a><\/p>\n<\/li>\n<li>\n<p><a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#3\" rel=\"noopener noreferrer nofollow\">NVMe-oF<\/a><\/p>\n<p>3.1 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#3.1\" rel=\"noopener noreferrer nofollow\">MDADM<\/a><\/p>\n<p>3.2 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#3.2\" rel=\"noopener noreferrer nofollow\">LVM<\/a><\/p>\n<p>3.3 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#3.3\" rel=\"noopener noreferrer nofollow\">ZFS<\/a><\/p>\n<\/li>\n<li>\n<p><a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#4\" rel=\"noopener noreferrer nofollow\">\u0412\u044b\u0432\u043e\u0434<\/a><\/p>\n<\/li>\n<\/ol>\n<p><a class=\"anchor\" name=\"1\" id=\"1\"><\/a><\/p>\n<h2>1. \u0422\u0435\u0441\u0442\u043e\u0432\u044b\u0439 \u0441\u0442\u0435\u043d\u0434:<\/h2>\n<p><strong>\u0421\u0435\u0440\u0432\u0435\u0440 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 1:<\/strong><\/p>\n<ul>\n<li>\n<p>Motherboard: Supermicro H11SSL-i<\/p>\n<\/li>\n<li>\n<p>CPU: EPYC 7302<\/p>\n<\/li>\n<li>\n<p>RAM: 4x64GB Micron 2933MHz<\/p>\n<\/li>\n<li>\n<p>\u0421\u0435\u0442\u044c: 40GbE ConnectX-3 Pro, 10GbE\/25GbE ConnectX-4 LN EN<\/p>\n<\/li>\n<li>\n<p>\u041e\u0421: ESXi 7U3 build 20036586<\/p>\n<\/li>\n<\/ul>\n<p> <strong>\u0421\u0435\u0440\u0432\u0435\u0440 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 2:<\/strong><\/p>\n<ul>\n<li>\n<p>Motherboard: Tyan S8030 (ver 1GbE)<\/p>\n<\/li>\n<li>\n<p>CPU: EPYC 7302<\/p>\n<\/li>\n<li>\n<p>RAM: 4x64GB Micron 2933MHz<\/p>\n<\/li>\n<li>\n<p>\u0421\u0435\u0442\u044c: 40GbE ConnectX-3 Pro x2 (\u041e\u0434\u0438\u043d \u0430\u0434\u0430\u043f\u0442\u0435\u0440 \u043f\u0440\u043e\u043a\u0438\u043d\u0443\u0442 \u0432 \u0412\u041c)<\/p>\n<\/li>\n<li>\n<p>\u041e\u0421: ESXi 7U3 build 21930508<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u0412\u041c \u0445\u0440\u0430\u043d\u0435\u043d\u0438\u044f \u043d\u0430 \u0441\u0435\u0440\u0432\u0435\u0440\u0435 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 #2<\/strong>, \u043a\u043e\u0442\u043e\u0440\u0430\u044f \u0431\u0443\u0434\u0435\u0442 \u043f\u0440\u0435\u0434\u043e\u0441\u0442\u0430\u0432\u043b\u044f\u0442\u044c \u0431\u043b\u043e\u0447\u043d\u044b\u0439 \u0434\u043e\u0441\u0442\u0443\u043f \u0434\u043b\u044f \u0445\u043e\u0441\u0442\u0430 1 \u0438 \u0445\u043e\u0441\u0442\u0430 2:<\/p>\n<ul>\n<li>\n<p>\u041e\u0421: Ubuntu 22.04, \u044f\u0434\u0440\u043e 6.2.0-26 (mitigations=off)<\/p>\n<\/li>\n<li>\n<p>vCPU: 8<\/p>\n<\/li>\n<li>\n<p>RAM: 64GB (\u0434\u043b\u044f ZFS \u0440\u0430\u0441\u0448\u0438\u0440\u0435\u043d\u0430 \u0434\u043e 128\u0413\u0411)<\/p>\n<\/li>\n<li>\n<p>\u0414\u0438\u0441\u043a\u0438: 6xPM9A3 1.92TB (\u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u044b \u0434\u043e \u0432\u0435\u0440\u0441\u0438\u0438 GDC5902Q)<\/p>\n<\/li>\n<\/ul>\n<details class=\"spoiler\">\n<summary>\u0421\u0445\u0435\u043c\u0430 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u044f:<\/summary>\n<div class=\"spoiler__content\">\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w780q1\/getpro\/habr\/upload_files\/1b5\/d4b\/41e\/1b5d4b41ed125677537df0e32a4a7917.jpg\" width=\"1042\" height=\"669\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/1b5\/d4b\/41e\/1b5d4b41ed125677537df0e32a4a7917.jpg\" data-blurred=\"true\"\/><\/figure>\n<\/p>\n<\/div>\n<\/details>\n<p>\u041f\u0440\u0438\u043c\u0435\u0447\u0430\u043d\u0438\u0435: \u0414\u043b\u044f HCIbench \u043e\u0442\u0441\u0443\u0442\u0441\u0442\u0432\u0443\u0435\u0442 \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0430\u0446\u0438\u044f \u043f\u043e \u0442\u043e\u043c\u0443 \u043a\u0430\u043a \u043f\u043e\u043c\u0435\u043d\u044f\u0442\u044c \u043a\u043e\u0434 \u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u043d\u0430 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435 \/dev\/nvme0nX, \u0432\u043c\u0435\u0441\u0442\u043e \/dev\/sdX, \u0432 \u0441\u0432\u044f\u0437\u0438 \u0441 \u044d\u0442\u0438\u043c \u0442\u0435\u0441\u0442\u044b \u043f\u0440\u043e\u0432\u043e\u0434\u044f\u0442\u0441\u044f \u0441 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435\u043c pvscsi.<br \/>\u0421\u043e\u0433\u043b\u0430\u0441\u043d\u043e \u0442\u0435\u0441\u0442\u0430\u043c \u0440\u0430\u0437\u043b\u0438\u0447\u0438\u0435 \u043c\u0435\u0436\u0434\u0443 pvscsi \u0438 nvme \u043a\u043e\u043d\u0442\u0440\u043e\u043b\u043b\u0435\u0440\u043e\u043c \u043b\u0435\u0436\u0438\u0442 \u0432 \u043f\u0440\u0435\u0434\u0435\u043b\u0430\u0445 \u043f\u043e\u0433\u0440\u0435\u0448\u043d\u043e\u0441\u0442\u0438 \u0438\u0437\u043c\u0435\u0440\u0435\u043d\u0438\u044f 5% (<a href=\"https:\/\/drive.google.com\/file\/d\/1D-wedRnr7drPkfB11swBiftDPwgiJACG\/view?usp=sharing\" rel=\"noopener noreferrer nofollow\">\u043f\u043e\u0434\u0440\u043e\u0431\u043d\u0435\u0435 \u0432 \u0442\u0430\u0431\u043b\u0438\u0446\u0435 \u0442\u0435\u0441\u0442\u043e\u0432<\/a>) \u0434\u043b\u044f \u0432\u0435\u0440\u0441\u0438\u0438 7U3, \u0442\u0430\u043a \u043a\u0430\u043a \u043f\u043e\u043b\u043d\u043e\u0446\u0435\u043d\u043d\u044b\u0439 NVMe back-to-back \u0431\u044b\u043b \u0440\u0435\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u043d \u0442\u043e\u043b\u044c\u043a\u043e \u0432 8U2, \u043a\u043e\u0442\u043e\u0440\u0430\u044f \u0443\u0436\u0435 \u043d\u0435 \u043f\u043e\u0434\u0434\u0435\u0440\u0436\u0438\u0432\u0430\u0435\u0442\u0441\u044f ConnectX-3 \ud83d\ude41<\/p>\n<p>\u041f\u0430\u0440\u0430\u043c\u0435\u0442\u0440 Prepare Virtual Disk Before Testing \u0432\u044b\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u043e \u0432 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 Random. \u0422\u0435\u0441\u0442\u044b \u043f\u0440\u043e\u0432\u043e\u0434\u044f\u0442\u0441\u044f \u0441 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435\u043c &#8212; 4 \u0412\u041c \u0441 \u0434\u0432\u0443\u043c\u044f \u0434\u0438\u0441\u043a\u0430\u043c\u0438 \u043f\u043e 100\u0413\u0411, 8 vCPU, 16GB RAM. <\/p>\n<p>\u0414\u043b\u044f \u0432\u044b\u0431\u043e\u0440\u0430 \u0442\u0435\u0441\u0442\u043e\u0432 \u043e\u0440\u0438\u0435\u0442\u0438\u0440\u043e\u043c \u0434\u043b\u044f \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u043e\u0432 \u0431\u044b\u043b\u0438 \u0432\u0437\u044f\u0442\u044b \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f \u0438\u0437 <a href=\"https:\/\/cloud.redhat.com\/blog\/software-design-pattern-benchmarking\" rel=\"noopener noreferrer nofollow\">\u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0435\u0433\u043e \u0438\u0441\u0442\u043e\u0447\u043d\u0438\u043a\u0430<\/a>.<\/p>\n<ul>\n<li>\n<p><strong>\u0422\u0435\u0441\u0442 1:<\/strong><\/p>\n<p>VDbench &#8212; 4k &#8212; 80% rng &#8212; 50\/50 r\/w &#8212; 8 thread per disk<\/p>\n<\/li>\n<li>\n<p><strong>\u0422\u0435\u0441\u0442 2:<\/strong><\/p>\n<p>VDbench &#8212; 8k &#8212; 80% rng &#8212; 75\/25 r\/w &#8212; 8 thread per disk<\/p>\n<\/li>\n<li>\n<p><strong>\u0422\u0435\u0441\u0442 3:<\/strong><\/p>\n<p>VDbench &#8212; 64k &#8212; 80% seq &#8212; 75\/25 r\/w &#8212; 8 thread per disk<\/p>\n<\/li>\n<li>\n<p><strong>\u0422\u0435\u0441\u0442 4:<\/strong><\/p>\n<p>VDbench &#8212; max_iops_read &#8212; 4k &#8212; 100% rng &#8212; 100\/0 r\/w &#8212; 8 thread per disk<\/p>\n<\/li>\n<li>\n<p><strong>\u0422\u0435\u0441\u0442 5:<\/strong><\/p>\n<p>VDbench &#8212; max_iops_write &#8212; 4k &#8212; 100% rng &#8212; 0\/100 r\/w &#8212; 8 thread per disk<\/p>\n<\/li>\n<\/ul>\n<p>\u0414\u043b\u044f \u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u043f\u0430\u0434\u0435\u043d\u0438\u044f \u0441\u043a\u043e\u0440\u043e\u0441\u0442\u0438 \u0432 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0435 \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d\u0438\u044f \u043f\u0440\u043e\u0441\u043b\u043e\u0439\u043a\u0438 \u0432 \u0432\u0438\u0434\u0435 \u0412\u041c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043b\u0438\u0441\u044c \u0442\u0435\u0441\u0442\u044b \u0438\u0437 \u043f\u0435\u0440\u0432\u043e\u0439 \u0447\u0430\u0441\u0442\u0438, \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u043d\u0438\u0436\u0435 \u0432 \u0442\u0430\u0431\u043b\u0438\u0446\u0435 \u0438 \u0433\u0440\u0430\u0444\u0438\u043a\u0430 (\u043f\u043e\u0433\u0440\u0435\u0448\u043d\u043e\u0441\u0442\u044c +-5%,):<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">SCSI controller (pvscsi)<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">NVMe controller<\/p>\n<\/td>\n<td>\n<p align=\"right\">% pvscsi=100%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">4603.11 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">4594.44 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">99.81%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">4618.22 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">4618.33 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">558.56 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">541.06 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">96.87%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">589.72 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">581.89 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">98.67%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">54.61 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">60.02 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">109.90%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">28.28 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">29.76 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">105.21%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p>\u0420\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u0442\u0435\u0441\u0442\u043e\u0432 iperf3 \u043c\u0435\u0436\u0434\u0443 ConnnectX-3 Pro, \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u043b\u0438\u0441\u044c \u043f\u0443\u0442\u0451\u043c \u043f\u0440\u043e\u0431\u0440\u043e\u0441\u0430 ConnectX-3 Pro \u0432 \u0412\u041c \u043d\u0430 \u043a\u0430\u0436\u0434\u043e\u043c \u0438\u0437 \u0445\u043e\u0441\u0442\u043e\u0432, \u0442.\u0435. \u0441\u0445\u0435\u043c\u0430 \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w780q1\/getpro\/habr\/upload_files\/678\/914\/2e5\/6789142e54e8a7bb4bf2ca2861007b80.jpg\" width=\"836\" height=\"506\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/678\/914\/2e5\/6789142e54e8a7bb4bf2ca2861007b80.jpg\" data-blurred=\"true\"\/><\/figure>\n<p>\u0420\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u043e\u043c \u0441\u0442\u0430\u043b\u043e:<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"351\" width=\"351\">\n<p align=\"left\">Sender <\/p>\n<\/td>\n<td>\n<p align=\"left\">34.93 Gbits\/sec<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"351\" width=\"351\">\n<p align=\"left\">Receiver <\/p>\n<\/td>\n<td>\n<p align=\"left\">34.21 Gbits\/sec<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p>\u041e\u043a\u0440\u0443\u0433\u043b\u044f\u044f \u0432 \u043c\u0435\u043d\u044c\u0448\u0443\u044e \u0441\u0442\u043e\u0440\u043e\u043d\u0443, \u043f\u043e\u043b\u0443\u0447\u0430\u0435\u043c:<\/p>\n<p>34 Gbits\/sec (\u0438\u043b\u0438 4.25 Gbytes\/s)<\/p>\n<p>\u041e\u0442\u043d\u043e\u0441\u0438\u0442\u0435\u043b\u044c\u043d\u043e OFED. \u0414\u043b\u044f \u0442\u0435\u0441\u0442\u043e\u0432 \u0442\u0430\u043a\u0436\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043b\u0430\u0441\u044c Ubuntu 20.04 \u0441 \u044f\u0434\u0440\u043e\u043c 5.5 \u0438 \u0443\u0441\u0442\u0430\u043d\u043e\u0432\u043b\u0435\u043d\u043d\u044b\u043c \u0434\u043b\u044f \u043d\u0435\u0433\u043e OFED, \u043d\u043e Ubuntu 22.04.03 \u0441 \u044f\u0434\u0440\u043e\u043c 6.2 \u043f\u0440\u043e\u0434\u0435\u043c\u043e\u043d\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043b\u0430 \u0432\u044b\u0448\u0435 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u0438 \u043f\u043e\u0432\u0442\u043e\u0440\u044f\u0435\u043c\u043e\u0441\u0442\u044c \u0432 iperf3.<\/p>\n<details class=\"spoiler\">\n<summary>iperf3<\/summary>\n<div class=\"spoiler__content\">\n<p>\u0422\u0435\u0441\u0442\u044b \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u043b\u0438\u0441\u044c \u0432 \u0441\u043e\u043e\u0442\u0432\u0435\u0442\u0441\u0442\u0432\u0438\u0438 \u0441 \u0434\u0430\u043d\u043d\u044b\u043c\u0438 \u0440\u0435\u043a\u043e\u043c\u0435\u043d\u0434\u0430\u0446\u0438\u044f\u043c\u0438 (<a href=\"https:\/\/fasterdata.es.net\/performance-testing\/network-troubleshooting-tools\/iperf\/multi-stream-iperf3\/\" rel=\"noopener noreferrer nofollow\">https:\/\/fasterdata.es.net\/performance-testing\/network-troubleshooting-tools\/iperf\/multi-stream-iperf3\/<\/a>)<\/p>\n<pre><code>s1:  [ ID] Interval           Transfer     Bitrate         Retr s1:  [  5]   0.00-10.00  sec  15.2 GBytes  13.0 Gbits\/sec  1062    sender s1: s1:  iperf Done. s2:  - - - - - - - - - - - - - - - - - - - - - - - - - s2:  [ ID] Interval           Transfer     Bitrate         Retr s2:  [  5]   0.00-10.00  sec  14.8 GBytes  12.7 Gbits\/sec  1856    sender s2: s2:  iperf Done. s3:  - - - - - - - - - - - - - - - - - - - - - - - - - s3:  [ ID] Interval           Transfer     Bitrate         Retr s3:  [  5]   0.00-10.00  sec  10.0 GBytes  8.59 Gbits\/sec  1027    sender<\/code><\/pre>\n<p>\u0421\u0443\u043c\u043c\u0430\u0440\u043d\u043e 34.29 Gbits\/sec<\/p>\n<p>\u0418 \u0441 \u0444\u043b\u0430\u0433\u043e\u043c -bidir<\/p>\n<pre><code>s1:  [ ID] Interval           Transfer     Bitrate         Retr s1:  [  5]   0.00-10.00  sec  13.4 GBytes  11.5 Gbits\/sec  1705    sender s1:  [  5]   0.00-10.04  sec  13.2 GBytes  11.3 Gbits\/sec          receiver s1: s1:  iperf Done. s2:  - - - - - - - - - - - - - - - - - - - - - - - - - s2:  [ ID] Interval           Transfer     Bitrate         Retr s2:  [  5]   0.00-10.00  sec  10.4 GBytes  8.93 Gbits\/sec  2171    sender s2:  [  5]   0.00-10.05  sec  10.1 GBytes  8.61 Gbits\/sec          receiver s3:  - - - - - - - - - - - - - - - - - - - - - - - - - s3:  [ ID] Interval           Transfer     Bitrate         Retr s3:  [  5]   0.00-10.00  sec  16.8 GBytes  14.5 Gbits\/sec  1513    sender s3:  [  5]   0.00-10.04  sec  16.7 GBytes  14.3 Gbits\/sec          receiver s3: s3:  iperf Done.<\/code><\/pre>\n<\/p>\n<\/div>\n<\/details>\n<p>\u0421\u0440\u0430\u0432\u043d\u0438\u0432\u0430\u043d\u0438\u044f \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c NVMe \u0434\u0438\u0441\u043a\u043e\u0432 \u043f\u0440\u0438 \u0438\u0445 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u0438 \u043d\u0430\u043f\u0440\u044f\u043c\u0443\u044e \u0432 \u041e\u0421 \u0438\u043b\u0438 \u0447\u0435\u0440\u0435\u0437 VMware \u0441 PCIe passthrough \u043c\u043e\u0436\u043d\u043e \u0433\u043e\u0432\u043e\u0440\u0438\u0442\u044c \u043e \u0442\u043e\u043c, \u0447\u0442\u043e \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u043d\u0435 \u043c\u0435\u043d\u044f\u0435\u0442\u0441\u044f, \u0442\u0430\u043a \u043a\u0430\u043a \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u043d\u0430\u0445\u043e\u0434\u044f\u0442\u0441\u044f \u0432 \u043f\u0440\u0435\u0434\u0435\u043b\u0430\u0445 2% \u043f\u043e\u0433\u0440\u0435\u0448\u043d\u043e\u0441\u0442\u0438. <\/p>\n<p><a href=\"https:\/\/drive.google.com\/file\/d\/1I7dCs_HX8VqKKpVgj8Tm8RqthC5PutwA\/view?usp=drive_link\" rel=\"noopener noreferrer nofollow\">\u0422\u0430\u0431\u043b\u0438\u0446\u0430 excel \u043f\u043e \u0442\u0435\u0441\u0442\u0430\u043c \u0434\u0438\u0441\u043a\u043e\u0432 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0451\u043d\u043d\u044b\u0445 \u043d\u0430\u043f\u0440\u044f\u043c\u0443\u044e \u043a \u0444\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u043e\u043c\u0443 \u0445\u043e\u0441\u0442\u0443 \u0438 \u043f\u0440\u043e\u043a\u0438\u043d\u0443\u0442\u044b\u0445 \u0432 \u0445\u043e\u0441\u0442 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438.<\/a><\/p>\n<details class=\"spoiler\">\n<summary>\u0422\u0435\u0441\u0442\u044b<\/summary>\n<div class=\"spoiler__content\">\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM1725 <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"left\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"left\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">1820.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">1826.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4518.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4553.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">1504.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">1502.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">3488.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">3514.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">148.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">172.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">42.90 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">45.45 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM9A3 (0369)<\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"left\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"left\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">5822.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5845.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">2807.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2806.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">4601.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4746.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">211.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">175.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">98.25 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">87.80 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM9A3 (6310)<\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"right\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">5846.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5835.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2806.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2807.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4610.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4743.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">211.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">178.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">98.30 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">87.85 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM9A3 (6314)<\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"right\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">5839.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5844.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2806.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2807.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4602.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4746.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">209.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">174.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">98.65 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">87.20 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM9A3 (3349)<\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"right\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">5844.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5835.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2807.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\"> 2806.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4598.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2328.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">208.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">201.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">98.10 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">94.90 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM9A3 (1091)<\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"right\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">5827.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5842.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2806.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2807.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4514.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4651.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">205.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">172.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">98.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">87.40 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">PM9A3 (0990)<\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"right\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2810.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">5849.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5857.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2797.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2807.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4489.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4646.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">206.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">183.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">97.20 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">88.05 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/details>\n<p>\u0420\u0430\u0441\u0441\u043c\u0430\u0442\u0440\u0438\u0432\u0430\u0442\u044c\u0441\u044f \u0431\u0443\u0434\u0435\u0442 2 \u0441\u0446\u0435\u043d\u0430\u0440\u0438\u044f:<\/p>\n<ol>\n<li>\n<p>\u0412\u044b\u0434\u0430\u0447\u0430 \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432\u0430 \u0447\u0435\u0440\u0435\u0437 iSER \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e LIO (\u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430 \u043e\u0441\u0443\u0449\u0435\u0441\u0442\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e targetcli-fb \u0441 \u043c\u0430\u0441\u0442\u0435\u0440 \u0432\u0435\u0442\u043a\u0438 \u0440\u0435\u043f\u043e\u0437\u0438\u0442\u043e\u0440\u0438\u044f)<\/p>\n<\/li>\n<li>\n<p>\u0412\u044b\u0434\u0430\u0447\u0430 \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432\u0430 \u0447\u0435\u0440\u0435\u0437 NVMe-oF \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e SPDK. <\/p>\n<p>\u041d\u0435\u0431\u043e\u043b\u044c\u0448\u043e\u0435 \u043e\u0442\u0441\u0442\u0443\u043f\u043b\u0435\u043d\u0438\u0435 &#8212; \u0441\u043f\u0435\u0440\u0432\u0430 \u043f\u043b\u0430\u043d\u0438\u0440\u043e\u0432\u0430\u043b\u043e\u0441\u044c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u043d\u0430\u0442\u0438\u0432\u043d\u0443\u044e \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044e \u044f\u0434\u0440\u0430 &#8212; nvmet (<a href=\"https:\/\/enterprise-support.nvidia.com\/s\/article\/howto-configure-nvme-over-fabrics--nvme-of--target-offload\" rel=\"noopener noreferrer nofollow\">https:\/\/enterprise-support.nvidia.com\/s\/article\/howto-configure-nvme-over-fabrics&#8212;nvme-of&#8212;target-offload<\/a>), \u043d\u043e \u043e\u043d\u0430 \u043d\u0435 \u043f\u043e\u0434\u0434\u0435\u0440\u0436\u0438\u0432\u0430\u0435\u0442\u0441\u044f VMware (<a href=\"https:\/\/koutoupis.com\/2022\/04\/22\/vmware-lightbits-labs-and-nvme-over-tcp\/\" rel=\"noopener noreferrer nofollow\">https:\/\/koutoupis.com\/2022\/04\/22\/vmware-lightbits-labs-and-nvme-over-tcp\/<\/a> <a href=\"https:\/\/communities.vmware.com\/t5\/ESXi-Discussions\/NVMEof-Datastore-Issues\/td-p\/2301440\" rel=\"noopener noreferrer nofollow\">https:\/\/communities.vmware.com\/t5\/ESXi-Discussions\/NVMEof-Datastore-Issues\/td-p\/2301440<\/a>), \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u0431\u044b\u043b \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d SPDK (<a href=\"https:\/\/spdk.io\/doc\/nvmf.html\" rel=\"noopener noreferrer nofollow\">https:\/\/spdk.io\/doc\/nvmf.html<\/a>, <a href=\"https:\/\/spdk.io\/doc\/bdev.html\" rel=\"noopener noreferrer nofollow\">https:\/\/spdk.io\/doc\/bdev.html<\/a>)<\/p>\n<\/li>\n<\/ol>\n<p><a class=\"anchor\" name=\"2\" id=\"2\"><\/a><\/p>\n<h3>2. iSER<\/h3>\n<p><u>\u0421\u043e \u0441\u0442\u043e\u0440\u043e\u043d\u044b Linux:<\/u><\/p>\n<p>\u0412 \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430\u0445 LIO \u043f\u0440\u043e\u043f\u0438\u0441\u044b\u0432\u0430\u0435\u0442\u0441\u044f enable_iser boolean=true \u043d\u0430 \u0443\u0440\u043e\u0432\u043d\u0435 \u043f\u043e\u0440\u0442\u0430\u043b\u0430:<\/p>\n<pre><code>set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1<\/code><\/pre>\n<p> \u0414\u043e\u043f\u043e\u043b\u043d\u0438\u0442\u0435\u043b\u044c\u043d\u043e \u0434\u043b\u044f \u0434\u0435\u0432\u0430\u0439\u0441\u0430 \u043f\u0440\u043e\u043f\u0438\u0441\u0430\u043d\u044b \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u0435 \u043e\u043f\u0446\u0438\u0438  (<a href=\"https:\/\/documentation.suse.com\/ses\/7\/html\/ses-all\/deploy-additional.html\" rel=\"noopener noreferrer nofollow\">https:\/\/documentation.suse.com\/ses\/7\/html\/ses-all\/deploy-additional.html<\/a>):<\/p>\n<pre><code>set attribute emulate_3pc=1 emulate_tpu=1 emulate_caw=1 max_write_same_len=65535 emulate_tpws=1 is_nonrot=1<\/code><\/pre>\n<p><u>\u0421\u043e \u0441\u0442\u043e\u0440\u043e\u043d\u044b VMware:<\/u><\/p>\n<pre><code>esxcli rdma iser add<\/code><\/pre>\n<p>\u0417\u0430\u0442\u0435\u043c \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430 \u0430\u043d\u0430\u043b\u043e\u0433\u0438\u0447\u043d\u0430 \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430\u043c \u043a\u0430\u043a \u0434\u043b\u044f iSCSI \u043c\u0430\u0441\u0441\u0438\u0432\u0430.<\/p>\n<p>\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c mdadm \u0441 \u0442\u0435\u0441\u0442\u0430\u043c\u0438 FIO \u0438\u0437 \u043f\u0435\u0440\u0432\u043e\u0439 \u0447\u0430\u0441\u0442\u0438 \u043d\u0430 1 \u0412\u041c \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">mdadm raid0 1\u0412\u041c<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">\u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">3632.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">866.20<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">3796.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">902.14<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">352.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">85736.44<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">559.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">136797.19<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">39.80 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">9712.06<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">26.10 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">6382.21<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p>\u041a\u043e\u043d\u0435\u0447\u043d\u044b\u0439 \u0433\u0440\u0430\u0444\u0438\u043a \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/9a2\/1f9\/937\/9a21f9937cf2a983b2e1ca16f162e624.png\" width=\"2647\" height=\"1086\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/9a2\/1f9\/937\/9a21f9937cf2a983b2e1ca16f162e624.png\"\/><\/figure>\n<p><a class=\"anchor\" name=\"2.1\" id=\"2.1\"><\/a><\/p>\n<h4>2.1 MDADM<\/h4>\n<pre><code>mdadm --create --verbose \/dev\/md0 --level=0 --raid-devices=6 \/dev\/nvme0n1 \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"295\" width=\"295\">\n<p align=\"left\">mdadm &#8212; iSER<\/p>\n<\/td>\n<td data-colwidth=\"232\" width=\"232\">\n<p align=\"left\">\u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"left\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"295\" width=\"295\">\n<p align=\"left\">4k-50rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"232\" width=\"232\">\n<p align=\"left\">486.78 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"left\">124615.40<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"295\" width=\"295\">\n<p align=\"left\">8k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"232\" width=\"232\">\n<p align=\"left\">941.52 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"left\">120514.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"295\" width=\"295\">\n<p align=\"left\">64k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"232\" width=\"232\">\n<p align=\"left\">3445.29 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"left\">55124.70<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"295\" width=\"295\">\n<p align=\"left\">4k-0rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"232\" width=\"232\">\n<p align=\"left\">495.13 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"left\">126752.90<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"295\" width=\"295\">\n<p align=\"left\">4k-100rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"232\" width=\"232\">\n<p align=\"left\">483.9 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"left\">123877.10<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p><a class=\"anchor\" name=\"2.2\" id=\"2.2\"><\/a><\/p>\n<h4>2.2 LVM<\/h4>\n<pre><code>lvcreate -i6 -I64 --type striped -l 100%VG -n nvme_stripe nvme<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"279\" width=\"279\">\n<p align=\"left\">LVM &#8212; iSER<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">\u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"279\" width=\"279\">\n<p align=\"left\">4k-50rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">514.27 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">131652.90<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"279\" width=\"279\">\n<p align=\"left\">8k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">957.10 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">122509.10<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"279\" width=\"279\">\n<p align=\"left\">64k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">3710.36 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">59365.80<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"279\" width=\"279\">\n<p align=\"left\">4k-0rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">535.9 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">137192.80<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"279\" width=\"279\">\n<p align=\"left\">4k-100rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">505.42 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">129386.40<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p><a class=\"anchor\" name=\"2.3\" id=\"2.3\"><\/a><\/p>\n<h4>2.3 ZFS<\/h4>\n<h3>v1 (without dedup)<\/h3>\n<pre><code>zpool create -o ashift=12 -O compression=lz4 -O atime=off -O recordsize=128k nvme \/dev\/nvme0n1 \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 zfs create -s -V 10T -o volblocksize=16k -o compression=lz4 nvme\/iser<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">ZFS &#8212; iSER<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">\u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-50rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">131.77 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">33734.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">8k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">343.34 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">43947.40<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">64k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">1418.62 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">22698.20<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-0rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">80.81 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">20690.60<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-100rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">251.38 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">64354.60<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<h3>v2 (with dedup)<\/h3>\n<pre><code>zpool create -o ashift=12 -O compression=lz4 -O atime=off -O dedup=on -O recordsize=128k nvme \/dev\/nvme0n1 \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 zfs create -s -V 10T -o volblocksize=16k -o compression=lz4 nvme\/iser<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">ZFS &#8212; iSER<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">\u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-50rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">56.75 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">14525.60<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">8k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">174.89 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">22385.60<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">64k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">384.76 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">6156.20<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-0rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">28.73 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">7355.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-100rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"234\" width=\"234\">\n<p align=\"right\">200.54 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">51337.80<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p><a class=\"anchor\" name=\"3\" id=\"3\"><\/a><\/p>\n<h3>3. NVMe-oF:<\/h3>\n<p><u>\u0421\u043e \u0441\u0442\u043e\u0440\u043e\u043d\u044b Linux:<\/u><\/p>\n<p>\u041f\u043e\u043c\u0438\u043c\u043e \u0431\u0430\u043d\u0430\u043b\u044c\u043d\u043e\u0439 \u0443\u0441\u0442\u0430\u043d\u043e\u0432\u043a\u0438 SPDK (<a href=\"https:\/\/spdk.io\/doc\/getting_started.html\" rel=\"noopener noreferrer nofollow\">https:\/\/spdk.io\/doc\/getting_started.html<\/a>)<s>, \u0435\u0441\u0442\u044c 1 \u043d\u044e\u0430\u043d\u0441. \u041d\u0430 \u043c\u043e\u043c\u0435\u043d\u0442 \u043d\u0430\u043f\u0438\u0441\u0430\u043d\u0438\u044f \u0441\u0442\u0430\u0442\u044c\u0438 SPDK \u0438\u0437 \u043c\u0430\u0441\u0442\u0435\u0440\u0430 \u043d\u0435 \u0440\u0430\u0431\u043e\u0442\u0430\u0435\u0442 \u0441 VMware \u043f\u043e \u043f\u0440\u0438\u0447\u0438\u043d\u0435 \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u043f\u0440\u043e\u0432\u0435\u0440\u043a\u0438 \u043d\u0430 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440 responder_resources == 0. \u0421\u043e \u0441\u0442\u043e\u0440\u043e\u043d\u044b VMware \u044d\u0442\u043e\u0442 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440 \u0440\u0430\u0432\u0435\u043d 1 (<\/s><a href=\"https:\/\/github.com\/spdk\/spdk\/issues\/3115\" rel=\"noopener noreferrer nofollow\"><s>https:\/\/github.com\/spdk\/spdk\/issues\/3115<\/s><\/a><s>). \u041f\u043e\u044d\u0442\u043e\u043c\u0443 \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u043e \u0441\u043e\u0431\u0440\u0430\u0442\u044c \u0432\u0435\u0440\u0441\u0438\u044e 23.05.x, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u043f\u0440\u043e\u0446\u0435\u0441\u0441 \u0443\u0441\u0442\u0430\u043d\u043e\u0432\u043a\u0438 spdk \u0431\u0443\u0434\u0435\u0442 \u043d\u0430\u0447\u0438\u043d\u0430\u0442\u044c\u0441\u044f \u043d\u0435 \u0441 &#8212;<\/s> \u043f\u043e\u0444\u0438\u043a\u0448\u0435\u043d\u043e \u0432 c8b9bba<\/p>\n<pre><code>git clone https:\/\/github.com\/spdk\/spdk --recursive modprobe nvme-rdma modprobe rdma_ucm modprobe rdma_cm scripts\/setup.sh screen - \u043b\u0438\u0431\u043e \u0434\u0440\u0443\u0433\u0430\u044f \u0441\u0435\u0441\u0441\u0438\u044f, \u043b\u0438\u0431\u043e \u043c\u043e\u0436\u043d\u043e \u043d\u0430\u043f\u0438\u0441\u0430\u0442\u044c \u0434\u0435\u043c\u043e\u043d\u0430, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0431\u0443\u0434\u0435\u0442 \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0442\u044c \u0435\u0433\u043e \u0441\u0430\u043c build\/bin\/nvmf_tgt ctrl+a d - \u043e\u0442\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u0435 \u043e\u0442 \u0441\u0435\u0441\u0441\u0438\u0438 screen \u0438 \u0432\u043e\u0437\u0432\u0440\u0430\u0442 \u0432 \u043a\u043e\u043d\u0441\u043e\u043b\u044c scripts\/rpc.py nvmf_create_transport -t RDMA -u 8192 -i 131072 -c 8192 scripts\/rpc.py nvmf_create_subsystem nqn.2016-06.io.spdk:cnode1 -a -s SPDK00000000000001 -d SPDK_Controller scripts\/rpc.py bdev_aio_create \/dev\/md0 md0 scripts\/rpc.py nvmf_subsystem_add_ns nqn.2016-06.io.spdk:cnode1 md0 scripts\/rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t rdma -a 10.20.0.1 -s 4420 scripts\/rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:cnode1 -t rdma -a 10.20.0.5 -s 4420<\/code><\/pre>\n<p><u>\u0421\u043e \u0441\u0442\u043e\u0440\u043e\u043d\u044b VMware:<\/u><\/p>\n<p>esxcli system module parameters set -m nmlx4_core -p &#171;enable_rocev2=1&#187; (\u0432 \u043f\u0440\u043e\u0442\u0438\u0432\u043d\u043e\u043c \u0441\u043b\u0443\u0447\u0430\u0435 \u0431\u0443\u0434\u0435\u0442 \u043e\u0448\u0438\u0431\u043a\u0430 &#171;Underlying device does not support requested gid\/RoCE type.&#187;)<\/p>\n<p>\u0417\u0430\u0442\u0435\u043c \u0432 \u0432\u0435\u0431 \u0441\u043e\u0437\u0434\u0430\u0451\u0442\u0441\u044f VMware NVME over RDMA Storage Adapter, \u0432 controller \u0437\u0430\u0434\u0430\u0451\u0442\u0441\u044f IP \u0430\u0434\u0440\u0435\u0441 \u0412\u041c, \u0443\u043a\u0430\u0437\u0430\u043d\u043d\u044b\u0439 \u0440\u0430\u043d\u0435\u0435, \u0432 \u043f\u043e\u043b\u0435 \u043f\u043e\u0440\u0442 &#8212; \u043f\u043e\u0440\u0442 4420<\/p>\n<p>\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c mdadm \u0441 \u0442\u0435\u0441\u0442\u0430\u043c\u0438 FIO \u0438\u0437 \u043f\u0435\u0440\u0432\u043e\u0439 \u0447\u0430\u0441\u0442\u0438 \u043d\u0430 1 \u0412\u041c \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"301\" width=\"301\">\n<p align=\"left\">mdadm raid0 1\u0412\u041c<\/p>\n<\/td>\n<td data-colwidth=\"233\" width=\"233\">\n<p align=\"right\">\u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"301\" width=\"301\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"233\" width=\"233\">\n<p align=\"right\">4593.00 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">1095.55<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"301\" width=\"301\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"233\" width=\"233\">\n<p align=\"right\">4618.00 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">1101.67<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"301\" width=\"301\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"233\" width=\"233\">\n<p align=\"right\">542.00 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">132417.04<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"301\" width=\"301\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"233\" width=\"233\">\n<p align=\"right\">575.50 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">140674.20<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"301\" width=\"301\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"233\" width=\"233\">\n<p align=\"right\">61.50 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">15013.28<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"301\" width=\"301\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"233\" width=\"233\">\n<p align=\"right\">30.35 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">7418.52<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p>\u041a\u043e\u043d\u0435\u0447\u043d\u044b\u0439 \u0433\u0440\u0430\u0444\u0438\u043a \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/2db\/daf\/56a\/2dbdaf56a51ae8412bba1892ce04cdef.png\" width=\"2647\" height=\"1086\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/2db\/daf\/56a\/2dbdaf56a51ae8412bba1892ce04cdef.png\"\/><\/figure>\n<p><a class=\"anchor\" name=\"3.1\" id=\"3.1\"><\/a><\/p>\n<h4>3.1 MDADM<\/h4>\n<pre><code>mdadm --create --verbose \/dev\/md0 --level=0 --raid-devices=6 \/dev\/nvme0n1 \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">mdadm &#8212; NVMe-oF<\/p>\n<\/td>\n<td data-colwidth=\"254\" width=\"254\">\n<p align=\"right\">MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-50rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"254\" width=\"254\">\n<p align=\"right\">570.59 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">146070.90<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">8k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"254\" width=\"254\">\n<p align=\"right\">1113.46 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">142522.10<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">64k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"254\" width=\"254\">\n<p align=\"right\">5775.06 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">92401.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-0rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"254\" width=\"254\">\n<p align=\"right\">593.87 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">152033.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-100rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"254\" width=\"254\">\n<p align=\"right\">540.54 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">138378.60<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p><a class=\"anchor\" name=\"3.2\" id=\"3.2\"><\/a><\/p>\n<h4>3.2 LVM<\/h4>\n<pre><code>lvcreate -i6 -I64 --type stripe  -l 100%VG -n lvm_stripe stripe<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">LVM &#8212; NVMe-oF<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-50rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">596.39 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">152675.70<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">8k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\"> 1160.72 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">148572.30<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">64k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">5826.39 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">93222.20<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-0rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">618.22 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">158267.10<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-100rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">559.45 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">143219.80<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p><a class=\"anchor\" name=\"3.3\" id=\"3.3\"><\/a><\/p>\n<h4>3.3 ZFS<\/h4>\n<h3>v1 (without dedup)<\/h3>\n<pre><code>zpool create -o ashift=12 -O compression=lz4 -O atime=off -O recordsize=128k nvme \/dev\/nvme0n1 \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 zfs create -s -V 10T -o volblocksize=16k -o compression=lz4 nvme\/iser<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">NVMe-oF ZFS<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-50rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">131.18 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">33585.30<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">8k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">344.21 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">44057.10<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">64k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">1409.13 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">22546.10<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-0rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">78.63 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">20129.20<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-100rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">267.21 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">68406.10<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<h3>v2 (with dedup)<\/h3>\n<pre><code>zpool create -o ashift=12 -O compression=lz4 -O atime=off -O dedup=on -O recordsize=128k nvme \/dev\/nvme0n1 \/dev\/nvme1n1 \/dev\/nvme2n1 \/dev\/nvme3n1 \/dev\/nvme4n1 \/dev\/nvme5n1 zfs create -s -V 10T -o volblocksize=16k -o compression=lz4 nvme\/iser<\/code><\/pre>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">NVMe-oF ZFS (dedup)<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">MB\/s <\/p>\n<\/td>\n<td>\n<p align=\"right\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-50rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">56.1 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">14362.50<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">8k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">183.32 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">23464.70<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">64k-75rdpct-80randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">355.83 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">5693.00<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-0rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">28.07 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">7185.10<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"265\" width=\"265\">\n<p align=\"left\">4k-100rdpct-100randompct<\/p>\n<\/td>\n<td data-colwidth=\"200\" width=\"200\">\n<p align=\"right\">211.7 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">54195.40<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p><a class=\"anchor\" name=\"4\" id=\"4\"><\/a><\/p>\n<h3>4. \u0412\u044b\u0432\u043e\u0434<\/h3>\n<p>LVM \u043f\u043e\u043a\u0430\u0437\u044b\u0432\u0430\u0435\u0442 \u0441\u0430\u043c\u044b\u0435 \u0432\u044b\u0441\u043e\u043a\u0438\u0435 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u0435\u0441\u043b\u0438 \u0431\u0440\u0430\u0442\u044c \u0435\u0433\u043e \u0437\u0430 100%, \u0442\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u0442\u0435\u0441\u0442\u043e\u0432 \u0431\u0443\u0434\u0443\u0442 \u0432\u044b\u0433\u043b\u044f\u0434\u0435\u0442\u044c \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">iSER<\/p>\n<\/td>\n<td>\n<p align=\"right\">LVM<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">mdadm<\/p>\n<\/td>\n<td data-colwidth=\"121\" width=\"121\">\n<p align=\"right\">ZFS<\/p>\n<\/td>\n<td>\n<p align=\"right\">ZFS dedup<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">4k-50rdpct-80randompct<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">94.65%<\/p>\n<\/td>\n<td data-colwidth=\"121\" width=\"121\">\n<p align=\"right\">25.62%<\/p>\n<\/td>\n<td>\n<p align=\"right\">11.04%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">8k-75rdpct-80randompct<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">98.37%<\/p>\n<\/td>\n<td data-colwidth=\"121\" width=\"121\">\n<p align=\"right\">35.87%<\/p>\n<\/td>\n<td>\n<p align=\"right\">18.27%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">64k-75rdpct-80randompct<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">92.86%<\/p>\n<\/td>\n<td data-colwidth=\"121\" width=\"121\">\n<p align=\"right\">38.23%<\/p>\n<\/td>\n<td>\n<p align=\"right\">10.37%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">4k-0rdpct-100randompct<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">92.39%<\/p>\n<\/td>\n<td data-colwidth=\"121\" width=\"121\">\n<p align=\"right\">15.08%<\/p>\n<\/td>\n<td>\n<p align=\"right\">5.36%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">4k-100rdpct-100randompct<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">95.74%<\/p>\n<\/td>\n<td data-colwidth=\"121\" width=\"121\">\n<p align=\"right\">49.74%<\/p>\n<\/td>\n<td>\n<p align=\"right\">39.68%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">NVMe-oF<\/p>\n<\/td>\n<td>\n<p align=\"right\">LVM<\/p>\n<\/td>\n<td>\n<p align=\"right\">mdadm<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">ZFS<\/p>\n<\/td>\n<td>\n<p align=\"right\">ZFS dedup<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">4k-50rdpct-80randompct<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">95.67%<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">22.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">9.41%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">8k-75rdpct-80randompct<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">95.93%<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">29.65%<\/p>\n<\/td>\n<td>\n<p align=\"right\">15.79%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">64k-75rdpct-80randompct<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">99.12%<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">24.19%<\/p>\n<\/td>\n<td>\n<p align=\"right\">6.11%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">4k-0rdpct-100randompct<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">96.06%<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">12.72%<\/p>\n<\/td>\n<td>\n<p align=\"right\">4.54%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"215\" width=\"215\">\n<p align=\"left\">4k-100rdpct-100randompct<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<td>\n<p align=\"right\">96.62%<\/p>\n<\/td>\n<td data-colwidth=\"122\" width=\"122\">\n<p align=\"right\">47.76%<\/p>\n<\/td>\n<td>\n<p align=\"right\">37.84%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p>ZFS \u043a\u0430\u043a \u0438 \u043e\u0436\u0438\u0434\u0430\u043b\u043e\u0441\u044c \u0434\u0435\u043c\u043e\u043d\u0441\u0442\u0440\u0438\u0440\u0443\u0435\u0442 \u043d\u0435 \u0441\u0430\u043c\u044b\u0435 \u0432\u044b\u0441\u043e\u043a\u0438\u0435 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b, \u043d\u043e \u044d\u0442\u043e, \u0432 \u0442\u043e\u043c \u0447\u0438\u0441\u043b\u0435, \u0441\u0432\u044f\u0437\u0430\u043d\u043e \u0441 \u043d\u0430\u0433\u0440\u0443\u0437\u043a\u043e\u0439 \u043f\u043e CPU, \u043d\u043e \u0438 \u0432 \u0446\u0435\u043b\u043e\u043c \u043d\u0430 \u044d\u0442\u043e \u0432\u043b\u0438\u044f\u0435\u0442 \u0442\u043e\u0442 \u0444\u0430\u043a\u0442, \u0447\u0442\u043e ZFS \u043d\u0435 \u043f\u0440\u0438\u0441\u043f\u043e\u0441\u043e\u0431\u043b\u0435\u043d \u0434\u043b\u044f \u0431\u044b\u0441\u0442\u0440\u043e\u0439 \u0440\u0430\u0431\u043e\u0442\u044b \u0441 NVMe. <\/p>\n<p>\u0412\u0441\u0435 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u0442\u0435\u0441\u0442\u043e\u0432 \u0434\u043e\u0441\u0442\u0443\u043f\u043d\u044b <a href=\"https:\/\/drive.google.com\/file\/d\/1ZdSmBC_AJVcpioPbcm7n0tmMJZ6FY5bn\/view?usp=sharing\" rel=\"noopener noreferrer nofollow\">\u0442\u0443\u0442<\/a> \u0438 <a href=\"https:\/\/drive.google.com\/file\/d\/1QRJlY5dFg8lknqI30KzPpXHxuWP1DNFr\/view?usp=sharing\" rel=\"noopener noreferrer nofollow\">\u0442\u0430\u0431\u043b\u0438\u0446\u0430 excel \u0442\u0443\u0442<\/a> (\u0434\u0430, \u0432\u0441\u0435 \u0442\u0430\u0431\u043b\u0438\u0446\u044b \u0432 \u0430\u0440\u0445\u0438\u0432\u0435, \u0442\u0430\u043a \u043a\u0430\u043a \u0433\u0443\u0433\u043b \u0442\u0430\u0431\u043b\u0438\u0446\u044b \u043b\u043e\u043c\u0430\u044e\u0442 \u0444\u043e\u0440\u043c\u0443\u043b\u044b). <\/p>\n<p>\u0413\u0440\u0430\u0444\u0438\u043a \u0432\u0441\u0435\u0445 \u0442\u0435\u0441\u0442\u043e\u0432 \u0441 \u0433\u0440\u0443\u043f\u043f\u0438\u0440\u043e\u0432\u043a\u043e\u0439 \u043f\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u043c\u043e\u043c\u0443 \u043f\u0440\u043e\u0442\u043e\u043a\u043e\u043b\u0443 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u044f \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c (\u0433\u0440\u0430\u0444\u0438\u043a \u0434\u0443\u0431\u043b\u0438\u0440\u0443\u0435\u0442\u0441\u044f \u0438\u0437 \u043f\u0443\u043d\u043a\u0442 2 \u0438 3)<\/p>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/c20\/641\/9fd\/c206419fd880b16d5fad48c6f68851d3.png\" alt=\"mdadm - \u0441\u0438\u043d\u0438\u0439, LVM - \u043e\u0440\u0430\u043d\u0436\u0435\u0432\u044b\u0439, ZFS - \u0441\u0435\u0440\u044b\u0439, ZFS-dedup - \u0436\u0451\u043b\u0442\u044b\u0439\" title=\"mdadm - \u0441\u0438\u043d\u0438\u0439, LVM - \u043e\u0440\u0430\u043d\u0436\u0435\u0432\u044b\u0439, ZFS - \u0441\u0435\u0440\u044b\u0439, ZFS-dedup - \u0436\u0451\u043b\u0442\u044b\u0439\" width=\"2647\" height=\"1086\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/c20\/641\/9fd\/c206419fd880b16d5fad48c6f68851d3.png\"\/><\/p>\n<div><figcaption>mdadm &#8212; \u0441\u0438\u043d\u0438\u0439, LVM &#8212; \u043e\u0440\u0430\u043d\u0436\u0435\u0432\u044b\u0439, ZFS &#8212; \u0441\u0435\u0440\u044b\u0439, ZFS-dedup &#8212; \u0436\u0451\u043b\u0442\u044b\u0439<\/figcaption><\/div>\n<\/figure>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/651\/794\/b80\/651794b8095591897b982c3ef1f0f04e.png\" alt=\"mdadm - \u0441\u0438\u043d\u0438\u0439, LVM - \u043e\u0440\u0430\u043d\u0436\u0435\u0432\u044b\u0439, ZFS - \u0441\u0435\u0440\u044b\u0439, ZFS-dedup - \u0436\u0451\u043b\u0442\u044b\u0439\" title=\"mdadm - \u0441\u0438\u043d\u0438\u0439, LVM - \u043e\u0440\u0430\u043d\u0436\u0435\u0432\u044b\u0439, ZFS - \u0441\u0435\u0440\u044b\u0439, ZFS-dedup - \u0436\u0451\u043b\u0442\u044b\u0439\" width=\"2647\" height=\"1086\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/651\/794\/b80\/651794b8095591897b982c3ef1f0f04e.png\"\/><\/p>\n<div><figcaption>mdadm &#8212; \u0441\u0438\u043d\u0438\u0439, LVM &#8212; \u043e\u0440\u0430\u043d\u0436\u0435\u0432\u044b\u0439, ZFS &#8212; \u0441\u0435\u0440\u044b\u0439, ZFS-dedup &#8212; \u0436\u0451\u043b\u0442\u044b\u0439<\/figcaption><\/div>\n<\/figure>\n<details class=\"spoiler\">\n<summary>\u0413\u0440\u0430\u0444\u0438\u043a \u0432\u0441\u0435\u0445 \u0442\u0435\u0441\u0442\u043e\u0432 \u0441 \u0433\u0440\u0443\u043f\u043f\u0438\u0440\u043e\u0432\u043a\u043e\u0439 \u043f\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u043c\u043e\u043c\u0443 \u0441\u043e\u0444\u0442 \u0440\u0435\u0439\u0434\u0443<\/summary>\n<div class=\"spoiler__content\">\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/e8e\/1e1\/aa6\/e8e1e1aa68f75b161e7f66488a313769.png\" alt=\"\u0421\u0438\u043d\u0435\u0435 - NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 - iSER\" title=\"\u0421\u0438\u043d\u0435\u0435 - NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 - iSER\" width=\"1901\" height=\"694\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/e8e\/1e1\/aa6\/e8e1e1aa68f75b161e7f66488a313769.png\"\/><\/p>\n<div><figcaption>\u0421\u0438\u043d\u0435\u0435 &#8212; NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 &#8212; iSER<\/figcaption><\/div>\n<\/figure>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/45a\/c29\/6bf\/45ac296bf0a1e4bafe3b653edbf43b23.png\" alt=\"\u0421\u0438\u043d\u0435\u0435 - NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 - iSER\" title=\"\u0421\u0438\u043d\u0435\u0435 - NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 - iSER\" width=\"1664\" height=\"695\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/45a\/c29\/6bf\/45ac296bf0a1e4bafe3b653edbf43b23.png\"\/><\/p>\n<div><figcaption>\u0421\u0438\u043d\u0435\u0435 &#8212; NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 &#8212; iSER<\/figcaption><\/div>\n<\/figure>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/882\/a50\/39f\/882a5039f79cca96067054e91b187936.png\" alt=\"\u0421\u0438\u043d\u0435\u0435 - NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 - iSER\" title=\"\u0421\u0438\u043d\u0435\u0435 - NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 - iSER\" width=\"1664\" height=\"695\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/882\/a50\/39f\/882a5039f79cca96067054e91b187936.png\"\/><\/p>\n<div><figcaption>\u0421\u0438\u043d\u0435\u0435 &#8212; NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 &#8212; iSER<\/figcaption><\/div>\n<\/figure>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/147\/f06\/e7d\/147f06e7d9c67a775a770b9eed609acd.png\" width=\"1664\" height=\"694\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/147\/f06\/e7d\/147f06e7d9c67a775a770b9eed609acd.png\"\/><\/figure>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/c9b\/358\/d4a\/c9b358d4ac84fa643fb6778109ea63cc.png\" alt=\"\u0421\u0438\u043d\u0435\u0435 - NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 - iSER\" title=\"\u0421\u0438\u043d\u0435\u0435 - NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 - iSER\" width=\"1664\" height=\"694\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/c9b\/358\/d4a\/c9b358d4ac84fa643fb6778109ea63cc.png\"\/><\/p>\n<div><figcaption>\u0421\u0438\u043d\u0435\u0435 &#8212; NVMe-oF, \u041e\u0440\u0430\u043d\u0436\u0435\u0432\u043e\u0435 &#8212; iSER<\/figcaption><\/div>\n<\/figure>\n<\/div>\n<\/details>\n<p>P.S. \u0415\u0441\u043b\u0438 \u0412\u044b \u0441\u0447\u0438\u0442\u0430\u0435\u0442\u0435, \u0447\u0442\u043e \u044d\u0442\u0438 \u0442\u0435\u0441\u0442\u044b \u043d\u0435\u043f\u043e\u043b\u043d\u044b\u0435, \u0432 \u043d\u0438\u0445 \u0447\u0442\u043e-\u0442\u043e \u0443\u043f\u0443\u0449\u0435\u043d\u043e \u0438\u043b\u0438 \u043e\u043d\u0438 \u043d\u0435\u043a\u043e\u0440\u0440\u0435\u043a\u0442\u043d\u044b &#8212; \u044f \u043e\u0442\u043a\u0440\u044b\u0442 \u043a \u043b\u044e\u0431\u044b\u043c \u043f\u0440\u0435\u0434\u043b\u043e\u0436\u0435\u043d\u0438\u044f\u043c \u043f\u043e \u0438\u0445 \u0434\u043e\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u044e.<\/p>\n<\/p>\n<\/div>\n<\/div>\n<\/div>\n<p><!----><!----><\/div>\n<p><!----><!----><br \/> \u0441\u0441\u044b\u043b\u043a\u0430 \u043d\u0430 \u043e\u0440\u0438\u0433\u0438\u043d\u0430\u043b \u0441\u0442\u0430\u0442\u044c\u0438 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/\"> https:\/\/habr.com\/ru\/articles\/763040\/<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<div><!--[--><!--]--><\/div>\n<div id=\"post-content-body\">\n<div>\n<div class=\"article-formatted-body article-formatted-body article-formatted-body_version-2\">\n<div xmlns=\"http:\/\/www.w3.org\/1999\/xhtml\">\n<p>\u041f\u0435\u0440\u0432\u0430\u044f \u0441\u0442\u0430\u0442\u044c\u044f \u0431\u044b\u043b\u0430 \u043f\u043e\u0441\u0432\u044f\u0449\u0435\u043d\u0430 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438 \u0441\u043e \u0441\u0442\u043e\u0440\u043e\u043d\u044b \u043b\u043e\u043a\u0430\u043b\u044c\u043d\u043e\u0439 \u0441\u0438\u0441\u0442\u0435\u043c\u044b (<a href=\"https:\/\/habr.com\/ru\/articles\/753322\/\" rel=\"noopener noreferrer nofollow\">https:\/\/habr.com\/ru\/articles\/753322\/<\/a>). <\/p>\n<p>\u0421\u043c\u044b\u0441\u043b \u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438 \u043e\u0441\u0442\u0430\u0451\u0442\u0441\u044f \u0441\u0445\u043e\u0436\u0435\u0439, \u043f\u043e\u043a\u0430\u0437\u0430\u0442\u044c \u043c\u0430\u043a\u0441\u0438\u043c\u0430\u043b\u044c\u043d\u0443\u044e \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c, \u043a\u043e\u0433\u0434\u0430 \u0432\u043e\u043f\u0440\u043e\u0441 \u0441\u043e\u0445\u0440\u0430\u043d\u043d\u043e\u0441\u0442\u0438 \u0434\u0430\u043d\u043d\u044b\u0445 \u0440\u0435\u0448\u0430\u0435\u0442\u0441\u044f \u0440\u0435\u043f\u043b\u0438\u043a\u0430\u043c\u0438 \u0438\u043b\u0438 \u0431\u044d\u043a\u0430\u043f\u0430\u043c\u0438.<\/p>\n<p>\u0426\u0435\u043b\u044c \u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438 \u043f\u0440\u043e\u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u0442\u0440\u0451\u0445 \u0441\u0438\u0441\u0442\u0435\u043c \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u0435\u043d\u0438\u044f \u0444\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0445 \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432 \u0432 \u043e\u0434\u043d\u0443 \u043b\u043e\u0433\u0438\u0447\u0435\u0441\u043a\u0443\u044e \u0433\u0440\u0443\u043f\u043f\u0443 \u043f\u0440\u0438 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0438 iSER \u0438 NVMe-oF.<\/p>\n<p>\u0412 \u0440\u0430\u043c\u043a\u0430\u0445 \u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0438 \u0431\u0443\u0434\u0443\u0442 \u0441\u0440\u0430\u0432\u043d\u0438\u0432\u0430\u0442\u044c\u0441\u044f \u0442\u0440\u0438 \u0441\u0438\u0441\u0442\u0435\u043c\u044b, \u043f\u0440\u043e\u0434\u0435\u043c\u043e\u0441\u0442\u0440\u0438\u0440\u0443\u044e\u0449\u0438\u0435 \u0441\u0430\u043c\u044b\u0435 \u0432\u044b\u0441\u043e\u043a\u0438\u0435 \u043f\u043e\u043a\u0430\u0437\u0430\u0442\u0435\u043b\u0438 \u043f\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0430\u043c \u0442\u0435\u0441\u0442\u043e\u0432 \u0438\u0437 \u043f\u0435\u0440\u0432\u043e\u0439 \u0447\u0430\u0441\u0442\u0438.<\/p>\n<ol>\n<li>\n<p>mdadm raid0<\/p>\n<\/li>\n<li>\n<p>LVM stripe<\/p>\n<\/li>\n<li>\n<p>ZFS stripe (default lz4)<\/p>\n<\/li>\n<\/ol>\n<p>\u0410 \u0442\u0430\u043a\u0436\u0435 \u0441\u0430\u043c\u043e\u0435 \u0444\u0443\u043d\u043a\u0446\u0438\u043e\u043d\u0430\u043b\u044c\u043d\u043e\u0435 \u0440\u0435\u0448\u0435\u043d\u0438\u0435 \u0438\u0437 \u0434\u043e\u0441\u0442\u0443\u043f\u043d\u044b\u0445 \u0431\u0435\u0441\u043f\u043b\u0430\u0442\u043d\u044b\u0445 \u0431\u044d\u043a\u044d\u043d\u0434\u043e\u0432:<\/p>\n<ol start=\"4\">\n<li>\n<p>ZFS stripe \u0441 \u043a\u043e\u043c\u043f\u0440\u0435\u0441\u0441\u0438\u0435\u0439 \u0438 \u0434\u0435\u0434\u0443\u043f\u043b\u0438\u043a\u0430\u0446\u0438\u0435\u0439<\/p>\n<\/li>\n<\/ol>\n<p>\u041f\u0440\u0438 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u0438 \u0447\u0435\u0440\u0435\u0437 iSER \u0438 NVMe-oF.<\/p>\n<p>\u041e\u0433\u043b\u0430\u0432\u043b\u0435\u043d\u0438\u0435:<\/p>\n<ol>\n<li>\n<p><a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#1\" rel=\"noopener noreferrer nofollow\">\u0422\u0435\u0441\u0442\u043e\u0432\u044b\u0439 \u0441\u0442\u0435\u043d\u0434<\/a><\/p>\n<\/li>\n<li>\n<p><a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#2\" rel=\"noopener noreferrer nofollow\">iSER<\/a><\/p>\n<p>2.1 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#2.1\" rel=\"noopener noreferrer nofollow\">MDADM<\/a><\/p>\n<p>2.2 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#2.2\" rel=\"noopener noreferrer nofollow\">LVM<\/a><\/p>\n<p>2.3 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#2.3\" rel=\"noopener noreferrer nofollow\">ZFS<\/a><\/p>\n<\/li>\n<li>\n<p><a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#3\" rel=\"noopener noreferrer nofollow\">NVMe-oF<\/a><\/p>\n<p>3.1 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#3.1\" rel=\"noopener noreferrer nofollow\">MDADM<\/a><\/p>\n<p>3.2 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#3.2\" rel=\"noopener noreferrer nofollow\">LVM<\/a><\/p>\n<p>3.3 <a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#3.3\" rel=\"noopener noreferrer nofollow\">ZFS<\/a><\/p>\n<\/li>\n<li>\n<p><a href=\"https:\/\/habr.com\/ru\/articles\/763040\/#4\" rel=\"noopener noreferrer nofollow\">\u0412\u044b\u0432\u043e\u0434<\/a><\/p>\n<\/li>\n<\/ol>\n<p><a class=\"anchor\" name=\"1\" id=\"1\"><\/a><\/p>\n<h2>1. \u0422\u0435\u0441\u0442\u043e\u0432\u044b\u0439 \u0441\u0442\u0435\u043d\u0434:<\/h2>\n<p><strong>\u0421\u0435\u0440\u0432\u0435\u0440 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 1:<\/strong><\/p>\n<ul>\n<li>\n<p>Motherboard: Supermicro H11SSL-i<\/p>\n<\/li>\n<li>\n<p>CPU: EPYC 7302<\/p>\n<\/li>\n<li>\n<p>RAM: 4x64GB Micron 2933MHz<\/p>\n<\/li>\n<li>\n<p>\u0421\u0435\u0442\u044c: 40GbE ConnectX-3 Pro, 10GbE\/25GbE ConnectX-4 LN EN<\/p>\n<\/li>\n<li>\n<p>\u041e\u0421: ESXi 7U3 build 20036586<\/p>\n<\/li>\n<\/ul>\n<p> <strong>\u0421\u0435\u0440\u0432\u0435\u0440 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 2:<\/strong><\/p>\n<ul>\n<li>\n<p>Motherboard: Tyan S8030 (ver 1GbE)<\/p>\n<\/li>\n<li>\n<p>CPU: EPYC 7302<\/p>\n<\/li>\n<li>\n<p>RAM: 4x64GB Micron 2933MHz<\/p>\n<\/li>\n<li>\n<p>\u0421\u0435\u0442\u044c: 40GbE ConnectX-3 Pro x2 (\u041e\u0434\u0438\u043d \u0430\u0434\u0430\u043f\u0442\u0435\u0440 \u043f\u0440\u043e\u043a\u0438\u043d\u0443\u0442 \u0432 \u0412\u041c)<\/p>\n<\/li>\n<li>\n<p>\u041e\u0421: ESXi 7U3 build 21930508<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u0412\u041c \u0445\u0440\u0430\u043d\u0435\u043d\u0438\u044f \u043d\u0430 \u0441\u0435\u0440\u0432\u0435\u0440\u0435 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 #2<\/strong>, \u043a\u043e\u0442\u043e\u0440\u0430\u044f \u0431\u0443\u0434\u0435\u0442 \u043f\u0440\u0435\u0434\u043e\u0441\u0442\u0430\u0432\u043b\u044f\u0442\u044c \u0431\u043b\u043e\u0447\u043d\u044b\u0439 \u0434\u043e\u0441\u0442\u0443\u043f \u0434\u043b\u044f \u0445\u043e\u0441\u0442\u0430 1 \u0438 \u0445\u043e\u0441\u0442\u0430 2:<\/p>\n<ul>\n<li>\n<p>\u041e\u0421: Ubuntu 22.04, \u044f\u0434\u0440\u043e 6.2.0-26 (mitigations=off)<\/p>\n<\/li>\n<li>\n<p>vCPU: 8<\/p>\n<\/li>\n<li>\n<p>RAM: 64GB (\u0434\u043b\u044f ZFS \u0440\u0430\u0441\u0448\u0438\u0440\u0435\u043d\u0430 \u0434\u043e 128\u0413\u0411)<\/p>\n<\/li>\n<li>\n<p>\u0414\u0438\u0441\u043a\u0438: 6xPM9A3 1.92TB (\u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u044b \u0434\u043e \u0432\u0435\u0440\u0441\u0438\u0438 GDC5902Q)<\/p>\n<\/li>\n<\/ul>\n<details class=\"spoiler\">\n<summary>\u0421\u0445\u0435\u043c\u0430 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u044f:<\/summary>\n<div class=\"spoiler__content\">\n<figure class=\"full-width\"><\/figure>\n<\/p>\n<\/div>\n<\/details>\n<p>\u041f\u0440\u0438\u043c\u0435\u0447\u0430\u043d\u0438\u0435: \u0414\u043b\u044f HCIbench \u043e\u0442\u0441\u0443\u0442\u0441\u0442\u0432\u0443\u0435\u0442 \u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0430\u0446\u0438\u044f \u043f\u043e \u0442\u043e\u043c\u0443 \u043a\u0430\u043a \u043f\u043e\u043c\u0435\u043d\u044f\u0442\u044c \u043a\u043e\u0434 \u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u043d\u0430 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435 \/dev\/nvme0nX, \u0432\u043c\u0435\u0441\u0442\u043e \/dev\/sdX, \u0432 \u0441\u0432\u044f\u0437\u0438 \u0441 \u044d\u0442\u0438\u043c \u0442\u0435\u0441\u0442\u044b \u043f\u0440\u043e\u0432\u043e\u0434\u044f\u0442\u0441\u044f \u0441 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435\u043c pvscsi.<br \/>\u0421\u043e\u0433\u043b\u0430\u0441\u043d\u043e \u0442\u0435\u0441\u0442\u0430\u043c \u0440\u0430\u0437\u043b\u0438\u0447\u0438\u0435 \u043c\u0435\u0436\u0434\u0443 pvscsi \u0438 nvme \u043a\u043e\u043d\u0442\u0440\u043e\u043b\u043b\u0435\u0440\u043e\u043c \u043b\u0435\u0436\u0438\u0442 \u0432 \u043f\u0440\u0435\u0434\u0435\u043b\u0430\u0445 \u043f\u043e\u0433\u0440\u0435\u0448\u043d\u043e\u0441\u0442\u0438 \u0438\u0437\u043c\u0435\u0440\u0435\u043d\u0438\u044f 5% (<a href=\"https:\/\/drive.google.com\/file\/d\/1D-wedRnr7drPkfB11swBiftDPwgiJACG\/view?usp=sharing\" rel=\"noopener noreferrer nofollow\">\u043f\u043e\u0434\u0440\u043e\u0431\u043d\u0435\u0435 \u0432 \u0442\u0430\u0431\u043b\u0438\u0446\u0435 \u0442\u0435\u0441\u0442\u043e\u0432<\/a>) \u0434\u043b\u044f \u0432\u0435\u0440\u0441\u0438\u0438 7U3, \u0442\u0430\u043a \u043a\u0430\u043a \u043f\u043e\u043b\u043d\u043e\u0446\u0435\u043d\u043d\u044b\u0439 NVMe back-to-back \u0431\u044b\u043b \u0440\u0435\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u043d \u0442\u043e\u043b\u044c\u043a\u043e \u0432 8U2, \u043a\u043e\u0442\u043e\u0440\u0430\u044f \u0443\u0436\u0435 \u043d\u0435 \u043f\u043e\u0434\u0434\u0435\u0440\u0436\u0438\u0432\u0430\u0435\u0442\u0441\u044f ConnectX-3 \ud83d\ude41<\/p>\n<p>\u041f\u0430\u0440\u0430\u043c\u0435\u0442\u0440 Prepare Virtual Disk Before Testing \u0432\u044b\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u043e \u0432 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 Random. \u0422\u0435\u0441\u0442\u044b \u043f\u0440\u043e\u0432\u043e\u0434\u044f\u0442\u0441\u044f \u0441 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435\u043c &#8212; 4 \u0412\u041c \u0441 \u0434\u0432\u0443\u043c\u044f \u0434\u0438\u0441\u043a\u0430\u043c\u0438 \u043f\u043e 100\u0413\u0411, 8 vCPU, 16GB RAM. <\/p>\n<p>\u0414\u043b\u044f \u0432\u044b\u0431\u043e\u0440\u0430 \u0442\u0435\u0441\u0442\u043e\u0432 \u043e\u0440\u0438\u0435\u0442\u0438\u0440\u043e\u043c \u0434\u043b\u044f \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u043e\u0432 \u0431\u044b\u043b\u0438 \u0432\u0437\u044f\u0442\u044b \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f \u0438\u0437 <a href=\"https:\/\/cloud.redhat.com\/blog\/software-design-pattern-benchmarking\" rel=\"noopener noreferrer nofollow\">\u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0435\u0433\u043e \u0438\u0441\u0442\u043e\u0447\u043d\u0438\u043a\u0430<\/a>.<\/p>\n<ul>\n<li>\n<p><strong>\u0422\u0435\u0441\u0442 1:<\/strong><\/p>\n<p>VDbench &#8212; 4k &#8212; 80% rng &#8212; 50\/50 r\/w &#8212; 8 thread per disk<\/p>\n<\/li>\n<li>\n<p><strong>\u0422\u0435\u0441\u0442 2:<\/strong><\/p>\n<p>VDbench &#8212; 8k &#8212; 80% rng &#8212; 75\/25 r\/w &#8212; 8 thread per disk<\/p>\n<\/li>\n<li>\n<p><strong>\u0422\u0435\u0441\u0442 3:<\/strong><\/p>\n<p>VDbench &#8212; 64k &#8212; 80% seq &#8212; 75\/25 r\/w &#8212; 8 thread per disk<\/p>\n<\/li>\n<li>\n<p><strong>\u0422\u0435\u0441\u0442 4:<\/strong><\/p>\n<p>VDbench &#8212; max_iops_read &#8212; 4k &#8212; 100% rng &#8212; 100\/0 r\/w &#8212; 8 thread per disk<\/p>\n<\/li>\n<li>\n<p><strong>\u0422\u0435\u0441\u0442 5:<\/strong><\/p>\n<p>VDbench &#8212; max_iops_write &#8212; 4k &#8212; 100% rng &#8212; 0\/100 r\/w &#8212; 8 thread per disk<\/p>\n<\/li>\n<\/ul>\n<p>\u0414\u043b\u044f \u0442\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u043f\u0430\u0434\u0435\u043d\u0438\u044f \u0441\u043a\u043e\u0440\u043e\u0441\u0442\u0438 \u0432 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0435 \u0434\u043e\u0431\u0430\u0432\u043b\u0435\u043d\u0438\u044f \u043f\u0440\u043e\u0441\u043b\u043e\u0439\u043a\u0438 \u0432 \u0432\u0438\u0434\u0435 \u0412\u041c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043b\u0438\u0441\u044c \u0442\u0435\u0441\u0442\u044b \u0438\u0437 \u043f\u0435\u0440\u0432\u043e\u0439 \u0447\u0430\u0441\u0442\u0438, \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u043d\u0438\u0436\u0435 \u0432 \u0442\u0430\u0431\u043b\u0438\u0446\u0435 \u0438 \u0433\u0440\u0430\u0444\u0438\u043a\u0430 (\u043f\u043e\u0433\u0440\u0435\u0448\u043d\u043e\u0441\u0442\u044c +-5%,):<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">SCSI controller (pvscsi)<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">NVMe controller<\/p>\n<\/td>\n<td>\n<p align=\"right\">% pvscsi=100%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">4603.11 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">4594.44 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">99.81%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">4618.22 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">4618.33 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">100.00%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">558.56 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">541.06 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">96.87%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">589.72 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">581.89 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">98.67%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">54.61 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">60.02 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">109.90%<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"281\" width=\"281\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"149\" width=\"149\">\n<p align=\"right\">28.28 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td data-colwidth=\"142\" width=\"142\">\n<p align=\"right\">29.76 \u041c\u0411\/\u0441<\/p>\n<\/td>\n<td>\n<p align=\"right\">105.21%<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p>\u0420\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u0442\u0435\u0441\u0442\u043e\u0432 iperf3 \u043c\u0435\u0436\u0434\u0443 ConnnectX-3 Pro, \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u043b\u0438\u0441\u044c \u043f\u0443\u0442\u0451\u043c \u043f\u0440\u043e\u0431\u0440\u043e\u0441\u0430 ConnectX-3 Pro \u0432 \u0412\u041c \u043d\u0430 \u043a\u0430\u0436\u0434\u043e\u043c \u0438\u0437 \u0445\u043e\u0441\u0442\u043e\u0432, \u0442.\u0435. \u0441\u0445\u0435\u043c\u0430 \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<figure class=\"full-width\"><\/figure>\n<p>\u0420\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u043e\u043c \u0441\u0442\u0430\u043b\u043e:<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"351\" width=\"351\">\n<p align=\"left\">Sender <\/p>\n<\/td>\n<td>\n<p align=\"left\">34.93 Gbits\/sec<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"351\" width=\"351\">\n<p align=\"left\">Receiver <\/p>\n<\/td>\n<td>\n<p align=\"left\">34.21 Gbits\/sec<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p>\u041e\u043a\u0440\u0443\u0433\u043b\u044f\u044f \u0432 \u043c\u0435\u043d\u044c\u0448\u0443\u044e \u0441\u0442\u043e\u0440\u043e\u043d\u0443, \u043f\u043e\u043b\u0443\u0447\u0430\u0435\u043c:<\/p>\n<p>34 Gbits\/sec (\u0438\u043b\u0438 4.25 Gbytes\/s)<\/p>\n<p>\u041e\u0442\u043d\u043e\u0441\u0438\u0442\u0435\u043b\u044c\u043d\u043e OFED. \u0414\u043b\u044f \u0442\u0435\u0441\u0442\u043e\u0432 \u0442\u0430\u043a\u0436\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043b\u0430\u0441\u044c Ubuntu 20.04 \u0441 \u044f\u0434\u0440\u043e\u043c 5.5 \u0438 \u0443\u0441\u0442\u0430\u043d\u043e\u0432\u043b\u0435\u043d\u043d\u044b\u043c \u0434\u043b\u044f \u043d\u0435\u0433\u043e OFED, \u043d\u043e Ubuntu 22.04.03 \u0441 \u044f\u0434\u0440\u043e\u043c 6.2 \u043f\u0440\u043e\u0434\u0435\u043c\u043e\u043d\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u043b\u0430 \u0432\u044b\u0448\u0435 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u0438 \u043f\u043e\u0432\u0442\u043e\u0440\u044f\u0435\u043c\u043e\u0441\u0442\u044c \u0432 iperf3.<\/p>\n<details class=\"spoiler\">\n<summary>iperf3<\/summary>\n<div class=\"spoiler__content\">\n<p>\u0422\u0435\u0441\u0442\u044b \u0432\u044b\u043f\u043e\u043b\u043d\u044f\u043b\u0438\u0441\u044c \u0432 \u0441\u043e\u043e\u0442\u0432\u0435\u0442\u0441\u0442\u0432\u0438\u0438 \u0441 \u0434\u0430\u043d\u043d\u044b\u043c\u0438 \u0440\u0435\u043a\u043e\u043c\u0435\u043d\u0434\u0430\u0446\u0438\u044f\u043c\u0438 (<a href=\"https:\/\/fasterdata.es.net\/performance-testing\/network-troubleshooting-tools\/iperf\/multi-stream-iperf3\/\" rel=\"noopener noreferrer nofollow\">https:\/\/fasterdata.es.net\/performance-testing\/network-troubleshooting-tools\/iperf\/multi-stream-iperf3\/<\/a>)<\/p>\n<pre><code>s1:  [ ID] Interval           Transfer     Bitrate         Retr s1:  [  5]   0.00-10.00  sec  15.2 GBytes  13.0 Gbits\/sec  1062    sender s1: s1:  iperf Done. s2:  - - - - - - - - - - - - - - - - - - - - - - - - - s2:  [ ID] Interval           Transfer     Bitrate         Retr s2:  [  5]   0.00-10.00  sec  14.8 GBytes  12.7 Gbits\/sec  1856    sender s2: s2:  iperf Done. s3:  - - - - - - - - - - - - - - - - - - - - - - - - - s3:  [ ID] Interval           Transfer     Bitrate         Retr s3:  [  5]   0.00-10.00  sec  10.0 GBytes  8.59 Gbits\/sec  1027    sender<\/code><\/pre>\n<p>\u0421\u0443\u043c\u043c\u0430\u0440\u043d\u043e 34.29 Gbits\/sec<\/p>\n<p>\u0418 \u0441 \u0444\u043b\u0430\u0433\u043e\u043c -bidir<\/p>\n<pre><code>s1:  [ ID] Interval           Transfer     Bitrate         Retr s1:  [  5]   0.00-10.00  sec  13.4 GBytes  11.5 Gbits\/sec  1705    sender s1:  [  5]   0.00-10.04  sec  13.2 GBytes  11.3 Gbits\/sec          receiver s1: s1:  iperf Done. s2:  - - - - - - - - - - - - - - - - - - - - - - - - - s2:  [ ID] Interval           Transfer     Bitrate         Retr s2:  [  5]   0.00-10.00  sec  10.4 GBytes  8.93 Gbits\/sec  2171    sender s2:  [  5]   0.00-10.05  sec  10.1 GBytes  8.61 Gbits\/sec          receiver s3:  - - - - - - - - - - - - - - - - - - - - - - - - - s3:  [ ID] Interval           Transfer     Bitrate         Retr s3:  [  5]   0.00-10.00  sec  16.8 GBytes  14.5 Gbits\/sec  1513    sender s3:  [  5]   0.00-10.04  sec  16.7 GBytes  14.3 Gbits\/sec          receiver s3: s3:  iperf Done.<\/code><\/pre>\n<\/p>\n<\/div>\n<\/details>\n<p>\u0421\u0440\u0430\u0432\u043d\u0438\u0432\u0430\u043d\u0438\u044f \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c NVMe \u0434\u0438\u0441\u043a\u043e\u0432 \u043f\u0440\u0438 \u0438\u0445 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u0438 \u043d\u0430\u043f\u0440\u044f\u043c\u0443\u044e \u0432 \u041e\u0421 \u0438\u043b\u0438 \u0447\u0435\u0440\u0435\u0437 VMware \u0441 PCIe passthrough \u043c\u043e\u0436\u043d\u043e \u0433\u043e\u0432\u043e\u0440\u0438\u0442\u044c \u043e \u0442\u043e\u043c, \u0447\u0442\u043e \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c \u043d\u0435 \u043c\u0435\u043d\u044f\u0435\u0442\u0441\u044f, \u0442\u0430\u043a \u043a\u0430\u043a \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u043d\u0430\u0445\u043e\u0434\u044f\u0442\u0441\u044f \u0432 \u043f\u0440\u0435\u0434\u0435\u043b\u0430\u0445 2% \u043f\u043e\u0433\u0440\u0435\u0448\u043d\u043e\u0441\u0442\u0438. <\/p>\n<p><a href=\"https:\/\/drive.google.com\/file\/d\/1I7dCs_HX8VqKKpVgj8Tm8RqthC5PutwA\/view?usp=drive_link\" rel=\"noopener noreferrer nofollow\">\u0422\u0430\u0431\u043b\u0438\u0446\u0430 excel \u043f\u043e \u0442\u0435\u0441\u0442\u0430\u043c \u0434\u0438\u0441\u043a\u043e\u0432 \u043f\u043e\u0434\u043a\u043b\u044e\u0447\u0451\u043d\u043d\u044b\u0445 \u043d\u0430\u043f\u0440\u044f\u043c\u0443\u044e \u043a \u0444\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u043e\u043c\u0443 \u0445\u043e\u0441\u0442\u0443 \u0438 \u043f\u0440\u043e\u043a\u0438\u043d\u0443\u0442\u044b\u0445 \u0432 \u0445\u043e\u0441\u0442 \u0432\u0438\u0440\u0442\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438.<\/a><\/p>\n<details class=\"spoiler\">\n<summary>\u0422\u0435\u0441\u0442\u044b<\/summary>\n<div class=\"spoiler__content\">\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM1725 <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"left\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"left\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">1820.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">1826.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4518.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4553.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">1504.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">1502.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">3488.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">3514.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">148.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">172.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">42.90 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">45.45 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM9A3 (0369)<\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"left\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"left\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">5822.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5845.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">2807.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2806.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">4601.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4746.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">211.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">175.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"124\" width=\"124\">\n<p align=\"right\">98.25 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">87.80 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM9A3 (6310)<\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"right\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">5846.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5835.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2806.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2807.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4610.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4743.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">211.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">178.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">98.30 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">87.85 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM9A3 (6314)<\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"right\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">5839.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5844.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2806.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2807.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4602.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4746.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">209.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">174.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">98.65 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">87.20 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM9A3 (3349)<\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"right\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">5844.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5835.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2807.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\"> 2806.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4598.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2328.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">208.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">201.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">98.10 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">94.90 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">PM9A3 (1091)<\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"right\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">5827.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5842.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2806.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2807.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4514.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4651.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">205.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">172.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"297\" width=\"297\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">98.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">87.40 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">PM9A3 (0990)<\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">\u0424\u0438\u0437\u0438\u0447\u0435\u0441\u043a\u0438\u0439<\/p>\n<\/td>\n<td>\n<p align=\"right\">\u0427\u0435\u0440\u0435\u0437 PCIe passthrough   <\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2810.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2811.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">5849.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">5857.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">2797.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">2807.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">4489.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">4646.00 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">206.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">183.50 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"298\" width=\"298\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1   <\/p>\n<\/td>\n<td data-colwidth=\"125\" width=\"125\">\n<p align=\"right\">97.20 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">88.05 \u041c\u0411\/c<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<\/div>\n<\/details>\n<p>\u0420\u0430\u0441\u0441\u043c\u0430\u0442\u0440\u0438\u0432\u0430\u0442\u044c\u0441\u044f \u0431\u0443\u0434\u0435\u0442 2 \u0441\u0446\u0435\u043d\u0430\u0440\u0438\u044f:<\/p>\n<ol>\n<li>\n<p>\u0412\u044b\u0434\u0430\u0447\u0430 \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432\u0430 \u0447\u0435\u0440\u0435\u0437 iSER \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e LIO (\u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430 \u043e\u0441\u0443\u0449\u0435\u0441\u0442\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e targetcli-fb \u0441 \u043c\u0430\u0441\u0442\u0435\u0440 \u0432\u0435\u0442\u043a\u0438 \u0440\u0435\u043f\u043e\u0437\u0438\u0442\u043e\u0440\u0438\u044f)<\/p>\n<\/li>\n<li>\n<p>\u0412\u044b\u0434\u0430\u0447\u0430 \u0431\u043b\u043e\u0447\u043d\u043e\u0433\u043e \u0443\u0441\u0442\u0440\u043e\u0439\u0441\u0442\u0432\u0430 \u0447\u0435\u0440\u0435\u0437 NVMe-oF \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e SPDK. <\/p>\n<p>\u041d\u0435\u0431\u043e\u043b\u044c\u0448\u043e\u0435 \u043e\u0442\u0441\u0442\u0443\u043f\u043b\u0435\u043d\u0438\u0435 &#8212; \u0441\u043f\u0435\u0440\u0432\u0430 \u043f\u043b\u0430\u043d\u0438\u0440\u043e\u0432\u0430\u043b\u043e\u0441\u044c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u043d\u0430\u0442\u0438\u0432\u043d\u0443\u044e \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044e \u044f\u0434\u0440\u0430 &#8212; nvmet (<a href=\"https:\/\/enterprise-support.nvidia.com\/s\/article\/howto-configure-nvme-over-fabrics--nvme-of--target-offload\" rel=\"noopener noreferrer nofollow\">https:\/\/enterprise-support.nvidia.com\/s\/article\/howto-configure-nvme-over-fabrics&#8212;nvme-of&#8212;target-offload<\/a>), \u043d\u043e \u043e\u043d\u0430 \u043d\u0435 \u043f\u043e\u0434\u0434\u0435\u0440\u0436\u0438\u0432\u0430\u0435\u0442\u0441\u044f VMware (<a href=\"https:\/\/koutoupis.com\/2022\/04\/22\/vmware-lightbits-labs-and-nvme-over-tcp\/\" rel=\"noopener noreferrer nofollow\">https:\/\/koutoupis.com\/2022\/04\/22\/vmware-lightbits-labs-and-nvme-over-tcp\/<\/a> <a href=\"https:\/\/communities.vmware.com\/t5\/ESXi-Discussions\/NVMEof-Datastore-Issues\/td-p\/2301440\" rel=\"noopener noreferrer nofollow\">https:\/\/communities.vmware.com\/t5\/ESXi-Discussions\/NVMEof-Datastore-Issues\/td-p\/2301440<\/a>), \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u0431\u044b\u043b \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d SPDK (<a href=\"https:\/\/spdk.io\/doc\/nvmf.html\" rel=\"noopener noreferrer nofollow\">https:\/\/spdk.io\/doc\/nvmf.html<\/a>, <a href=\"https:\/\/spdk.io\/doc\/bdev.html\" rel=\"noopener noreferrer nofollow\">https:\/\/spdk.io\/doc\/bdev.html<\/a>)<\/p>\n<\/li>\n<\/ol>\n<p><a class=\"anchor\" name=\"2\" id=\"2\"><\/a><\/p>\n<h3>2. iSER<\/h3>\n<p><u>\u0421\u043e \u0441\u0442\u043e\u0440\u043e\u043d\u044b Linux:<\/u><\/p>\n<p>\u0412 \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430\u0445 LIO \u043f\u0440\u043e\u043f\u0438\u0441\u044b\u0432\u0430\u0435\u0442\u0441\u044f enable_iser boolean=true \u043d\u0430 \u0443\u0440\u043e\u0432\u043d\u0435 \u043f\u043e\u0440\u0442\u0430\u043b\u0430:<\/p>\n<pre><code>set attribute authentication=0 demo_mode_write_protect=0 generate_node_acls=1 cache_dynamic_acls=1<\/code><\/pre>\n<p> \u0414\u043e\u043f\u043e\u043b\u043d\u0438\u0442\u0435\u043b\u044c\u043d\u043e \u0434\u043b\u044f \u0434\u0435\u0432\u0430\u0439\u0441\u0430 \u043f\u0440\u043e\u043f\u0438\u0441\u0430\u043d\u044b \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u0435 \u043e\u043f\u0446\u0438\u0438  (<a href=\"https:\/\/documentation.suse.com\/ses\/7\/html\/ses-all\/deploy-additional.html\" rel=\"noopener noreferrer nofollow\">https:\/\/documentation.suse.com\/ses\/7\/html\/ses-all\/deploy-additional.html<\/a>):<\/p>\n<pre><code>set attribute emulate_3pc=1 emulate_tpu=1 emulate_caw=1 max_write_same_len=65535 emulate_tpws=1 is_nonrot=1<\/code><\/pre>\n<p><u>\u0421\u043e \u0441\u0442\u043e\u0440\u043e\u043d\u044b VMware:<\/u><\/p>\n<pre><code>esxcli rdma iser add<\/code><\/pre>\n<p>\u0417\u0430\u0442\u0435\u043c \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430 \u0430\u043d\u0430\u043b\u043e\u0433\u0438\u0447\u043d\u0430 \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430\u043c \u043a\u0430\u043a \u0434\u043b\u044f iSCSI \u043c\u0430\u0441\u0441\u0438\u0432\u0430.<\/p>\n<p>\u041f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u044c mdadm \u0441 \u0442\u0435\u0441\u0442\u0430\u043c\u0438 FIO \u0438\u0437 \u043f\u0435\u0440\u0432\u043e\u0439 \u0447\u0430\u0441\u0442\u0438 \u043d\u0430 1 \u0412\u041c \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<div>\n<div class=\"table\">\n<table>\n<tbody>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">mdadm raid0 1\u0412\u041c<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">\u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">IOPS<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">3632.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">866.20<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u041f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4M qd=32<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">3796.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">902.14<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">352.00 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">85736.44<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=128 jobs=16<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">559.50 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">136797.19<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u0430\u044f \u0437\u0430\u043f\u0438\u0441\u044c 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">39.80 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">9712.06<\/p>\n<\/td>\n<\/tr>\n<tr>\n<td data-colwidth=\"343\" width=\"343\">\n<p align=\"left\">\u0421\u043b\u0443\u0447\u0430\u0439\u043d\u043e\u0435 \u0447\u0442\u0435\u043d\u0438\u0435 4k qd=1 fsync=1<\/p>\n<\/td>\n<td data-colwidth=\"202\" width=\"202\">\n<p align=\"right\">26.10 \u041c\u0411\/c<\/p>\n<\/td>\n<td>\n<p align=\"right\">6382.21<\/p>\n<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<\/div>\n<\/div>\n<p>\u041a\u043e\u043d\u0435\u0447\u043d\u044b\u0439 \u0433\u0440\u0430\u0444\u0438\u043a<\/p>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[],"tags":[],"class_list":["post-355695","post","type-post","status-publish","format-standard","hentry"],"_links":{"self":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts\/355695","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=355695"}],"version-history":[{"count":0,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts\/355695\/revisions"}],"wp:attachment":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=355695"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=355695"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=355695"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}