{"id":356495,"date":"2024-05-20T23:30:56","date_gmt":"2024-05-20T23:30:56","guid":{"rendered":"http:\/\/savepearlharbor.com\/?p=356495"},"modified":"-0001-11-30T00:00:00","modified_gmt":"2024-05-20T23:30:56","slug":"","status":"publish","type":"post","link":"https:\/\/savepearlharbor.com\/?p=356495","title":{"rendered":"<span>\u041d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c, \u0433\u0435\u043d\u0435\u0440\u0438\u0440\u0443\u044e\u0449\u0430\u044f \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438. \u0427\u0430\u0441\u0442\u044c 2. RL \u0430\u0433\u0435\u043d\u0442 \u0441\u043e\u0437\u0434\u0430\u0451\u0442 \u0441\u0432\u043e\u0438 \u043f\u0435\u0440\u0432\u044b\u0435 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438<\/span>"},"content":{"rendered":"<div><!--[--><!--]--><\/div>\n<div id=\"post-content-body\">\n<div>\n<div class=\"article-formatted-body article-formatted-body article-formatted-body_version-2\">\n<div xmlns=\"http:\/\/www.w3.org\/1999\/xhtml\">\n<h2>\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435<\/h2>\n<p>\u041d\u0430\u043a\u043e\u043d\u0435\u0446 \u043f\u0440\u0438\u0448\u043b\u043e \u0432\u0440\u0435\u043c\u044f \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u0438\u0442\u044c \u043a\u043e\u0434 \u0438\u0437\u00a0\u043f\u0440\u0435\u0434\u044b\u0434\u0443\u0449\u0435\u0439 \u0447\u0430\u0441\u0442\u0438, \u0432\u00a0\u043a\u043e\u0442\u043e\u0440\u043e\u0439 \u043c\u044b \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u043b\u0438 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c \u043f\u043e\u00a0\u0441\u043f\u0438\u0441\u043a\u0443 \u0441\u043b\u043e\u0451\u0432, \u0441\u00a0RL \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c\u0430\u043c\u0438! \u0421\u0435\u0433\u043e\u0434\u043d\u044f \u043c\u044b \u043f\u043e\u0441\u0442\u0430\u0432\u0438\u043c \u0437\u0430\u0434\u0430\u0447\u0443 \u0434\u043b\u044f\u00a0\u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0441\u00a0\u043f\u043e\u0434\u043a\u0440\u0435\u043f\u043b\u0435\u043d\u0438\u0435\u043c, \u043e\u043f\u0438\u0448\u0435\u043c, \u043a\u0430\u043a\u00a0\u0431\u0443\u0434\u0435\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u044c\u0441\u044f \u0432\u0437\u0430\u0438\u043c\u043e\u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0435 \u0430\u0433\u0435\u043d\u0442\u0430 \u0441\u043e \u0441\u0440\u0435\u0434\u043e\u0439 \u0438 \u043d\u0430\u00a0\u043f\u0440\u0430\u043a\u0442\u0438\u043a\u0435 \u0440\u0435\u0430\u043b\u0438\u0437\u0443\u0435\u043c \u043a\u043e\u0434 \u043d\u0430\u00a0\u044f\u0437\u044b\u043a\u0435 python.<\/p>\n<p><strong>\u0412\u0430\u0436\u043d\u043e:<\/strong> \u043d\u0430\u0441\u0442\u043e\u044f\u0442\u0435\u043b\u044c\u043d\u043e \u0440\u0435\u043a\u043e\u043c\u0435\u043d\u0434\u0443\u044e \u043e\u0437\u043d\u0430\u043a\u043e\u043c\u0438\u0442\u044c\u0441\u044f \u0441 <a href=\"https:\/\/habr.com\/ru\/articles\/763706\/\" rel=\"noopener noreferrer nofollow\">\u043f\u0435\u0440\u0432\u043e\u0439 \u0447\u0430\u0441\u0442\u044c\u044e<\/a>, \u0432\u00a0\u043a\u043e\u0442\u043e\u0440\u043e\u0439 \u043c\u044b \u0440\u0430\u0437\u043e\u0431\u0440\u0430\u043b\u0438 \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u0435 \u0441\u0435\u0442\u0438 \u043f\u043e\u00a0\u0441\u043f\u0438\u0441\u043a\u0443 \u0441\u043b\u043e\u0451\u0432, \u0432\u00a0\u044d\u0442\u043e\u0439 \u0447\u0430\u0441\u0442\u0438 \u0430\u043a\u0446\u0435\u043d\u0442 \u0441\u0438\u043b\u044c\u043d\u043e \u0441\u043c\u0435\u0449\u0451\u043d \u0432\u00a0\u0441\u0442\u043e\u0440\u043e\u043d\u0443 RL, \u0438 \u0440\u0430\u0437\u0434\u0443\u0432\u0430\u043d\u0438\u044f \u0445\u0440\u043e\u043d\u043e\u043c\u0435\u0442\u0440\u0430\u0436\u0430 \u0437\u0430\u00a0\u0441\u0447\u0451\u0442 \u043f\u043e\u0432\u0442\u043e\u0440\u0435\u043d\u0438\u0439 \u043f\u0440\u043e\u0439\u0434\u0435\u043d\u043d\u043e\u0433\u043e \u043d\u0435\u00a0\u0431\u0443\u0434\u0435\u0442.<\/p>\n<h2>\u041d\u0430\u0447\u043d\u0451\u043c \u0441 \u043f\u043e\u0441\u0442\u0430\u043d\u043e\u0432\u043a\u0438 \u0437\u0430\u0434\u0430\u0447\u0438<\/h2>\n<p>\u041f\u0435\u0440\u0432\u044b\u043c \u0434\u0435\u043b\u043e\u043c \u043e\u043f\u0438\u0448\u0435\u043c, \u0447\u0442\u043e\u00a0\u043c\u044b \u0445\u043e\u0442\u0438\u043c \u043f\u043e\u043b\u0443\u0447\u0438\u0442\u044c \u0432\u00a0\u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0435. <\/p>\n<p>\u0412\u00a0\u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u043c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u043e \u0436\u0438\u0437\u043d\u0435\u0441\u043f\u043e\u0441\u043e\u0431\u043d\u043e\u0433\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0430 \u0431\u0443\u0434\u0435\u043c \u0441\u0447\u0438\u0442\u0430\u0442\u044c \u0441\u0438\u0441\u0442\u0435\u043c\u0443, \u0432\u00a0\u043a\u043e\u0442\u043e\u0440\u043e\u0439 \u0430\u0433\u0435\u043d\u0442 \u043f\u043e\u00a0\u0432\u0445\u043e\u0434\u043d\u044b\u043c \u0434\u0430\u043d\u043d\u044b\u043c \u0431\u0443\u0434\u0435\u0442 \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u0442\u044c \u0441\u043f\u0438\u0441\u043e\u043a \u0441\u043b\u043e\u0451\u0432 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438, \u0441\u0440\u0435\u0434\u0430 \u0431\u0443\u0434\u0435\u0442 \u043f\u0435\u0440\u0435\u0432\u043e\u0434\u0438\u0442\u044c \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0435 \u0430\u0433\u0435\u043d\u0442\u0430 \u0432\u00a0\u0443\u0434\u043e\u0431\u043d\u044b\u0439 \u0434\u043b\u044f\u00a0\u043d\u0435\u0451 \u0444\u043e\u0440\u043c\u0430\u0442, \u043f\u0440\u043e\u0441\u0442\u0440\u0430\u0438\u0432\u0430\u0442\u044c \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c, \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0442\u044c \u0446\u0438\u043a\u043b \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0438 \u0432\u043e\u0437\u0440\u0430\u0449\u0430\u0442\u044c \u0435\u0433\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442 \u0430\u0433\u0435\u043d\u0442\u0443 \u0432\u00a0\u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u044f. <\/p>\n<h2>\u0423\u043f\u0440\u043e\u0449\u0435\u043d\u0438\u044f \u0434\u043b\u044f \u043f\u0435\u0440\u0432\u043e\u0439 \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438<\/h2>\n<p>\u0422\u0430\u043a \u0437\u0430\u0434\u0430\u0447\u0430 \u0438\u043c\u0435\u0435\u0442 \u043c\u043d\u043e\u0433\u043e \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u043e\u0432, \u043e\u0442 \u043a\u043e\u0442\u043e\u0440\u044b\u0445 \u0437\u0430\u0432\u0438\u0441\u0438\u0442 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0430, \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u043e \u0438\u0445 \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u0438\u0442\u044c \u0438 \u043d\u0435\u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u0437\u0430\u0444\u0438\u043a\u0441\u0438\u0440\u043e\u0432\u0430\u0442\u044c.<\/p>\n<h4>\u0413\u0438\u043f\u0435\u0440\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b \u0434\u043b\u044f \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438<\/h4>\n<p>learning_rate=0.001<br \/>batch_size=32<br \/>input_shape=[B, C, H, W]<br \/>num_classes=10 (\u0434\u0430\u0442\u0430\u0441\u0435\u0442 CIFAR10)<br \/>train_epochs=10<br \/>device=&#8217;cuda&#8217; (\u0422\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043b\u043e\u0441\u044c \u043d\u0430 T4 GPU \u0432 google colab)<\/p>\n<h4>\u0413\u0438\u043f\u0435\u0440\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b \u0441\u0440\u0435\u0434\u044b<\/h4>\n<p>layers_amount=5 &#8212; \u0434\u043b\u0438\u043d\u0430 \u043f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438 \u0441\u043b\u043e\u0451\u0432, \u0433\u0435\u043d\u0435\u0440\u0438\u0440\u0443\u0435\u043c\u043e\u0439 \u0430\u0433\u0435\u043d\u0442\u043e\u043c.<\/p>\n<p>memory_len=10 &#8212; \u0441\u043a\u043e\u043b\u044c\u043a\u043e \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u0438\u0445 \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u0439 \u0431\u0443\u0434\u0443\u0442 \u0432\u044b\u0434\u0430\u0432\u0430\u0442\u044c\u0441\u044f \u0430\u0433\u0435\u043d\u0442\u0443, \u0434\u043b\u0438\u043d\u0430 \u043a\u043e\u043b\u044c\u0446\u0435\u0432\u043e\u0433\u043e \u0431\u0443\u0444\u0435\u0440\u0430, \u043d\u043e\u0432\u044b\u0435 \u0438\u0437\u043c\u0435\u0440\u0435\u043d\u0438\u044f \u0434\u043e\u0431\u0430\u0432\u043b\u044f\u044e\u0442\u0441\u044f \u0432 \u043a\u043e\u043d\u0435\u0446.<\/p>\n<p>metrics_amount=2 (\u0411\u0443\u0434\u0435\u043c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u043f\u0440\u043e\u0441\u0442\u043e train \u0438 valid MSELosses, \u0434\u0430, \u043d\u0435 \u043e\u043f\u0442\u0438\u043c\u0430\u043b\u044c\u043d\u043e, \u043d\u043e \u044d\u0442\u043e \u0432 \u043f\u0435\u0440\u0432\u043e\u0439 \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u043d\u0435 \u043f\u0440\u0438\u043d\u0446\u0438\u043f\u0438\u0430\u043b\u044c\u043d\u044b\u0439 \u043c\u043e\u043c\u0435\u043d\u0442).<\/p>\n<h2>Observation, action, reward<\/h2>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/c31\/d7b\/aaa\/c31d7baaabf939f6b71c1829e8dc68f8.png\" alt=\"\u0423\u043f\u0440\u043e\u0449\u0451\u043d\u043d\u043e\u0435 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0435 \u0446\u0438\u043a\u043b\u0430 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0430\u0433\u0435\u043d\u0442\u0430\" title=\"\u0423\u043f\u0440\u043e\u0449\u0451\u043d\u043d\u043e\u0435 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0435 \u0446\u0438\u043a\u043b\u0430 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0430\u0433\u0435\u043d\u0442\u0430\" width=\"1205\" height=\"654\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/c31\/d7b\/aaa\/c31d7baaabf939f6b71c1829e8dc68f8.png\"\/><\/p>\n<div><figcaption>\u0423\u043f\u0440\u043e\u0449\u0451\u043d\u043d\u043e\u0435 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0435 \u0446\u0438\u043a\u043b\u0430 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0430\u0433\u0435\u043d\u0442\u0430<\/figcaption><\/div>\n<\/figure>\n<p>\u0412 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u0432\u0445\u043e\u0434\u043d\u044b\u0445 \u0434\u0430\u043d\u043d\u044b\u0445 \u0434\u043b\u044f \u0430\u0433\u0435\u043d\u0442\u0430 <strong>(observation)<\/strong> \u043c\u044b \u0432\u043e\u0437\u044c\u043c\u0451\u043c \u043c\u0435\u0442\u0440\u0438\u043a\u0438, \u043f\u043e\u043b\u0443\u0447\u0435\u043d\u043d\u044b\u0435 \u0432 \u0445\u043e\u0434\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f, \u0438 \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u044b \u0432 \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u0435, \u0440\u0430\u0432\u043d\u043e\u043c \u0434\u043b\u0438\u043d\u0435 \u043f\u0430\u043c\u044f\u0442\u0438. \u0422.\u0435. \u0430\u0433\u0435\u043d\u0442 \u043f\u0440\u0438\u043d\u0438\u043c\u0430\u0435\u0442 \u0440\u0435\u0448\u0435\u043d\u0438\u0435 \u043e \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u0438 \u043d\u043e\u0432\u044b\u0445 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0435\u0439 \u043d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 \u043f\u0430\u0440: (\u043f\u0440\u0435\u0434\u044b\u0434\u0443\u0449\u0430\u044f \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u0430, \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u043f\u043e\u0441\u043b\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f)<\/p>\n<p>obs={<br \/> &#8216;metrics&#8217;: shape=[mem_len, amount_of_metrics, train_epochs], <br \/>&#8216;architectures&#8217;: shape=[mem_lem, layers_amount],<br \/>}<\/p>\n<p>\u0414\u0435\u0439\u0441\u0442\u0432\u0438\u0435 \u0430\u0433\u0435\u043d\u0442\u0430 <strong>(action)<\/strong> \u0431\u0443\u0434\u0435\u0442 \u0432\u044b\u0433\u043b\u044f\u0434\u0435\u0442\u044c \u043a\u0430\u043a \u043d\u0430\u0431\u043e\u0440 \u0447\u0438\u0441\u0435\u043b \u0444\u0438\u043a\u0441\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u043e\u0439 \u0434\u043b\u0438\u043d\u044b, \u043a\u0430\u0436\u0434\u043e\u043c\u0443 \u0438\u0437 \u043a\u043e\u0442\u043e\u0440\u044b\u0445 \u0441\u043e\u043e\u0442\u0432\u0435\u0442\u0441\u0442\u0432\u0443\u0435\u0442 \u0441\u043b\u043e\u0439 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438 \u0438\u0437 \u0441\u043f\u0438\u0441\u043a\u0430 \u0432\u043e\u0437\u043c\u043e\u0436\u043d\u044b\u0445 (actions_set). <\/p>\n<p>action = []<br \/>action.shape=[max_layers_amount]<\/p>\n<p>\u041d\u0430\u0433\u0440\u0430\u0434\u0430 \u0430\u0433\u0435\u043d\u0442\u0430 <strong>(reward)<\/strong> &#8212; \u0437\u0430\u0447\u0430\u0441\u0442\u0443\u044e \u0441\u0430\u043c\u0430\u044f \u0441\u043b\u043e\u0436\u043d\u0430\u044f \u0434\u043b\u044f \u0444\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u0447\u0430\u0441\u0442\u044c, \u0442\u0430\u043a \u043a\u0430\u043a \u043e\u0442 \u044d\u0442\u043e\u0433\u043e \u0437\u0430\u0432\u0438\u0441\u0438\u0442, \u0447\u0442\u043e \u0430\u0433\u0435\u043d\u0442 \u043f\u0440\u0438\u043c\u0435\u0442 \u0437\u0430 \u0445\u043e\u0440\u043e\u0448\u043e, \u0430 \u0447\u0442\u043e \u0437\u0430 \u043f\u043b\u043e\u0445\u043e. \u041c\u044b \u0431\u0443\u0434\u0435\u043c \u043d\u0430\u0433\u0440\u0430\u0436\u0434\u0430\u0442\u044c \u0430\u0433\u0435\u043d\u0442\u0430 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<ul>\n<li>\n<p><strong>metrics_optimization_reward<\/strong> &#8212; \u0432 \u0441\u043b\u0443\u0447\u0430\u0435, \u043a\u043e\u0433\u0434\u0430 \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u0443\u043b\u0443\u0447\u0448\u0430\u044e\u0442\u0441\u044f, \u0430\u0433\u0435\u043d\u0442 \u0437\u0430\u0441\u043b\u0443\u0436\u0438\u0432\u0430\u0435\u0442 \u043f\u043e\u043e\u0449\u0440\u0435\u043d\u0438\u0435. \u0440\u0430\u0441\u0441\u0447\u0438\u0442\u0430\u0435\u043c \u043d\u0430\u0433\u0440\u0430\u0434\u0443 \u043a\u0430\u043a \u0440\u0430\u0437\u043d\u0438\u0446\u0443 \u043c\u0435\u0436\u0434\u0443 \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u0438\u043c\u0438 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f\u043c\u0438 \u043e\u0442\u0441\u043b\u0435\u0436\u0438\u0432\u0430\u0435\u043c\u044b\u0445 \u043c\u0435\u0442\u0440\u0438\u043a \u0438 \u043d\u043e\u0432\u044b\u043c\u0438, \u0432 \u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e\u0441\u0442\u0438 \u043e\u0442 \u0437\u043d\u0430\u043a\u0430 \u043f\u043e\u043b\u0443\u0447\u0438\u0432\u0448\u0435\u0433\u043e\u0441\u044f \u0447\u0438\u0441\u043b\u0430 \u043d\u0430\u0433\u0440\u0430\u0434\u0430 \u043c\u043e\u0436\u0435\u0442 \u043f\u0440\u0435\u0432\u0440\u0430\u0442\u0438\u0442\u044c\u0441\u044f \u0432 \u043d\u0430\u043a\u0430\u0437\u0430\u043d\u0438\u0435 \u0437\u0430 \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u0435 \u043c\u043e\u0434\u0435\u043b\u0438 \u0445\u0443\u0436\u0435, \u0447\u0435\u043c \u0432 \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u0438\u0439 \u0440\u0430\u0437.<br \/><em>\u0417\u0430\u043c\u0435\u0447\u0430\u043d\u0438\u0435 1:<\/em> \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u043f\u0435\u0440\u0432\u043e\u0439 \u044d\u043f\u043e\u0445\u0438 \u043c\u043e\u0433\u0443\u0442 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u044f\u0442\u044c \u0441\u043e\u0431\u043e\u0439 \u0432\u044b\u0431\u0440\u043e\u0441\u044b \u0431\u043e\u043b\u044c\u0448\u043e\u0439 \u0432\u0435\u043b\u0438\u0447\u0438\u043d\u044b, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u0438\u0445 \u043c\u044b \u0431\u0443\u0434\u0435\u043c \u043f\u0440\u043e\u043f\u0443\u0441\u043a\u0430\u0442\u044c, \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u044f \u043f\u043e \u0441\u0443\u0442\u0438 train_epochs + 1 \u044d\u043f\u043e\u0445 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f.<br \/><em>\u0417\u0430\u043c\u0435\u0447\u0430\u043d\u0438\u0435 2:<\/em> \u041d\u0435 \u0441\u0430\u043c\u044b\u0435 \u043e\u043f\u0442\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u0435 \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u044b \u043c\u043e\u0433\u0443\u0442 \u0432\u044b\u0434\u0430\u0432\u0430\u0442\u044c \u043e\u0433\u0440\u043e\u043c\u043d\u044b\u0435 \u043f\u043e\u043a\u0430\u0437\u0430\u0442\u0435\u043b\u0438 \u043c\u0435\u0442\u0440\u0438\u043a\u0438. \u0414\u043b\u044f \u0441\u0433\u043b\u0430\u0436\u0438\u0432\u0430\u043d\u0438\u044f \u044d\u0442\u043e\u0433\u043e \u043c\u043e\u043c\u0435\u043d\u0442\u0430 \u043c\u044b \u0431\u0443\u0434\u0435\u043c \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0438\u0432\u0430\u0442\u044c \u0441\u0432\u0435\u0440\u0445\u0443 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f \u044d\u043c\u043f\u0438\u0440\u0438\u0447\u0435\u0441\u043a\u0438 \u043f\u043e\u0434\u043e\u0431\u0440\u0430\u043d\u043d\u044b\u043c \u043f\u043e\u0440\u043e\u0433\u043e\u043c (\u0432 \u0447\u0430\u0441\u0442\u043d\u043e\u0441\u0442\u0438 \u0431\u044b\u043b\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u043e \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 100).<br \/><em>\u0417\u0430\u043c\u0435\u0447\u0430\u043d\u0438\u0435 3:<\/em> \u0438\u043d\u043e\u0433\u0434\u0430 \u0432 \u043e\u0441\u043e\u0431\u043e \u043f\u0435\u0447\u0430\u043b\u044c\u043d\u044b\u0445 \u0441\u043b\u0443\u0447\u0430\u044f\u0445 \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u0432\u043e\u043e\u0431\u0449\u0435 \u043c\u043e\u0433\u0443\u0442 \u043f\u0440\u0438\u043d\u044f\u0442\u044c \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 nan \u0438\u043b\u0438 inf, \u0431\u0443\u0434\u0435\u043c \u0442\u0430\u043a\u0436\u0435 \u0437\u0430\u043c\u0435\u043d\u044f\u0442\u044c \u0442\u0430\u043a\u0438\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f \u043d\u0430 NAN_NUM = 100.<\/p>\n<\/li>\n<li>\n<p><strong>creation_successfull_reward<\/strong> &#8212; \u0435\u0441\u043b\u0438 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c \u0432\u043e\u043e\u0431\u0449\u0435 \u043f\u043e\u043b\u0443\u0447\u0438\u043b\u043e\u0441\u044c \u0441\u043e\u0437\u0434\u0430\u0442\u044c, \u0443\u0436\u0435 \u043d\u0435\u043f\u043b\u043e\u0445\u043e, \u0431\u0443\u0434\u0435\u043c \u043d\u0430\u0433\u0440\u0430\u0436\u0434\u0430\u0442\u044c \u0430\u0433\u0435\u043d\u0442\u0430 \u0437\u0430 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u043f\u043e\u043b\u0443\u0447\u0438\u043b\u043e\u0441\u044c \u043a\u043e\u0440\u0440\u0435\u043a\u0442\u043d\u043e \u043f\u043e\u0441\u0442\u0440\u043e\u0438\u0442\u044c \u0438 \u0448\u0442\u0440\u0430\u0444\u043e\u0432\u0430\u0442\u044c \u0432 \u043e\u0431\u0440\u0430\u0442\u043d\u043e\u043c \u0441\u043b\u0443\u0447\u0430\u0435. \u0414\u043b\u044f \u044d\u0442\u043e\u0433\u043e \u0431\u044b\u043b\u0438 \u0441\u043e\u0437\u0434\u0430\u043d\u044b \u043f\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u044b\u0435 NN_CREATE_SUCCESS_REWARD = 1<br \/>NN_CREATE_NOT_SUCCESS_PENALTY = -1<\/p>\n<\/li>\n<li>\n<p><strong>optimal_depth_reward<\/strong> &#8212; \u044d\u0442\u0430 \u0447\u0430\u0441\u0442\u044c \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0437\u0430\u0434\u0435\u043b\u043e\u043c \u043d\u0430 \u0431\u0443\u0434\u0443\u0449\u0435\u0435, \u043d\u0430 \u0442\u0435\u043a\u0443\u0449\u0438\u0439 \u043c\u043e\u043c\u0435\u043d\u0442 \u043d\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442\u0441\u044f<\/p>\n<\/li>\n<\/ul>\n<h2>RL Custom environment<\/h2>\n<p>\u0414\u043b\u044f \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u0431\u0443\u0434\u0435\u0442 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c\u0441\u044f python \u0441 \u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0430\u043c\u0438 <a href=\"https:\/\/gymnasium.farama.org\/content\/basic_usage\/\" rel=\"noopener noreferrer nofollow\">gymnasium<\/a> \u0438 <a href=\"https:\/\/stable-baselines3.readthedocs.io\/en\/master\/guide\/examples.html\" rel=\"noopener noreferrer nofollow\">stable-baselines3<\/a>.<\/p>\n<p>\u041d\u0430\u043f\u043e\u043c\u043d\u044e \u043e\u0441\u043d\u043e\u0432\u043d\u0443\u044e \u0441\u0442\u0440\u0443\u043a\u0442\u0443\u0440\u0443 \u0441\u0440\u0435\u0434\u044b \u0432 gymnasium:<\/p>\n<pre><code>class Env(gym.Env):   def __init__(self):     # init all required variables     # action and observation spaces must be here!      self.action_space = spaces.Box(...)     self.observation_space = spaces.MultiDiscrete(...)     pass    def seed(self):      seed = 42      return [seed]    def step(self, action):     # most important metrics     # do manipulations with action,      # return new observation and reward     return obs, reward, _, done, info    def reset(self):     # set variables to start state     return obs    def render(self):     # all GUI parts should be here     # may be empty     pass<\/code><\/pre>\n<p>\u041f\u0440\u043e\u0439\u0434\u0451\u043c\u0441\u044f \u043f\u043e \u043a\u0430\u0436\u0434\u043e\u043c\u0443 \u043f\u0443\u043d\u043a\u0442\u0443.<\/p>\n<h3>__init__, \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0435 \u043f\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u044b\u0445<\/h3>\n<pre><code>  def __init__(self, opt_cls=None, crit=None, trn_ldr=None, vld_ldr=None, render_mode=None):       '''       Initialization of environment variables       '''        # here is learning params       self.NN_PARAMS = {           'lr': 0.001,           'num_classes': 10, # TODO adaptive num_classes           'train_epochs': 10,           'last_nets_metrics_memory_len': 10,           'layers_amount': 5,           'amount_of_metrics': 2,       }        # here contains NN learning metrics       # uses for observations       # shape=[MEMORY_LEN, N_METRICS, EPOCHS]       self.NN_PARAMS['metrics'] = np.zeros((           self.NN_PARAMS['last_nets_metrics_memory_len'],           self.NN_PARAMS['amount_of_metrics'],           self.NN_PARAMS['train_epochs'],       ))        # here contains last architectures       # uses for observations       # shape=[MEMORY, N_LAYERS]       self.NN_PARAMS['last_nets_architectures'] = np.zeros((              self.NN_PARAMS['last_nets_metrics_memory_len'],              self.NN_PARAMS['layers_amount'],           ))        # Variables for NN       self.Net = self.NN()       self.train_dataloader = trn_ldr       self.valid_dataloader = vld_ldr       self.optimizer_class = opt_cls       self.optimizer = None       self.criterion = crit       self.device = 'cuda'        self.nngenerator = self.nnGenerator()        self.last_obs = None        # Action space describes what agent will give to environment       # shape=[ACTION_SET_SIZE, N_LAYERS]       self.action_space = spaces.MultiDiscrete(         [len(self.actions_set)] * self.NN_PARAMS['layers_amount'],          seed=42)        # Observation space describes what agent       # will take from enviromnent as observation       # shape=dict{       #  METRICS, shape as NN_PARAMS['metrics'],       #  ARCHITECTURES, shape as NN_PARAMS['layers_amount'],       # }       self.observation_space = spaces.Dict(           {           'last_nets_metrics_memory': spaces.Box(               low=0,               high=100,               shape=(self.NN_PARAMS['last_nets_metrics_memory_len'],                      self.NN_PARAMS['amount_of_metrics'],                      self.NN_PARAMS['train_epochs'],                      )),           'last_nets_architectures': spaces.Box(               low=0,               high=len(self.actions_set),               shape=(self.NN_PARAMS['last_nets_metrics_memory_len'],                      self.NN_PARAMS['layers_amount'],                      ))           }       )        # some variable for collecting statistics        self.statistics = {         'episode_rewards': [],         'global_rewards': [],         'made_steps': [],       }        self.seed()       assert render_mode is None or render_mode in self.metadata[\"render_modes\"]       self.render_mode = render_mode <\/code><\/pre>\n<p>\u041d\u0430\u00a0\u0432\u0445\u043e\u0434 \u043f\u0440\u0438\u043d\u0438\u043c\u0430\u044e\u0442\u0441\u044f \u043e\u043f\u0442\u0438\u043c\u0430\u0439\u0437\u0435\u0440, \u0444\u2011\u044f \u043e\u0448\u0438\u0431\u043a\u0438 \u0438 \u0434\u0430\u0442\u0430\u043b\u043e\u0430\u0434\u0435\u0440\u044b.<\/p>\n<p>\u0417\u0434\u0435\u0441\u044c \u0443\u043a\u0430\u0437\u0430\u043d\u044b \u0432\u0441\u0435 \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u044b\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b, \u0441\u043e\u0437\u0434\u0430\u0451\u0442\u0441\u044f \u043e\u0431\u044a\u0435\u043a\u0442 \u043f\u0443\u0441\u0442\u043e\u0439 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438 Net, \u0432\u00a0\u043a\u043e\u0442\u043e\u0440\u0443\u044e \u043c\u044b \u043f\u043e\u0437\u0436\u0435 \u0431\u0443\u0434\u0435\u0442 \u043f\u043e\u043c\u0435\u0449\u0430\u0442\u044c \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u043d\u0443\u044e \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c, \u0430\u00a0\u0442\u0430\u043a\u0436\u0435 nnGenerator&#8217;a, \u0447\u0442\u043e\u00a0\u0437\u0430\u043d\u0438\u043c\u0430\u0435\u0442\u0441\u044f \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u043e\u0439 \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u044f \u0430\u0433\u0435\u043d\u0442\u0430 \u0438 \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u044f \u0441\u0430\u043c\u043e\u0439 \u0441\u0435\u0442\u0438. \u041e\u0431\u044f\u0437\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u043c\u0438 \u043f\u043e\u043b\u044f\u043c\u0438 \u044f\u0432\u043b\u044f\u044e\u0442\u0441\u044f action_space \u0438 observation_space. \u0411\u0435\u0437\u00a0\u0438\u0445 \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u0438\u044f \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u0435 \u0441\u0440\u0435\u0434\u044b \u043d\u0435\u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e.<\/p>\n<p><em>\u0417\u0430\u043c\u0435\u0447\u0430\u043d\u0438\u0435: \u043a\u00a0\u0432\u044b\u0431\u043e\u0440\u0443 \u043f\u0440\u043e\u0441<\/em>\u0442\u0440\u0430\u043d\u0441\u0442\u0432\u0430 \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0439 \u0438 \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u0439 \u0441\u043b\u0435\u0434\u0443\u0435\u0442 \u043f\u043e\u0434\u0445\u043e\u0434\u0438\u0442\u044c \u0441\u00a0\u0443\u043c\u043e\u043c, \u0442\u0430\u043a \u043a\u0430\u043a\u00a0\u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c\u044b \u0430\u0433\u0435\u043d\u0442\u0430 \u0438\u0437\u00a0stable\u2011baselines3\u00a0\u043c\u043e\u0433\u0443\u0442 \u043d\u0435\u00a0\u043f\u043e\u0434\u0434\u0435\u0440\u0436\u0438\u0432\u0430\u0442\u044c \u0442\u043e\u0442 \u0444\u043e\u0440\u043c\u0430\u0442, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0432\u044b \u0432\u044b\u0431\u0435\u0440\u0435\u0442\u0435. \u041e\u00a0\u0442\u043e\u043c, \u043a\u0430\u043a\u0438\u0435 \u043f\u0440\u043e\u0441\u0442\u0440\u0430\u043d\u0441\u0442\u0432\u0430 \u043c\u043e\u0436\u043d\u043e \u0432\u044b\u0431\u0440\u0430\u0442\u044c \u0434\u043b\u044f\u00a0\u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c\u043e\u0432, \u043c\u043e\u0436\u043d\u043e \u043f\u043e\u0441\u043c\u043e\u0442\u0440\u0435\u0442\u044c <a href=\"https:\/\/stable-baselines3.readthedocs.io\/en\/master\/guide\/algos.html\" rel=\"noopener noreferrer nofollow\">\u0442\u0443\u0442.<\/a><strong><br \/><\/strong>\u041c\u044b \u0431\u0443\u0434\u0435\u043c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c PPO(&#8216;MultiInputPolicy&#8217;)\u00a0\u2014 proximal policy optimization \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c, \u043e\u00a0\u043a\u043e\u0442\u043e\u0440\u043e\u043c \u043c\u043e\u0436\u043d\u043e \u043f\u0440\u043e\u0447\u0438\u0442\u0430\u0442\u044c <a href=\"https:\/\/stable-baselines3.readthedocs.io\/en\/master\/modules\/ppo.html\" rel=\"noopener noreferrer nofollow\">\u0432\u00a0\u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0430\u0446\u0438\u0438 \u0442\u0443\u0442.<\/a> <\/p>\n<h3>reset, \u043e\u0431\u043d\u0443\u043b\u0435\u043d\u0438\u0435 \u043f\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u044b\u0445, \u043f\u0435\u0440\u0435\u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0441\u0440\u0435\u0434\u044b<\/h3>\n<pre><code>   def reset(self, seed=None, options=None):       '''       Reset the env,       Set all changed in training proccess variables to zero       (or noise, dependse on your realization)        seed: list, list of random seeds (depricated)       options: list, additional options (future)       '''       super().reset(seed=seed)       self.Net = self.NN()        current_obs = {           'last_nets_metrics_memory': np.zeros((               self.NN_PARAMS['last_nets_metrics_memory_len'],               self.NN_PARAMS['amount_of_metrics'],               self.NN_PARAMS['train_epochs'],              ) ),           'last_nets_architectures': np.zeros((              self.NN_PARAMS['last_nets_metrics_memory_len'],              self.NN_PARAMS['layers_amount'],           ) )            }       self.NN_PARAMS['metrics'] = np.zeros((           self.NN_PARAMS['last_nets_metrics_memory_len'],           self.NN_PARAMS['amount_of_metrics'],           self.NN_PARAMS['train_epochs'],       ))       self.NN_PARAMS['last_nets_architectures'] = np.zeros((              self.NN_PARAMS['last_nets_metrics_memory_len'],              self.NN_PARAMS['layers_amount'],           ))        self.last_obs = current_obs       self.episode_reward = 0       self.current_it = 1        self.statistics['episode_rewards'] = []       self.statistics['made_steps'] = []        return current_obs, {'none': None} <\/code><\/pre>\n<p>\u0422\u0443\u0442 \u043d\u0438\u0447\u0435\u0433\u043e \u043d\u0435\u043e\u0431\u044b\u0447\u043d\u043e\u0433\u043e, \u0437\u0430\u043d\u0443\u043b\u044f\u0435\u043c \u043c\u0435\u0442\u0440\u0438\u043a\u0438, \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u044b, \u0443\u0434\u0430\u043b\u044f\u0435\u043c \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c, \u0441\u043e\u0437\u0434\u0430\u0451\u043c \u043f\u0443\u0441\u0442\u043e\u0435 \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u0435. \u0412 \u043f\u043b\u0430\u043d\u0430\u0445 \u0440\u0435\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u0442\u044c \u0437\u0430\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u0435 \u043f\u0435\u0440\u0432\u043e\u0433\u043e \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u044f \u0448\u0443\u043c\u043e\u043c.<\/p>\n<h3>step, \u043f\u0435\u0440\u0435\u0432\u043e\u0434\u0438\u043c action \u0432 new_obs \u0438 reward<\/h3>\n<pre><code>  def step(self, action):       '''           Main environment function            takes action, creates NN, train NN, calc new obs and reward            action: list, shape of action is like action_space        return:       obs: np.array,       reward: float,       done = False, end of agent training flag, needed when your actions       achieved some finish state       info: dict, you may need to add some extra information, put it here        Algorithm       1) parse action       2) generate NN       3) update optimizer, prepare for training       4) train NN, collect metrics       5) calculate reward       6) collect statistics       7) create new observation       8) return obs, reward, done, info        '''       reward = 0       done = False       info = {}       self.nngenerator.parse_action(action, self.actions_set)       test_b = self.get_test_batch()       success_state, net = self.nngenerator.generateNN(n_classes=self.NN_PARAMS['num_classes'], test_batch=test_b)        new_metrics = None       if success_state == True: # NN created_correctly         self.NN_PARAMS['last_nets_architectures'] = np.roll(self.NN_PARAMS['last_nets_architectures'], -1, axis=0)         self.NN_PARAMS['last_nets_architectures'][-1] = action         self.Net.layers = net         display(self.Net)         self.Net = self.Net.to(self.device)         self.optimizer = self.optimizer_class(self.Net.parameters(), lr=self.NN_PARAMS['lr'])         new_metrics = self.train()         reward = self.calc_reward(success_state,                                 self.nngenerator.get_nn_len(),                                 new_metrics,                                 )         current_obs = self.create_obs()        self.last_obs = current_obs        print('Reward: ', reward)       self.episode_reward = reward       self.current_it += 1       return current_obs, self.episode_reward, None, done, info <\/code><\/pre>\n<p>\u0412 \u044d\u0442\u043e\u043c \u043c\u0435\u0442\u043e\u0434\u0435 \u043f\u0440\u043e\u0438\u0441\u0445\u043e\u0434\u0438\u0442 \u0432\u0441\u044f \u043c\u0430\u0433\u0438\u044f. \u0421\u043e\u0437\u0434\u0430\u0451\u043c \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c, \u043e\u0431\u0443\u0447\u0430\u0435\u043c, \u0441\u043e\u0431\u0438\u0440\u0430\u0435\u043c \u043c\u0435\u0442\u0440\u0438\u043a\u0438, \u0441\u0447\u0438\u0442\u0430\u0435\u043c \u043d\u0430\u0433\u0440\u0430\u0434\u0443, \u0444\u0438\u043a\u0441\u0438\u0440\u0443\u0435\u043c \u043f\u0440\u0438\u0431\u044b\u043b\u044c \u30fd( \u2580\u033f \u0139\u032f \u2580\u033f)\u30ce.<\/p>\n<p>\u0412\u0441\u043f\u043e\u043c\u043e\u0433\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u043c\u0438 \u043c\u0435\u0442\u043e\u0434\u0430\u043c\u0438 \u0437\u0434\u0435\u0441\u044c \u044f\u0432\u043b\u044f\u044e\u0442\u0441\u044f get_test_batch, calc_reward \u0438 create_obs.<br \/>\u0412\u044b\u0433\u043b\u044f\u0434\u044f\u0442 \u043e\u043d\u0438 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<pre><code>  def calc_reward(self, nn_created_correctly_flag, nn_len, last_train_metrics):     '''        calculate agent reward         nn_created_correctly_flag: bool,        if True - NN was built successfully,        we can calculate other parts of reward,        otherwise - agent takes NN_CREATE_NOT_SUCCESS_PENALTY only         nn_len: int,        that var needed for depth decreasing reward        # Not used (future) #         last_train_metrcis: np.array,        contains last training loop metrics        for metrics_optimization_reward         return reward: float, sum of all reward parts      '''      reward = 0     # TODO reward for decreasing nn depth     optimal_depth_reward = 0     # reward by metrics     metrics_optimization_reward = 0     # reward for successfull nn creation     creation_successfull_reward = 0     if nn_created_correctly_flag == True:       # do not reward agent if creation is not succeed       creation_successfull_reward += self.NN_CREATE_SUCCESS_REWARD        last_metrics = self.NN_PARAMS['metrics'][-1]       last_train_metrics = np.nan_to_num(x=last_train_metrics, nan=self.NAN_NUM)       last_train_metrics = np.minimum(last_train_metrics,                           np.ones(shape=(self.NN_PARAMS['amount_of_metrics'], self.NN_PARAMS['train_epochs'])) * self.NAN_NUM                           )        tmp_r = np.min(last_metrics,axis=1) - np.min(np.array(last_train_metrics), axis=1)         self.NN_PARAMS['metrics'] = np.roll(self.NN_PARAMS['metrics'], -1, axis=0)       self.NN_PARAMS['metrics'][-1] = last_train_metrics       metrics_optimization_reward = np.sum(self.METRICS_OPTIMIZATION_FACTOR * tmp_r)        optimal_depth_reward = (self.NN_PARAMS['layers_amount'] - nn_len)       optimal_depth_reward *= self.DEPTH_REWARD_FACTOR      else:       creation_successfull_reward += self.NN_CREATE_NOT_SUCCESS_PENALTY      reward += optimal_depth_reward     reward += metrics_optimization_reward     reward += creation_successfull_reward      return reward      def get_test_batch(self):     '''         return: batch: np.array, one batch         for input_shape in NN building algorithm      '''     batch = None     for  b, _ in self.train_dataloader:       batch = b     return batch       def create_obs(self):     '''        return obs: dict, new observation     '''      obs = {           'last_nets_metrics_memory': self.NN_PARAMS['metrics'],           'last_nets_architectures': self.NN_PARAMS['last_nets_architectures']            }     return obs<\/code><\/pre>\n<p>get_test_batch \u0434\u043e\u0441\u0442\u0430\u0451\u0442 \u0438\u0437 \u0434\u0430\u0442\u0430\u0441\u0435\u0442\u0430 \u043e\u0434\u0438\u043d \u0431\u0430\u0442\u0447, \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u044b\u0439 \u0434\u043b\u044f \u0432\u044b\u0447\u0438\u0441\u043b\u0435\u043d\u0438\u044f \u0432\u0445\u043e\u0434\u043d\u043e\u0433\u043e shape&#8217;a \u0434\u0430\u043d\u043d\u044b\u0445 \u043f\u0440\u0438 \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u0438 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438.<\/p>\n<p>create_obs \u0432\u043e\u0437\u0432\u0440\u0430\u0449\u0430\u0435\u0442 \u043d\u043e\u0432\u043e\u0435 \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u0435, \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u044f\u044f \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u0438 \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u044b \u0432 \u0441\u043b\u043e\u0432\u0430\u0440\u044c \u0444\u043e\u0440\u043c\u0430\u0442\u0430, \u0437\u0430\u0434\u0430\u043d\u043d\u043e\u0433\u043e \u0432 observation_space.<\/p>\n<p>calc_reward \u043f\u043e\u0441\u0447\u0438\u0442\u044b\u0432\u0430\u0435\u0442, \u043a\u0430\u043a\u043e\u0439 \u043d\u0430\u0433\u0440\u0430\u0434\u044b (\u0438\u043b\u0438 \u043d\u0430\u043a\u0430\u0437\u0430\u043d\u0438\u044f) \u0431\u0443\u0434\u0435\u0442 \u0443\u0434\u043e\u0441\u0442\u043e\u0435\u043d \u0430\u0433\u0435\u043d\u0442. <\/p>\n<h2>\u0422\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u0435 <\/h2>\n<p>\u041f\u0435\u0440\u0432\u044b\u043c \u0434\u0435\u043b\u043e\u043c \u043d\u0443\u0436\u043d\u043e \u0437\u0430\u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u043d\u0430\u0448\u0443 \u0441\u0430\u043c\u043e\u0434\u0435\u043b\u044c\u043d\u0443\u044e \u0441\u0440\u0435\u0434\u0443 \u0432 gymnasium \u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0435.<br \/>\u0414\u043b\u044f \u044d\u0442\u043e\u0433\u043e \u043c\u044b \u043e\u0431\u0435\u0440\u043d\u0451\u043c \u043d\u0430\u0448 \u043a\u043e\u0434 \u0441\u0440\u0435\u0434\u044b \u043e\u0441\u043e\u0431\u044b\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c, \u0447\u0442\u043e\u0431\u044b \u0432\u0441\u0435 \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u044b\u0435 \u0444\u0430\u0439\u043b\u044b \u0441\u0433\u0435\u043d\u0435\u0440\u0438\u0440\u043e\u0432\u0430\u043b\u0438\u0441\u044c. \u0412\u043e\u0442 \u0442\u0430\u043a \u044d\u0442\u043e \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441 \u0448\u0430\u0431\u043b\u043e\u043d\u043e\u043c gym \u0441\u0440\u0435\u0434\u044b, \u043e\u043f\u0438\u0441\u0430\u043d\u043d\u043e\u043c \u0432\u044b\u0448\u0435:<\/p>\n<pre><code>import os   def make_dirs(env_name: str):     os.makedirs('gym_{}'.format(env_name))     os.makedirs('gym_{}\/envs'.format(env_name))     pass   def make_files(env_name: str, stage_names):     registry_init = 'gym_{}\/__init__.py'.format(env_name)     with open(registry_init, 'w') as f:         f.write('from gym.envs.registration import register\\n\\n')         for name in stage_names:             register_str = \"register(\\n    id='{}',\\n    entry_point='gym_{}.envs:{}',\\n)\\n\".format(name,                                                                                                      env_name,                                                                                                     ''.join([s.capitalize()for s in name.split('_')]))             f.write(register_str)     envs_init = 'gym_{}\/envs\/__init__.py'.format(env_name)     with open(envs_init, 'w') as f:         for name in stage_names:             import_str = \"from gym_{}.envs.{} import {}\\n\".format(env_name, name,                                                                   ''.join([s.capitalize() for s in name.split('_')]))             f.write(import_str)      head = \"\"\"#all required imports here  \"\"\"      tail = \"\"\"   def __init__(self):     # init all required variables     # action and observation spaces must be here!      self.action_space = spaces.Box(...)     self.observation_space = spaces.MultiDiscrete(...)     pass    def seed(self):      seed = 42      return [seed]    def step(self, action):     # most important metrics     # do manipulations with action,      # return new observation and reward     return obs, reward, _, done, info    def reset(self):     # set variables to start state     return obs    def render(self):     # all GUI parts should be here     # may be empty     pass         \"\"\"     for name in stage_names:         env_class = 'class {}(gym.Env):'.format(''.join([s.capitalize() for s in name.split('_')]))         with open('gym_{}\/envs\/{}.py'.format(env_name, name), 'w') as f:             f.write(head + env_class + tail)     pass   def make_setup(env_name):     with open('setup.py', 'w') as f:         tmp = [             'from setuptools import setup\\n\\n',             \"setup(name='gym_{}',\\n\".format(env_name),             \"    version='0.0.1',\\n\",             \"    install_requires=['gym']  # And any other dependencies foo needs\\n\",             \")\\n\",         ]         f.writelines(tmp)   if __name__ == '__main__':     env_name = 'env'# input('Please type your env\\'s name (string):')     env_stage_num = 1 #int(input('Type the stage number of your Env (int):'))      print(         'Please type yout stage names\\n(start with alphabet, lowercase and number, split by underscore(_), e.g. my_env_v1)')     env_stage_names = ['env']     #for i in range(env_stage_num):     #    env_stage_names.append(input(\"Stage {}s name:\".format(i)))      make_dirs(env_name)     make_files(env_name, env_stage_names)     #generate_setup = input('Generate `setup.py`?[y\/n]')      generate_setup = 'y'     if generate_setup.lower() == 'y':         make_setup(env_name) <\/code><\/pre>\n<p>\u0414\u0430\u043b\u0435\u0435 \u043c\u044b \u0440\u0435\u0433\u0438\u0441\u0442\u0440\u0438\u0440\u0443\u0435\u043c \u043d\u0430\u0448\u0443 \u0441\u0440\u0435\u0434\u0443 \u0432 gym, \u0441\u043e\u0437\u0434\u0430\u0451\u043c \u0435\u0451 \u0438 \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0435\u043c \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u0435:<\/p>\n<pre><code># for google colab #!pip install stable_baselines3 #!pip install shimmy==0.2.1 #!pip install gymnasium from stable_baselines3 import * from stable_baselines3.common.vec_env import DummyVecEnv, SubprocVecEnv from stable_baselines3.common.env_util import make_vec_env import gymnasium as gym import numpy as np import torch import torchvision.datasets as dsets from torchvision import transforms from torch.utils.data import Dataset from gymnasium.envs.registration import register  register(     id='gym_env\/env_v1-v0',     entry_point='gym_env.envs:Env',     max_episode_steps=300, )  # download pytorch CIFAR10 Dataset train_data = dsets.CIFAR10(root = '.\/data', train = True,                         transform = transforms.ToTensor(), download = True)  test_data = dsets.CIFAR10(root = '.\/data', train = False,                        transform = transforms.ToTensor())  train_samples = np.array(train_data.data)[:15000].transpose((0,3,1,2)) # convert to [B, C, H, W] train_labels = np.array(train_data.targets)[:15000]  # simple custom dataset template class myDataset(Dataset):     def __init__(self, X, y):        self.X = X       self.y = y      def __len__(self):         return len(self.y)      def __getitem__(self, idx):         x_ = self.X[idx]         y_ = self.y[idx]         return x_, y_  dataset = myDataset(X=train_samples, y=train_labels)  train_set, valid_set = torch.utils.data.random_split(dataset, [0.8, 0.2], generator=torch.Generator().manual_seed(42))   train_loader = torch.utils.data.DataLoader(   train_set,   batch_size=32,   shuffle=True,   drop_last=True)  valid_loader = torch.utils.data.DataLoader(   valid_set,   batch_size=32,   drop_last=True,   shuffle=True)   # some environment params # optimizer should be class # because every NN building needs # pass model.params() to optimizer, # so we will recreate optimizer every # time new NN will be created successfully  env_params = {     'opt_cls': torch.optim.RAdam,     'trn_ldr': train_loader,     'vld_ldr': valid_loader,     'crit': torch.nn.MSELoss(), } env = gymnasium.make('gym_env\/env_v1', **env_params) # Proximal Policy Optimization # doc: https:\/\/stable-baselines3.readthedocs.io\/en\/master\/modules\/ppo.html  model = PPO(\"MultiInputPolicy\", env, verbose=1)  model.learn(total_timesteps=10000)   <\/code><\/pre>\n<p>\u041f\u0440\u043e\u0446\u0435\u0441\u0441 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0432\u044b\u0433\u043b\u044f\u0434\u0438\u0442 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<pre><code>Reward:  -1 Reward:  -1 Reward:  -1 Reward:  -1 Reward:  -1 NN(   (layers): Sequential(     (0): Sequential(       (0): AvgPool2d(kernel_size=2, stride=2, padding=0)       (1): Dropout2d(p=0.1, inplace=False)       (2): Dropout2d(p=0.1, inplace=False)       (3): Conv2d(3, 2, kernel_size=(5, 5), stride=(1, 1))       (4): ReLU(inplace=True)       (5): Conv2d(2, 16, kernel_size=(5, 5), stride=(1, 1))       (6): ReLU(inplace=True)     )     (1): Flatten(start_dim=1, end_dim=-1)     (2): Linear(in_features=1024, out_features=10, bias=True)   ) ) 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 11\/11 [00:10&lt;00:00,  1.04it\/s]Reward:  -3.344358348846436  NN(   (layers): Sequential(     (0): Sequential(       (0): Conv2d(3, 2, kernel_size=(5, 5), stride=(1, 1))       (1): ReLU(inplace=True)       (2): Dropout2d(p=0.1, inplace=False)       (3): Dropout2d(p=0.1, inplace=False)       (4): Dropout2d(p=0.2, inplace=False)       (5): AvgPool2d(kernel_size=2, stride=2, padding=0)     )     (1): Flatten(start_dim=1, end_dim=-1)     (2): Linear(in_features=392, out_features=10, bias=True)   ) ) 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 11\/11 [00:09&lt;00:00,  1.12it\/s] Reward:  -5.165955924987793 Reward:  -1 Reward:  -1 Reward:  -1 Reward:  -1 Reward:  -1 Reward:  -1 NN(   (layers): Sequential(     (0): Sequential(       (0): Conv2d(3, 1, kernel_size=(5, 5), stride=(1, 1))       (1): ReLU(inplace=True)       (2): Dropout2d(p=0.2, inplace=False)       (3): BatchNorm2d(1, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)       (4): Conv2d(1, 8, kernel_size=(5, 5), stride=(1, 1))       (5): ReLU(inplace=True)       (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)     )     (1): Flatten(start_dim=1, end_dim=-1)     (2): Linear(in_features=1152, out_features=10, bias=True)   ) ) 100%|\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 11\/11 [00:11&lt;00:00,  1.06s\/it] Reward:  10.80641279220581<\/code><\/pre>\n<h2>\u0417\u0430\u0434\u0435\u043b \u043d\u0430 \u0431\u0443\u0434\u0443\u0449\u0435\u0435<\/h2>\n<p>\u0415\u0441\u0442\u0435\u0441\u0442\u0432\u0435\u043d\u043d\u043e, \u044d\u0442\u0430 \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044f \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u043c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u043e \u0440\u0430\u0431\u043e\u0447\u0438\u043c \u0432\u0430\u0440\u0438\u0430\u043d\u0442\u043e\u043c, \u043c\u0430\u0441\u0448\u0442\u0430\u0431 \u043c\u043e\u0434\u0438\u0444\u0438\u043a\u0430\u0446\u0438\u0439 \u0438 \u0438\u0434\u0435\u0439 \u0434\u043b\u044f\u00a0\u0443\u043b\u0443\u0447\u0448\u0435\u043d\u0438\u044f \u0441\u043b\u043e\u0436\u043d\u043e \u043e\u0445\u0432\u0430\u0442\u0438\u0442\u044c \u043e\u0434\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0451\u0439. \u0412\u00a0\u043f\u043e\u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u0445 \u0441\u0442\u0430\u0442\u044c\u044f\u0445 \u0431\u0443\u0434\u0435\u043c \u0443\u043b\u0443\u0447\u0448\u0430\u0442\u044c \u0442\u0435\u043a\u0443\u0449\u0443\u044e \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u044e, \u043f\u043e\u043f\u0440\u043e\u0431\u0443\u0435\u043c \u0432\u0430\u0440\u0438\u0430\u043d\u0442 \u0433\u0435\u043d\u0435\u0440\u0430\u0446\u0438\u0438 \u0441\u0435\u0442\u0438 \u043f\u043e\u00a0\u0441\u043b\u043e\u044f\u043c \u0432\u043c\u0435\u0441\u0442\u043e \u0446\u0435\u043b\u043e\u0439 \u0441\u0435\u0442\u0438 \u0437\u0430\u00a0\u0440\u0430\u0437, \u0434\u043e\u0431\u0430\u0432\u0438\u043c \u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u0438 \u0434\u043b\u044f\u00a0\u0440\u0435\u0448\u0435\u043d\u0438\u044f \u0437\u0430\u0434\u0430\u0447 \u0434\u0435\u0442\u0435\u043a\u0446\u0438\u0438, \u0441\u0435\u0433\u043c\u0435\u043d\u0442\u0430\u0446\u0438\u0438, \u0438 \u043c\u043d\u043e\u0433\u043e \u0447\u0435\u0433\u043e \u0435\u0449\u0451! <\/p>\n<h2>\u0417\u0430\u043a\u043b\u044e\u0447\u0435\u043d\u0438\u0435<\/h2>\n<p>\u041e\u0431\u0443\u0447\u0435\u043d\u0438\u0435 RL \u043c\u043e\u0434\u0435\u043b\u0435\u0439\u00a0\u2014 \u043f\u0440\u043e\u0446\u0435\u0441\u0441 \u0434\u043e\u0432\u043e\u043b\u044c\u043d\u043e \u0434\u043e\u043b\u0433\u0438\u0439, \u043f\u043e\u043b\u043e\u0436\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0439 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442 \u043c\u043e\u0436\u0435\u0442 \u043f\u0440\u0438\u0439\u0442\u0438 \u0434\u0430\u043b\u0435\u043a\u043e \u043d\u0435\u00a0\u0441\u0440\u0430\u0437\u0443. \u041e\u0434\u043d\u0430\u043a\u043e \u0432\u00a0\u043f\u0435\u0440\u0441\u043f\u0435\u043a\u0442\u0438\u0432\u0435 \u043f\u043e\u043b\u0443\u0447\u0438\u0442\u044c \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c, \u043a\u043e\u0442\u043e\u0440\u0430\u044f \u0441\u0430\u043c\u0430 \u0431\u0443\u0434\u0435\u0442 \u0440\u0435\u0448\u0430\u0442\u044c \u0437\u0430\u0434\u0430\u0447\u0443 \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u044f \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438, \u0432\u043f\u043e\u043b\u043d\u0435 \u0440\u0435\u0430\u043b\u044c\u043d\u043e. \u041a\u0442\u043e \u0437\u043d\u0430\u0435\u0442, \u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e \u0441\u043a\u043e\u0440\u043e \u0432\u043c\u0435\u0441\u0442\u043e ML \u0438\u043d\u0436\u0435\u043d\u0435\u0440\u043e\u0432 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438 \u0431\u0443\u0434\u0443\u0442 \u0441\u0442\u0440\u043e\u0438\u0442\u044c \u0441\u0430\u043c\u0438 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438? <\/p>\n<p><a href=\"https:\/\/github.com\/Simsi\/RL_NN_Generator\" rel=\"noopener noreferrer nofollow\">\u0420\u0435\u043f\u043e \u0441 \u0444\u0430\u0439\u043b\u043e\u043c \u0438\u0441\u0445\u043e\u0434\u043d\u043e\u0433\u043e \u043a\u043e\u0434\u0430 \u0432 \u0444\u043e\u0440\u043c\u0430\u0442\u0435 ipynb<\/a><\/p>\n<\/p>\n<\/div>\n<\/div>\n<\/div>\n<p><!----><!----><\/div>\n<p><!----><!----><br \/> \u0441\u0441\u044b\u043b\u043a\u0430 \u043d\u0430 \u043e\u0440\u0438\u0433\u0438\u043d\u0430\u043b \u0441\u0442\u0430\u0442\u044c\u0438 <a href=\"https:\/\/habr.com\/ru\/articles\/764986\/\"> https:\/\/habr.com\/ru\/articles\/764986\/<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<div><!--[--><!--]--><\/div>\n<div id=\"post-content-body\">\n<div>\n<div class=\"article-formatted-body article-formatted-body article-formatted-body_version-2\">\n<div xmlns=\"http:\/\/www.w3.org\/1999\/xhtml\">\n<h2>\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435<\/h2>\n<p>\u041d\u0430\u043a\u043e\u043d\u0435\u0446 \u043f\u0440\u0438\u0448\u043b\u043e \u0432\u0440\u0435\u043c\u044f \u043e\u0431\u044a\u0435\u0434\u0438\u043d\u0438\u0442\u044c \u043a\u043e\u0434 \u0438\u0437\u00a0\u043f\u0440\u0435\u0434\u044b\u0434\u0443\u0449\u0435\u0439 \u0447\u0430\u0441\u0442\u0438, \u0432\u00a0\u043a\u043e\u0442\u043e\u0440\u043e\u0439 \u043c\u044b \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u043b\u0438 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c \u043f\u043e\u00a0\u0441\u043f\u0438\u0441\u043a\u0443 \u0441\u043b\u043e\u0451\u0432, \u0441\u00a0RL \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c\u0430\u043c\u0438! \u0421\u0435\u0433\u043e\u0434\u043d\u044f \u043c\u044b \u043f\u043e\u0441\u0442\u0430\u0432\u0438\u043c \u0437\u0430\u0434\u0430\u0447\u0443 \u0434\u043b\u044f\u00a0\u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0441\u00a0\u043f\u043e\u0434\u043a\u0440\u0435\u043f\u043b\u0435\u043d\u0438\u0435\u043c, \u043e\u043f\u0438\u0448\u0435\u043c, \u043a\u0430\u043a\u00a0\u0431\u0443\u0434\u0435\u0442 \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u0438\u0442\u044c\u0441\u044f \u0432\u0437\u0430\u0438\u043c\u043e\u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0435 \u0430\u0433\u0435\u043d\u0442\u0430 \u0441\u043e \u0441\u0440\u0435\u0434\u043e\u0439 \u0438 \u043d\u0430\u00a0\u043f\u0440\u0430\u043a\u0442\u0438\u043a\u0435 \u0440\u0435\u0430\u043b\u0438\u0437\u0443\u0435\u043c \u043a\u043e\u0434 \u043d\u0430\u00a0\u044f\u0437\u044b\u043a\u0435 python.<\/p>\n<p><strong>\u0412\u0430\u0436\u043d\u043e:<\/strong> \u043d\u0430\u0441\u0442\u043e\u044f\u0442\u0435\u043b\u044c\u043d\u043e \u0440\u0435\u043a\u043e\u043c\u0435\u043d\u0434\u0443\u044e \u043e\u0437\u043d\u0430\u043a\u043e\u043c\u0438\u0442\u044c\u0441\u044f \u0441 <a href=\"https:\/\/habr.com\/ru\/articles\/763706\/\" rel=\"noopener noreferrer nofollow\">\u043f\u0435\u0440\u0432\u043e\u0439 \u0447\u0430\u0441\u0442\u044c\u044e<\/a>, \u0432\u00a0\u043a\u043e\u0442\u043e\u0440\u043e\u0439 \u043c\u044b \u0440\u0430\u0437\u043e\u0431\u0440\u0430\u043b\u0438 \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u0435 \u0441\u0435\u0442\u0438 \u043f\u043e\u00a0\u0441\u043f\u0438\u0441\u043a\u0443 \u0441\u043b\u043e\u0451\u0432, \u0432\u00a0\u044d\u0442\u043e\u0439 \u0447\u0430\u0441\u0442\u0438 \u0430\u043a\u0446\u0435\u043d\u0442 \u0441\u0438\u043b\u044c\u043d\u043e \u0441\u043c\u0435\u0449\u0451\u043d \u0432\u00a0\u0441\u0442\u043e\u0440\u043e\u043d\u0443 RL, \u0438 \u0440\u0430\u0437\u0434\u0443\u0432\u0430\u043d\u0438\u044f \u0445\u0440\u043e\u043d\u043e\u043c\u0435\u0442\u0440\u0430\u0436\u0430 \u0437\u0430\u00a0\u0441\u0447\u0451\u0442 \u043f\u043e\u0432\u0442\u043e\u0440\u0435\u043d\u0438\u0439 \u043f\u0440\u043e\u0439\u0434\u0435\u043d\u043d\u043e\u0433\u043e \u043d\u0435\u00a0\u0431\u0443\u0434\u0435\u0442.<\/p>\n<h2>\u041d\u0430\u0447\u043d\u0451\u043c \u0441 \u043f\u043e\u0441\u0442\u0430\u043d\u043e\u0432\u043a\u0438 \u0437\u0430\u0434\u0430\u0447\u0438<\/h2>\n<p>\u041f\u0435\u0440\u0432\u044b\u043c \u0434\u0435\u043b\u043e\u043c \u043e\u043f\u0438\u0448\u0435\u043c, \u0447\u0442\u043e\u00a0\u043c\u044b \u0445\u043e\u0442\u0438\u043c \u043f\u043e\u043b\u0443\u0447\u0438\u0442\u044c \u0432\u00a0\u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0435. <\/p>\n<p>\u0412\u00a0\u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u043c\u0438\u043d\u0438\u043c\u0430\u043b\u044c\u043d\u043e \u0436\u0438\u0437\u043d\u0435\u0441\u043f\u043e\u0441\u043e\u0431\u043d\u043e\u0433\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0430 \u0431\u0443\u0434\u0435\u043c \u0441\u0447\u0438\u0442\u0430\u0442\u044c \u0441\u0438\u0441\u0442\u0435\u043c\u0443, \u0432\u00a0\u043a\u043e\u0442\u043e\u0440\u043e\u0439 \u0430\u0433\u0435\u043d\u0442 \u043f\u043e\u00a0\u0432\u0445\u043e\u0434\u043d\u044b\u043c \u0434\u0430\u043d\u043d\u044b\u043c \u0431\u0443\u0434\u0435\u0442 \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u0442\u044c \u0441\u043f\u0438\u0441\u043e\u043a \u0441\u043b\u043e\u0451\u0432 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438, \u0441\u0440\u0435\u0434\u0430 \u0431\u0443\u0434\u0435\u0442 \u043f\u0435\u0440\u0435\u0432\u043e\u0434\u0438\u0442\u044c \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0435 \u0430\u0433\u0435\u043d\u0442\u0430 \u0432\u00a0\u0443\u0434\u043e\u0431\u043d\u044b\u0439 \u0434\u043b\u044f\u00a0\u043d\u0435\u0451 \u0444\u043e\u0440\u043c\u0430\u0442, \u043f\u0440\u043e\u0441\u0442\u0440\u0430\u0438\u0432\u0430\u0442\u044c \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c, \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0442\u044c \u0446\u0438\u043a\u043b \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0438 \u0432\u043e\u0437\u0440\u0430\u0449\u0430\u0442\u044c \u0435\u0433\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442 \u0430\u0433\u0435\u043d\u0442\u0443 \u0432\u00a0\u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u044f. <\/p>\n<h2>\u0423\u043f\u0440\u043e\u0449\u0435\u043d\u0438\u044f \u0434\u043b\u044f \u043f\u0435\u0440\u0432\u043e\u0439 \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438<\/h2>\n<p>\u0422\u0430\u043a \u0437\u0430\u0434\u0430\u0447\u0430 \u0438\u043c\u0435\u0435\u0442 \u043c\u043d\u043e\u0433\u043e \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u043e\u0432, \u043e\u0442 \u043a\u043e\u0442\u043e\u0440\u044b\u0445 \u0437\u0430\u0432\u0438\u0441\u0438\u0442 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u043e \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u0430, \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u043e \u0438\u0445 \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u0438\u0442\u044c \u0438 \u043d\u0435\u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u0437\u0430\u0444\u0438\u043a\u0441\u0438\u0440\u043e\u0432\u0430\u0442\u044c.<\/p>\n<h4>\u0413\u0438\u043f\u0435\u0440\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b \u0434\u043b\u044f \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438<\/h4>\n<p>learning_rate=0.001<br \/>batch_size=32<br \/>input_shape=[B, C, H, W]<br \/>num_classes=10 (\u0434\u0430\u0442\u0430\u0441\u0435\u0442 CIFAR10)<br \/>train_epochs=10<br \/>device=&#8217;cuda&#8217; (\u0422\u0435\u0441\u0442\u0438\u0440\u043e\u0432\u0430\u043b\u043e\u0441\u044c \u043d\u0430 T4 GPU \u0432 google colab)<\/p>\n<h4>\u0413\u0438\u043f\u0435\u0440\u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b \u0441\u0440\u0435\u0434\u044b<\/h4>\n<p>layers_amount=5 &#8212; \u0434\u043b\u0438\u043d\u0430 \u043f\u043e\u0441\u043b\u0435\u0434\u043e\u0432\u0430\u0442\u0435\u043b\u044c\u043d\u043e\u0441\u0442\u0438 \u0441\u043b\u043e\u0451\u0432, \u0433\u0435\u043d\u0435\u0440\u0438\u0440\u0443\u0435\u043c\u043e\u0439 \u0430\u0433\u0435\u043d\u0442\u043e\u043c.<\/p>\n<p>memory_len=10 &#8212; \u0441\u043a\u043e\u043b\u044c\u043a\u043e \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u0438\u0445 \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u0439 \u0431\u0443\u0434\u0443\u0442 \u0432\u044b\u0434\u0430\u0432\u0430\u0442\u044c\u0441\u044f \u0430\u0433\u0435\u043d\u0442\u0443, \u0434\u043b\u0438\u043d\u0430 \u043a\u043e\u043b\u044c\u0446\u0435\u0432\u043e\u0433\u043e \u0431\u0443\u0444\u0435\u0440\u0430, \u043d\u043e\u0432\u044b\u0435 \u0438\u0437\u043c\u0435\u0440\u0435\u043d\u0438\u044f \u0434\u043e\u0431\u0430\u0432\u043b\u044f\u044e\u0442\u0441\u044f \u0432 \u043a\u043e\u043d\u0435\u0446.<\/p>\n<p>metrics_amount=2 (\u0411\u0443\u0434\u0435\u043c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c \u043f\u0440\u043e\u0441\u0442\u043e train \u0438 valid MSELosses, \u0434\u0430, \u043d\u0435 \u043e\u043f\u0442\u0438\u043c\u0430\u043b\u044c\u043d\u043e, \u043d\u043e \u044d\u0442\u043e \u0432 \u043f\u0435\u0440\u0432\u043e\u0439 \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u043d\u0435 \u043f\u0440\u0438\u043d\u0446\u0438\u043f\u0438\u0430\u043b\u044c\u043d\u044b\u0439 \u043c\u043e\u043c\u0435\u043d\u0442).<\/p>\n<h2>Observation, action, reward<\/h2>\n<figure class=\"full-width\">\n<div><figcaption>\u0423\u043f\u0440\u043e\u0449\u0451\u043d\u043d\u043e\u0435 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u0435\u043d\u0438\u0435 \u0446\u0438\u043a\u043b\u0430 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f \u0430\u0433\u0435\u043d\u0442\u0430<\/figcaption><\/div>\n<\/figure>\n<p>\u0412 \u043a\u0430\u0447\u0435\u0441\u0442\u0432\u0435 \u0432\u0445\u043e\u0434\u043d\u044b\u0445 \u0434\u0430\u043d\u043d\u044b\u0445 \u0434\u043b\u044f \u0430\u0433\u0435\u043d\u0442\u0430 <strong>(observation)<\/strong> \u043c\u044b \u0432\u043e\u0437\u044c\u043c\u0451\u043c \u043c\u0435\u0442\u0440\u0438\u043a\u0438, \u043f\u043e\u043b\u0443\u0447\u0435\u043d\u043d\u044b\u0435 \u0432 \u0445\u043e\u0434\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f, \u0438 \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u044b \u0432 \u043a\u043e\u043b\u0438\u0447\u0435\u0441\u0442\u0432\u0435, \u0440\u0430\u0432\u043d\u043e\u043c \u0434\u043b\u0438\u043d\u0435 \u043f\u0430\u043c\u044f\u0442\u0438. \u0422.\u0435. \u0430\u0433\u0435\u043d\u0442 \u043f\u0440\u0438\u043d\u0438\u043c\u0430\u0435\u0442 \u0440\u0435\u0448\u0435\u043d\u0438\u0435 \u043e \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u0438 \u043d\u043e\u0432\u044b\u0445 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0435\u0439 \u043d\u0430 \u043e\u0441\u043d\u043e\u0432\u0435 \u043f\u0430\u0440: (\u043f\u0440\u0435\u0434\u044b\u0434\u0443\u0449\u0430\u044f \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u0430, \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u043f\u043e\u0441\u043b\u0435 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f)<\/p>\n<p>obs={<br \/> &#8216;metrics&#8217;: shape=[mem_len, amount_of_metrics, train_epochs], <br \/>&#8216;architectures&#8217;: shape=[mem_lem, layers_amount],<br \/>}<\/p>\n<p>\u0414\u0435\u0439\u0441\u0442\u0432\u0438\u0435 \u0430\u0433\u0435\u043d\u0442\u0430 <strong>(action)<\/strong> \u0431\u0443\u0434\u0435\u0442 \u0432\u044b\u0433\u043b\u044f\u0434\u0435\u0442\u044c \u043a\u0430\u043a \u043d\u0430\u0431\u043e\u0440 \u0447\u0438\u0441\u0435\u043b \u0444\u0438\u043a\u0441\u0438\u0440\u043e\u0432\u0430\u043d\u043d\u043e\u0439 \u0434\u043b\u0438\u043d\u044b, \u043a\u0430\u0436\u0434\u043e\u043c\u0443 \u0438\u0437 \u043a\u043e\u0442\u043e\u0440\u044b\u0445 \u0441\u043e\u043e\u0442\u0432\u0435\u0442\u0441\u0442\u0432\u0443\u0435\u0442 \u0441\u043b\u043e\u0439 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438 \u0438\u0437 \u0441\u043f\u0438\u0441\u043a\u0430 \u0432\u043e\u0437\u043c\u043e\u0436\u043d\u044b\u0445 (actions_set). <\/p>\n<p>action = []<br \/>action.shape=[max_layers_amount]<\/p>\n<p>\u041d\u0430\u0433\u0440\u0430\u0434\u0430 \u0430\u0433\u0435\u043d\u0442\u0430 <strong>(reward)<\/strong> &#8212; \u0437\u0430\u0447\u0430\u0441\u0442\u0443\u044e \u0441\u0430\u043c\u0430\u044f \u0441\u043b\u043e\u0436\u043d\u0430\u044f \u0434\u043b\u044f \u0444\u043e\u0440\u043c\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u0447\u0430\u0441\u0442\u044c, \u0442\u0430\u043a \u043a\u0430\u043a \u043e\u0442 \u044d\u0442\u043e\u0433\u043e \u0437\u0430\u0432\u0438\u0441\u0438\u0442, \u0447\u0442\u043e \u0430\u0433\u0435\u043d\u0442 \u043f\u0440\u0438\u043c\u0435\u0442 \u0437\u0430 \u0445\u043e\u0440\u043e\u0448\u043e, \u0430 \u0447\u0442\u043e \u0437\u0430 \u043f\u043b\u043e\u0445\u043e. \u041c\u044b \u0431\u0443\u0434\u0435\u043c \u043d\u0430\u0433\u0440\u0430\u0436\u0434\u0430\u0442\u044c \u0430\u0433\u0435\u043d\u0442\u0430 \u0441\u043b\u0435\u0434\u0443\u044e\u0449\u0438\u043c \u043e\u0431\u0440\u0430\u0437\u043e\u043c:<\/p>\n<ul>\n<li>\n<p><strong>metrics_optimization_reward<\/strong> &#8212; \u0432 \u0441\u043b\u0443\u0447\u0430\u0435, \u043a\u043e\u0433\u0434\u0430 \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u0443\u043b\u0443\u0447\u0448\u0430\u044e\u0442\u0441\u044f, \u0430\u0433\u0435\u043d\u0442 \u0437\u0430\u0441\u043b\u0443\u0436\u0438\u0432\u0430\u0435\u0442 \u043f\u043e\u043e\u0449\u0440\u0435\u043d\u0438\u0435. \u0440\u0430\u0441\u0441\u0447\u0438\u0442\u0430\u0435\u043c \u043d\u0430\u0433\u0440\u0430\u0434\u0443 \u043a\u0430\u043a \u0440\u0430\u0437\u043d\u0438\u0446\u0443 \u043c\u0435\u0436\u0434\u0443 \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u0438\u043c\u0438 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f\u043c\u0438 \u043e\u0442\u0441\u043b\u0435\u0436\u0438\u0432\u0430\u0435\u043c\u044b\u0445 \u043c\u0435\u0442\u0440\u0438\u043a \u0438 \u043d\u043e\u0432\u044b\u043c\u0438, \u0432 \u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e\u0441\u0442\u0438 \u043e\u0442 \u0437\u043d\u0430\u043a\u0430 \u043f\u043e\u043b\u0443\u0447\u0438\u0432\u0448\u0435\u0433\u043e\u0441\u044f \u0447\u0438\u0441\u043b\u0430 \u043d\u0430\u0433\u0440\u0430\u0434\u0430 \u043c\u043e\u0436\u0435\u0442 \u043f\u0440\u0435\u0432\u0440\u0430\u0442\u0438\u0442\u044c\u0441\u044f \u0432 \u043d\u0430\u043a\u0430\u0437\u0430\u043d\u0438\u0435 \u0437\u0430 \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u0435 \u043c\u043e\u0434\u0435\u043b\u0438 \u0445\u0443\u0436\u0435, \u0447\u0435\u043c \u0432 \u043f\u043e\u0441\u043b\u0435\u0434\u043d\u0438\u0439 \u0440\u0430\u0437.<br \/><em>\u0417\u0430\u043c\u0435\u0447\u0430\u043d\u0438\u0435 1:<\/em> \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u043f\u0435\u0440\u0432\u043e\u0439 \u044d\u043f\u043e\u0445\u0438 \u043c\u043e\u0433\u0443\u0442 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u044f\u0442\u044c \u0441\u043e\u0431\u043e\u0439 \u0432\u044b\u0431\u0440\u043e\u0441\u044b \u0431\u043e\u043b\u044c\u0448\u043e\u0439 \u0432\u0435\u043b\u0438\u0447\u0438\u043d\u044b, \u043f\u043e\u044d\u0442\u043e\u043c\u0443 \u0438\u0445 \u043c\u044b \u0431\u0443\u0434\u0435\u043c \u043f\u0440\u043e\u043f\u0443\u0441\u043a\u0430\u0442\u044c, \u043f\u0440\u043e\u0438\u0437\u0432\u043e\u0434\u044f \u043f\u043e \u0441\u0443\u0442\u0438 train_epochs + 1 \u044d\u043f\u043e\u0445 \u043e\u0431\u0443\u0447\u0435\u043d\u0438\u044f.<br \/><em>\u0417\u0430\u043c\u0435\u0447\u0430\u043d\u0438\u0435 2:<\/em> \u041d\u0435 \u0441\u0430\u043c\u044b\u0435 \u043e\u043f\u0442\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u0435 \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u044b \u043c\u043e\u0433\u0443\u0442 \u0432\u044b\u0434\u0430\u0432\u0430\u0442\u044c \u043e\u0433\u0440\u043e\u043c\u043d\u044b\u0435 \u043f\u043e\u043a\u0430\u0437\u0430\u0442\u0435\u043b\u0438 \u043c\u0435\u0442\u0440\u0438\u043a\u0438. \u0414\u043b\u044f \u0441\u0433\u043b\u0430\u0436\u0438\u0432\u0430\u043d\u0438\u044f \u044d\u0442\u043e\u0433\u043e \u043c\u043e\u043c\u0435\u043d\u0442\u0430 \u043c\u044b \u0431\u0443\u0434\u0435\u043c \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0438\u0432\u0430\u0442\u044c \u0441\u0432\u0435\u0440\u0445\u0443 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f \u044d\u043c\u043f\u0438\u0440\u0438\u0447\u0435\u0441\u043a\u0438 \u043f\u043e\u0434\u043e\u0431\u0440\u0430\u043d\u043d\u044b\u043c \u043f\u043e\u0440\u043e\u0433\u043e\u043c (\u0432 \u0447\u0430\u0441\u0442\u043d\u043e\u0441\u0442\u0438 \u0431\u044b\u043b\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u043e \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 100).<br \/><em>\u0417\u0430\u043c\u0435\u0447\u0430\u043d\u0438\u0435 3:<\/em> \u0438\u043d\u043e\u0433\u0434\u0430 \u0432 \u043e\u0441\u043e\u0431\u043e \u043f\u0435\u0447\u0430\u043b\u044c\u043d\u044b\u0445 \u0441\u043b\u0443\u0447\u0430\u044f\u0445 \u043c\u0435\u0442\u0440\u0438\u043a\u0438 \u0432\u043e\u043e\u0431\u0449\u0435 \u043c\u043e\u0433\u0443\u0442 \u043f\u0440\u0438\u043d\u044f\u0442\u044c \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u0435 nan \u0438\u043b\u0438 inf, \u0431\u0443\u0434\u0435\u043c \u0442\u0430\u043a\u0436\u0435 \u0437\u0430\u043c\u0435\u043d\u044f\u0442\u044c \u0442\u0430\u043a\u0438\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f \u043d\u0430 NAN_NUM = 100.<\/p>\n<\/li>\n<li>\n<p><strong>creation_successfull_reward<\/strong> &#8212; \u0435\u0441\u043b\u0438 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c \u0432\u043e\u043e\u0431\u0449\u0435 \u043f\u043e\u043b\u0443\u0447\u0438\u043b\u043e\u0441\u044c \u0441\u043e\u0437\u0434\u0430\u0442\u044c, \u0443\u0436\u0435 \u043d\u0435\u043f\u043b\u043e\u0445\u043e, \u0431\u0443\u0434\u0435\u043c \u043d\u0430\u0433\u0440\u0430\u0436\u0434\u0430\u0442\u044c \u0430\u0433\u0435\u043d\u0442\u0430 \u0437\u0430 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u043f\u043e\u043b\u0443\u0447\u0438\u043b\u043e\u0441\u044c \u043a\u043e\u0440\u0440\u0435\u043a\u0442\u043d\u043e \u043f\u043e\u0441\u0442\u0440\u043e\u0438\u0442\u044c \u0438 \u0448\u0442\u0440\u0430\u0444\u043e\u0432\u0430\u0442\u044c \u0432 \u043e\u0431\u0440\u0430\u0442\u043d\u043e\u043c \u0441\u043b\u0443\u0447\u0430\u0435. \u0414\u043b\u044f \u044d\u0442\u043e\u0433\u043e \u0431\u044b\u043b\u0438 \u0441\u043e\u0437\u0434\u0430\u043d\u044b \u043f\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u044b\u0435 NN_CREATE_SUCCESS_REWARD = 1<br \/>NN_CREATE_NOT_SUCCESS_PENALTY = -1<\/p>\n<\/li>\n<li>\n<p><strong>optimal_depth_reward<\/strong> &#8212; \u044d\u0442\u0430 \u0447\u0430\u0441\u0442\u044c \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f \u0437\u0430\u0434\u0435\u043b\u043e\u043c \u043d\u0430 \u0431\u0443\u0434\u0443\u0449\u0435\u0435, \u043d\u0430 \u0442\u0435\u043a\u0443\u0449\u0438\u0439 \u043c\u043e\u043c\u0435\u043d\u0442 \u043d\u0435 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442\u0441\u044f<\/p>\n<\/li>\n<\/ul>\n<h2>RL Custom environment<\/h2>\n<p>\u0414\u043b\u044f \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438 \u0431\u0443\u0434\u0435\u0442 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c\u0441\u044f python \u0441 \u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0430\u043c\u0438 <a href=\"https:\/\/gymnasium.farama.org\/content\/basic_usage\/\" rel=\"noopener noreferrer nofollow\">gymnasium<\/a> \u0438 <a href=\"https:\/\/stable-baselines3.readthedocs.io\/en\/master\/guide\/examples.html\" rel=\"noopener noreferrer nofollow\">stable-baselines3<\/a>.<\/p>\n<p>\u041d\u0430\u043f\u043e\u043c\u043d\u044e \u043e\u0441\u043d\u043e\u0432\u043d\u0443\u044e \u0441\u0442\u0440\u0443\u043a\u0442\u0443\u0440\u0443 \u0441\u0440\u0435\u0434\u044b \u0432 gymnasium:<\/p>\n<pre><code>class Env(gym.Env):   def __init__(self):     # init all required variables     # action and observation spaces must be here!      self.action_space = spaces.Box(...)     self.observation_space = spaces.MultiDiscrete(...)     pass    def seed(self):      seed = 42      return [seed]    def step(self, action):     # most important metrics     # do manipulations with action,      # return new observation and reward     return obs, reward, _, done, info    def reset(self):     # set variables to start state     return obs    def render(self):     # all GUI parts should be here     # may be empty     pass<\/code><\/pre>\n<p>\u041f\u0440\u043e\u0439\u0434\u0451\u043c\u0441\u044f \u043f\u043e \u043a\u0430\u0436\u0434\u043e\u043c\u0443 \u043f\u0443\u043d\u043a\u0442\u0443.<\/p>\n<h3>__init__, \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0435 \u043f\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u044b\u0445<\/h3>\n<pre><code>  def __init__(self, opt_cls=None, crit=None, trn_ldr=None, vld_ldr=None, render_mode=None):       '''       Initialization of environment variables       '''        # here is learning params       self.NN_PARAMS = {           'lr': 0.001,           'num_classes': 10, # TODO adaptive num_classes           'train_epochs': 10,           'last_nets_metrics_memory_len': 10,           'layers_amount': 5,           'amount_of_metrics': 2,       }        # here contains NN learning metrics       # uses for observations       # shape=[MEMORY_LEN, N_METRICS, EPOCHS]       self.NN_PARAMS['metrics'] = np.zeros((           self.NN_PARAMS['last_nets_metrics_memory_len'],           self.NN_PARAMS['amount_of_metrics'],           self.NN_PARAMS['train_epochs'],       ))        # here contains last architectures       # uses for observations       # shape=[MEMORY, N_LAYERS]       self.NN_PARAMS['last_nets_architectures'] = np.zeros((              self.NN_PARAMS['last_nets_metrics_memory_len'],              self.NN_PARAMS['layers_amount'],           ))        # Variables for NN       self.Net = self.NN()       self.train_dataloader = trn_ldr       self.valid_dataloader = vld_ldr       self.optimizer_class = opt_cls       self.optimizer = None       self.criterion = crit       self.device = 'cuda'        self.nngenerator = self.nnGenerator()        self.last_obs = None        # Action space describes what agent will give to environment       # shape=[ACTION_SET_SIZE, N_LAYERS]       self.action_space = spaces.MultiDiscrete(         [len(self.actions_set)] * self.NN_PARAMS['layers_amount'],          seed=42)        # Observation space describes what agent       # will take from enviromnent as observation       # shape=dict{       #  METRICS, shape as NN_PARAMS['metrics'],       #  ARCHITECTURES, shape as NN_PARAMS['layers_amount'],       # }       self.observation_space = spaces.Dict(           {           'last_nets_metrics_memory': spaces.Box(               low=0,               high=100,               shape=(self.NN_PARAMS['last_nets_metrics_memory_len'],                      self.NN_PARAMS['amount_of_metrics'],                      self.NN_PARAMS['train_epochs'],                      )),           'last_nets_architectures': spaces.Box(               low=0,               high=len(self.actions_set),               shape=(self.NN_PARAMS['last_nets_metrics_memory_len'],                      self.NN_PARAMS['layers_amount'],                      ))           }       )        # some variable for collecting statistics        self.statistics = {         'episode_rewards': [],         'global_rewards': [],         'made_steps': [],       }        self.seed()       assert render_mode is None or render_mode in self.metadata[\"render_modes\"]       self.render_mode = render_mode <\/code><\/pre>\n<p>\u041d\u0430\u00a0\u0432\u0445\u043e\u0434 \u043f\u0440\u0438\u043d\u0438\u043c\u0430\u044e\u0442\u0441\u044f \u043e\u043f\u0442\u0438\u043c\u0430\u0439\u0437\u0435\u0440, \u0444\u2011\u044f \u043e\u0448\u0438\u0431\u043a\u0438 \u0438 \u0434\u0430\u0442\u0430\u043b\u043e\u0430\u0434\u0435\u0440\u044b.<\/p>\n<p>\u0417\u0434\u0435\u0441\u044c \u0443\u043a\u0430\u0437\u0430\u043d\u044b \u0432\u0441\u0435 \u043d\u0435\u043e\u0431\u0445\u043e\u0434\u0438\u043c\u044b\u0435 \u043f\u0430\u0440\u0430\u043c\u0435\u0442\u0440\u044b, \u0441\u043e\u0437\u0434\u0430\u0451\u0442\u0441\u044f \u043e\u0431\u044a\u0435\u043a\u0442 \u043f\u0443\u0441\u0442\u043e\u0439 \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u0438 Net, \u0432\u00a0\u043a\u043e\u0442\u043e\u0440\u0443\u044e \u043c\u044b \u043f\u043e\u0437\u0436\u0435 \u0431\u0443\u0434\u0435\u0442 \u043f\u043e\u043c\u0435\u0449\u0430\u0442\u044c \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u043d\u0443\u044e \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c, \u0430\u00a0\u0442\u0430\u043a\u0436\u0435 nnGenerator&#8217;a, \u0447\u0442\u043e\u00a0\u0437\u0430\u043d\u0438\u043c\u0430\u0435\u0442\u0441\u044f \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u043e\u0439 \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u044f \u0430\u0433\u0435\u043d\u0442\u0430 \u0438 \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u044f \u0441\u0430\u043c\u043e\u0439 \u0441\u0435\u0442\u0438. \u041e\u0431\u044f\u0437\u0430\u0442\u0435\u043b\u044c\u043d\u044b\u043c\u0438 \u043f\u043e\u043b\u044f\u043c\u0438 \u044f\u0432\u043b\u044f\u044e\u0442\u0441\u044f action_space \u0438 observation_space. \u0411\u0435\u0437\u00a0\u0438\u0445 \u043e\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u0438\u044f \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u0435 \u0441\u0440\u0435\u0434\u044b \u043d\u0435\u0432\u043e\u0437\u043c\u043e\u0436\u043d\u043e.<\/p>\n<p><em>\u0417\u0430\u043c\u0435\u0447\u0430\u043d\u0438\u0435: \u043a\u00a0\u0432\u044b\u0431\u043e\u0440\u0443 \u043f\u0440\u043e\u0441<\/em>\u0442\u0440\u0430\u043d\u0441\u0442\u0432\u0430 \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0439 \u0438 \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u0439 \u0441\u043b\u0435\u0434\u0443\u0435\u0442 \u043f\u043e\u0434\u0445\u043e\u0434\u0438\u0442\u044c \u0441\u00a0\u0443\u043c\u043e\u043c, \u0442\u0430\u043a \u043a\u0430\u043a\u00a0\u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c\u044b \u0430\u0433\u0435\u043d\u0442\u0430 \u0438\u0437\u00a0stable\u2011baselines3\u00a0\u043c\u043e\u0433\u0443\u0442 \u043d\u0435\u00a0\u043f\u043e\u0434\u0434\u0435\u0440\u0436\u0438\u0432\u0430\u0442\u044c \u0442\u043e\u0442 \u0444\u043e\u0440\u043c\u0430\u0442, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0432\u044b \u0432\u044b\u0431\u0435\u0440\u0435\u0442\u0435. \u041e\u00a0\u0442\u043e\u043c, \u043a\u0430\u043a\u0438\u0435 \u043f\u0440\u043e\u0441\u0442\u0440\u0430\u043d\u0441\u0442\u0432\u0430 \u043c\u043e\u0436\u043d\u043e \u0432\u044b\u0431\u0440\u0430\u0442\u044c \u0434\u043b\u044f\u00a0\u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c\u043e\u0432, \u043c\u043e\u0436\u043d\u043e \u043f\u043e\u0441\u043c\u043e\u0442\u0440\u0435\u0442\u044c <a href=\"https:\/\/stable-baselines3.readthedocs.io\/en\/master\/guide\/algos.html\" rel=\"noopener noreferrer nofollow\">\u0442\u0443\u0442.<\/a><strong><br \/><\/strong>\u041c\u044b \u0431\u0443\u0434\u0435\u043c \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u044c PPO(&#8216;MultiInputPolicy&#8217;)\u00a0\u2014 proximal policy optimization \u0430\u043b\u0433\u043e\u0440\u0438\u0442\u043c, \u043e\u00a0\u043a\u043e\u0442\u043e\u0440\u043e\u043c \u043c\u043e\u0436\u043d\u043e \u043f\u0440\u043e\u0447\u0438\u0442\u0430\u0442\u044c <a href=\"https:\/\/stable-baselines3.readthedocs.io\/en\/master\/modules\/ppo.html\" rel=\"noopener noreferrer nofollow\">\u0432\u00a0\u0434\u043e\u043a\u0443\u043c\u0435\u043d\u0442\u0430\u0446\u0438\u0438 \u0442\u0443\u0442.<\/a> <\/p>\n<h3>reset, \u043e\u0431\u043d\u0443\u043b\u0435\u043d\u0438\u0435 \u043f\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u044b\u0445, \u043f\u0435\u0440\u0435\u0437\u0430\u0433\u0440\u0443\u0437\u043a\u0430 \u0441\u0440\u0435\u0434\u044b<\/h3>\n<pre><code>   def reset(self, seed=None, options=None):       '''       Reset the env,       Set all changed in training proccess variables to zero       (or noise, dependse on your realization)        seed: list, list of random seeds (depricated)       options: list, additional options (future)       '''       super().reset(seed=seed)       self.Net = self.NN()        current_obs = {           'last_nets_metrics_memory': np.zeros((               self.NN_PARAMS['last_nets_metrics_memory_len'],               self.NN_PARAMS['amount_of_metrics'],               self.NN_PARAMS['train_epochs'],              ) ),           'last_nets_architectures': np.zeros((              self.NN_PARAMS['last_nets_metrics_memory_len'],              self.NN_PARAMS['layers_amount'],           ) )            }       self.NN_PARAMS['metrics'] = np.zeros((           self.NN_PARAMS['last_nets_metrics_memory_len'],           self.NN_PARAMS['amount_of_metrics'],           self.NN_PARAMS['train_epochs'],       ))       self.NN_PARAMS['last_nets_architectures'] = np.zeros((              self.NN_PARAMS['last_nets_metrics_memory_len'],              self.NN_PARAMS['layers_amount'],           ))        self.last_obs = current_obs       self.episode_reward = 0       self.current_it = 1        self.statistics['episode_rewards'] = []       self.statistics['made_steps'] = []        return current_obs, {'none': None} <\/code><\/pre>\n<p>\u0422\u0443\u0442 \u043d\u0438\u0447\u0435\u0433\u043e \u043d\u0435\u043e\u0431\u044b\u0447\u043d\u043e\u0433\u043e, \u0437\u0430\u043d\u0443\u043b\u044f\u0435\u043c \u043c\u0435\u0442\u0440\u0438\u043a\u0438, \u0430\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u044b, \u0443\u0434\u0430\u043b\u044f\u0435\u043c \u043d\u0435\u0439\u0440\u043e\u0441\u0435\u0442\u044c, \u0441\u043e\u0437\u0434\u0430\u0451\u043c \u043f\u0443\u0441\u0442\u043e\u0435 \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u0435. \u0412 \u043f\u043b\u0430\u043d\u0430\u0445 \u0440\u0435\u0430\u043b\u0438\u0437\u043e\u0432\u0430\u0442\u044c \u0437\u0430\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u0435 \u043f\u0435\u0440\u0432\u043e\u0433\u043e \u043d\u0430\u0431\u043b\u044e\u0434\u0435\u043d\u0438\u044f \u0448\u0443\u043c\u043e\u043c.<\/p>\n<h3>step, \u043f\u0435\u0440\u0435\u0432\u043e\u0434\u0438\u043c action \u0432 new_obs \u0438 reward<\/h3>\n<pre><code>  def step(self, action):       '''           Main environment function            takes action, creates NN, train NN, calc new obs and reward            action: list, shape of action is like action_space  <\/code><\/pre>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[],"tags":[],"class_list":["post-356495","post","type-post","status-publish","format-standard","hentry"],"_links":{"self":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts\/356495","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=356495"}],"version-history":[{"count":0,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts\/356495\/revisions"}],"wp:attachment":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=356495"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=356495"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=356495"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}