{"id":4110,"date":"2022-07-22T11:39:37","date_gmt":"2022-07-22T11:39:37","guid":{"rendered":"https:\/\/otus.ru\/journal\/?p=4110"},"modified":"2022-08-03T14:01:30","modified_gmt":"2022-08-03T14:01:30","slug":"vvedenie-v-orkestraciju","status":"publish","type":"post","link":"https:\/\/otus.ru\/journal\/vvedenie-v-orkestraciju\/","title":{"rendered":"\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435 \u0432 \u043e\u0440\u043a\u0435\u0441\u0442\u0440\u0430\u0446\u0438\u044e"},"content":{"rendered":"<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_61 counter-hierarchy ez-toc-counter ez-toc-grey ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title \" >\u0421\u043e\u0434\u0435\u0440\u0436\u0430\u043d\u0438\u0435<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/otus.ru\/journal\/vvedenie-v-orkestraciju\/#%D0%97%D0%B0%D0%BF%D0%B8%D1%81%D1%8C_%D0%BE%D1%82%D0%BA%D1%80%D1%8B%D1%82%D0%BE%D0%B3%D0%BE_%D0%B7%D0%B0%D0%BD%D1%8F%D1%82%D0%B8%D1%8F_%D0%BA%D1%83%D1%80%D1%81%D0%B0_%C2%ABData_Engineer%C2%BB\" title=\"\u0417\u0430\u043f\u0438\u0441\u044c \u043e\u0442\u043a\u0440\u044b\u0442\u043e\u0433\u043e \u0437\u0430\u043d\u044f\u0442\u0438\u044f \u043a\u0443\u0440\u0441\u0430 \u00abData Engineer\u00bb\">\u0417\u0430\u043f\u0438\u0441\u044c \u043e\u0442\u043a\u0440\u044b\u0442\u043e\u0433\u043e \u0437\u0430\u043d\u044f\u0442\u0438\u044f \u043a\u0443\u0440\u0441\u0430 \u00abData Engineer\u00bb<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/otus.ru\/journal\/vvedenie-v-orkestraciju\/#%D0%A7%D0%B8%D1%82%D0%B0%D0%B9%D1%82%D0%B5_%D0%BD%D0%B0_%D0%A5%D0%B0%D0%B1%D1%80\" title=\"\u0427\u0438\u0442\u0430\u0439\u0442\u0435 \u043d\u0430 \u0425\u0430\u0431\u0440:\">\u0427\u0438\u0442\u0430\u0439\u0442\u0435 \u043d\u0430 \u0425\u0430\u0431\u0440:<\/a><\/li><\/ul><\/li><\/ul><\/li><\/ul><\/nav><\/div>\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%D0%97%D0%B0%D0%BF%D0%B8%D1%81%D1%8C_%D0%BE%D1%82%D0%BA%D1%80%D1%8B%D1%82%D0%BE%D0%B3%D0%BE_%D0%B7%D0%B0%D0%BD%D1%8F%D1%82%D0%B8%D1%8F_%D0%BA%D1%83%D1%80%D1%81%D0%B0_%C2%ABData_Engineer%C2%BB\"><\/span>\u0417\u0430\u043f\u0438\u0441\u044c \u043e\u0442\u043a\u0440\u044b\u0442\u043e\u0433\u043e \u0437\u0430\u043d\u044f\u0442\u0438\u044f \u043a\u0443\u0440\u0441\u0430 <a href=\"https:\/\/otus.ru\/lessons\/data-engineer\/?utm_source=oj&amp;utm_medium=affilate&amp;utm_campaign=data-engineer&amp;mxm=[[hash_metrika]]&amp;relogin=True&amp;token=[[token]]\" target=\"_blank\" rel=\"noreferrer noopener\">\u00abData Engineer\u00bb<\/a><span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<figure class=\"wp-block-embed is-type-video is-provider-youtube wp-block-embed-youtube wp-embed-aspect-16-9 wp-has-aspect-ratio\"><div class=\"wp-block-embed__wrapper\">\n<iframe loading=\"lazy\" title=\"\u0414\u0435\u043c\u043e-\u0437\u0430\u043d\u044f\u0442\u0438\u0435 \u043a\u0443\u0440\u0441\u0430 \u00abData Engineer\u00bb\" width=\"770\" height=\"433\" src=\"https:\/\/www.youtube.com\/embed\/PPKfa0B1q68?start=65&#038;feature=oembed\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen><\/iframe>\n<\/div><\/figure>\n\n\n\n<p>\u041d\u0430 \u0441\u0435\u0433\u043e\u0434\u043d\u044f\u0448\u043d\u0438\u0439 \u0434\u0435\u043d\u044c \u0438\u043d\u0441\u0442\u0440\u0443\u043c\u0435\u043d\u0442\u044b \u041e\u0440\u043a\u0435\u0441\u0442\u0440\u0430\u0446\u0438\u0438 \u2014 \u044d\u0442\u043e \u043e\u0442\u0440\u0430\u0441\u043b\u0435\u0432\u043e\u0439 \u0441\u0442\u0430\u043d\u0434\u0430\u0440\u0442 \u0434\u043b\u044f \u043e\u0440\u0433\u0430\u043d\u0438\u0437\u0430\u0446\u0438\u0438 \u043f\u043e\u043b\u0443\u0447\u0435\u043d\u0438\u044f, \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0438 \u0438 \u0441\u043e\u0445\u0440\u0430\u043d\u0435\u043d\u0438\u044f \u0434\u0430\u043d\u043d\u044b\u0445 \u0438\u0437 \u0441\u043e\u0442\u0435\u043d \u0438 \u0434\u0430\u0436\u0435 \u0442\u044b\u0441\u044f\u0447 \u0440\u0430\u0437\u043d\u043e\u0440\u043e\u0434\u043d\u044b\u0445 \u0438\u0441\u0442\u043e\u0447\u043d\u0438\u043a\u043e\u0432 \u0441 \u0440\u0430\u0437\u043d\u043e\u043e\u0431\u0440\u0430\u0437\u043d\u043e\u0439 \u0447\u0430\u0441\u0442\u043e\u0442\u043e\u0439 \u043e\u0431\u043d\u043e\u0432\u043b\u0435\u043d\u0438\u044f \u0438 \u043f\u0440\u0438\u0440\u043e\u0434\u043e\u0439.<\/p>\n\n\n\n<p>\u0421\u0446\u0435\u043d\u0430\u0440\u0438\u0438 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u044f \u043f\u043b\u0430\u0442\u0444\u043e\u0440\u043c \u041e\u0440\u043a\u0435\u0441\u0442\u0440\u0430\u0446\u0438\u0438 \u0440\u0430\u0437\u043d\u043e\u043e\u0431\u0440\u0430\u0437\u043d\u044b: \u0432\u044b \u043c\u043e\u0436\u0435\u0442\u0435 \u043f\u0440\u043e\u0441\u0442\u043e \u043e\u0440\u0433\u0430\u043d\u0438\u0437\u043e\u0432\u0430\u0442\u044c \u0440\u0435\u0433\u0443\u043b\u044f\u0440\u043d\u044b\u0435 <code>select -&gt; group by-&gt; insert<\/code> \u0438\u0437 production \u0431\u0430\u0437\u044b \u0434\u0430\u043d\u043d\u044b\u0445 \u0432 \u00ab\u0445\u043e\u043b\u043e\u0434\u043d\u0443\u044e\u00bb \u0440\u0435\u043f\u043b\u0438\u043a\u0443 \u043f\u043e\u0434 \u0430\u043d\u0430\u043b\u0438\u0442\u0438\u043a\u0443, \u0430 \u043c\u043e\u0436\u0435\u0442\u0435 \u043d\u0430\u043f\u0438\u0441\u0430\u0442\u044c \u0446\u0435\u043b\u044b\u0439 \u0441\u0435\u0440\u0432\u0438\u0441, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0431\u0443\u0434\u0435\u0442 \u0440\u0430\u0437 \u0432 \u0447\u0430\u0441 \u043e\u0431\u043d\u043e\u0432\u043b\u044f\u0442\u044c \u0434\u0430\u043d\u043d\u044b\u0435, \u0434\u043e\u043e\u0431\u0443\u0447\u0430\u0442\u044c ML-\u043c\u043e\u0434\u0435\u043b\u044c \u0438 \u043f\u043e\u0441\u0442\u0430\u0432\u043b\u044f\u0442\u044c \u0430\u043a\u0442\u0443\u0430\u043b\u044c\u043d\u044b\u0435 \u043f\u0440\u043e\u0433\u043d\u043e\u0437\u043d\u044b\u0435 \u0437\u043d\u0430\u0447\u0435\u043d\u0438\u044f \u043a\u043e\u043d\u0435\u0447\u043d\u044b\u043c \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f\u043c.<\/p>\n\n\n\n<p>\u0414\u0430\u043d\u043d\u0430\u044f \u0442\u0435\u0445\u043d\u043e\u043b\u043e\u0433\u0438\u044f \u2014 \u043d\u0435\u043e\u0442\u044a\u0435\u043c\u043b\u0435\u043c\u044b\u0439 \u0438\u043d\u0441\u0442\u0440\u0443\u043c\u0435\u043d\u0442 \u0432 \u0430\u0440\u0441\u0435\u043d\u0430\u043b\u0435 \u0441\u043e\u0432\u0440\u0435\u043c\u0435\u043d\u043d\u043e\u0433\u043e Data Engineer \u0438 \u0430\u0434\u043c\u0438\u043d\u0438\u0441\u0442\u0440\u0430\u0442\u043e\u0440\u0430 \u0432\u044b\u0447\u0438\u0441\u043b\u0438\u0442\u0435\u043b\u044c\u043d\u043e\u0433\u043e \u043a\u043b\u0430\u0441\u0442\u0435\u0440\u0430.<\/p>\n\n\n\n<p>\u041d\u0430 \u043e\u0442\u043a\u0440\u044b\u0442\u043e\u043c \u0443\u0440\u043e\u043a\u0435 \u043c\u044b \u043f\u043e\u0434\u0440\u043e\u0431\u043d\u043e \u0440\u0430\u0437\u043e\u0431\u0440\u0430\u043b\u0438, \u0447\u0442\u043e \u0442\u0430\u043a\u043e\u0435 \u043f\u043b\u0430\u0442\u0444\u043e\u0440\u043c\u044b \u041e\u0440\u043a\u0435\u0441\u0442\u0440\u0430\u0446\u0438\u0438, \u043a\u0430\u043a\u0438\u0435 \u0440\u0435\u0448\u0435\u043d\u0438\u044f \u0435\u0441\u0442\u044c \u0441\u0435\u0433\u043e\u0434\u043d\u044f \u043d\u0430 \u0440\u044b\u043d\u043a\u0435 \u0438 \u0443\u0433\u043b\u0443\u0431\u0438\u043b\u0438\u0441\u044c \u0432 \u043f\u0440\u0430\u043a\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u043f\u0440\u0438\u043c\u0435\u0440 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u044f \u043e\u0434\u043d\u043e\u0439 \u0438\u0437 \u0441\u0430\u043c\u044b\u0445 \u0440\u0430\u0441\u043f\u0440\u043e\u0441\u0442\u0440\u0430\u043d\u0435\u043d\u043d\u044b\u0445 \u043f\u043b\u0430\u0442\u0444\u043e\u0440\u043c \u043d\u0430 \u0441\u0435\u0433\u043e\u0434\u043d\u044f: Apache Airflow.<\/p>\n\n\n\n<div class=\"wp-block-columns are-vertically-aligned-center is-layout-flex wp-container-core-columns-layout-1 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-vertically-aligned-center is-layout-flow wp-block-column-is-layout-flow\" style=\"flex-basis:15%\">\n<div class=\"wp-block-image\"><figure class=\"aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/otus.ru\/journal\/wp-content\/uploads\/2022\/07\/Maksim-Migutin_80h80.png\" alt=\"\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435 \u0432 \u043e\u0440\u043a\u0435\u0441\u0442\u0440\u0430\u0446\u0438\u044e\" class=\"wp-image-4111\" width=\"100\" height=\"100\"\/><\/figure><\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-center is-layout-flow wp-block-column-is-layout-flow\" style=\"flex-basis:2%\"><\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-center is-layout-flow wp-block-column-is-layout-flow\" style=\"flex-basis:55%\">\n<p class=\"has-text-align-left\" style=\"line-height:1.5\"><strong>\u0421\u043f\u0438\u043a\u0435\u0440: \u041c\u0430\u043a\u0441\u0438\u043c \u041c\u0438\u0433\u0443\u0442\u0438\u043d<\/strong> <\/p>\n\n\n\n<p class=\"has-text-align-left\" style=\"line-height:1.5\">\u0411\u043e\u043b\u0435\u0435 5-\u0442\u0438 \u043b\u0435\u0442 \u043e\u043f\u044b\u0442\u0430 \u0432 \u0438\u043d\u0434\u0443\u0441\u0442\u0440\u0438\u0438 \u0414\u0430\u043d\u043d\u044b\u0445 &amp; \u0410\u043d\u0430\u043b\u0438\u0442\u0438\u043a\u0438 \u0432 \u0440\u043e\u043b\u0438 \u0432\u043d\u0435\u0448\u043d\u0435\u0433\u043e \u043a\u043e\u0441\u0443\u043b\u044c\u0442\u0430\u043d\u0442\u0430 \u0438 in-house \u043b\u0438\u0434\u0435\u0440\u0430 Data Engineering \u0438 Data Science-\u043f\u0440\u043e\u0435\u043a\u0442\u043e\u0432<\/p>\n<\/div>\n<\/div>\n\n\n\n<hr class=\"wp-block-separator\"\/>\n\n\n\n<h4 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%D0%A7%D0%B8%D1%82%D0%B0%D0%B9%D1%82%D0%B5_%D0%BD%D0%B0_%D0%A5%D0%B0%D0%B1%D1%80\"><\/span>\u0427\u0438\u0442\u0430\u0439\u0442\u0435 \u043d\u0430 \u0425\u0430\u0431\u0440:<span class=\"ez-toc-section-end\"><\/span><\/h4>\n\n\n\n<p><a href=\"https:\/\/habr.com\/ru\/company\/otus\/blog\/667274\/\" target=\"_blank\" rel=\"noreferrer noopener\">\u0421\u043e\u0432\u0440\u0435\u043c\u0435\u043d\u043d\u044b\u0439 \u0434\u0430\u0442\u0430-\u0441\u0442\u0435\u043a: \u043f\u0440\u043e\u0448\u043b\u043e\u0435, \u043d\u0430\u0441\u0442\u043e\u044f\u0449\u0435\u0435 \u0438 \u0431\u0443\u0434\u0443\u0449\u0435\u0435<\/a><\/p>\n\n\n\n<p><a href=\"https:\/\/habr.com\/ru\/company\/otus\/blog\/650471\/\" target=\"_blank\" rel=\"noreferrer noopener\">\u0412\u0440\u0435\u0434\u043d\u044b\u0435 \u0441\u043e\u0432\u0435\u0442\u044b \u043f\u0440\u0438 \u043f\u043e\u0441\u0442\u0440\u043e\u0435\u043d\u0438\u0438 \u0410\u043d\u0430\u043b\u0438\u0442\u0438\u043a\u0438 (Data Lake \/ DWH \/ BI) \u2013 \u0446\u0435\u043b\u0435\u043f\u043e\u043b\u0430\u0433\u0430\u043d\u0438\u0435 \u0438 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b<\/a><\/p>\n\n\n\n<p><a href=\"https:\/\/habr.com\/ru\/company\/otus\/blog\/665642\/\" target=\"_blank\" rel=\"noreferrer noopener\">\u041a\u0442\u043e \u0442\u0430\u043a\u043e\u0439 Analytics Engineer \u2013 E2E-\u0440\u0435\u0448\u0435\u043d\u0438\u0435 \u0441 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435\u043c bash + dbt + Looker<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u041d\u0430 \u043e\u0442\u043a\u0440\u044b\u0442\u043e\u043c \u0443\u0440\u043e\u043a\u0435 \u043c\u044b \u043f\u043e\u0434\u0440\u043e\u0431\u043d\u043e \u0440\u0430\u0437\u043e\u0431\u0440\u0430\u043b\u0438, \u0447\u0442\u043e \u0442\u0430\u043a\u043e\u0435 \u043f\u043b\u0430\u0442\u0444\u043e\u0440\u043c\u044b \u041e\u0440\u043a\u0435\u0441\u0442\u0440\u0430\u0446\u0438\u0438, \u043a\u0430\u043a\u0438\u0435 \u0440\u0435\u0448\u0435\u043d\u0438\u044f \u0435\u0441\u0442\u044c \u0441\u0435\u0433\u043e\u0434\u043d\u044f \u043d\u0430 \u0440\u044b\u043d\u043a\u0435 \u0438 \u0443\u0433\u043b\u0443\u0431\u0438\u043b\u0438\u0441\u044c \u0432 \u043f\u0440\u0430\u043a\u0442\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u043f\u0440\u0438\u043c\u0435\u0440 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u044f \u043e\u0434\u043d\u043e\u0439 \u0438\u0437 \u0441\u0430\u043c\u044b\u0445 \u0440\u0430\u0441\u043f\u0440\u043e\u0441\u0442\u0440\u0430\u043d\u0435\u043d\u043d\u044b\u0445 \u043f\u043b\u0430\u0442\u0444\u043e\u0440\u043c \u043d\u0430 \u0441\u0435\u0433\u043e\u0434\u043d\u044f: Apache Airflow.<\/p>\n","protected":false},"author":10,"featured_media":4244,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"om_disable_all_campaigns":false,"footnotes":""},"categories":[1],"tags":[201],"aioseo_notices":[],"_links":{"self":[{"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/posts\/4110"}],"collection":[{"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/users\/10"}],"replies":[{"embeddable":true,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/comments?post=4110"}],"version-history":[{"count":1,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/posts\/4110\/revisions"}],"predecessor-version":[{"id":4246,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/posts\/4110\/revisions\/4246"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/media\/4244"}],"wp:attachment":[{"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/media?parent=4110"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/categories?post=4110"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/tags?post=4110"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}