{"id":4114,"date":"2022-07-22T11:52:34","date_gmt":"2022-07-22T11:52:34","guid":{"rendered":"https:\/\/otus.ru\/journal\/?p=4114"},"modified":"2022-08-03T13:59:58","modified_gmt":"2022-08-03T13:59:58","slug":"extract-load-s-pomoshhju-saas-servisa-vs-cobstvennoe-reshenie-poisk-balansa-i-dzen","status":"publish","type":"post","link":"https:\/\/otus.ru\/journal\/extract-load-s-pomoshhju-saas-servisa-vs-cobstvennoe-reshenie-poisk-balansa-i-dzen\/","title":{"rendered":"Extract &#8212; Load \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e SaaS-\u0441\u0435\u0440\u0432\u0438\u0441\u0430 vs. C\u043e\u0431\u0441\u0442\u0432\u0435\u043d\u043d\u043e\u0435 \u0440\u0435\u0448\u0435\u043d\u0438\u0435. \u041f\u043e\u0438\u0441\u043a \u0431\u0430\u043b\u0430\u043d\u0441\u0430 \u0438 \u0434\u0437\u0435\u043d"},"content":{"rendered":"<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_61 counter-hierarchy ez-toc-counter ez-toc-grey ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title \" >\u0421\u043e\u0434\u0435\u0440\u0436\u0430\u043d\u0438\u0435<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #999;color:#999\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #999;color:#999\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"https:\/\/otus.ru\/journal\/extract-load-s-pomoshhju-saas-servisa-vs-cobstvennoe-reshenie-poisk-balansa-i-dzen\/#%D0%97%D0%B0%D0%BF%D0%B8%D1%81%D1%8C_%D0%B4%D0%B2%D1%83%D1%85%D0%B4%D0%BD%D0%B5%D0%B2%D0%BD%D0%BE%D0%B3%D0%BE_%D0%B8%D0%BD%D1%82%D0%B5%D0%BD%D1%81%D0%B8%D0%B2%D0%B0_%D0%BA%D1%83%D1%80%D1%81%D0%B0_%C2%ABData_Engineer%C2%BB\" title=\"\u0417\u0430\u043f\u0438\u0441\u044c \u0434\u0432\u0443\u0445\u0434\u043d\u0435\u0432\u043d\u043e\u0433\u043e \u0438\u043d\u0442\u0435\u043d\u0441\u0438\u0432\u0430 \u043a\u0443\u0440\u0441\u0430 \u00abData Engineer\u00bb\">\u0417\u0430\u043f\u0438\u0441\u044c \u0434\u0432\u0443\u0445\u0434\u043d\u0435\u0432\u043d\u043e\u0433\u043e \u0438\u043d\u0442\u0435\u043d\u0441\u0438\u0432\u0430 \u043a\u0443\u0440\u0441\u0430 \u00abData Engineer\u00bb<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"https:\/\/otus.ru\/journal\/extract-load-s-pomoshhju-saas-servisa-vs-cobstvennoe-reshenie-poisk-balansa-i-dzen\/#%D0%97%D0%B0%D0%BF%D0%B8%D1%81%D0%B8_%D0%B4%D1%80%D1%83%D0%B3%D0%B8%D1%85_%D0%BE%D1%82%D0%BA%D1%80%D1%8B%D1%82%D1%8B%D1%85_%D1%83%D1%80%D0%BE%D0%BA%D0%BE%D0%B2\" title=\"\u0417\u0430\u043f\u0438\u0441\u0438 \u0434\u0440\u0443\u0433\u0438\u0445 \u043e\u0442\u043a\u0440\u044b\u0442\u044b\u0445 \u0443\u0440\u043e\u043a\u043e\u0432:\">\u0417\u0430\u043f\u0438\u0441\u0438 \u0434\u0440\u0443\u0433\u0438\u0445 \u043e\u0442\u043a\u0440\u044b\u0442\u044b\u0445 \u0443\u0440\u043e\u043a\u043e\u0432:<\/a><\/li><\/ul><\/li><\/ul><\/li><\/ul><\/nav><\/div>\n\n<h2 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%D0%97%D0%B0%D0%BF%D0%B8%D1%81%D1%8C_%D0%B4%D0%B2%D1%83%D1%85%D0%B4%D0%BD%D0%B5%D0%B2%D0%BD%D0%BE%D0%B3%D0%BE_%D0%B8%D0%BD%D1%82%D0%B5%D0%BD%D1%81%D0%B8%D0%B2%D0%B0_%D0%BA%D1%83%D1%80%D1%81%D0%B0_%C2%ABData_Engineer%C2%BB\"><\/span>\u0417\u0430\u043f\u0438\u0441\u044c \u0434\u0432\u0443\u0445\u0434\u043d\u0435\u0432\u043d\u043e\u0433\u043e \u0438\u043d\u0442\u0435\u043d\u0441\u0438\u0432\u0430 \u043a\u0443\u0440\u0441\u0430 \u00abData Engineer\u00bb<span class=\"ez-toc-section-end\"><\/span><\/h2>\n\n\n\n<figure class=\"wp-block-embed is-type-video is-provider-youtube wp-block-embed-youtube wp-embed-aspect-16-9 wp-has-aspect-ratio\"><div class=\"wp-block-embed__wrapper\">\n<iframe loading=\"lazy\" title=\"\u0414\u0435\u043c\u043e-\u0437\u0430\u043d\u044f\u0442\u0438\u0435 \u043a\u0443\u0440\u0441\u0430 \u00abData Engineer\u00bb. \u0414\u0435\u043d\u044c 1\" width=\"770\" height=\"433\" src=\"https:\/\/www.youtube.com\/embed\/CR32LFCgtGE?feature=oembed\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen><\/iframe>\n<\/div><figcaption>\u0418\u043d\u0442\u0435\u043d\u0441\u0438\u0432. \u0414\u0435\u043d\u044c 1<\/figcaption><\/figure>\n\n\n\n<figure class=\"wp-block-embed is-type-video is-provider-youtube wp-block-embed-youtube wp-embed-aspect-16-9 wp-has-aspect-ratio\"><div class=\"wp-block-embed__wrapper\">\n<iframe loading=\"lazy\" title=\"\u0414\u0435\u043c\u043e-\u0437\u0430\u043d\u044f\u0442\u0438\u0435 \u043a\u0443\u0440\u0441\u0430 \u00abData Engineer\u00bb  2 \u0414\u0435\u043d\u044c\" width=\"770\" height=\"433\" src=\"https:\/\/www.youtube.com\/embed\/dSm2kDsRhOI?feature=oembed\" frameborder=\"0\" allow=\"accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture\" allowfullscreen><\/iframe>\n<\/div><figcaption>\u0418\u043d\u0442\u0435\u043d\u0441\u0438\u0432. \u0414\u0435\u043d\u044c 2<\/figcaption><\/figure>\n\n\n\n<p>\u041d\u0430 \u0431\u0435\u0441\u043f\u043b\u0430\u0442\u043d\u043e\u043c \u0434\u0432\u0443\u0445\u0434\u043d\u0435\u0432\u043d\u043e\u043c \u0438\u043d\u0442\u0435\u043d\u0441\u0438\u0432\u0435 \u043e\u0431\u0441\u0443\u0434\u0438\u043b\u0438:<\/p>\n\n\n\n<ul><li>Extract-Load \u0447\u0435\u0440\u0435\u0437 SaaS \u0440\u0435\u0448\u0435\u043d\u0438\u044f. \u0412\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u0438 \u0433\u043e\u0442\u043e\u0432\u044b\u0445 \u0441\u0435\u0440\u0432\u0438\u0441\u043e\u0432, \u0438\u0445 \u043d\u0430\u0434\u0435\u0436\u043d\u043e\u0441\u0442\u044c \u0438 \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f;<\/li><li>Extract-Load \u0447\u0435\u0440\u0435\u0437 API-\u0432\u044b\u0437\u043e\u0432\u044b, \u043e\u0431\u0440\u0430\u0449\u0435\u043d\u0438\u044f \u043a \u0421\u0423\u0411\u0414 \u0438 CDC \u2013 \u043e\u043f\u0442\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u0435 \u0441\u043f\u043e\u0441\u043e\u0431\u044b \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438;<\/li><li>\u0410\u0432\u0442\u043e\u043c\u0430\u0442\u0438\u0437\u0430\u0446\u0438\u044f \u0432\u044b\u0433\u0440\u0443\u0437\u043a\u0438, \u043f\u043e\u0432\u0442\u043e\u0440\u043d\u044b\u0435 \u043f\u043e\u043f\u044b\u0442\u043a\u0438 (retries), \u043f\u043e\u043b\u0443\u0447\u0435\u043d\u0438\u0435 \u0443\u0432\u0435\u0434\u043e\u043c\u043b\u0435\u043d\u0438\u0439 \u0432 Slack (notifications) \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e Airflow;<\/li><li>\u041d\u0430\u043a\u043e\u043f\u043b\u0435\u043d\u0438\u0435 \u0438\u0441\u0442\u043e\u0440\u0438\u0438 \u0432\u044b\u0433\u0440\u0443\u0437\u043e\u043a \u0438 \u043e\u0440\u0433\u0430\u043d\u0438\u0437\u0430\u0446\u0438\u044f Data Lake \u0432 S3 \u043f\u0435\u0440\u0435\u0434 DWH.<\/li><\/ul>\n\n\n\n<div class=\"wp-block-columns are-vertically-aligned-center is-layout-flex wp-container-core-columns-layout-1 wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-vertically-aligned-center is-layout-flow wp-block-column-is-layout-flow\" style=\"flex-basis:15%\">\n<div class=\"wp-block-image\"><figure class=\"aligncenter size-large\"><img loading=\"lazy\" decoding=\"async\" width=\"80\" height=\"80\" src=\"https:\/\/otus.ru\/journal\/wp-content\/uploads\/2022\/07\/Artemij_Kozyr_80h80.png\" alt=\"Extract - Load \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e SaaS-\u0441\u0435\u0440\u0432\u0438\u0441\u0430 vs. C\u043e\u0431\u0441\u0442\u0432\u0435\u043d\u043d\u043e\u0435 \u0440\u0435\u0448\u0435\u043d\u0438\u0435. \u041f\u043e\u0438\u0441\u043a \u0431\u0430\u043b\u0430\u043d\u0441\u0430 \u0438 \u0434\u0437\u0435\u043d\" class=\"wp-image-4115\"\/><\/figure><\/div>\n<\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-center is-layout-flow wp-block-column-is-layout-flow\" style=\"flex-basis:2%\"><\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-center is-layout-flow wp-block-column-is-layout-flow\" style=\"flex-basis:55%\">\n<p class=\"has-text-align-left\" style=\"line-height:1.5\"><strong>\u0421\u043f\u0438\u043a\u0435\u0440: \u0410\u0440\u0442\u0435\u043c\u0438\u0439 \u041a\u043e\u0437\u044b\u0440\u044c<\/strong> <\/p>\n\n\n\n<p>Analytics Engineer. 7+ \u043b\u0435\u0442 \u043e\u043f\u044b\u0442\u0430 \u0432 \u043e\u0431\u043b\u0430\u0441\u0442\u0438 \u0425\u0440\u0430\u043d\u0438\u043b\u0438\u0449 \u0414\u0430\u043d\u043d\u044b\u0445, ELT pipelines, \u0410\u043d\u0430\u043b\u0438\u0437\u0430 \u0434\u0430\u043d\u043d\u044b\u0445 \u0438 \u0432\u0438\u0437\u0443\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438.<\/p>\n<\/div>\n<\/div>\n\n\n\n<hr class=\"wp-block-separator\"\/>\n\n\n\n<h4 class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"%D0%97%D0%B0%D0%BF%D0%B8%D1%81%D0%B8_%D0%B4%D1%80%D1%83%D0%B3%D0%B8%D1%85_%D0%BE%D1%82%D0%BA%D1%80%D1%8B%D1%82%D1%8B%D1%85_%D1%83%D1%80%D0%BE%D0%BA%D0%BE%D0%B2\"><\/span>\u0417\u0430\u043f\u0438\u0441\u0438 \u0434\u0440\u0443\u0433\u0438\u0445 \u043e\u0442\u043a\u0440\u044b\u0442\u044b\u0445 \u0443\u0440\u043e\u043a\u043e\u0432:<span class=\"ez-toc-section-end\"><\/span><\/h4>\n\n\n\n<p><a href=\"https:\/\/otus.ru\/journal\/vvedenie-v-orkestraciju\/\" target=\"_blank\" rel=\"noreferrer noopener\">\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435 \u0432 \u043e\u0440\u043a\u0435\u0441\u0442\u0440\u0430\u0446\u0438\u044e<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u041d\u0430 \u0431\u0435\u0441\u043f\u043b\u0430\u0442\u043d\u043e\u043c \u0434\u0432\u0443\u0445\u0434\u043d\u0435\u0432\u043d\u043e\u043c \u0438\u043d\u0442\u0435\u043d\u0441\u0438\u0432\u0435 \u043e\u0431\u0441\u0443\u0434\u0438\u043b\u0438: Extract-Load \u0447\u0435\u0440\u0435\u0437 SaaS \u0440\u0435\u0448\u0435\u043d\u0438\u044f. \u0412\u043e\u0437\u043c\u043e\u0436\u043d\u043e\u0441\u0442\u0438 \u0433\u043e\u0442\u043e\u0432\u044b\u0445 \u0441\u0435\u0440\u0432\u0438\u0441\u043e\u0432, \u0438\u0445 \u043d\u0430\u0434\u0435\u0436\u043d\u043e\u0441\u0442\u044c \u0438 \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u044f; Extract-Load \u0447\u0435\u0440\u0435\u0437 API-\u0432\u044b\u0437\u043e\u0432\u044b, \u043e\u0431\u0440\u0430\u0449\u0435\u043d\u0438\u044f \u043a \u0421\u0423\u0411\u0414 \u0438 CDC \u2013 \u043e\u043f\u0442\u0438\u043c\u0430\u043b\u044c\u043d\u044b\u0435 \u0441\u043f\u043e\u0441\u043e\u0431\u044b \u0440\u0435\u0430\u043b\u0438\u0437\u0430\u0446\u0438\u0438; \u0410\u0432\u0442\u043e\u043c\u0430\u0442\u0438\u0437\u0430\u0446\u0438\u044f \u0432\u044b\u0433\u0440\u0443\u0437\u043a\u0438, \u043f\u043e\u0432\u0442\u043e\u0440\u043d\u044b\u0435 \u043f\u043e\u043f\u044b\u0442\u043a\u0438 (retries), \u043f\u043e\u043b\u0443\u0447\u0435\u043d\u0438\u0435 \u0443\u0432\u0435\u0434\u043e\u043c\u043b\u0435\u043d\u0438\u0439 \u0432 Slack (notifications) \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e Airflow; \u041d\u0430\u043a\u043e\u043f\u043b\u0435\u043d\u0438\u0435 \u0438\u0441\u0442\u043e\u0440\u0438\u0438 \u0432\u044b\u0433\u0440\u0443\u0437\u043e\u043a \u0438 \u043e\u0440\u0433\u0430\u043d\u0438\u0437\u0430\u0446\u0438\u044f Data Lake \u0432 S3 \u043f\u0435\u0440\u0435\u0434 DWH.<\/p>\n","protected":false},"author":10,"featured_media":4241,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"om_disable_all_campaigns":false,"footnotes":""},"categories":[1],"tags":[201],"aioseo_notices":[],"_links":{"self":[{"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/posts\/4114"}],"collection":[{"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/users\/10"}],"replies":[{"embeddable":true,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/comments?post=4114"}],"version-history":[{"count":1,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/posts\/4114\/revisions"}],"predecessor-version":[{"id":4243,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/posts\/4114\/revisions\/4243"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/media\/4241"}],"wp:attachment":[{"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/media?parent=4114"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/categories?post=4114"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/otus.ru\/journal\/wp-json\/wp\/v2\/tags?post=4114"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}