{"id":375241,"date":"2024-05-21T06:19:46","date_gmt":"2024-05-21T06:19:46","guid":{"rendered":"http:\/\/savepearlharbor.com\/?p=375241"},"modified":"-0001-11-30T00:00:00","modified_gmt":"-0001-11-29T21:00:00","slug":"","status":"publish","type":"post","link":"https:\/\/savepearlharbor.com\/?p=375241","title":{"rendered":"Apache Airflow: \u043f\u0440\u0435\u0438\u043c\u0443\u0449\u0435\u0441\u0442\u0432\u0430 \u0438 \u043d\u0435\u0434\u043e\u0441\u0442\u0430\u0442\u043a\u0438<\/span>"},"content":{"rendered":"
<\/div>\n
\n
\n
\n
\n
\n

\u041e\u0440\u043a\u0435\u0441\u0442\u0440\u0430\u0442\u043e\u0440 \u0437\u0430\u0434\u0430\u0447<\/em><\/strong>\u00a0\u2014 \u0438\u043d\u0441\u0442\u0440\u0443\u043c\u0435\u043d\u0442 \u0438\u043b\u0438\u00a0\u0441\u0438\u0441\u0442\u0435\u043c\u0430, \u043f\u0440\u0435\u0434\u043d\u0430\u0437\u043d\u0430\u0447\u0435\u043d\u043d\u044b\u0435 \u0434\u043b\u044f\u00a0\u0443\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u0438\u044f \u0438 \u043a\u043e\u043d\u0442\u0440\u043e\u043b\u044f \u0432\u044b\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u0435\u043c \u0437\u0430\u0434\u0430\u0447 \u0432\u00a0\u0441\u043b\u043e\u0436\u043d\u044b\u0445 \u0432\u044b\u0447\u0438\u0441\u043b\u0438\u0442\u0435\u043b\u044c\u043d\u044b\u0445 \u0438 \u0438\u043d\u0444\u043e\u0440\u043c\u0430\u0446\u0438\u043e\u043d\u043d\u044b\u0445 \u0441\u0440\u0435\u0434\u0430\u0445. \u041e\u043d \u043e\u0431\u043b\u0435\u0433\u0447\u0430\u044e\u0442 \u043f\u0440\u043e\u0446\u0435\u0441\u0441 \u0440\u0430\u0437\u0432\u0435\u0440\u0442\u044b\u0432\u0430\u043d\u0438\u044f, \u0430\u0432\u0442\u043e\u043c\u0430\u0442\u0438\u0437\u0430\u0446\u0438\u0438 \u0438 \u0443\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u0438\u044f \u0432\u044b\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u0435\u043c \u0437\u0430\u0434\u0430\u0447, \u0447\u0442\u043e\u00a0\u043f\u043e\u0437\u0432\u043e\u043b\u044f\u0435\u0442 \u043f\u043e\u0432\u044b\u0441\u0438\u0442\u044c \u044d\u0444\u0444\u0435\u043a\u0442\u0438\u0432\u043d\u043e\u0441\u0442\u044c \u0440\u0430\u0431\u043e\u0442\u044b \u0438 \u043e\u043f\u0442\u0438\u043c\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0440\u0435\u0441\u0443\u0440\u0441\u044b. <\/p>\n<\/blockquote>\n

\u041e\u0434\u043d\u0438\u043c \u0438\u0437\u00a0\u043f\u043e\u043f\u0443\u043b\u044f\u0440\u043d\u044b\u0445 \u043e\u0440\u043a\u0435\u0441\u0442\u0440\u0430\u0442\u043e\u0440\u043e\u043c \u0437\u0430\u0434\u0430\u0447 \u044f\u0432\u043b\u044f\u0435\u0442\u0441\u044f Apache Airflow. \u041e\u043d, \u043a\u0430\u043a\u00a0\u0438 \u0432\u0441\u0435 \u0438\u043d\u0441\u0442\u0440\u0443\u043c\u0435\u043d\u0442\u044b, \u0438\u043c\u0435\u0435\u0442 \u0441\u0432\u043e\u0438 \u043f\u0440\u0435\u0438\u043c\u0443\u0449\u0435\u0441\u0442\u0432\u0430 \u0438 \u043d\u0435\u0434\u043e\u0441\u0442\u0430\u0442\u043a\u0438, \u043e\u00a0\u043a\u043e\u0442\u043e\u0440\u044b\u0445 \u043f\u043e\u0439\u0434\u0435\u0442 \u0440\u0435\u0447\u044c \u0432\u00a0\u0434\u0430\u043d\u043d\u043e\u0439 \u0441\u0442\u0430\u0442\u044c\u0435<\/p>\n

Apache Airflow<\/h2>\n
\n

Apache Airflow<\/em><\/strong> — \u044d\u0442\u043e \u043f\u043b\u0430\u0442\u0444\u043e\u0440\u043c\u0430 \u0434\u043b\u044f \u043f\u0440\u043e\u0433\u0440\u0430\u043c\u043c\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f, \u043f\u043b\u0430\u043d\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u0438 \u043c\u043e\u043d\u0438\u0442\u043e\u0440\u0438\u043d\u0433\u0430 \u0440\u0430\u0431\u043e\u0447\u0438\u0445 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0432 (Workflows). \u041e\u043d \u043f\u043e\u0437\u0432\u043e\u043b\u044f\u0435\u0442 \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u0442\u044c \u0438 \u0443\u043f\u0440\u0430\u0432\u043b\u044f\u0442\u044c \u0441\u043b\u043e\u0436\u043d\u044b\u043c\u0438 \u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e\u0441\u0442\u044f\u043c\u0438 \u043c\u0435\u0436\u0434\u0443 \u0437\u0430\u0434\u0430\u0447\u0430\u043c\u0438, \u0430\u0432\u0442\u043e\u043c\u0430\u0442\u0438\u0437\u0438\u0440\u043e\u0432\u0430\u0442\u044c \u0438\u0445 \u0432\u044b\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u0435 \u0438 \u043c\u043e\u043d\u0438\u0442\u043e\u0440\u0438\u0442\u044c \u043f\u0440\u043e\u0446\u0435\u0441\u0441 \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0438 \u0434\u0430\u043d\u043d\u044b\u0445. <\/p>\n<\/blockquote>\n

Airflow<\/em> \u043f\u0440\u0435\u0434\u043e\u0441\u0442\u0430\u0432\u043b\u044f\u0435\u0442 \u0433\u0440\u0430\u0444\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0438\u043d\u0442\u0435\u0440\u0444\u0435\u0439\u0441 \u0434\u043b\u044f\u00a0\u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f, \u0432\u044b\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u044f \u0438 \u043c\u043e\u043d\u0438\u0442\u043e\u0440\u0438\u043d\u0433\u0430 \u0440\u0430\u0431\u043e\u0447\u0438\u0445 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u043e\u0432, \u0442\u0430\u043a\u0436\u0435 \u043e\u043d \u043f\u043e\u0437\u0432\u043e\u043b\u044f\u0435\u0442 \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u0442\u044c \u0440\u0430\u0431\u043e\u0447\u0438\u0435 \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u044b \u0432\u00a0\u0444\u043e\u0440\u043c\u0435 \u043d\u0430\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u043d\u044b\u0445 \u0430\u0446\u0438\u043a\u043b\u0438\u0447\u043d\u044b\u0445 \u0433\u0440\u0430\u0444\u043e\u0432 (DAG<\/em>), \u0433\u0434\u0435 \u043a\u0430\u0436\u0434\u044b\u0439 \u0443\u0437\u0435\u043b \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u044f\u0435\u0442 \u0441\u043e\u0431\u043e\u0439 \u043e\u0442\u0434\u0435\u043b\u044c\u043d\u0443\u044e \u0437\u0430\u0434\u0430\u0447\u0443. \u042d\u0442\u043e \u043e\u0431\u0435\u0441\u043f\u0435\u0447\u0438\u0432\u0430\u0435\u0442 \u0443\u0434\u043e\u0431\u0441\u0442\u0432\u043e \u0438 \u0433\u0438\u0431\u043a\u043e\u0441\u0442\u044c \u0432\u00a0\u0443\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u0438\u0438 \u0437\u0430\u0434\u0430\u0447\u0430\u043c\u0438 \u0438 \u0437\u0430\u0432\u0438\u0441\u0438\u043c\u043e\u0441\u0442\u044f\u043c\u0438 \u043c\u0435\u0436\u0434\u0443 \u043d\u0438\u043c\u0438.<\/p>\n

Airflow \u0448\u0438\u0440\u043e\u043a\u043e \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442\u0441\u044f \u0434\u043b\u044f\u00a0\u0430\u0432\u0442\u043e\u043c\u0430\u0442\u0438\u0437\u0430\u0446\u0438\u0438 \u0440\u0430\u0441\u043f\u0440\u0435\u0434\u0435\u043b\u0435\u043d\u043d\u044b\u0445 \u0432\u044b\u0447\u0438\u0441\u043b\u0435\u043d\u0438\u0439, \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0438 \u0434\u0430\u043d\u043d\u044b\u0445, \u0443\u043f\u0440\u0430\u0432\u043b\u0435\u043d\u0438\u044f ETL<\/em> (Extract, Transform, Load) \u043f\u0440\u043e\u0446\u0435\u0441\u0441\u0430\u043c\u0438, \u043f\u043b\u0430\u043d\u0438\u0440\u043e\u0432\u0430\u043d\u0438\u044f \u0437\u0430\u0434\u0430\u0447 \u0438 \u0434\u0440\u0443\u0433\u0438\u0445 \u0441\u0446\u0435\u043d\u0430\u0440\u0438\u0435\u0432 \u0440\u0430\u0431\u043e\u0442\u044b \u0441\u00a0\u0434\u0430\u043d\u043d\u044b\u043c\u0438. <\/p>\n

\n

\u041f\u043e \u0443\u043c\u043e\u043b\u0447\u0430\u043d\u0438\u044e Apache Airflow \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u0435\u0442 SQLite, \u043a\u0430\u043a \u0431\u0430\u0437\u0443 \u0434\u0430\u043d\u043d\u044b\u0445.<\/p>\n<\/blockquote>\n

\u041e\u0434\u0438\u043d \u0438\u0437 \u0441\u043f\u043e\u0441\u043e\u0431\u043e\u0432 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u044f Apache Airflow — docker \u0444\u0430\u0439\u043b.<\/p>\n

\ndocker-compose<\/summary>\n
\n
version: '3.8' x-airflow-common:   &airflow-common   image: ${AIRFLOW_IMAGE_NAME:-apache\/airflow:2.7.3}   environment:     &airflow-common-env     AIRFLOW__CORE__EXECUTOR: LocalExecutor     AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2:\/\/airflow:airflow@postgres\/airflow     AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2:\/\/airflow:airflow@postgres\/airflow     AIRFLOW__CORE__FERNET_KEY: ''     AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'     AIRFLOW__CORE__LOAD_EXAMPLES: 'false'     AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session'     AIRFLOW__SCHEDULER__ENABLE_HEALTH_CHECK: 'true'     _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-}   volumes:     - ${AIRFLOW_PROJ_DIR:-.}\/dags:\/opt\/airflow\/dags     - ${AIRFLOW_PROJ_DIR:-.}\/logs:\/opt\/airflow\/logs     - ${AIRFLOW_PROJ_DIR:-.}\/config:\/opt\/airflow\/config     - ${AIRFLOW_PROJ_DIR:-.}\/plugins:\/opt\/airflow\/plugins   user: \"${AIRFLOW_UID:-50000}:0\"   depends_on:     &airflow-common-depends-on     postgres:       condition: service_healthy  services:   postgres:     image: postgres:13     environment:       POSTGRES_USER: airflow       POSTGRES_PASSWORD: airflow       POSTGRES_DB: airflow     volumes:       - postgres-db-volume:\/var\/lib\/postgresql\/data     healthcheck:       test: [\"CMD\", \"pg_isready\", \"-U\", \"airflow\"]       interval: 10s       retries: 5       start_period: 5s     restart: always    airflow-webserver:     <<: *airflow-common     command: webserver     ports:       - \"8080:8080\"     healthcheck:       test: [\"CMD\", \"curl\", \"--fail\", \"http:\/\/localhost:8080\/health\"]       interval: 30s       timeout: 10s       retries: 5       start_period: 30s     restart: always     depends_on:       <<: *airflow-common-depends-on       airflow-init:         condition: service_completed_successfully    airflow-scheduler:     <<: *airflow-common     command: scheduler     healthcheck:       test: [\"CMD\", \"curl\", \"--fail\", \"http:\/\/localhost:8974\/health\"]       interval: 30s       timeout: 10s       retries: 5       start_period: 30s     restart: always     depends_on:       <<: *airflow-common-depends-on       airflow-init:         condition: service_completed_successfully    airflow-init:     <<: *airflow-common     entrypoint: \/bin\/bash     # yamllint disable rule:line-length     command:       - -c       - |         function ver() {           printf \"%04d%04d%04d%04d\" $${1\/\/.\/ }         }         airflow_version=$$(AIRFLOW__LOGGING__LOGGING_LEVEL=INFO && gosu airflow airflow version)         airflow_version_comparable=$$(ver $${airflow_version})         min_airflow_version=2.2.0         min_airflow_version_comparable=$$(ver $${min_airflow_version})         if (( airflow_version_comparable < min_airflow_version_comparable )); then           echo           echo -e \"\\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\\e[0m\"           echo \"The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!\"           echo           exit 1         fi         if [[ -z \"${AIRFLOW_UID}\" ]]; then           echo           echo -e \"\\033[1;33mWARNING!!!: AIRFLOW_UID not set!\\e[0m\"           echo \"If you are on Linux, you SHOULD follow the instructions below to set \"           echo \"AIRFLOW_UID environment variable, otherwise files will be owned by root.\"           echo \"For other operating systems you can get rid of the warning with manually created .env file:\"           echo \"    See: https:\/\/airflow.apache.org\/docs\/apache-airflow\/stable\/howto\/docker-compose\/index.html#setting-the-right-airflow-user\"           echo         fi         one_meg=1048576         mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) \/ one_meg))         cpus_available=$$(grep -cE 'cpu[0-9]+' \/proc\/stat)         disk_available=$$(df \/ | tail -1 | awk '{print $$4}')         warning_resources=\"false\"         if (( mem_available < 4000 )) ; then           echo           echo -e \"\\033[1;33mWARNING!!!: Not enough memory available for Docker.\\e[0m\"           echo \"At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))\"           echo           warning_resources=\"true\"         fi         if (( cpus_available < 2 )); then           echo           echo -e \"\\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\\e[0m\"           echo \"At least 2 CPUs recommended. You have $${cpus_available}\"           echo           warning_resources=\"true\"         fi         if (( disk_available < one_meg * 10 )); then           echo           echo -e \"\\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\\e[0m\"           echo \"At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))\"           echo           warning_resources=\"true\"         fi         if [[ $${warning_resources} == \"true\" ]]; then           echo           echo -e \"\\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\\e[0m\"           echo \"Please follow the instructions to increase amount of resources available:\"           echo \"   https:\/\/airflow.apache.org\/docs\/apache-airflow\/stable\/howto\/docker-compose\/index.html#before-you-begin\"           echo         fi         mkdir -p \/sources\/logs \/sources\/dags \/sources\/plugins         chown -R \"${AIRFLOW_UID}:0\" \/sources\/{logs,dags,plugins}         exec \/entrypoint airflow version     environment:       <<: *airflow-common-env       _AIRFLOW_DB_MIGRATE: 'true'       _AIRFLOW_WWW_USER_CREATE: 'true'       _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}       _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-qwerty12345!}       _PIP_ADDITIONAL_REQUIREMENTS: ''     user: \"0:0\"     volumes:       - ${AIRFLOW_PROJ_DIR:-.}:\/sources    airflow-cli:     <<: *airflow-common     profiles:       - debug     environment:       <<: *airflow-common-env       CONNECTION_CHECK_MAX_COUNT: \"0\"     command:       - bash       - -c       - airflow  volumes:   postgres-db-volume:<\/code><\/pre>\n<\/p>\n<\/div>\n<\/details>\n
\u041d\u0435\u0441\u043a\u043e\u043b\u044c\u043a\u043e \u043f\u043e\u043f\u0443\u043b\u044f\u0440\u043d\u044b\u0445 \u043e\u043f\u0435\u0440\u0430\u0442\u043e\u0440\u043e\u0432 \u0432 Apache Airflow:<\/p>\n
EmptyOperator<\/h3>\n
\n
\u042d\u0442\u043e\u0442 \u043e\u043f\u0435\u0440\u0430\u0442\u043e\u0440 \u043f\u0440\u0435\u0434\u0441\u0442\u0430\u0432\u043b\u044f\u0435\u0442 \u0441\u043e\u0431\u043e\u0439 \u043e\u0447\u0435\u043d\u044c \u043f\u0440\u043e\u0441\u0442\u043e\u0439 \u043e\u043f\u0435\u0440\u0430\u0442\u043e\u0440 \u0432\u00a0Apache Airflow, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u043d\u0435\u00a0\u0432\u044b\u043f\u043e\u043b\u043d\u044f\u0435\u0442 \u043d\u0438\u043a\u0430\u043a\u0438\u0445 \u0440\u0435\u0430\u043b\u044c\u043d\u044b\u0445 \u0434\u0435\u0439\u0441\u0442\u0432\u0438\u0439.<\/p>\n<\/blockquote>\n