{"id":332411,"date":"2022-04-26T03:00:23","date_gmt":"2022-04-26T03:00:23","guid":{"rendered":"http:\/\/savepearlharbor.com\/?p=332411"},"modified":"-0001-11-30T00:00:00","modified_gmt":"-0001-11-29T21:00:00","slug":"","status":"publish","type":"post","link":"https:\/\/savepearlharbor.com\/?p=332411","title":{"rendered":"<span>\u041f\u0430\u0440\u0441\u0438\u043d\u0433 \u0434\u043b\u044f \u0432\u0437\u0440\u043e\u0441\u043b\u044b\u0445 \u0438\u043b\u0438 \u0418\u043d\u0444\u0440\u0430\u0441\u0442\u0440\u0443\u043a\u0442\u0443\u0440\u0430 \u0434\u043b\u044f \u043f\u0440\u043e\u043c\u044b\u0448\u043b\u0435\u043d\u043d\u043e\u0433\u043e \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430<\/span>"},"content":{"rendered":"<div><\/div>\n<div id=\"post-content-body\">\n<div>\n<div class=\"article-formatted-body article-formatted-body article-formatted-body_version-2\">\n<div xmlns=\"http:\/\/www.w3.org\/1999\/xhtml\">\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w780q1\/getpro\/habr\/upload_files\/ff4\/a20\/325\/ff4a20325fd58c17b3fb5236b83fee86.jpg\" alt=\"\u0421\u0442\u0430\u0442\u044c\u044f \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u0439 \u043f\u043e \u0432\u043e\u0437\u0440\u0430\u0441\u0442\u0443\" title=\"\u0421\u0442\u0430\u0442\u044c\u044f \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u0439 \u043f\u043e \u0432\u043e\u0437\u0440\u0430\u0441\u0442\u0443\" width=\"863\" height=\"360\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/ff4\/a20\/325\/ff4a20325fd58c17b3fb5236b83fee86.jpg\" data-blurred=\"true\"\/><figcaption>\u0421\u0442\u0430\u0442\u044c\u044f \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u0439 \u043f\u043e \u0432\u043e\u0437\u0440\u0430\u0441\u0442\u0443<\/figcaption><\/figure>\n<p>\u0426\u0435\u043b\u044c \u0441\u0442\u0430\u0442\u044c\u0438 &#8212; \u043e\u043f\u0438\u0441\u0430\u0442\u044c \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u0435 \u0438\u043d\u0444\u0440\u0430\u0441\u0442\u0440\u0443\u043a\u0442\u0443\u0440\u044b \u0434\u043b\u044f \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430 \u043d\u0430 \u0431\u0430\u0437\u0435 python, Django, Celery \u0438 Docker.<\/p>\n<h2>\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435<\/h2>\n<p>\u0412 \u0441\u0442\u0443\u0434\u0435\u043d\u0447\u0435\u0441\u043a\u0438\u0435 \u0433\u043e\u0434\u044b \u044f \u043d\u0430\u043f\u0438\u0441\u0430\u043b \u043d\u0430 \u0437\u0430\u043a\u0430\u0437 \u043c\u043d\u043e\u0433\u043e \u043f\u0430\u0440\u0441\u0435\u0440\u043e\u0432 \u043c\u0430\u0433\u0430\u0437\u0438\u043d\u043e\u0432 \u0438 \u0441\u043e\u0446\u0438\u0430\u043b\u044c\u043d\u044b\u0445 \u0441\u0435\u0442\u0435\u0439. \u0421\u043e \u0432\u0440\u0435\u043c\u0435\u043d\u0435\u043c \u043f\u0430\u0440\u0441\u0435\u0440\u044b \u0443\u0441\u043b\u043e\u0436\u043d\u044f\u043b\u0438\u0441\u044c \u0438 \u0438\u0437 \u0441\u043a\u0440\u0438\u043f\u0442\u043e\u0432 \u043f\u0440\u0435\u0432\u0440\u0430\u0449\u0430\u043b\u0438\u0441\u044c \u0432 \u043f\u043e\u043b\u043d\u043e\u0446\u0435\u043d\u043d\u044b\u0435 \u0432\u0435\u0431-\u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f c \u0431\u0430\u0437\u043e\u0439 \u0434\u0430\u043d\u043d\u044b\u0445 \u0438 Rest API. \u0412 \u0441\u0442\u0430\u0442\u044c\u0435 \u043e\u043f\u0438\u0441\u0430\u043d \u0448\u0430\u0431\u043b\u043e\u043d \u0432\u0435\u0431-\u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e \u0434\u043b\u044f \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u043f\u0430\u0440\u0441\u0435\u043e\u0432.<\/p>\n<p>\u0418\u0437 \u0441\u0442\u0430\u0442\u044c\u0438 \u043c\u044b \u0443\u0437\u043d\u0430\u0435\u043c<\/p>\n<ol>\n<li>\n<p>\u041a\u0430\u043a \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u0442\u044c Rest API \u043d\u0430 \u0431\u0430\u0437\u0435 Django Rest Framework<\/p>\n<\/li>\n<li>\n<p>\u041a\u0430\u043a \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u0442\u044c \u0430\u0441\u0438\u043d\u0445\u0440\u043e\u043d\u043d\u044b\u0435 \u0437\u0430\u0434\u0430\u0447\u0438 \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e Celery<\/p>\n<\/li>\n<li>\n<p>\u041a\u0430\u043a \u0434\u0435\u043f\u043b\u043e\u0438\u0442\u044c \u0432\u0435\u0431-\u043f\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u0441 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435\u043c Docker-compose<\/p>\n<\/li>\n<\/ol>\n<p><strong>\u0410\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u0430 \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f \u0441\u043e\u0434\u0435\u0440\u0436\u0438\u0442 3 \u043e\u0441\u043d\u043e\u0432\u044b\u0435 \u0447\u0430\u0441\u0442\u0438:<\/strong><\/p>\n<ol>\n<li>\n<p>Django \u0434\u043b\u044f \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0438 HTTP \u0437\u0430\u043f\u0440\u043e\u0441\u043e\u0432 \u0438 \u0445\u0440\u0430\u043d\u0435\u043d\u0438\u044f \u0434\u0430\u043d\u043d\u044b\u0445<\/p>\n<\/li>\n<li>\n<p> Redis &#8212; \u0442\u0440\u0430\u043d\u0441\u043f\u043e\u0440\u0442 (\u0431\u0440\u043e\u043a\u0435\u0440\u0430 \u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u0439)<\/p>\n<\/li>\n<li>\n<p>Celery &#8212; \u0434\u043b\u044f \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u043e\u0447\u0435\u0440\u0435\u0434\u0435\u0439 \u0437\u0430\u0434\u0430\u0447 (\u0437\u0430\u0434\u0430\u0447 \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430)<\/p>\n<\/li>\n<\/ol>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/18b\/652\/e9f\/18b652e9f8cb80b15756d6e768a9b38c.png\" width=\"934\" height=\"145\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/18b\/652\/e9f\/18b652e9f8cb80b15756d6e768a9b38c.png\"\/><figcaption><\/figcaption><\/figure>\n<p><strong>UseCase:<\/strong><\/p>\n<ul>\n<li>\n<p>\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u0434\u0435\u043b\u0430\u0435\u0442 HTTP POST \u0437\u0430\u043f\u0440\u043e\u0441 \/task <\/p>\n<\/li>\n<li>\n<p>\u0415\u0441\u043b\u0438 \u0437\u0430\u043f\u0440\u043e\u0441 \u0441\u043e\u0434\u0435\u0440\u0436\u0438\u0442 \u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u044b\u0435 \u0434\u0430\u043d\u043d\u044b\u0435, \u0442\u043e \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0435\u0442 \u0437\u0430\u0434\u0430\u0447\u0443 \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430<\/p>\n<\/li>\n<li>\n<p>\u0412 \u0431\u0430\u0437\u0443 \u0434\u0430\u043d\u043d\u044b\u0445 \u0441\u043e\u0445\u0440\u0430\u043d\u044f\u044e\u0442\u0441\u044f \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u0437\u0430\u0434\u0430\u0447\u0438<\/p>\n<\/li>\n<\/ul>\n<h2>1 \u041d\u0430\u0447\u043d\u0435\u043c \u0441 \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f Django \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f<\/h2>\n<pre><code class=\"bash\"># Create Django application python3.9 -m venv venv # create virtual environment source venv\/bin\/activate # activate the environment  pip install Django==4.0.0 # install django library mkdir project  cd project django-admin startproject core_app . # create django app with initial settings <\/code><\/pre>\n<p>\u0414\u043b\u044f \u0430\u043a\u043a\u0443\u0440\u0430\u0442\u043d\u043e\u0441\u0442\u0438  \u0441\u043e\u0437\u0434\u0430\u0434\u0438\u043c \u043e\u0442\u0434\u0435\u043b\u044c\u043d\u043e\u0435 \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435\u00a0(\u0434\u0438\u0440\u0435\u043a\u0442\u043e\u0440\u0438\u044e), \u043a\u043e\u0434 \u043a\u043e\u0442\u043e\u0440\u043e\u0433\u043e \u0431\u0443\u0434\u0435\u0442 \u043e\u0442\u0432\u0435\u0447\u0430\u0442\u044c \u0437\u0430 \u043b\u043e\u0433\u0438\u043a\u0443 \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f (\u043f\u0430\u0440\u0441\u0438\u043d\u0433)<\/p>\n<pre><code># create a new app in \/project python manage.py startapp parser_app<\/code><\/pre>\n<p>\u041f\u043e\u0441\u043b\u0435 \u0432\u044b\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u044f \u043a\u043e\u043c\u0430\u043d\u0434 \u0443 \u043d\u0430\u0441 \u0434\u043e\u043b\u0436\u0435\u043d \u043f\u043e\u043b\u0443\u0447\u0438\u0442\u044c\u0441\u044f \u0442\u0430\u043a\u043e\u0439 \u043f\u0440\u043e\u0435\u043a\u0442:<\/p>\n<pre><code>project\/ \u251c\u2500\u2500 core_app \u2502\u00a0\u00a0 \u251c\u2500\u2500 asgi.py \u2502\u00a0\u00a0 \u251c\u2500\u2500 __init__.py \u2502\u00a0\u00a0 \u251c\u2500\u2500 __pycache__ \u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 __init__.cpython-39.pyc \u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 settings.cpython-39.pyc \u2502\u00a0\u00a0 \u251c\u2500\u2500 settings.py \u2502\u00a0\u00a0 \u251c\u2500\u2500 urls.py \u2502\u00a0\u00a0 \u2514\u2500\u2500 wsgi.py \u251c\u2500\u2500 manage.py \u2514\u2500\u2500 parser_app     \u251c\u2500\u2500 admin.py     \u251c\u2500\u2500 apps.py     \u251c\u2500\u2500 __init__.py     \u251c\u2500\u2500 migrations     \u2502\u00a0\u00a0 \u2514\u2500\u2500 __init__.py     \u251c\u2500\u2500 models.py     \u251c\u2500\u2500 tests.py     \u2514\u2500\u2500 views.py<\/code><\/pre>\n<h2>2 \u0417\u0430\u043f\u0443\u0441\u0442\u0438\u043c \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u0432 Docker<\/h2>\n<p><strong>2.1 \u041f\u043e\u0434\u0433\u043e\u0442\u043e\u0432\u043a\u0430 PostgreSQL \u0431\u0430\u0437\u044b \u0434\u0430\u043d\u043d\u044b\u0445<\/strong><\/p>\n<pre><code class=\"bash\">sudo apt-get update sudo apt-get install python3-dev libpq-dev postgresql postgresql-contrib  sudo -u postgres psql CREATE DATABASE parsing_db; CREATE USER postgres WITH PASSWORD 'post222'; # postgres is username GRANT ALL PRIVILEGES ON DATABASE parsing_db TO postgres; \\q <\/code><\/pre>\n<p><strong>2.2 \u041f\u043e\u0434\u0433\u043e\u0442\u043e\u0432\u043a\u0430 docker-compose<\/strong><\/p>\n<p>\u0421\u043e\u0437\u0434\u0430\u0435\u043c .env \u0444\u0430\u0439\u043b \u0441 \u043f\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u044b\u043c\u0438 \u043e\u043a\u0440\u0443\u0436\u0435\u043d\u0438\u044f<\/p>\n<pre><code class=\"bash\"># .env DB_USER=postgres DB_PASSWORD=post222 DB_NAME=parsing_db DB_PORT=5444  DATABASE_URL=postgres:\/\/postgres:post222@db:5432\/parsing_db\" DEBUG=1<\/code><\/pre>\n<p>\u041c\u043e\u0434\u0438\u0444\u0438\u0446\u0438\u0440\u0443\u0435\u043c \u0444\u0430\u0439\u043b \u0441 \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430\u043c\u0438<\/p>\n<pre><code class=\"python\"># core_app\/settings.py  import os import environ  env = environ.Env()  ALLOWED_HOSTS = ['127.0.0.1', '0.0.0.0']  # Application definition INSTALLED_APPS = [     'django.contrib.admin',     'django.contrib.auth',     'django.contrib.contenttypes',     'django.contrib.sessions',     'django.contrib.messages',     'django.contrib.staticfiles',     # packages     'rest_framework',      # my apps     'parser_app', ]   DATABASES = {     'default': {         'ENGINE': 'django.db.backends.postgresql_psycopg2',         'NAME': os.environ['DB_NAME'],         'USER': os.environ['DB_USER'],         'PASSWORD': os.environ['DB_PASSWORD'],         'HOST': 'db',         'PORT': 5432,     } }  STATIC_URL = '\/static\/' STATIC_ROOT = BASE_DIR \/ 'static'<\/code><\/pre>\n<p>\u041e\u043f\u0438\u0441\u044b\u0432\u0430\u0435\u043c \u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u043f\u043e\u0442\u0440\u0435\u0431\u0443\u044e\u0442\u0441\u044f \u0434\u043b\u044f \u0437\u0430\u043f\u0443\u0441\u043a\u0430 \u043f\u0440\u043e\u0435\u043a\u0442<\/p>\n<pre><code># project\/requirements.txt Django==4.0 celery==5.2.6 djangorestframework==3.13.1 redis==3.4.1 django-environ==0.8.1 gunicorn==20.1.0 psycopg2==2.9.3 celery==5.2.6 redis==3.4.1 requests==2.23.0 lxml==4.8.0<\/code><\/pre>\n<p>\u041e\u043f\u0438\u0441\u044b\u0432\u0430\u0435\u043c \u0438\u043d\u0441\u0442\u0440\u0443\u043a\u0446\u0438\u0438 \u0434\u043b\u044f \u0441\u0431\u043e\u0440\u043a\u0438 \u0434\u043e\u043a\u0435\u0440-\u043e\u0431\u0440\u0430\u0437\u0430 \u0432 Dockerfile<\/p>\n<pre><code class=\"bash\"># project\/Dockerfile FROM python:3.9-slim-bullseye  WORKDIR \/project  # forbid .pyc file recording # forbid bufferization ENV PYTHONDONTWRITEBYTECODE=1 \\     PYTHONBUFFERED=1  COPY . .  RUN apt-get update &amp;&amp; apt-get install --no-install-recommends -y \\     gcc libc-dev libpq-dev  python-dev libxml2-dev libxslt1-dev python3-lxml &amp;&amp; apt-get install -y cron &amp;&amp;\\     pip install --no-cache-dir -r requirements.txt<\/code><\/pre>\n<p>\u041a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0438\u0440\u0443\u0435\u043c nginx<\/p>\n<pre><code># project\/nginx-conf.d\/nginx-conf.conf  upstream app {     server django:8000; }  server {     listen 80;     server_name 127.0.0.1;      location \/ {         proxy_pass http:\/\/django:8000;         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;         proxy_set_header Host $host;         proxy_redirect off;     }      location \/static\/ {         alias \/var\/www\/html\/static\/;     } } <\/code><\/pre>\n<p>\u0421\u043e\u0437\u0434\u0430\u0435\u043c docker-compose.yml \u043d\u0430 \u043e\u0434\u043d\u043e\u043c \u0443\u0440\u043e\u0432\u043d\u0435 \u0441 \u043f\u0430\u043f\u043a\u043e\u0439 project\/<\/p>\n<pre><code class=\"yaml\">#  docker-compose.yml  version: '3.9'  services:   django:     build: .\/project # path to Dockerfile     command: sh -c \"       python manage.py makemigrations       &amp;&amp; python manage.py migrate         &amp;&amp; gunicorn --bind 0.0.0.0:8000 core_app.wsgi\"     volumes:       - .\/project:\/project       - .\/project\/static:\/project\/static     expose:       - 8000     env_file:       - .env      db:     image: postgres:13-alpine     volumes:       - pg_data:\/var\/lib\/postgresql\/data\/     expose:        - 5432     env_file:       - .env     environment:       - POSTGRES_USER=${DB_USER}       - POSTGRES_PASSWORD=${DB_PASSWORD}       - POSTGRES_DB=${DB_NAME}      nginx:     image: nginx:1.19.8-alpine     depends_on:        - django     ports:        - \"80:80\"     volumes:       - .\/project\/static:\/var\/www\/html\/static       - .\/project\/nginx-conf.d\/:\/etc\/nginx\/conf.d    volumes:     pg_data:     static: <\/code><\/pre>\n<p>\u0417\u0430\u043f\u0443\u0441\u043a\u0430\u0435\u043c \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435<\/p>\n<pre><code>docker-compose up --build -d<\/code><\/pre>\n<p>\u0414\u0430\u043b\u0435\u0435 \u0437\u0430\u0439\u0434\u0435\u043c \u0432 \u0434\u043e\u043a\u0435\u0440-\u043a\u043e\u043d\u0442\u0435\u0439\u043d\u0435\u0440 \u0438 \u0441\u043e\u0437\u0434\u0430\u0434\u0438\u043c \u0441\u0443\u043f\u0435\u0440\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f<\/p>\n<pre><code class=\"bash\">docker ps # intended to find django container name --> code_django_1 >> 6f5db39cfa3b   code_django                   \"sh -c ' python mana\u2026\"   47 seconds ago   Up 46 seconds                   8000\/tcp                 code_django_1  docker exec -ti code_django_1 bash # go into the container python manage.py createsuperuser # create admin user in django python manage.py collectstatic # intended to load css and js files exit<\/code><\/pre>\n<p>\u041f\u043e\u0441\u043b\u0435 \u044d\u0442\u043e\u0433\u043e \u0437\u0430\u0445\u043e\u0434\u0438\u043c \u043d\u0430 127.0.0.1 \u0414\u043e\u043b\u0436\u0435\u043d \u0431\u044b\u0442\u044c \u0442\u0430\u043a\u043e\u0439 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442:<\/p>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/b54\/69c\/a95\/b5469ca9539cbbe53dd29bee84294c33.png\" width=\"1567\" height=\"800\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/b54\/69c\/a95\/b5469ca9539cbbe53dd29bee84294c33.png\"\/><figcaption><\/figcaption><\/figure>\n<h2>3 \u041f\u043e\u0434\u043a\u043b\u044e\u0447\u0430\u0435\u043c Celery<\/h2>\n<p>\u0414\u043e\u0431\u0430\u0432\u043b\u044f\u0435\u043c \u0432  docker-compose.yml \u0444\u0430\u0439\u043b \u043d\u043e\u0432\u044b\u0435 \u0441\u0435\u0440\u0432\u0438\u0441\u044b<\/p>\n<pre><code class=\"yaml\">celery:     build: .\/project     command: celery -A parser_app worker  --loglevel=info     volumes:       - .\/project:\/usr\/src\/app     env_file:       - .env     environment:     # environment variables declared in the environment section override env_file       - DEBUG=1       - DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1]       - CELERY_BROKER=redis:\/\/redis:6379\/0       - CELERY_BACKEND=redis:\/\/redis:6379\/0     depends_on:       - django       - redis    redis:     image: redis:5-alpine  volumes:     pg_data:     static:<\/code><\/pre>\n<p>\u0414\u043e\u0431\u0430\u0432\u043b\u044f\u0435\u043c \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0438 \u0434\u043b\u044f celery<\/p>\n<pre><code class=\"python\"># settings.py CELERY_BROKER_URL = os.environ.get(\"CELERY_BROKER\", \"redis:\/\/redis:6379\/0\") CELERY_RESULT_BACKEND = os.environ.get(\"CELERY_BROKER\", \"redis:\/\/redis:6379\/0\") CELERY_IMPORTS = (\"parser_app.celery\",)<\/code><\/pre>\n<p>\u0421\u043e\u0437\u0434\u0430\u0435\u043c \u043e\u0431\u044a\u0435\u043a\u0442 \u0434\u043b\u044f \u0440\u0430\u0431\u043e\u0442\u044b \u0441 celery<\/p>\n<pre><code class=\"python\"># parser_app\/celery.py \"\"\" Celery config file  https:\/\/docs.celeryproject.org\/en\/stable\/django\/first-steps-with-django.html  \"\"\" from __future__ import absolute_import import os from celery import Celery  from core_app.settings import INSTALLED_APPS  # this code copied from manage.py # set the default Django settings module for the 'celery' app. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core_app.settings')  # you change the name here app = Celery(\"parser_app\")  # read config from Django settings, the CELERY namespace would make celery  # config keys has `CELERY` prefix app.config_from_object('django.conf:settings', namespace='CELERY')  # load tasks.py in django apps app.autodiscover_tasks(lambda: INSTALLED_APPS) <\/code><\/pre>\n<p>\u0418\u043c\u043f\u043e\u0440\u0442\u0438\u0440\u0443\u0435\u043c celery \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438, \u0447\u0442\u043e\u0431\u044b \u043e\u043d\u043e \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u043b\u043e\u0441\u044c \u0432\u043c\u0435\u0441\u0442\u0435 \u0441 django<\/p>\n<pre><code class=\"python\"># parser_app\/__init__.py from __future__ import absolute_import, unicode_literals  # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app  __all__ = ('celery_app',)<\/code><\/pre>\n<h2>4 \u041f\u0435\u0440\u0435\u0445\u043e\u0434\u0438\u043c \u043a \u043b\u043e\u0433\u0438\u043a\u0435 \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430<\/h2>\n<p>\u041c\u044b \u0431\u0443\u0434\u0435\u043c \u043f\u0430\u0440\u0441\u0438\u0442\u044c \u0441\u0430\u0439\u0442 <a href=\"https:\/\/books.toscrape.com\/\" rel=\"noopener noreferrer nofollow\">https:\/\/books.toscrape.com\/<\/a> \u0442\u0430\u043a \u043a\u0430\u043a \u0441\u0430\u0439\u0442 \u043f\u0440\u0435\u0434\u043d\u0430\u0437\u043d\u0430\u0447\u0435\u043d \u0434\u043b\u044f \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430 \u0438 \u043d\u0435 \u0438\u0437\u043c\u0435\u043d\u0438\u0442\u0441\u044f. \u0412 HTTP POST \u0437\u0430\u043f\u0440\u043e\u0441\u0435 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u0431\u0443\u0434\u0435\u0442 \u043f\u0435\u0440\u0435\u0434\u0430\u0432\u0430\u0442\u044c \u043d\u0430\u0437\u0432\u0430\u043d\u0438\u0435 \u043a\u0430\u0442\u0435\u0433\u043e\u0440\u0438\u0438 (\u043d\u0430\u043f\u0440\u043c\u0435\u0440, mystery_3)<\/p>\n<p>\u041f\u0440\u043e\u0433\u0440\u0430\u043c\u043c\u0430 \u0431\u0443\u0434\u0435\u0442 \u0441\u043e\u0445\u0440\u0430\u043d\u044f\u0442\u044c \u043d\u0430\u0437\u0432\u0430\u043d\u0438\u044f \u043a\u043d\u0438\u0433 \u0438\u0437 \u044d\u0442\u043e\u0439 \u043a\u0430\u0442\u0435\u0433\u043e\u0440\u0438\u0438<\/p>\n<p>\u041d\u0430\u0447\u043d\u0435\u043c \u0441 \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u043c\u043e\u0434\u0435\u043b\u0438 \u0434\u0430\u043d\u043d\u044b\u0445<\/p>\n<pre><code class=\"python\">from django.db import models   class BaseTask(models.Model):     \"\"\" Celery task info\"\"\"     name = models.CharField(max_length=100)     is_success = models.BooleanField(default=False)          created_at = models.DateTimeField(auto_now_add=True)     updated_at = models.DateTimeField(auto_now=True)      def __str__(self):         return self.name   class BaseParsingResult(models.Model):     \"\"\" Parsing result details\"\"\"     task_id = models.ForeignKey(         BaseTask,         blank=True,         null=True,         on_delete=models.PROTECT     )     data = models.TextField(blank=True)     task_type = models.CharField(blank=True, max_length=64) <\/code><\/pre>\n<p>\u0418\u0437\u043c\u0435\u043d\u0435\u043d\u0438\u044f \u0432 urls.py \u0444\u0430\u0439\u043b\u0430\u0445. \u0423 \u043d\u0432\u0441 \u0431\u0443\u0434\u0435\u0442 \u0432\u0441\u0435\u0433\u043e \u043e\u0434\u0438\u043d \u0437\u0430\u043f\u0440\u043e\u0441 task\/ &#8212; \u043f\u043e\u0441\u0442\u0430\u043d\u043e\u0432\u043a\u0430 \u0437\u0430\u0434\u0430\u0447\u0438 \u043d\u0430 \u043f\u0430\u0441\u0438\u043d\u0433 \u0438 \u043c\u043e\u043d\u0438\u0442\u043e\u0440\u043d\u0433 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u043e\u0432<\/p>\n<pre><code class=\"python\"># core_app\/urls.py from django.contrib import admin from django.urls import path, include  urlpatterns = [     path('admin\/', admin.site.urls),     path('', include('parser_app.urls'))  ]<\/code><\/pre>\n<pre><code class=\"python\"># parser_app\/urls.py from django.urls import path from . import views   urlpatterns = [     path('task', views.task, name='task'), ] <\/code><\/pre>\n<p>\u0412 \u0444\u0430\u0439\u043b\u0435 views.py \u043e\u043f\u0438\u0441\u044b\u0432\u0430\u0435\u043c \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a \u0437\u0430\u043f\u0440\u043e\u0441\u0430 task\/<\/p>\n<p>GET &#8212; \u0432\u043e\u0437\u0432\u0440\u0430\u0449\u0430\u0435\u0442 \u0441\u0442\u0430\u0442\u0443\u0441 \u0437\u0430\u0434\u0430\u0447\u0438, POST &#8212; \u0441\u0442\u0430\u0432\u0438\u0442 \u0437\u0430\u0434\u0430\u0447\u0443 \u043d\u0430 \u043f\u0430\u0440\u0441\u0438\u043d\u0433<\/p>\n<pre><code class=\"python\">from django.shortcuts import render from rest_framework.decorators import api_view from rest_framework.response import Response from celery.result import AsyncResult  from parser_app.tasks import create_task   @api_view(['GET', 'POST']) def task(request):     if request.method == 'POST':         if \"type\" in request.data:             category_name = request.data[\"type\"]             task = create_task.delay(category_name) # create celery task             return Response({\"message\": \"Create task\", \"task_id\": task.id, \"data\": request.data})         else:             return Response({\"message\": \"Error, not found 'type' in POST request\"})     if request.method == 'GET': # get task status         if \"task_id\" in request.data:             task_id = request.data[\"task_id\"]             task_result = AsyncResult(task_id)             result = {                 \"task_id\": task_id,                 \"task_status\": task_result.status,                 \"task_result\": task_result.result             }             return Response(result)         else:             return Response({\"message\": \"Error, not found 'task_id' in GET request\"}) <\/code><\/pre>\n<p>\u0421\u043e\u0437\u0434\u0430\u0435\u043c \u0444\u0430\u0439\u043b tasks.py \u0441 \u043e\u043f\u0438\u0441\u0430\u043d\u0438\u0435 \u043b\u043e\u0433\u0438\u043a\u0438 \u0440\u0430\u0431\u043e\u0442\u044b \u0437\u0430\u0434\u0430\u0447\u0438 \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430<\/p>\n<pre><code class=\"python\">import requests import time  from lxml import etree from datetime import datetime from parser_app.models import BaseTask, BaseParsingResult from core_app.celery import app  from django.core.cache import cache   def parse_data(celery_task_id: str, category_name: str):     new_task = BaseTask.objects.create(         name=celery_task_id,     )     new_task.save()     try:         response = requests.get(             f\"https:\/\/books.toscrape.com\/catalogue\/category\/books\/{category_name}\/\"         )         if response.status_code == 200:             tree = etree.HTML(response.content)             results = tree.xpath(\"\/\/article\/h3\/a\")             for cur in results:                 cur_parsing_res = BaseParsingResult.objects.create(                     task_id=new_task,                     data=cur.text,                     task_type=category_name                 )                 cur_parsing_res.save()     except Exception as e:         print(\"Error: \", e)     else:         new_task.is_success = True         new_task.save()   @app.task(name='create_task', bind=True) def create_task(self, category_name):     parse_data(self.request.id, category_name)     return True<\/code><\/pre>\n<p>\u0414\u043e\u0431\u0430\u0432\u043b\u044f\u0435\u043c \u0432 \u0430\u0434\u043c\u0438\u043d\u0443 \u0434\u0430\u043d\u043d\u044b\u0435<\/p>\n<pre><code class=\"python\">from django.contrib import admin  from .models import BaseTask, BaseParsingResult   @admin.register(BaseTask) class BaseTaskAdmin(admin.ModelAdmin):     list_display = ['id', 'name', 'is_success', 'created_at']     readonly_fields = ['created_at']     list_filter = ['is_success']  @admin.register(BaseParsingResult) class BaseResultAdmin(admin.ModelAdmin):     list_display = ['id', 'task_id', 'data', 'task_type']<\/code><\/pre>\n<p>\u041f\u0435\u0440\u0435\u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0435\u043c \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435.<\/p>\n<h2>5 \u0418\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435 \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f<\/h2>\n<p>5.1 \u0417\u0430\u043f\u0440\u043e\u0441 \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430 POST<\/p>\n<pre><code># POST http:\/\/127.0.0.1:80\/task {     \"type\": \"philosophy_7\" }  RESPONSE: {     \"message\": \"Create task\",     \"task_id\": \"062ac81f-dafe-4e2c-95e9-c042936e85f3\",     \"data\": {         \"type\": \"philosophy_7\"     } }<\/code><\/pre>\n<p>5.2 \u041f\u043e\u0441\u043c\u043e\u0442\u0440\u0438\u043c \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442 \u0437\u0430\u0434\u0430\u0447\u0438<\/p>\n<pre><code># GET http:\/\/127.0.0.1:80\/task {     \"task_id\": \"062ac81f-dafe-4e2c-95e9-c042936e85f3\" }  RESPONSE: {     \"task_id\": \"062ac81f-dafe-4e2c-95e9-c042936e85f3\",     \"task_status\": \"SUCCESS\",     \"task_result\": true }<\/code><\/pre>\n<p>5.3 \u0420\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u0440\u0430\u0431\u043e\u0442\u044b \u0432 \u0430\u0434\u043c\u0438\u043d\u043a\u0435<\/p>\n<figure class=\"full-width\"><img loading=\"lazy\" decoding=\"async\" src=\"https:\/\/habrastorage.org\/r\/w1560\/getpro\/habr\/upload_files\/0fc\/b93\/a0c\/0fcb93a0c2b27a5ee9375f2ee6e8f3b6.png\" width=\"1807\" height=\"767\" data-src=\"https:\/\/habrastorage.org\/getpro\/habr\/upload_files\/0fc\/b93\/a0c\/0fcb93a0c2b27a5ee9375f2ee6e8f3b6.png\"\/><figcaption><\/figcaption><\/figure>\n<p>\u041f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u0433\u043e\u0442\u043e\u0432\u043e!<\/p>\n<p>\u0411\u0443\u0434\u0443 \u043f\u0440\u0438\u0437\u043d\u0430\u0442\u0435\u043b\u0435\u043d \u0437\u0430 \u0444\u0438\u0434\u0431\u0435\u043a \ud83d\ude42<\/p>\n<\/p>\n<\/div>\n<\/div>\n<\/div>\n<div class=\"v-portal\" style=\"display:none;\"><\/div>\n<\/div>\n<p> <!----> <!----><br \/> \u0441\u0441\u044b\u043b\u043a\u0430 \u043d\u0430 \u043e\u0440\u0438\u0433\u0438\u043d\u0430\u043b \u0441\u0442\u0430\u0442\u044c\u0438 <a href=\"https:\/\/habr.com\/ru\/post\/662928\/\"> https:\/\/habr.com\/ru\/post\/662928\/<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<div><\/div>\n<div id=\"post-content-body\">\n<div>\n<div class=\"article-formatted-body article-formatted-body article-formatted-body_version-2\">\n<div xmlns=\"http:\/\/www.w3.org\/1999\/xhtml\">\n<figure class=\"full-width\"><figcaption>\u0421\u0442\u0430\u0442\u044c\u044f \u043d\u0435 \u0438\u043c\u0435\u0435\u0442 \u043e\u0433\u0440\u0430\u043d\u0438\u0447\u0435\u043d\u0438\u0439 \u043f\u043e \u0432\u043e\u0437\u0440\u0430\u0441\u0442\u0443<\/figcaption><\/figure>\n<p>\u0426\u0435\u043b\u044c \u0441\u0442\u0430\u0442\u044c\u0438 &#8212; \u043e\u043f\u0438\u0441\u0430\u0442\u044c \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u0435 \u0438\u043d\u0444\u0440\u0430\u0441\u0442\u0440\u0443\u043a\u0442\u0443\u0440\u044b \u0434\u043b\u044f \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430 \u043d\u0430 \u0431\u0430\u0437\u0435 python, Django, Celery \u0438 Docker.<\/p>\n<h2>\u0412\u0432\u0435\u0434\u0435\u043d\u0438\u0435<\/h2>\n<p>\u0412 \u0441\u0442\u0443\u0434\u0435\u043d\u0447\u0435\u0441\u043a\u0438\u0435 \u0433\u043e\u0434\u044b \u044f \u043d\u0430\u043f\u0438\u0441\u0430\u043b \u043d\u0430 \u0437\u0430\u043a\u0430\u0437 \u043c\u043d\u043e\u0433\u043e \u043f\u0430\u0440\u0441\u0435\u0440\u043e\u0432 \u043c\u0430\u0433\u0430\u0437\u0438\u043d\u043e\u0432 \u0438 \u0441\u043e\u0446\u0438\u0430\u043b\u044c\u043d\u044b\u0445 \u0441\u0435\u0442\u0435\u0439. \u0421\u043e \u0432\u0440\u0435\u043c\u0435\u043d\u0435\u043c \u043f\u0430\u0440\u0441\u0435\u0440\u044b \u0443\u0441\u043b\u043e\u0436\u043d\u044f\u043b\u0438\u0441\u044c \u0438 \u0438\u0437 \u0441\u043a\u0440\u0438\u043f\u0442\u043e\u0432 \u043f\u0440\u0435\u0432\u0440\u0430\u0449\u0430\u043b\u0438\u0441\u044c \u0432 \u043f\u043e\u043b\u043d\u043e\u0446\u0435\u043d\u043d\u044b\u0435 \u0432\u0435\u0431-\u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f c \u0431\u0430\u0437\u043e\u0439 \u0434\u0430\u043d\u043d\u044b\u0445 \u0438 Rest API. \u0412 \u0441\u0442\u0430\u0442\u044c\u0435 \u043e\u043f\u0438\u0441\u0430\u043d \u0448\u0430\u0431\u043b\u043e\u043d \u0432\u0435\u0431-\u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f, \u043a\u043e\u0442\u043e\u0440\u044b\u0439 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u0443\u044e \u0434\u043b\u044f \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u043f\u0430\u0440\u0441\u0435\u043e\u0432.<\/p>\n<p>\u0418\u0437 \u0441\u0442\u0430\u0442\u044c\u0438 \u043c\u044b \u0443\u0437\u043d\u0430\u0435\u043c<\/p>\n<ol>\n<li>\n<p>\u041a\u0430\u043a \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u0442\u044c Rest API \u043d\u0430 \u0431\u0430\u0437\u0435 Django Rest Framework<\/p>\n<\/li>\n<li>\n<p>\u041a\u0430\u043a \u0441\u043e\u0437\u0434\u0430\u0432\u0430\u0442\u044c \u0430\u0441\u0438\u043d\u0445\u0440\u043e\u043d\u043d\u044b\u0435 \u0437\u0430\u0434\u0430\u0447\u0438 \u0441 \u043f\u043e\u043c\u043e\u0449\u044c\u044e Celery<\/p>\n<\/li>\n<li>\n<p>\u041a\u0430\u043a \u0434\u0435\u043f\u043b\u043e\u0438\u0442\u044c \u0432\u0435\u0431-\u043f\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u0441 \u0438\u0441\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u043d\u0438\u0435\u043c Docker-compose<\/p>\n<\/li>\n<\/ol>\n<p><strong>\u0410\u0440\u0445\u0438\u0442\u0435\u043a\u0442\u0443\u0440\u0430 \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f \u0441\u043e\u0434\u0435\u0440\u0436\u0438\u0442 3 \u043e\u0441\u043d\u043e\u0432\u044b\u0435 \u0447\u0430\u0441\u0442\u0438:<\/strong><\/p>\n<ol>\n<li>\n<p>Django \u0434\u043b\u044f \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u043a\u0438 HTTP \u0437\u0430\u043f\u0440\u043e\u0441\u043e\u0432 \u0438 \u0445\u0440\u0430\u043d\u0435\u043d\u0438\u044f \u0434\u0430\u043d\u043d\u044b\u0445<\/p>\n<\/li>\n<li>\n<p> Redis &#8212; \u0442\u0440\u0430\u043d\u0441\u043f\u043e\u0440\u0442 (\u0431\u0440\u043e\u043a\u0435\u0440\u0430 \u0441\u043e\u043e\u0431\u0449\u0435\u043d\u0438\u0439)<\/p>\n<\/li>\n<li>\n<p>Celery &#8212; \u0434\u043b\u044f \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u043e\u0447\u0435\u0440\u0435\u0434\u0435\u0439 \u0437\u0430\u0434\u0430\u0447 (\u0437\u0430\u0434\u0430\u0447 \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430)<\/p>\n<\/li>\n<\/ol>\n<figure class=\"full-width\"><figcaption><\/figcaption><\/figure>\n<p><strong>UseCase:<\/strong><\/p>\n<ul>\n<li>\n<p>\u041f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u0434\u0435\u043b\u0430\u0435\u0442 HTTP POST \u0437\u0430\u043f\u0440\u043e\u0441 \/task <\/p>\n<\/li>\n<li>\n<p>\u0415\u0441\u043b\u0438 \u0437\u0430\u043f\u0440\u043e\u0441 \u0441\u043e\u0434\u0435\u0440\u0436\u0438\u0442 \u043f\u0440\u0430\u0432\u0438\u043b\u044c\u043d\u044b\u0435 \u0434\u0430\u043d\u043d\u044b\u0435, \u0442\u043e \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u0435\u0442 \u0437\u0430\u0434\u0430\u0447\u0443 \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430<\/p>\n<\/li>\n<li>\n<p>\u0412 \u0431\u0430\u0437\u0443 \u0434\u0430\u043d\u043d\u044b\u0445 \u0441\u043e\u0445\u0440\u0430\u043d\u044f\u044e\u0442\u0441\u044f \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u044b \u0437\u0430\u0434\u0430\u0447\u0438<\/p>\n<\/li>\n<\/ul>\n<h2>1 \u041d\u0430\u0447\u043d\u0435\u043c \u0441 \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f Django \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f<\/h2>\n<pre><code class=\"bash\"># Create Django application python3.9 -m venv venv # create virtual environment source venv\/bin\/activate # activate the environment  pip install Django==4.0.0 # install django library mkdir project  cd project django-admin startproject core_app . # create django app with initial settings <\/code><\/pre>\n<p>\u0414\u043b\u044f \u0430\u043a\u043a\u0443\u0440\u0430\u0442\u043d\u043e\u0441\u0442\u0438  \u0441\u043e\u0437\u0434\u0430\u0434\u0438\u043c \u043e\u0442\u0434\u0435\u043b\u044c\u043d\u043e\u0435 \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435\u00a0(\u0434\u0438\u0440\u0435\u043a\u0442\u043e\u0440\u0438\u044e), \u043a\u043e\u0434 \u043a\u043e\u0442\u043e\u0440\u043e\u0433\u043e \u0431\u0443\u0434\u0435\u0442 \u043e\u0442\u0432\u0435\u0447\u0430\u0442\u044c \u0437\u0430 \u043b\u043e\u0433\u0438\u043a\u0443 \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u044f (\u043f\u0430\u0440\u0441\u0438\u043d\u0433)<\/p>\n<pre><code># create a new app in \/project python manage.py startapp parser_app<\/code><\/pre>\n<p>\u041f\u043e\u0441\u043b\u0435 \u0432\u044b\u043f\u043e\u043b\u043d\u0435\u043d\u0438\u044f \u043a\u043e\u043c\u0430\u043d\u0434 \u0443 \u043d\u0430\u0441 \u0434\u043e\u043b\u0436\u0435\u043d \u043f\u043e\u043b\u0443\u0447\u0438\u0442\u044c\u0441\u044f \u0442\u0430\u043a\u043e\u0439 \u043f\u0440\u043e\u0435\u043a\u0442:<\/p>\n<pre><code>project\/ \u251c\u2500\u2500 core_app \u2502\u00a0\u00a0 \u251c\u2500\u2500 asgi.py \u2502\u00a0\u00a0 \u251c\u2500\u2500 __init__.py \u2502\u00a0\u00a0 \u251c\u2500\u2500 __pycache__ \u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u251c\u2500\u2500 __init__.cpython-39.pyc \u2502\u00a0\u00a0 \u2502\u00a0\u00a0 \u2514\u2500\u2500 settings.cpython-39.pyc \u2502\u00a0\u00a0 \u251c\u2500\u2500 settings.py \u2502\u00a0\u00a0 \u251c\u2500\u2500 urls.py \u2502\u00a0\u00a0 \u2514\u2500\u2500 wsgi.py \u251c\u2500\u2500 manage.py \u2514\u2500\u2500 parser_app     \u251c\u2500\u2500 admin.py     \u251c\u2500\u2500 apps.py     \u251c\u2500\u2500 __init__.py     \u251c\u2500\u2500 migrations     \u2502\u00a0\u00a0 \u2514\u2500\u2500 __init__.py     \u251c\u2500\u2500 models.py     \u251c\u2500\u2500 tests.py     \u2514\u2500\u2500 views.py<\/code><\/pre>\n<h2>2 \u0417\u0430\u043f\u0443\u0441\u0442\u0438\u043c \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435 \u0432 Docker<\/h2>\n<p><strong>2.1 \u041f\u043e\u0434\u0433\u043e\u0442\u043e\u0432\u043a\u0430 PostgreSQL \u0431\u0430\u0437\u044b \u0434\u0430\u043d\u043d\u044b\u0445<\/strong><\/p>\n<pre><code class=\"bash\">sudo apt-get update sudo apt-get install python3-dev libpq-dev postgresql postgresql-contrib  sudo -u postgres psql CREATE DATABASE parsing_db; CREATE USER postgres WITH PASSWORD 'post222'; # postgres is username GRANT ALL PRIVILEGES ON DATABASE parsing_db TO postgres; \\q <\/code><\/pre>\n<p><strong>2.2 \u041f\u043e\u0434\u0433\u043e\u0442\u043e\u0432\u043a\u0430 docker-compose<\/strong><\/p>\n<p>\u0421\u043e\u0437\u0434\u0430\u0435\u043c .env \u0444\u0430\u0439\u043b \u0441 \u043f\u0435\u0440\u0435\u043c\u0435\u043d\u043d\u044b\u043c\u0438 \u043e\u043a\u0440\u0443\u0436\u0435\u043d\u0438\u044f<\/p>\n<pre><code class=\"bash\"># .env DB_USER=postgres DB_PASSWORD=post222 DB_NAME=parsing_db DB_PORT=5444  DATABASE_URL=postgres:\/\/postgres:post222@db:5432\/parsing_db\" DEBUG=1<\/code><\/pre>\n<p>\u041c\u043e\u0434\u0438\u0444\u0438\u0446\u0438\u0440\u0443\u0435\u043c \u0444\u0430\u0439\u043b \u0441 \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0430\u043c\u0438<\/p>\n<pre><code class=\"python\"># core_app\/settings.py  import os import environ  env = environ.Env()  ALLOWED_HOSTS = ['127.0.0.1', '0.0.0.0']  # Application definition INSTALLED_APPS = [     'django.contrib.admin',     'django.contrib.auth',     'django.contrib.contenttypes',     'django.contrib.sessions',     'django.contrib.messages',     'django.contrib.staticfiles',     # packages     'rest_framework',      # my apps     'parser_app', ]   DATABASES = {     'default': {         'ENGINE': 'django.db.backends.postgresql_psycopg2',         'NAME': os.environ['DB_NAME'],         'USER': os.environ['DB_USER'],         'PASSWORD': os.environ['DB_PASSWORD'],         'HOST': 'db',         'PORT': 5432,     } }  STATIC_URL = '\/static\/' STATIC_ROOT = BASE_DIR \/ 'static'<\/code><\/pre>\n<p>\u041e\u043f\u0438\u0441\u044b\u0432\u0430\u0435\u043c \u0431\u0438\u0431\u043b\u0438\u043e\u0442\u0435\u043a\u0438, \u043a\u043e\u0442\u043e\u0440\u044b\u0435 \u043f\u043e\u0442\u0440\u0435\u0431\u0443\u044e\u0442\u0441\u044f \u0434\u043b\u044f \u0437\u0430\u043f\u0443\u0441\u043a\u0430 \u043f\u0440\u043e\u0435\u043a\u0442<\/p>\n<pre><code># project\/requirements.txt Django==4.0 celery==5.2.6 djangorestframework==3.13.1 redis==3.4.1 django-environ==0.8.1 gunicorn==20.1.0 psycopg2==2.9.3 celery==5.2.6 redis==3.4.1 requests==2.23.0 lxml==4.8.0<\/code><\/pre>\n<p>\u041e\u043f\u0438\u0441\u044b\u0432\u0430\u0435\u043c \u0438\u043d\u0441\u0442\u0440\u0443\u043a\u0446\u0438\u0438 \u0434\u043b\u044f \u0441\u0431\u043e\u0440\u043a\u0438 \u0434\u043e\u043a\u0435\u0440-\u043e\u0431\u0440\u0430\u0437\u0430 \u0432 Dockerfile<\/p>\n<pre><code class=\"bash\"># project\/Dockerfile FROM python:3.9-slim-bullseye  WORKDIR \/project  # forbid .pyc file recording # forbid bufferization ENV PYTHONDONTWRITEBYTECODE=1 \\     PYTHONBUFFERED=1  COPY . .  RUN apt-get update &amp;&amp; apt-get install --no-install-recommends -y \\     gcc libc-dev libpq-dev  python-dev libxml2-dev libxslt1-dev python3-lxml &amp;&amp; apt-get install -y cron &amp;&amp;\\     pip install --no-cache-dir -r requirements.txt<\/code><\/pre>\n<p>\u041a\u043e\u043d\u0444\u0438\u0433\u0443\u0440\u0438\u0440\u0443\u0435\u043c nginx<\/p>\n<pre><code># project\/nginx-conf.d\/nginx-conf.conf  upstream app {     server django:8000; }  server {     listen 80;     server_name 127.0.0.1;      location \/ {         proxy_pass http:\/\/django:8000;         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;         proxy_set_header Host $host;         proxy_redirect off;     }      location \/static\/ {         alias \/var\/www\/html\/static\/;     } } <\/code><\/pre>\n<p>\u0421\u043e\u0437\u0434\u0430\u0435\u043c docker-compose.yml \u043d\u0430 \u043e\u0434\u043d\u043e\u043c \u0443\u0440\u043e\u0432\u043d\u0435 \u0441 \u043f\u0430\u043f\u043a\u043e\u0439 project\/<\/p>\n<pre><code class=\"yaml\">#  docker-compose.yml  version: '3.9'  services:   django:     build: .\/project # path to Dockerfile     command: sh -c \"       python manage.py makemigrations       &amp;&amp; python manage.py migrate         &amp;&amp; gunicorn --bind 0.0.0.0:8000 core_app.wsgi\"     volumes:       - .\/project:\/project       - .\/project\/static:\/project\/static     expose:       - 8000     env_file:       - .env      db:     image: postgres:13-alpine     volumes:       - pg_data:\/var\/lib\/postgresql\/data\/     expose:        - 5432     env_file:       - .env     environment:       - POSTGRES_USER=${DB_USER}       - POSTGRES_PASSWORD=${DB_PASSWORD}       - POSTGRES_DB=${DB_NAME}      nginx:     image: nginx:1.19.8-alpine     depends_on:        - django     ports:        - \"80:80\"     volumes:       - .\/project\/static:\/var\/www\/html\/static       - .\/project\/nginx-conf.d\/:\/etc\/nginx\/conf.d    volumes:     pg_data:     static: <\/code><\/pre>\n<p>\u0417\u0430\u043f\u0443\u0441\u043a\u0430\u0435\u043c \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438\u0435<\/p>\n<pre><code>docker-compose up --build -d<\/code><\/pre>\n<p>\u0414\u0430\u043b\u0435\u0435 \u0437\u0430\u0439\u0434\u0435\u043c \u0432 \u0434\u043e\u043a\u0435\u0440-\u043a\u043e\u043d\u0442\u0435\u0439\u043d\u0435\u0440 \u0438 \u0441\u043e\u0437\u0434\u0430\u0434\u0438\u043c \u0441\u0443\u043f\u0435\u0440\u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044f<\/p>\n<pre><code class=\"bash\">docker ps # intended to find django container name --> code_django_1 >> 6f5db39cfa3b   code_django                   \"sh -c ' python mana\u2026\"   47 seconds ago   Up 46 seconds                   8000\/tcp                 code_django_1  docker exec -ti code_django_1 bash # go into the container python manage.py createsuperuser # create admin user in django python manage.py collectstatic # intended to load css and js files exit<\/code><\/pre>\n<p>\u041f\u043e\u0441\u043b\u0435 \u044d\u0442\u043e\u0433\u043e \u0437\u0430\u0445\u043e\u0434\u0438\u043c \u043d\u0430 127.0.0.1 \u0414\u043e\u043b\u0436\u0435\u043d \u0431\u044b\u0442\u044c \u0442\u0430\u043a\u043e\u0439 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442:<\/p>\n<figure class=\"full-width\"><figcaption><\/figcaption><\/figure>\n<h2>3 \u041f\u043e\u0434\u043a\u043b\u044e\u0447\u0430\u0435\u043c Celery<\/h2>\n<p>\u0414\u043e\u0431\u0430\u0432\u043b\u044f\u0435\u043c \u0432  docker-compose.yml \u0444\u0430\u0439\u043b \u043d\u043e\u0432\u044b\u0435 \u0441\u0435\u0440\u0432\u0438\u0441\u044b<\/p>\n<pre><code class=\"yaml\">celery:     build: .\/project     command: celery -A parser_app worker  --loglevel=info     volumes:       - .\/project:\/usr\/src\/app     env_file:       - .env     environment:     # environment variables declared in the environment section override env_file       - DEBUG=1       - DJANGO_ALLOWED_HOSTS=localhost 127.0.0.1 [::1]       - CELERY_BROKER=redis:\/\/redis:6379\/0       - CELERY_BACKEND=redis:\/\/redis:6379\/0     depends_on:       - django       - redis    redis:     image: redis:5-alpine  volumes:     pg_data:     static:<\/code><\/pre>\n<p>\u0414\u043e\u0431\u0430\u0432\u043b\u044f\u0435\u043c \u043d\u0430\u0441\u0442\u0440\u043e\u0439\u043a\u0438 \u0434\u043b\u044f celery<\/p>\n<pre><code class=\"python\"># settings.py CELERY_BROKER_URL = os.environ.get(\"CELERY_BROKER\", \"redis:\/\/redis:6379\/0\") CELERY_RESULT_BACKEND = os.environ.get(\"CELERY_BROKER\", \"redis:\/\/redis:6379\/0\") CELERY_IMPORTS = (\"parser_app.celery\",)<\/code><\/pre>\n<p>\u0421\u043e\u0437\u0434\u0430\u0435\u043c \u043e\u0431\u044a\u0435\u043a\u0442 \u0434\u043b\u044f \u0440\u0430\u0431\u043e\u0442\u044b \u0441 celery<\/p>\n<pre><code class=\"python\"># parser_app\/celery.py \"\"\" Celery config file  https:\/\/docs.celeryproject.org\/en\/stable\/django\/first-steps-with-django.html  \"\"\" from __future__ import absolute_import import os from celery import Celery  from core_app.settings import INSTALLED_APPS  # this code copied from manage.py # set the default Django settings module for the 'celery' app. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core_app.settings')  # you change the name here app = Celery(\"parser_app\")  # read config from Django settings, the CELERY namespace would make celery  # config keys has `CELERY` prefix app.config_from_object('django.conf:settings', namespace='CELERY')  # load tasks.py in django apps app.autodiscover_tasks(lambda: INSTALLED_APPS) <\/code><\/pre>\n<p>\u0418\u043c\u043f\u043e\u0440\u0442\u0438\u0440\u0443\u0435\u043c celery \u043f\u0440\u0438\u043b\u043e\u0436\u0435\u043d\u0438, \u0447\u0442\u043e\u0431\u044b \u043e\u043d\u043e \u0437\u0430\u043f\u0443\u0441\u043a\u0430\u043b\u043e\u0441\u044c \u0432\u043c\u0435\u0441\u0442\u0435 \u0441 django<\/p>\n<pre><code class=\"python\"># parser_app\/__init__.py from __future__ import absolute_import, unicode_literals  # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app  __all__ = ('celery_app',)<\/code><\/pre>\n<h2>4 \u041f\u0435\u0440\u0435\u0445\u043e\u0434\u0438\u043c \u043a \u043b\u043e\u0433\u0438\u043a\u0435 \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430<\/h2>\n<p>\u041c\u044b \u0431\u0443\u0434\u0435\u043c \u043f\u0430\u0440\u0441\u0438\u0442\u044c \u0441\u0430\u0439\u0442 <a href=\"https:\/\/books.toscrape.com\/\" rel=\"noopener noreferrer nofollow\">https:\/\/books.toscrape.com\/<\/a> \u0442\u0430\u043a \u043a\u0430\u043a \u0441\u0430\u0439\u0442 \u043f\u0440\u0435\u0434\u043d\u0430\u0437\u043d\u0430\u0447\u0435\u043d \u0434\u043b\u044f \u043f\u0430\u0440\u0441\u0438\u043d\u0433\u0430 \u0438 \u043d\u0435 \u0438\u0437\u043c\u0435\u043d\u0438\u0442\u0441\u044f. \u0412 HTTP POST \u0437\u0430\u043f\u0440\u043e\u0441\u0435 \u043f\u043e\u043b\u044c\u0437\u043e\u0432\u0430\u0442\u0435\u043b\u044c \u0431\u0443\u0434\u0435\u0442 \u043f\u0435\u0440\u0435\u0434\u0430\u0432\u0430\u0442\u044c \u043d\u0430\u0437\u0432\u0430\u043d\u0438\u0435 \u043a\u0430\u0442\u0435\u0433\u043e\u0440\u0438\u0438 (\u043d\u0430\u043f\u0440\u043c\u0435\u0440, mystery_3)<\/p>\n<p>\u041f\u0440\u043e\u0433\u0440\u0430\u043c\u043c\u0430 \u0431\u0443\u0434\u0435\u0442 \u0441\u043e\u0445\u0440\u0430\u043d\u044f\u0442\u044c \u043d\u0430\u0437\u0432\u0430\u043d\u0438\u044f \u043a\u043d\u0438\u0433 \u0438\u0437 \u044d\u0442\u043e\u0439 \u043a\u0430\u0442\u0435\u0433\u043e\u0440\u0438\u0438<\/p>\n<p>\u041d\u0430\u0447\u043d\u0435\u043c \u0441 \u0441\u043e\u0437\u0434\u0430\u043d\u0438\u044f \u043c\u043e\u0434\u0435\u043b\u0438 \u0434\u0430\u043d\u043d\u044b\u0445<\/p>\n<pre><code class=\"python\">from django.db import models   class BaseTask(models.Model):     \"\"\" Celery task info\"\"\"     name = models.CharField(max_length=100)     is_success = models.BooleanField(default=False)          created_at = models.DateTimeField(auto_now_add=True)     updated_at = models.DateTimeField(auto_now=True)      def __str__(self):         return self.name   class BaseParsingResult(models.Model):     \"\"\" Parsing result details\"\"\"     task_id = models.ForeignKey(         BaseTask,         blank=True,         null=True,         on_delete=models.PROTECT     )     data = models.TextField(blank=True)     task_type = models.CharField(blank=True, max_length=64) <\/code><\/pre>\n<p>\u0418\u0437\u043c\u0435\u043d\u0435\u043d\u0438\u044f \u0432 urls.py \u0444\u0430\u0439\u043b\u0430\u0445. \u0423 \u043d\u0432\u0441 \u0431\u0443\u0434\u0435\u0442 \u0432\u0441\u0435\u0433\u043e \u043e\u0434\u0438\u043d \u0437\u0430\u043f\u0440\u043e\u0441 task\/ &#8212; \u043f\u043e\u0441\u0442\u0430\u043d\u043e\u0432\u043a\u0430 \u0437\u0430\u0434\u0430\u0447\u0438 \u043d\u0430 \u043f\u0430\u0441\u0438\u043d\u0433 \u0438 \u043c\u043e\u043d\u0438\u0442\u043e\u0440\u043d\u0433 \u0440\u0435\u0437\u0443\u043b\u044c\u0442\u0430\u0442\u043e\u0432<\/p>\n<pre><code class=\"python\"># core_app\/urls.py from django.contrib import admin from django.urls import path, include  urlpatterns = [     path('admin\/', admin.site.urls),     path('', include('parser_app.urls'))  ]<\/code><\/pre>\n<pre><code class=\"python\"># parser_app\/urls.py from django.urls import path from . import views   urlpatterns = [     path('task', views.task, name='task'), ] <\/code><\/pre>\n<p>\u0412 \u0444\u0430\u0439\u043b\u0435 views.py \u043e\u043f\u0438\u0441\u044b\u0432\u0430\u0435\u043c \u043e\u0431\u0440\u0430\u0431\u043e\u0442\u0447\u0438\u043a \u0437\u0430\u043f\u0440\u043e\u0441\u0430 task\/<\/p>\n<p>GET &#8212; \u0432\u043e\u0437\u0432\u0440\u0430\u0449\u0430\u0435\u0442 \u0441\u0442\u0430\u0442\u0443\u0441 \u0437\u0430\u0434\u0430\u0447\u0438, POST &#8212; \u0441\u0442\u0430\u0432\u0438\u0442 \u0437\u0430\u0434\u0430\u0447\u0443 \u043d\u0430 \u043f\u0430\u0440\u0441\u0438\u043d\u0433<\/p>\n<pre><code class=\"python\">from django.shortcuts import render from rest_framework.decorators import api_view from rest_framework.response import Response from celery.result import AsyncResult  from parser_app.tasks import create_task   @api_view(['GET', 'POST']) def task(request):     if request.method == 'POST':         if \"type\" in request.data:             category_name = request.data[\"type\"]             task = create_task.delay(category_name) # create celery task             return Response({\"message\": \"Create task\", \"task_id\": task.id, \"data\": request.data})         else:             return Response({\"message\": \"Error,<\/code><\/pre>\n<\/div>\n<\/div>\n<\/div>\n<\/div>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[],"tags":[],"class_list":["post-332411","post","type-post","status-publish","format-standard","hentry"],"_links":{"self":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts\/332411","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=332411"}],"version-history":[{"count":0,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=\/wp\/v2\/posts\/332411\/revisions"}],"wp:attachment":[{"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=332411"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=332411"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/savepearlharbor.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=332411"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}