О том, как я качал музыку с VK

def authorize():     payload = {         'client_id': settings.APP_ID,         'scope': ','.join(settings.APP_PERMISSIONS),         # TODO: сменить нах         'redirect_uri': 'http://127.0.0.1:8000/vkapi/authorize',         'response_type': 'code',         'v': settings.APP_API_VERSION     }     return 'https://oauth.vk.com/authorize?%s' % urllib.urlencode(payload) 

def vk_authorize(request):     return redirect(authorize()) 

def vk_authorize(request):     # подумать как перенести в мидлварь     if request.GET.get('code'):         code = request.GET['code']         r = access_token_get(code)         print r.text         data = r.json()                  if data.get('error'):             raise Http404("Error: %s. Desc: %s" % (data['error'], data['error_description']))                  data['date'] = datetime.now()         request.session['vkapi'] = data         return redirect('main')              elif request.GET.get('error'):         error = request.GET['error']         error_description = request.GET.get('error_description')         raise Http404("Error: %s. Desc: %s" % (error, error_description)) 

def access_token_get(code):     payload = {         'client_id': settings.APP_ID,         'client_secret': settings.APP_SECRET,         'code': code,         'redirect_uri': 'http://127.0.0.1:8000/vkapi/authorize',     }          return requests.get('https://oauth.vk.com/access_token', params=payload) 

def request_api(method, access_token, params):     """     Для того чтобы вызвать метод API ВКонтакте, Вам необходимо осуществить     POST или GET запрос по протоколу HTTPS на указанный URL:     https://api.vk.com/method/'''METHOD_NAME'''?'''PARAMETERS'''&access_token='''ACCESS_TOKEN'''          METHOD_NAME – название метода из списка функций API (http://vk.com/dev/methods),     PARAMETERS – параметры соответствующего метода API,     ACCESS_TOKEN – ключ доступа, полученный в результате успешной авторизации приложения.     """     payload = {         'access_token': access_token,         'v': settings.APP_API_VERSION,     }     payload.update(params)     r = requests.get('https://api.vk.com/method/%s' % method, params=payload)     return r.json().get('response', {})   def audio_get(session):          params = {         'owner_id': session['user_id'],         'count': 6000,     }     return request_api('audio.get', session['access_token'], params) 

@render_to("downloader/vk_audios.html") def vk_audios(request):     audios = []     if request.session.get('vkapi'):         # TODO: мидлварь, которая будет обновлять access_token         audios = audio_get(request.session['vkapi'])              return {         'audios': audios,     } 

 # *  coding: utf8 * from datetime import datetime, timedelta from django.conf import settings from django.shortcuts import redirect from downloader.vkapi import access_token_get   class AccessTokenMiddleware(object):      def process_request(self, request):                  if request.session.get('vkapi'):             data = request.session['vkapi']             expired = data['date']  timedelta(seconds=int(data['expires_in']))             if (expired  datetime.now()).seconds < 3600:                 return redirect('vk_authorize')                  return None 

def authorize_require(view_func):          def check_session(request, *args, **kwargs):         if request.session.get('vkapi'):             data = request.session['vkapi']             expired = data['date']  timedelta(seconds=int(data['expires_in']))             if (expired - datetime.now()).seconds < 3600:                 return redirect('vk_authorize')         else:             return redirect('vk_authorize')         return view_func(request, *args, **kwargs)      return check_session 

Обычно использую такую регистрацию, когда пишу для себя. Быстро и дешево, да и хватает с головой.

В urls.py добавляю строку:

url(r'^registration/$', 'downloader.views.registration', name='registration'), 

views.py пополняется такой вьюхой:

@render_to("registration/registration.html") def registration(request):          form = RegistrationForm()          if request.method == "POST":         form = RegistrationForm(data=request.POST)         if form.is_valid():             user = User(_db=request.db,                         is_active=True,                         is_superuser=False,                         type='user',                         permissions=[],                         groups=[], )             username = form.cleaned_data['username']             password = form.cleaned_data['password']             user.update({"username": username, 'password': make_password(password)})             user.create('u_%s' % slugify(username))                          auth_user = authenticate(username=username, password=password)             login(request, auth_user)             return redirect('main')                  return {         "form": form     } 

Вьюха создает нового пользователя в CouchDB и сразу же его авторизирует, после чего кидает на 1 страницу.

Форма RegistrationForm выглядит вот так:
forms.py

class RegistrationForm(forms.Form):          username = forms.EmailField(label=_('Username'), max_length=30)     password = forms.CharField(widget=forms.PasswordInput(), label=_('Password'))      def clean_username(self):         username = self.cleaned_data['username']         user = django_couch.db().view('auth/auth', key=username).rows         if user:             raise forms.ValidationError(_("Username already in use"))         return username 

registration/registration.html

{% extends "base.html" %} {% load i18n %} {% load bootstrap_toolkit %}  {% block title %}  - {% trans "Registration" %} {% endblock %}  {% block lib %}     <link rel="stylesheet" href="/media/css/authentification.css" /> {% endblock %}   {% block body %} <div class="container">     <div id="register-form">         <form action="" method="post" class="form-horizontal">             <legend>                 <h2>{% trans 'Registration' %}</h2>             </legend>             {{ form|as_bootstrap }}{% csrf_token %}             <div class="form-actions">                 <button class="btn btn-primary" type="submit">{% trans 'Register' %}</button>                   <small>                     <a href="{% url login %}"> {% trans 'Login' %}</a>                 </small>             </div>         </form>     </div> </div> {% endblock %} 

Осенило меня в общем… если у пользователя будет over100500 аудиозаписей, то браузер просто загнется при рендеринге и было решено добавить пагинацию.

Функция audio_get преобразилась примерно до такого вида, что дало возможность сделать пагинацию:

def audio_get(session, album_id='', offset=0, count=100):          params = {         'owner_id': session['user_id'],         'offset': offset,         'count': count,         'album_id': album_id,     }     return request_api('audio.get', session['access_token'], params) 

vk_audios в файле view.py приобрела примерно такой вид:

@authorize_require @render_to("downloader/vk_audios.html") def vk_audios(request, album_id=''):            try:         page = int(request.GET.get('page', 1))     except:         raise Http404("Error page param: %s" % request.GET['page'])          offset = 100 * (int(page) - 1)          response = audio_get(request.session['vkapi'], album_id=album_id, offset=offset)          audios = response.get('items', [])     audios_count = response.get('count')           return {         'album_id': album_id,         'audios_count': audios_count,         'page': page,         'offset': offset,         'audios': audios,      } 

Был добавлен inclusion_tag, который принимал количество аудиозаписей, страницу на которой находится пользователь и id альбома, что бы рендерить страницы.

@register.inclusion_tag('snippets/pagination.html') def render_pagination(audios_count, page, album_id=False):          pages_count = int(math.ceil(audios_count / 100.0))  1          pages = range(1, pages_count)          return {         "pages": pages,         "page": page,         "album_id": album_id,     } 

И добавлен html-файл (snippets/pagination.html):

{% load i18n %}  {% if pages|length > 1 %} <div class="pagination pagination-right">     <ul>     {% for p in pages %}         <li {% ifequal p page %}class="active"{% endifequal %}>             {% if album_id %}             <a href="{% url vk_audios album_id %}?page={{ p }}">{{ p }}</a>             {% else %}             <a href="{% url vk_audios %}?page={{ p }}">{{ p }}</a>             {% endif %}         </li>     {% endfor %}     </ul> </div> {% endif %} 

Итого я ограничил себя скачиванием по 100 файлов. Осталось их скачать.

class RequestsForm(forms.Form):          username = forms.EmailField(label=_('E-mail'),                                 help_text=_("Archive with audios will be send to this email")) 

* _id - r_<hash> * status - new * username - test@test.com * is_active - true * audios - [    {        "url": "<url>",        "processed": true,        "title": "Three Days Grace - I Hate Everything About You"    } ] * date_created - 2013-10-20 11:27:21.208492 * type - request 

class DownloadRequest(django_couch.Document):          def __init__(self, *args, **kwargs):         self._db = django_couch.db('db_requests')         self.type = 'request'         self.is_active = True         self.status = 'new'         self.date_created = datetime.now().isoformat(' ')         super(DownloadRequest, self).__init__(*args, **kwargs)              @staticmethod     def load(resp_id):         db = django_couch.db('db_requests')                  if resp_id in db:             doc = DownloadRequest(db[resp_id])             assert doc.type == 'request', _("Invalid data loaded")                          return doc                      else:             raise Http404(_("Can't find download request with id '%s'") % id)              @staticmethod     def get_list(email):         pass 

def nsq_push(topic, message, fmt='json'):     url = "http://%s/put?topic=%s" % (random.choice(settings.NSQD_HTTP_ADDRESSES), topic)          if fmt == 'json':         message_encoded = json.dumps(message)     elif fmt == 'raw':         message_encoded = message     else:         raise Exception("Unsupported message encode format: %s" % fmt)          r = requests.post(url, data=message_encoded)          return r.ok 

@authorize_require @render_to("downloader/vk_audios.html") def vk_audios(request, album_id=''):          try:         page = int(request.GET.get('page', 1))     except:         page = 1         messages.error(request, _("Error page: %s. Changed to 1") % request.GET.get('page'))          offset = 100 * (int(page) - 1)          response = audio_get(request.session['vkapi'], album_id=album_id, offset=offset)          audios = response.get('items', [])     audios_count = response.get('count')          if request.user.is_authenticated():         initial_data = request.user     else:         initial_data = {'username': ''}          form = RequestsForm(request.POST or None, initial=initial_data)          if form.is_valid():         request_doc = DownloadRequest()         request_doc.update(form.cleaned_data)                  formated_audios = []         for audio in audios:             formated_data = {                 'title': "%s - %s" % (audio['artist'], audio['title']),                 'url': audio['url'],                 'processed': False,             }             formated_audios.append(formated_data)                  request_doc.update({'audios': formated_audios})         request_doc.create('r', force_random_suffix=True)         messages.success(request, _("Download request successfully created"))         nsq_push('download_requests', request_doc.id, fmt="raw")              return {         'album_id': album_id,         'audios_count': audios_count,         'page': page,         'offset': offset,         'audios': audios,         'form': form,     } 

И принялся я писать вьюху для их отображения. Была использована таже форма, что и выше, для отбора по email’у. В CouchDB был создан дизайн док, который строит индекс с ключем [email, дата создания]:

function(doc) {     if (doc.type == 'request' && doc.is_active) {         emit([doc.username, doc.date_created])     } } 

А в аппликацию добавлена вьюха requests_list:

@render_to("downloader/requests.html") def requests_list(request):          requests = []          if request.user.is_authenticated():         initial_data = request.user     else:         initial_data = {'username': ''}          form = RequestsForm(request.GET or None, initial=initial_data)          if form.is_valid():         requests = DownloadRequest.get_list(form.cleaned_data['username'])          return {         'form': form,         'requests': requests,     } 

И дописана функция get_list в модель DownloadRequest:

@staticmethod def get_list(email):     db = django_couch.db('db_requests')          requests = db.view('requests/list', include_docs=True, startkey=[email], endkey=[email, {}]).rows     return [DownloadRequest(request) for request in requests] 

Хэх! Теперь я еще вижу и статусы, осталось написать nsq-обработчик, который собственно будет скачивать…

#!/usr/bin/env python # -*- coding: utf-8 -*- import os import sys import nsq import signal import requests import django_couch  from django.conf import settings from django.core.management.base import BaseCommand from logger import logger  from downloader.models import DownloadRequest           class Command(BaseCommand):          def handle(self, *args, **options):                  self.log = logger('download_request', int(options['verbosity']) > 1, settings.LOG_DIR)                  signal.signal(signal.SIGINT, self.signal_callback)         signal.signal(signal.SIGTERM, self.signal_callback)                  self.db = django_couch.db('db_requests')                  nsq.Reader({"message_callback": self.message_callback},                    "download_requests",                    "download_requests",                    nsqd_tcp_addresses=settings.NSQD_TCP_ADDRESSES)         self.log.debug("Starting NSQ...")         nsq.run()              def process_request(self, request):         self.log.debug("Setting status '%s' to 'processing.'" % request.status)                  request.status = 'processing'         request.save()                  user_path = os.path.join(settings.DOWNLOAD_AUDIOS_DIR, request.username)         if not os.path.exists(user_path):             os.mkdir(user_path)                  self.log.debug("User dir: %s" % user_path)                      request_path = os.path.join(user_path, request.id)         if not os.path.exists(request_path):             os.mkdir(request_path)                  self.log.debug("Download request dir: %s" % request_path)                  for audio in request.audios:             self.log.debug("Title: %s. Url: %s", audio['title'], audio['url'])             if audio.get('processed', False):                 self.log.debug("Already processed")                 continue                          self.log.debug("Downloading file..")             response = requests.get(audio['url'])             self.log.debug("Downloaded")                          filename = os.path.join(request_path, "%s.mp3" % audio['title'])             self.log.debug("Writing to filename: %s" % filename)                          with open(filename, 'wb') as f:                 f.write(response.content)             self.log.debug("Setting audio to processed")             audio['processed'] = True                  request.save()      def message_callback(self, message):         self.log.debug("Processing message id: %s", message.id)                  self.log.debug("Message data: %s", message.body)                  try:             request = DownloadRequest.load(message.body)             self.log.info("Document loaded. Audios count: %s" % len(request.audios))             self.process_request(request)             self.log.debug("Setting status '%s' to 'processed.'" % request.status)             request.status = 'processed'             request.save()             self.log.debug("Request successfullly processed.")         except:             return False         return True      def signal_callback(self, signal_number, stack_frame):         self.log.critical("Signal %d received. Shutting down", signal_number)                  sys.exit(-1) 

#!/usr/bin/env python # -*- coding: utf-8 -*- import os import sys import nsq import json import time import shutil import signal import requests import django_couch import multiprocessing  from logger import logger from datetime import datetime  from django.conf import settings from django.core.mail import send_mail from django.core.urlresolvers import reverse from django.template.loader import render_to_string from django.utils.translation import ugettext_lazy as _ from django.core.management.base import BaseCommand  from downloader.models import DownloadRequest from downloader.views import nsq_push   class DownloadRequestProcess(multiprocessing.Process):          def __init__(self, log, message, *args, **kwargs):         self.log = log         self.message = message         super(DownloadRequestProcess, self).__init__(*args, **kwargs)              def process_request(self, drequest):         self.log.debug("Setting status '%s' to 'processing'. For doc: %s" % (drequest.status, drequest.id))                  drequest.status = 'processing'         drequest.save()                      request_path = os.path.join(settings.DOWNLOAD_AUDIOS_DIR, drequest.id)         if not os.path.exists(request_path):             os.mkdir(request_path)                  self.log.debug("Download request dir: %s" % request_path)                  for audio in drequest.audios:             self.log.debug("Title: %s. Url: %s", audio['title'], audio['url'])             if audio.get('processed', False):                 self.log.debug("Audio already processed")                 continue                          filename = os.path.join(request_path, "%s.mp3" % audio['title'])             self.log.debug("Filename: %s" % filename)                          self.log.debug("Downloading file...")             response = requests.get(audio['url'])             self.log.debug("Downloaded")                          with open(filename, 'wb') as f:                 f.write(response.content)             self.log.debug("Setting audio to processed")             audio['processed'] = True                  archive_path = None         if drequest.get('archive'):             archive_path = os.path.exists(os.path.join(request_path, drequest.archive))                  if not drequest.get('archive') and not archive_path:             self.log.debug("Writing archive")             archive = shutil.make_archive(request_path, 'gztar', settings.DOWNLOAD_AUDIOS_DIR, drequest.id)             self.log.debug("Archive: %s" % archive)             drequest['archive'] = os.path.basename(archive)                  self.log.debug("Deleting download path dir")         shutil.rmtree(request_path)                  self.log.debug("Setting status '%s' to 'processed.'" % drequest.status)         drequest['date_processed'] = datetime.now().isoformat(' ')         drequest.status = 'processed'         drequest.save()              def run(self):         self.log.debug("Message data: %s", self.message.body)                  data = json.loads(self.message.body)                  attempts = data.get('attempts', 1)         if (attempts > 5):             self.log.debug("Attempts limit reached, dropping this request")             return                  drequest = DownloadRequest.load(data['id'])         self.log.info("Document loaded. Audios count: %s" % len(drequest.audios))                  if drequest.get('processed'):             self.log.debug("Download request already processed")             return                  try:             self.process_request(drequest)             self.log.debug("Download request successfullly processed. Sending mail.")                          if drequest.get('archive'):                 archive_link = 'http://%s%s' % (data['host'], reverse('archive_link', args=[drequest.archive]))                 self.log.debug("Link to archive: %s" % archive_link)                 send_mail(_("Take you archive"),                           render_to_string("mail/archive_mail.html", {'archive_link': archive_link}),                           settings.SERVER_EMAIL,                           [drequest.username])                 self.log.debug("Mail sent")         except:             self.log.debug("Error occured: %s. Setting status to error" % sys.exc_info()[1])             drequest.status = 'error'             drequest.status_verbose = "%s" % sys.exc_info()[1]             drequest.save()                          sleep_time = 30 * attempts             self.log.debug("Pushing it back to nsq in %s seconds. Topic: download_requests" % sleep_time)             time.sleep(sleep_time)             nsq_push('download_requests', {"host": data['host'], 'id': drequest.id, 'attempts': attempts + 1})               class Command(BaseCommand):          def handle(self, *args, **options):                  self.log = logger('download_request', int(options['verbosity']) > 1, settings.LOG_DIR)                  signal.signal(signal.SIGINT, self.signal_callback)         signal.signal(signal.SIGTERM, self.signal_callback)                  self.db = django_couch.db('db_requests')                  nsq.Reader({"message_callback": self.message_callback},                    "download_requests",                    "download_requests",                    nsqd_tcp_addresses=settings.NSQD_TCP_ADDRESSES)         self.log.debug("Starting NSQ...")         self.processes = []         nsq.run()          def message_callback(self, message):         self.log.debug("Processing message id: %s", message.id)                  message.enable_async()                  process = DownloadRequestProcess(self.log, message)         process.start()                  self.log.debug("Process: %s", process)         message.finish()                  self.processes.append(process)      def signal_callback(self, signal_number, stack_frame):         self.log.critical("Signal %d received. Shutting down", signal_number)                  for process in self.processes:             if process.is_alive():                 process.join()                 process.terminate()                  sys.exit(-1) 

#!/usr/bin/env python # -*- coding: utf-8 -*- import os import django_couch from datetime import datetime, timedelta  from logger import logger  from django.conf import settings from django.core.management.base import BaseCommand   class Command(BaseCommand):      help = u'prepare and send fax document'          def handle(self, *args, **options):                  self.log = logger('delete_in_24', int(options['verbosity']) > 1, settings.LOG_DIR)         self.db = django_couch.db("db_requests")                  requests = self.db.view("request_processed/list", include_docs=True).rows         self.log.debug("Founded %s processed download requests", len(requests))         for req in requests:             self.log.debug("%s", req.value)             self.log.debug("ID: %s. Archive: %s", req.id, req.value)                          now = datetime.now()             date_expired = datetime.strptime(req.key, settings.DATETIME_FMT) + timedelta(hours=24)             self.log.debug("Now: %s. Expired: %s", now.strftime(settings.DATETIME_FMT), date_expired.strftime(settings.DATETIME_FMT))                          if now < date_expired:                 self.log.debug("Passing this doc")                 continue                           archive_path = os.path.join(settings.DOWNLOAD_AUDIOS_DIR, req.value)             self.log.debug("Archive path: %s", archive_path)                            if os.path.exists(archive_path):                 self.log.debug("Deleting file: %s", archive_path)                 os.unlink(archive_path)             else:                 self.log.warning("Path doesn't exists")                            doc = req.doc             self.log.debug("Settings status '%s' to 'deleted'", doc.status)             doc.status = 'deleted'             doc.save(self.db) 

function(doc) {     if (doc.type == 'request' && doc.archive     && doc.date_processed && doc.status != 'deleted') {         emit(doc.date_processed.slice(0, 19), doc.archive);     } }