From f4524961cda009aad7332684abffd2b43d6c7840 Mon Sep 17 00:00:00 2001 From: Oliver Falk Date: Thu, 12 Jul 2018 15:00:59 +0200 Subject: [PATCH] Feature 'Import libravatar export (mails + photos)' --- ivatar/context_processors.py | 2 +- ivatar/ivataraccount/forms.py | 88 +++++++++++++++---- ivatar/ivataraccount/models.py | 28 +++++- .../ivataraccount/read_libravatar_export.py | 64 ++++++++++++++ .../templates/upload_libravatar_export.html | 20 +++++ ivatar/ivataraccount/urls.py | 3 +- ivatar/ivataraccount/views.py | 22 ++++- 7 files changed, 206 insertions(+), 21 deletions(-) create mode 100644 ivatar/ivataraccount/read_libravatar_export.py create mode 100644 ivatar/ivataraccount/templates/upload_libravatar_export.html diff --git a/ivatar/context_processors.py b/ivatar/context_processors.py index bd7573d..c4c7518 100644 --- a/ivatar/context_processors.py +++ b/ivatar/context_processors.py @@ -30,5 +30,5 @@ def basepage(request): unconfirmed = request.user.unconfirmedemail_set.count() if unconfirmed >= MAX_NUM_UNCONFIRMED_EMAILS: context['max_emails'] = True - + return context diff --git a/ivatar/ivataraccount/forms.py b/ivatar/ivataraccount/forms.py index 2093c8c..ad82553 100644 --- a/ivatar/ivataraccount/forms.py +++ b/ivatar/ivataraccount/forms.py @@ -2,21 +2,26 @@ Classes for our ivatar.ivataraccount.forms ''' from urllib.parse import urlsplit, urlunsplit +from io import BytesIO from django import forms from django.utils.translation import ugettext_lazy as _ from django.urls import reverse from django.template.loader import render_to_string from django.core.mail import send_mail +from django.contrib import messages from ipware import get_client_ip from ivatar import settings from ivatar.settings import MIN_LENGTH_EMAIL, MAX_LENGTH_EMAIL from ivatar.settings import MIN_LENGTH_URL, MAX_LENGTH_URL +from ivatar.settings import JPEG_QUALITY from . models import UnconfirmedEmail, ConfirmedEmail, Photo from . models import UnconfirmedOpenId, ConfirmedOpenId from . models import UserPreference +from . models import pil_format, file_format +from . read_libravatar_export import read_gzdata as libravatar_read_gzdata MAX_NUM_UNCONFIRMED_EMAILS_DEFAULT = 5 @@ -76,22 +81,7 @@ class AddEmailForm(forms.Form): unconfirmed.email = self.cleaned_data['email'] unconfirmed.user = user unconfirmed.save() - - link = request.build_absolute_uri('/')[:-1] + \ - reverse( - 'confirm_email', - kwargs={'verification_key': unconfirmed.verification_key}) - email_subject = _('Confirm your email address on %s') % \ - settings.SITE_NAME - email_body = render_to_string('email_confirmation.txt', { - 'verification_link': link, - 'site_name': settings.SITE_NAME, - }) - # if settings.DEBUG: - # print('DEBUG: %s' % link) - send_mail( - email_subject, email_body, settings.SERVER_EMAIL, - [unconfirmed.email]) + unconfirmed.send_confirmation_mail(url=request.build_absolute_uri('/')[:-1]) return True @@ -182,6 +172,7 @@ class AddOpenIDForm(forms.Form): return unconfirmed.pk + class UpdatePreferenceForm(forms.ModelForm): ''' Form for updating user preferences @@ -193,3 +184,68 @@ class UpdatePreferenceForm(forms.ModelForm): ''' model = UserPreference fields = ['theme'] + + +class UploadLibravatarExportForm(forms.Form): + ''' + Form handling libravatar user export upload + ''' + export_file = forms.FileField( + label=_('Export file'), + error_messages={'required': _('You must choose an export file to upload.')}) + not_porn = forms.BooleanField( + label=_('suitable for all ages (i.e. no offensive content)'), + required=True, + error_messages={ + 'required': + _('We only host "G-rated" images and so this field must\ + be checked.') + }) + can_distribute = forms.BooleanField( + label=_('can be freely copied'), + required=True, + error_messages={ + 'required': + _('This field must be checked since we need to be able to\ + distribute photos to third parties.') + }) + + @staticmethod + def save(request, data): + ''' + Save the models and assign it to the current user + ''' + items = libravatar_read_gzdata(data.read()) + for email in items['emails']: + if not ConfirmedEmail.objects.filter(email=email) and \ + not UnconfirmedEmail.objects.filter(email=email): # pylint: disable=no-member + try: + unconfirmed = UnconfirmedEmail.objects.create( # pylint: disable=no-member + user=request.user, + email=email + ) + unconfirmed.save() + unconfirmed.send_confirmation_mail(url=request.build_absolute_uri('/')[:-1]) + except Exception as e: # pylint: disable=broad-except,invalid-name + # Debugging only + print('Exception while trying to import mail addresses: %s' % e) + if 'openids' in items: + messages.warning( + request, + _('You have OpenIDs in your export file, but we cannot\ + import those at the moment')) + + for pilobj in items['photos']: + # Is there a reasonable way to check if that photo already exists!? + try: + data = BytesIO() + pilobj.save(data, pilobj.format, quality=JPEG_QUALITY) + data.seek(0) + photo = Photo() + photo.user = request.user + photo.ip_address = get_client_ip(request)[0] + photo.format = file_format(pilobj.format) + photo.data = data.read() + photo.save() + except Exception: # pylint: disable=broad-except + pass diff --git a/ivatar/ivataraccount/models.py b/ivatar/ivataraccount/models.py index 5563551..c39af65 100644 --- a/ivatar/ivataraccount/models.py +++ b/ivatar/ivataraccount/models.py @@ -17,9 +17,11 @@ from django.contrib import messages from django.db import models from django.utils import timezone from django.http import HttpResponseRedirect -from django.urls import reverse_lazy +from django.urls import reverse_lazy, reverse from django.utils.translation import ugettext_lazy as _ from django.core.exceptions import ObjectDoesNotExist +from django.core.mail import send_mail +from django.template.loader import render_to_string from openid.association import Association as OIDAssociation from openid.store import nonce as oidnonce from openid.store.interface import OpenIDStore @@ -29,6 +31,7 @@ from libravatar import libravatar_url from ivatar.settings import MAX_LENGTH_EMAIL, logger from ivatar.settings import MAX_PIXELS, AVATAR_MAX_SIZE, JPEG_QUALITY from ivatar.settings import MAX_LENGTH_URL +from ivatar.settings import SECURE_BASE_URL, SITE_NAME, SERVER_EMAIL from .gravatar import get_photo as get_gravatar_photo @@ -257,7 +260,7 @@ class Photo(BaseAccountModel): return HttpResponseRedirect(reverse_lazy('profile')) def __str__(self): - return '%s (%i) from %s' % (self.format, self.pk, self.user) + return '%s (%i) from %s' % (self.format, self.pk or 0, self.user) # pylint: disable=too-few-public-methods @@ -361,6 +364,27 @@ class UnconfirmedEmail(BaseAccountModel): using, update_fields) + def send_confirmation_mail(self, url=SECURE_BASE_URL): + ''' + Send confirmation mail to that mail address + ''' + link = url + \ + reverse( + 'confirm_email', + kwargs={'verification_key': self.verification_key}) + email_subject = _('Confirm your email address on %s') % \ + SITE_NAME + email_body = render_to_string('email_confirmation.txt', { + 'verification_link': link, + 'site_name': SITE_NAME, + }) + # if settings.DEBUG: + # print('DEBUG: %s' % link) + send_mail( + email_subject, email_body, SERVER_EMAIL, + [self.email]) + return True + def __str__(self): return '%s (%i) from %s' % (self.email, self.pk, self.user) diff --git a/ivatar/ivataraccount/read_libravatar_export.py b/ivatar/ivataraccount/read_libravatar_export.py new file mode 100644 index 0000000..775bb5f --- /dev/null +++ b/ivatar/ivataraccount/read_libravatar_export.py @@ -0,0 +1,64 @@ +''' +Reading libravatar export +''' + +from io import BytesIO +import gzip +import xml.etree.ElementTree +import base64 +from PIL import Image + +SCHEMAROOT = 'https://www.libravatar.org/schemas/export/0.2' + +def read_gzdata(gzdata=None): + ''' + Read gzipped data file + ''' + emails = [] # pylint: disable=invalid-name + openids = [] # pylint: disable=invalid-name + photos = [] # pylint: disable=invalid-name + + if not gzdata: + return False + + fh = gzip.open(BytesIO(gzdata), 'rb') # pylint: disable=invalid-name + content = fh.read() + fh.close() + root = xml.etree.ElementTree.fromstring(content) + if not root.tag == '{%s}user' % SCHEMAROOT: + print('Unknown export format: %s' % root.tag) + exit(-1) + + # Emails + for email in root.findall('{%s}emails' % SCHEMAROOT)[0]: + if email.tag == '{%s}email' % SCHEMAROOT: + emails.append(email.text) + + # OpenIDs + for openid in root.findall('{%s}openids' % SCHEMAROOT)[0]: + if openid.tag == '{%s}openid' % SCHEMAROOT: + openids.append(openid.text) + + # Photos + for photo in root.findall('{%s}photos' % SCHEMAROOT)[0]: + if photo.tag == '{%s}photo' % SCHEMAROOT: + try: + data = base64.decodebytes(bytes(photo.text, 'utf-8')) + except Exception as e: # pylint: disable=broad-except,invalid-name + print('Cannot decode photo; Encoding: %s, Format: %s: %s' % ( + photo.attrib['encoding'], photo.attrib['format'], e)) + continue + try: + img = Image.open(BytesIO(data)) + except Exception as e: # pylint: disable=broad-except,invalid-name + print('Cannot decode photo; Encoding: %s, Format: %s: %s' % ( + photo.attrib['encoding'], photo.attrib['format'], e)) + continue + + photos.append(img) + + return { + 'emails': emails, + 'openids': openids, + 'photos': photos, + } diff --git a/ivatar/ivataraccount/templates/upload_libravatar_export.html b/ivatar/ivataraccount/templates/upload_libravatar_export.html new file mode 100644 index 0000000..ddd2d2b --- /dev/null +++ b/ivatar/ivataraccount/templates/upload_libravatar_export.html @@ -0,0 +1,20 @@ +{% extends 'base.html' %} +{% load i18n %} +{% load static %} +{% load bootstrap4 %} + +{% block title %}{% trans 'Upload an export from libravatar' %} - ivatar{% endblock title %} + +{% block content %} +

{% trans 'Upload an export from libravatar' %}

+ +
+
{% csrf_token %} + {% bootstrap_form form %} + {% buttons %} + + + {% endbuttons %} +
+
+{% endblock content %} diff --git a/ivatar/ivataraccount/urls.py b/ivatar/ivataraccount/urls.py index 9c14baa..cd6c901 100644 --- a/ivatar/ivataraccount/urls.py +++ b/ivatar/ivataraccount/urls.py @@ -16,7 +16,7 @@ from . views import ImportPhotoView, RawImageView, DeletePhotoView from . views import UploadPhotoView, AssignPhotoOpenIDView from . views import AddOpenIDView, RedirectOpenIDView, ConfirmOpenIDView from . views import CropPhotoView -from . views import UserPreferenceView +from . views import UserPreferenceView, UploadLibravatarExportView # Define URL patterns, self documenting # To see the fancy, colorful evaluation of these use: @@ -84,4 +84,5 @@ urlpatterns = [ # pylint: disable=invalid-name url(r'raw_image/(?P\d+)', RawImageView.as_view(), name='raw_image'), url(r'crop_photo/(?P\d+)', CropPhotoView.as_view(), name='crop_photo'), url(r'pref/$', UserPreferenceView.as_view(), name='user_preference'), + url(r'upload_export/$', UploadLibravatarExportView.as_view(), name='upload_export'), ] diff --git a/ivatar/ivataraccount/views.py b/ivatar/ivataraccount/views.py index a19aafa..9b94dec 100644 --- a/ivatar/ivataraccount/views.py +++ b/ivatar/ivataraccount/views.py @@ -5,6 +5,7 @@ import io from urllib.request import urlopen from django.contrib.auth.decorators import login_required +from django.contrib.auth.models import User from django.utils.decorators import method_decorator from django.contrib.messages.views import SuccessMessageMixin from django.contrib import messages @@ -31,7 +32,7 @@ from .gravatar import get_photo as get_gravatar_photo from ivatar.settings import MAX_NUM_PHOTOS, MAX_PHOTO_SIZE from .forms import AddEmailForm, UploadPhotoForm, AddOpenIDForm -from .forms import UpdatePreferenceForm +from .forms import UpdatePreferenceForm, UploadLibravatarExportForm from .models import UnconfirmedEmail, ConfirmedEmail, Photo from .models import UnconfirmedOpenId, ConfirmedOpenId, DjangoOpenIDStore from .models import UserPreference @@ -690,6 +691,7 @@ class CropPhotoView(TemplateView): return photo.perform_crop(request, dimensions, email, openid) + @method_decorator(login_required, name='dispatch') # pylint: disable=too-many-ancestors class UserPreferenceView(FormView, UpdateView): ''' @@ -702,3 +704,21 @@ class UserPreferenceView(FormView, UpdateView): def get_object(self, queryset=None): return self.request.user.userpreference + + +@method_decorator(login_required, name='dispatch') +class UploadLibravatarExportView(SuccessMessageMixin, FormView): + ''' + View class responsible for libravatar user data export upload + ''' + template_name = 'upload_libravatar_export.html' + form_class = UploadLibravatarExportForm + success_message = _('Successfully uploaded') + success_url = reverse_lazy('profile') + model = User + + def form_valid(self, form): + data = self.request.FILES['export_file'] + + form.save(self.request, data) + return super().form_valid(form)