From e02e064998ffc986a5b6970f16c94d4abd3088a3 Mon Sep 17 00:00:00 2001 From: Oliver Falk Date: Fri, 23 Nov 2018 13:14:33 +0100 Subject: [PATCH 1/5] Add script, able run export all data from libravatar --- exportaccounts.py | 129 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100755 exportaccounts.py diff --git a/exportaccounts.py b/exportaccounts.py new file mode 100755 index 0000000..3a2ac6f --- /dev/null +++ b/exportaccounts.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (C) 2018 Oliver Falk +# +# This file is part of Libravatar +# +# Libravatar is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Libravatar is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with Libravatar. If not, see . + +import base64 +import gzip +import json +import os +import sys +from xml.sax import saxutils +import hashlib + +# pylint: disable=relative-import +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "libravatar.settings") +import libravatar.settings as settings +from libravatar.utils import create_logger, is_hex +import django +django.setup() +from django.contrib.auth.models import User + +os.umask(022) +LOGGER = create_logger('exportaccount') + +SCHEMA_ROOT = 'https://www.libravatar.org/schemas/export/0.2' +SCHEMA_XSD = '%s/export.xsd' % SCHEMA_ROOT + + +def xml_header(): + return ''' +\n''' % (SCHEMA_ROOT, SCHEMA_XSD, SCHEMA_ROOT) + + +def xml_footer(): + return '\n' + + +def xml_account(username): + escaped_username = saxutils.quoteattr(username) + escaped_site_url = saxutils.quoteattr(settings.SITE_URL) + return ' \n' % (escaped_username, escaped_site_url) + + +def xml_list(list_type, list_elements): + s = ' <%ss>\n' % list_type + for element in list_elements: + element = element.encode('utf-8') + s += ' <%s>%s\n' % (list_type, saxutils.escape(element), list_type) + s += ' \n' % list_type + return s + + +def xml_photos(photos): + s = ' \n' + for photo in photos: + (photo_filename, photo_format) = photo + encoded_photo = encode_photo(photo_filename, photo_format) + if encoded_photo: + s += ''' +%s + \n''' % (saxutils.quoteattr(photo_format), encoded_photo) + s += ' \n' + return s + + +def encode_photo(photo_filename, photo_format): + filename = settings.USER_FILES_ROOT + photo_filename + '.' + photo_format + if not os.path.isfile(filename): + LOGGER.warning('Photo not found: %s', filename) + return None + + photo_content = None + with open(filename) as photo: + photo_content = photo.read() + + if not photo_content: + LOGGER.warning('Could not read photo: %s', filename) + return None + + return base64.b64encode(photo_content) + + +def main(argv=None): + if argv is None: + argv = sys.argv + + for user in User.objects.all(): + hash_object = hashlib.new('sha256') + hash_object.update(user.username + user.password) + file_hash = hash_object.hexdigest() + emails = list(user.confirmed_emails.values_list('email', flat=True)) + openids = list(user.confirmed_openids.values_list('openid', flat=True)) + photos = [] + for photo in user.photos.all(): + photo_details = (photo.filename, photo.format) + photos.append(photo_details) + username = user.username + + dest_filename = settings.EXPORT_FILES_ROOT + file_hash + '.xml.gz' + destination = gzip.open(dest_filename, 'w') + destination.write(xml_header()) + destination.write(xml_account(username)) + destination.write(xml_list('email', emails)) + destination.write(xml_list('openid', openids)) + destination.write(xml_photos(photos)) + destination.write(xml_footer()) + destination.close() + print(dest_filename) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From f4249b312c92da2a2fddda6dcf1e30927513c7a3 Mon Sep 17 00:00:00 2001 From: Oliver Falk Date: Tue, 8 Jan 2019 10:08:25 +0100 Subject: [PATCH 2/5] Export relations for merge request !97 --- exportaccounts.py | 64 +++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/exportaccounts.py b/exportaccounts.py index 3a2ac6f..ff94afc 100755 --- a/exportaccounts.py +++ b/exportaccounts.py @@ -56,25 +56,30 @@ def xml_account(username): escaped_site_url = saxutils.quoteattr(settings.SITE_URL) return ' \n' % (escaped_username, escaped_site_url) +def xml_email(emails): + returnstring = " \n" + for email in emails: + returnstring += ' ' + email.email.encode('utf-8') + '' + "\n" + returnstring += " \n" + return returnstring -def xml_list(list_type, list_elements): - s = ' <%ss>\n' % list_type - for element in list_elements: - element = element.encode('utf-8') - s += ' <%s>%s\n' % (list_type, saxutils.escape(element), list_type) - s += ' \n' % list_type - return s +def xml_openid(openids): + returnstring = " \n" + for openid in openids: + returnstring += ' ' + openid.openid.encode('utf-8') + '' + "\n" + returnstring += " \n" + return returnstring def xml_photos(photos): s = ' \n' for photo in photos: - (photo_filename, photo_format) = photo + (photo_filename, photo_format, id) = photo encoded_photo = encode_photo(photo_filename, photo_format) if encoded_photo: - s += ''' + s += ''' %s - \n''' % (saxutils.quoteattr(photo_format), encoded_photo) + \n''' % (id, saxutils.quoteattr(photo_format), encoded_photo) s += ' \n' return s @@ -96,32 +101,43 @@ def encode_photo(photo_filename, photo_format): return base64.b64encode(photo_content) -def main(argv=None): +def main(argv=None, dryrun=False): if argv is None: argv = sys.argv - for user in User.objects.all(): + if(len(sys.argv) > 1): + userobjs = User.objects.filter(username=sys.argv[1]) + else: + userobjs = User.objects.all() + if(len(sys.argv) > 2): + dryrun = True + + for user in userobjs: hash_object = hashlib.new('sha256') hash_object.update(user.username + user.password) file_hash = hash_object.hexdigest() - emails = list(user.confirmed_emails.values_list('email', flat=True)) - openids = list(user.confirmed_openids.values_list('openid', flat=True)) photos = [] for photo in user.photos.all(): - photo_details = (photo.filename, photo.format) + photo_details = (photo.filename, photo.format, photo.id) photos.append(photo_details) username = user.username dest_filename = settings.EXPORT_FILES_ROOT + file_hash + '.xml.gz' - destination = gzip.open(dest_filename, 'w') - destination.write(xml_header()) - destination.write(xml_account(username)) - destination.write(xml_list('email', emails)) - destination.write(xml_list('openid', openids)) - destination.write(xml_photos(photos)) - destination.write(xml_footer()) - destination.close() - print(dest_filename) + data = '' + data += xml_header() + data += xml_account(username) + data += xml_email(user.confirmed_emails.all()) + data += xml_openid(user.confirmed_openids.all()) + data += xml_photos(photos) + data += xml_footer() + + if not dryrun: + destination = gzip.open(dest_filename, 'w') + destination.write(data) + destination.close() + print(dest_filename) + else: + print(data) return 0 From 90635d64a7c53352ea48923791aea4a81f1a3033 Mon Sep 17 00:00:00 2001 From: Oliver Falk Date: Tue, 8 Jan 2019 10:17:30 +0100 Subject: [PATCH 3/5] Remove dryrun, since saving the data to a string, will be problematic with ascii and sh*t --- exportaccounts.py | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/exportaccounts.py b/exportaccounts.py index ff94afc..82e36e9 100755 --- a/exportaccounts.py +++ b/exportaccounts.py @@ -101,7 +101,7 @@ def encode_photo(photo_filename, photo_format): return base64.b64encode(photo_content) -def main(argv=None, dryrun=False): +def main(argv=None): if argv is None: argv = sys.argv @@ -109,8 +109,6 @@ def main(argv=None, dryrun=False): userobjs = User.objects.filter(username=sys.argv[1]) else: userobjs = User.objects.all() - if(len(sys.argv) > 2): - dryrun = True for user in userobjs: hash_object = hashlib.new('sha256') @@ -123,21 +121,16 @@ def main(argv=None, dryrun=False): username = user.username dest_filename = settings.EXPORT_FILES_ROOT + file_hash + '.xml.gz' - data = '' - data += xml_header() - data += xml_account(username) - data += xml_email(user.confirmed_emails.all()) - data += xml_openid(user.confirmed_openids.all()) - data += xml_photos(photos) - data += xml_footer() - if not dryrun: - destination = gzip.open(dest_filename, 'w') - destination.write(data) - destination.close() - print(dest_filename) - else: - print(data) + destination = gzip.open(dest_filename, 'w') + destination.write(xml_header()) + destination.write(xml_account(username)) + destination.write(xml_email(user.confirmed_emails.all())) + destination.write(xml_openid(user.confirmed_openids.all())) + destination.write(xml_photos(photos)) + destination.write(xml_footer()) + destination.close() + print(dest_filename) return 0 From 9cbe84e0051934b31bf7c894666c4bcd4454a607 Mon Sep 17 00:00:00 2001 From: Oliver Falk Date: Tue, 8 Jan 2019 12:29:51 +0100 Subject: [PATCH 4/5] Also export password - not sure if this will always work... --- exportaccounts.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/exportaccounts.py b/exportaccounts.py index 82e36e9..b921b99 100755 --- a/exportaccounts.py +++ b/exportaccounts.py @@ -51,10 +51,11 @@ def xml_footer(): return '\n' -def xml_account(username): +def xml_account(username, password): escaped_username = saxutils.quoteattr(username) escaped_site_url = saxutils.quoteattr(settings.SITE_URL) - return ' \n' % (escaped_username, escaped_site_url) + escaped_password = saxutils.quoteattr(password) + return ' \n' % (escaped_username, escaped_password, escaped_site_url) def xml_email(emails): returnstring = " \n" @@ -124,7 +125,7 @@ def main(argv=None): destination = gzip.open(dest_filename, 'w') destination.write(xml_header()) - destination.write(xml_account(username)) + destination.write(xml_account(username, user.password)) destination.write(xml_email(user.confirmed_emails.all())) destination.write(xml_openid(user.confirmed_openids.all())) destination.write(xml_photos(photos)) From 58a7738fe754aa8055d6848c43763840cf079b35 Mon Sep 17 00:00:00 2001 From: Oliver Falk Date: Thu, 21 Feb 2019 10:17:42 +0100 Subject: [PATCH 5/5] Move exportaccounts to attic - only worked with old libravatar version on Python 2 --- exportaccounts.py | 139 ---------------------------------------------- 1 file changed, 139 deletions(-) delete mode 100755 exportaccounts.py diff --git a/exportaccounts.py b/exportaccounts.py deleted file mode 100755 index b921b99..0000000 --- a/exportaccounts.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (C) 2018 Oliver Falk -# -# This file is part of Libravatar -# -# Libravatar is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Libravatar is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with Libravatar. If not, see . - -import base64 -import gzip -import json -import os -import sys -from xml.sax import saxutils -import hashlib - -# pylint: disable=relative-import -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "libravatar.settings") -import libravatar.settings as settings -from libravatar.utils import create_logger, is_hex -import django -django.setup() -from django.contrib.auth.models import User - -os.umask(022) -LOGGER = create_logger('exportaccount') - -SCHEMA_ROOT = 'https://www.libravatar.org/schemas/export/0.2' -SCHEMA_XSD = '%s/export.xsd' % SCHEMA_ROOT - - -def xml_header(): - return ''' -\n''' % (SCHEMA_ROOT, SCHEMA_XSD, SCHEMA_ROOT) - - -def xml_footer(): - return '\n' - - -def xml_account(username, password): - escaped_username = saxutils.quoteattr(username) - escaped_site_url = saxutils.quoteattr(settings.SITE_URL) - escaped_password = saxutils.quoteattr(password) - return ' \n' % (escaped_username, escaped_password, escaped_site_url) - -def xml_email(emails): - returnstring = " \n" - for email in emails: - returnstring += ' ' + email.email.encode('utf-8') + '' + "\n" - returnstring += " \n" - return returnstring - -def xml_openid(openids): - returnstring = " \n" - for openid in openids: - returnstring += ' ' + openid.openid.encode('utf-8') + '' + "\n" - returnstring += " \n" - return returnstring - - -def xml_photos(photos): - s = ' \n' - for photo in photos: - (photo_filename, photo_format, id) = photo - encoded_photo = encode_photo(photo_filename, photo_format) - if encoded_photo: - s += ''' -%s - \n''' % (id, saxutils.quoteattr(photo_format), encoded_photo) - s += ' \n' - return s - - -def encode_photo(photo_filename, photo_format): - filename = settings.USER_FILES_ROOT + photo_filename + '.' + photo_format - if not os.path.isfile(filename): - LOGGER.warning('Photo not found: %s', filename) - return None - - photo_content = None - with open(filename) as photo: - photo_content = photo.read() - - if not photo_content: - LOGGER.warning('Could not read photo: %s', filename) - return None - - return base64.b64encode(photo_content) - - -def main(argv=None): - if argv is None: - argv = sys.argv - - if(len(sys.argv) > 1): - userobjs = User.objects.filter(username=sys.argv[1]) - else: - userobjs = User.objects.all() - - for user in userobjs: - hash_object = hashlib.new('sha256') - hash_object.update(user.username + user.password) - file_hash = hash_object.hexdigest() - photos = [] - for photo in user.photos.all(): - photo_details = (photo.filename, photo.format, photo.id) - photos.append(photo_details) - username = user.username - - dest_filename = settings.EXPORT_FILES_ROOT + file_hash + '.xml.gz' - - destination = gzip.open(dest_filename, 'w') - destination.write(xml_header()) - destination.write(xml_account(username, user.password)) - destination.write(xml_email(user.confirmed_emails.all())) - destination.write(xml_openid(user.confirmed_openids.all())) - destination.write(xml_photos(photos)) - destination.write(xml_footer()) - destination.close() - print(dest_filename) - return 0 - - -if __name__ == "__main__": - sys.exit(main())