Add new script to import a CSV file with users

fix: validation for trusted urls

See merge request oliver/ivatar!207

.gitlab-ci.yml

@@ -14,6 +14,7 @@ before_script:
 
 test_and_coverage:
   stage: test
+  coverage: '/^TOTAL.*\s+(\d+\%)$/'
   script:
     - echo 'from ivatar.settings import TEMPLATES' > config_local.py
     - echo 'TEMPLATES[0]["OPTIONS"]["debug"] = True' >> config_local.py

.pre-commit-config.yaml

@@ -4,16 +4,16 @@ repos:
     hooks:
     -   id: check-useless-excludes
 -   repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.4.0
+    rev: v2.6.2
     hooks:
     -   id: prettier
         files: \.(css|js|md|markdown|json)
 -   repo: https://github.com/python/black
-    rev: 21.9b0
+    rev: 22.3.0
     hooks:
     -   id: black
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.2.0
     hooks:
     -   id: check-added-large-files
     -   id: check-ast
@@ -57,11 +57,11 @@ repos:
         types:
         - shell
 -   repo: https://github.com/asottile/blacken-docs
-    rev: v1.11.0
+    rev: v1.12.1
     hooks:
     -   id: blacken-docs
 -   repo: https://github.com/hcodes/yaspeller.git
-    rev: v7.0.0
+    rev: v8.0.1
     hooks:
     -   id: yaspeller
 

INSTALL.md

@@ -1,6 +1,6 @@
 # Installation
 
-## Prequisits
+## Prerequisites
 
 Python 3.x + virtualenv
 
@@ -10,6 +10,13 @@ Python 3.x + virtualenv
 yum install python34-virtualenv.noarch
 ```
 
+### Debian 11
+
+```
+sudo apt-get update
+sudo apt-get install git python3-virtualenv libmariadb-dev libldap2-dev libsasl2-dev
+```
+
 ## Checkout
 
 ~~~~bash
@@ -22,8 +29,8 @@ cd ivatar
 ~~~~bash
 virtualenv -p python3 .virtualenv 
 source .virtualenv/bin/activate
-pip install -r requirements.txt
 pip install pillow
+pip install -r requirements.txt
 ~~~~
 
 ## (SQL) Migrations

config.py

@@ -211,19 +211,72 @@ CACHE_RESPONSE = True
 
 # Trusted URLs for default redirection
 TRUSTED_DEFAULT_URLS = [
-    "https://ui-avatars.com/api/",
-    "http://gravatar.com/avatar/",
-    "https://gravatar.com/avatar/",
-    "http://www.gravatar.org/avatar/",
-    "https://www.gravatar.org/avatar/",
-    "https://secure.gravatar.com/avatar/",
-    "http://0.gravatar.com/avatar/",
-    "https://0.gravatar.com/avatar/",
-    "https://avatars.dicebear.com/api/",
-    "https://badges.fedoraproject.org/static/img/",
-    "http://www.planet-libre.org/themes/planetlibre/images/",
-    "https://www.azuracast.com/img/",
-    "https://reps.mozilla.org/static/base/img/remo/",
+    {
+        "schemes": [
+            "https"
+        ],
+        "host_equals": "ui-avatars.com",
+        "path_prefix": "/api/"
+    },
+    {
+        "schemes": [
+            "http",
+            "https"
+        ],
+        "host_equals": "gravatar.com",
+        "path_prefix": "/avatar/"
+    },
+    {
+        "schemes": [
+            "http",
+            "https"
+        ],
+        "host_suffix": ".gravatar.com",
+        "path_prefix": "/avatar/"
+    },
+    {
+        "schemes": [
+            "http",
+            "https"
+        ],
+        "host_equals": "www.gravatar.org",
+        "path_prefix": "/avatar/"
+    },
+    {
+        "schemes": [
+            "https"
+        ],
+        "host_equals": "avatars.dicebear.com",
+        "path_prefix": "/api/"
+    },
+    {
+        "schemes": [
+            "https"
+        ],
+        "host_equals": "badges.fedoraproject.org",
+        "path_prefix": "/static/img/"
+    },
+    {
+        "schemes": [
+            "http",
+        ],
+        "host_equals": "www.planet-libre.org",
+        "path_prefix": "/themes/planetlibre/images/"
+    },
+    {
+        "schemes": [
+            "https"
+        ],
+        "host_equals": "www.azuracast.com",
+        "path_prefix": "/img/"
+    },
+    {
+        "schemes": [
+            "https"
+        ],
+        "host_equals": "reps.mozilla.org",
+        "path_prefix": "/static/base/img/remo/"
+    }
 ]
 
 # This MUST BE THE LAST!

gpg-pubkey

@@ -1 +0,0 @@
-1B4A3476CB99010178CEAB5C00C0EF248E1F4575

import_csv.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Import a CSV - Format as follows:
+    <mailaddr>,<path_to_image>
+
+Example:
+    myuser@mydomain.tld,myphoto.jpeg
+
+This will create or update an existing user and assign the image
+to the given address.
+"""
+
+import os
+from os.path import isfile
+import sys
+from io import BytesIO
+import csv
+import django
+
+os.environ.setdefault(
+    "DJANGO_SETTINGS_MODULE", "ivatar.settings"
+)  # pylint: disable=wrong-import-position
+django.setup()  # pylint: disable=wrong-import-position
+from django.contrib.auth.models import User
+from PIL import Image
+from ivatar.settings import JPEG_QUALITY
+from ivatar.ivataraccount.models import ConfirmedEmail
+from ivatar.ivataraccount.models import Photo
+from ivatar.ivataraccount.models import file_format
+
+if len(sys.argv) < 2:
+    print("First argument to '%s' must be the path to the CSV" % sys.argv[0])
+    exit(-255)
+
+if not isfile(sys.argv[1]):
+    print("First argument to '%s' must be a path to the CSV" % sys.argv[0])
+    exit(-255)
+
+PATH = sys.argv[1]
+with open(PATH, newline="") as csvfile:
+    contactreader = csv.reader(csvfile, delimiter=",")
+    for row in contactreader:
+        mailaddr = row[0]
+        image = row[1]
+
+        if not isfile(image):
+            print("File '%s' doesn't exist - cannot add" % image)
+            continue
+
+        print("Adding: %s" % mailaddr)
+
+        (user, created) = User.objects.get_or_create(username=mailaddr)
+        if not user.confirmedemail_set.count() < 1:
+            ConfirmedEmail.objects.get_or_create(
+                email=mailaddr,
+                user=user,
+            )
+        user.save()
+        with open(image, "rb") as avatar:
+            pilobj = Image.open(avatar)
+            out = BytesIO()
+            pilobj.save(out, pilobj.format, quality=JPEG_QUALITY)
+            out.seek(0)
+            photo = None
+            if user.photo_set.count() < 1:
+                photo = Photo()
+                photo.user = user
+            else:
+                photo = user.photo_set.first()
+            photo.ip_address = "0.0.0.0"
+            photo.format = file_format(pilobj.format)
+            photo.data = out.read()
+            photo.save()
+            print("xxx: %s" % user.confirmedemail_set.first())
+            confirmed_email = user.confirmedemail_set.first()
+            confirmed_email.photo_id = photo.id
+            confirmed_email.save()

ivatar/ivataraccount/templates/profile.html

@@ -113,7 +113,7 @@
               </li>
               <li class="email-delete">
                 <button type="submit" class="nobutton" onclick="return confirm('{% trans 'Are you sure that you want to delete this email address?' %}')">
-                  Delete Email Adress
+                  Delete Email Address
                 </button>
               </li>
             </ul>
@@ -135,7 +135,7 @@
               </li>
               <li class="email-delete">
                 <button type="submit" class="nobutton" onclick="return confirm('{% trans 'Are you sure that you want to delete this email address?' %}')">
-                  Delete Email Adress
+                  Delete Email Address
                 </button>
               </li>
             </ul>

ivatar/ivataraccount/test_views.py

@@ -1830,4 +1830,63 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
         Test if uploading export works
         """
 
-        self.client.get(reverse("upload_export"))
+        # Ensure we have data in place
+        self.test_export()
+
+        self.login()
+        self.client.get(reverse("export"))
+        response = self.client.post(
+            reverse("export"),
+            {},
+            follow=False,
+        )
+        self.assertIsInstance(response.content, bytes)
+
+        fh_gzip = gzip.open(BytesIO(response.content), "rb")
+        fh = BytesIO(response.content)
+
+        response = self.client.post(
+            reverse("upload_export"),
+            data={"not_porn": "on", "can_distribute": "on", "export_file": fh_gzip},
+            follow=True,
+        )
+        fh_gzip.close()
+        self.assertEqual(response.status_code, 200, "Upload worked")
+        self.assertContains(
+            response,
+            "Unable to parse file: Not a gzipped file",
+            1,
+            200,
+            "Upload didn't work?",
+        )
+
+        # Second test - correctly gzipped content
+        response = self.client.post(
+            reverse("upload_export"),
+            data={"not_porn": "on", "can_distribute": "on", "export_file": fh},
+            follow=True,
+        )
+        fh.close()
+
+        self.assertEqual(response.status_code, 200, "Upload worked")
+        self.assertContains(
+            response,
+            "Choose items to be imported",
+            1,
+            200,
+            "Upload didn't work?",
+        )
+        self.assertContains(
+            response,
+            "asdf@asdf.local",
+            2,
+            200,
+            "Upload didn't work?",
+        )
+
+    def test_prefs_page(self):
+        """
+        Test if preferences page works
+        """
+
+        self.client.get(reverse("user_preference"))

ivatar/test_utils.py

@@ -5,7 +5,7 @@ Test our utils from ivatar.utils
 
 from django.test import TestCase
 
-from ivatar.utils import openid_variations
+from ivatar.utils import is_trusted_url, openid_variations
 
 
 class Tester(TestCase):
@@ -45,3 +45,60 @@ class Tester(TestCase):
         self.assertEqual(openid_variations(openid3)[1], openid1)
         self.assertEqual(openid_variations(openid3)[2], openid2)
         self.assertEqual(openid_variations(openid3)[3], openid3)
+
+    def test_is_trusted_url(self):
+        test1 = is_trusted_url("https://gravatar.com/avatar/63a75a80e6b1f4adfdb04c1ca02e596c", [
+            {
+                "schemes": [
+                    "http",
+                    "https"
+                ],
+                "host_equals": "gravatar.com",
+                "path_prefix": "/avatar/"
+            }
+        ])
+        self.assertTrue(test1)
+
+        test2 = is_trusted_url("https://gravatar.com.example.org/avatar/63a75a80e6b1f4adfdb04c1ca02e596c", [
+            {
+                "schemes": [
+                    "http",
+                    "https"
+                ],
+                "host_suffix": ".gravatar.com",
+                "path_prefix": "/avatar/"
+            }
+        ])
+        self.assertFalse(test2)
+
+        # Test against open redirect with valid URL in query params
+        test3 = is_trusted_url("https://github.com/SethFalco/?boop=https://secure.gravatar.com/avatar/205e460b479e2e5b48aec07710c08d50", [
+            {
+                "schemes": [
+                    "http",
+                    "https"
+                ],
+                "host_suffix": ".gravatar.com",
+                "path_prefix": "/avatar/"
+            }
+        ])
+        self.assertFalse(test3)
+
+        test4 = is_trusted_url("https://ui-avatars.com/api/blah", [
+            {
+                "schemes": [
+                    "https"
+                ],
+                "host_equals": "ui-avatars.com",
+                "path_prefix": "/api/"
+            },
+            {
+                "schemes": [
+                    "http",
+                    "https"
+                ],
+                "host_suffix": ".gravatar.com",
+                "path_prefix": "/avatar/"
+            }
+        ])
+        self.assertTrue(test4)

ivatar/utils.py

@@ -5,6 +5,7 @@ Simple module providing reusable random_string function
 import random
 import string
 from PIL import Image, ImageDraw
+from urllib.parse import urlparse
 
 
 def random_string(length=10):
@@ -112,3 +113,42 @@ def mm_ng(
     )
 
     return image
+
+
+def is_trusted_url(url, url_filters):
+    """
+    Check if a URL is valid and considered a trusted URL.
+    If the URL is malformed, returns False.
+
+    Based on: https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/events/UrlFilter
+    """
+    (scheme, netloc, path, params, query, fragment) = urlparse(url)
+
+    for filter in url_filters:
+        if "schemes" in filter:
+            schemes = filter["schemes"]
+
+            if scheme not in schemes:
+                continue
+
+        if "host_equals" in filter:
+            host_equals = filter["host_equals"]
+
+            if netloc != host_equals:
+                continue
+
+        if "host_suffix" in filter:
+            host_suffix = filter["host_suffix"]
+
+            if not netloc.endswith(host_suffix):
+                continue
+
+        if "path_prefix" in filter:
+            path_prefix = filter["path_prefix"]
+
+            if not path.startswith(path_prefix):
+                continue
+
+        return True
+
+    return False

ivatar/views.py

@@ -34,7 +34,7 @@ from .ivataraccount.models import ConfirmedEmail, ConfirmedOpenId
 from .ivataraccount.models import UnconfirmedEmail, UnconfirmedOpenId
 from .ivataraccount.models import Photo
 from .ivataraccount.models import pil_format, file_format
-from .utils import mm_ng
+from .utils import is_trusted_url, mm_ng
 
 URL_TIMEOUT = 5  # in seconds
 
@@ -146,7 +146,9 @@ class AvatarImageView(TemplateView):
         # Check for :// (schema)
         if default is not None and default.find("://") > 0:
             # Check if it's trusted, if not, reset to None
-            if not any(x in default for x in TRUSTED_DEFAULT_URLS):
+            trusted_url = is_trusted_url(default, TRUSTED_DEFAULT_URLS)
+
+            if not trusted_url:
                 print(
                     "Default URL is not in trusted URLs: '%s' ; Kicking it!" % default
                 )

ssh-pubkeys

@@ -1,2 +0,0 @@
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC6t7d4wsHf/4Ymwo8gnqxsTM2BiqsqEJzuGOOI00uqQNI5s50oalsAjRBzLa4Lum8nmA6tJLf7uk/N0atkF/80x6g9n0VayJnXhGjVz/c2UNL2bPbO9J0Zx1Lrelr1QjlSq3Rf/VoWO2vf63UNW5VOXRCSmCT8UJFUh7eaPs+jXI9AMgSorEEGNSa/Be+bWDVR5Y7K9KT2XcUYZH5c6wASGIl3huscQDcMa/znaruER/21sk3/LAnhHVTjaEjXBbFrL+7mk4up+nlTEwOYupOkEn2CpKc8YuURH6GoVQ/HIYf7CPOKOrVAM3k43rbNb67u1yoHERM4ykMCUhsVCczR falko@home
-ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDnNQpIpD+b1ER1Gg0H+rSvWSg7M9aZIxHYNwWpuvpBOF95zzRbnkswABD1LobU43XLs1mUFca5Fmh+DU02PpnRnyYqzc16O3dFZbClre9Z1eNDcodQSVZqy0L8VM56qnUjD3NF7AExEwG6meSozQLluyHHrg4LnuSoQ2sOKeDSOdxkndE4SPlAwyogvYkglQlrFClxptQfCEH7zLu4f+Y8/ycUpSwSUxy/GCahWNyKQ9mGBkpU+04ZlLjstO0Xaa8KCBREn5KkHRfnk5kjJMv29fz1GRkLaOp0UnZjb6Srzx+LO+e0+wl7gS0ff9FJixEgS23lCYP3p4d8pduu9yX3 ofalk@work