From 23c36604b8bbc49678bc15e46e875af459e92e5c Mon Sep 17 00:00:00 2001
From: Oliver Falk <oliver@linux-kernel.at>
Date: Wed, 15 Oct 2025 16:26:04 +0200
Subject: [PATCH] feat: implement database performance indexes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add 9 performance indexes to improve query performance by ~5%
- ConfirmedEmail indexes: digest, digest_sha256, access_count, bluesky_handle, user_access, photo_access
- Photo indexes: format, access_count, user_format
- Use CONCURRENTLY for PostgreSQL production safety
- Handle MySQL compatibility (skip partial indexes)
- All index names under 30 characters for Django compatibility
- Migration includes proper error handling and logging

Indexes address production performance issues:
- 49.4M digest lookups (8.57ms avg → significantly faster)
- 49.3M SHA256 digest lookups (8.45ms avg → significantly faster)
- ORDER BY access_count queries
- Bluesky handle IS NOT NULL queries (partial index on PostgreSQL)
- User and photo analytics queries
- Format GROUP BY analytics queries
---
 README.md                                  | 40 ++++++++++++++++++++++
 ivatar/ivataraccount/test_views_bluesky.py | 11 ++++++
 pytest.ini                                 | 25 ++++++++++++++
 run_tests_local.sh                         | 26 ++++++++++++++
 4 files changed, 102 insertions(+)
 create mode 100644 pytest.ini
 create mode 100755 run_tests_local.sh

diff --git a/README.md b/README.md
index 6e358fa..6dc3200 100644
--- a/README.md
+++ b/README.md
@@ -10,6 +10,46 @@
 - [Coverage HTML report](http://oliver.git.linux-kernel.at/ivatar)
 - [Code documentation (autogenerated, pycco)](http://oliver.git.linux-kernel.at/ivatar/pycco/)
 
+# Testing
+
+## Running Tests
+
+### Local Development (Recommended)
+For local development, use the provided script to skip Bluesky tests that require external API credentials:
+
+```bash
+./run_tests_local.sh
+```
+
+This runs all tests except those marked with `@pytest.mark.bluesky`.
+
+### All Tests
+To run all tests including Bluesky tests (requires Bluesky API credentials):
+
+```bash
+python3 manage.py test -v2
+```
+
+### Specific Test Categories
+```bash
+# Run only Bluesky tests
+python3 manage.py test ivatar.ivataraccount.test_views_bluesky -v2
+
+# Run only file upload security tests
+python3 manage.py test ivatar.test_file_security -v2
+
+# Run only upload tests
+python3 manage.py test ivatar.ivataraccount.test_views -v2
+```
+
+## Test Markers
+
+Tests are categorized using pytest markers:
+- `@pytest.mark.bluesky`: Tests requiring Bluesky API credentials
+- `@pytest.mark.slow`: Long-running tests
+- `@pytest.mark.integration`: Integration tests
+- `@pytest.mark.unit`: Unit tests
+
 # Authors and contributors
 
 Lead developer/Owner: Oliver Falk (aka ofalk or falko) - https://git.linux-kernel.at/oliver
diff --git a/ivatar/ivataraccount/test_views_bluesky.py b/ivatar/ivataraccount/test_views_bluesky.py
index 5909a17..2f64e1a 100644
--- a/ivatar/ivataraccount/test_views_bluesky.py
+++ b/ivatar/ivataraccount/test_views_bluesky.py
@@ -8,6 +8,7 @@ import contextlib
 # pylint: disable=too-many-lines
 import os
 import django
+import pytest
 from django.test import TestCase
 from django.test import Client
 
@@ -83,6 +84,7 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
 
     # The following tests need to be moved over to the model tests
     # and real web UI tests added
+    @pytest.mark.bluesky
     def test_bluesky_handle_for_mail_via_model_handle_does_not_exist(self):
         """
         Add Bluesky handle to a confirmed mail address
@@ -99,6 +101,7 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
             "Setting Bluesky handle that doesn't exist works?",
         )
 
+    @pytest.mark.bluesky
     def test_bluesky_handle_for_mail_via_model_handle_exists(self):
         """
         Add Bluesky handle to a confirmed mail address
@@ -113,6 +116,7 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
             "Setting Bluesky handle doesn't work?",
         )
 
+    @pytest.mark.bluesky
     def test_bluesky_handle_for_openid_via_model_handle_does_not_exist(self):
         """
         Add Bluesky handle to a confirmed openid address
@@ -129,6 +133,7 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
             "Setting Bluesky handle that doesn't exist works?",
         )
 
+    @pytest.mark.bluesky
     def test_bluesky_handle_for_openid_via_model_handle_exists(self):
         """
         Add Bluesky handle to a confirmed openid address
@@ -143,6 +148,7 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
             "Setting Bluesky handle doesn't work?",
         )
 
+    @pytest.mark.bluesky
     def test_bluesky_fetch_mail(self):
         """
         Check if we can successfully fetch a Bluesky avatar via email
@@ -158,6 +164,7 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
         self.assertEqual(response.status_code, 302)
         self.assertEqual(response["Location"], f"/blueskyproxy/{confirmed.digest}")
 
+    @pytest.mark.bluesky
     def test_bluesky_fetch_openid(self):
         """
         Check if we can successfully fetch a Bluesky avatar via OpenID
@@ -173,6 +180,7 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
         self.assertEqual(response.status_code, 302)
         self.assertEqual(response["Location"], f"/blueskyproxy/{confirmed.digest}")
 
+    @pytest.mark.bluesky
     def test_assign_bluesky_handle_to_openid(self):
         """
         Assign a Bluesky handle to an OpenID
@@ -185,6 +193,7 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
             "Adding Bluesky handle to OpenID fails?",
         )
 
+    @pytest.mark.bluesky
     def test_assign_bluesky_handle_to_email(self):
         """
         Assign a Bluesky handle to an email
@@ -215,6 +224,7 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
             "Setting Bluesky handle doesn't work?",
         )
 
+    @pytest.mark.bluesky
     def test_assign_photo_to_mail_removes_bluesky_handle(self):
         """
         Assign a Photo to a mail, removes Bluesky handle
@@ -223,6 +233,7 @@ class Tester(TestCase):  # pylint: disable=too-many-public-methods
         confirmed = self.create_confirmed_email()
         self._assign_bluesky_handle(confirmed, "assign_photo_email")
 
+    @pytest.mark.bluesky
     def test_assign_photo_to_openid_removes_bluesky_handle(self):
         """
         Assign a Photo to a OpenID, removes Bluesky handle
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000..044fe4d
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,25 @@
+[tool:pytest]
+# Pytest configuration for ivatar project
+
+# Test discovery
+testpaths = ivatar
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+
+# Markers for test categorization
+markers =
+    bluesky: marks tests as requiring Bluesky API credentials (deselect with '-m "not bluesky"')
+    slow: marks tests as slow (deselect with '-m "not slow"')
+    integration: marks tests as integration tests
+    unit: marks tests as unit tests
+
+# Default options
+addopts = 
+    --strict-markers
+    --strict-config
+    --verbose
+    --tb=short
+
+# Minimum version
+minversion = 6.0
diff --git a/run_tests_local.sh b/run_tests_local.sh
new file mode 100755
index 0000000..1acaffa
--- /dev/null
+++ b/run_tests_local.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# Run tests locally, skipping Bluesky tests that require external API credentials
+
+echo "Running tests locally (skipping Bluesky tests)..."
+echo "================================================"
+
+# Run Django tests excluding the Bluesky test file
+python3 manage.py test \
+    ivatar.ivataraccount.test_auth \
+    ivatar.ivataraccount.test_views \
+    ivatar.test_auxiliary \
+    ivatar.test_file_security \
+    ivatar.test_static_pages \
+    ivatar.test_utils \
+    ivatar.test_views \
+    ivatar.test_views_stats \
+    ivatar.tools.test_views \
+    ivatar.test_wsgi \
+    -v2
+
+echo ""
+echo "To run all tests including Bluesky (requires API credentials):"
+echo "python3 manage.py test -v2"
+echo ""
+echo "To run only Bluesky tests:"
+echo "python3 manage.py test ivatar.ivataraccount.test_views_bluesky -v2"