diff --git a/config.py b/config.py
index ac00796..56c44e7 100644
--- a/config.py
+++ b/config.py
@@ -31,7 +31,7 @@ INSTALLED_APPS.extend(
 
 MIDDLEWARE.extend(
     [
-        "django.middleware.locale.LocaleMiddleware",
+        "ivatar.middleware.RemoveVaryForImagesMiddleware",
     ]
 )
 MIDDLEWARE.insert(
diff --git a/ivatar/middleware.py b/ivatar/middleware.py
index 8f3ce10..d3e51bc 100644
--- a/ivatar/middleware.py
+++ b/ivatar/middleware.py
@@ -4,6 +4,26 @@ Middleware classes
 """
 
 from django.utils.deprecation import MiddlewareMixin
+from django.middleware.locale import LocaleMiddleware
+
+
+class RemoveVaryForImagesMiddleware(LocaleMiddleware):
+    """
+    Middleware that extends LocaleMiddleware to skip Vary header processing for image URLs
+    """
+
+    def process_response(self, request, response):
+        # Check if this is an image-related URL
+        path = request.path
+        if any(
+            path.startswith(prefix)
+            for prefix in ["/avatar/", "/gravatarproxy/", "/blueskyproxy/"]
+        ):
+            # Skip the parent's process_response to avoid adding Accept-Language to Vary
+            return response
+
+        # For all other URLs, use the parent's behavior
+        return super().process_response(request, response)
 
 
 class MultipleProxyMiddleware(
diff --git a/ivatar/settings.py b/ivatar/settings.py
index bf4b3ee..30e0232 100644
--- a/ivatar/settings.py
+++ b/ivatar/settings.py
@@ -43,7 +43,6 @@ MIDDLEWARE = [
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
-    "django.middleware.locale.LocaleMiddleware",
 ]
 
 ROOT_URLCONF = "ivatar.urls"
diff --git a/ivatar/views.py b/ivatar/views.py
index 562d90c..d63d082 100644
--- a/ivatar/views.py
+++ b/ivatar/views.py
@@ -118,16 +118,13 @@ class AvatarImageView(TemplateView):
             if centry := caches["filesystem"].get(uri):
                 # For DEBUG purpose only
                 # print('Cached entry for %s' % uri)
-                response = HttpResponse(
+                return HttpResponse(
                     centry["content"],
                     content_type=centry["content_type"],
                     status=centry["status"],
                     reason=centry["reason"],
                     charset=centry["charset"],
                 )
-                # Remove Vary header for images since language doesn't matter
-                response["Vary"] = ""
-                return response
 
         # In case no digest at all is provided, return to home page
         if "digest" not in kwargs: