diff --git a/config.py b/config.py
index ac00796..56c44e7 100644
--- a/config.py
+++ b/config.py
@@ -31,7 +31,7 @@ INSTALLED_APPS.extend(
 
 MIDDLEWARE.extend(
     [
-        "django.middleware.locale.LocaleMiddleware",
+        "ivatar.middleware.RemoveVaryForImagesMiddleware",
     ]
 )
 MIDDLEWARE.insert(
diff --git a/ivatar/middleware.py b/ivatar/middleware.py
index 8f3ce10..d3e51bc 100644
--- a/ivatar/middleware.py
+++ b/ivatar/middleware.py
@@ -4,6 +4,26 @@ Middleware classes
 """
 
 from django.utils.deprecation import MiddlewareMixin
+from django.middleware.locale import LocaleMiddleware
+
+
+class RemoveVaryForImagesMiddleware(LocaleMiddleware):
+    """
+    Middleware that extends LocaleMiddleware to skip Vary header processing for image URLs
+    """
+
+    def process_response(self, request, response):
+        # Check if this is an image-related URL
+        path = request.path
+        if any(
+            path.startswith(prefix)
+            for prefix in ["/avatar/", "/gravatarproxy/", "/blueskyproxy/"]
+        ):
+            # Skip the parent's process_response to avoid adding Accept-Language to Vary
+            return response
+
+        # For all other URLs, use the parent's behavior
+        return super().process_response(request, response)
 
 
 class MultipleProxyMiddleware(
diff --git a/ivatar/settings.py b/ivatar/settings.py
index bf4b3ee..30e0232 100644
--- a/ivatar/settings.py
+++ b/ivatar/settings.py
@@ -43,7 +43,6 @@ MIDDLEWARE = [
     "django.contrib.auth.middleware.AuthenticationMiddleware",
     "django.contrib.messages.middleware.MessageMiddleware",
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
-    "django.middleware.locale.LocaleMiddleware",
 ]
 
 ROOT_URLCONF = "ivatar.urls"
diff --git a/ivatar/views.py b/ivatar/views.py
index 562d90c..0a9abfe 100644
--- a/ivatar/views.py
+++ b/ivatar/views.py
@@ -125,8 +125,6 @@ class AvatarImageView(TemplateView):
                     reason=centry["reason"],
                     charset=centry["charset"],
                 )
-                # Remove Vary header for images since language doesn't matter
-                response["Vary"] = ""
                 return response
 
         # In case no digest at all is provided, return to home page
@@ -297,8 +295,6 @@ class AvatarImageView(TemplateView):
             imgformat = "jpeg"
         response = CachingHttpResponse(uri, data, content_type=f"image/{imgformat}")
         response["Cache-Control"] = "max-age=%i" % CACHE_IMAGES_MAX_AGE
-        # Remove Vary header for images since language doesn't matter
-        response["Vary"] = ""
         return response
 
     def _redirect_static_w_size(self, arg0, size):
@@ -317,8 +313,6 @@ class AvatarImageView(TemplateView):
         data.seek(0)
         response = CachingHttpResponse(uri, data, content_type="image/png")
         response["Cache-Control"] = "max-age=%i" % CACHE_IMAGES_MAX_AGE
-        # Remove Vary header for images since language doesn't matter
-        response["Vary"] = ""
         return response
 
     def _return_cached_png(self, arg0, data, uri):
@@ -414,8 +408,6 @@ class GravatarProxyView(View):
                 data.read(), content_type=f"image/{file_format(img.format)}"
             )
             response["Cache-Control"] = "max-age=%i" % CACHE_IMAGES_MAX_AGE
-            # Remove Vary header for images since language doesn't matter
-            response["Vary"] = ""
             return response
 
         except ValueError as exc:
@@ -535,8 +527,6 @@ class BlueskyProxyView(View):
                 data.read(), content_type=f"image/{file_format(format)}"
             )
             response["Cache-Control"] = "max-age=%i" % CACHE_IMAGES_MAX_AGE
-            # Remove Vary header for images since language doesn't matter
-            response["Vary"] = ""
             return response
         except ValueError as exc:
             print(f"Value error: {exc}")