diff --git a/dj_rest_auth/serializers.py b/dj_rest_auth/serializers.py
index 3e0132d0..08ecf976 100644
--- a/dj_rest_auth/serializers.py
+++ b/dj_rest_auth/serializers.py
@@ -239,11 +239,12 @@ def __init__(self, *args, **kwargs):
         )
         super(PasswordChangeSerializer, self).__init__(*args, **kwargs)
 
-        if not self.old_password_field_enabled:
-            self.fields.pop('old_password')
-
         self.request = self.context.get('request')
         self.user = getattr(self.request, 'user', None)
+        
+        if not self.old_password_field_enabled or not self.user.has_usable_password():
+            self.fields.pop('old_password')
+
 
     def validate_old_password(self, value):
         invalid_password_conditions = (