diff --git a/lib/tower/logger_handler.ex b/lib/tower/logger_handler.ex
index d3cc052..ab58b58 100644
--- a/lib/tower/logger_handler.ex
+++ b/lib/tower/logger_handler.ex
@@ -44,6 +44,8 @@ defmodule Tower.LoggerHandler do
   end
 
   def removing_handler(_config) do
+    rate_limiter_delete()
+
     :ok
   end
 
@@ -178,6 +180,10 @@ defmodule Tower.LoggerHandler do
     RateLimiter.new(@handler_id, burst_limit_period, burst_limit_hits)
   end
 
+  defp rate_limiter_delete do
+    RateLimiter.delete(@handler_id)
+  end
+
   defp rate_limiter do
     RateLimiter.get!(@handler_id)
   end