diff --git a/modules/agent/pkg/deployer/manager.go b/modules/agent/pkg/deployer/manager.go
index abb7679703..7e203667f4 100644
--- a/modules/agent/pkg/deployer/manager.go
+++ b/modules/agent/pkg/deployer/manager.go
@@ -62,7 +62,7 @@ func (m *Manager) Cleanup() error {
 		bundleDeployment, err := m.bundleDeploymentCache.Get(m.fleetNamespace, deployed.BundleID)
 		if apierror.IsNotFound(err) {
 			logrus.Infof("Deleting orphan bundle ID %s, release %s", deployed.BundleID, deployed.ReleaseName)
-			if err := m.deployer.Delete(deployed.BundleID, ""); err != nil {
+			if err := m.deployer.Delete(deployed.BundleID, deployed.ReleaseName); err != nil {
 				return err
 			}
 		} else if err != nil {
diff --git a/pkg/helmdeployer/deployer.go b/pkg/helmdeployer/deployer.go
index bf1acf87ee..517d667385 100644
--- a/pkg/helmdeployer/deployer.go
+++ b/pkg/helmdeployer/deployer.go
@@ -428,10 +428,23 @@ func (h *helm) Resources(bundleID, resourcesID string) (*deployer.Resources, err
 }
 
 func (h *helm) Delete(bundleID, releaseName string) error {
-	if releaseName != "" {
-		return h.deleteByRelease(bundleID, releaseName)
+	if releaseName == "" {
+		deployments, err := h.ListDeployments()
+		if err != nil {
+			return err
+		}
+		for _, deployment := range deployments {
+			if deployment.BundleID == bundleID {
+				releaseName = deployment.ReleaseName
+				break
+			}
+		}
+	}
+	if releaseName == "" {
+		// Never found anything to delete
+		return nil
 	}
-	return h.delete(bundleID, fleet.BundleDeploymentOptions{}, false)
+	return h.deleteByRelease(bundleID, releaseName)
 }
 
 func (h *helm) deleteByRelease(bundleID, releaseName string) error {