Skip to content

Commit

Permalink
feat(vmsnapshot): add new events (#647)
Browse files Browse the repository at this point in the history 
Signed-off-by: Roman Sysoev <[email protected]>
  • Loading branch information
@hardcoretime
hardcoretime authored and Roman Sysoev committed Jan 23, 2025
1 parent 774058d commit 6aa904d
Show file tree
Hide file tree
Showing 5 changed files with 171 additions and 45 deletions.
21 changes: 21 additions & 0 deletions api/core/v1alpha2/events.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,4 +95,25 @@ const (

// ReasonDataSourceDiskProvisioningFailed is event reason that DataSource disk provisioning is failed.
ReasonDataSourceDiskProvisioningFailed = "DataSourceImportDiskProvisioningFailed"

// ReasonVMSnapshottingStarted is event reason that VirtualMachine snapshotting is started.
ReasonVMSnapshottingStarted = "VirtualMachineSnapshottingStarted"

// ReasonVMSnapshottingFrozen is event reason that the file system of VirtualMachine is frozen.
ReasonVMSnapshottingFrozen = "VirtualMachineSnapshottingFrozen"

// ReasonVMSnapshottingInProgress is event reason that VirtualMachine snapshotting is in progress.
ReasonVMSnapshottingInProgress = "VirtualMachineSnapshottingInProgress"

// ReasonVMSnapshottingThawed is event reason that the file system of VirtualMachine is thawed.
ReasonVMSnapshottingThawed = "VirtualMachineSnapshottingThawed"

// ReasonVMSnapshottingPending is event reason that VirtualMachine is not ready for snapshotting.
ReasonVMSnapshottingPending = "VirtualMachineSnapshottingPending"

// ReasonVMSnapshottingCompleted is event reason that VirtualMachine snapshotting is completed.
ReasonVMSnapshottingCompleted = "VirtualMachineSnapshottingCompleted"

// ReasonVMSnapshottingFailed is event reason that VirtualMachine snapshotting is failed.
ReasonVMSnapshottingFailed = "VirtualMachineSnapshottingFailed"
)
160 changes: 128 additions & 32 deletions images/virtualization-artifact/pkg/controller/vmsnapshot/internal/life_cycle.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (

"github.com/deckhouse/virtualization-controller/pkg/controller/conditions"
"github.com/deckhouse/virtualization-controller/pkg/controller/service"
"github.com/deckhouse/virtualization-controller/pkg/eventrecord"
"github.com/deckhouse/virtualization-controller/pkg/logger"
virtv2 "github.com/deckhouse/virtualization/api/core/v1alpha2"
"github.com/deckhouse/virtualization/api/core/v1alpha2/vdcondition"
Expand All @@ -38,12 +39,14 @@ import (
)

type LifeCycleHandler struct {
recorder eventrecord.EventRecorderLogger
snapshotter Snapshotter
storer Storer
}

func NewLifeCycleHandler(snapshotter Snapshotter, storer Storer) *LifeCycleHandler {
func NewLifeCycleHandler(recorder eventrecord.EventRecorderLogger, snapshotter Snapshotter, storer Storer) *LifeCycleHandler {
return &LifeCycleHandler{
recorder: recorder,
snapshotter: snapshotter,
storer: storer,
}
Expand All @@ -61,7 +64,7 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual

vm, err := h.snapshotter.GetVirtualMachine(ctx, vmSnapshot.Spec.VirtualMachineName, vmSnapshot.Namespace)
if err != nil {
setPhaseConditionToFailed(cb, &vmSnapshot.Status.Phase, err)
h.setPhaseConditionToFailed(cb, vmSnapshot, err)
return reconcile.Result{}, err
}

Expand All @@ -74,7 +77,7 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual

_, err = h.unfreezeVirtualMachineIfCan(ctx, vmSnapshot, vm)
if err != nil {
setPhaseConditionToFailed(cb, &vmSnapshot.Status.Phase, err)
h.setPhaseConditionToFailed(cb, vmSnapshot, err)
return reconcile.Result{}, err
}

Expand All @@ -97,7 +100,7 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual
for _, vdSnapshotName := range vmSnapshot.Status.VirtualDiskSnapshotNames {
vdSnapshot, err := h.snapshotter.GetVirtualDiskSnapshot(ctx, vdSnapshotName, vmSnapshot.Namespace)
if err != nil {
setPhaseConditionToFailed(cb, &vmSnapshot.Status.Phase, err)
h.setPhaseConditionToFailed(cb, vmSnapshot, err)
return reconcile.Result{}, err
}

Expand All @@ -113,9 +116,23 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual
vmSnapshot.Status.Phase = virtv2.VirtualMachineSnapshotPhaseFailed
cb.Status(metav1.ConditionFalse).Reason(vmscondition.VirtualDiskSnapshotLost)
if len(lostVDSnapshots) == 1 {
cb.Message(fmt.Sprintf("The underlieng virtual disk snapshot (%s) is lost.", lostVDSnapshots[0]))
msg := fmt.Sprintf("The underlieng virtual disk snapshot (%s) is lost.", lostVDSnapshots[0])
h.recorder.Event(
vmSnapshot,
corev1.EventTypeWarning,
virtv2.ReasonVMSnapshottingFailed,
msg,
)
cb.Message(msg)
} else {
cb.Message(fmt.Sprintf("The underlieng virtual disk snapshots (%s) are lost.", strings.Join(lostVDSnapshots, ", ")))
msg := fmt.Sprintf("The underlieng virtual disk snapshots (%s) are lost.", strings.Join(lostVDSnapshots, ", "))
h.recorder.Event(
vmSnapshot,
corev1.EventTypeWarning,
virtv2.ReasonVMSnapshottingFailed,
msg,
)
cb.Message(msg)
}
return reconcile.Result{}, nil
}
Expand All @@ -131,10 +148,17 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual
virtualMachineReadyCondition, _ := conditions.GetCondition(vmscondition.VirtualMachineReadyType, vmSnapshot.Status.Conditions)
if vm == nil || virtualMachineReadyCondition.Status != metav1.ConditionTrue {
vmSnapshot.Status.Phase = virtv2.VirtualMachineSnapshotPhasePending
msg := fmt.Sprintf("Waiting for the virtual machine %q to be ready for snapshotting.", vmSnapshot.Spec.VirtualMachineName)
h.recorder.Event(
vmSnapshot,
corev1.EventTypeNormal,
virtv2.ReasonVMSnapshottingPending,
msg,
)
cb.
Status(metav1.ConditionFalse).
Reason(vmscondition.WaitingForTheVirtualMachine).
Message(fmt.Sprintf("Waiting for the virtual machine %q to be ready for snapshotting.", vmSnapshot.Spec.VirtualMachineName))
Message(msg)
return reconcile.Result{}, nil
}

Expand All @@ -144,26 +168,39 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual
case err == nil:
case errors.Is(err, ErrBlockDevicesNotReady), errors.Is(err, ErrVirtualDiskNotReady), errors.Is(err, ErrVirtualDiskResizing):
vmSnapshot.Status.Phase = virtv2.VirtualMachineSnapshotPhasePending
msg := service.CapitalizeFirstLetter(err.Error() + ".")
h.recorder.Event(
vmSnapshot,
corev1.EventTypeNormal,
virtv2.ReasonVMSnapshottingPending,
msg,
)
cb.
Status(metav1.ConditionFalse).
Reason(vmscondition.BlockDevicesNotReady).
Message(service.CapitalizeFirstLetter(err.Error() + "."))
Message(msg)
return reconcile.Result{}, nil
default:
setPhaseConditionToFailed(cb, &vmSnapshot.Status.Phase, err)
h.setPhaseConditionToFailed(cb, vmSnapshot, err)
return reconcile.Result{}, err
}

// 2. Ensure there are no RestartAwaitingChanges.
if len(vm.Status.RestartAwaitingChanges) > 0 {
vmSnapshot.Status.Phase = virtv2.VirtualMachineSnapshotPhasePending
msg := fmt.Sprintf(
"Waiting for the restart and approval of changes to virtual machine %q before taking the snapshot.",
vm.Name,
)
h.recorder.Event(
vmSnapshot,
corev1.EventTypeNormal,
virtv2.ReasonVMSnapshottingPending,
msg,
)
cb.
Status(metav1.ConditionFalse).
Reason(vmscondition.RestartAwaitingChanges).Message(
fmt.Sprintf(
"Waiting for the restart and approval of changes to virtual machine %q before taking the snapshot.",
vm.Name,
))
Reason(vmscondition.RestartAwaitingChanges).Message(msg)
return reconcile.Result{}, nil
}

Expand All @@ -172,19 +209,32 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual
isAwaitingConsistency := needToFreeze && !h.snapshotter.CanFreeze(vm) && vmSnapshot.Spec.RequiredConsistency
if isAwaitingConsistency {
vmSnapshot.Status.Phase = virtv2.VirtualMachineSnapshotPhasePending
msg := fmt.Sprintf(
"The snapshotting of virtual machine %q might result in an inconsistent snapshot: "+
"waiting for the virtual machine to be %s",
vm.Name, virtv2.MachineStopped,
)
h.recorder.Event(
vmSnapshot,
corev1.EventTypeNormal,
virtv2.ReasonVMSnapshottingPending,
msg,
)
cb.
Status(metav1.ConditionFalse).
Reason(vmscondition.PotentiallyInconsistent).
Message(fmt.Sprintf(
"The snapshotting of virtual machine %q might result in an inconsistent snapshot: "+
"waiting for the virtual machine to be %s",
vm.Name, virtv2.MachineStopped,
))
Message(msg)
return reconcile.Result{}, nil
}

if vmSnapshot.Status.Phase == virtv2.VirtualMachineSnapshotPhasePending {
vmSnapshot.Status.Phase = virtv2.VirtualMachineSnapshotPhaseInProgress
h.recorder.Event(
vmSnapshot,
corev1.EventTypeNormal,
virtv2.ReasonVMSnapshottingStarted,
"Virtual machine snapshotting process is started.",
)
cb.
Status(metav1.ConditionFalse).
Reason(vmscondition.FileSystemFreezing).
Expand All @@ -196,9 +246,9 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual

// 3. Ensure the virtual machine is consistent for snapshotting.
if needToFreeze {
hasFrozen, err = h.freezeVirtualMachine(ctx, vm)
hasFrozen, err = h.freezeVirtualMachine(ctx, vm, vmSnapshot)
if err != nil {
setPhaseConditionToFailed(cb, &vmSnapshot.Status.Phase, err)
h.setPhaseConditionToFailed(cb, vmSnapshot, err)
return reconcile.Result{}, err
}
}
Expand All @@ -215,7 +265,7 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual
// 4. Create secret.
err = h.ensureSecret(ctx, vm, vmSnapshot)
if err != nil {
setPhaseConditionToFailed(cb, &vmSnapshot.Status.Phase, err)
h.setPhaseConditionToFailed(cb, vmSnapshot, err)
return reconcile.Result{}, err
}

Expand All @@ -229,6 +279,12 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual
case errors.Is(err, ErrVolumeSnapshotClassNotFound), errors.Is(err, ErrCannotTakeSnapshot):
vmSnapshot.Status.Phase = virtv2.VirtualMachineSnapshotPhaseFailed
msg := service.CapitalizeFirstLetter(err.Error())
h.recorder.Event(
vmSnapshot,
corev1.EventTypeWarning,
virtv2.ReasonVMSnapshottingFailed,
msg,
)
if !strings.HasSuffix(msg, ".") {
msg += "."
}
Expand All @@ -238,26 +294,40 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual
Message(msg)
return reconcile.Result{}, nil
default:
setPhaseConditionToFailed(cb, &vmSnapshot.Status.Phase, err)
h.setPhaseConditionToFailed(cb, vmSnapshot, err)
return reconcile.Result{}, err
}

// 7. Wait for VirtualDiskSnapshots to be Ready.
readyCount := h.countReadyVirtualDiskSnapshots(vdSnapshots)
msg := fmt.Sprintf(
"Waiting for the virtual disk snapshots to be taken for "+
"the block devices of the virtual machine %q (%d/%d).",
vm.Name, readyCount, len(vdSnapshots),
)

if readyCount != len(vdSnapshots) {
log.Debug("Waiting for the virtual disk snapshots to be taken for the block devices of the virtual machine")

vmSnapshot.Status.Phase = virtv2.VirtualMachineSnapshotPhaseInProgress
h.recorder.Event(
vmSnapshot,
corev1.EventTypeNormal,
virtv2.ReasonVMSnapshottingInProgress,
msg,
)
cb.
Status(metav1.ConditionFalse).
Reason(vmscondition.Snapshotting).
Message(fmt.Sprintf(
"Waiting for the virtual disk snapshots to be taken for "+
"the block devices of the virtual machine %q (%d/%d).",
vm.Name, readyCount, len(vdSnapshots),
))
Message(msg)
return reconcile.Result{}, nil
} else {
h.recorder.Event(
vmSnapshot,
corev1.EventTypeNormal,
virtv2.ReasonVMSnapshottingInProgress,
msg,
)
}

vmSnapshot.Status.Consistent = ptr.To(true)
Expand All @@ -268,14 +338,20 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual
// 8. Unfreeze VirtualMachine if can.
unfrozen, err := h.unfreezeVirtualMachineIfCan(ctx, vmSnapshot, vm)
if err != nil {
setPhaseConditionToFailed(cb, &vmSnapshot.Status.Phase, err)
h.setPhaseConditionToFailed(cb, vmSnapshot, err)
return reconcile.Result{}, err
}

// 9. Move to Ready phase.
log.Debug("The virtual disk snapshots are taken: the virtual machine snapshot is Ready now", "unfrozen", unfrozen)

vmSnapshot.Status.Phase = virtv2.VirtualMachineSnapshotPhaseReady
h.recorder.Event(
vmSnapshot,
corev1.EventTypeNormal,
virtv2.ReasonVMSnapshottingCompleted,
"Virtual machine snapshotting process is completed.",
)
cb.
Status(metav1.ConditionTrue).
Reason(vmscondition.VirtualMachineReady).
Expand All @@ -284,8 +360,14 @@ func (h LifeCycleHandler) Handle(ctx context.Context, vmSnapshot *virtv2.Virtual
return reconcile.Result{}, nil
}

func setPhaseConditionToFailed(cb *conditions.ConditionBuilder, phase *virtv2.VirtualMachineSnapshotPhase, err error) {
*phase = virtv2.VirtualMachineSnapshotPhaseFailed
func (h LifeCycleHandler) setPhaseConditionToFailed(cb *conditions.ConditionBuilder, vmSnapshot *virtv2.VirtualMachineSnapshot, err error) {
vmSnapshot.Status.Phase = virtv2.VirtualMachineSnapshotPhaseFailed
h.recorder.Event(
vmSnapshot,
corev1.EventTypeWarning,
virtv2.ReasonVMSnapshottingFailed,
err.Error()+".",
)
cb.
Status(metav1.ConditionFalse).
Reason(vmscondition.VirtualMachineSnapshotFailed).
Expand Down Expand Up @@ -423,7 +505,7 @@ func (h LifeCycleHandler) needToFreeze(vm *virtv2.VirtualMachine) bool {
return true
}

func (h LifeCycleHandler) freezeVirtualMachine(ctx context.Context, vm *virtv2.VirtualMachine) (bool, error) {
func (h LifeCycleHandler) freezeVirtualMachine(ctx context.Context, vm *virtv2.VirtualMachine, vmSnapshot *virtv2.VirtualMachineSnapshot) (bool, error) {
if vm.Status.Phase != virtv2.MachineRunning {
return false, errors.New("cannot freeze not Running virtual machine")
}
Expand All @@ -433,6 +515,13 @@ func (h LifeCycleHandler) freezeVirtualMachine(ctx context.Context, vm *virtv2.V
return false, fmt.Errorf("freeze the virtual machine %q: %w", vm.Name, err)
}

h.recorder.Event(
vmSnapshot,
corev1.EventTypeNormal,
virtv2.ReasonVMSnapshottingFrozen,
fmt.Sprintf("The file system of the virtual machine %q is frozen.", vm.Name),
)

return true, nil
}

Expand All @@ -455,6 +544,13 @@ func (h LifeCycleHandler) unfreezeVirtualMachineIfCan(ctx context.Context, vmSna
return false, fmt.Errorf("unfreeze the virtual machine %q: %w", vm.Name, err)
}

h.recorder.Event(
vmSnapshot,
corev1.EventTypeNormal,
virtv2.ReasonVMSnapshottingThawed,
fmt.Sprintf("The file system of the virtual machine %q is thawed.", vm.Name),
)

return true, nil
}

Expand Down
Loading

0 comments on commit 6aa904d

Please sign in to comment.