Skip to content

Commit

Permalink
initiator: support engine upgrade
Browse files Browse the repository at this point in the history 
Longhorn 6001

Signed-off-by: Derek Su <[email protected]>
  • Loading branch information
@derekbit
derekbit committed Apr 9, 2024
1 parent a46b670 commit 7298152
Showing 1 changed file with 66 additions and 25 deletions.
91 changes: 66 additions & 25 deletions pkg/nvme/initiator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,14 +110,35 @@ func (i *Initiator) replaceDmDeviceTarget() error {
return nil
}

func (i *Initiator) isCleanUpDmDeviceRequired(upgradeRequired bool) (bool, error) {
if !upgradeRequired {
return true, nil
}

dmDeviceExists, err := i.isLinearDmDeviceExist()
if err != nil {
return false, errors.Wrapf(err, "failed to check linear dm device for NVMe initiator %s", i.Name)
}
if !dmDeviceExists {
return false, fmt.Errorf("linear dm device doesn't exist for NVMe initiator %s", i.Name)
}

return false, nil
}

// Start starts the NVMe initiator with the given transportAddress and transportServiceID
func (i *Initiator) Start(transportAddress, transportServiceID string, needDmDeviceCleanup bool) (dmDeviceBusy bool, err error) {
func (i *Initiator) Start(transportAddress, transportServiceID string, upgradeRequired bool) (dmDeviceBusy bool, err error) {
defer func() {
if err != nil {
err = errors.Wrapf(err, "failed to start NVMe initiator %s", i.Name)
}
}()

dmDeviceCleanupRequired, err := i.isCleanUpDmDeviceRequired(upgradeRequired)
if err != nil {
return false, errors.Wrapf(err, "failed to check if cleaning up linear device is required for NVMe initiator %s", i.Name)
}

if transportAddress == "" || transportServiceID == "" {
return false, fmt.Errorf("invalid TransportAddress %s and TransportServiceID %s for initiator %s start", transportAddress, transportServiceID, i.Name)
}
Expand Down Expand Up @@ -149,7 +170,7 @@ func (i *Initiator) Start(transportAddress, transportServiceID string, needDmDev
}

i.logger.Infof("Stopping NVMe initiator blindly before starting")
dmDeviceBusy, err = i.stopWithoutLock(needDmDeviceCleanup, false, false)
dmDeviceBusy, err = i.stopWithoutLock(dmDeviceCleanupRequired, false, false)
if err != nil {
return dmDeviceBusy, errors.Wrapf(err, "failed to stop the mismatching NVMe initiator %s before starting", i.Name)
}
Expand Down Expand Up @@ -191,31 +212,40 @@ func (i *Initiator) Start(transportAddress, transportServiceID string, needDmDev
return dmDeviceBusy, errors.Wrapf(err, "failed to load device info after starting NVMe initiator %s", i.Name)
}

needMakeEndpoint := true
if needDmDeviceCleanup {
if dmDeviceBusy {
// Endpoint is already created, just replace the target device
needMakeEndpoint = false
i.logger.Infof("Linear dm device %s is busy, trying the best to replace the target device for NVMe initiator %s", i.Name, i.Name)
if err := i.replaceDmDeviceTarget(); err != nil {
i.logger.WithError(err).Warnf("Failed to replace the target device for NVMe initiator %s", i.Name)
} else {
i.logger.Infof("Successfully replaced the target device for NVMe initiator %s", i.Name)
dmDeviceBusy = false
}
makeEndpointRequired := true

logSuccess := func() {
i.logger.Infof("Successfully replaced the target device for NVMe initiator %s", i.Name)
dmDeviceBusy = false
}
logFailure := func(err error) {
i.logger.WithError(err).Warnf("Failed to replace the target device for NVMe initiator %s", i.Name)
}

if dmDeviceCleanupRequired || upgradeRequired {
i.logger.Infof("Linear dm device %s is busy, attempting to replace the target device for NVMe initiator %s", i.Name, i.Name)

makeEndpointRequired = false

if err := i.replaceDmDeviceTarget(); err != nil {
logFailure(err)
} else {
i.logger.Infof("Creating linear dm device for NVMe initiator %s", i.Name)
if err := i.createLinearDmDevice(); err != nil {
return dmDeviceBusy, errors.Wrapf(err, "failed to create linear dm device for NVMe initiator %s", i.Name)
}
logSuccess()
}
} else {
}

if dmDeviceCleanupRequired && !dmDeviceBusy {
i.logger.Infof("Creating linear dm device for NVMe initiator %s", i.Name)
if err := i.createLinearDmDevice(); err != nil {
return dmDeviceBusy, errors.Wrapf(err, "failed to create linear dm device for NVMe initiator %s", i.Name)
}
} else if !dmDeviceCleanupRequired && !upgradeRequired {
i.logger.Infof("Skipping creating linear dm device for NVMe initiator %s", i.Name)
i.dev.Export = i.dev.Nvme
}

if needMakeEndpoint {
i.logger.Infof("Creating endpoint %v", i.Endpoint)
if makeEndpointRequired {
i.logger.Infof("Creating endpoint for NVMe initiator %s", i.Endpoint)
if err := i.makeEndpoint(); err != nil {
return dmDeviceBusy, err
}
Expand All @@ -226,7 +256,7 @@ func (i *Initiator) Start(transportAddress, transportServiceID string, needDmDev
return dmDeviceBusy, nil
}

func (i *Initiator) Stop(needDmDeviceCleanup, deferDmDeviceCleanup, errOnBusyDmDevice bool) (bool, error) {
func (i *Initiator) Stop(dmDeviceCleanupRequired, deferDmDeviceCleanup, errOnBusyDmDevice bool) (bool, error) {
if i.hostProc != "" {
lock := nsfilelock.NewLockWithTimeout(util.GetHostNamespacePath(i.hostProc), LockFile, LockTimeout)
if err := lock.Lock(); err != nil {
Expand All @@ -235,7 +265,7 @@ func (i *Initiator) Stop(needDmDeviceCleanup, deferDmDeviceCleanup, errOnBusyDmD
defer lock.Unlock()
}

return i.stopWithoutLock(needDmDeviceCleanup, deferDmDeviceCleanup, errOnBusyDmDevice)
return i.stopWithoutLock(dmDeviceCleanupRequired, deferDmDeviceCleanup, errOnBusyDmDevice)
}

func (i *Initiator) removeDmDeviceAndEndpoint(deferDmDeviceCleanup, errOnBusyDmDevice bool) (bool, error) {
Expand All @@ -254,9 +284,9 @@ func (i *Initiator) removeDmDeviceAndEndpoint(deferDmDeviceCleanup, errOnBusyDmD
return false, nil
}

func (i *Initiator) stopWithoutLock(needDmDeviceCleanup, deferDmDeviceCleanup, errOnBusyDmDevice bool) (bool, error) {
func (i *Initiator) stopWithoutLock(dmDeviceCleanupRequired, deferDmDeviceCleanup, errOnBusyDmDevice bool) (bool, error) {
dmDeviceBusy := false
if needDmDeviceCleanup {
if dmDeviceCleanupRequired {
var err error
dmDeviceBusy, err = i.removeDmDeviceAndEndpoint(deferDmDeviceCleanup, errOnBusyDmDevice)
if err != nil {
Expand Down Expand Up @@ -501,6 +531,17 @@ func (i *Initiator) reloadLinearDmDevice() error {
return util.DmsetupReload(i.Name, table, i.executor)
}

func (i *Initiator) isLinearDmDeviceExist() (bool, error) {
dmDevPath := getDmDevicePath(i.Name)
if _, err := os.Stat(dmDevPath); err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
return true, nil
}

func getDmDevicePath(name string) string {
return fmt.Sprintf("/dev/mapper/%s", name)
}

0 comments on commit 7298152

Please sign in to comment.