Managed Infrastructure Maintenance Operator - Milestone 1 #3571
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
Please rebase pull request. |
jaitaiwan
reviewed
May 23, 2024
Comment on lines
+94
to
+78
| if err, ok := err.(*cosmosdb.Error); ok && err.StatusCode == http.StatusConflict { | ||
| err.StatusCode = http.StatusPreconditionFailed | ||
| } |
There was a problem hiding this comment.
Why are we overwriting the http status condition?
There was a problem hiding this comment.
It looks like to me it's because of line 143. We're saying that in case of a conflict we want to change it to a status that will have the cosmosdb Retry function retry the request. If this is the case I think commenting this would be helpful in-case the functionality of functions that use the cosmosdb Retry function change in the future.
There was a problem hiding this comment.
I think a lot of the code here which is copy pasted we should probably open up things to improve this consistently.
github-actions
bot
added
needs-rebase
ArrisLee
reviewed
Jun 15, 2024
ArrisLee
reviewed
Jun 15, 2024
ArrisLee
reviewed
Jun 18, 2024
ArrisLee
reviewed
Jun 18, 2024
pkg/mimo/actuator/manager.go
Outdated
Comment on lines
74
to
258
| docs, err := i.Next(ctx, -1) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| if docs == nil { | ||
| break | ||
| } | ||
|
|
||
| docList = append(docList, docs.MaintenanceManifestDocuments...) | ||
| } | ||
|
|
||
| manifestsToAction := make([]*api.MaintenanceManifestDocument, 0) | ||
|
|
||
| sort.SliceStable(docList, func(i, j int) bool { | ||
| if docList[i].MaintenanceManifest.RunAfter != docList[j].MaintenanceManifest.RunAfter { | ||
| return docList[i].MaintenanceManifest.Priority < docList[j].MaintenanceManifest.Priority | ||
| } | ||
|
|
||
| return docList[i].MaintenanceManifest.RunAfter < docList[j].MaintenanceManifest.RunAfter | ||
| }) | ||
|
|
||
| evaluationTime := a.now() | ||
|
|
||
| // Check for manifests that have timed out first | ||
| for _, doc := range docList { | ||
| if evaluationTime.After(time.Unix(int64(doc.MaintenanceManifest.RunBefore), 0)) { | ||
| // timed out, mark as such | ||
| a.log.Infof("marking %v as outdated: %v older than %v", doc.ID, doc.MaintenanceManifest.RunBefore, evaluationTime.UTC()) | ||
|
|
||
| _, err := a.mmf.Patch(ctx, a.clusterID, doc.ID, func(d *api.MaintenanceManifestDocument) error { | ||
| d.MaintenanceManifest.State = api.MaintenanceManifestStateTimedOut | ||
| d.MaintenanceManifest.StatusText = fmt.Sprintf("timed out at %s", evaluationTime.UTC()) | ||
| return nil | ||
| }) | ||
| if err != nil { | ||
| a.log.Error(err) | ||
| } | ||
| } else { | ||
| // not timed out, do something about it | ||
| manifestsToAction = append(manifestsToAction, doc) | ||
| } | ||
| } | ||
|
|
||
| // Nothing to do, don't dequeue | ||
| if len(manifestsToAction) == 0 { | ||
| return false, nil | ||
| } | ||
|
|
||
| // Dequeue the document | ||
| oc, err := a.oc.Get(ctx, a.clusterID) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| oc, err = a.oc.DoDequeue(ctx, oc) | ||
| if err != nil { | ||
| return false, err // This will include StatusPreconditionFaileds | ||
| } | ||
|
|
||
| taskContext := newTaskContext(a.env, a.log, oc) | ||
|
|
||
| // Execute on the manifests we want to action | ||
| for _, doc := range manifestsToAction { | ||
| // here | ||
| f, ok := a.tasks[doc.MaintenanceManifest.MaintenanceSetID] | ||
| if !ok { | ||
| a.log.Infof("not found %v", doc.MaintenanceManifest.MaintenanceSetID) | ||
| continue | ||
| } | ||
|
|
||
| // Attempt a dequeue | ||
| doc, err = a.mmf.Lease(ctx, a.clusterID, doc.ID) | ||
| if err != nil { | ||
| // log and continue if it doesn't work | ||
| a.log.Error(err) | ||
| continue | ||
| } | ||
|
|
||
| // if we've tried too many times, give up | ||
| if doc.Dequeues > maxDequeueCount { | ||
| err := fmt.Errorf("dequeued %d times, failing", doc.Dequeues) | ||
| _, leaseErr := a.mmf.EndLease(ctx, doc.ClusterID, doc.ID, api.MaintenanceManifestStateTimedOut, to.StringPtr(err.Error())) | ||
| if leaseErr != nil { | ||
| a.log.Error(err) | ||
| } | ||
| continue | ||
| } | ||
|
|
||
| // Perform the task | ||
| state, msg := f(ctx, taskContext, doc, oc) | ||
| _, err = a.mmf.EndLease(ctx, doc.ClusterID, doc.ID, state, &msg) | ||
| if err != nil { | ||
| a.log.Error(err) | ||
| } | ||
| } | ||
|
|
||
| // release the OpenShiftCluster | ||
| _, err = a.oc.EndLease(ctx, a.clusterID, oc.OpenShiftCluster.Properties.ProvisioningState, api.ProvisioningStateMaintenance, nil) | ||
| return true, err | ||
| } |
There was a problem hiding this comment.
suggest to split the logic into private funcs to improve readability, something like:
func (a *actuator) Process(ctx context.Context) (bool, error) {
// Fetch manifests
manifests, err := a.fetchManifests(ctx)
if err != nil {
return false, err
}
// Evaluate and segregate manifests
expiredManifests, actionableManifests := a.evaluateManifests(manifests)
// Handle expired manifests
a.handleExpiredManifests(ctx, expiredManifests)
// If no actionable manifests, return
if len(actionableManifests) == 0 {
return false, nil
}
// Dequeue the cluster document
oc, err := a.oc.DequeueCluster(ctx, a.clusterID)
if err != nil {
return false, err
}
// Execute tasks
taskContext := newTaskContext(a.env, a.log, oc)
a.executeTasks(ctx, taskContext, actionableManifests)
// Release the cluster lease
return true, a.oc.EndClusterLease(ctx, a.clusterID, oc)
}
func (a *actuator) fetchManifests(ctx context.Context) ([]*api.MaintenanceManifestDocument, error) {
// Fetch manifests logic here
}
func (a *actuator) evaluateManifests(manifests []*api.MaintenanceManifestDocument) ([]*api.MaintenanceManifestDocument, []*api.MaintenanceManifestDocument) {
// Evaluation logic here
}
func (a *actuator) handleExpiredManifests(ctx context.Context, expiredManifests []*api.MaintenanceManifestDocument) {
// Handling expired manifests logic here
}
func (a *actuator) executeTasks(ctx context.Context, taskContext tasks.TaskContext, manifests []*api.MaintenanceManifestDocument) {
// Task execution logic here
}
yjst2012
reviewed
Jun 18, 2024
ArrisLee
reviewed
Jun 25, 2024
ArrisLee
reviewed
Jun 25, 2024
ArrisLee
reviewed
Jun 25, 2024
pkg/mimo/actuator/manager.go
Outdated
| // timed out, mark as such | ||
| a.log.Infof("marking %v as outdated: %v older than %v", doc.ID, doc.MaintenanceManifest.RunBefore, evaluationTime.UTC()) | ||
|
|
||
| _, err := a.mmf.Patch(ctx, a.clusterID, doc.ID, func(d *api.MaintenanceManifestDocument) error { |
There was a problem hiding this comment.
shall we implement a retry logic here? just to make the patch action more robust?
ArrisLee
previously approved these changes
Jun 25, 2024
hawkowl
force-pushed
the
hawkowl/mimo-m1
branch
4 times, most recently
from
6ab5f5d to
e5c05b6
Compare
Co-authored-by: Kipp Morris <[email protected]>
hawkowl
force-pushed
the
hawkowl/mimo-m1
branch
from
e7b9b5e to
e93a3f1
Compare
jaitaiwan
approved these changes
Dec 9, 2024
pkg/api/mimo.go
Outdated
Comment on lines
9
to
14
| MaintenanceManifestStatePending MaintenanceManifestState = "Pending" | ||
| MaintenanceManifestStateInProgress MaintenanceManifestState = "InProgress" | ||
| MaintenanceManifestStateCompleted MaintenanceManifestState = "Completed" | ||
| MaintenanceManifestStateFailed MaintenanceManifestState = "Failed" | ||
| MaintenanceManifestStateTimedOut MaintenanceManifestState = "TimedOut" | ||
| MaintenanceManifestStateCancelled MaintenanceManifestState = "Cancelled" |
There was a problem hiding this comment.
We've got these consts in 2 places, can we shrink that down to 1?
pkg/api/mimo.go
Outdated
| StatusText string `json:"statusText,omitempty"` | ||
|
|
||
| MaintenanceSetID string `json:"maintenanceSetID,omitempty"` | ||
| Priority int `json:"priority,omitempty"` |
There was a problem hiding this comment.
Probably need some documentation on what order or meaning Priority has, for example is there negatives? Does a higher value have more priority than a lower one? I know there will be code for the priority somewhere but having it documented alongside the definition would be helpful I think.
| LSN int `json:"_lsn,omitempty"` | ||
| Metadata map[string]interface{} `json:"_metadata,omitempty"` | ||
|
|
||
| ClusterResourceID string `json:"clusterResourceID,omitempty"` |
There was a problem hiding this comment.
What's the difference between ClusterResourceID and ResourceID
|
|
||
| LeaseOwner string `json:"leaseOwner,omitempty" deep:"-"` | ||
| LeaseExpires int `json:"leaseExpires,omitempty" deep:"-"` | ||
| Dequeues int `json:"dequeues,omitempty"` |
There was a problem hiding this comment.
What does Dequeues mean? A count of times the manifest has not been executed?
ehvs
pushed a commit
that referenced
this pull request
Jan 23, 2025
Adds the MIMO milestone 1 ready for work in deployment testing
ArrisLee
pushed a commit
that referenced
this pull request
Feb 9, 2025
Adds the MIMO milestone 1 ready for work in deployment testing
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Which issue this PR addresses:
Part of https://issues.redhat.com/browse/ARO-4895.
What this PR does / why we need it:
This PR is the initial feature branch for the MIMO M1 milestone.
Is there any documentation that needs to be updated for this PR?
Yes, see https://issues.redhat.com/browse/ARO-4895 .
How do you know this will function as expected in production?
Telemetry, monitoring, and documentation will need to be fleshed out. See https://issues.redhat.com/browse/ARO-4895 for details.