-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
12 changed files
with
318 additions
and
362 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
62 changes: 0 additions & 62 deletions
62
grails-app/services/io/xh/hoist/monitor/MonitoringEmailService.groovy
This file was deleted.
Oops, something went wrong.
104 changes: 104 additions & 0 deletions
104
grails-app/services/io/xh/hoist/monitor/MonitoringReportService.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,104 @@ | ||
| /* | ||
| * This file belongs to Hoist, an application development toolkit | ||
| * developed by Extremely Heavy Industries (www.xh.io | [email protected]) | ||
| * | ||
| * Copyright © 2023 Extremely Heavy Industries Inc. | ||
| */ | ||
| package io.xh.hoist.monitor | ||
|
|
||
| import io.xh.hoist.BaseService | ||
| import io.xh.hoist.util.Utils | ||
|
|
||
| import static grails.util.Environment.isDevelopmentMode | ||
| import static io.xh.hoist.monitor.MonitorStatus.WARN | ||
| import static io.xh.hoist.util.DateTimeUtils.MINUTES | ||
| import static io.xh.hoist.util.DateTimeUtils.intervalElapsed | ||
| import static java.lang.System.currentTimeMillis | ||
|
|
||
| /** | ||
| * Listens for status monitor change events from MonitoringService and generates a report. | ||
| * Reports generated periodically, and also when status changes after certain thresholds. | ||
| * | ||
| * Also emails status updates to a configurable list of recipients. | ||
| */ | ||
| class MonitoringReportService extends BaseService { | ||
|
|
||
| def emailService, | ||
| configService | ||
|
|
||
| // Notification state for primary instance to manage | ||
| // If primary instance goes down, may get extra notification -- that's ok | ||
| private Long lastNotified = null | ||
| private boolean alertMode = false | ||
|
|
||
| void noteResultsUpdated(Collection<MonitorResults> results) { | ||
| if (!isPrimary) return; | ||
|
|
||
| def failThreshold = config.failNotifyThreshold, | ||
| warnThreshold = config.warnNotifyThreshold | ||
|
|
||
| // 1) Calc new alert mode, true if crossed thresholds or already alerting and still have problems | ||
| boolean newAlertMode = (alertMode && results?.any {it.status >= WARN}) || | ||
| results?.any { it.cyclesAsFail >= failThreshold || it.cyclesAsWarn >= warnThreshold } | ||
|
|
||
| // 2) Generate report if we have a change, or still alerting and interval has elapsed | ||
| if (newAlertMode != alertMode || | ||
| (newAlertMode && intervalElapsed(config.monitorRepeatNotifyMins * MINUTES, lastNotified)) | ||
| ) { | ||
| lastNotified = currentTimeMillis() | ||
| alertMode = newAlertMode | ||
| generateStatusReport(results) | ||
| } | ||
| } | ||
|
|
||
| //------------------------ | ||
| // Implementation | ||
| //------------------------ | ||
| private MonitorStatusReport generateStatusReport(results) { | ||
| def report = new MonitorStatusReport(results: results) | ||
| logDebug("Emitting monitor status report: ${report.title}") | ||
| getTopic('xhMonitorStatusReport').publishAsync(report) | ||
| if (isDevelopmentMode()) { | ||
| emailReport(report) | ||
| } | ||
| } | ||
|
|
||
| private void emailReport(MonitorStatusReport report) { | ||
| def to = emailService.parseMailConfig('xhMonitorEmailRecipients') | ||
| if (to) { | ||
| emailService.sendEmail( | ||
| to: to, | ||
| subject: report.title, | ||
| html: formatHtml(report), | ||
| async: true | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| private String formatHtml(MonitorStatusReport report) { | ||
| def results = report.results | ||
|
|
||
| results.sort{it.name} | ||
| results.sort{it.status} | ||
|
|
||
| if (report.status < WARN) return "There are no alerting monitors for ${Utils.appName}." | ||
|
|
||
| return results.findAll{it.status >= WARN}.collect { | ||
| "+ $it.name | ${it.message ? it.message + ' | ' : ''}Minutes in [${it.status}]: ${it.minsInStatus}" | ||
| }.join('<br>') | ||
| } | ||
|
|
||
| private Map getConfig() { | ||
| configService.getMap('xhMonitorConfig') | ||
| } | ||
|
|
||
| Map getAdminStats() {[ | ||
| config: [ | ||
| toAddress: emailService.parseMailConfig('xhMonitorEmailRecipients'), | ||
| *: configService.getForAdminStats('xhMonitorConfig') | ||
| ], | ||
| lastNotifed: lastNotified, | ||
| alertMode: alertMode | ||
| ]} | ||
|
|
||
| } |
Oops, something went wrong.