Skip to content

Commit

Permalink
context added mustache added, frequency change taken into account, co…
Browse files Browse the repository at this point in the history 
…mpletion of data instead of deletion
  • Loading branch information
@marcghaly
marcghaly committed Jun 4, 2022
1 parent 7ed5a33 commit 458297e
Show file tree
Hide file tree
Showing 6 changed files with 102 additions and 69 deletions.
5 changes: 3 additions & 2 deletions classes/metric/base.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,11 +191,12 @@ public function is_backfillable(): bool {
/**
* Returns records for backfilled metric.
*
* @param int $starttime Time from which sample is to be retrieved.
* @param int $backwardperiod Time from which sample is to be retrieved.
* @param int $finishtime If data is being completed argument is passed here.
*
* @return array|null
*/
public function generate_metric_items(int $starttime): ?array {
public function generate_metric_items(int $backwardperiod, int $finishtime = null): ?array {
return null;
}

Expand Down
43 changes: 26 additions & 17 deletions classes/metric/online_users_metric.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,16 +65,31 @@ public function is_backfillable(): bool {
/**
* Returns records for backfilled metric.
*
* @param int $starttime Time from which sample is to be retrieved.
* @param int $finishtime Time to which sample is to be retrieved.
* @param int $interval Index representing interval to retrieve.
* @param int $sample Sample to retrieve (TODO check use).
* @param int $backwardperiod Time from which sample is to be retrieved.
* @param int $finishtime If data is being completed argument is passed here.
*
* @return array|null
* @return array
*/
public function generate_metric_items($starttime): ?array {
public function generate_metric_items($backwardperiod, $finishtime = null): array {
global $DB;
$starttime = time() - $starttime;
// Get start time from period selection.
$starttime = time() - $backwardperiod;
// Allows data to be completed instead of retrieving all data again unless frequency change.
$finishtime = $finishtime ?? time();
$frequency = $this->get_frequency();
$rangeretrieved = $this->get_range_retrieved();
if ($rangeretrieved) {
$freqretrieved = (int) explode('-', $rangeretrieved)[2];
} else {
$freqretrieved = false;
}
// In case frequency change we want to override data.
if ($frequency !== $freqretrieved) {
$finishtime = time();
}
if ($finishtime < $starttime) {
return [];
}
$secondsinterval = [
manager::FREQ_MIN => MINSECS,
manager::FREQ_5MIN => MINSECS * 5,
Expand All @@ -88,21 +103,14 @@ public function generate_metric_items($starttime): ?array {
manager::FREQ_MONTH => WEEKSECS * 4
];

$frequency = $this->get_frequency();
if ($frequency) {
$interval = $secondsinterval[$frequency] ?? MINSECS * 5;
} else {
// If form is empty we do not want to backfill anything.
return null;
}
$interval = $secondsinterval[$frequency];
$sql = 'SELECT floor(timecreated / :interval ) * :intervaldup AS time,
COUNT(DISTINCT userid) as value
FROM {logstore_standard_log} WHERE timecreated >= :starttime AND timecreated <= :finishtime
GROUP BY "time"
ORDER BY "time" ASC';
// If interval is small amount of data retrieved could be high hence use of get_recordset_sql.
$rs = $DB->get_recordset_sql($sql,
['interval' => $interval, 'intervaldup' => $interval, 'starttime' => $starttime, 'finishtime' => time()]);
['interval' => $interval, 'intervaldup' => $interval, 'starttime' => $starttime, 'finishtime' => $finishtime]);
$metricitems = [];
foreach ($rs as $r) {
$metricitems[] = new metric_item($this->get_name(), $r->time, $r->value, $this);
Expand All @@ -112,6 +120,7 @@ public function generate_metric_items($starttime): ?array {
$this->mintimestamp = !empty($metricitems) ? $metricitems[0]->time : null;
$this->maxtimestamp = !empty($metricitems) ? end($metricitems)->time : null;
$this->interval = $frequency;

return $metricitems;

}
Expand All @@ -124,7 +133,7 @@ public function set_data_sent_config() {
// Store what data has been sent min and max timestamp range + sample.
$currentconfig = $this->mintimestamp . '-' . $this->maxtimestamp . '-' . $this->interval;
$this->sameconfig = ($this->get_range_retrieved() === $currentconfig) ? true : false;
if (!$this->sameconfig) {
if (!$this->sameconfig && isset($this->mintimestamp) && isset($this->maxtimestamp) && isset($this->interval)) {
$this->set_range_retrieved($currentconfig);
}
}
Expand Down
56 changes: 28 additions & 28 deletions collector/database/chart.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,20 +80,20 @@
$rangeretrieved = $metrics[$metricname]->get_range_retrieved();
if ($rangeretrieved) {
$datametricsaved = explode('-', $rangeretrieved);
if ($datametricsaved) {
$backfilledfrom = userdate($datametricsaved[0], get_string('strftimedatetime', 'cltr_database'), $CFG->timezone);
$backfilledto = userdate($datametricsaved[1], get_string('strftimedatetime', 'cltr_database'), $CFG->timezone);
$backfilledinterval = (int)$datametricsaved[2];
if ($backfilledinterval !== $metrics[$metricname]->get_frequency()) {
$isdifferentfreq = true;
} else {
$isdifferentfreq = false;
}
$context['dataindb'] = get_string('data_in_db', 'tool_cloudmetrics', ['dbstart' => $backfilledfrom, 'dbend' => $backfilledto]);
}
} else {
$emptydb = get_string('data_empty', 'tool_cloudmetrics');
}
if ($datametricsaved) {
$backfilledfrom = userdate($datametricsaved[0], get_string('strftimedatetime', 'cltr_database'), $CFG->timezone);
$backfilledto = userdate($datametricsaved[1], get_string('strftimedatetime', 'cltr_database'), $CFG->timezone);
$backfilledinterval = (int)$datametricsaved[2];
if ($backfilledinterval !== $metrics[$metricname]->get_frequency()) {
$isdifferentfreq = true;
} else {
$isdifferentfreq = false;
}
$context['dataindb'] = get_string('data_in_db', 'tool_cloudmetrics', ['dbstart' => $backfilledfrom, 'dbend' => $backfilledto]);
}
// Gets available data to backfill.
$daterange = $metrics[$metricname]->get_range_log_available();
$startdate = userdate($daterange->min, get_string('strftimedatetime', 'cltr_database'), $CFG->timezone);
Expand All @@ -102,15 +102,15 @@
$context['form'] = $mform->render();
if ($fromform = $mform->get_data()) {
$periodretrieval = $fromform->periodretrieval;
$metricitems = $metrics[$metricname]->generate_metric_items($periodretrieval);
$finishtime = $datametricsaved[0] ?? null;
$metricitems = $metrics[$metricname]->generate_metric_items($periodretrieval, $finishtime);
if ($collector->supports_saved_metric_record()) {
$collector->record_saved_metrics($metrics[$metricname], $metricitems);
}
}
}

// Prepare time window selector.

$periods = [
HOURSECS => get_string('one_hour', 'tool_cloudmetrics'),
DAYSECS => get_string('one_day', 'tool_cloudmetrics'),
Expand Down Expand Up @@ -138,30 +138,30 @@

$periodselect->set_label(get_string('select_graph_period', 'cltr_database'));

$records = $collector->get_metrics($metricname, $defaultperiod);
if ($metricbackfill != 1) {
$records = $collector->get_metrics($metricname, $defaultperiod);
$values = [];
$labels = [];

$values = [];
$labels = [];
foreach ($records as $record) {
$values[] = (float) $record->value;
$labels[] = userdate($record->time, get_string('strftimedatetime', 'cltr_database'), $CFG->timezone);
}

foreach ($records as $record) {
$values[] = (float) $record->value;
$labels[] = userdate($record->time, get_string('strftimedatetime', 'cltr_database'), $CFG->timezone);
$chartseries = new chart_series($metriclabels[$metricname], $values);
$chart = new chart_line();
$chart->add_series($chartseries);
$chart->set_labels($labels);
$context['chart'] = $OUTPUT->render($chart);
$context['selector'] = $OUTPUT->render($select);
$context['periodselect'] = $OUTPUT->render($periodselect);
}

$renderer = $PAGE->get_renderer('tool_cloudmetrics');

$chartseries = new chart_series($metriclabels[$metricname], $values);

$chart = new chart_line();
$chart->add_series($chartseries);
$chart->set_labels($labels);

$context['dataperiod'] = get_string('data_period', 'tool_cloudmetrics', ['startdate' => $startdate ?? 0, 'enddate' => $enddate ?? 0]);
$context['emptydb'] = $emptydb ?? false;
$context['linktochart'] = $url;
$context['selector'] = $OUTPUT->render($select);
$context['periodselect'] = $OUTPUT->render($periodselect);
$context['chart'] = $OUTPUT->render($chart);
$context['metriclabel'] = $metric->get_label();
$context['metricdescription'] = $metric->get_description();
$context['isbackfilled'] = ($metricbackfill == 1 && $backfilledmetric && is_siteadmin()) ? true : false;
Expand Down
20 changes: 10 additions & 10 deletions collector/database/classes/collector.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,16 +103,16 @@ public function record_saved_metrics(\tool_cloudmetrics\metric\base $metricclass
$transaction = $DB->start_delegated_transaction();
// Sets what data has been sent to collector.
$metricclass->set_data_sent_config();
if (is_array($metricitems) && !$metricclass->sameconfig) {
// if ($metricitems[0]->time > $maxbackdate) {
// foreach ($metricitems as $key => $item) {
// if ($item->time > $maxbackdate) {
// unset($metricitems[$key]);
// }
// }
// }
$this->delete_metrics($metricclass->get_name());
$this->record_metrics($metricitems);
if (is_array($metricitems) && count($metricitems) != 0 && !$metricclass->sameconfig) {
// When a further date is passed we only want to add missing data.
$iscompleted = ($metricitems[0]->time < $maxbackdate) ? true : false;
if ($iscompleted) {
// Trick to avoid erasing data when it only needs to be completed.
$this->record_metrics($metricitems);
} else {
$this->delete_metrics($metricclass->get_name());
$this->record_metrics($metricitems);
}
}
$transaction->allow_commit();
}
Expand Down
15 changes: 4 additions & 11 deletions collector/database/tests/cltr_database_test.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,13 +134,6 @@ public function test_backfillable_metric() {

$onlinebackfill = $onlinemetric->is_backfillable();
$activemetricbackfill = $activemetric->is_backfillable();
// 5 min interval and sample of 10, starting Tuesday, May 26, 2020 and ending Thursday, May 26, 2022.
$data = [
'interval' => 0,
'sample' => 0,
'starttime' => 1590465600,
'finishtime' => 1653537600
];

$rec = $DB->get_records(lib::TABLE);
$this->assertEquals(0, count($rec));
Expand All @@ -152,8 +145,8 @@ public function test_backfillable_metric() {
$rec = $DB->get_records(lib::TABLE);
$this->assertEquals(0, count($rec));
$dataobjects = [];

for ($i = 1; $i <= 100; $i++) {
$res = (1653537600 - 1590465600) / 100;
for ($i = 1590465600; $i < 1653537600; $i += $res) {
$dataobjects[] = [
'eventname' => '\core\event\user_created',
'component' => 'core',
Expand All @@ -166,7 +159,7 @@ public function test_backfillable_metric() {
'contextinstanceid' => 0,
'userid' => $i,
'anonymous' => 0,
'timecreated' => rand(1590465600, 1653537600)
'timecreated' => $i
];
}
set_config('enabled_stores', 'logstore_standard', 'tool_log');
Expand All @@ -175,7 +168,7 @@ public function test_backfillable_metric() {
$DB->insert_records('logstore_standard_log', $dataobjects);
$rec = $DB->get_records('logstore_standard_log');
$this->assertEquals(100, count($rec));
$collector->record_saved_metrics($onlinemetric, $onlinemetric->generate_metric_items(1590465600, 1653537600, 0, 0));
$collector->record_saved_metrics($onlinemetric, $onlinemetric->generate_metric_items(1590465600, 1653537600));
$rec = $DB->get_records(lib::TABLE);
foreach ($rec as $r) {
$remainder = $r->time % 300;
Expand Down
32 changes: 31 additions & 1 deletion templates/chart_page.mustache
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,40 @@
{{!
@template tool_cloudmetrics/chart_page
This template renders the list item for each category.
This template renders chart for different metrics.
Context variables for this template:
* isbackfilled Boolean Whether to display backfilled metric form or chart.
* metriclabel String - Label for the metric.
* metricdescription String - Description for the metric.
* selector String - Metric selector.
* periodselect String - Chart period selector.
* chart String - Chart display for metric data.
* backfillable Boolean - Whether or not selected metric supports data retrieval.
* backfillurl String - Url redirecting to backfill form.
* form String - Form for retrieving data.
* linktochart String - Url redirecting to chart display.
* emptydb String | Boolean - Whether metric has already been backfilled in db.
* dataperiod String - Available data to retrieve for logstore log standard table.
* dataindb String - Already retrieved data stored.
* isdifferentfreq Boolean - Whether frequency change since last data retrieval.
Example context (json):
{
"isbackfilled": "false",
"metriclabel": "Online users",
"metricdescription": "Users that are currently online.",
"selector": true,
"periodselect": "<div class="singleselect d-inline-block">...</div>",
"chart": "<div class="chart-area" id="chart-area-629a7399c40fd629a7399b9f9b6"></div>",
"backfillable": true,
"backfillurl": "http://moodle.com/admin/tool/cloudmetrics/collector/database/chart.php?metric=onlineusers&metric_backfill=1",
"form": "<form autocomplete="off"> ... </form>",
"linktochart": "http://moodle.com/admin/tool/cloudmetrics/collector/database/chart.php?metric=onlineusers",
"emptydb": false,
"dataperiod": "Current information shows data can be retrieved from 22 Oct 2020, 17:10 to 3 Jun 2022, 08:33.",
"dataindb": "Your database contains data from 2 Jun 2021, 20:00 to 9 Feb 2022, 19:00.",
"isdifferentfreq": false
}
}}
{{^isbackfilled}}
Expand Down

0 comments on commit 458297e

Please sign in to comment.