Skip to content
This repository has been archived by the owner on Apr 20, 2024. It is now read-only.

Commit

Permalink
Merge pull request #52 from eastgenomics/storage_minify
Browse files Browse the repository at this point in the history 
Storage minify (#52)
  • Loading branch information
@jethror1
jethror1 authored Apr 12, 2023
2 parents b81ab57 + a992257 commit 8c62e72
Show file tree
Hide file tree
Showing 20 changed files with 3,156 additions and 3,564 deletions.
15 changes: 7 additions & 8 deletions Ploutos/Ploutos/settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,13 @@

from pathlib import Path
import os
from dotenv import load_dotenv

load_dotenv()

# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent

# Read in credentials file in same directory
# datafile = BASE_DIR / "Ploutos" / "CREDENTIALS.json"
# with open(datafile, 'r') as credentials_file:
# CREDENTIALS = json.load(credentials_file)

# Assign keys to data from the file
try:
# Database Env
Expand Down Expand Up @@ -182,9 +180,10 @@


# Logging (Docker)
ERROR_LOG = "/logs/ploutos-error.log" # Edit in local Runserver
DEBUG_LOG = "/logs/ploutos-debug.log" # Edit in local Runserver
EXECUTION_LOG = "/logs/executions_log.log" # Edit in local Runserver
LOGGING_PATHWAY = '/logs/'
ERROR_LOG = "ploutos-error.log" # Edit in local Runserver
DEBUG_LOG = "ploutos-debug.log" # Edit in local Runserver
EXECUTION_LOG = "executions_log.log" # Edit in local Runserver

with open(ERROR_LOG, "a+"):
pass
Expand Down
37 changes: 23 additions & 14 deletions Ploutos/dashboard/forms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,30 +262,29 @@ def clean(self):
start = self.cleaned_data["start"]
end = self.cleaned_data["end"]

# Check whether >1 entries are in both proj and assay type by comma
if project_type and assay_type:
if (project_type.find(",") != -1) or (assay_type.find(",") != -1):
raise ValidationError(
"If using both project type and assay type filters, "
"please only enter one of each"
)

# Check both start and end included
if start == "---" and end != "---":
raise ValidationError(
"Please include a start month if entering an end month"
)

if end == "---" and start != "---":
elif end == "---" and start != "---":
raise ValidationError(
"Please include an end month if entering a start month"
)

# Check end month not before start
if end < start:
elif end < start:
# Check end month not before start
raise ValidationError(
"Please ensure start month is before end month"
)
elif (
project_type
and assay_type
and len(project_type.split(",")) > 1
and len(assay_type.split(",")) > 1
):
raise ValidationError(
"If using both project type and assay type filters, "
"please only enter one of each"
)

return self.cleaned_data

Expand Down Expand Up @@ -479,19 +478,29 @@ class LeaderboardForm(forms.Form):
label="Earliest Month",
choices=[],
required=True,
widget=forms.Select(attrs={"style": "width: 350px"}),
)

end = forms.ChoiceField(
label="Latest Month",
choices=[],
required=True,
widget=forms.Select(attrs={"style": "width: 350px"}),
)

def clean(self):
# Used to sanitize inputted data to check for invalid inputs
start = self.cleaned_data["start"]
end = self.cleaned_data["end"]

project_len = len([n for n in self.cleaned_data["project_type"].split(',') if n.strip()])
user_len = len([n for n in self.cleaned_data['user_type'].split(',') if n.strip()])

if project_len > 5 or user_len > 5:
raise ValidationError(
"Maximum 5 for project type or user type"
)

# Check both start and end included
if start == "---" and end != "---":
raise ValidationError(
Expand Down
18 changes: 18 additions & 0 deletions Ploutos/dashboard/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,3 +104,21 @@ class ComputeCosts(models.Model):

def __str__(self):
return self.dx_id


class StagingRun(models.Model):
"""
Model representing Staging52 Tars
"""

name = models.CharField(max_length=200)
file_id = models.CharField(max_length=200)
directory = models.CharField(max_length=200)
created = models.ForeignKey(
Dates, on_delete=models.CASCADE, related_name="staging_run_created"
)
added_on = models.ForeignKey(
Dates, on_delete=models.CASCADE, related_name="staging_run_added_on"
)
archival_state = models.CharField(max_length=20)
size = models.PositiveBigIntegerField()
12 changes: 6 additions & 6 deletions Ploutos/dashboard/templates/dashboard/base.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,22 +72,22 @@
<div class="collapse navbar-collapse" id="navbarNavDropdown">
<ul class="navbar-nav mx-auto">
<li id ="link-index" class="nav-item">
<a id='home' class="nav-link active" aria-current="page" href="{% url 'index' %}">Running Totals</a>
<a id='home' class="nav-link {% if nbar == 'home' %}active{% endif %}" aria-current="page" href="{% url 'index' %}">Running Totals</a>
</li>
<li id ="link-storage" class="nav-item">
<a id='storage' class="nav-link" href="{% url 'storage' %}">Storage Cost</a>
<a id='storage' class="nav-link {% if nbar == 'storage' %}active{% endif %}" href="{% url 'storage' %}">Storage Cost</a>
</li>
<li id ="link-files" class="nav-item">
<a id='files' class="nav-link" href="{% url 'files' %}">File Type Storage</a>
<a id='files' class="nav-link {% if nbar == 'files' %}active{% endif %}" href="{% url 'files' %}">File Type Size</a>
</li>
<li id ="link-compute" class="nav-item">
<a id='compute' class="nav-link" href="{% url 'compute' %}">Compute</a>
<a id='compute' class="nav-link {% if nbar == 'compute' %}active{% endif %}" href="{% url 'compute' %}">Compute</a>
</li>
<li id ="link-leaderboard" class="nav-item">
<a id='leaderboard' class="nav-link" href="{% url 'leaderboard' %}">Leaderboard</a>
<a id='leaderboard' class="nav-link {% if nbar == 'leaderboard' %}active{% endif %}" href="{% url 'leaderboard' %}">Leaderboard</a>
</li>
<li id ="link-tars" class="nav-item">
<a id='tars' class="nav-link" href="{% url 'tars' %}">Tars</a>
<a id='tars' class="nav-link {% if nbar == 'tars' %}active{% endif %}" href="{% url 'tars' %}">Tars</a>
</li>
</ul>
<ul class="navbar-nav ml-auto">
Expand Down
144 changes: 108 additions & 36 deletions Ploutos/dashboard/templates/dashboard/compute.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,60 +18,132 @@ <h2>
{% csrf_token %}
{{ form|crispy }}

<input type="submit" class="button btn-sm btn-primary" name="Daily" value="Daily"/>
<input type="submit" class="button btn-sm btn-info" name="Monthly" value="Monthly"/>
<input type="submit" class="button btn-sm btn-warning" name="Reset" value="Reset"/>
<input type="submit" class="button btn-sm btn-primary" name="DAILY" value="Daily"/>
<input type="submit" class="button btn-sm btn-info" name="MONTHLY" value="Monthly"/>
<input type="submit" class="button btn-sm btn-warning" name="RESET" value="Reset"/>
</form>
</div>

<div id="compute_data" class="mt-5"></div>
{% if detail_html %}
<h2 class="mt-5">Table Details</h2>
<div>
{{ detail_html|safe }}
</div>
{% endif %}
{% if chart_html %}
<h2 class="mt-5">Project Level Details</h2>
<div>
{{ chart_html|safe }}
</div>
{% endif %}
<div id="compute-plot" class="mt-5"></div>
<table class="table table-striped table-bordered border rounded mt-5" id="detail-datatable">
<thead>
{% if pivot %}
<tr>
<th rowspan="1">Date</th>
<th colspan="{{ cols|length }}">Projects / Assays</th>
</tr>
<tr>
<th colspan="1"></th>
{% for col in cols %}
<th colspan="1">{{ col }}</th>
{% endfor %}
</tr>
{% else %}
<tr>
<th>Date</th>
<th>Compute Cost in $</th>
</tr>
{% endif %}
</thead>
</table>
<h2 class="mt-5">Project Level Details</h2>
<table class="table table-striped table-bordered border rounded mt-5" id="project-datatable">
<thead>
<tr>
<th>DNAnexus ID</th>
<th>Date</th>
<th>Project</th>
<th>Total Cost ($)</th>
<th>User</th>
<th>Executable</th>
<th>Runtime</th>
</tr>
</thead>
</table>

</div>
{% endblock content %}

{% block script %}
<script>
Highcharts.chart('compute_data', {{ compute_data|safe }});
var data = JSON.parse('{{context|escapejs}}');

if (data['min_xaxis'] == null) {
var xAxis = {
"tickWidth": 1,
"minPadding": 0.5,
"maxPadding": 0.5,
"min": data['min_xaxis'],
"max": data['max_xaxis'],
"categories": data['xaxis']
}
} else {
var xAxis = {
"type": "datetime",
"dateTimeLabelFormats": {"month": "%b", "year": "%Y"},
"tickInterval": 5 * 24 * 3600 * 1000,
"tickWidth": 1,
"minPadding": 0.5,
"maxPadding": 0.5,
"min": data['min_xaxis'],
"max": data['max_xaxis']
}
}

Highcharts.chart(
'compute-plot', {
chart: {
"zoomType": "x",
"type": "column",
"style": {"float": "center"}},
title: {
"text": "Compute Costs"
},
xAxis: xAxis,
yAxis: {
"allowDecimals": "false",
"min": 0,
"title": {
"text": "Daily Compute Cost ($)"}
},
setOptions: {
"lang": {
"thousandsSep": ",",
"noData": "No data to display"}
},
plotOptions: {
"series": {
"stacking": "normal"
}
},
series: data['plot_series'],
tooltip: {
"pointFormat": "{series.name}: <b>${point.y:.2f} </b><br>{series.options.stack}<br>"
}
});

// Sets up the DataTable
$(document).ready(function () {
$('#project-level-detail').DataTable({
deferRender: true,
$('#detail-datatable').DataTable({
data: data['detail_datatable'],
paging: true,
dom: 'Blfrtip',
dom: 'Bfrtip',
buttons: [
'copy',
'csv'
'copyHtml5', 'csv'
],
ordering: true
autoWidth: false,
order: [[0, 'desc']]
});

$('#detail_html').DataTable({
deferRender: true,
$('#project-datatable').DataTable({
data: data['project_datatable'],
paging: true,
ordering: true
dom: 'Bfrtip',
buttons: [
'copyHtml5', 'csv'
],
autoWidth: false,
});
});

// below control the navbar active
var jobs = document.getElementById("compute");
jobs.classList.add('active');
jobs.setAttribute("aria-current", "page");

var home = document.getElementById("home");
home.classList.remove('active');
home.setAttribute("aria-current","none");

</script>
{% endblock script %}
Loading

0 comments on commit 8c62e72

Please sign in to comment.