Merge branch 'dev'

.flake8

@@ -4,3 +4,4 @@ exclude =
     __pycache__,
     build
 max-complexity = 10
+max-line-length = 120

pyproject.toml

@@ -20,6 +20,7 @@ dependencies = [
     'pydantic',
     'panel',
     'altair',
+    'aind-data-schema',
     'aind-data-access-api[docdb]',
     'aind-data-schema-models',
     'flask',

src/aind_metadata_viz/app.py

@@ -1,21 +1,38 @@
 import panel as pn
 import altair as alt
 from aind_metadata_viz import docdb
+from aind_metadata_viz.docdb import _get_all
+from aind_data_schema import __version__ as ads_version
+
+_get_all(test_mode=True)
 
 pn.extension(design="material")
 pn.extension("vega")
 alt.themes.enable("ggplot2")
 
 color_options = {
-    "default": ["grey", "red", "black"],
-    "lemonade": ["#FFEF00", "pink", "black"],
+    "default": {
+        "valid": "green",
+        "present": "grey",
+        "optional": "grey",
+        "missing": "red",
+        "excluded": "white",
+    },
+    "lemonade": {
+        "valid": "#9FF2F5",
+        "present": "#F49FD7",
+        "optional": "grey",
+        "missing": "#F49FD7",
+        "excluded": "white",
+    },
 }
 
 colors = (
     color_options[pn.state.location.query_params["color"]]
     if "color" in pn.state.location.query_params
     else color_options["default"]
 )
+color_list = list(colors.values())
 
 db = docdb.Database()
 
@@ -24,14 +41,15 @@
 )
 
 top_selector = pn.widgets.Select(
-    name="Select metadata file:", options=docdb.EXPECTED_FILES
+    name="Select metadata file:", options=docdb.ALL_FILES
 )
 
-mid_selector = pn.widgets.Select(name="Sub-select for field:", options=[])
+field_selector = pn.widgets.Select(name="Sub-select for field:", options=[])
 
 missing_selector = pn.widgets.Select(
-    name="Value state", options=["Missing", "Present"]
+    name="Value state", options=["Not Valid/Present", "Valid/Present"]
 )
+missing_selector.value = "Not Valid/Present"
 
 derived_selector = pn.widgets.Select(
     name="Filter for:",
@@ -41,61 +59,37 @@
 
 pn.state.location.sync(modality_selector, {"value": "modality"})
 pn.state.location.sync(top_selector, {"value": "file"})
-pn.state.location.sync(mid_selector, {"value": "field"})
+pn.state.location.sync(field_selector, {"value": "field"})
 pn.state.location.sync(missing_selector, {"value": "missing"})
 pn.state.location.sync(derived_selector, {"value": "derived"})
 
 
 def file_present_chart():
-    (expected_files, excluded_files) = db.get_expected_files()
-    sum_longform_df = db.get_file_presence(expected_files, excluded_files)
+    sum_longform_df = db.get_file_presence()
+    # print(sum_longform_df)
+    local_states = sum_longform_df["state"].unique()
+    local_color_list = [colors[state] for state in local_states]
 
     chart = (
         alt.Chart(sum_longform_df)
         .mark_bar()
         .encode(
-            x=alt.X("column:N", title=None, axis=alt.Axis(grid=False)),
+            x=alt.X("file:N", title=None, axis=alt.Axis(grid=False)),
             y=alt.Y(
-                "count:Q",
+                "sum:Q",
                 title="Metadata assets (n)",
                 axis=alt.Axis(grid=False),
             ),
             color=alt.Color(
-                "category:N",
-                scale=alt.Scale(domain=["present", "absent", "excluded"], range=colors),
-                legend=None,
-            ),
-        )
-        .properties(title="Metadata files")
-    )
-
-    pane = pn.pane.Vega(chart)
-
-    return pane
-
-
-def notfile_present_chart():
-    sum_longform_df = db.get_field_presence()
-
-    chart = (
-        alt.Chart(sum_longform_df)
-        .mark_bar()
-        .encode(
-            x=alt.X("column:N", title=None, axis=alt.Axis(grid=False)),
-            y=alt.Y(
-                "count:Q",
-                title=None,
-                axis=alt.Axis(grid=False),
-            ),
-            color=alt.Color(
-                "category:N",
+                "state:N",
                 scale=alt.Scale(
-                    domain=["present", "absent", "excluded"], range=colors
+                    domain=local_states,
+                    range=local_color_list,
                 ),
                 legend=None,
             ),
         )
-        .properties(title="Other fields")
+        .properties(title="Metadata files")
     )
 
     pane = pn.pane.Vega(chart)
@@ -110,19 +104,17 @@ def build_csv_jscode(event):
     """
     Create the javascript code and append it to the page.
     """
-    csv = db.get_csv(
-        top_selector.value, mid_selector.value, missing_selector.value
-    )
+    csv = db.get_csv(missing_selector.value)
     csv_escaped = csv.replace("\n", "\\n").replace(
         '"', '\\"'
     )  # Escape newlines and double quotes
 
-    get_missing = missing_selector.value == "Missing"
-    missing_text = "missing" if get_missing else "present"
+    get_missing = missing_selector.value == "Not Valid/Present"
+    missing_text = "bad" if get_missing else "good"
 
-    if not mid_selector.value == " ":
+    if not field_selector.value == " ":
         filename = (
-            f"{top_selector.value}-{mid_selector.value}-{missing_text}.csv"
+            f"{top_selector.value}-{field_selector.value}-{missing_text}.csv"
         )
     else:
         filename = f"{top_selector.value}-{missing_text}.csv"
@@ -168,12 +160,15 @@ def build_mid(selected_file, derived_filter, **args):
         .encode(
             x=alt.X("column:N", title=None, axis=alt.Axis(grid=False)),
             y=alt.Y(
-                "count:Q", title="Metadata assets (n)", axis=alt.Axis(grid=False)
+                "count:Q",
+                title="Metadata assets (n)",
+                axis=alt.Axis(grid=False),
             ),
             color=alt.Color(
                 "category:N",
                 scale=alt.Scale(
-                    domain=["present", "absent", "excluded"], range=colors
+                    domain=["valid", "present", "missing", "excluded"],
+                    range=color_list,
                 ),
                 legend=None,
             ),
@@ -182,21 +177,28 @@ def build_mid(selected_file, derived_filter, **args):
     )
 
     # Also update the selected list
-    if len(db.mid_list) > 0:
-        option_list = [" "] + list(db.mid_list[0].keys())
-    else:
-        option_list = []
+    option_list = [" "] + db.field_list
 
-    mid_selector.options = option_list
+    field_selector.options = option_list
 
     return pn.pane.Vega(chart)
 
 
-header = f"""
-# Missing metadata viewer
+def hd_style(text):
+    return (
+        f"<span style='font-weight: bold; color:{colors[text]}'>{text}</span>"
+    )
 
-This app steps through all of the metadata stored in DocDB and checks whether every dictionary key's value is <span style="color:{colors[0]}">present</span> or <span style="color:{colors[1]}">missing</span>
-"""
+
+header = (
+    f"# Metadata Portal\n\n"
+    "This app steps through all of the metadata stored in DocDB and determines whether every record's fields "
+    "(and subfields) are "
+    f"{hd_style('valid')} for aind-data-schema v{ads_version}, "
+    f"{hd_style('present')} but invalid or {hd_style('optional')}, "
+    f"{hd_style('missing')}, or "
+    f"{hd_style('excluded')} for the record's modality."
+)
 
 download_md = """
 **Download options**
@@ -212,7 +214,7 @@ def build_mid(selected_file, derived_filter, **args):
     top_selector,
     derived_selector,
     download_pane,
-    mid_selector,
+    field_selector,
     missing_selector,
     download_button,
     width=400,
@@ -223,7 +225,7 @@ def build_row(selected_modality, derived_filter):
     db.modality_filter = selected_modality
     db.derived_filter = derived_filter
 
-    return pn.Row(file_present_chart, notfile_present_chart)
+    return file_present_chart
 
 
 top_row = pn.bind(
@@ -242,4 +244,6 @@ def build_row(selected_modality, derived_filter):
 # Put everything in a column and buffer it
 main_col = pn.Column(top_row, mid_plot, sizing_mode="stretch_width")
 
-pn.Row(left_col, main_col, pn.layout.HSpacer()).servable(title="Metadata Viz")
+pn.Row(left_col, main_col, pn.layout.HSpacer()).servable(
+    title="Metadata Portal"
+)