From 0e7e21cd31f702971444bd6a99f9d8e030c378cd Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Thu, 12 Dec 2024 23:32:43 +0000
Subject: [PATCH 01/68] first draft of venn

---
 code/pages/3_AIND data access playground.py | 93 +++++++++++++++++++--
 1 file changed, 85 insertions(+), 8 deletions(-)

diff --git a/code/pages/3_AIND data access playground.py b/code/pages/3_AIND data access playground.py
index 03324c7..f7f0c42 100644
--- a/code/pages/3_AIND data access playground.py	
+++ b/code/pages/3_AIND data access playground.py	
@@ -2,10 +2,24 @@
 '''
 
 import logging
+import re
 
+from matplotlib_venn import venn2, venn3
+import matplotlib.pyplot as plt
 import streamlit as st
 from streamlit_dynamic_filters import DynamicFilters
-from util.fetch_data_docDB import load_data_from_docDB
+
+from util.fetch_data_docDB import load_data_from_docDB, load_client
+
+st.markdown(
+"""
+<style>
+    .stMultiSelect [data-baseweb=select] span{
+        max-width: 1000px;
+    }
+</style>""",
+unsafe_allow_html=True,
+)
 
 try:
     st.set_page_config(layout="wide", 
@@ -19,12 +33,75 @@
 except:
     pass
 
-df = load_data_from_docDB()
 
-st.markdown(f'### Note: the dataframe showing here has been merged in to the master table on the Home page!')
+client = load_client()
+
+queries = {
+    "'dynamic_foraging' in software name, raw": {
+        "session.data_streams.software.name": "dynamic-foraging-task",
+        "name": {"$not": {"$regex": ".*processed.*"}},
+    },
+    "'dynamic_foraging' in software name, processed": {
+        "session.data_streams.software.name": "dynamic-foraging-task",
+        "name": {"$regex": ".*processed.*"},
+    },
+    "'fib' in 'data_description.modality', raw": {
+        "data_description.modality.abbreviation": "fib",
+        "name": {"$not": {"$regex": ".*processed.*"}},
+    },
+    "'fib' in 'rig.modalities', raw": {
+        "rig.modalities.abbreviation": "fib",
+        "name": {"$not": {"$regex": ".*processed.*"}},        
+    },
+    "'fib' in 'session.data_streams', raw": {
+        "session.data_streams.stream_modalities.abbreviation": "fib",
+        "name": {"$not": {"$regex": ".*processed.*"}},
+    }
+}
+
+@st.cache_data(ttl=3600 * 12)  # Cache the df_docDB up to 12 hours
+def get_session_from_query(query):
+    results = client.retrieve_docdb_records(
+        filter_query=query,
+        projection={"name": 1, "_id": 1},
+    )
+    
+    sessions = [re.sub(r'_processed.*$', '', r["name"]) for r in results]
+    return sessions
+
+# Multiselect for selecting queries up to three
+query_keys = list(queries.keys())
+selected_queries = st.multiselect(
+    "Select queries to filter sessions",
+    query_keys,
+    default=query_keys[:3],
+    key="selected_queries",
+)
+
+# Generage venn diagram of the selected queries
+query_results = {key: set(get_session_from_query(queries[key])) for key in selected_queries}
+
+fig, ax = plt.subplots()
+if len(selected_queries) == 2:
+    venn2(
+        [query_results[key] for key in selected_queries],
+        set_labels=selected_queries,
+    )
+else:
+    venn3(
+        [query_results[key] for key in selected_queries],
+        set_labels=selected_queries,
+    )
+    
+st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
+
+
+# df = load_data_from_docDB()
+
+# st.markdown(f'### Note: the dataframe showing here has been merged in to the master table on the Home page!')
 
-dynamic_filters = DynamicFilters(
-    df=df, 
-    filters=['subject_id', 'subject_genotype'])
-dynamic_filters.display_filters()
-dynamic_filters.display_df()
+# dynamic_filters = DynamicFilters(
+#     df=df, 
+#     filters=['subject_id', 'subject_genotype'])
+# dynamic_filters.display_filters()
+# dynamic_filters.display_df()

From 92c3fd34457f7964fea2c2c83cea12d2255a8747 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 20 Dec 2024 20:57:19 +0000
Subject: [PATCH 02/68] add stimulus_epochs

---
 code/pages/3_AIND data access playground.py | 26 ++++++++++++++++-----
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/code/pages/3_AIND data access playground.py b/code/pages/3_AIND data access playground.py
index f7f0c42..be6aca7 100644
--- a/code/pages/3_AIND data access playground.py	
+++ b/code/pages/3_AIND data access playground.py	
@@ -37,23 +37,37 @@
 client = load_client()
 
 queries = {
-    "'dynamic_foraging' in software name, raw": {
-        "session.data_streams.software.name": "dynamic-foraging-task",
+    "raw, 'dynamic_foraging' in ANY software name": {
+        "$or":[
+            {"session.data_streams.software.name": "dynamic-foraging-task"},
+            {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+        ],
         "name": {"$not": {"$regex": ".*processed.*"}},
     },
-    "'dynamic_foraging' in software name, processed": {
+    "raw, 'dynamic_foraging' in data_streams software name": {
         "session.data_streams.software.name": "dynamic-foraging-task",
+        "name": {"$not": {"$regex": ".*processed.*"}},
+    },
+    "raw, 'dynamic_foraging' in stimulus_epochs software name": {
+        "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
+        "name": {"$not": {"$regex": ".*processed.*"}},
+    },
+    "processed, 'dynamic_foraging' in ANY software name": {
+        "$or":[
+            {"session.data_streams.software.name": "dynamic-foraging-task"},
+            {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+        ],
         "name": {"$regex": ".*processed.*"},
     },
-    "'fib' in 'data_description.modality', raw": {
+    "raw, 'fib' in 'data_description.modality'": {
         "data_description.modality.abbreviation": "fib",
         "name": {"$not": {"$regex": ".*processed.*"}},
     },
-    "'fib' in 'rig.modalities', raw": {
+    "raw, 'fib' in 'rig.modalities'": {
         "rig.modalities.abbreviation": "fib",
         "name": {"$not": {"$regex": ".*processed.*"}},        
     },
-    "'fib' in 'session.data_streams', raw": {
+    "raw, 'fib' in 'session.data_streams'": {
         "session.data_streams.stream_modalities.abbreviation": "fib",
         "name": {"$not": {"$regex": ".*processed.*"}},
     }

From 9d9e36dc63463c366249a9c8b29a96a9318e0951 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 20 Dec 2024 21:19:02 +0000
Subject: [PATCH 03/68] refactor

---
 code/pages/3_AIND data access playground.py | 71 +++++++++++++--------
 1 file changed, 43 insertions(+), 28 deletions(-)

diff --git a/code/pages/3_AIND data access playground.py b/code/pages/3_AIND data access playground.py
index be6aca7..f9dacd2 100644
--- a/code/pages/3_AIND data access playground.py	
+++ b/code/pages/3_AIND data access playground.py	
@@ -83,39 +83,54 @@ def get_session_from_query(query):
     sessions = [re.sub(r'_processed.*$', '', r["name"]) for r in results]
     return sessions
 
-# Multiselect for selecting queries up to three
-query_keys = list(queries.keys())
-selected_queries = st.multiselect(
-    "Select queries to filter sessions",
-    query_keys,
-    default=query_keys[:3],
-    key="selected_queries",
-)
 
-# Generage venn diagram of the selected queries
-query_results = {key: set(get_session_from_query(queries[key])) for key in selected_queries}
+def app():
 
-fig, ax = plt.subplots()
-if len(selected_queries) == 2:
-    venn2(
-        [query_results[key] for key in selected_queries],
-        set_labels=selected_queries,
-    )
-else:
-    venn3(
-        [query_results[key] for key in selected_queries],
-        set_labels=selected_queries,
+    # Multiselect for selecting queries up to three
+    query_keys = list(queries.keys())
+    selected_queries = st.multiselect(
+        "Select queries to filter sessions",
+        query_keys,
+        default=query_keys[:3],
+        key="selected_queries",
     )
+
+    # Generage venn diagram of the selected queries
+    query_results = {key: set(get_session_from_query(queries[key])) for key in selected_queries}
+
+
+    # -- Show venn --
+    fig, ax = plt.subplots()
+    if len(selected_queries) == 2:
+        venn2(
+            [query_results[key] for key in selected_queries],
+            set_labels=selected_queries,
+        )
+    else:
+        venn3(
+            [query_results[key] for key in selected_queries],
+            set_labels=selected_queries,
+        )
+        
+    st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
+    
+    
+    # -- Show dataframe that summarize the selected queries --
+    st.markdown(f"### Summary of selected queries")
+    query_results
     
-st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
 
 
-# df = load_data_from_docDB()
+    # df = load_data_from_docDB()
 
-# st.markdown(f'### Note: the dataframe showing here has been merged in to the master table on the Home page!')
+    # st.markdown(f'### Note: the dataframe showing here has been merged in to the master table on the Home page!')
 
-# dynamic_filters = DynamicFilters(
-#     df=df, 
-#     filters=['subject_id', 'subject_genotype'])
-# dynamic_filters.display_filters()
-# dynamic_filters.display_df()
+    # dynamic_filters = DynamicFilters(
+    #     df=df, 
+    #     filters=['subject_id', 'subject_genotype'])
+    # dynamic_filters.display_filters()
+    # dynamic_filters.display_df()
+    
+    
+if __name__ == "__main__":
+    app()

From 629bf712d012fdfb7d361d821d4ad9728a4efc86 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 20 Dec 2024 21:54:20 +0000
Subject: [PATCH 04/68] change page name

---
 .../{3_AIND data access playground.py => 3_docDB inventory.py}    | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename code/pages/{3_AIND data access playground.py => 3_docDB inventory.py} (100%)

diff --git a/code/pages/3_AIND data access playground.py b/code/pages/3_docDB inventory.py
similarity index 100%
rename from code/pages/3_AIND data access playground.py
rename to code/pages/3_docDB inventory.py

From bc45811af1b44a58a8ffab12b108e035537b8971 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 20 Dec 2024 23:24:23 +0000
Subject: [PATCH 05/68] reorder page; show queries

---
 ...ocDB inventory.py => 0_docDB inventory.py} | 32 +++++++++++--------
 1 file changed, 19 insertions(+), 13 deletions(-)
 rename code/pages/{3_docDB inventory.py => 0_docDB inventory.py} (89%)

diff --git a/code/pages/3_docDB inventory.py b/code/pages/0_docDB inventory.py
similarity index 89%
rename from code/pages/3_docDB inventory.py
rename to code/pages/0_docDB inventory.py
index f9dacd2..f1b68f1 100644
--- a/code/pages/3_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -11,16 +11,6 @@
 
 from util.fetch_data_docDB import load_data_from_docDB, load_client
 
-st.markdown(
-"""
-<style>
-    .stMultiSelect [data-baseweb=select] span{
-        max-width: 1000px;
-    }
-</style>""",
-unsafe_allow_html=True,
-)
-
 try:
     st.set_page_config(layout="wide", 
                     page_title='Foraging behavior browser',
@@ -33,10 +23,19 @@
 except:
     pass
 
+st.markdown(
+"""
+<style>
+    .stMultiSelect [data-baseweb=select] span{
+        max-width: 1000px;
+    }
+</style>""",
+unsafe_allow_html=True,
+)
 
 client = load_client()
 
-queries = {
+QUERY_PRESET = {
     "raw, 'dynamic_foraging' in ANY software name": {
         "$or":[
             {"session.data_streams.software.name": "dynamic-foraging-task"},
@@ -85,9 +84,16 @@ def get_session_from_query(query):
 
 
 def app():
+    
+    # 
+    with st.expander("Show docDB queries", expanded=False):
+        st.write('See how to use these queries [here](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
+        for key, query in QUERY_PRESET.items():
+            st.markdown(f"**{key}**")
+            st.code(query)
 
     # Multiselect for selecting queries up to three
-    query_keys = list(queries.keys())
+    query_keys = list(QUERY_PRESET.keys())
     selected_queries = st.multiselect(
         "Select queries to filter sessions",
         query_keys,
@@ -96,7 +102,7 @@ def app():
     )
 
     # Generage venn diagram of the selected queries
-    query_results = {key: set(get_session_from_query(queries[key])) for key in selected_queries}
+    query_results = {key: set(get_session_from_query(QUERY_PRESET[key])) for key in selected_queries}
 
 
     # -- Show venn --

From bb79299c3a084a51e0849e56fbcd2a4d70449f6a Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Sat, 21 Dec 2024 00:22:22 +0000
Subject: [PATCH 06/68] add util.reformat.split_nwb_name

---
 code/util/reformat.py | 43 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 code/util/reformat.py

diff --git a/code/util/reformat.py b/code/util/reformat.py
new file mode 100644
index 0000000..efa8dcd
--- /dev/null
+++ b/code/util/reformat.py
@@ -0,0 +1,43 @@
+
+""" Helper functions to reformat the data
+"""
+import re
+
+
+# Function to split the `nwb_name` column
+def split_nwb_name(nwb_name):
+    """Turn the nwb_name into subject_id, session_date, nwb_suffix in order to be merged to
+    the main df.
+
+    Parameters
+    ----------
+    nwb_name : str. The name of the nwb file. This function can handle the following formats:
+        "721403_2024-08-09_08-39-12.nwb"
+        "685641_2023-10-04.nwb",
+        "behavior_754280_2024-11-14_11-06-24.nwb",
+        "behavior_1_2024-08-05_15-48-54",
+        "
+        ...
+
+    Returns
+    -------
+    subject_id : str. The subject ID
+    session_date : str. The session date
+    nwb_suffix : int. The nwb suffix (converted from session time if available, otherwise 0)
+    """
+
+    pattern = R"(?:\w+_)?(?P<subject_id>\d+)_(?P<date>\d{4}-\d{2}-\d{2})(?:_(?P<time>\d{2}-\d{2}-\d{2}))?(?:.*)?"
+    matches = re.search(pattern, nwb_name)
+    
+    if not matches:
+        return None, None, 0
+
+    subject_id = matches.group("subject_id")
+    session_date = matches.group("date")
+    session_time = matches.group("time")
+    if session_time:
+        nwb_suffix = int(session_time.replace("-", ""))
+    else:
+        nwb_suffix = 0
+
+    return subject_id, session_date, nwb_suffix
\ No newline at end of file

From 3677098131828684741005316d0cb7377454d466 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Sat, 21 Dec 2024 00:50:00 +0000
Subject: [PATCH 07/68] outer merge queries

---
 code/pages/0_docDB inventory.py | 55 +++++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 16 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index f1b68f1..6beab25 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -3,13 +3,16 @@
 
 import logging
 import re
+from functools import reduce
 
 from matplotlib_venn import venn2, venn3
 import matplotlib.pyplot as plt
 import streamlit as st
 from streamlit_dynamic_filters import DynamicFilters
+import pandas as pd
 
 from util.fetch_data_docDB import load_data_from_docDB, load_client
+from util.reformat import split_nwb_name
 
 try:
     st.set_page_config(layout="wide", 
@@ -43,6 +46,14 @@
         ],
         "name": {"$not": {"$regex": ".*processed.*"}},
     },
+    "raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)": {
+        "$or":[
+            {"session.data_streams.software.name": "dynamic-foraging-task"},
+            {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+        ],
+        "data_description.modality.abbreviation": "ecephys",
+        "name": {"$not": {"$regex": ".*processed.*"}},        
+    },
     "raw, 'dynamic_foraging' in data_streams software name": {
         "session.data_streams.software.name": "dynamic-foraging-task",
         "name": {"$not": {"$regex": ".*processed.*"}},
@@ -73,10 +84,10 @@
 }
 
 @st.cache_data(ttl=3600 * 12)  # Cache the df_docDB up to 12 hours
-def get_session_from_query(query):
+def query_sessions_from_docDB(query):
     results = client.retrieve_docdb_records(
         filter_query=query,
-        projection={"name": 1, "_id": 1},
+        projection={"name": 1, "_id": 0},
     )
     
     sessions = [re.sub(r'_processed.*$', '', r["name"]) for r in results]
@@ -84,8 +95,8 @@ def get_session_from_query(query):
 
 
 def app():
-    
-    # 
+
+    #
     with st.expander("Show docDB queries", expanded=False):
         st.write('See how to use these queries [here](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
         for key, query in QUERY_PRESET.items():
@@ -102,8 +113,27 @@ def app():
     )
 
     # Generage venn diagram of the selected queries
-    query_results = {key: set(get_session_from_query(QUERY_PRESET[key])) for key in selected_queries}
+    query_results = {key: query_sessions_from_docDB(QUERY_PRESET[key]) for key in selected_queries}
+
+    # -- Show dataframe that summarize the selected queries --
+    st.markdown(f"#### Merged dataframe")
+
+    dfs = []
+    for query_name, query_result in query_results.items():
+        df = pd.DataFrame(query_result, columns=[f"nwb_name_{query_name}"])
+
+        # Apply the function to create new columns
+        df[["subject_id", "session_date", "nwb_suffix"]] = df["nwb_name"].apply(
+            lambda x: pd.Series(split_nwb_name(x))
+        )
+        
+        df[query_name] = True
+        dfs.append(df)
 
+    # Out merge all dataframes
+    df_merged = reduce(
+        lambda left, right: pd.merge(left, right, on=["subject_id", "session_date", "nwb_suffix"], how="outer"), dfs
+    )
 
     # -- Show venn --
     fig, ax = plt.subplots()
@@ -117,26 +147,19 @@ def app():
             [query_results[key] for key in selected_queries],
             set_labels=selected_queries,
         )
-        
-    st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
-    
-    
-    # -- Show dataframe that summarize the selected queries --
-    st.markdown(f"### Summary of selected queries")
-    query_results
-    
 
+    st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
 
     # df = load_data_from_docDB()
 
     # st.markdown(f'### Note: the dataframe showing here has been merged in to the master table on the Home page!')
 
     # dynamic_filters = DynamicFilters(
-    #     df=df, 
+    #     df=df,
     #     filters=['subject_id', 'subject_genotype'])
     # dynamic_filters.display_filters()
     # dynamic_filters.display_df()
-    
-    
+
+
 if __name__ == "__main__":
     app()

From e2d9a80722d2e59bd7ed030a5eaca08703f6a359 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Sat, 21 Dec 2024 01:16:00 +0000
Subject: [PATCH 08/68] refactor: add download button; use combine_first

---
 code/pages/0_docDB inventory.py | 34 ++++++++++++++++++++++++++-------
 1 file changed, 27 insertions(+), 7 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 6beab25..7656ee6 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -94,6 +94,18 @@ def query_sessions_from_docDB(query):
     return sessions
 
 
+def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
+    """ Download df as csv """
+    csv = df.to_csv(index=True)
+    
+    # Create download buttons
+    st.download_button(
+        label=label,
+        data=csv,
+        file_name=file_name,
+        mime='text/csv'
+    )
+
 def app():
 
     #
@@ -120,20 +132,28 @@ def app():
 
     dfs = []
     for query_name, query_result in query_results.items():
-        df = pd.DataFrame(query_result, columns=[f"nwb_name_{query_name}"])
+        df = pd.DataFrame(query_result, columns=[f"nwb_name"])
 
-        # Apply the function to create new columns
-        df[["subject_id", "session_date", "nwb_suffix"]] = df["nwb_name"].apply(
+        # Create index that can be joined with other dfs and my main df
+        df[["subject_id", "session_date", "nwb_suffix"]] = df[f"nwb_name"].apply(
             lambda x: pd.Series(split_nwb_name(x))
         )
         
         df[query_name] = True
+        df.set_index(["subject_id", "session_date", "nwb_suffix"], inplace=True)
         dfs.append(df)
 
-    # Out merge all dataframes
-    df_merged = reduce(
-        lambda left, right: pd.merge(left, right, on=["subject_id", "session_date", "nwb_suffix"], how="outer"), dfs
-    )
+    # # Out merge all dataframes
+    # df_merged = reduce(
+    #     lambda left, right: left.join(right, how="outer"), dfs
+    # )
+    
+    df_merged = dfs[0]
+    for df in dfs[1:]:
+        df_merged = df_merged.combine_first(df)
+    
+    st.write(df_merged)
+    download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
 
     # -- Show venn --
     fig, ax = plt.subplots()

From d6bc352701ddad45e4291540ae716becda2ac8e0 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Sat, 21 Dec 2024 01:53:00 +0000
Subject: [PATCH 09/68] add more fields

---
 code/pages/0_docDB inventory.py | 58 +++++++++++++++++----------------
 1 file changed, 30 insertions(+), 28 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 7656ee6..69dd112 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -39,14 +39,14 @@
 client = load_client()
 
 QUERY_PRESET = {
-    "raw, 'dynamic_foraging' in ANY software name": {
+    "{raw, 'dynamic_foraging' in ANY software name}": {
         "$or":[
             {"session.data_streams.software.name": "dynamic-foraging-task"},
             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
         ],
         "name": {"$not": {"$regex": ".*processed.*"}},
     },
-    "raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)": {
+    "{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}": {
         "$or":[
             {"session.data_streams.software.name": "dynamic-foraging-task"},
             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
@@ -54,30 +54,30 @@
         "data_description.modality.abbreviation": "ecephys",
         "name": {"$not": {"$regex": ".*processed.*"}},        
     },
-    "raw, 'dynamic_foraging' in data_streams software name": {
+    "{raw, 'dynamic_foraging' in data_streams software name}": {
         "session.data_streams.software.name": "dynamic-foraging-task",
         "name": {"$not": {"$regex": ".*processed.*"}},
     },
-    "raw, 'dynamic_foraging' in stimulus_epochs software name": {
+    "{raw, 'dynamic_foraging' in stimulus_epochs software name}": {
         "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
         "name": {"$not": {"$regex": ".*processed.*"}},
     },
-    "processed, 'dynamic_foraging' in ANY software name": {
+    "{processed, 'dynamic_foraging' in ANY software name}": {
         "$or":[
             {"session.data_streams.software.name": "dynamic-foraging-task"},
             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
         ],
         "name": {"$regex": ".*processed.*"},
     },
-    "raw, 'fib' in 'data_description.modality'": {
+    "{raw, 'fib' in 'data_description.modality'}": {
         "data_description.modality.abbreviation": "fib",
         "name": {"$not": {"$regex": ".*processed.*"}},
     },
-    "raw, 'fib' in 'rig.modalities'": {
+    "{raw, 'fib' in 'rig.modalities'}": {
         "rig.modalities.abbreviation": "fib",
         "name": {"$not": {"$regex": ".*processed.*"}},        
     },
-    "raw, 'fib' in 'session.data_streams'": {
+    "{raw, 'fib' in 'session.data_streams'}": {
         "session.data_streams.stream_modalities.abbreviation": "fib",
         "name": {"$not": {"$regex": ".*processed.*"}},
     }
@@ -85,13 +85,15 @@
 
 @st.cache_data(ttl=3600 * 12)  # Cache the df_docDB up to 12 hours
 def query_sessions_from_docDB(query):
-    results = client.retrieve_docdb_records(
+    return client.retrieve_docdb_records(
         filter_query=query,
-        projection={"name": 1, "_id": 0},
+        projection={
+            "_id": 0,
+            "name": 1,
+            "rig.rig_id": 1,
+            "session.experimenter_full_name": 1,
+        },
     )
-    
-    sessions = [re.sub(r'_processed.*$', '', r["name"]) for r in results]
-    return sessions
 
 
 def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
@@ -124,34 +126,34 @@ def app():
         key="selected_queries",
     )
 
-    # Generage venn diagram of the selected queries
-    query_results = {key: query_sessions_from_docDB(QUERY_PRESET[key]) for key in selected_queries}
+    # Generate combined dataframe
+    dfs = []
 
-    # -- Show dataframe that summarize the selected queries --
-    st.markdown(f"#### Merged dataframe")
+    progress_bar = st.progress(0, text="Querying docDB")
+    
+    for n, key in enumerate(selected_queries):
+        progress_bar.progress(n / len(selected_queries), text=f"Querying docDB: {key}")
 
-    dfs = []
-    for query_name, query_result in query_results.items():
-        df = pd.DataFrame(query_result, columns=[f"nwb_name"])
+        # Query docDB
+        query_result = query_sessions_from_docDB(QUERY_PRESET[key])
+        df = pd.json_normalize(query_result)
 
         # Create index that can be joined with other dfs and my main df
-        df[["subject_id", "session_date", "nwb_suffix"]] = df[f"nwb_name"].apply(
+        df[["subject_id", "session_date", "nwb_suffix"]] = df[f"name"].apply(
             lambda x: pd.Series(split_nwb_name(x))
         )
         
-        df[query_name] = True
+        df[key] = True
         df.set_index(["subject_id", "session_date", "nwb_suffix"], inplace=True)
         dfs.append(df)
-
-    # # Out merge all dataframes
-    # df_merged = reduce(
-    #     lambda left, right: left.join(right, how="outer"), dfs
-    # )
+        
+        progress_bar.progress((n+1) / len(selected_queries), text=f"Querying docDB: {key}")
     
     df_merged = dfs[0]
     for df in dfs[1:]:
-        df_merged = df_merged.combine_first(df)
+        df_merged = df_merged.combine_first(df)  # Combine nwb_names
     
+    st.markdown(f"#### Merged dataframe")
     st.write(df_merged)
     download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
 

From 973b02009441a1ea399d8316d27ddd0e3ad6b7ea Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Sat, 21 Dec 2024 02:09:18 +0000
Subject: [PATCH 10/68] put venn back

---
 code/pages/0_docDB inventory.py | 69 +++++++++++++++++++++------------
 1 file changed, 44 insertions(+), 25 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 69dd112..71aca91 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -108,27 +108,11 @@ def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
         mime='text/csv'
     )
 
-def app():
-
-    #
-    with st.expander("Show docDB queries", expanded=False):
-        st.write('See how to use these queries [here](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
-        for key, query in QUERY_PRESET.items():
-            st.markdown(f"**{key}**")
-            st.code(query)
-
-    # Multiselect for selecting queries up to three
-    query_keys = list(QUERY_PRESET.keys())
-    selected_queries = st.multiselect(
-        "Select queries to filter sessions",
-        query_keys,
-        default=query_keys[:3],
-        key="selected_queries",
-    )
 
-    # Generate combined dataframe
+def get_merged_queries(selected_queries):
+    """ Get merged queries from selected queries """
+    
     dfs = []
-
     progress_bar = st.progress(0, text="Querying docDB")
     
     for n, key in enumerate(selected_queries):
@@ -156,20 +140,55 @@ def app():
     st.markdown(f"#### Merged dataframe")
     st.write(df_merged)
     download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
+    
+    return df_merged
 
-    # -- Show venn --
+def venn(df, columns_to_venn):
+    """ Show venn diagram """
+    if len(columns_to_venn) > 3:
+        st.write("Venn diagram only supports up to 3 columns.")
+        return
+    
     fig, ax = plt.subplots()
-    if len(selected_queries) == 2:
+    
+    if len(columns_to_venn) == 2:
         venn2(
-            [query_results[key] for key in selected_queries],
-            set_labels=selected_queries,
+            [set(df.index[df[col]==True]) for col in columns_to_venn],
+            set_labels=columns_to_venn,
         )
     else:
         venn3(
-            [query_results[key] for key in selected_queries],
-            set_labels=selected_queries,
+            [set(df.index[df[col]==True]) for col in columns_to_venn],
+            set_labels=columns_to_venn,
         )
+    return fig
 
+
+def app():
+
+    #
+    with st.expander("Show docDB queries", expanded=False):
+        st.write('See how to use these queries [here](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
+        for key, query in QUERY_PRESET.items():
+            st.markdown(f"**{key}**")
+            st.code(query)
+
+    # Multiselect for selecting queries up to three
+    query_keys = list(QUERY_PRESET.keys())
+    selected_queries = st.multiselect(
+        "Select queries to filter sessions",
+        query_keys,
+        default=query_keys[:3],
+        key="selected_queries",
+    )
+
+    # Generate combined dataframe
+    df_merged = get_merged_queries(selected_queries)
+    
+    columns_to_venn = selected_queries
+
+    # -- Show venn --
+    fig = venn(df_merged, columns_to_venn)
     st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
 
     # df = load_data_from_docDB()

From 0fa14a83b83521fd0bb6e51b486ff4f981d0c8c4 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 24 Dec 2024 04:35:36 +0000
Subject: [PATCH 11/68] use ThreadPoolExecutor to query docDB in parallel

---
 code/pages/0_docDB inventory.py | 50 +++++++++++++++++++++------------
 1 file changed, 32 insertions(+), 18 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 71aca91..8ba800e 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -3,7 +3,8 @@
 
 import logging
 import re
-from functools import reduce
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import threading
 
 from matplotlib_venn import venn2, venn3
 import matplotlib.pyplot as plt
@@ -108,6 +109,21 @@ def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
         mime='text/csv'
     )
 
+@st.cache_data(ttl=3600 * 12)  # Cache the df_docDB up to 12 hours
+def query_single_query(key):
+    """ Query a single query from QUERY_PRESET and process the result """
+    query_result = query_sessions_from_docDB(QUERY_PRESET[key])
+    df = pd.json_normalize(query_result)
+    
+    # Create index that can be joined with other dfs and main df
+    df[["subject_id", "session_date", "nwb_suffix"]] = df[f"name"].apply(
+        lambda x: pd.Series(split_nwb_name(x))
+    )
+    df[key] = True
+    df.set_index(["subject_id", "session_date", "nwb_suffix"], inplace=True)
+    
+    return df
+
 
 def get_merged_queries(selected_queries):
     """ Get merged queries from selected queries """
@@ -115,24 +131,19 @@ def get_merged_queries(selected_queries):
     dfs = []
     progress_bar = st.progress(0, text="Querying docDB")
     
-    for n, key in enumerate(selected_queries):
-        progress_bar.progress(n / len(selected_queries), text=f"Querying docDB: {key}")
-
-        # Query docDB
-        query_result = query_sessions_from_docDB(QUERY_PRESET[key])
-        df = pd.json_normalize(query_result)
-
-        # Create index that can be joined with other dfs and my main df
-        df[["subject_id", "session_date", "nwb_suffix"]] = df[f"name"].apply(
-            lambda x: pd.Series(split_nwb_name(x))
-        )
-        
-        df[key] = True
-        df.set_index(["subject_id", "session_date", "nwb_suffix"], inplace=True)
-        dfs.append(df)
-        
-        progress_bar.progress((n+1) / len(selected_queries), text=f"Querying docDB: {key}")
+    with ThreadPoolExecutor(max_workers=len(selected_queries)) as executor:
+        future_to_query = {executor.submit(query_single_query, key): key for key in selected_queries}
+        for i, future in enumerate(as_completed(future_to_query), 1):
+            key = future_to_query[future]
+            try:
+                df = future.result()
+                dfs.append(df)
+            except Exception as e:
+                logging.error(f"Error querying {key}: {e}")
+            finally:
+                progress_bar.progress(i / len(selected_queries), text=f"Querying docDB ... {key} done!")           
     
+    # Combine queried dfs
     df_merged = dfs[0]
     for df in dfs[1:]:
         df_merged = df_merged.combine_first(df)  # Combine nwb_names
@@ -183,7 +194,10 @@ def app():
     )
 
     # Generate combined dataframe
+    import time
+    start = time.time()
     df_merged = get_merged_queries(selected_queries)
+    st.write(f"Time taken to query and merge: {time.time() - start:.2f} seconds")
     
     columns_to_venn = selected_queries
 

From 60d2d6aeeb60a5afb99997938cb05841c4e162e3 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 24 Dec 2024 04:57:25 +0000
Subject: [PATCH 12/68] fetch all queries

---
 code/pages/0_docDB inventory.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 8ba800e..f23cb7e 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -125,14 +125,14 @@ def query_single_query(key):
     return df
 
 
-def get_merged_queries(selected_queries):
+def get_merged_queries(queries_to_merge):
     """ Get merged queries from selected queries """
     
     dfs = []
     progress_bar = st.progress(0, text="Querying docDB")
     
-    with ThreadPoolExecutor(max_workers=len(selected_queries)) as executor:
-        future_to_query = {executor.submit(query_single_query, key): key for key in selected_queries}
+    with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
+        future_to_query = {executor.submit(query_single_query, key): key for key in queries_to_merge}
         for i, future in enumerate(as_completed(future_to_query), 1):
             key = future_to_query[future]
             try:
@@ -141,7 +141,7 @@ def get_merged_queries(selected_queries):
             except Exception as e:
                 logging.error(f"Error querying {key}: {e}")
             finally:
-                progress_bar.progress(i / len(selected_queries), text=f"Querying docDB ... {key} done!")           
+                progress_bar.progress(i / len(queries_to_merge), text=f"Fetching  docDB ({i}/{len(queries_to_merge)})... {key} done!")           
     
     # Combine queried dfs
     df_merged = dfs[0]
@@ -196,7 +196,7 @@ def app():
     # Generate combined dataframe
     import time
     start = time.time()
-    df_merged = get_merged_queries(selected_queries)
+    df_merged = get_merged_queries(queries_to_merge=QUERY_PRESET.keys())
     st.write(f"Time taken to query and merge: {time.time() - start:.2f} seconds")
     
     columns_to_venn = selected_queries

From a1e22e67102b3cc0752673250a02289953f08483 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 24 Dec 2024 06:01:28 +0000
Subject: [PATCH 13/68] put query to sidebar

---
 code/pages/0_docDB inventory.py | 43 +++++++++++++++------------------
 1 file changed, 19 insertions(+), 24 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index f23cb7e..59e4840 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -5,6 +5,7 @@
 import re
 from concurrent.futures import ThreadPoolExecutor, as_completed
 import threading
+import json
 
 from matplotlib_venn import venn2, venn3
 import matplotlib.pyplot as plt
@@ -84,7 +85,6 @@
     }
 }
 
-@st.cache_data(ttl=3600 * 12)  # Cache the df_docDB up to 12 hours
 def query_sessions_from_docDB(query):
     return client.retrieve_docdb_records(
         filter_query=query,
@@ -94,9 +94,9 @@ def query_sessions_from_docDB(query):
             "rig.rig_id": 1,
             "session.experimenter_full_name": 1,
         },
+        paginate=False,
     )
 
-
 def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
     """ Download df as csv """
     csv = df.to_csv(index=True)
@@ -109,7 +109,6 @@ def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
         mime='text/csv'
     )
 
-@st.cache_data(ttl=3600 * 12)  # Cache the df_docDB up to 12 hours
 def query_single_query(key):
     """ Query a single query from QUERY_PRESET and process the result """
     query_result = query_sessions_from_docDB(QUERY_PRESET[key])
@@ -124,13 +123,13 @@ def query_single_query(key):
     
     return df
 
-
+@st.cache_data(ttl=3600 * 24)
 def get_merged_queries(queries_to_merge):
     """ Get merged queries from selected queries """
     
     dfs = []
-    progress_bar = st.progress(0, text="Querying docDB")
     
+    # Fetch data in parallel
     with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
         future_to_query = {executor.submit(query_single_query, key): key for key in queries_to_merge}
         for i, future in enumerate(as_completed(future_to_query), 1):
@@ -139,19 +138,13 @@ def get_merged_queries(queries_to_merge):
                 df = future.result()
                 dfs.append(df)
             except Exception as e:
-                logging.error(f"Error querying {key}: {e}")
-            finally:
-                progress_bar.progress(i / len(queries_to_merge), text=f"Fetching  docDB ({i}/{len(queries_to_merge)})... {key} done!")           
+                print(f"Error querying {key}: {e}")
     
     # Combine queried dfs
     df_merged = dfs[0]
     for df in dfs[1:]:
         df_merged = df_merged.combine_first(df)  # Combine nwb_names
-    
-    st.markdown(f"#### Merged dataframe")
-    st.write(df_merged)
-    download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
-    
+        
     return df_merged
 
 def venn(df, columns_to_venn):
@@ -178,11 +171,21 @@ def venn(df, columns_to_venn):
 def app():
 
     #
-    with st.expander("Show docDB queries", expanded=False):
-        st.write('See how to use these queries [here](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
+    with st.sidebar:
+        st.markdown('## docDB query presets ')
+        st.markdown('#### See how to use these queries [in this doc.](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
         for key, query in QUERY_PRESET.items():
             st.markdown(f"**{key}**")
-            st.code(query)
+            # Turn query to json with indent=4
+            query_json = json.dumps(query, indent=4)
+            st.code(query_json)
+
+    # Generate combined dataframe
+    df_merged = get_merged_queries(queries_to_merge=list(QUERY_PRESET.keys()))
+    
+    st.markdown(f"#### Merged dataframe")
+    st.write(df_merged)
+    download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
 
     # Multiselect for selecting queries up to three
     query_keys = list(QUERY_PRESET.keys())
@@ -192,16 +195,8 @@ def app():
         default=query_keys[:3],
         key="selected_queries",
     )
-
-    # Generate combined dataframe
-    import time
-    start = time.time()
-    df_merged = get_merged_queries(queries_to_merge=QUERY_PRESET.keys())
-    st.write(f"Time taken to query and merge: {time.time() - start:.2f} seconds")
     
     columns_to_venn = selected_queries
-
-    # -- Show venn --
     fig = venn(df_merged, columns_to_venn)
     st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
 

From 164766799d4fb356b43127b0e44e33ec3cae33de Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 24 Dec 2024 17:05:37 +0000
Subject: [PATCH 14/68] minor tweak of sidebar

---
 code/pages/0_docDB inventory.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 59e4840..edf506d 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -175,10 +175,10 @@ def app():
         st.markdown('## docDB query presets ')
         st.markdown('#### See how to use these queries [in this doc.](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
         for key, query in QUERY_PRESET.items():
-            st.markdown(f"**{key}**")
-            # Turn query to json with indent=4
-            query_json = json.dumps(query, indent=4)
-            st.code(query_json)
+            with st.expander(f"{key}"):
+                # Turn query to json with indent=4
+                query_json = json.dumps(query, indent=4)
+                st.code(query_json)
 
     # Generate combined dataframe
     df_merged = get_merged_queries(queries_to_merge=list(QUERY_PRESET.keys()))

From d9e0933b800d730bdac7bc71ed5cbbade349686c Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 24 Dec 2024 17:19:11 +0000
Subject: [PATCH 15/68] add query

---
 code/pages/0_docDB inventory.py | 36 ++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 16 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index edf506d..4ead19b 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -48,14 +48,6 @@
         ],
         "name": {"$not": {"$regex": ".*processed.*"}},
     },
-    "{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}": {
-        "$or":[
-            {"session.data_streams.software.name": "dynamic-foraging-task"},
-            {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-        ],
-        "data_description.modality.abbreviation": "ecephys",
-        "name": {"$not": {"$regex": ".*processed.*"}},        
-    },
     "{raw, 'dynamic_foraging' in data_streams software name}": {
         "session.data_streams.software.name": "dynamic-foraging-task",
         "name": {"$not": {"$regex": ".*processed.*"}},
@@ -64,13 +56,6 @@
         "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
         "name": {"$not": {"$regex": ".*processed.*"}},
     },
-    "{processed, 'dynamic_foraging' in ANY software name}": {
-        "$or":[
-            {"session.data_streams.software.name": "dynamic-foraging-task"},
-            {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-        ],
-        "name": {"$regex": ".*processed.*"},
-    },
     "{raw, 'fib' in 'data_description.modality'}": {
         "data_description.modality.abbreviation": "fib",
         "name": {"$not": {"$regex": ".*processed.*"}},
@@ -82,7 +67,26 @@
     "{raw, 'fib' in 'session.data_streams'}": {
         "session.data_streams.stream_modalities.abbreviation": "fib",
         "name": {"$not": {"$regex": ".*processed.*"}},
-    }
+    },
+    "{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}": {
+        "$or":[
+            {"session.data_streams.software.name": "dynamic-foraging-task"},
+            {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+        ],
+        "data_description.modality.abbreviation": "ecephys",
+        "name": {"$not": {"$regex": ".*processed.*"}},        
+    },
+    "{raw, 'FIP' in name}":{
+        "name": {"$regex": "^FIP.*"},
+        "name": {"$not": {"$regex": ".*processed.*"}}
+    },
+    "{processed, 'dynamic_foraging' in ANY software name}": {
+        "$or":[
+            {"session.data_streams.software.name": "dynamic-foraging-task"},
+            {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+        ],
+        "name": {"$regex": ".*processed.*"},
+    },
 }
 
 def query_sessions_from_docDB(query):

From 2471bc5e114aa8f42fdaa2c44e5aaaaff6b3bcb9 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 24 Dec 2024 17:29:56 +0000
Subject: [PATCH 16/68] refactor PRESET

---
 code/pages/0_docDB inventory.py | 141 ++++++++++++++++----------------
 1 file changed, 71 insertions(+), 70 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 4ead19b..9e5c1d0 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -1,6 +1,3 @@
-'''Migrated from David's toy app https://codeocean.allenneuraldynamics.org/capsule/9532498/tree
-'''
-
 import logging
 import re
 from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -40,58 +37,69 @@
 
 client = load_client()
 
-QUERY_PRESET = {
-    "{raw, 'dynamic_foraging' in ANY software name}": {
-        "$or":[
-            {"session.data_streams.software.name": "dynamic-foraging-task"},
-            {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-        ],
-        "name": {"$not": {"$regex": ".*processed.*"}},
-    },
-    "{raw, 'dynamic_foraging' in data_streams software name}": {
-        "session.data_streams.software.name": "dynamic-foraging-task",
-        "name": {"$not": {"$regex": ".*processed.*"}},
-    },
-    "{raw, 'dynamic_foraging' in stimulus_epochs software name}": {
-        "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
-        "name": {"$not": {"$regex": ".*processed.*"}},
-    },
-    "{raw, 'fib' in 'data_description.modality'}": {
-        "data_description.modality.abbreviation": "fib",
-        "name": {"$not": {"$regex": ".*processed.*"}},
-    },
-    "{raw, 'fib' in 'rig.modalities'}": {
-        "rig.modalities.abbreviation": "fib",
-        "name": {"$not": {"$regex": ".*processed.*"}},        
-    },
-    "{raw, 'fib' in 'session.data_streams'}": {
-        "session.data_streams.stream_modalities.abbreviation": "fib",
-        "name": {"$not": {"$regex": ".*processed.*"}},
-    },
-    "{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}": {
-        "$or":[
-            {"session.data_streams.software.name": "dynamic-foraging-task"},
-            {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-        ],
-        "data_description.modality.abbreviation": "ecephys",
-        "name": {"$not": {"$regex": ".*processed.*"}},        
-    },
-    "{raw, 'FIP' in name}":{
-        "name": {"$regex": "^FIP.*"},
-        "name": {"$not": {"$regex": ".*processed.*"}}
-    },
-    "{processed, 'dynamic_foraging' in ANY software name}": {
-        "$or":[
-            {"session.data_streams.software.name": "dynamic-foraging-task"},
-            {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-        ],
-        "name": {"$regex": ".*processed.*"},
-    },
-}
+QUERY_PRESET = [
+    {"alias": "{raw, 'dynamic_foraging' in ANY software name}",
+     "filter": {
+         "$or":[
+             {"session.data_streams.software.name": "dynamic-foraging-task"},
+             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+         ],
+         "name": {"$not": {"$regex": ".*processed.*"}},
+     }},
+    {"alias": "{raw, 'dynamic_foraging' in data_streams software name}",
+     "filter": {
+         "session.data_streams.software.name": "dynamic-foraging-task",
+         "name": {"$not": {"$regex": ".*processed.*"}},
+     }},
+    {"alias": "{raw, 'dynamic_foraging' in stimulus_epochs software name}",
+     "filter": {
+         "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
+         "name": {"$not": {"$regex": ".*processed.*"}},
+     }},
+    {"alias": "{raw, 'fib' in 'data_description.modality'}",
+     "filter": {
+         "data_description.modality.abbreviation": "fib",
+         "name": {"$not": {"$regex": ".*processed.*"}},
+     }},
+    {"alias": "{raw, 'fib' in 'rig.modalities'}",
+     "filter": {
+         "rig.modalities.abbreviation": "fib",
+         "name": {"$not": {"$regex": ".*processed.*"}},        
+     }},
+    {"alias": "{raw, 'fib' in 'session.data_streams'}",
+     "filter": {
+         "session.data_streams.stream_modalities.abbreviation": "fib",
+         "name": {"$not": {"$regex": ".*processed.*"}},
+     }},
+    {"alias": "{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
+     "filter": {
+         "$or":[
+             {"session.data_streams.software.name": "dynamic-foraging-task"},
+             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+         ],
+         "data_description.modality.abbreviation": "ecephys",
+         "name": {"$not": {"$regex": ".*processed.*"}},        
+     }},
+    {"alias": "{raw, 'FIP' in name}",
+     "filter": {
+        "name": {
+            "$regex": "^FIP.*",
+            "$not": {"$regex": ".*processed.*"}
+        }
+    }},
+    {"alias": "{processed, 'dynamic_foraging' in ANY software name}",
+     "filter": {
+         "$or":[
+             {"session.data_streams.software.name": "dynamic-foraging-task"},
+             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+         ],
+         "name": {"$regex": ".*processed.*"},
+     }},
+]
 
 def query_sessions_from_docDB(query):
     return client.retrieve_docdb_records(
-        filter_query=query,
+        filter_query=query["filter"],
         projection={
             "_id": 0,
             "name": 1,
@@ -115,14 +123,14 @@ def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
 
 def query_single_query(key):
     """ Query a single query from QUERY_PRESET and process the result """
-    query_result = query_sessions_from_docDB(QUERY_PRESET[key])
+    query_result = query_sessions_from_docDB(key)
     df = pd.json_normalize(query_result)
     
     # Create index that can be joined with other dfs and main df
     df[["subject_id", "session_date", "nwb_suffix"]] = df[f"name"].apply(
         lambda x: pd.Series(split_nwb_name(x))
     )
-    df[key] = True
+    df[key["alias"]] = True
     df.set_index(["subject_id", "session_date", "nwb_suffix"], inplace=True)
     
     return df
@@ -148,6 +156,9 @@ def get_merged_queries(queries_to_merge):
     df_merged = dfs[0]
     for df in dfs[1:]:
         df_merged = df_merged.combine_first(df)  # Combine nwb_names
+    
+    # Recover the order of QUERY_PRESET
+    df_merged = df_merged.reindex(columns=[query["alias"] for query in queries_to_merge])
         
     return df_merged
 
@@ -178,21 +189,21 @@ def app():
     with st.sidebar:
         st.markdown('## docDB query presets ')
         st.markdown('#### See how to use these queries [in this doc.](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
-        for key, query in QUERY_PRESET.items():
-            with st.expander(f"{key}"):
+        for query in QUERY_PRESET:
+            with st.expander(f"{query['alias']}"):
                 # Turn query to json with indent=4
-                query_json = json.dumps(query, indent=4)
+                query_json = json.dumps(query["filter"], indent=4)
                 st.code(query_json)
 
     # Generate combined dataframe
-    df_merged = get_merged_queries(queries_to_merge=list(QUERY_PRESET.keys()))
+    df_merged = get_merged_queries(queries_to_merge=QUERY_PRESET)
     
     st.markdown(f"#### Merged dataframe")
     st.write(df_merged)
     download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
 
     # Multiselect for selecting queries up to three
-    query_keys = list(QUERY_PRESET.keys())
+    query_keys = [query["alias"] for query in QUERY_PRESET]
     selected_queries = st.multiselect(
         "Select queries to filter sessions",
         query_keys,
@@ -204,16 +215,6 @@ def app():
     fig = venn(df_merged, columns_to_venn)
     st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
 
-    # df = load_data_from_docDB()
-
-    # st.markdown(f'### Note: the dataframe showing here has been merged in to the master table on the Home page!')
-
-    # dynamic_filters = DynamicFilters(
-    #     df=df,
-    #     filters=['subject_id', 'subject_genotype'])
-    # dynamic_filters.display_filters()
-    # dynamic_filters.display_df()
-
 
 if __name__ == "__main__":
-    app()
+    app()
\ No newline at end of file

From 6566026de508a4c5c109421173733734d0563c60 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 24 Dec 2024 17:35:44 +0000
Subject: [PATCH 17/68] add number of returns

---
 code/pages/0_docDB inventory.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 9e5c1d0..6db7f19 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -185,20 +185,22 @@ def venn(df, columns_to_venn):
 
 def app():
 
-    #
+    # Generate combined dataframe
+    df_merged = get_merged_queries(queries_to_merge=QUERY_PRESET)
+    
+    # Sidebar
     with st.sidebar:
         st.markdown('## docDB query presets ')
         st.markdown('#### See how to use these queries [in this doc.](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
         for query in QUERY_PRESET:
-            with st.expander(f"{query['alias']}"):
+            results_this = df_merged[query["alias"]].sum()
+            with st.expander(f"n = {results_this}, {query['alias']}"):
                 # Turn query to json with indent=4
                 query_json = json.dumps(query["filter"], indent=4)
                 st.code(query_json)
 
-    # Generate combined dataframe
-    df_merged = get_merged_queries(queries_to_merge=QUERY_PRESET)
     
-    st.markdown(f"#### Merged dataframe")
+    st.markdown(f"#### Merged dataframe (n = {len(df_merged)})")
     st.write(df_merged)
     download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
 

From e93d88a4b6b1ace096014616be80e9cde0be6f04 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 31 Dec 2024 01:30:11 +0000
Subject: [PATCH 18/68] find multiple sessions per day

---
 code/pages/0_docDB inventory.py | 54 +++++++++++++++++++++------------
 1 file changed, 34 insertions(+), 20 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 6db7f19..2adaf36 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -121,46 +121,60 @@ def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
         mime='text/csv'
     )
 
-def query_single_query(key):
-    """ Query a single query from QUERY_PRESET and process the result """
+def fetch_single_query(key):
+    """ Query a single query from QUERY_PRESET and process the result 
+    
+    Return: 
+    - df: the full dataframe
+    - df_multi_sessions_per_day: the dataframe with multiple sessions per day
+    
+    """
     query_result = query_sessions_from_docDB(key)
     df = pd.json_normalize(query_result)
-    
+
     # Create index that can be joined with other dfs and main df
     df[["subject_id", "session_date", "nwb_suffix"]] = df[f"name"].apply(
         lambda x: pd.Series(split_nwb_name(x))
     )
     df[key["alias"]] = True
-    df.set_index(["subject_id", "session_date", "nwb_suffix"], inplace=True)
+    df = df.set_index(["subject_id", "session_date", "nwb_suffix"]).sort_index()
     
-    return df
+
+    # Get cases where one mouse has multiple records per day
+    subject_date = df.index.to_frame(index=False)[["subject_id", "session_date"]]
+    df_multi_sessions_per_day = df[subject_date.duplicated(keep=False).values]
+
+    return df, df_multi_sessions_per_day
 
 @st.cache_data(ttl=3600 * 24)
-def get_merged_queries(queries_to_merge):
+def fetch_all_queries_from_docDB(queries_to_merge):
     """ Get merged queries from selected queries """
-    
-    dfs = []
-    
+
+    dfs = {}
+
     # Fetch data in parallel
     with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
-        future_to_query = {executor.submit(query_single_query, key): key for key in queries_to_merge}
+        future_to_query = {executor.submit(fetch_single_query, key): key for key in queries_to_merge}
         for i, future in enumerate(as_completed(future_to_query), 1):
             key = future_to_query[future]
             try:
-                df = future.result()
-                dfs.append(df)
+                df, df_multi_sessions_per_day = future.result()
+                dfs[key["alias"]] = {
+                    "df": df,
+                    "df_multi_sessions_per_day": df_multi_sessions_per_day,
+                }
             except Exception as e:
                 print(f"Error querying {key}: {e}")
-    
+
     # Combine queried dfs
-    df_merged = dfs[0]
-    for df in dfs[1:]:
+    df_merged = dfs[queries_to_merge[0]["alias"]]["df"]
+    for df in [dfs[query["alias"]]["df"] for query in queries_to_merge[1:]]:
         df_merged = df_merged.combine_first(df)  # Combine nwb_names
-    
+
     # Recover the order of QUERY_PRESET
     df_merged = df_merged.reindex(columns=[query["alias"] for query in queries_to_merge])
-        
-    return df_merged
+
+    return df_merged, dfs
 
 def venn(df, columns_to_venn):
     """ Show venn diagram """
@@ -186,7 +200,7 @@ def venn(df, columns_to_venn):
 def app():
 
     # Generate combined dataframe
-    df_merged = get_merged_queries(queries_to_merge=QUERY_PRESET)
+    df_merged, dfs = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
     
     # Sidebar
     with st.sidebar:
@@ -219,4 +233,4 @@ def app():
 
 
 if __name__ == "__main__":
-    app()
\ No newline at end of file
+    app()

From fc5cfada0d9c9656033a24e9f8f41a301764ce96 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 31 Dec 2024 01:30:40 +0000
Subject: [PATCH 19/68] remove invalid subject_id

---
 code/pages/0_docDB inventory.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 2adaf36..a55fb71 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -139,6 +139,8 @@ def fetch_single_query(key):
     df[key["alias"]] = True
     df = df.set_index(["subject_id", "session_date", "nwb_suffix"]).sort_index()
     
+    # Remove invalid subject_id
+    df = df[df.index.get_level_values("subject_id").astype(int) > 300000]
 
     # Get cases where one mouse has multiple records per day
     subject_date = df.index.to_frame(index=False)[["subject_id", "session_date"]]

From 56d79a1090a8a19f9b1be3949ae1e83f830d3bef Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Thu, 2 Jan 2025 20:08:34 +0000
Subject: [PATCH 20/68] mark duplicates and return a duplicates-removed df

---
 code/pages/0_docDB inventory.py | 122 ++++++++++++++++++--------------
 1 file changed, 70 insertions(+), 52 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index a55fb71..0fc39da 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -51,50 +51,50 @@
          "session.data_streams.software.name": "dynamic-foraging-task",
          "name": {"$not": {"$regex": ".*processed.*"}},
      }},
-    {"alias": "{raw, 'dynamic_foraging' in stimulus_epochs software name}",
-     "filter": {
-         "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
-         "name": {"$not": {"$regex": ".*processed.*"}},
-     }},
-    {"alias": "{raw, 'fib' in 'data_description.modality'}",
-     "filter": {
-         "data_description.modality.abbreviation": "fib",
-         "name": {"$not": {"$regex": ".*processed.*"}},
-     }},
-    {"alias": "{raw, 'fib' in 'rig.modalities'}",
-     "filter": {
-         "rig.modalities.abbreviation": "fib",
-         "name": {"$not": {"$regex": ".*processed.*"}},        
-     }},
-    {"alias": "{raw, 'fib' in 'session.data_streams'}",
-     "filter": {
-         "session.data_streams.stream_modalities.abbreviation": "fib",
-         "name": {"$not": {"$regex": ".*processed.*"}},
-     }},
-    {"alias": "{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
-     "filter": {
-         "$or":[
-             {"session.data_streams.software.name": "dynamic-foraging-task"},
-             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-         ],
-         "data_description.modality.abbreviation": "ecephys",
-         "name": {"$not": {"$regex": ".*processed.*"}},        
-     }},
-    {"alias": "{raw, 'FIP' in name}",
-     "filter": {
-        "name": {
-            "$regex": "^FIP.*",
-            "$not": {"$regex": ".*processed.*"}
-        }
-    }},
-    {"alias": "{processed, 'dynamic_foraging' in ANY software name}",
-     "filter": {
-         "$or":[
-             {"session.data_streams.software.name": "dynamic-foraging-task"},
-             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-         ],
-         "name": {"$regex": ".*processed.*"},
-     }},
+    # {"alias": "{raw, 'dynamic_foraging' in stimulus_epochs software name}",
+    #  "filter": {
+    #      "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
+    #      "name": {"$not": {"$regex": ".*processed.*"}},
+    #  }},
+    # {"alias": "{raw, 'fib' in 'data_description.modality'}",
+    #  "filter": {
+    #      "data_description.modality.abbreviation": "fib",
+    #      "name": {"$not": {"$regex": ".*processed.*"}},
+    #  }},
+    # {"alias": "{raw, 'fib' in 'rig.modalities'}",
+    #  "filter": {
+    #      "rig.modalities.abbreviation": "fib",
+    #      "name": {"$not": {"$regex": ".*processed.*"}},        
+    #  }},
+    # {"alias": "{raw, 'fib' in 'session.data_streams'}",
+    #  "filter": {
+    #      "session.data_streams.stream_modalities.abbreviation": "fib",
+    #      "name": {"$not": {"$regex": ".*processed.*"}},
+    #  }},
+    # {"alias": "{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
+    #  "filter": {
+    #      "$or":[
+    #          {"session.data_streams.software.name": "dynamic-foraging-task"},
+    #          {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+    #      ],
+    #      "data_description.modality.abbreviation": "ecephys",
+    #      "name": {"$not": {"$regex": ".*processed.*"}},        
+    #  }},
+    # {"alias": "{raw, 'FIP' in name}",
+    #  "filter": {
+    #     "name": {
+    #         "$regex": "^FIP.*",
+    #         "$not": {"$regex": ".*processed.*"}
+    #     }
+    # }},
+    # {"alias": "{processed, 'dynamic_foraging' in ANY software name}",
+    #  "filter": {
+    #      "$or":[
+    #          {"session.data_streams.software.name": "dynamic-foraging-task"},
+    #          {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+    #      ],
+    #      "name": {"$regex": ".*processed.*"},
+    #  }},
 ]
 
 def query_sessions_from_docDB(query):
@@ -145,8 +145,15 @@ def fetch_single_query(key):
     # Get cases where one mouse has multiple records per day
     subject_date = df.index.to_frame(index=False)[["subject_id", "session_date"]]
     df_multi_sessions_per_day = df[subject_date.duplicated(keep=False).values]
+    
+    # Create a new column to mark duplicates
+    df.loc[df_multi_sessions_per_day.index, "multiple_sessions_per_day"] = True
+    
+    # Also return a dataframe that removes duplicates (only keep subject_id and session_date as indices)
+    df_removed_multi_sessions_per_day = df[~subject_date.duplicated(keep="last").values]
+    df_removed_multi_sessions_per_day
 
-    return df, df_multi_sessions_per_day
+    return df, df_multi_sessions_per_day, df_removed_multi_sessions_per_day
 
 @st.cache_data(ttl=3600 * 24)
 def fetch_all_queries_from_docDB(queries_to_merge):
@@ -173,8 +180,8 @@ def fetch_all_queries_from_docDB(queries_to_merge):
     for df in [dfs[query["alias"]]["df"] for query in queries_to_merge[1:]]:
         df_merged = df_merged.combine_first(df)  # Combine nwb_names
 
-    # Recover the order of QUERY_PRESET
-    df_merged = df_merged.reindex(columns=[query["alias"] for query in queries_to_merge])
+    # Recover the column order of QUERY_PRESET
+    # df_merged = df_merged.reindex(columns=[query["alias"] for query in queries_to_merge])
 
     return df_merged, dfs
 
@@ -203,19 +210,30 @@ def app():
 
     # Generate combined dataframe
     df_merged, dfs = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
-    
+
     # Sidebar
     with st.sidebar:
         st.markdown('## docDB query presets ')
         st.markdown('#### See how to use these queries [in this doc.](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
         for query in QUERY_PRESET:
-            results_this = df_merged[query["alias"]].sum()
-            with st.expander(f"n = {results_this}, {query['alias']}"):
+            with st.expander(f"{query['alias']}"):
                 # Turn query to json with indent=4
                 query_json = json.dumps(query["filter"], indent=4)
                 st.code(query_json)
 
-    
+                # Show records
+                df = dfs[query["alias"]]["df"]
+                n_records = df_merged[query["alias"]].sum()
+
+                n_multiple_sessions_per_day = len(
+                    dfs[query["alias"]]["df_multi_sessions_per_day"].index.droplevel("nwb_suffix").unique()
+                )
+                st.markdown(
+                    f"{n_records} from merged, {len(df)} from df, {n_multiple_sessions_per_day} has multiple sessions per day"
+                )
+
+                
+
     st.markdown(f"#### Merged dataframe (n = {len(df_merged)})")
     st.write(df_merged)
     download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
@@ -228,7 +246,7 @@ def app():
         default=query_keys[:3],
         key="selected_queries",
     )
-    
+
     columns_to_venn = selected_queries
     fig = venn(df_merged, columns_to_venn)
     st.columns([1, 1])[0].pyplot(fig, use_container_width=True)

From 0a0728cda8d1380e248f1ddc5d14d7f537a35d71 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Thu, 2 Jan 2025 20:50:15 +0000
Subject: [PATCH 21/68] add df_unique_mouse_date; add download buttons

---
 code/pages/0_docDB inventory.py | 72 ++++++++++++++++++++++++---------
 1 file changed, 52 insertions(+), 20 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 0fc39da..8315e31 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -127,7 +127,7 @@ def fetch_single_query(key):
     Return: 
     - df: the full dataframe
     - df_multi_sessions_per_day: the dataframe with multiple sessions per day
-    
+    - df_unique_mouse_date: the dataframe with multiple sessions per day combined
     """
     query_result = query_sessions_from_docDB(key)
     df = pd.json_normalize(query_result)
@@ -138,22 +138,26 @@ def fetch_single_query(key):
     )
     df[key["alias"]] = True
     df = df.set_index(["subject_id", "session_date", "nwb_suffix"]).sort_index()
-    
+
     # Remove invalid subject_id
     df = df[df.index.get_level_values("subject_id").astype(int) > 300000]
 
     # Get cases where one mouse has multiple records per day
     subject_date = df.index.to_frame(index=False)[["subject_id", "session_date"]]
     df_multi_sessions_per_day = df[subject_date.duplicated(keep=False).values]
-    
+
     # Create a new column to mark duplicates
     df.loc[df_multi_sessions_per_day.index, "multiple_sessions_per_day"] = True
-    
-    # Also return a dataframe that removes duplicates (only keep subject_id and session_date as indices)
-    df_removed_multi_sessions_per_day = df[~subject_date.duplicated(keep="last").values]
-    df_removed_multi_sessions_per_day
 
-    return df, df_multi_sessions_per_day, df_removed_multi_sessions_per_day
+    # Also return a dataframe with unique mouse-dates,
+    # i.e.,  multiple sessions per day are combined in a list of 'name'
+    df_unique_mouse_date = (
+        df.reset_index()
+        .groupby(["subject_id", "session_date"])
+        .agg({"name": list, **{col: "first" for col in df.columns if col != "name"}})
+    )
+
+    return df, df_multi_sessions_per_day, df_unique_mouse_date
 
 @st.cache_data(ttl=3600 * 24)
 def fetch_all_queries_from_docDB(queries_to_merge):
@@ -167,21 +171,28 @@ def fetch_all_queries_from_docDB(queries_to_merge):
         for i, future in enumerate(as_completed(future_to_query), 1):
             key = future_to_query[future]
             try:
-                df, df_multi_sessions_per_day = future.result()
+                df, df_multi_sessions_per_day, df_unique_mouse_date = future.result()
                 dfs[key["alias"]] = {
                     "df": df,
                     "df_multi_sessions_per_day": df_multi_sessions_per_day,
+                    "df_unique_mouse_date": df_unique_mouse_date,
                 }
             except Exception as e:
                 print(f"Error querying {key}: {e}")
 
-    # Combine queried dfs
-    df_merged = dfs[queries_to_merge[0]["alias"]]["df"]
-    for df in [dfs[query["alias"]]["df"] for query in queries_to_merge[1:]]:
+    # Combine queried dfs using df_unique_mouse_date (on index "subject_id", "session_date" only)
+    df_merged = dfs[queries_to_merge[0]["alias"]]["df_unique_mouse_date"]
+    for df in [dfs[query["alias"]]["df_unique_mouse_date"] for query in queries_to_merge[1:]]:
         df_merged = df_merged.combine_first(df)  # Combine nwb_names
 
     # Recover the column order of QUERY_PRESET
-    # df_merged = df_merged.reindex(columns=[query["alias"] for query in queries_to_merge])
+    query_cols = [query["alias"] for query in queries_to_merge]
+    df_merged = df_merged.reindex(
+        columns=[
+            other_col for other_col in df_merged.columns if other_col not in query_cols
+        ]
+        + query_cols
+    ) 
 
     return df_merged, dfs
 
@@ -223,16 +234,37 @@ def app():
 
                 # Show records
                 df = dfs[query["alias"]]["df"]
-                n_records = df_merged[query["alias"]].sum()
+                df_multi_sessions_per_day = dfs[query["alias"]]["df_multi_sessions_per_day"]
+                df_unique_mouse_date = dfs[query["alias"]]["df_unique_mouse_date"]
 
-                n_multiple_sessions_per_day = len(
-                    dfs[query["alias"]]["df_multi_sessions_per_day"].index.droplevel("nwb_suffix").unique()
-                )
-                st.markdown(
-                    f"{n_records} from merged, {len(df)} from df, {n_multiple_sessions_per_day} has multiple sessions per day"
+                cols = st.columns([2, 1])
+                cols[0].markdown(f"{len(df)} returned")
+                with cols[1]:
+                    download_df(df, label="Download as CSV", file_name=f"{query['alias']}.csv")
+
+                cols = st.columns([2, 1])
+                cols[0].markdown(
+                    f"{df_unique_mouse_date.multiple_sessions_per_day.sum()} have multiple sessions per day")
+                with cols[1]:
+                    download_df(
+                        df_multi_sessions_per_day,
+                        label="Download as CSV",
+                        file_name=f"{query['alias']}_multi_sessions_per_day.csv",
+                    )
+
+                cols = st.columns([2, 1])
+                cols[0].markdown(
+                    f"{len(df_unique_mouse_date)} unique mouse-date pairs"
                 )
+                with cols[1]:
+                    download_df(
+                        df_unique_mouse_date,
+                        label="Download as CSV",
+                        file_name=f"{query['alias']}_unique_mouse_date.csv",
+                    )
 
-                
+                if len(df_unique_mouse_date) != df_merged[query["alias"]].sum():
+                    st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')
 
     st.markdown(f"#### Merged dataframe (n = {len(df_merged)})")
     st.write(df_merged)

From 86b39ea2f191aec31bb0e00a7dfde00d210646fa Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Thu, 2 Jan 2025 20:52:38 +0000
Subject: [PATCH 22/68] add back other queries

---
 code/pages/0_docDB inventory.py | 89 ++++++++++++++++-----------------
 1 file changed, 44 insertions(+), 45 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 8315e31..ca397bf 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -51,50 +51,50 @@
          "session.data_streams.software.name": "dynamic-foraging-task",
          "name": {"$not": {"$regex": ".*processed.*"}},
      }},
-    # {"alias": "{raw, 'dynamic_foraging' in stimulus_epochs software name}",
-    #  "filter": {
-    #      "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
-    #      "name": {"$not": {"$regex": ".*processed.*"}},
-    #  }},
-    # {"alias": "{raw, 'fib' in 'data_description.modality'}",
-    #  "filter": {
-    #      "data_description.modality.abbreviation": "fib",
-    #      "name": {"$not": {"$regex": ".*processed.*"}},
-    #  }},
-    # {"alias": "{raw, 'fib' in 'rig.modalities'}",
-    #  "filter": {
-    #      "rig.modalities.abbreviation": "fib",
-    #      "name": {"$not": {"$regex": ".*processed.*"}},        
-    #  }},
-    # {"alias": "{raw, 'fib' in 'session.data_streams'}",
-    #  "filter": {
-    #      "session.data_streams.stream_modalities.abbreviation": "fib",
-    #      "name": {"$not": {"$regex": ".*processed.*"}},
-    #  }},
-    # {"alias": "{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
-    #  "filter": {
-    #      "$or":[
-    #          {"session.data_streams.software.name": "dynamic-foraging-task"},
-    #          {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-    #      ],
-    #      "data_description.modality.abbreviation": "ecephys",
-    #      "name": {"$not": {"$regex": ".*processed.*"}},        
-    #  }},
-    # {"alias": "{raw, 'FIP' in name}",
-    #  "filter": {
-    #     "name": {
-    #         "$regex": "^FIP.*",
-    #         "$not": {"$regex": ".*processed.*"}
-    #     }
-    # }},
-    # {"alias": "{processed, 'dynamic_foraging' in ANY software name}",
-    #  "filter": {
-    #      "$or":[
-    #          {"session.data_streams.software.name": "dynamic-foraging-task"},
-    #          {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-    #      ],
-    #      "name": {"$regex": ".*processed.*"},
-    #  }},
+    {"alias": "{raw, 'dynamic_foraging' in stimulus_epochs software name}",
+     "filter": {
+         "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
+         "name": {"$not": {"$regex": ".*processed.*"}},
+     }},
+    {"alias": "{raw, 'fib' in 'data_description.modality'}",
+     "filter": {
+         "data_description.modality.abbreviation": "fib",
+         "name": {"$not": {"$regex": ".*processed.*"}},
+     }},
+    {"alias": "{raw, 'fib' in 'rig.modalities'}",
+     "filter": {
+         "rig.modalities.abbreviation": "fib",
+         "name": {"$not": {"$regex": ".*processed.*"}},        
+     }},
+    {"alias": "{raw, 'fib' in 'session.data_streams'}",
+     "filter": {
+         "session.data_streams.stream_modalities.abbreviation": "fib",
+         "name": {"$not": {"$regex": ".*processed.*"}},
+     }},
+    {"alias": "{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
+     "filter": {
+         "$or":[
+             {"session.data_streams.software.name": "dynamic-foraging-task"},
+             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+         ],
+         "data_description.modality.abbreviation": "ecephys",
+         "name": {"$not": {"$regex": ".*processed.*"}},        
+     }},
+    {"alias": "{raw, 'FIP' in name}",
+     "filter": {
+        "name": {
+            "$regex": "^FIP.*",
+            "$not": {"$regex": ".*processed.*"}
+        }
+    }},
+    {"alias": "{processed, 'dynamic_foraging' in ANY software name}",
+     "filter": {
+         "$or":[
+             {"session.data_streams.software.name": "dynamic-foraging-task"},
+             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
+         ],
+         "name": {"$regex": ".*processed.*"},
+     }},
 ]
 
 def query_sessions_from_docDB(query):
@@ -193,7 +193,6 @@ def fetch_all_queries_from_docDB(queries_to_merge):
         ]
         + query_cols
     ) 
-
     return df_merged, dfs
 
 def venn(df, columns_to_venn):

From 46c740783f44845e60f20aa03ad93dcb43b6017c Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 00:17:51 +0000
Subject: [PATCH 23/68] refactor

---
 code/pages/0_docDB inventory.py | 91 +++++++++++++++++++++------------
 1 file changed, 57 insertions(+), 34 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index ca397bf..e09c791 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -9,6 +9,7 @@
 import streamlit as st
 from streamlit_dynamic_filters import DynamicFilters
 import pandas as pd
+import time
 
 from util.fetch_data_docDB import load_data_from_docDB, load_client
 from util.reformat import split_nwb_name
@@ -97,18 +98,6 @@
      }},
 ]
 
-def query_sessions_from_docDB(query):
-    return client.retrieve_docdb_records(
-        filter_query=query["filter"],
-        projection={
-            "_id": 0,
-            "name": 1,
-            "rig.rig_id": 1,
-            "session.experimenter_full_name": 1,
-        },
-        paginate=False,
-    )
-
 def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
     """ Download df as csv """
     csv = df.to_csv(index=True)
@@ -121,65 +110,95 @@ def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
         mime='text/csv'
     )
 
-def fetch_single_query(key):
-    """ Query a single query from QUERY_PRESET and process the result 
+def fetch_single_query(query):
+    """ Fetch a query from docDB and process the result into dataframe
     
     Return: 
-    - df: the full dataframe
+    - df: dataframe of the original returned records
     - df_multi_sessions_per_day: the dataframe with multiple sessions per day
     - df_unique_mouse_date: the dataframe with multiple sessions per day combined
     """
-    query_result = query_sessions_from_docDB(key)
-    df = pd.json_normalize(query_result)
 
-    # Create index that can be joined with other dfs and main df
+    # --- Fetch data from docDB ---
+    results = client.retrieve_docdb_records(
+        filter_query=query["filter"],
+        projection={
+            "_id": 0,
+            "name": 1,
+            "rig.rig_id": 1,
+            "session.experimenter_full_name": 1,
+        },
+        paginate=False,
+    )
+
+    # --- Process data into dataframe ---
+    df = pd.json_normalize(results)
+
+    # Create index of subject_id, session_date, nwb_suffix by parsing nwb_name
     df[["subject_id", "session_date", "nwb_suffix"]] = df[f"name"].apply(
         lambda x: pd.Series(split_nwb_name(x))
     )
-    df[key["alias"]] = True
     df = df.set_index(["subject_id", "session_date", "nwb_suffix"]).sort_index()
+    df[query["alias"]] = True  # Add a column to mark the query
 
     # Remove invalid subject_id
     df = df[df.index.get_level_values("subject_id").astype(int) > 300000]
 
-    # Get cases where one mouse has multiple records per day
-    subject_date = df.index.to_frame(index=False)[["subject_id", "session_date"]]
-    df_multi_sessions_per_day = df[subject_date.duplicated(keep=False).values]
-
-    # Create a new column to mark duplicates
-    df.loc[df_multi_sessions_per_day.index, "multiple_sessions_per_day"] = True
-
-    # Also return a dataframe with unique mouse-dates,
-    # i.e.,  multiple sessions per day are combined in a list of 'name'
+    # --- Handle multiple sessions per day ---
+    # Build a dataframe with unique mouse-dates.
+    # If multiple sessions per day, combine them into a list of 'name'
     df_unique_mouse_date = (
         df.reset_index()
         .groupby(["subject_id", "session_date"])
         .agg({"name": list, **{col: "first" for col in df.columns if col != "name"}})
     )
+    # Add a new column to indicate multiple sessions per day
+    df_unique_mouse_date["multiple_sessions_per_day"] = df_unique_mouse_date["name"].apply(
+        lambda x: len(x) > 1
+    )
 
-    return df, df_multi_sessions_per_day, df_unique_mouse_date
+    # Also return the dataframe with multiple sessions per day
+    df_multi_sessions_per_day = df_unique_mouse_date[df_unique_mouse_date["multiple_sessions_per_day"]]
+
+    # Create a new column to mark duplicates in the original df
+    df.loc[
+        df.index.droplevel("nwb_suffix").isin(df_multi_sessions_per_day.index), 
+        "multiple_sessions_per_day"
+    ] = True
+
+    print(f"Done querying {query['alias']}!")
+    return df, df_unique_mouse_date, df_multi_sessions_per_day
 
 @st.cache_data(ttl=3600 * 24)
 def fetch_all_queries_from_docDB(queries_to_merge):
     """ Get merged queries from selected queries """
 
-    dfs = {}
-
+    dfs = {} 
+    
     # Fetch data in parallel
     with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
         future_to_query = {executor.submit(fetch_single_query, key): key for key in queries_to_merge}
         for i, future in enumerate(as_completed(future_to_query), 1):
             key = future_to_query[future]
             try:
-                df, df_multi_sessions_per_day, df_unique_mouse_date = future.result()
-                dfs[key["alias"]] = {
+                df, df_unique_mouse_date, df_multi_sessions_per_day = future.result()
+                dfs[key["alias"]] = { 
                     "df": df,
-                    "df_multi_sessions_per_day": df_multi_sessions_per_day,
                     "df_unique_mouse_date": df_unique_mouse_date,
+                    "df_multi_sessions_per_day": df_multi_sessions_per_day,
                 }
             except Exception as e:
                 print(f"Error querying {key}: {e}")
 
+    # Fetch data in serial 
+    # for query in queries_to_merge:
+    #     df, df_unique_mouse_date, df_multi_sessions_per_day = fetch_single_query(query)
+    #     dfs[query["alias"]] = {
+    #         "df": df,
+    #         "df_unique_mouse_date": df_unique_mouse_date,
+    #         "df_multi_sessions_per_day": df_multi_sessions_per_day,
+    #     }
+
     # Combine queried dfs using df_unique_mouse_date (on index "subject_id", "session_date" only)
     df_merged = dfs[queries_to_merge[0]["alias"]]["df_unique_mouse_date"]
     for df in [dfs[query["alias"]]["df_unique_mouse_date"] for query in queries_to_merge[1:]]:
@@ -219,6 +238,7 @@ def venn(df, columns_to_venn):
 def app():
 
     # Generate combined dataframe
+    start_time = time.time()
     df_merged, dfs = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
 
     # Sidebar
@@ -264,6 +284,9 @@ def app():
 
                 if len(df_unique_mouse_date) != df_merged[query["alias"]].sum():
                     st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')
+                    
+        st.markdown(f"Retrieving data took {time.time() - start_time} secs")
+
 
     st.markdown(f"#### Merged dataframe (n = {len(df_merged)})")
     st.write(df_merged)

From 7aff726766a5c2d872116f159d0293e0d5d6e5e8 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 00:32:13 +0000
Subject: [PATCH 24/68] change format of sidebar

---
 code/pages/0_docDB inventory.py | 70 ++++++++++++++++-----------------
 1 file changed, 33 insertions(+), 37 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index e09c791..f908539 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -99,7 +99,7 @@
 ]
 
 def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
-    """ Download df as csv """
+    """ Add a button to download df as csv """
     csv = df.to_csv(index=True)
     
     # Create download buttons
@@ -246,44 +246,40 @@ def app():
         st.markdown('## docDB query presets ')
         st.markdown('#### See how to use these queries [in this doc.](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
         for query in QUERY_PRESET:
-            with st.expander(f"{query['alias']}"):
-                # Turn query to json with indent=4
-                query_json = json.dumps(query["filter"], indent=4)
-                st.code(query_json)
-
-                # Show records
-                df = dfs[query["alias"]]["df"]
-                df_multi_sessions_per_day = dfs[query["alias"]]["df_multi_sessions_per_day"]
-                df_unique_mouse_date = dfs[query["alias"]]["df_unique_mouse_date"]
-
-                cols = st.columns([2, 1])
-                cols[0].markdown(f"{len(df)} returned")
-                with cols[1]:
-                    download_df(df, label="Download as CSV", file_name=f"{query['alias']}.csv")
-
-                cols = st.columns([2, 1])
-                cols[0].markdown(
-                    f"{df_unique_mouse_date.multiple_sessions_per_day.sum()} have multiple sessions per day")
-                with cols[1]:
-                    download_df(
-                        df_multi_sessions_per_day,
-                        label="Download as CSV",
-                        file_name=f"{query['alias']}_multi_sessions_per_day.csv",
-                    )
-
-                cols = st.columns([2, 1])
-                cols[0].markdown(
-                    f"{len(df_unique_mouse_date)} unique mouse-date pairs"
+            st.markdown(f"### {query['alias']}")
+            
+            # Turn query to json with indent=4
+            # with st.expander("Show docDB query"):
+            query_json = json.dumps(query["filter"], indent=4)
+            st.code(query_json)
+
+            # Show records
+            df = dfs[query["alias"]]["df"]
+            df_multi_sessions_per_day = dfs[query["alias"]]["df_multi_sessions_per_day"]
+            df_unique_mouse_date = dfs[query["alias"]]["df_unique_mouse_date"]
+
+            with st.expander(f"{len(df)} records returned from docDB"):
+                st.write(df)
+                download_df(df, label="Download as CSV", file_name=f"{query['alias']}.csv")
+
+            with st.expander(f"{len(df_multi_sessions_per_day)} have multiple sessions per day"):
+                st.write(df_multi_sessions_per_day)
+                download_df(
+                    df_multi_sessions_per_day,
+                    label="Download as CSV",
+                    file_name=f"{query['alias']}_multi_sessions_per_day.csv",
+                )
+            
+            with st.expander(f"{len(df_unique_mouse_date)} unique mouse-date pairs"):
+                st.write(df_unique_mouse_date)
+                download_df(
+                    df_unique_mouse_date,
+                    label="Download as CSV",
+                    file_name=f"{query['alias']}_unique_mouse_date.csv",
                 )
-                with cols[1]:
-                    download_df(
-                        df_unique_mouse_date,
-                        label="Download as CSV",
-                        file_name=f"{query['alias']}_unique_mouse_date.csv",
-                    )
 
-                if len(df_unique_mouse_date) != df_merged[query["alias"]].sum():
-                    st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')
+            if len(df_unique_mouse_date) != df_merged[query["alias"]].sum():
+                st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')
                     
         st.markdown(f"Retrieving data took {time.time() - start_time} secs")
 

From c69ffbd8daf7b3c61268cba7b482cd1bbce44768 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 00:43:24 +0000
Subject: [PATCH 25/68] nested expanders

---
 code/pages/0_docDB inventory.py | 61 +++++++++++++++++----------------
 1 file changed, 31 insertions(+), 30 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index f908539..7572b1b 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -10,6 +10,7 @@
 from streamlit_dynamic_filters import DynamicFilters
 import pandas as pd
 import time
+import streamlit_nested_layout
 
 from util.fetch_data_docDB import load_data_from_docDB, load_client
 from util.reformat import split_nwb_name
@@ -246,37 +247,37 @@ def app():
         st.markdown('## docDB query presets ')
         st.markdown('#### See how to use these queries [in this doc.](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
         for query in QUERY_PRESET:
-            st.markdown(f"### {query['alias']}")
+            with st.expander(f"### {query['alias']}"):
             
-            # Turn query to json with indent=4
-            # with st.expander("Show docDB query"):
-            query_json = json.dumps(query["filter"], indent=4)
-            st.code(query_json)
-
-            # Show records
-            df = dfs[query["alias"]]["df"]
-            df_multi_sessions_per_day = dfs[query["alias"]]["df_multi_sessions_per_day"]
-            df_unique_mouse_date = dfs[query["alias"]]["df_unique_mouse_date"]
-
-            with st.expander(f"{len(df)} records returned from docDB"):
-                st.write(df)
-                download_df(df, label="Download as CSV", file_name=f"{query['alias']}.csv")
-
-            with st.expander(f"{len(df_multi_sessions_per_day)} have multiple sessions per day"):
-                st.write(df_multi_sessions_per_day)
-                download_df(
-                    df_multi_sessions_per_day,
-                    label="Download as CSV",
-                    file_name=f"{query['alias']}_multi_sessions_per_day.csv",
-                )
-            
-            with st.expander(f"{len(df_unique_mouse_date)} unique mouse-date pairs"):
-                st.write(df_unique_mouse_date)
-                download_df(
-                    df_unique_mouse_date,
-                    label="Download as CSV",
-                    file_name=f"{query['alias']}_unique_mouse_date.csv",
-                )
+                # Turn query to json with indent=4
+                # with st.expander("Show docDB query"):
+                query_json = json.dumps(query["filter"], indent=4)
+                st.code(query_json)
+
+                # Show records
+                df = dfs[query["alias"]]["df"]
+                df_multi_sessions_per_day = dfs[query["alias"]]["df_multi_sessions_per_day"]
+                df_unique_mouse_date = dfs[query["alias"]]["df_unique_mouse_date"]
+
+                with st.expander(f"{len(df)} records returned from docDB"):
+                    download_df(df, label="Download as CSV", file_name=f"{query['alias']}.csv")
+                    st.write(df)
+
+                with st.expander(f"{len(df_multi_sessions_per_day)} have multiple sessions per day"):
+                    download_df(
+                        df_multi_sessions_per_day,
+                        label="Download as CSV",
+                        file_name=f"{query['alias']}_multi_sessions_per_day.csv",
+                    )
+                    st.write(df_multi_sessions_per_day)
+                
+                with st.expander(f"{len(df_unique_mouse_date)} unique mouse-date pairs"):
+                    download_df(
+                        df_unique_mouse_date,
+                        label="Download as CSV",
+                        file_name=f"{query['alias']}_unique_mouse_date.csv",
+                    )
+                    st.write(df_unique_mouse_date)
 
             if len(df_unique_mouse_date) != df_merged[query["alias"]].sum():
                 st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')

From 9e3fdb045b977ae17b1d022b36878360da414864 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 01:08:27 +0000
Subject: [PATCH 26/68] refactor and use aggrid; improve documents

---
 code/Home.py                    |  4 ++--
 code/pages/0_docDB inventory.py | 16 +++++++++++-----
 code/util/streamlit.py          | 11 +++++------
 3 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/code/Home.py b/code/Home.py
index 31b8ecb..cf07ab7 100644
--- a/code/Home.py
+++ b/code/Home.py
@@ -33,7 +33,7 @@
 from util.streamlit import (_plot_population_x_y, add_auto_train_manager,
                             add_dot_property_mapper, add_session_filter,
                             add_xy_selector, add_xy_setting,
-                            aggrid_interactive_table_curriculum,
+                            aggrid_interactive_table_basic,
                             aggrid_interactive_table_session, data_selector,
                             add_footnote)
 from util.url_query_helper import (checkbox_wrapper_for_url_query,
@@ -745,7 +745,7 @@ def app():
                     pre_selected_rows = None
             
             # Show df_curriculum       
-            aggrid_interactive_table_curriculum(df=df_curriculums,
+            aggrid_interactive_table_basic(df=df_curriculums,
                                                 pre_selected_rows=pre_selected_rows)        
 
             
diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 7572b1b..ea3d30c 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -12,6 +12,7 @@
 import time
 import streamlit_nested_layout
 
+from util.streamlit import aggrid_interactive_table_basic
 from util.fetch_data_docDB import load_data_from_docDB, load_client
 from util.reformat import split_nwb_name
 
@@ -244,8 +245,9 @@ def app():
 
     # Sidebar
     with st.sidebar:
-        st.markdown('## docDB query presets ')
-        st.markdown('#### See how to use these queries [in this doc.](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb)')
+        st.markdown('# Data sources:')
+        st.markdown('## 1. From docDB queries')
+        st.markdown('#### See [how to use these queries](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb) in your own code.')
         for query in QUERY_PRESET:
             with st.expander(f"### {query['alias']}"):
             
@@ -284,10 +286,14 @@ def app():
                     
         st.markdown(f"Retrieving data took {time.time() - start_time} secs")
 
+    # --- Main contents ---
+    st.markdown(f"# Data inventory for dynamic foraging")
+    cols = st.columns([1, 5])
+    cols[0].markdown(f"### Merged dataframe (n = {len(df_merged)})")
+    with cols[1]:
+        download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
 
-    st.markdown(f"#### Merged dataframe (n = {len(df_merged)})")
-    st.write(df_merged)
-    download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
+    aggrid_interactive_table_basic(df_merged.reset_index(), height=400)
 
     # Multiselect for selecting queries up to three
     query_keys = [query["alias"] for query in QUERY_PRESET]
diff --git a/code/util/streamlit.py b/code/util/streamlit.py
index d1a3bec..95250de 100644
--- a/code/util/streamlit.py
+++ b/code/util/streamlit.py
@@ -106,8 +106,9 @@ def aggrid_interactive_table_session(df: pd.DataFrame, table_height: int = 400):
     
     return selection
 
-def aggrid_interactive_table_curriculum(df: pd.DataFrame,
-                                        pre_selected_rows: list = None):
+def aggrid_interactive_table_basic(df: pd.DataFrame,
+                                   height: int = 200,
+                                   pre_selected_rows: list = None):
     """Creates an st-aggrid interactive table based on a dataframe.
 
     Args:
@@ -121,10 +122,9 @@ def aggrid_interactive_table_curriculum(df: pd.DataFrame,
     )
 
     options.configure_side_bar()
-    
     options.configure_selection(selection_mode=None,
                                 pre_selected_rows=pre_selected_rows)
-        
+
     selection = AgGrid(
         df,
         enable_enterprise_modules=True,
@@ -132,11 +132,10 @@ def aggrid_interactive_table_curriculum(df: pd.DataFrame,
         theme="balham",
         update_mode=GridUpdateMode.NO_UPDATE,
         allow_unsafe_jscode=True,
-        height=200,
+        height=height,
         columns_auto_size_mode=ColumnsAutoSizeMode.FIT_CONTENTS,
         custom_css=custom_css,
     )
-    
     return selection
 
 def aggrid_interactive_table_units(df: pd.DataFrame):

From ddf4917f47da3f10dc854087549362e9771fe784 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 01:10:15 +0000
Subject: [PATCH 27/68] minor

---
 code/pages/0_docDB inventory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index ea3d30c..bcc6237 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -284,7 +284,7 @@ def app():
             if len(df_unique_mouse_date) != df_merged[query["alias"]].sum():
                 st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')
                     
-        st.markdown(f"Retrieving data took {time.time() - start_time} secs")
+        st.markdown(f"Retrieving data from docDB (or st.cache) took {time.time() - start_time:.3f} secs.")
 
     # --- Main contents ---
     st.markdown(f"# Data inventory for dynamic foraging")

From eb529403897a84398ec626bda6eff54e4cb03c59 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 01:12:51 +0000
Subject: [PATCH 28/68] refactor

---
 code/pages/0_docDB inventory.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index bcc6237..71e87b2 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -236,13 +236,7 @@ def venn(df, columns_to_venn):
         )
     return fig
 
-
-def app():
-
-    # Generate combined dataframe
-    start_time = time.time()
-    df_merged, dfs = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
-
+def add_sidebar(dfs, df_merged, data_retrieve_time):
     # Sidebar
     with st.sidebar:
         st.markdown('# Data sources:')
@@ -283,8 +277,19 @@ def app():
 
             if len(df_unique_mouse_date) != df_merged[query["alias"]].sum():
                 st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')
+                
+        st.markdown(f"Retrieving data from docDB (or st.cache) took {data_retrieve_time:.3f} secs.")
+
                     
-        st.markdown(f"Retrieving data from docDB (or st.cache) took {time.time() - start_time:.3f} secs.")
+def app():
+
+    # --- Generate combined dataframe from docDB ---
+    start_time = time.time()
+    df_merged, dfs = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
+    data_retrieve_time = time.time() - start_time
+    
+    # --- Add sidebar ---
+    add_sidebar(dfs, df_merged, data_retrieve_time)
 
     # --- Main contents ---
     st.markdown(f"# Data inventory for dynamic foraging")

From 9ae44a5589761652956fe3342233b00123794ed6 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 01:36:10 +0000
Subject: [PATCH 29/68] remove invalid subject_id from the master df

---
 code/Home.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/code/Home.py b/code/Home.py
index cf07ab7..c2947cf 100644
--- a/code/Home.py
+++ b/code/Home.py
@@ -377,6 +377,10 @@ def _get_data_source(rig):
     _df.dropna(subset=['session'], inplace=True) # Remove rows with no session number (only leave the nwb file with the largest finished_trials for now)
     _df.drop(_df.query('session < 1').index, inplace=True)
     
+    # Remove invalid subject_id
+    _df = _df[(999999 > _df["subject_id"].astype(int)) 
+              & (_df["subject_id"].astype(int) > 300000)]
+    
     # Remove abnormal values
     _df.loc[_df['weight_after'] > 100, 
             ['weight_after', 'weight_after_ratio', 'water_in_session_total', 'water_after_session', 'water_day_total']

From 83c73e9c2abc5a80e36e9760118a777202cdf0b1 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 01:47:46 +0000
Subject: [PATCH 30/68] merge in Han's temp pipeline

---
 code/pages/0_docDB inventory.py | 42 ++++++++++++++++++++++++++-------
 code/util/streamlit.py          |  7 +++++-
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 71e87b2..84652d1 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -15,6 +15,8 @@
 from util.streamlit import aggrid_interactive_table_basic
 from util.fetch_data_docDB import load_data_from_docDB, load_client
 from util.reformat import split_nwb_name
+from Home import init
+
 
 try:
     st.set_page_config(layout="wide", 
@@ -140,11 +142,13 @@ def fetch_single_query(query):
     df[["subject_id", "session_date", "nwb_suffix"]] = df[f"name"].apply(
         lambda x: pd.Series(split_nwb_name(x))
     )
+    df["session_date"] = pd.to_datetime(df["session_date"])
     df = df.set_index(["subject_id", "session_date", "nwb_suffix"]).sort_index()
     df[query["alias"]] = True  # Add a column to mark the query
 
     # Remove invalid subject_id
-    df = df[df.index.get_level_values("subject_id").astype(int) > 300000]
+    df = df[(df.index.get_level_values("subject_id").astype(int) > 300000) 
+            & (df.index.get_level_values("subject_id").astype(int) < 999999)]
 
     # --- Handle multiple sessions per day ---
     # Build a dataframe with unique mouse-dates.
@@ -236,7 +240,7 @@ def venn(df, columns_to_venn):
         )
     return fig
 
-def add_sidebar(dfs, df_merged, data_retrieve_time):
+def add_sidebar(dfs, df_merged, docDB_retrieve_time):
     # Sidebar
     with st.sidebar:
         st.markdown('# Data sources:')
@@ -278,18 +282,25 @@ def add_sidebar(dfs, df_merged, data_retrieve_time):
             if len(df_unique_mouse_date) != df_merged[query["alias"]].sum():
                 st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')
                 
-        st.markdown(f"Retrieving data from docDB (or st.cache) took {data_retrieve_time:.3f} secs.")
+        st.markdown(f"Retrieving data from docDB (or st.cache) took {docDB_retrieve_time:.3f} secs.")
+
 
-                    
 def app():
 
     # --- Generate combined dataframe from docDB ---
     start_time = time.time()
     df_merged, dfs = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
-    data_retrieve_time = time.time() - start_time
-    
+    docDB_retrieve_time = time.time() - start_time
+
+    # --- Merge in the master df in the Home page (Han's temporary pipeline) ---
+    # Only keep subject_id and session_date as index
+    Han_mouse_dates = st.session_state.df["sessions_bonsai"].set_index(["subject_id", "session_date"]).index
+    df_Han_pipeline = pd.DataFrame(index=Han_mouse_dates, columns=["Han_temp_pipeline"], data=True)
+    # Merged with df_merged
+    df_merged = df_merged.combine_first(df_Han_pipeline) 
+
     # --- Add sidebar ---
-    add_sidebar(dfs, df_merged, data_retrieve_time)
+    add_sidebar(dfs, df_merged, docDB_retrieve_time)
 
     # --- Main contents ---
     st.markdown(f"# Data inventory for dynamic foraging")
@@ -298,7 +309,17 @@ def app():
     with cols[1]:
         download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
 
-    aggrid_interactive_table_basic(df_merged.reset_index(), height=400)
+    aggrid_interactive_table_basic(
+        df_merged.reset_index(),
+        height=400,
+        configure_columns=[
+            dict(
+                field="session_date",
+                type=["customDateTimeFormat"],
+                custom_format_string="yyyy-MM-dd",
+            )
+        ],
+    )
 
     # Multiselect for selecting queries up to three
     query_keys = [query["alias"] for query in QUERY_PRESET]
@@ -315,4 +336,9 @@ def app():
 
 
 if __name__ == "__main__":
+    
+    # Share the same master df as the Home page
+    if "df" not in st.session_state or "sessions_bonsai" not in st.session_state.df.keys():
+        init(if_load_docDB_override=False)
+
     app()
diff --git a/code/util/streamlit.py b/code/util/streamlit.py
index 95250de..aac2342 100644
--- a/code/util/streamlit.py
+++ b/code/util/streamlit.py
@@ -108,7 +108,8 @@ def aggrid_interactive_table_session(df: pd.DataFrame, table_height: int = 400):
 
 def aggrid_interactive_table_basic(df: pd.DataFrame,
                                    height: int = 200,
-                                   pre_selected_rows: list = None):
+                                   pre_selected_rows: list = None,
+                                   configure_columns: list = None,):
     """Creates an st-aggrid interactive table based on a dataframe.
 
     Args:
@@ -125,6 +126,10 @@ def aggrid_interactive_table_basic(df: pd.DataFrame,
     options.configure_selection(selection_mode=None,
                                 pre_selected_rows=pre_selected_rows)
 
+    if configure_columns:
+        for col in configure_columns:
+            options.configure_column(**col)
+
     selection = AgGrid(
         df,
         enable_enterprise_modules=True,

From 874336258b43fa6d4707a4743c545ed3bf11c7f5 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 02:03:29 +0000
Subject: [PATCH 31/68] minor

---
 code/pages/0_docDB inventory.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 84652d1..e9eecdf 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -9,6 +9,7 @@
 import streamlit as st
 from streamlit_dynamic_filters import DynamicFilters
 import pandas as pd
+import numpy as np
 import time
 import streamlit_nested_layout
 
@@ -159,17 +160,17 @@ def fetch_single_query(query):
         .agg({"name": list, **{col: "first" for col in df.columns if col != "name"}})
     )
     # Add a new column to indicate multiple sessions per day
-    df_unique_mouse_date["multiple_sessions_per_day"] = df_unique_mouse_date["name"].apply(
+    df_unique_mouse_date["multiple_sessions_per_day_in_docDB"] = df_unique_mouse_date["name"].apply(
         lambda x: len(x) > 1
     )
 
     # Also return the dataframe with multiple sessions per day
-    df_multi_sessions_per_day = df_unique_mouse_date[df_unique_mouse_date["multiple_sessions_per_day"]]
+    df_multi_sessions_per_day = df_unique_mouse_date[df_unique_mouse_date["multiple_sessions_per_day_in_docDB"]]
 
     # Create a new column to mark duplicates in the original df
     df.loc[
         df.index.droplevel("nwb_suffix").isin(df_multi_sessions_per_day.index), 
-        "multiple_sessions_per_day"
+        "multiple_sessions_per_day_in_docDB"
     ] = True
 
     print(f"Done querying {query['alias']}!")
@@ -295,7 +296,10 @@ def app():
     # --- Merge in the master df in the Home page (Han's temporary pipeline) ---
     # Only keep subject_id and session_date as index
     Han_mouse_dates = st.session_state.df["sessions_bonsai"].set_index(["subject_id", "session_date"]).index
-    df_Han_pipeline = pd.DataFrame(index=Han_mouse_dates, columns=["Han_temp_pipeline"], data=True)
+    df_Han_pipeline = pd.DataFrame(index=Han_mouse_dates)
+    df_Han_pipeline["Han_temp_pipeline"] = True
+    df_Han_pipeline["multiple_sessions_per_day_in_docDB"] = np.nan
+
     # Merged with df_merged
     df_merged = df_merged.combine_first(df_Han_pipeline) 
 

From e3ef227a7919f011316ec5ef847e5b812469de09 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 02:19:52 +0000
Subject: [PATCH 32/68] add bpod sessions from Han pipeline

---
 code/Home.py                    |  9 ++++++---
 code/pages/0_docDB inventory.py | 21 ++++++++++++++++-----
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/code/Home.py b/code/Home.py
index c2947cf..7abe455 100644
--- a/code/Home.py
+++ b/code/Home.py
@@ -280,7 +280,7 @@ def show_curriculums():
     pass
 
 # ------- Layout starts here -------- #
-def init(if_load_docDB_override=None):
+def init(if_load_bpod_data_override=None, if_load_docDB_override=None):
     
     # Clear specific session state and all filters
     for key in st.session_state:
@@ -296,11 +296,14 @@ def init(if_load_docDB_override=None):
     # Because sync_URL_to_session_state() needs df to be loaded (for dynamic column filtering),
     # 'if_load_bpod_sessions' has not been synced from URL to session state yet.
     # So here we need to manually get it from URL or session state.
-    if (st.query_params['if_load_bpod_sessions'].lower() == 'true'
+    _if_load_bpod = if_load_bpod_data_override if if_load_bpod_data_override is not None else (
+        st.query_params['if_load_bpod_sessions'].lower() == 'true'
         if 'if_load_bpod_sessions' in st.query_params
         else st.session_state.if_load_bpod_sessions 
         if 'if_load_bpod_sessions' in st.session_state
-        else False):
+        else False)
+    
+    if _if_load_bpod:
         df_bpod = load_data(['sessions'], data_source='bpod')
         
         # For historial reason, the suffix of df['sessions_bonsai'] just mean the data of the Home.py page
diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index e9eecdf..9208a93 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -294,11 +294,22 @@ def app():
     docDB_retrieve_time = time.time() - start_time
 
     # --- Merge in the master df in the Home page (Han's temporary pipeline) ---
+    # Data from Home.init (all sessions from Janelia bpod + AIND bpod + AIND bonsai)
+    df_from_Home = st.session_state.df["sessions_bonsai"]
+    # Only keep AIND sessions
+    df_from_Home = df_from_Home.query("institute == 'AIND'")
+    df_from_Home.loc[df_from_Home.hardware == "bpod", "Han_temp_pipeline (bpod)"] = True
+    df_from_Home.loc[df_from_Home.hardware == "bonsai", "Han_temp_pipeline (bonsai)"] = True
+
     # Only keep subject_id and session_date as index
-    Han_mouse_dates = st.session_state.df["sessions_bonsai"].set_index(["subject_id", "session_date"]).index
-    df_Han_pipeline = pd.DataFrame(index=Han_mouse_dates)
-    df_Han_pipeline["Han_temp_pipeline"] = True
-    df_Han_pipeline["multiple_sessions_per_day_in_docDB"] = np.nan
+    df_Han_pipeline = df_from_Home[
+        [
+            "subject_id",
+            "session_date",
+            "Han_temp_pipeline (bpod)",
+            "Han_temp_pipeline (bonsai)",
+        ]
+    ].set_index(["subject_id", "session_date"])
 
     # Merged with df_merged
     df_merged = df_merged.combine_first(df_Han_pipeline) 
@@ -343,6 +354,6 @@ def app():
     
     # Share the same master df as the Home page
     if "df" not in st.session_state or "sessions_bonsai" not in st.session_state.df.keys():
-        init(if_load_docDB_override=False)
+        init(if_load_docDB_override=False, if_load_bpod_data_override=True)
 
     app()

From fb00f07d94f3ba094994da33e8031fe51d504e82 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 02:39:38 +0000
Subject: [PATCH 33/68] add Han temp pipeline to venn diagram

---
 code/pages/0_docDB inventory.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 9208a93..0a4ed30 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -158,14 +158,16 @@ def fetch_single_query(query):
         df.reset_index()
         .groupby(["subject_id", "session_date"])
         .agg({"name": list, **{col: "first" for col in df.columns if col != "name"}})
-    )
+    ).rename(columns={"name": "name_in_docDB"})
     # Add a new column to indicate multiple sessions per day
-    df_unique_mouse_date["multiple_sessions_per_day_in_docDB"] = df_unique_mouse_date["name"].apply(
-        lambda x: len(x) > 1
-    )
+    df_unique_mouse_date["multiple_sessions_per_day_in_docDB"] = df_unique_mouse_date[
+        "name_in_docDB"
+    ].apply(lambda x: True if len(x) > 1 else np.nan)
 
     # Also return the dataframe with multiple sessions per day
-    df_multi_sessions_per_day = df_unique_mouse_date[df_unique_mouse_date["multiple_sessions_per_day_in_docDB"]]
+    df_multi_sessions_per_day = df_unique_mouse_date[
+        df_unique_mouse_date["multiple_sessions_per_day_in_docDB"].notnull()
+    ]
 
     # Create a new column to mark duplicates in the original df
     df.loc[
@@ -180,7 +182,7 @@ def fetch_single_query(query):
 def fetch_all_queries_from_docDB(queries_to_merge):
     """ Get merged queries from selected queries """
 
-    dfs = {} 
+    dfs = {}
     
     # Fetch data in parallel
     with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
@@ -218,7 +220,7 @@ def fetch_all_queries_from_docDB(queries_to_merge):
             other_col for other_col in df_merged.columns if other_col not in query_cols
         ]
         + query_cols
-    ) 
+    )
     return df_merged, dfs
 
 def venn(df, columns_to_venn):
@@ -337,7 +339,10 @@ def app():
     )
 
     # Multiselect for selecting queries up to three
-    query_keys = [query["alias"] for query in QUERY_PRESET]
+    query_keys = [
+        "Han_temp_pipeline (bpod)",
+        "Han_temp_pipeline (bonsai)",
+    ] + [query["alias"] for query in QUERY_PRESET]
     selected_queries = st.multiselect(
         "Select queries to filter sessions",
         query_keys,

From b02c14c72b3a7eb1fc7c5f324d69c80c694d2c98 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 02:56:51 +0000
Subject: [PATCH 34/68] add Han pipeline to sidebar

---
 code/pages/0_docDB inventory.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 0a4ed30..b1d2709 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -243,15 +243,16 @@ def venn(df, columns_to_venn):
         )
     return fig
 
-def add_sidebar(dfs, df_merged, docDB_retrieve_time):
+def add_sidebar(dfs, df_merged, docDB_retrieve_time, df_Han_pipeline):
     # Sidebar
     with st.sidebar:
         st.markdown('# Data sources:')
+
         st.markdown('## 1. From docDB queries')
         st.markdown('#### See [how to use these queries](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb) in your own code.')
         for query in QUERY_PRESET:
             with st.expander(f"### {query['alias']}"):
-            
+
                 # Turn query to json with indent=4
                 # with st.expander("Show docDB query"):
                 query_json = json.dumps(query["filter"], indent=4)
@@ -273,7 +274,7 @@ def add_sidebar(dfs, df_merged, docDB_retrieve_time):
                         file_name=f"{query['alias']}_multi_sessions_per_day.csv",
                     )
                     st.write(df_multi_sessions_per_day)
-                
+
                 with st.expander(f"{len(df_unique_mouse_date)} unique mouse-date pairs"):
                     download_df(
                         df_unique_mouse_date,
@@ -284,9 +285,22 @@ def add_sidebar(dfs, df_merged, docDB_retrieve_time):
 
             if len(df_unique_mouse_date) != df_merged[query["alias"]].sum():
                 st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')
-                
         st.markdown(f"Retrieving data from docDB (or st.cache) took {docDB_retrieve_time:.3f} secs.")
 
+        st.markdown('''## 2. From Han's temporary pipeline (the "Home" page)''')
+        hardwares = ["bonsai", "bpod"]
+        for hardware in hardwares:
+            df_this_hardware = df_Han_pipeline[
+                df_Han_pipeline[f"Han_temp_pipeline ({hardware})"].notnull()
+            ]
+            with st.expander(f"### {len(df_this_hardware)} {hardware} sessions"):
+                download_df(
+                    df_this_hardware,
+                    label="Download as CSV",
+                    file_name=f"Han_temp_pipeline_{hardware}.csv",
+                )
+                st.write(df_this_hardware)
+
 
 def app():
 
@@ -317,7 +331,7 @@ def app():
     df_merged = df_merged.combine_first(df_Han_pipeline) 
 
     # --- Add sidebar ---
-    add_sidebar(dfs, df_merged, docDB_retrieve_time)
+    add_sidebar(dfs, df_merged, docDB_retrieve_time, df_Han_pipeline)
 
     # --- Main contents ---
     st.markdown(f"# Data inventory for dynamic foraging")

From 03fe4d07c12e67df4efb59d4da1a3ff9f2019a30 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 03:06:42 +0000
Subject: [PATCH 35/68] minor

---
 code/pages/0_docDB inventory.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index b1d2709..6455913 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -293,7 +293,10 @@ def add_sidebar(dfs, df_merged, docDB_retrieve_time, df_Han_pipeline):
             df_this_hardware = df_Han_pipeline[
                 df_Han_pipeline[f"Han_temp_pipeline ({hardware})"].notnull()
             ]
-            with st.expander(f"### {len(df_this_hardware)} {hardware} sessions"):
+            with st.expander(
+                f"### {len(df_this_hardware)} {hardware} sessions"
+                + (" (old data, not growing)" if hardware == "bpod" else "")
+            ):
                 download_df(
                     df_this_hardware,
                     label="Download as CSV",

From 98fb9a87b21b1d888c9ee6eff422af192bcbbb5e Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Fri, 3 Jan 2025 03:07:20 +0000
Subject: [PATCH 36/68] minor

---
 code/pages/0_docDB inventory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 6455913..aa31a3b 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -246,7 +246,7 @@ def venn(df, columns_to_venn):
 def add_sidebar(dfs, df_merged, docDB_retrieve_time, df_Han_pipeline):
     # Sidebar
     with st.sidebar:
-        st.markdown('# Data sources:')
+        st.markdown('# Metadata sources:')
 
         st.markdown('## 1. From docDB queries')
         st.markdown('#### See [how to use these queries](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb) in your own code.')

From 9958ce77ac1225b07c386e7ffcd2ffe87625805d Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Sat, 4 Jan 2025 01:44:24 +0000
Subject: [PATCH 37/68] fetch raw data list from VAST

---
 code/pages/0_docDB inventory.py | 134 +++++++++++++++++++++-----------
 code/util/aws_s3.py             |   6 ++
 2 files changed, 96 insertions(+), 44 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index aa31a3b..b6238fb 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -15,6 +15,7 @@
 
 from util.streamlit import aggrid_interactive_table_basic
 from util.fetch_data_docDB import load_data_from_docDB, load_client
+from util.aws_s3 import load_raw_sessions_on_VAST
 from util.reformat import split_nwb_name
 from Home import init
 
@@ -115,37 +116,23 @@ def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
         mime='text/csv'
     )
 
-def fetch_single_query(query):
-    """ Fetch a query from docDB and process the result into dataframe
-    
-    Return: 
-    - df: dataframe of the original returned records
-    - df_multi_sessions_per_day: the dataframe with multiple sessions per day
-    - df_unique_mouse_date: the dataframe with multiple sessions per day combined
+def _formatting_metadata_df(df, source_prefix="in_docDB"):
+    """Formatting metadata dataframe
+    Given a dataframe with a column of "name" that contains nwb names
+    1. parse the nwb names into subject_id, session_date, nwb_suffix.
+    2. remove invalid subject_id
+    3. handle multiple sessions per day
     """
-
-    # --- Fetch data from docDB ---
-    results = client.retrieve_docdb_records(
-        filter_query=query["filter"],
-        projection={
-            "_id": 0,
-            "name": 1,
-            "rig.rig_id": 1,
-            "session.experimenter_full_name": 1,
-        },
-        paginate=False,
-    )
-
-    # --- Process data into dataframe ---
-    df = pd.json_normalize(results)
-
+    
+    new_name_field = f"name_{source_prefix}"
+    df.rename(columns={"name": new_name_field}, inplace=True)
+    
     # Create index of subject_id, session_date, nwb_suffix by parsing nwb_name
-    df[["subject_id", "session_date", "nwb_suffix"]] = df[f"name"].apply(
+    df[["subject_id", "session_date", "nwb_suffix"]] = df[new_name_field].apply(
         lambda x: pd.Series(split_nwb_name(x))
     )
     df["session_date"] = pd.to_datetime(df["session_date"])
     df = df.set_index(["subject_id", "session_date", "nwb_suffix"]).sort_index()
-    df[query["alias"]] = True  # Add a column to mark the query
 
     # Remove invalid subject_id
     df = df[(df.index.get_level_values("subject_id").astype(int) > 300000) 
@@ -157,32 +144,62 @@ def fetch_single_query(query):
     df_unique_mouse_date = (
         df.reset_index()
         .groupby(["subject_id", "session_date"])
-        .agg({"name": list, **{col: "first" for col in df.columns if col != "name"}})
-    ).rename(columns={"name": "name_in_docDB"})
+        .agg({new_name_field: list, **{col: "first" for col in df.columns if col != new_name_field}})
+    )
     # Add a new column to indicate multiple sessions per day
-    df_unique_mouse_date["multiple_sessions_per_day_in_docDB"] = df_unique_mouse_date[
-        "name_in_docDB"
-    ].apply(lambda x: True if len(x) > 1 else np.nan)
+    df_unique_mouse_date[f"multiple_sessions_per_day_{source_prefix}"] = df_unique_mouse_date[
+        new_name_field
+    ].apply(lambda x: len(x) > 1)
 
     # Also return the dataframe with multiple sessions per day
     df_multi_sessions_per_day = df_unique_mouse_date[
-        df_unique_mouse_date["multiple_sessions_per_day_in_docDB"].notnull()
+        df_unique_mouse_date[f"multiple_sessions_per_day_{source_prefix}"]
     ]
 
     # Create a new column to mark duplicates in the original df
     df.loc[
         df.index.droplevel("nwb_suffix").isin(df_multi_sessions_per_day.index), 
-        "multiple_sessions_per_day_in_docDB"
+        f"multiple_sessions_per_day_{source_prefix}"
     ] = True
 
+    return df, df_unique_mouse_date, df_multi_sessions_per_day
+
+def fetch_single_query(query):
+    """ Fetch a query from docDB and process the result into dataframe
+    
+    Return: 
+    - df: dataframe of the original returned records
+    - df_multi_sessions_per_day: the dataframe with multiple sessions per day
+    - df_unique_mouse_date: the dataframe with multiple sessions per day combined
+    """
+
+    # --- Fetch data from docDB ---
+    results = client.retrieve_docdb_records(
+        filter_query=query["filter"],
+        projection={
+            "_id": 0,
+            "name": 1,
+            "rig.rig_id": 1,
+            "session.experimenter_full_name": 1,
+        },
+        paginate=False,
+    )
     print(f"Done querying {query['alias']}!")
+
+    # --- Process data into dataframe ---
+    df = pd.json_normalize(results)
+    df[query["alias"]] = True  # Add a column to mark the query
+    
+    # Formatting dataframe
+    df, df_unique_mouse_date, df_multi_sessions_per_day = _formatting_metadata_df(df)
+
     return df, df_unique_mouse_date, df_multi_sessions_per_day
 
 @st.cache_data(ttl=3600 * 24)
 def fetch_all_queries_from_docDB(queries_to_merge):
     """ Get merged queries from selected queries """
 
-    dfs = {}
+    dfs = {} 
     
     # Fetch data in parallel
     with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
@@ -206,7 +223,7 @@ def fetch_all_queries_from_docDB(queries_to_merge):
     #         "df": df,
     #         "df_unique_mouse_date": df_unique_mouse_date,
     #         "df_multi_sessions_per_day": df_multi_sessions_per_day,
-    #     }
+    #     } 
 
     # Combine queried dfs using df_unique_mouse_date (on index "subject_id", "session_date" only)
     df_merged = dfs[queries_to_merge[0]["alias"]]["df_unique_mouse_date"]
@@ -243,7 +260,7 @@ def venn(df, columns_to_venn):
         )
     return fig
 
-def add_sidebar(dfs, df_merged, docDB_retrieve_time, df_Han_pipeline):
+def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_retrieve_time):
     # Sidebar
     with st.sidebar:
         st.markdown('# Metadata sources:')
@@ -259,9 +276,9 @@ def add_sidebar(dfs, df_merged, docDB_retrieve_time, df_Han_pipeline):
                 st.code(query_json)
 
                 # Show records
-                df = dfs[query["alias"]]["df"]
-                df_multi_sessions_per_day = dfs[query["alias"]]["df_multi_sessions_per_day"]
-                df_unique_mouse_date = dfs[query["alias"]]["df_unique_mouse_date"]
+                df = dfs_docDB[query["alias"]]["df"]
+                df_multi_sessions_per_day = dfs_docDB[query["alias"]]["df_multi_sessions_per_day"]
+                df_unique_mouse_date = dfs_docDB[query["alias"]]["df_unique_mouse_date"]
 
                 with st.expander(f"{len(df)} records returned from docDB"):
                     download_df(df, label="Download as CSV", file_name=f"{query['alias']}.csv")
@@ -307,12 +324,12 @@ def add_sidebar(dfs, df_merged, docDB_retrieve_time, df_Han_pipeline):
 
 def app():
 
-    # --- Generate combined dataframe from docDB ---
+    # --- 1. Generate combined dataframe from docDB queries ---
     start_time = time.time()
-    df_merged, dfs = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
+    df_merged, dfs_docDB = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
     docDB_retrieve_time = time.time() - start_time
 
-    # --- Merge in the master df in the Home page (Han's temporary pipeline) ---
+    # --- 2. Merge in the master df in the Home page (Han's temporary pipeline) ---
     # Data from Home.init (all sessions from Janelia bpod + AIND bpod + AIND bonsai)
     df_from_Home = st.session_state.df["sessions_bonsai"]
     # Only keep AIND sessions
@@ -331,14 +348,43 @@ def app():
     ].set_index(["subject_id", "session_date"])
 
     # Merged with df_merged
-    df_merged = df_merged.combine_first(df_Han_pipeline) 
+    df_merged = df_merged.combine_first(df_Han_pipeline)
+
+    # --- 3. Get raw data on VAST ---
+    raw_sessions_on_VAST = load_raw_sessions_on_VAST()
+
+    # Example entry of raw_sessions_on_VAST:
+    #     Z:\svc_aind_behavior_transfer\447-3-D\751153\behavior_751153_2024-10-20_17-13-34\behavior
+    #     Z:\svc_aind_behavior_transfer\2023late_DataNoMeta_Reorganized\687553_2023-11-20_09-48-24\behavior
+    #     Z:\svc_aind_behavior_transfer\2023late_DataNoMeta_Reorganized\687553_2023-11-13_11-09-55\687553_2023-12-01_09-41-43\TrainingFolder
+    #     Let's find the strings between two \\s that precede "behavior" or "TrainingFolder"
+
+    import re
+    re_pattern = R"\\([^\\]*)\\(?:behavior|TrainingFolder)$"
+    session_names = [re.findall(re_pattern, path)[0] for path in raw_sessions_on_VAST]
+
+    df_raw_sessions_on_VAST = pd.DataFrame(raw_sessions_on_VAST, columns=["raw_full_path_on_VAST"])
+    df_raw_sessions_on_VAST["name"] = session_names
+    df_raw_sessions_on_VAST["raw_data_on_VAST"] = True 
+
+    (
+        df_raw_sessions_on_VAST, 
+        df_raw_sessions_on_VAST_unique_mouse_date,
+        df_raw_sessions_on_VAST_multi_sessions_per_day
+    ) = _formatting_metadata_df(df_raw_sessions_on_VAST, source_prefix="on_VAST")
+    
+    dfs_raw_on_VAST = {
+        "df": df_raw_sessions_on_VAST,
+        "df_unique_mouse_date": df_raw_sessions_on_VAST_unique_mouse_date,
+        "df_multi_sessions_per_day": df_raw_sessions_on_VAST_multi_sessions_per_day,
+    }
 
     # --- Add sidebar ---
-    add_sidebar(dfs, df_merged, docDB_retrieve_time, df_Han_pipeline)
+    add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_retrieve_time)
 
     # --- Main contents ---
     st.markdown(f"# Data inventory for dynamic foraging")
-    cols = st.columns([1, 5])
+    cols = st.columns([1, 2])
     cols[0].markdown(f"### Merged dataframe (n = {len(df_merged)})")
     with cols[1]:
         download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
diff --git a/code/util/aws_s3.py b/code/util/aws_s3.py
index 17f258b..300dffc 100644
--- a/code/util/aws_s3.py
+++ b/code/util/aws_s3.py
@@ -37,6 +37,12 @@ def load_data(tables=['sessions'], data_source = 'bonsai'):
                      width=500)
     return df
 
+@st.cache_data(ttl=12*3600)
+def load_raw_sessions_on_VAST():
+    file_name = s3_nwb_folder['bonsai'] + 'raw_sessions_on_VAST.json'
+    with fs.open(file_name) as f:
+        raw_sessions_on_VAST = json.load(f)
+    return raw_sessions_on_VAST
 
 def draw_session_plots_quick_preview(df_to_draw_session):
 

From 751743f0792db0223b7d4da8f883100ca6cfaf7e Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Sat, 4 Jan 2025 01:51:12 +0000
Subject: [PATCH 38/68] use prefix for all

---
 code/pages/0_docDB inventory.py | 34 ++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index b6238fb..dca9ea2 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -45,7 +45,7 @@
 client = load_client()
 
 QUERY_PRESET = [
-    {"alias": "{raw, 'dynamic_foraging' in ANY software name}",
+    {"alias": "docDB_{raw, 'dynamic_foraging' in ANY software name}",
      "filter": {
          "$or":[
              {"session.data_streams.software.name": "dynamic-foraging-task"},
@@ -53,32 +53,32 @@
          ],
          "name": {"$not": {"$regex": ".*processed.*"}},
      }},
-    {"alias": "{raw, 'dynamic_foraging' in data_streams software name}",
+    {"alias": "docDB_{raw, 'dynamic_foraging' in data_streams software name}",
      "filter": {
          "session.data_streams.software.name": "dynamic-foraging-task",
          "name": {"$not": {"$regex": ".*processed.*"}},
      }},
-    {"alias": "{raw, 'dynamic_foraging' in stimulus_epochs software name}",
+    {"alias": "docDB_{raw, 'dynamic_foraging' in stimulus_epochs software name}",
      "filter": {
          "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
          "name": {"$not": {"$regex": ".*processed.*"}},
      }},
-    {"alias": "{raw, 'fib' in 'data_description.modality'}",
+    {"alias": "docDB_{raw, 'fib' in 'data_description.modality'}",
      "filter": {
          "data_description.modality.abbreviation": "fib",
          "name": {"$not": {"$regex": ".*processed.*"}},
      }},
-    {"alias": "{raw, 'fib' in 'rig.modalities'}",
+    {"alias": "docDB_{raw, 'fib' in 'rig.modalities'}",
      "filter": {
          "rig.modalities.abbreviation": "fib",
          "name": {"$not": {"$regex": ".*processed.*"}},        
      }},
-    {"alias": "{raw, 'fib' in 'session.data_streams'}",
+    {"alias": "docDB_{raw, 'fib' in 'session.data_streams'}",
      "filter": {
          "session.data_streams.stream_modalities.abbreviation": "fib",
          "name": {"$not": {"$regex": ".*processed.*"}},
      }},
-    {"alias": "{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
+    {"alias": "docDB_{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
      "filter": {
          "$or":[
              {"session.data_streams.software.name": "dynamic-foraging-task"},
@@ -87,14 +87,14 @@
          "data_description.modality.abbreviation": "ecephys",
          "name": {"$not": {"$regex": ".*processed.*"}},        
      }},
-    {"alias": "{raw, 'FIP' in name}",
+    {"alias": "docDB_{raw, 'FIP' in name}",
      "filter": {
         "name": {
             "$regex": "^FIP.*",
             "$not": {"$regex": ".*processed.*"}
         }
     }},
-    {"alias": "{processed, 'dynamic_foraging' in ANY software name}",
+    {"alias": "docDB_{processed, 'dynamic_foraging' in ANY software name}",
      "filter": {
          "$or":[
              {"session.data_streams.software.name": "dynamic-foraging-task"},
@@ -116,7 +116,7 @@ def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
         mime='text/csv'
     )
 
-def _formatting_metadata_df(df, source_prefix="in_docDB"):
+def _formatting_metadata_df(df, source_prefix="docDB_"):
     """Formatting metadata dataframe
     Given a dataframe with a column of "name" that contains nwb names
     1. parse the nwb names into subject_id, session_date, nwb_suffix.
@@ -124,8 +124,8 @@ def _formatting_metadata_df(df, source_prefix="in_docDB"):
     3. handle multiple sessions per day
     """
     
-    new_name_field = f"name_{source_prefix}"
-    df.rename(columns={"name": new_name_field}, inplace=True)
+    df.rename(columns={col: f"{source_prefix}_{col}" for col in df.columns}, inplace=True)
+    new_name_field = f"{source_prefix}_name"
     
     # Create index of subject_id, session_date, nwb_suffix by parsing nwb_name
     df[["subject_id", "session_date", "nwb_suffix"]] = df[new_name_field].apply(
@@ -147,19 +147,19 @@ def _formatting_metadata_df(df, source_prefix="in_docDB"):
         .agg({new_name_field: list, **{col: "first" for col in df.columns if col != new_name_field}})
     )
     # Add a new column to indicate multiple sessions per day
-    df_unique_mouse_date[f"multiple_sessions_per_day_{source_prefix}"] = df_unique_mouse_date[
+    df_unique_mouse_date[f"{source_prefix}_multiple_sessions_per_day"] = df_unique_mouse_date[
         new_name_field
     ].apply(lambda x: len(x) > 1)
 
     # Also return the dataframe with multiple sessions per day
     df_multi_sessions_per_day = df_unique_mouse_date[
-        df_unique_mouse_date[f"multiple_sessions_per_day_{source_prefix}"]
+        df_unique_mouse_date[f"{source_prefix}_multiple_sessions_per_day"]
     ]
 
     # Create a new column to mark duplicates in the original df
     df.loc[
         df.index.droplevel("nwb_suffix").isin(df_multi_sessions_per_day.index), 
-        f"multiple_sessions_per_day_{source_prefix}"
+        f"{source_prefix}_multiple_sessions_per_day"
     ] = True
 
     return df, df_unique_mouse_date, df_multi_sessions_per_day
@@ -363,7 +363,7 @@ def app():
     re_pattern = R"\\([^\\]*)\\(?:behavior|TrainingFolder)$"
     session_names = [re.findall(re_pattern, path)[0] for path in raw_sessions_on_VAST]
 
-    df_raw_sessions_on_VAST = pd.DataFrame(raw_sessions_on_VAST, columns=["raw_full_path_on_VAST"])
+    df_raw_sessions_on_VAST = pd.DataFrame(raw_sessions_on_VAST, columns=["VAST_full_path"])
     df_raw_sessions_on_VAST["name"] = session_names
     df_raw_sessions_on_VAST["raw_data_on_VAST"] = True 
 
@@ -371,7 +371,7 @@ def app():
         df_raw_sessions_on_VAST, 
         df_raw_sessions_on_VAST_unique_mouse_date,
         df_raw_sessions_on_VAST_multi_sessions_per_day
-    ) = _formatting_metadata_df(df_raw_sessions_on_VAST, source_prefix="on_VAST")
+    ) = _formatting_metadata_df(df_raw_sessions_on_VAST, source_prefix="VAST")
     
     dfs_raw_on_VAST = {
         "df": df_raw_sessions_on_VAST,

From ab76acc2526a87e6a38a00b7f6a66e43cf3e2dff Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Sat, 4 Jan 2025 01:59:50 +0000
Subject: [PATCH 39/68] merged in VAST

---
 code/pages/0_docDB inventory.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index dca9ea2..4ed617b 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -116,7 +116,7 @@ def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
         mime='text/csv'
     )
 
-def _formatting_metadata_df(df, source_prefix="docDB_"):
+def _formatting_metadata_df(df, source_prefix="docDB"):
     """Formatting metadata dataframe
     Given a dataframe with a column of "name" that contains nwb names
     1. parse the nwb names into subject_id, session_date, nwb_suffix.
@@ -188,10 +188,10 @@ def fetch_single_query(query):
 
     # --- Process data into dataframe ---
     df = pd.json_normalize(results)
-    df[query["alias"]] = True  # Add a column to mark the query
     
     # Formatting dataframe
     df, df_unique_mouse_date, df_multi_sessions_per_day = _formatting_metadata_df(df)
+    df_unique_mouse_date[query["alias"]] = True  # Add a column to prepare for merging
 
     return df, df_unique_mouse_date, df_multi_sessions_per_day
 
@@ -199,7 +199,7 @@ def fetch_single_query(query):
 def fetch_all_queries_from_docDB(queries_to_merge):
     """ Get merged queries from selected queries """
 
-    dfs = {} 
+    dfs = {}
     
     # Fetch data in parallel
     with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
@@ -359,20 +359,23 @@ def app():
     #     Z:\svc_aind_behavior_transfer\2023late_DataNoMeta_Reorganized\687553_2023-11-13_11-09-55\687553_2023-12-01_09-41-43\TrainingFolder
     #     Let's find the strings between two \\s that precede "behavior" or "TrainingFolder"
 
-    import re
+    # Parse "name" from full path on VAST
     re_pattern = R"\\([^\\]*)\\(?:behavior|TrainingFolder)$"
     session_names = [re.findall(re_pattern, path)[0] for path in raw_sessions_on_VAST]
-
-    df_raw_sessions_on_VAST = pd.DataFrame(raw_sessions_on_VAST, columns=["VAST_full_path"])
+    df_raw_sessions_on_VAST = pd.DataFrame(raw_sessions_on_VAST, columns=["full_path"])
     df_raw_sessions_on_VAST["name"] = session_names
     df_raw_sessions_on_VAST["raw_data_on_VAST"] = True 
 
+    # Formatting metadata dataframe
     (
         df_raw_sessions_on_VAST, 
         df_raw_sessions_on_VAST_unique_mouse_date,
         df_raw_sessions_on_VAST_multi_sessions_per_day
     ) = _formatting_metadata_df(df_raw_sessions_on_VAST, source_prefix="VAST")
     
+    # Merging with df_merged (using the unique mouse-date dataframe)
+    df_merged = df_merged.combine_first(df_raw_sessions_on_VAST_unique_mouse_date)
+    
     dfs_raw_on_VAST = {
         "df": df_raw_sessions_on_VAST,
         "df_unique_mouse_date": df_raw_sessions_on_VAST_unique_mouse_date,

From afb8d4c0c68f60dba11dd9152c59665cc9d7d9d5 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Sat, 4 Jan 2025 02:27:24 +0000
Subject: [PATCH 40/68] sort according to session_date

---
 code/pages/0_docDB inventory.py | 42 ++++++++++++++++++++++-----------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 4ed617b..aabc977 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -123,16 +123,19 @@ def _formatting_metadata_df(df, source_prefix="docDB"):
     2. remove invalid subject_id
     3. handle multiple sessions per day
     """
-    
+
     df.rename(columns={col: f"{source_prefix}_{col}" for col in df.columns}, inplace=True)
     new_name_field = f"{source_prefix}_name"
-    
+
     # Create index of subject_id, session_date, nwb_suffix by parsing nwb_name
     df[["subject_id", "session_date", "nwb_suffix"]] = df[new_name_field].apply(
         lambda x: pd.Series(split_nwb_name(x))
     )
     df["session_date"] = pd.to_datetime(df["session_date"])
-    df = df.set_index(["subject_id", "session_date", "nwb_suffix"]).sort_index()
+    df = df.set_index(["subject_id", "session_date", "nwb_suffix"]).sort_index(
+        level=["session_date", "subject_id", "nwb_suffix"],
+        ascending=[False, False, False],
+    )
 
     # Remove invalid subject_id
     df = df[(df.index.get_level_values("subject_id").astype(int) > 300000) 
@@ -145,6 +148,9 @@ def _formatting_metadata_df(df, source_prefix="docDB"):
         df.reset_index()
         .groupby(["subject_id", "session_date"])
         .agg({new_name_field: list, **{col: "first" for col in df.columns if col != new_name_field}})
+    ).sort_index(
+        level=["session_date", "subject_id"], # Restore order 
+        ascending=[False, False],
     )
     # Add a new column to indicate multiple sessions per day
     df_unique_mouse_date[f"{source_prefix}_multiple_sessions_per_day"] = df_unique_mouse_date[
@@ -338,14 +344,21 @@ def app():
     df_from_Home.loc[df_from_Home.hardware == "bonsai", "Han_temp_pipeline (bonsai)"] = True
 
     # Only keep subject_id and session_date as index
-    df_Han_pipeline = df_from_Home[
-        [
-            "subject_id",
-            "session_date",
-            "Han_temp_pipeline (bpod)",
-            "Han_temp_pipeline (bonsai)",
+    df_Han_pipeline = (
+        df_from_Home[
+            [
+                "subject_id",
+                "session_date",
+                "Han_temp_pipeline (bpod)",
+                "Han_temp_pipeline (bonsai)",
+            ]
         ]
-    ].set_index(["subject_id", "session_date"])
+        .set_index(["subject_id", "session_date"])
+        .sort_index(
+            level=["session_date", "subject_id"],
+            ascending=[False, False],
+        )
+    )
 
     # Merged with df_merged
     df_merged = df_merged.combine_first(df_Han_pipeline)
@@ -372,16 +385,17 @@ def app():
         df_raw_sessions_on_VAST_unique_mouse_date,
         df_raw_sessions_on_VAST_multi_sessions_per_day
     ) = _formatting_metadata_df(df_raw_sessions_on_VAST, source_prefix="VAST")
-    
-    # Merging with df_merged (using the unique mouse-date dataframe)
-    df_merged = df_merged.combine_first(df_raw_sessions_on_VAST_unique_mouse_date)
-    
+
     dfs_raw_on_VAST = {
         "df": df_raw_sessions_on_VAST,
         "df_unique_mouse_date": df_raw_sessions_on_VAST_unique_mouse_date,
         "df_multi_sessions_per_day": df_raw_sessions_on_VAST_multi_sessions_per_day,
     }
 
+    # Merging with df_merged (using the unique mouse-date dataframe)
+    df_merged = df_merged.combine_first(df_raw_sessions_on_VAST_unique_mouse_date)
+    df_merged.sort_index(level=["session_date", "subject_id"], ascending=[False, False], inplace=True)
+
     # --- Add sidebar ---
     add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_retrieve_time)
 

From 8286468b6730d5146ed3e2992cc2f280b96dce36 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Mon, 6 Jan 2025 19:15:16 +0000
Subject: [PATCH 41/68] add a flag in ss for bpod load status

---
 code/Home.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/code/Home.py b/code/Home.py
index 7abe455..f4c7646 100644
--- a/code/Home.py
+++ b/code/Home.py
@@ -303,8 +303,10 @@ def init(if_load_bpod_data_override=None, if_load_docDB_override=None):
         if 'if_load_bpod_sessions' in st.session_state
         else False)
     
+    st.session_state.bpod_loaded = False
     if _if_load_bpod:
         df_bpod = load_data(['sessions'], data_source='bpod')
+        st.session_state.bpod_loaded = True
         
         # For historial reason, the suffix of df['sessions_bonsai'] just mean the data of the Home.py page
         df['sessions_bonsai'] = pd.concat([df['sessions_bonsai'], df_bpod['sessions_bonsai']], axis=0)

From 632ec489a8d2573e3ecb10570cfa1b58f38c1f6a Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Mon, 6 Jan 2025 19:22:47 +0000
Subject: [PATCH 42/68] add VAST to sidebar

---
 code/pages/0_docDB inventory.py | 65 +++++++++++++++++++--------------
 1 file changed, 37 insertions(+), 28 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index aabc977..2ade1b6 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -266,6 +266,34 @@ def venn(df, columns_to_venn):
         )
     return fig
 
+def _show_records_on_sidebar(dfs, file_name_prefix, source_str="docDB"):
+    df = dfs["df"]
+    df_multi_sessions_per_day = dfs["df_multi_sessions_per_day"]
+    df_unique_mouse_date = dfs["df_unique_mouse_date"]
+
+    with st.expander(f"{len(df)} records from {source_str}"):
+        download_df(df, label="Download as CSV", file_name=f"{file_name_prefix}.csv")
+        st.write(df)
+
+    with st.expander(f"{len(df_multi_sessions_per_day)} have multiple sessions per day"):
+        download_df(
+            df_multi_sessions_per_day,
+            label="Download as CSV",
+            file_name=f"{file_name_prefix}_multi_sessions_per_day.csv",
+        )
+        st.write(df_multi_sessions_per_day)
+
+    with st.expander(f"{len(df_unique_mouse_date)} unique mouse-date pairs"):
+        download_df(
+            df_unique_mouse_date,
+            label="Download as CSV",
+            file_name=f"{file_name_prefix}_unique_mouse_date.csv",
+        )
+        st.write(df_unique_mouse_date)
+        
+    # if len(df_unique_mouse_date) != df_merged[query["alias"]].sum(): 
+    #     st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')
+
 def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_retrieve_time):
     # Sidebar
     with st.sidebar:
@@ -281,33 +309,9 @@ def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_re
                 query_json = json.dumps(query["filter"], indent=4)
                 st.code(query_json)
 
-                # Show records
-                df = dfs_docDB[query["alias"]]["df"]
-                df_multi_sessions_per_day = dfs_docDB[query["alias"]]["df_multi_sessions_per_day"]
-                df_unique_mouse_date = dfs_docDB[query["alias"]]["df_unique_mouse_date"]
-
-                with st.expander(f"{len(df)} records returned from docDB"):
-                    download_df(df, label="Download as CSV", file_name=f"{query['alias']}.csv")
-                    st.write(df)
-
-                with st.expander(f"{len(df_multi_sessions_per_day)} have multiple sessions per day"):
-                    download_df(
-                        df_multi_sessions_per_day,
-                        label="Download as CSV",
-                        file_name=f"{query['alias']}_multi_sessions_per_day.csv",
-                    )
-                    st.write(df_multi_sessions_per_day)
-
-                with st.expander(f"{len(df_unique_mouse_date)} unique mouse-date pairs"):
-                    download_df(
-                        df_unique_mouse_date,
-                        label="Download as CSV",
-                        file_name=f"{query['alias']}_unique_mouse_date.csv",
-                    )
-                    st.write(df_unique_mouse_date)
+                # Show records                
+                _show_records_on_sidebar(dfs_docDB[query["alias"]], file_name_prefix=query["alias"], source_str="docDB")
 
-            if len(df_unique_mouse_date) != df_merged[query["alias"]].sum():
-                st.warning('''len(df_unique_mouse_date) != df_merged[query["alias"]].sum()!''')
         st.markdown(f"Retrieving data from docDB (or st.cache) took {docDB_retrieve_time:.3f} secs.")
 
         st.markdown('''## 2. From Han's temporary pipeline (the "Home" page)''')
@@ -326,6 +330,10 @@ def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_re
                     file_name=f"Han_temp_pipeline_{hardware}.csv",
                 )
                 st.write(df_this_hardware)
+                
+        st.markdown('''## 3. From VAST: existing raw data''')
+        _show_records_on_sidebar(dfs_raw_on_VAST, file_name_prefix="raw_on_VAST", source_str="VAST /scratch")
+        
 
 
 def app():
@@ -402,7 +410,7 @@ def app():
     # --- Main contents ---
     st.markdown(f"# Data inventory for dynamic foraging")
     cols = st.columns([1, 2])
-    cols[0].markdown(f"### Merged dataframe (n = {len(df_merged)})")
+    cols[0].markdown(f"### Merged metadata (n = {len(df_merged)}, see the sidebar for details)")
     with cols[1]:
         download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
 
@@ -422,6 +430,7 @@ def app():
     query_keys = [
         "Han_temp_pipeline (bpod)",
         "Han_temp_pipeline (bonsai)",
+        "VAST_raw_data_on_VAST",
     ] + [query["alias"] for query in QUERY_PRESET]
     selected_queries = st.multiselect(
         "Select queries to filter sessions",
@@ -438,7 +447,7 @@ def app():
 if __name__ == "__main__":
     
     # Share the same master df as the Home page
-    if "df" not in st.session_state or "sessions_bonsai" not in st.session_state.df.keys():
+    if "df" not in st.session_state or "sessions_bonsai" not in st.session_state.df.keys() or not st.session_state.bpod_loaded:
         init(if_load_docDB_override=False, if_load_bpod_data_override=True)
 
     app()

From a73adffa6c2283f02f5071dc5d0bb10803be8870 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Mon, 6 Jan 2025 20:46:51 +0000
Subject: [PATCH 43/68] load preset from json

---
 code/pages/0_docDB inventory.py  | 64 ++------------------------------
 data_inventory_QUERY_PRESET.json | 59 +++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 60 deletions(-)
 create mode 100644 data_inventory_QUERY_PRESET.json

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 2ade1b6..f78fa10 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -44,65 +44,9 @@
 
 client = load_client()
 
-QUERY_PRESET = [
-    {"alias": "docDB_{raw, 'dynamic_foraging' in ANY software name}",
-     "filter": {
-         "$or":[
-             {"session.data_streams.software.name": "dynamic-foraging-task"},
-             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-         ],
-         "name": {"$not": {"$regex": ".*processed.*"}},
-     }},
-    {"alias": "docDB_{raw, 'dynamic_foraging' in data_streams software name}",
-     "filter": {
-         "session.data_streams.software.name": "dynamic-foraging-task",
-         "name": {"$not": {"$regex": ".*processed.*"}},
-     }},
-    {"alias": "docDB_{raw, 'dynamic_foraging' in stimulus_epochs software name}",
-     "filter": {
-         "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
-         "name": {"$not": {"$regex": ".*processed.*"}},
-     }},
-    {"alias": "docDB_{raw, 'fib' in 'data_description.modality'}",
-     "filter": {
-         "data_description.modality.abbreviation": "fib",
-         "name": {"$not": {"$regex": ".*processed.*"}},
-     }},
-    {"alias": "docDB_{raw, 'fib' in 'rig.modalities'}",
-     "filter": {
-         "rig.modalities.abbreviation": "fib",
-         "name": {"$not": {"$regex": ".*processed.*"}},        
-     }},
-    {"alias": "docDB_{raw, 'fib' in 'session.data_streams'}",
-     "filter": {
-         "session.data_streams.stream_modalities.abbreviation": "fib",
-         "name": {"$not": {"$regex": ".*processed.*"}},
-     }},
-    {"alias": "docDB_{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
-     "filter": {
-         "$or":[
-             {"session.data_streams.software.name": "dynamic-foraging-task"},
-             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-         ],
-         "data_description.modality.abbreviation": "ecephys",
-         "name": {"$not": {"$regex": ".*processed.*"}},        
-     }},
-    {"alias": "docDB_{raw, 'FIP' in name}",
-     "filter": {
-        "name": {
-            "$regex": "^FIP.*",
-            "$not": {"$regex": ".*processed.*"}
-        }
-    }},
-    {"alias": "docDB_{processed, 'dynamic_foraging' in ANY software name}",
-     "filter": {
-         "$or":[
-             {"session.data_streams.software.name": "dynamic-foraging-task"},
-             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"},            
-         ],
-         "name": {"$regex": ".*processed.*"},
-     }},
-]
+# Load QUERY_PRESET from json
+with open("data_inventory_QUERY_PRESET.json", "r") as f:
+    QUERY_PRESET = json.load(f)
 
 def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
     """ Add a button to download df as csv """
@@ -331,7 +275,7 @@ def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_re
                 )
                 st.write(df_this_hardware)
                 
-        st.markdown('''## 3. From VAST: existing raw data''')
+        st.markdown('''## 3. From VAST /scratch: existing raw data''')
         _show_records_on_sidebar(dfs_raw_on_VAST, file_name_prefix="raw_on_VAST", source_str="VAST /scratch")
         
 
diff --git a/data_inventory_QUERY_PRESET.json b/data_inventory_QUERY_PRESET.json
new file mode 100644
index 0000000..f89451e
--- /dev/null
+++ b/data_inventory_QUERY_PRESET.json
@@ -0,0 +1,59 @@
+[
+    {"alias": "docDB_{raw, 'dynamic_foraging' in ANY software name}",
+     "filter": {
+         "$or":[
+             {"session.data_streams.software.name": "dynamic-foraging-task"},
+             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"}           
+         ],
+         "name": {"$not": {"$regex": ".*processed.*"}}
+     }},
+    {"alias": "docDB_{raw, 'dynamic_foraging' in data_streams software name}",
+     "filter": {
+         "session.data_streams.software.name": "dynamic-foraging-task",
+         "name": {"$not": {"$regex": ".*processed.*"}}
+     }},
+    {"alias": "docDB_{raw, 'dynamic_foraging' in stimulus_epochs software name}",
+     "filter": {
+         "session.stimulus_epochs.software.name": "dynamic-foraging-task",            
+         "name": {"$not": {"$regex": ".*processed.*"}}
+     }},
+    {"alias": "docDB_{raw, 'fib' in 'data_description.modality'}",
+     "filter": {
+         "data_description.modality.abbreviation": "fib",
+         "name": {"$not": {"$regex": ".*processed.*"}}
+     }},
+    {"alias": "docDB_{raw, 'fib' in 'rig.modalities'}",
+     "filter": {
+         "rig.modalities.abbreviation": "fib",
+         "name": {"$not": {"$regex": ".*processed.*"}}      
+     }},
+    {"alias": "docDB_{raw, 'fib' in 'session.data_streams'}",
+     "filter": {
+         "session.data_streams.stream_modalities.abbreviation": "fib",
+         "name": {"$not": {"$regex": ".*processed.*"}}
+     }},
+    {"alias": "docDB_{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
+     "filter": {
+         "$or":[
+             {"session.data_streams.software.name": "dynamic-foraging-task"},
+             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"}           
+         ],
+         "data_description.modality.abbreviation": "ecephys",
+         "name": {"$not": {"$regex": ".*processed.*"}}      
+     }},
+    {"alias": "docDB_{raw, 'FIP' in name}",
+     "filter": {
+        "name": {
+            "$regex": "^FIP.*",
+            "$not": {"$regex": ".*processed.*"}
+        }
+    }},
+    {"alias": "docDB_{processed, 'dynamic_foraging' in ANY software name}",
+     "filter": {
+         "$or":[
+             {"session.data_streams.software.name": "dynamic-foraging-task"},
+             {"session.stimulus_epochs.software.name": "dynamic-foraging-task"}            
+         ],
+         "name": {"$regex": ".*processed.*"}
+     }}
+]
\ No newline at end of file

From e5e4378bbda2d6d3a7cfd66ee0d0606e9bb95f27 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Mon, 6 Jan 2025 20:54:17 +0000
Subject: [PATCH 44/68] add venn presets

---
 code/pages/0_docDB inventory.py | 18 +++++++++++++++++-
 data_inventory_VENN_PRESET.json | 12 ++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 data_inventory_VENN_PRESET.json

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index f78fa10..65acec8 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -369,8 +369,24 @@ def app():
             )
         ],
     )
-
+    
+    # --- Venn diagram from presets ---
+    with open("data_inventory_VENN_PRESET.json", "r") as f:
+        VENN_PRESET = json.load(f)
+
+    n_col = 2
+    if VENN_PRESET:
+        st.markdown("## Venn diagram from presets")
+        cols = st.columns([1] * n_col)
+        for i, venn_preset in enumerate(VENN_PRESET):
+            cols[i % n_col].markdown(f"#### {venn_preset['name']}")
+            fig = venn(df_merged, venn_preset['pairs'])
+            cols[i % n_col].pyplot(fig, use_container_width=True)
+    
+    # --- User-defined Venn diagram ---
     # Multiselect for selecting queries up to three
+    st.markdown('---')
+    st.markdown("## Venn diagram from user-selected queries")
     query_keys = [
         "Han_temp_pipeline (bpod)",
         "Han_temp_pipeline (bonsai)",
diff --git a/data_inventory_VENN_PRESET.json b/data_inventory_VENN_PRESET.json
new file mode 100644
index 0000000..c2f4f94
--- /dev/null
+++ b/data_inventory_VENN_PRESET.json
@@ -0,0 +1,12 @@
+[
+    {"name": "Overview (Han temp pipeline and docDB dynamic foraging sessions)",
+    "pairs":[
+        "Han_temp_pipeline (bpod)",
+        "Han_temp_pipeline (bonsai)",
+        "docDB_{raw, 'dynamic_foraging' in ANY software name}"
+    ]
+}
+
+
+    
+]
\ No newline at end of file

From c6333f7739a7b6e0204cbe8d4dbcc7aeab91d45a Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Mon, 6 Jan 2025 21:00:22 +0000
Subject: [PATCH 45/68] add more preset

---
 code/pages/0_docDB inventory.py |  4 ++--
 data_inventory_VENN_PRESET.json | 34 +++++++++++++++++++++++----------
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_docDB inventory.py
index 65acec8..405155c 100644
--- a/code/pages/0_docDB inventory.py	
+++ b/code/pages/0_docDB inventory.py	
@@ -379,9 +379,9 @@ def app():
         st.markdown("## Venn diagram from presets")
         cols = st.columns([1] * n_col)
         for i, venn_preset in enumerate(VENN_PRESET):
-            cols[i % n_col].markdown(f"#### {venn_preset['name']}")
+            cols[i % n_col].markdown(f"##### ({i+1}). {venn_preset['name']}")
             fig = venn(df_merged, venn_preset['pairs'])
-            cols[i % n_col].pyplot(fig, use_container_width=True)
+            cols[i % n_col].pyplot(fig, use_container_width=True) 
     
     # --- User-defined Venn diagram ---
     # Multiselect for selecting queries up to three
diff --git a/data_inventory_VENN_PRESET.json b/data_inventory_VENN_PRESET.json
index c2f4f94..99b112c 100644
--- a/data_inventory_VENN_PRESET.json
+++ b/data_inventory_VENN_PRESET.json
@@ -1,12 +1,26 @@
 [
-    {"name": "Overview (Han temp pipeline and docDB dynamic foraging sessions)",
-    "pairs":[
-        "Han_temp_pipeline (bpod)",
-        "Han_temp_pipeline (bonsai)",
-        "docDB_{raw, 'dynamic_foraging' in ANY software name}"
-    ]
-}
-
-
-    
+    {
+        "name": "Han temp pipeline and docDB dynamic foraging session)",
+        "pairs": [
+            "Han_temp_pipeline (bpod)",
+            "Han_temp_pipeline (bonsai)",
+            "docDB_{raw, 'dynamic_foraging' in ANY software name}"
+        ]
+    },
+    {
+        "name": "Han temp pipeline and Kenta's uploaded FIP* sessions",
+        "pairs": [
+            "Han_temp_pipeline (bpod)",
+            "Han_temp_pipeline (bonsai)",
+            "docDB_{raw, 'FIP' in name}"
+        ]
+    },
+    {
+        "name": "Raw data lost (Han temp pipeline, docDB dynamic foraging, and raw data on VAST)",
+        "pairs": [
+            "Han_temp_pipeline (bonsai)",
+            "docDB_{raw, 'dynamic_foraging' in ANY software name}",
+            "VAST_raw_data_on_VAST"
+        ]
+    }
 ]
\ No newline at end of file

From 2ad50b1cbd2b064551d5cadcbdd3748662141616 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Mon, 6 Jan 2025 22:35:58 +0000
Subject: [PATCH 46/68] change name

---
 code/pages/{0_docDB inventory.py => 0_Data inventory.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename code/pages/{0_docDB inventory.py => 0_Data inventory.py} (100%)

diff --git a/code/pages/0_docDB inventory.py b/code/pages/0_Data inventory.py
similarity index 100%
rename from code/pages/0_docDB inventory.py
rename to code/pages/0_Data inventory.py

From 6277f86e59f8b1e6d9c4c8b820c418b48b22e975 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Mon, 6 Jan 2025 22:36:57 +0000
Subject: [PATCH 47/68] minor

---
 code/pages/0_Data inventory.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 405155c..4b2592b 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -146,7 +146,7 @@ def fetch_single_query(query):
     return df, df_unique_mouse_date, df_multi_sessions_per_day
 
 @st.cache_data(ttl=3600 * 24)
-def fetch_all_queries_from_docDB(queries_to_merge):
+def fetch_all_queries_from_docDB_in_parallel(queries_to_merge):
     """ Get merged queries from selected queries """
 
     dfs = {}
@@ -256,7 +256,7 @@ def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_re
                 # Show records                
                 _show_records_on_sidebar(dfs_docDB[query["alias"]], file_name_prefix=query["alias"], source_str="docDB")
 
-        st.markdown(f"Retrieving data from docDB (or st.cache) took {docDB_retrieve_time:.3f} secs.")
+        st.markdown(f"Retrieving data from docDB in parallel (or st.cache) took {docDB_retrieve_time:.3f} secs.")
 
         st.markdown('''## 2. From Han's temporary pipeline (the "Home" page)''')
         hardwares = ["bonsai", "bpod"]
@@ -284,7 +284,7 @@ def app():
 
     # --- 1. Generate combined dataframe from docDB queries ---
     start_time = time.time()
-    df_merged, dfs_docDB = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
+    df_merged, dfs_docDB = fetch_all_queries_from_docDB_in_parallel(queries_to_merge=QUERY_PRESET)
     docDB_retrieve_time = time.time() - start_time
 
     # --- 2. Merge in the master df in the Home page (Han's temporary pipeline) ---

From adab85ee342511048159cdefe49a05c407d560e8 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 00:15:56 +0000
Subject: [PATCH 48/68] add edge color control; refactor code

---
 code/pages/0_Data inventory.py  | 130 ++++++++++++++++++++------------
 data_inventory_VENN_PRESET.json |  21 +++++-
 2 files changed, 101 insertions(+), 50 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 4b2592b..9fef7fd 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -4,7 +4,7 @@
 import threading
 import json
 
-from matplotlib_venn import venn2, venn3
+from matplotlib_venn import venn2, venn3, venn2_circles, venn3_circles
 import matplotlib.pyplot as plt
 import streamlit as st
 from streamlit_dynamic_filters import DynamicFilters
@@ -114,6 +114,7 @@ def _formatting_metadata_df(df, source_prefix="docDB"):
 
     return df, df_unique_mouse_date, df_multi_sessions_per_day
 
+@st.cache_data(ttl=3600*24)
 def fetch_single_query(query):
     """ Fetch a query from docDB and process the result into dataframe
     
@@ -145,36 +146,43 @@ def fetch_single_query(query):
 
     return df, df_unique_mouse_date, df_multi_sessions_per_day
 
-@st.cache_data(ttl=3600 * 24)
-def fetch_all_queries_from_docDB_in_parallel(queries_to_merge):
+
+def fetch_all_queries_from_docDB(queries_to_merge, parallel=False):
     """ Get merged queries from selected queries """
 
     dfs = {}
     
     # Fetch data in parallel
-    with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
-        future_to_query = {executor.submit(fetch_single_query, key): key for key in queries_to_merge}
-        for i, future in enumerate(as_completed(future_to_query), 1):
-            key = future_to_query[future]
-            try:
-                df, df_unique_mouse_date, df_multi_sessions_per_day = future.result()
-                dfs[key["alias"]] = { 
-                    "df": df,
-                    "df_unique_mouse_date": df_unique_mouse_date,
-                    "df_multi_sessions_per_day": df_multi_sessions_per_day,
-                }
-            except Exception as e:
-                print(f"Error querying {key}: {e}")
-
-    # Fetch data in serial 
-    # for query in queries_to_merge:
-    #     df, df_unique_mouse_date, df_multi_sessions_per_day = fetch_single_query(query)
-    #     dfs[query["alias"]] = {
-    #         "df": df,
-    #         "df_unique_mouse_date": df_unique_mouse_date,
-    #         "df_multi_sessions_per_day": df_multi_sessions_per_day,
-    #     } 
-
+    if parallel:
+        with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
+            future_to_query = {executor.submit(fetch_single_query, key): key for key in queries_to_merge}
+            for i, future in enumerate(as_completed(future_to_query), 1):
+                key = future_to_query[future]
+                try:
+                    df, df_unique_mouse_date, df_multi_sessions_per_day = future.result()
+                    dfs[key["alias"]] = { 
+                        "df": df,
+                        "df_unique_mouse_date": df_unique_mouse_date,
+                        "df_multi_sessions_per_day": df_multi_sessions_per_day,
+                    }
+                except Exception as e:
+                    print(f"Error querying {key}: {e}")
+    else:
+        # Fetch data in serial
+        p_bar = st.progress(0, text="Querying docDB...")
+        for i, query in enumerate(queries_to_merge):
+            df, df_unique_mouse_date, df_multi_sessions_per_day = fetch_single_query(query)
+            dfs[query["alias"]] = {
+                "df": df,
+                "df_unique_mouse_date": df_unique_mouse_date,
+                "df_multi_sessions_per_day": df_multi_sessions_per_day,
+            }
+            p_bar.progress((i+1) / len(queries_to_merge), text=f"Querying docDB... ({i+1}/{len(queries_to_merge)})")
+
+    if not dfs:
+        st.warning("Querying docDB error! Refresh the page to try again or ask Han.")
+        return None, None
+    
     # Combine queried dfs using df_unique_mouse_date (on index "subject_id", "session_date" only)
     df_merged = dfs[queries_to_merge[0]["alias"]]["df_unique_mouse_date"]
     for df in [dfs[query["alias"]]["df_unique_mouse_date"] for query in queries_to_merge[1:]]:
@@ -190,24 +198,40 @@ def fetch_all_queries_from_docDB_in_parallel(queries_to_merge):
     )
     return df_merged, dfs
 
-def venn(df, columns_to_venn):
+def generate_venn(df, columns_to_venn, edge_colors=None):
     """ Show venn diagram """
     if len(columns_to_venn) > 3:
         st.write("Venn diagram only supports up to 3 columns.")
         return
     
     fig, ax = plt.subplots()
-    
     if len(columns_to_venn) == 2:
-        venn2(
-            [set(df.index[df[col]==True]) for col in columns_to_venn],
-            set_labels=columns_to_venn,
-        )
+        v_func = venn2
+        c_func = venn2_circles
+    elif len(columns_to_venn) == 3:
+        v_func = venn3
+        c_func = venn3_circles
     else:
-        venn3(
-            [set(df.index[df[col]==True]) for col in columns_to_venn],
-            set_labels=columns_to_venn,
-        )
+        st.warning("Number of columns to venn should be 2 or 3.")
+        return None
+        
+    v = v_func(
+        [set(df.index[df[col]==True]) for col in columns_to_venn],
+        set_labels=columns_to_venn,
+    )
+    c = c_func(
+        [set(df.index[df[col]==True]) for col in columns_to_venn],
+    )
+        
+    # Set edge colur
+    if edge_colors: 
+        for i, circle in enumerate(c):
+            c[i].set_edgecolor(edge_colors[i])
+    
+    for patch in v.patches:
+        if patch:  # Some patches might be None
+            patch.set_facecolor('none')
+
     return fig
 
 def _show_records_on_sidebar(dfs, file_name_prefix, source_str="docDB"):
@@ -241,10 +265,6 @@ def _show_records_on_sidebar(dfs, file_name_prefix, source_str="docDB"):
 def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_retrieve_time):
     # Sidebar
     with st.sidebar:
-        st.markdown('# Metadata sources:')
-
-        st.markdown('## 1. From docDB queries')
-        st.markdown('#### See [how to use these queries](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb) in your own code.')
         for query in QUERY_PRESET:
             with st.expander(f"### {query['alias']}"):
 
@@ -255,8 +275,7 @@ def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_re
 
                 # Show records                
                 _show_records_on_sidebar(dfs_docDB[query["alias"]], file_name_prefix=query["alias"], source_str="docDB")
-
-        st.markdown(f"Retrieving data from docDB in parallel (or st.cache) took {docDB_retrieve_time:.3f} secs.")
+        st.markdown('#### See [how to use above queries](https://aind-data-access-api.readthedocs.io/en/latest/UserGuide.html#document-database-docdb) in your own code.')
 
         st.markdown('''## 2. From Han's temporary pipeline (the "Home" page)''')
         hardwares = ["bonsai", "bpod"]
@@ -283,9 +302,25 @@ def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_re
 def app():
 
     # --- 1. Generate combined dataframe from docDB queries ---
-    start_time = time.time()
-    df_merged, dfs_docDB = fetch_all_queries_from_docDB_in_parallel(queries_to_merge=QUERY_PRESET)
-    docDB_retrieve_time = time.time() - start_time
+    with st.sidebar:
+        st.markdown('# Metadata sources:')
+        st.markdown('## 1. From docDB queries')
+
+        cols = st.columns([1.5, 1])
+        with cols[0]:
+            start_time = time.time()
+            df_merged, dfs_docDB = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
+            docDB_retrieve_time = time.time() - start_time
+            st.markdown(f"Finished in {docDB_retrieve_time:.3f} secs.")
+            
+        with cols[1]:
+            if st.button('Re-fetch docDB queries'):
+                st.cache_data.clear()
+                st.rerun()
+
+    if df_merged is None:
+        st.cache_data.clear()
+        return
 
     # --- 2. Merge in the master df in the Home page (Han's temporary pipeline) ---
     # Data from Home.init (all sessions from Janelia bpod + AIND bpod + AIND bonsai)
@@ -380,7 +415,7 @@ def app():
         cols = st.columns([1] * n_col)
         for i, venn_preset in enumerate(VENN_PRESET):
             cols[i % n_col].markdown(f"##### ({i+1}). {venn_preset['name']}")
-            fig = venn(df_merged, venn_preset['pairs'])
+            fig = generate_venn(df_merged, venn_preset['columns'], edge_colors=venn_preset.get("edge_colors", None))
             cols[i % n_col].pyplot(fig, use_container_width=True) 
     
     # --- User-defined Venn diagram ---
@@ -400,7 +435,7 @@ def app():
     )
 
     columns_to_venn = selected_queries
-    fig = venn(df_merged, columns_to_venn)
+    fig = generate_venn(df_merged, columns_to_venn)
     st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
 
 
@@ -408,6 +443,7 @@ def app():
     
     # Share the same master df as the Home page
     if "df" not in st.session_state or "sessions_bonsai" not in st.session_state.df.keys() or not st.session_state.bpod_loaded:
+        st.spinner("Loading data from Han temp pipeline...")
         init(if_load_docDB_override=False, if_load_bpod_data_override=True)
 
     app()
diff --git a/data_inventory_VENN_PRESET.json b/data_inventory_VENN_PRESET.json
index 99b112c..ec7b038 100644
--- a/data_inventory_VENN_PRESET.json
+++ b/data_inventory_VENN_PRESET.json
@@ -1,26 +1,41 @@
 [
     {
         "name": "Han temp pipeline and docDB dynamic foraging session)",
-        "pairs": [
+        "columns": [
             "Han_temp_pipeline (bpod)",
             "Han_temp_pipeline (bonsai)",
             "docDB_{raw, 'dynamic_foraging' in ANY software name}"
+        ],
+        "edge_colors": [
+            "lightblue",
+            "blue",
+            "green"
         ]
     },
     {
         "name": "Han temp pipeline and Kenta's uploaded FIP* sessions",
-        "pairs": [
+        "columns": [
             "Han_temp_pipeline (bpod)",
             "Han_temp_pipeline (bonsai)",
             "docDB_{raw, 'FIP' in name}"
+        ],
+        "edge_colors": [
+            "lightblue",
+            "blue",
+            "lightgreen"
         ]
     },
     {
         "name": "Raw data lost (Han temp pipeline, docDB dynamic foraging, and raw data on VAST)",
-        "pairs": [
+        "columns": [
             "Han_temp_pipeline (bonsai)",
             "docDB_{raw, 'dynamic_foraging' in ANY software name}",
             "VAST_raw_data_on_VAST"
+        ],
+        "edge_colors": [
+            "blue",
+            "green",
+            "red"
         ]
     }
 ]
\ No newline at end of file

From ad2e2dba2825a03aed7e839d2cbb9236d8dbf282 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 01:51:11 +0000
Subject: [PATCH 49/68] add edge styles; add more venn diagrams

---
 code/pages/0_Data inventory.py  |  51 +++++++----
 data_inventory_VENN_PRESET.json | 155 ++++++++++++++++++++++++++++++--
 2 files changed, 183 insertions(+), 23 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 9fef7fd..990c085 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -198,7 +198,7 @@ def fetch_all_queries_from_docDB(queries_to_merge, parallel=False):
     )
     return df_merged, dfs
 
-def generate_venn(df, columns_to_venn, edge_colors=None):
+def generate_venn(df, columns_to_venn, edge_colors=None, edge_styles=None, patch_settings=None):
     """ Show venn diagram """
     if len(columns_to_venn) > 3:
         st.write("Venn diagram only supports up to 3 columns.")
@@ -223,14 +223,29 @@ def generate_venn(df, columns_to_venn, edge_colors=None):
         [set(df.index[df[col]==True]) for col in columns_to_venn],
     )
         
-    # Set edge colur
+    # Set edge color
     if edge_colors: 
         for i, circle in enumerate(c):
             c[i].set_edgecolor(edge_colors[i])
+            v.get_label_by_id(["A", "B", "C"][i]).set_color(edge_colors[i])
+            
+    # Set edge style
+    if edge_styles:
+        for i, circle in enumerate(c):
+            c[i].set_linestyle(edge_styles[i])
     
+    # Set patch color
     for patch in v.patches:
         if patch:  # Some patches might be None
             patch.set_facecolor('none')
+    if patch_settings:
+        for patch_setting in patch_settings:
+            # Set color
+            for patch_id in patch_setting["patch_ids"]:
+                if v.get_patch_by_id(patch_id):
+                    v.get_patch_by_id(patch_id).set_color(patch_setting["color"])
+            # Add notes
+            st.markdown(f"#### :{patch_setting['emoji']}: :{patch_setting['color']}[{patch_setting['notes']}]")
 
     return fig
 
@@ -243,6 +258,7 @@ def _show_records_on_sidebar(dfs, file_name_prefix, source_str="docDB"):
         download_df(df, label="Download as CSV", file_name=f"{file_name_prefix}.csv")
         st.write(df)
 
+    st.markdown(":heavy_exclamation_mark: :red[Multiple sessions per day should be resolved!]")
     with st.expander(f"{len(df_multi_sessions_per_day)} have multiple sessions per day"):
         download_df(
             df_multi_sessions_per_day,
@@ -296,7 +312,6 @@ def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_re
                 
         st.markdown('''## 3. From VAST /scratch: existing raw data''')
         _show_records_on_sidebar(dfs_raw_on_VAST, file_name_prefix="raw_on_VAST", source_str="VAST /scratch")
-        
 
 
 def app():
@@ -312,7 +327,7 @@ def app():
             df_merged, dfs_docDB = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
             docDB_retrieve_time = time.time() - start_time
             st.markdown(f"Finished in {docDB_retrieve_time:.3f} secs.")
-            
+
         with cols[1]:
             if st.button('Re-fetch docDB queries'):
                 st.cache_data.clear()
@@ -388,10 +403,8 @@ def app():
 
     # --- Main contents ---
     st.markdown(f"# Data inventory for dynamic foraging")
-    cols = st.columns([1, 2])
-    cols[0].markdown(f"### Merged metadata (n = {len(df_merged)}, see the sidebar for details)")
-    with cols[1]:
-        download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
+    st.markdown(f"### Merged metadata (n = {len(df_merged)}, see the sidebar for details)")
+    download_df(df_merged, label="Download merged df as CSV", file_name="df_docDB_queries.csv")
 
     aggrid_interactive_table_basic(
         df_merged.reset_index(),
@@ -404,20 +417,28 @@ def app():
             )
         ],
     )
-    
+
     # --- Venn diagram from presets ---
     with open("data_inventory_VENN_PRESET.json", "r") as f:
         VENN_PRESET = json.load(f)
 
-    n_col = 2
     if VENN_PRESET:
         st.markdown("## Venn diagram from presets")
-        cols = st.columns([1] * n_col)
         for i, venn_preset in enumerate(VENN_PRESET):
-            cols[i % n_col].markdown(f"##### ({i+1}). {venn_preset['name']}")
-            fig = generate_venn(df_merged, venn_preset['columns'], edge_colors=venn_preset.get("edge_colors", None))
-            cols[i % n_col].pyplot(fig, use_container_width=True) 
-    
+            st.markdown(f"### ({i+1}). {venn_preset['name']}")
+            cols = st.columns([1.5, 1])
+            fig = generate_venn(
+                    df_merged,
+                    venn_preset["columns"],
+                    edge_colors=venn_preset.get("edge_colors", None),
+                    edge_styles=venn_preset.get("edge_styles", None),
+                    patch_settings=venn_preset.get("patch_settings", None),
+                )
+            with cols[0]:
+                st.pyplot(fig, use_container_width=True)
+            
+            st.markdown("---")
+
     # --- User-defined Venn diagram ---
     # Multiselect for selecting queries up to three
     st.markdown('---')
diff --git a/data_inventory_VENN_PRESET.json b/data_inventory_VENN_PRESET.json
index ec7b038..e2dfc5c 100644
--- a/data_inventory_VENN_PRESET.json
+++ b/data_inventory_VENN_PRESET.json
@@ -1,41 +1,180 @@
 [
     {
-        "name": "Han temp pipeline and docDB dynamic foraging session)",
+        "name": "Bonsai-based dynamic foraging sessions that are missing in docDB/CO",
         "columns": [
             "Han_temp_pipeline (bpod)",
             "Han_temp_pipeline (bonsai)",
             "docDB_{raw, 'dynamic_foraging' in ANY software name}"
         ],
         "edge_colors": [
-            "lightblue",
-            "blue",
+            "grey",
+            "black",
             "green"
+        ],
+        "edge_styles": [
+            "dashed",
+            "solid",
+            "solid"
+        ],
+        "patch_settings":[
+            {
+                "patch_ids": ["010"],
+                "color": "red",
+                "emoji": "heavy_exclamation_mark",
+                "notes": "Missing sessions on docDB (bonsai)"
+            },
+            {
+                "patch_ids": ["001"],
+                "color": "orange",
+                "emoji": "large_yellow_square",
+                "notes": "Dynamic foraging sessions uploaded by some ephys rigs that Han's pipeline didn't ingest (which is fine)"
+            }
         ]
     },
     {
-        "name": "Han temp pipeline and Kenta's uploaded FIP* sessions",
+        "name": "Bpod-based (old) dynamic foraging sessions that are missing in docDB/CO",
         "columns": [
             "Han_temp_pipeline (bpod)",
             "Han_temp_pipeline (bonsai)",
             "docDB_{raw, 'FIP' in name}"
         ],
         "edge_colors": [
-            "lightblue",
-            "blue",
-            "lightgreen"
+            "grey",
+            "black",
+            "limegreen"
+        ],
+        "edge_styles": [
+            "solid",
+            "dashed",
+            "solid"
+        ],    
+        "patch_settings":[
+            {
+                "patch_ids": ["100"],
+                "color": "red",
+                "emoji": "heavy_exclamation_mark",
+                "notes": "Missing sessions on docDB (bpod)"
+            }
         ]
     },
     {
-        "name": "Raw data lost (Han temp pipeline, docDB dynamic foraging, and raw data on VAST)",
+        "name": "We may have lost some raw data...",
         "columns": [
             "Han_temp_pipeline (bonsai)",
             "docDB_{raw, 'dynamic_foraging' in ANY software name}",
             "VAST_raw_data_on_VAST"
         ],
+        "edge_colors": [
+            "black",
+            "green",
+            "blue"
+        ],
+        "patch_settings":[
+            {
+                "patch_ids": ["100"],
+                "color": "red",
+                "emoji": "x",
+                "notes": "Raw data may lost! (Note that Han's temp pipeline archives the behavior data only)"
+            }
+        ]
+    },
+    {
+        "name": "Inconsistent metadata when looking for \"FIB\" sessions",
+        "columns": [
+            "docDB_{raw, 'fib' in 'rig.modalities'}",
+            "docDB_{raw, 'fib' in 'data_description.modality'}",
+            "docDB_{raw, 'fib' in 'session.data_streams'}"
+        ],
         "edge_colors": [
             "blue",
             "green",
             "red"
+        ],
+        "patch_settings":[
+            {
+                "patch_ids": ["010", "011"],
+                "color": "red",
+                "emoji": "x",
+                "notes": "Green and black circles should be a subset of the blue one; green and black circles should be the same."
+            }
+        ]
+    },
+    {
+        "name": "Inconsistent metadata when looking for \"dynamic_foraging\" sessions",
+        "columns": [
+            "docDB_{raw, 'dynamic_foraging' in data_streams software name}",
+            "docDB_{raw, 'dynamic_foraging' in stimulus_epochs software name}",
+            "Han_temp_pipeline (bonsai)"
+        ],
+        "edge_colors": [
+            "red",
+            "blue",
+            "black"
+        ],
+        "edge_styles": [
+            "solid",
+            "solid",
+            "dashed"
+        ],
+        "patch_settings":[
+            {
+                "patch_ids": ["100", "101", "110", "111"],
+                "color": "red",
+                "emoji": "x",
+                "notes": "Behavior acquisition software name should always be in stimulus_epoches, not data_streams."
+            }
+        ]
+    },   
+    {
+        "name": "Failed pipeline processing",
+        "columns": [
+            "docDB_{processed, 'dynamic_foraging' in ANY software name}",
+            "docDB_{raw, 'dynamic_foraging' in ANY software name}",
+            "Han_temp_pipeline (bonsai)"
+        ],
+        "edge_colors": [
+            "red",
+            "blue",
+            "black"
+        ],
+        "edge_styles": [
+            "solid",
+            "solid",
+            "dashed"
+        ],
+        "patch_settings":[
+            {
+                "patch_ids": ["010", "011"],
+                "color": "red",
+                "emoji": "x",
+                "notes": "Raw data uploaded but failed to process in CO."
+            }
+        ]
+    },
+    {
+        "name": "Check ephys-behavior sessions",
+        "columns": [
+            "docDB_{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
+            "docDB_{raw, 'dynamic_foraging' in ANY software name}",
+            "Han_temp_pipeline (bonsai)"
+        ],
+        "edge_colors": [
+            "orange",
+            "blue",
+            "black"
+        ],
+        "edge_styles": [
+            "solid",
+            "solid",
+            "dashed"
+        ],
+        "patch_settings":[
+            {
+                "patch_ids": ["111", "100", "101"],
+                "color": "orange",
+                "emoji": "large_yellow_square",
+                "notes": "We should check the data integrity of sessions uploaded by the ephys pipeline."
+            }
         ]
     }
 ]
\ No newline at end of file

From bcc8a34e512862bd9ac48a452b2634c0239b3ad1 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 01:54:15 +0000
Subject: [PATCH 50/68] minor

---
 data_inventory_VENN_PRESET.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/data_inventory_VENN_PRESET.json b/data_inventory_VENN_PRESET.json
index e2dfc5c..d7af988 100644
--- a/data_inventory_VENN_PRESET.json
+++ b/data_inventory_VENN_PRESET.json
@@ -95,7 +95,7 @@
                 "patch_ids": ["010", "011"],
                 "color": "red",
                 "emoji": "x",
-                "notes": "Green and black circles should be a subset of the blue one; green and black circles should be the same."
+                "notes": "\"FIB\" session must have \"FIB\" in rig; red and green circles should be the same."
             }
         ]
     },

From 9e9796df06f3edce36bf738333b7c0fa85b792f0 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 04:01:49 +0000
Subject: [PATCH 51/68] use pagination to avoid 503 error

---
 code/pages/0_Data inventory.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 990c085..05c163d 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -133,7 +133,8 @@ def fetch_single_query(query):
             "rig.rig_id": 1,
             "session.experimenter_full_name": 1,
         },
-        paginate=False,
+        paginate=True,
+        paginate_batch_size=5000,
     )
     print(f"Done querying {query['alias']}!")
 

From 5a6052eb0a4725edd624f61c3722f90ec1b4f25b Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 04:25:06 +0000
Subject: [PATCH 52/68] show df of each Venn

---
 code/pages/0_Data inventory.py | 36 ++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 05c163d..66316c4 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -48,6 +48,12 @@
 with open("data_inventory_QUERY_PRESET.json", "r") as f:
     QUERY_PRESET = json.load(f)
 
+meta_columns = [
+    "Han_temp_pipeline (bpod)",
+    "Han_temp_pipeline (bonsai)",
+    "VAST_raw_data_on_VAST",
+] + [query["alias"] for query in QUERY_PRESET]
+
 def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
     """ Add a button to download df as csv """
     csv = df.to_csv(index=True)
@@ -424,8 +430,9 @@ def app():
         VENN_PRESET = json.load(f)
 
     if VENN_PRESET:
-        st.markdown("## Venn diagram from presets")
+        st.markdown("## Venn diagrams from presets")
         for i, venn_preset in enumerate(VENN_PRESET):
+            # -- Venn diagrams --
             st.markdown(f"### ({i+1}). {venn_preset['name']}")
             cols = st.columns([1.5, 1])
             fig = generate_venn(
@@ -437,22 +444,35 @@ def app():
                 )
             with cols[0]:
                 st.pyplot(fig, use_container_width=True)
+
+            # -- Time view --
+            # Show histogram over time for the columns and patches in preset
+            this_columns = venn_preset["columns"] + [
+                col
+                for col in df_merged.columns
+                if col not in meta_columns  # Other extra info
+            ]
+            # Trim columns
+            df_this_presest = df_merged[this_columns]
+            # Filter out rows that have at least one True in this Venn
+            df_this_presest = df_this_presest[df_this_presest[venn_preset["columns"]].any(axis=1)]
             
+            with cols[1]:
+                with st.expander(f'Show dataframe for this Venn diagram, n = {len(df_this_presest)} '
+                                 f'(the master df_merged filtered by "at least one True in this Venn")'):
+                    download_df(df_this_presest, label="Download as CSV", file_name=f"df_{venn_preset['name']}.csv")
+                    st.write(df_this_presest)
+
             st.markdown("---")
 
     # --- User-defined Venn diagram ---
     # Multiselect for selecting queries up to three
     st.markdown('---')
     st.markdown("## Venn diagram from user-selected queries")
-    query_keys = [
-        "Han_temp_pipeline (bpod)",
-        "Han_temp_pipeline (bonsai)",
-        "VAST_raw_data_on_VAST",
-    ] + [query["alias"] for query in QUERY_PRESET]
     selected_queries = st.multiselect(
         "Select queries to filter sessions",
-        query_keys,
-        default=query_keys[:3],
+        meta_columns,
+        default=meta_columns[:3],
         key="selected_queries",
     )
 

From 8e0b87bd273641d7a3a36a0a927b6b61f7b6c9ef Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 05:05:59 +0000
Subject: [PATCH 53/68] add plot time-histogram

---
 code/pages/0_Data inventory.py | 91 +++++++++++++++++++++++++++++-----
 1 file changed, 78 insertions(+), 13 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 66316c4..895d40d 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -205,6 +205,25 @@ def fetch_all_queries_from_docDB(queries_to_merge, parallel=False):
     )
     return df_merged, dfs
 
+def _filter_df_by_patch_ids(df, patch_ids):
+    """ Filter df by patch_ids [110, 001] etc"""
+    # Turn NAN to False
+    df = df.fillna(False)
+    conditions = []
+    for patch_id in patch_ids:
+        # Convert patch_id string to boolean conditions
+        condition = True
+        for i, col in enumerate(df.columns):
+            if patch_id[i] == "1":  # Include rows where the column is True
+                condition &= df[col]
+            elif patch_id[i] == "0":  # Include rows where the column is not True
+                condition &= ~df[col] 
+        conditions.append(condition)
+    
+    # Combine all conditions with OR
+    final_condition = pd.concat(conditions, axis=1).any(axis=1)
+    return df[final_condition].index
+
 def generate_venn(df, columns_to_venn, edge_colors=None, edge_styles=None, patch_settings=None):
     """ Show venn diagram """
     if len(columns_to_venn) > 3:
@@ -445,23 +464,69 @@ def app():
             with cols[0]:
                 st.pyplot(fig, use_container_width=True)
 
-            # -- Time view --
+            # -- Show and download df for this Venn --
             # Show histogram over time for the columns and patches in preset
-            this_columns = venn_preset["columns"] + [
-                col
-                for col in df_merged.columns
-                if col not in meta_columns  # Other extra info
-            ]
-            # Trim columns
-            df_this_presest = df_merged[this_columns]
+            df_this_preset = df_merged[venn_preset["columns"]]
             # Filter out rows that have at least one True in this Venn
-            df_this_presest = df_this_presest[df_this_presest[venn_preset["columns"]].any(axis=1)]
-            
+            df_this_preset = df_this_preset[df_this_preset.any(axis=1)]
+
+            # Create a new column to indicate sessions in patches specified by patch_ids like ["100", "101", "110", "111"]
+            for patch_setting in venn_preset.get("patch_settings", []):
+                idx = _filter_df_by_patch_ids(
+                    df_this_preset[venn_preset["columns"]],
+                    patch_setting["patch_ids"]
+                )
+                df_this_preset.loc[idx, str(patch_setting["patch_ids"])] = True 
+                
+            # Join in other extra columns
+            df_this_preset = df_this_preset.join(
+                df_merged[[col for col in df_merged.columns if col not in meta_columns]], how="left"
+            )
+
             with cols[1]:
-                with st.expander(f'Show dataframe for this Venn diagram, n = {len(df_this_presest)} '
+                with st.expander(f'Show dataframe for this Venn diagram, n = {len(df_this_preset)} '
                                  f'(the master df_merged filtered by "at least one True in this Venn")'):
-                    download_df(df_this_presest, label="Download as CSV", file_name=f"df_{venn_preset['name']}.csv")
-                    st.write(df_this_presest)
+                    download_df(df_this_preset, label="Download as CSV", file_name=f"df_{venn_preset['name']}.csv")
+                    st.write(df_this_preset)
+
+            # -- Show histogram over time --
+            import numpy as np
+            import plotly.graph_objects as go
+            from plotly.subplots import make_subplots
+
+            # Simulating the dataframe from the screenshot
+            # Grouping by different time periods (e.g., daily, weekly, quarterly)
+            def plot_histogram_over_time(df, venn_preset, time_period="Daily"):
+                df["Daily"] = df["session_date"]
+                df["Weekly"] = df["session_date"].dt.to_period("W").dt.start_time
+                df["Quarterly"] = df["session_date"].dt.to_period("Q").dt.start_time
+
+                # Function to count "True" values for a given column over a specific time period
+                def count_true_values(df, time_period, column):
+                    return df.groupby(time_period)[column].apply(lambda x: (x == True).sum())
+
+                # Preparing subplots for each circle/patch in venn
+                columns = venn_preset["columns"] + [str(p_s["patch_ids"]) for p_s in venn_preset.get("patch_settings", [])]
+                fig = make_subplots(rows=len(columns), cols=1, shared_xaxes=True, vertical_spacing=0.05, subplot_titles=columns)
+
+                # Adding traces for each column 
+                for i, column in enumerate(columns):
+                    counts = count_true_values(df, time_period, column)
+                    fig.add_trace(go.Bar(x=counts.index, y=counts.values, name=column), row=i + 1, col=1)
+
+                # Updating layout
+                fig.update_layout(
+                    height=800,
+                    yaxis_title=f"{time_period} count of True",
+                    showlegend=False
+                )
+                return fig
+
+            fig = plot_histogram_over_time(
+                df=df_this_preset.reset_index(),
+                venn_preset=venn_preset,
+                )
+            cols[1].plotly_chart(fig)
 
             st.markdown("---")
 

From 2e8da80516551ebe1271db373a7f125f31749b94 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 05:47:21 +0000
Subject: [PATCH 54/68] refactor

---
 code/pages/0_Data inventory.py | 127 ++++++++++++++++++++++-----------
 1 file changed, 84 insertions(+), 43 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 895d40d..084713f 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -10,6 +10,10 @@
 from streamlit_dynamic_filters import DynamicFilters
 import pandas as pd
 import numpy as np
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+from streamlit_plotly_events import plotly_events
+
 import time
 import streamlit_nested_layout
 
@@ -224,6 +228,7 @@ def _filter_df_by_patch_ids(df, patch_ids):
     final_condition = pd.concat(conditions, axis=1).any(axis=1)
     return df[final_condition].index
 
+@st.cache_data(ttl=3600*24)
 def generate_venn(df, columns_to_venn, edge_colors=None, edge_styles=None, patch_settings=None):
     """ Show venn diagram """
     if len(columns_to_venn) > 3:
@@ -265,15 +270,16 @@ def generate_venn(df, columns_to_venn, edge_colors=None, edge_styles=None, patch
         if patch:  # Some patches might be None
             patch.set_facecolor('none')
     if patch_settings:
+        notes = []
         for patch_setting in patch_settings:
             # Set color
             for patch_id in patch_setting["patch_ids"]:
                 if v.get_patch_by_id(patch_id):
                     v.get_patch_by_id(patch_id).set_color(patch_setting["color"])
             # Add notes
-            st.markdown(f"#### :{patch_setting['emoji']}: :{patch_setting['color']}[{patch_setting['notes']}]")
+            notes.append(f"#### :{patch_setting['emoji']}: :{patch_setting['color']}[{patch_setting['notes']}]")
 
-    return fig
+    return fig, notes
 
 def _show_records_on_sidebar(dfs, file_name_prefix, source_str="docDB"):
     df = dfs["df"]
@@ -339,9 +345,56 @@ def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_re
         st.markdown('''## 3. From VAST /scratch: existing raw data''')
         _show_records_on_sidebar(dfs_raw_on_VAST, file_name_prefix="raw_on_VAST", source_str="VAST /scratch")
 
+@st.cache_data(ttl=3600*24)
+def plot_histogram_over_time(df, venn_preset, time_period="Daily", if_overlay=False):
+    """Generate histogram over time for the columns and patches in preset
+    """
+    df["Daily"] = df["session_date"]
+    df["Weekly"] = df["session_date"].dt.to_period("W").dt.start_time
+    df["Monthly"] = df["session_date"].dt.to_period("M").dt.start_time
+    df["Quarterly"] = df["session_date"].dt.to_period("Q").dt.start_time
+
+    # Function to count "True" values for a given column over a specific time period
+    def count_true_values(df, time_period, column):
+        return df.groupby(time_period)[column].apply(lambda x: (x == True).sum())
+
+    # Preparing subplots for each circle/patch in venn
+    if not if_overlay:
+        columns = venn_preset["columns"] + [str(p_s["patch_ids"]) for p_s in venn_preset.get("patch_settings", [])]
+        fig = make_subplots(
+            rows=len(columns),
+            cols=1,
+            shared_xaxes=True,
+            vertical_spacing=0.05,
+            subplot_titles=columns,
+        )
 
-def app():
+        # Adding traces for each column
+        colors = venn_preset.get("edge_colors", ["blue", "green", "red"]) + [
+            p_s["color"] for p_s in venn_preset.get("patch_settings", [])
+        ]
+        for i, column in enumerate(columns):
+            counts = count_true_values(df, time_period, column)
+            fig.add_trace(
+                go.Bar(
+                    x=counts.index,
+                    y=counts.values,
+                    name=column,
+                    marker=dict(color=colors[i]),
+                ),
+                row=i + 1,
+                col=1,
+            )
 
+    # Updating layout
+    fig.update_layout(
+        height=200 * len(columns) if not if_overlay else 200,
+        yaxis_title=f"{time_period} count of True",
+        showlegend=False,
+    )
+    return fig
+
+def app():
     # --- 1. Generate combined dataframe from docDB queries ---
     with st.sidebar:
         st.markdown('# Metadata sources:')
@@ -450,11 +503,11 @@ def app():
 
     if VENN_PRESET:
         st.markdown("## Venn diagrams from presets")
-        for i, venn_preset in enumerate(VENN_PRESET):
+        for i_venn, venn_preset in enumerate(VENN_PRESET):
             # -- Venn diagrams --
-            st.markdown(f"### ({i+1}). {venn_preset['name']}")
+            st.markdown(f"### ({i_venn+1}). {venn_preset['name']}")
             cols = st.columns([1.5, 1])
-            fig = generate_venn(
+            fig, notes = generate_venn(
                     df_merged,
                     venn_preset["columns"],
                     edge_colors=venn_preset.get("edge_colors", None),
@@ -462,6 +515,8 @@ def app():
                     patch_settings=venn_preset.get("patch_settings", None),
                 )
             with cols[0]:
+                for note in notes:
+                    st.markdown(note)
                 st.pyplot(fig, use_container_width=True)
 
             # -- Show and download df for this Venn --
@@ -477,7 +532,7 @@ def app():
                     patch_setting["patch_ids"]
                 )
                 df_this_preset.loc[idx, str(patch_setting["patch_ids"])] = True 
-                
+
             # Join in other extra columns
             df_this_preset = df_this_preset.join(
                 df_merged[[col for col in df_merged.columns if col not in meta_columns]], how="left"
@@ -489,44 +544,30 @@ def app():
                     download_df(df_this_preset, label="Download as CSV", file_name=f"df_{venn_preset['name']}.csv")
                     st.write(df_this_preset)
 
-            # -- Show histogram over time --
-            import numpy as np
-            import plotly.graph_objects as go
-            from plotly.subplots import make_subplots
-
-            # Simulating the dataframe from the screenshot
-            # Grouping by different time periods (e.g., daily, weekly, quarterly)
-            def plot_histogram_over_time(df, venn_preset, time_period="Daily"):
-                df["Daily"] = df["session_date"]
-                df["Weekly"] = df["session_date"].dt.to_period("W").dt.start_time
-                df["Quarterly"] = df["session_date"].dt.to_period("Q").dt.start_time
-
-                # Function to count "True" values for a given column over a specific time period
-                def count_true_values(df, time_period, column):
-                    return df.groupby(time_period)[column].apply(lambda x: (x == True).sum())
-
-                # Preparing subplots for each circle/patch in venn
-                columns = venn_preset["columns"] + [str(p_s["patch_ids"]) for p_s in venn_preset.get("patch_settings", [])]
-                fig = make_subplots(rows=len(columns), cols=1, shared_xaxes=True, vertical_spacing=0.05, subplot_titles=columns)
-
-                # Adding traces for each column 
-                for i, column in enumerate(columns):
-                    counts = count_true_values(df, time_period, column)
-                    fig.add_trace(go.Bar(x=counts.index, y=counts.values, name=column), row=i + 1, col=1)
-
-                # Updating layout
-                fig.update_layout(
-                    height=800,
-                    yaxis_title=f"{time_period} count of True",
-                    showlegend=False
-                )
-                return fig
+                # -- Show histogram over time --
+                if i_venn == 0:
+                    cols_1 = st.columns([1, 1])
+                    time_periods = ["Daily", "Weekly", "Monthly", "Quarterly"]
+                    time_period = cols_1[0].selectbox(
+                        "Select time period", time_periods, key=f"time_period_{i_venn}", index=1
+                    )
 
-            fig = plot_histogram_over_time(
-                df=df_this_preset.reset_index(),
-                venn_preset=venn_preset,
+                    if_overlay = cols_1[1].checkbox("Overlay all histograms", value=False)
+
+                fig = plot_histogram_over_time(
+                    df=df_this_preset.reset_index(),
+                    venn_preset=venn_preset,
+                    time_period=time_period,
+                    if_overlay=if_overlay,
+                )
+                plotly_events(
+                    fig,
+                    click_event=False,
+                    hover_event=False,
+                    select_event=False,
+                    override_height=fig.layout.height * 1.1,
+                    override_width=fig.layout.width,
                 )
-            cols[1].plotly_chart(fig)
 
             st.markdown("---")
 

From 87ce818f15dcf9f65525ec5a301f453da98b823d Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 06:15:36 +0000
Subject: [PATCH 55/68] refactor json

---
 code/pages/0_Data inventory.py  |  76 +++++-----
 data_inventory_VENN_PRESET.json | 253 +++++++++++++++++++-------------
 2 files changed, 183 insertions(+), 146 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 084713f..66f9d63 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -229,55 +229,51 @@ def _filter_df_by_patch_ids(df, patch_ids):
     return df[final_condition].index
 
 @st.cache_data(ttl=3600*24)
-def generate_venn(df, columns_to_venn, edge_colors=None, edge_styles=None, patch_settings=None):
+def generate_venn(df, venn_preset):
     """ Show venn diagram """
-    if len(columns_to_venn) > 3:
-        st.write("Venn diagram only supports up to 3 columns.")
-        return
+    circle_settings = venn_preset["circle_settings"]
     
     fig, ax = plt.subplots()
-    if len(columns_to_venn) == 2:
+    if len(circle_settings) == 2:
         v_func = venn2
         c_func = venn2_circles
-    elif len(columns_to_venn) == 3:
+    elif len(circle_settings) == 3:
         v_func = venn3
         c_func = venn3_circles
     else:
         st.warning("Number of columns to venn should be 2 or 3.")
-        return None
+        return None, None
         
     v = v_func(
-        [set(df.index[df[col]==True]) for col in columns_to_venn],
-        set_labels=columns_to_venn,
+        [set(df.index[df[c_s["column"]]==True]) for c_s in circle_settings],
+        set_labels=[c_s["column"] for c_s in circle_settings],
     )
     c = c_func(
-        [set(df.index[df[col]==True]) for col in columns_to_venn],
+        [set(df.index[df[c_s["column"]]==True]) for c_s in circle_settings],
     )
         
-    # Set edge color
-    if edge_colors: 
-        for i, circle in enumerate(c):
-            c[i].set_edgecolor(edge_colors[i])
-            v.get_label_by_id(["A", "B", "C"][i]).set_color(edge_colors[i])
-            
-    # Set edge style
-    if edge_styles:
-        for i, circle in enumerate(c):
-            c[i].set_linestyle(edge_styles[i])
+    # Set edge color and style
+    for i, c_s in enumerate(circle_settings):
+        edge_color = c_s.get("edge_color", "black")
+        edge_style = c_s.get("edge_style", "solid")
+        
+        c[i].set_edgecolor(edge_color)
+        c[i].set_linestyle(edge_style)
+        v.get_label_by_id(["A", "B", "C"][i]).set_color(edge_color)
     
-    # Set patch color
+    # Clear all patch color
     for patch in v.patches:
         if patch:  # Some patches might be None
             patch.set_facecolor('none')
-    if patch_settings:
-        notes = []
-        for patch_setting in patch_settings:
-            # Set color
-            for patch_id in patch_setting["patch_ids"]:
-                if v.get_patch_by_id(patch_id):
-                    v.get_patch_by_id(patch_id).set_color(patch_setting["color"])
-            # Add notes
-            notes.append(f"#### :{patch_setting['emoji']}: :{patch_setting['color']}[{patch_setting['notes']}]")
+
+    notes = []
+    for patch_setting in venn_preset["patch_settings"]:
+        # Set color
+        for patch_id in patch_setting["patch_ids"]:
+            if v.get_patch_by_id(patch_id):
+                v.get_patch_by_id(patch_id).set_color(patch_setting["color"])
+        # Add notes
+        notes.append(f"#### :{patch_setting['emoji']}: :{patch_setting['color']}[{patch_setting['notes']}]")
 
     return fig, notes
 
@@ -360,7 +356,9 @@ def count_true_values(df, time_period, column):
 
     # Preparing subplots for each circle/patch in venn
     if not if_overlay:
-        columns = venn_preset["columns"] + [str(p_s["patch_ids"]) for p_s in venn_preset.get("patch_settings", [])]
+        columns = [c_s["column"] for c_s in venn_preset["circle_settings"]] + [
+            str(p_s["patch_ids"]) for p_s in venn_preset.get("patch_settings", [])
+        ]
         fig = make_subplots(
             rows=len(columns),
             cols=1,
@@ -370,8 +368,8 @@ def count_true_values(df, time_period, column):
         )
 
         # Adding traces for each column
-        colors = venn_preset.get("edge_colors", ["blue", "green", "red"]) + [
-            p_s["color"] for p_s in venn_preset.get("patch_settings", [])
+        colors = [c_s["edge_color"] for c_s in venn_preset["circle_settings"]] + [
+            p_s["color"] for p_s in venn_preset["patch_settings"]
         ]
         for i, column in enumerate(columns):
             counts = count_true_values(df, time_period, column)
@@ -509,10 +507,7 @@ def app():
             cols = st.columns([1.5, 1])
             fig, notes = generate_venn(
                     df_merged,
-                    venn_preset["columns"],
-                    edge_colors=venn_preset.get("edge_colors", None),
-                    edge_styles=venn_preset.get("edge_styles", None),
-                    patch_settings=venn_preset.get("patch_settings", None),
+                    venn_preset
                 )
             with cols[0]:
                 for note in notes:
@@ -520,15 +515,16 @@ def app():
                 st.pyplot(fig, use_container_width=True)
 
             # -- Show and download df for this Venn --
+            circle_columns = [c_s["column"] for c_s in venn_preset["circle_settings"]]
             # Show histogram over time for the columns and patches in preset
-            df_this_preset = df_merged[venn_preset["columns"]]
+            df_this_preset = df_merged[circle_columns]
             # Filter out rows that have at least one True in this Venn
             df_this_preset = df_this_preset[df_this_preset.any(axis=1)]
 
             # Create a new column to indicate sessions in patches specified by patch_ids like ["100", "101", "110", "111"]
             for patch_setting in venn_preset.get("patch_settings", []):
                 idx = _filter_df_by_patch_ids(
-                    df_this_preset[venn_preset["columns"]],
+                    df_this_preset[circle_columns],
                     patch_setting["patch_ids"]
                 )
                 df_this_preset.loc[idx, str(patch_setting["patch_ids"])] = True 
@@ -552,7 +548,7 @@ def app():
                         "Select time period", time_periods, key=f"time_period_{i_venn}", index=1
                     )
 
-                    if_overlay = cols_1[1].checkbox("Overlay all histograms", value=False)
+                    if_overlay = cols_1[1].checkbox("Overlay all histograms", value=False, disabled=True)
 
                 fig = plot_histogram_over_time(
                     df=df_this_preset.reset_index(),
diff --git a/data_inventory_VENN_PRESET.json b/data_inventory_VENN_PRESET.json
index d7af988..d82c203 100644
--- a/data_inventory_VENN_PRESET.json
+++ b/data_inventory_VENN_PRESET.json
@@ -1,30 +1,36 @@
 [
     {
         "name": "Bonsai-based dynamic foraging sessions that are missing in docDB/CO",
-        "columns": [
-            "Han_temp_pipeline (bpod)",
-            "Han_temp_pipeline (bonsai)",
-            "docDB_{raw, 'dynamic_foraging' in ANY software name}"
-        ],
-        "edge_colors": [
-            "grey",
-            "black",
-            "green"
-        ],
-        "edge_styles": [
-            "dashed",
-            "solid",
-            "solid"
+        "circle_settings": [
+            {
+                "column": "Han_temp_pipeline (bonsai)",
+                "edge_color": "black",
+                "edge_style": "solid"
+            },
+            {
+                "column": "docDB_{raw, 'dynamic_foraging' in ANY software name}",
+                "edge_color": "green",
+                "edge_style": "solid"
+            },
+            {
+                "column": "Han_temp_pipeline (bpod)",
+                "edge_color": "grey",
+                "edge_style": "dashed"
+            }
         ],
-        "patch_settings":[
+        "patch_settings": [
             {
-                "patch_ids": ["010"],
+                "patch_ids": [
+                    "100"
+                ],
                 "color": "red",
                 "emoji": "heavy_exclamation_mark",
                 "notes": "Missing sessions on docDB (bonsai)"
             },
             {
-                "patch_ids": ["001"],
+                "patch_ids": [
+                    "010"
+                ],
                 "color": "orange",
                 "emoji": "large_yellow_square",
                 "notes": "Dynamic foraging sessions uploaded by some ephys rigs that Han's pipeline didn't ingest (which is fine)"
@@ -33,24 +39,28 @@
     },
     {
         "name": "Bpod-based (old) dynamic foraging sessions that are missing in docDB/CO",
-        "columns": [
-            "Han_temp_pipeline (bpod)",
-            "Han_temp_pipeline (bonsai)",
-            "docDB_{raw, 'FIP' in name}"
-        ],
-        "edge_colors": [
-            "grey",
-            "black",
-            "limegreen"
+        "circle_settings": [
+            {
+                "column": "Han_temp_pipeline (bpod)",
+                "edge_color": "grey",
+                "edge_style": "solid"
+            },
+            {
+                "column": "docDB_{raw, 'FIP' in name}",
+                "edge_color": "limegreen",
+                "edge_style": "solid"
+            },
+            {
+                "column": "Han_temp_pipeline (bonsai)",
+                "edge_color": "black",
+                "edge_style": "dashed"
+            }
         ],
-        "edge_styles": [
-            "solid",
-            "dashed",
-            "solid"
-        ],    
-        "patch_settings":[
-            {
-                "patch_ids": ["100"],
+        "patch_settings": [
+            {
+                "patch_ids": [
+                    "100"
+                ],
                 "color": "red",
                 "emoji": "heavy_exclamation_mark",
                 "notes": "Missing sessions on docDB (bpod)"
@@ -59,19 +69,25 @@
     },
     {
         "name": "We may have lost some raw data...",
-        "columns": [
-            "Han_temp_pipeline (bonsai)",
-            "docDB_{raw, 'dynamic_foraging' in ANY software name}",
-            "VAST_raw_data_on_VAST"
-        ],
-        "edge_colors": [
-            "black",
-            "green",
-            "blue"
+        "circle_settings": [
+            {
+                "column": "Han_temp_pipeline (bonsai)",
+                "edge_color": "black"
+            },
+            {
+                "column": "docDB_{raw, 'dynamic_foraging' in ANY software name}",
+                "edge_color": "green"
+            },
+            {
+                "column": "VAST_raw_data_on_VAST",
+                "edge_color": "blue"
+            }
         ],
-        "patch_settings":[
+        "patch_settings": [
             {
-                "patch_ids": ["100"],
+                "patch_ids": [
+                    "100"
+                ],
                 "color": "red",
                 "emoji": "x",
                 "notes": "Raw data may lost! (Note that Han's temp pipeline archives the behavior data only)"
@@ -80,71 +96,90 @@
     },
     {
         "name": "Inconsistent metadata when looking for \"FIB\" sessions",
-        "columns": [
-            "docDB_{raw, 'fib' in 'rig.modalities'}",
-            "docDB_{raw, 'fib' in 'data_description.modality'}",
-            "docDB_{raw, 'fib' in 'session.data_streams'}"
-        ],
-        "edge_colors": [
-            "blue",
-            "green",
-            "red"
+        "circle_settings": [
+            {
+                "column": "docDB_{raw, 'fib' in 'rig.modalities'}",
+                "edge_color": "blue"
+            },
+            {
+                "column": "docDB_{raw, 'fib' in 'data_description.modality'}",
+                "edge_color": "green"
+            },
+            {
+                "column": "docDB_{raw, 'fib' in 'session.data_streams'}",
+                "edge_color": "limegreen"
+            }
         ],
-        "patch_settings":[
+        "patch_settings": [
             {
-                "patch_ids": ["010", "011"],
+                "patch_ids": [
+                    "010",
+                    "011"
+                ],
                 "color": "red",
                 "emoji": "x",
-                "notes": "\"FIB\" session must have \"FIB\" in rig; red and green circles should be the same."
+                "notes": "\"FIB\" session must have \"FIB\" in rig (two greens should be the same and inside the blue)"
             }
         ]
     },
     {
         "name": "Inconsistent metadata when looking for \"dynamic_foraging\" sessions",
-        "columns": [
-            "docDB_{raw, 'dynamic_foraging' in data_streams software name}",
-            "docDB_{raw, 'dynamic_foraging' in stimulus_epochs software name}",
-            "Han_temp_pipeline (bonsai)"
-        ],
-        "edge_colors": [
-            "red",
-            "blue",
-            "black"
-        ],
-        "edge_styles": [
-            "solid",
-            "solid",
-            "dashed"
+        "circle_settings": [
+            {
+                "column": "docDB_{raw, 'dynamic_foraging' in data_streams software name}",
+                "edge_color": "blue",
+                "edge_style": "solid"
+            },
+            {
+                "column": "docDB_{raw, 'dynamic_foraging' in stimulus_epochs software name}",
+                "edge_color": "green",
+                "edge_style": "solid"
+            },
+            {
+                "column": "Han_temp_pipeline (bonsai)",
+                "edge_color": "black",
+                "edge_style": "dashed"
+            }
         ],
-        "patch_settings":[
+        "patch_settings": [
             {
-                "patch_ids": ["100", "101", "110", "111"],
+                "patch_ids": [
+                    "100",
+                    "101",
+                    "110",
+                    "111"
+                ],
                 "color": "red",
                 "emoji": "x",
                 "notes": "Behavior acquisition software name should always be in stimulus_epoches, not data_streams."
             }
         ]
-    },   
+    },
     {
         "name": "Failed pipeline processing",
-        "columns": [
-            "docDB_{processed, 'dynamic_foraging' in ANY software name}",
-            "docDB_{raw, 'dynamic_foraging' in ANY software name}",
-            "Han_temp_pipeline (bonsai)"
-        ],
-        "edge_colors": [
-            "red",
-            "blue",
-            "black"
-        ],
-        "edge_styles": [
-            "solid",
-            "solid",
-            "dashed"
+        "circle_settings": [
+            {
+                "column": "docDB_{processed, 'dynamic_foraging' in ANY software name}",
+                "edge_color": "green",
+                "edge_style": "solid"
+            },
+            {
+                "column": "docDB_{raw, 'dynamic_foraging' in ANY software name}",
+                "edge_color": "blue",
+                "edge_style": "solid"
+            },
+            {
+                "column": "Han_temp_pipeline (bonsai)",
+                "edge_color": "black",
+                "edge_style": "dashed"
+            }
         ],
-        "patch_settings":[
+        "patch_settings": [
             {
-                "patch_ids": ["010", "011"],
+                "patch_ids": [
+                    "010",
+                    "011"
+                ],
                 "color": "red",
                 "emoji": "x",
                 "notes": "Raw data uploaded but failed to process in CO."
@@ -153,24 +188,30 @@
     },
     {
         "name": "Check ephys-behavior sessions",
-        "columns": [
-            "docDB_{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
-            "docDB_{raw, 'dynamic_foraging' in ANY software name}",
-            "Han_temp_pipeline (bonsai)"
-        ],
-        "edge_colors": [
-            "orange",
-            "blue",
-            "black"
-        ],
-        "edge_styles": [
-            "solid",
-            "solid",
-            "dashed"
+        "circle_settings": [
+            {
+                "column": "Han_temp_pipeline (bonsai)",
+                "edge_color": "black",
+                "edge_style": "dashed"
+            },
+            {
+                "column": "docDB_{raw, 'dynamic_foraging' in ANY software name}",
+                "edge_color": "blue",
+                "edge_style": "solid"
+            },
+            {
+                "column": "docDB_{raw, ('dynamic_foraging' in ANY software name) AND ('ecephys' in data_description.modality)}",
+                "edge_color": "orange",
+                "edge_style": "solid"
+            }
         ],
-        "patch_settings":[
+        "patch_settings": [
             {
-                "patch_ids": ["111", "100", "101"],
+                "patch_ids": [
+                    "111",
+                    "101",
+                    "011"
+                ],
                 "color": "orange",
                 "emoji": "large_yellow_square",
                 "notes": "We should check the data integrity of sessions uploaded by the ephys pipeline."

From 2401b961f69253b9ba774c204e95ebbeafd3d6c0 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 07:19:16 +0000
Subject: [PATCH 56/68] done overlay or not

---
 code/pages/0_Data inventory.py | 134 ++++++++++++++++++++++-----------
 1 file changed, 90 insertions(+), 44 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 66f9d63..7442216 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -342,7 +342,7 @@ def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_re
         _show_records_on_sidebar(dfs_raw_on_VAST, file_name_prefix="raw_on_VAST", source_str="VAST /scratch")
 
 @st.cache_data(ttl=3600*24)
-def plot_histogram_over_time(df, venn_preset, time_period="Daily", if_overlay=False):
+def plot_histogram_over_time(df, venn_preset, time_period="Daily", if_sync_y_limits=True, if_separate_plots=False):
     """Generate histogram over time for the columns and patches in preset
     """
     df["Daily"] = df["session_date"]
@@ -355,10 +355,13 @@ def count_true_values(df, time_period, column):
         return df.groupby(time_period)[column].apply(lambda x: (x == True).sum())
 
     # Preparing subplots for each circle/patch in venn
-    if not if_overlay:
-        columns = [c_s["column"] for c_s in venn_preset["circle_settings"]] + [
-            str(p_s["patch_ids"]) for p_s in venn_preset.get("patch_settings", [])
-        ]
+    columns = [c_s["column"] for c_s in venn_preset["circle_settings"]] + [
+        str(p_s["patch_ids"]) for p_s in venn_preset.get("patch_settings", [])
+    ]
+    colors = [c_s["edge_color"] for c_s in venn_preset["circle_settings"]] + [
+        p_s["color"] for p_s in venn_preset["patch_settings"]
+    ]
+    if if_separate_plots:
         fig = make_subplots(
             rows=len(columns),
             cols=1,
@@ -368,11 +371,10 @@ def count_true_values(df, time_period, column):
         )
 
         # Adding traces for each column
-        colors = [c_s["edge_color"] for c_s in venn_preset["circle_settings"]] + [
-            p_s["color"] for p_s in venn_preset["patch_settings"]
-        ]
+        max_counts = 0
         for i, column in enumerate(columns):
             counts = count_true_values(df, time_period, column)
+            max_counts = max(max_counts, counts.max())
             fig.add_trace(
                 go.Bar(
                     x=counts.index,
@@ -384,12 +386,44 @@ def count_true_values(df, time_period, column):
                 col=1,
             )
 
-    # Updating layout
-    fig.update_layout(
-        height=200 * len(columns) if not if_overlay else 200,
-        yaxis_title=f"{time_period} count of True",
-        showlegend=False,
-    )
+        # Sync y limits
+        if if_sync_y_limits:
+            fig.update_yaxes(range=[0, max_counts * 1.1])
+
+        # Updating layout
+        fig.update_layout(
+            height=200 * len(columns),
+            showlegend=False,
+            title=f"{time_period} counts",
+        )
+    else:  # side-by-side histograms in the same plot
+        fig = go.Figure()
+        for i, column in enumerate(columns):
+            fig.add_trace(go.Histogram( 
+                x=df[df[column]==True]["session_date"],
+                xbins=dict(size="M1"), # Only monthly bins look good
+                name=column,
+                marker_color=colors[i],
+                opacity=0.75
+            ))
+
+        # Update layout for grouped histogram
+        fig.update_layout(
+            height=500,
+            bargap=0.05,  # Gap between bars of adjacent locations
+            bargroupgap=0.1,  # Gap between bars of the same location
+            barmode='group',  # Grouped style
+            showlegend=True,
+            legend=dict(
+                orientation="h",  # Horizontal legend
+                y=-0.2,  # Position below the plot
+                x=0.5,  # Center the legend
+                xanchor="center",  # Anchor the legend's x position
+                yanchor="top"  # Anchor the legend's y position
+            ),
+            title="Monthly counts"
+        )
+        
     return fig
 
 def app():
@@ -500,18 +534,34 @@ def app():
         VENN_PRESET = json.load(f)
 
     if VENN_PRESET:
-        st.markdown("## Venn diagrams from presets")
+
+        cols = st.columns([2, 1])
+        cols[0].markdown("## Venn diagrams from presets")
+        with cols[1].expander("Time view settings", expanded=True):
+            cols_1 = st.columns([1, 1])                    
+            if_separate_plots = cols_1[0].checkbox("Separate in subplots", value=True)
+            if_sync_y_limits = cols_1[0].checkbox(
+                "Sync Y limits", value=True, disabled=not if_separate_plots
+            )
+            time_period = cols_1[1].selectbox(
+                "Bin size",
+                ["Daily", "Weekly", "Monthly", "Quarterly"],
+                index=1,
+                disabled=not if_separate_plots,
+            )
+
         for i_venn, venn_preset in enumerate(VENN_PRESET):
             # -- Venn diagrams --
             st.markdown(f"### ({i_venn+1}). {venn_preset['name']}")
-            cols = st.columns([1.5, 1])
             fig, notes = generate_venn(
                     df_merged,
                     venn_preset
                 )
+            for note in notes:
+                st.markdown(note)
+
+            cols = st.columns([1, 1])
             with cols[0]:
-                for note in notes:
-                    st.markdown(note)
                 st.pyplot(fig, use_container_width=True)
 
             # -- Show and download df for this Venn --
@@ -534,27 +584,23 @@ def app():
                 df_merged[[col for col in df_merged.columns if col not in meta_columns]], how="left"
             )
 
-            with cols[1]:
-                with st.expander(f'Show dataframe for this Venn diagram, n = {len(df_this_preset)} '
-                                 f'(the master df_merged filtered by "at least one True in this Venn")'):
-                    download_df(df_this_preset, label="Download as CSV", file_name=f"df_{venn_preset['name']}.csv")
+            with cols[0]:
+                download_df(
+                    df_this_preset,
+                    label="Download as CSV for this Venn diagram",
+                    file_name=f"df_{venn_preset['name']}.csv",
+                )
+                with st.expander(f"Show dataframe, n = {len(df_this_preset)}"):
                     st.write(df_this_preset)
 
+            with cols[1]:
                 # -- Show histogram over time --
-                if i_venn == 0:
-                    cols_1 = st.columns([1, 1])
-                    time_periods = ["Daily", "Weekly", "Monthly", "Quarterly"]
-                    time_period = cols_1[0].selectbox(
-                        "Select time period", time_periods, key=f"time_period_{i_venn}", index=1
-                    )
-
-                    if_overlay = cols_1[1].checkbox("Overlay all histograms", value=False, disabled=True)
-
                 fig = plot_histogram_over_time(
                     df=df_this_preset.reset_index(),
                     venn_preset=venn_preset,
                     time_period=time_period,
-                    if_overlay=if_overlay,
+                    if_sync_y_limits=if_sync_y_limits,
+                    if_separate_plots=if_separate_plots,
                 )
                 plotly_events(
                     fig,
@@ -569,18 +615,18 @@ def app():
 
     # --- User-defined Venn diagram ---
     # Multiselect for selecting queries up to three
-    st.markdown('---')
-    st.markdown("## Venn diagram from user-selected queries")
-    selected_queries = st.multiselect(
-        "Select queries to filter sessions",
-        meta_columns,
-        default=meta_columns[:3],
-        key="selected_queries",
-    )
-
-    columns_to_venn = selected_queries
-    fig = generate_venn(df_merged, columns_to_venn)
-    st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
+    # st.markdown('---')
+    # st.markdown("## Venn diagram from user-selected queries")
+    # selected_queries = st.multiselect(
+    #     "Select queries to filter sessions",
+    #     meta_columns,
+    #     default=meta_columns[:3],
+    #     key="selected_queries",
+    # )
+
+    # columns_to_venn = selected_queries
+    # fig = generate_venn(df_merged, columns_to_venn)
+    # st.columns([1, 1])[0].pyplot(fig, use_container_width=True)
 
 
 if __name__ == "__main__":

From 81854c99bae7828f943acc831d07fd0907f14e6e Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 07:22:37 +0000
Subject: [PATCH 57/68] skip timeline

---
 code/pages/0_Data inventory.py  | 1 +
 data_inventory_VENN_PRESET.json | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 7442216..0d09d6f 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -357,6 +357,7 @@ def count_true_values(df, time_period, column):
     # Preparing subplots for each circle/patch in venn
     columns = [c_s["column"] for c_s in venn_preset["circle_settings"]] + [
         str(p_s["patch_ids"]) for p_s in venn_preset.get("patch_settings", [])
+        if not p_s.get("skip_timeline", False)
     ]
     colors = [c_s["edge_color"] for c_s in venn_preset["circle_settings"]] + [
         p_s["color"] for p_s in venn_preset["patch_settings"]
diff --git a/data_inventory_VENN_PRESET.json b/data_inventory_VENN_PRESET.json
index d82c203..81e6c4e 100644
--- a/data_inventory_VENN_PRESET.json
+++ b/data_inventory_VENN_PRESET.json
@@ -214,7 +214,8 @@
                 ],
                 "color": "orange",
                 "emoji": "large_yellow_square",
-                "notes": "We should check the data integrity of sessions uploaded by the ephys pipeline."
+                "notes": "We should check the data integrity of sessions uploaded by the ephys pipeline.",
+                "skip_timeline": true
             }
         ]
     }

From 91401fb560e8df9b17e19e1a3a9193e2cb1798e8 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 07:32:37 +0000
Subject: [PATCH 58/68] add docDB fetch settings

---
 code/pages/0_Data inventory.py | 38 ++++++++++++++++++++++++++--------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 0d09d6f..4825ef5 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -125,7 +125,7 @@ def _formatting_metadata_df(df, source_prefix="docDB"):
     return df, df_unique_mouse_date, df_multi_sessions_per_day
 
 @st.cache_data(ttl=3600*24)
-def fetch_single_query(query):
+def fetch_single_query(query, pagination, paginate_batch_size):
     """ Fetch a query from docDB and process the result into dataframe
     
     Return: 
@@ -143,8 +143,8 @@ def fetch_single_query(query):
             "rig.rig_id": 1,
             "session.experimenter_full_name": 1,
         },
-        paginate=True,
-        paginate_batch_size=5000,
+        paginate=pagination,
+        paginate_batch_size=paginate_batch_size,
     )
     print(f"Done querying {query['alias']}!")
 
@@ -158,15 +158,23 @@ def fetch_single_query(query):
     return df, df_unique_mouse_date, df_multi_sessions_per_day
 
 
-def fetch_all_queries_from_docDB(queries_to_merge, parallel=False):
+def fetch_all_queries_from_docDB(queries_to_merge, parallel=False, pagination=False, paginate_batch_size=5000):
     """ Get merged queries from selected queries """
 
     dfs = {}
-    
+
     # Fetch data in parallel
     if parallel:
         with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
-            future_to_query = {executor.submit(fetch_single_query, key): key for key in queries_to_merge}
+            future_to_query = {
+                executor.submit(
+                    fetch_single_query,
+                    key,
+                    pagination=pagination,
+                    paginate_batch_size=paginate_batch_size,
+                ): key
+                for key in queries_to_merge
+            }
             for i, future in enumerate(as_completed(future_to_query), 1):
                 key = future_to_query[future]
                 try:
@@ -182,7 +190,9 @@ def fetch_all_queries_from_docDB(queries_to_merge, parallel=False):
         # Fetch data in serial
         p_bar = st.progress(0, text="Querying docDB...")
         for i, query in enumerate(queries_to_merge):
-            df, df_unique_mouse_date, df_multi_sessions_per_day = fetch_single_query(query)
+            df, df_unique_mouse_date, df_multi_sessions_per_day = fetch_single_query(
+                query, pagination=pagination, paginate_batch_size=paginate_batch_size
+            )
             dfs[query["alias"]] = {
                 "df": df,
                 "df_unique_mouse_date": df_unique_mouse_date,
@@ -193,7 +203,7 @@ def fetch_all_queries_from_docDB(queries_to_merge, parallel=False):
     if not dfs:
         st.warning("Querying docDB error! Refresh the page to try again or ask Han.")
         return None, None
-    
+
     # Combine queried dfs using df_unique_mouse_date (on index "subject_id", "session_date" only)
     df_merged = dfs[queries_to_merge[0]["alias"]]["df_unique_mouse_date"]
     for df in [dfs[query["alias"]]["df_unique_mouse_date"] for query in queries_to_merge[1:]]:
@@ -433,10 +443,20 @@ def app():
         st.markdown('# Metadata sources:')
         st.markdown('## 1. From docDB queries')
 
+        with st.expander("MetadataDbClient settings"):
+            parallel = st.checkbox("Parallel fetching", value=False)
+            pagination = st.checkbox("Pagination", value=True)
+            paginate_batch_size = st.number_input("Pagination batch size", value=5000, disabled=not pagination)
+
         cols = st.columns([1.5, 1])
         with cols[0]:
             start_time = time.time()
-            df_merged, dfs_docDB = fetch_all_queries_from_docDB(queries_to_merge=QUERY_PRESET)
+            df_merged, dfs_docDB = fetch_all_queries_from_docDB(
+                queries_to_merge=QUERY_PRESET,
+                parallel=parallel,
+                pagination=pagination,
+                paginate_batch_size=paginate_batch_size,
+            )
             docDB_retrieve_time = time.time() - start_time
             st.markdown(f"Finished in {docDB_retrieve_time:.3f} secs.")
 

From 3b64efc5d2e5743ee2a5cc2eeaba5610c1c66ec4 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 07:34:03 +0000
Subject: [PATCH 59/68] add spinner

---
 code/pages/0_Data inventory.py | 45 +++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 4825ef5..5caace0 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -165,30 +165,31 @@ def fetch_all_queries_from_docDB(queries_to_merge, parallel=False, pagination=Fa
 
     # Fetch data in parallel
     if parallel:
-        with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
-            future_to_query = {
-                executor.submit(
-                    fetch_single_query,
-                    key,
-                    pagination=pagination,
-                    paginate_batch_size=paginate_batch_size,
-                ): key
-                for key in queries_to_merge
-            }
-            for i, future in enumerate(as_completed(future_to_query), 1):
-                key = future_to_query[future]
-                try:
-                    df, df_unique_mouse_date, df_multi_sessions_per_day = future.result()
-                    dfs[key["alias"]] = { 
-                        "df": df,
-                        "df_unique_mouse_date": df_unique_mouse_date,
-                        "df_multi_sessions_per_day": df_multi_sessions_per_day,
-                    }
-                except Exception as e:
-                    print(f"Error querying {key}: {e}")
+        with st.spinner("Querying docDB in parallel..."):
+            with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
+                future_to_query = {
+                    executor.submit(
+                        fetch_single_query,
+                        key,
+                        pagination=pagination,
+                        paginate_batch_size=paginate_batch_size,
+                    ): key
+                    for key in queries_to_merge
+                }
+                for i, future in enumerate(as_completed(future_to_query), 1):
+                    key = future_to_query[future]
+                    try:
+                        df, df_unique_mouse_date, df_multi_sessions_per_day = future.result()
+                        dfs[key["alias"]] = { 
+                            "df": df,
+                            "df_unique_mouse_date": df_unique_mouse_date,
+                            "df_multi_sessions_per_day": df_multi_sessions_per_day,
+                        }
+                    except Exception as e:
+                        print(f"Error querying {key}: {e}")
     else:
         # Fetch data in serial
-        p_bar = st.progress(0, text="Querying docDB...")
+        p_bar = st.progress(0, text="Querying docDB in serial...")
         for i, query in enumerate(queries_to_merge):
             df, df_unique_mouse_date, df_multi_sessions_per_day = fetch_single_query(
                 query, pagination=pagination, paginate_batch_size=paginate_batch_size

From 6b1264db18720705880b1018df16b5ef813449a7 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 07:46:18 +0000
Subject: [PATCH 60/68] refactor

---
 code/pages/0_Data inventory.py | 107 +++++++++++++++++++--------------
 1 file changed, 61 insertions(+), 46 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 5caace0..be6d456 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -158,53 +158,63 @@ def fetch_single_query(query, pagination, paginate_batch_size):
     return df, df_unique_mouse_date, df_multi_sessions_per_day
 
 
-def fetch_all_queries_from_docDB(queries_to_merge, parallel=False, pagination=False, paginate_batch_size=5000):
+def fetch_queries_from_docDB(queries_to_merge, pagination=False, paginate_batch_size=5000):  
     """ Get merged queries from selected queries """
-
+    
     dfs = {}
 
-    # Fetch data in parallel
-    if parallel:
-        with st.spinner("Querying docDB in parallel..."):
-            with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
-                future_to_query = {
-                    executor.submit(
-                        fetch_single_query,
-                        key,
-                        pagination=pagination,
-                        paginate_batch_size=paginate_batch_size,
-                    ): key
-                    for key in queries_to_merge
-                }
-                for i, future in enumerate(as_completed(future_to_query), 1):
-                    key = future_to_query[future]
-                    try:
-                        df, df_unique_mouse_date, df_multi_sessions_per_day = future.result()
-                        dfs[key["alias"]] = { 
-                            "df": df,
-                            "df_unique_mouse_date": df_unique_mouse_date,
-                            "df_multi_sessions_per_day": df_multi_sessions_per_day,
-                        }
-                    except Exception as e:
-                        print(f"Error querying {key}: {e}")
-    else:
-        # Fetch data in serial
-        p_bar = st.progress(0, text="Querying docDB in serial...")
-        for i, query in enumerate(queries_to_merge):
-            df, df_unique_mouse_date, df_multi_sessions_per_day = fetch_single_query(
-                query, pagination=pagination, paginate_batch_size=paginate_batch_size
-            )
-            dfs[query["alias"]] = {
-                "df": df,
-                "df_unique_mouse_date": df_unique_mouse_date,
-                "df_multi_sessions_per_day": df_multi_sessions_per_day,
-            }
-            p_bar.progress((i+1) / len(queries_to_merge), text=f"Querying docDB... ({i+1}/{len(queries_to_merge)})")
+    # Fetch data in serial
+    p_bar = st.progress(0, text="Querying docDB in serial...")
+    for i, query in enumerate(queries_to_merge):
+        df, df_unique_mouse_date, df_multi_sessions_per_day = fetch_single_query(
+            query, pagination=pagination, paginate_batch_size=paginate_batch_size
+        )
+        dfs[query["alias"]] = {
+            "df": df,
+            "df_unique_mouse_date": df_unique_mouse_date,
+            "df_multi_sessions_per_day": df_multi_sessions_per_day,
+        }
+        p_bar.progress((i+1) / len(queries_to_merge), text=f"Querying docDB... ({i+1}/{len(queries_to_merge)})")
 
     if not dfs:
         st.warning("Querying docDB error! Refresh the page to try again or ask Han.")
-        return None, None
+        return None
+    
+    return dfs
 
+@st.cache_data(ttl=3600*12)
+def fetch_queries_from_docDB_parallel(queries_to_merge, pagination=False, paginate_batch_size=5000):
+    """ Get merged queries from selected queries """
+
+    dfs = {}
+
+    # Fetch data in parallel
+    with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
+        future_to_query = {
+            executor.submit(
+                fetch_single_query,
+                key,
+                pagination=pagination,
+                paginate_batch_size=paginate_batch_size,
+            ): key
+            for key in queries_to_merge
+        }
+        for i, future in enumerate(as_completed(future_to_query), 1):
+            key = future_to_query[future]
+            try:
+                df, df_unique_mouse_date, df_multi_sessions_per_day = future.result()
+                dfs[key["alias"]] = { 
+                    "df": df,
+                    "df_unique_mouse_date": df_unique_mouse_date,
+                    "df_multi_sessions_per_day": df_multi_sessions_per_day,
+                }
+            except Exception as e:
+                print(f"Error querying {key}: {e}")
+                    
+    return dfs
+
+@st.cache_data(ttl=3600*12)
+def merge_queried_dfs(dfs, queries_to_merge):
     # Combine queried dfs using df_unique_mouse_date (on index "subject_id", "session_date" only)
     df_merged = dfs[queries_to_merge[0]["alias"]]["df_unique_mouse_date"]
     for df in [dfs[query["alias"]]["df_unique_mouse_date"] for query in queries_to_merge[1:]]:
@@ -218,7 +228,8 @@ def fetch_all_queries_from_docDB(queries_to_merge, parallel=False, pagination=Fa
         ]
         + query_cols
     )
-    return df_merged, dfs
+    return df_merged
+
 
 def _filter_df_by_patch_ids(df, patch_ids):
     """ Filter df by patch_ids [110, 001] etc"""
@@ -445,19 +456,23 @@ def app():
         st.markdown('## 1. From docDB queries')
 
         with st.expander("MetadataDbClient settings"):
-            parallel = st.checkbox("Parallel fetching", value=False)
-            pagination = st.checkbox("Pagination", value=True)
-            paginate_batch_size = st.number_input("Pagination batch size", value=5000, disabled=not pagination)
+            with st.form("MetadataDbClient settings"):
+                parallel = st.checkbox("Parallel fetching", value=False)
+                pagination = st.checkbox("Pagination", value=True)
+                paginate_batch_size = st.number_input("Pagination batch size", value=5000, disabled=not pagination)
+                st.form_submit_button("OK")
 
         cols = st.columns([1.5, 1])
         with cols[0]:
             start_time = time.time()
-            df_merged, dfs_docDB = fetch_all_queries_from_docDB(
+            fetch_fun = fetch_queries_from_docDB_parallel if parallel else fetch_queries_from_docDB
+            dfs_docDB = fetch_fun(
                 queries_to_merge=QUERY_PRESET,
-                parallel=parallel,
                 pagination=pagination,
                 paginate_batch_size=paginate_batch_size,
             )
+            df_merged = merge_queried_dfs(dfs_docDB, QUERY_PRESET)
+            
             docDB_retrieve_time = time.time() - start_time
             st.markdown(f"Finished in {docDB_retrieve_time:.3f} secs.")
 

From ff873b9f30e9bae3887949a83a462f1ebe7535fd Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 07:58:18 +0000
Subject: [PATCH 61/68] refactor!

---
 code/pages/0_Data inventory.py | 172 ++-------------------------------
 code/util/fetch_data_docDB.py  |  93 ++++++++++++++++++
 code/util/reformat.py          |  59 ++++++++++-
 code/util/streamlit.py         |  13 +++
 4 files changed, 172 insertions(+), 165 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index be6d456..fd1cdcf 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -17,10 +17,13 @@
 import time
 import streamlit_nested_layout
 
-from util.streamlit import aggrid_interactive_table_basic
-from util.fetch_data_docDB import load_data_from_docDB, load_client
+from util.streamlit import (aggrid_interactive_table_basic, download_df)
+from util.fetch_data_docDB import (
+    fetch_queries_from_docDB,
+    fetch_queries_from_docDB_parallel,
+)
+from util.reformat import formatting_metadata_df
 from util.aws_s3 import load_raw_sessions_on_VAST
-from util.reformat import split_nwb_name
 from Home import init
 
 
@@ -46,173 +49,16 @@
 unsafe_allow_html=True,
 )
 
-client = load_client()
-
 # Load QUERY_PRESET from json
 with open("data_inventory_QUERY_PRESET.json", "r") as f:
     QUERY_PRESET = json.load(f)
 
-meta_columns = [
+META_COLUMNS = [
     "Han_temp_pipeline (bpod)",
     "Han_temp_pipeline (bonsai)",
     "VAST_raw_data_on_VAST",
 ] + [query["alias"] for query in QUERY_PRESET]
 
-def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
-    """ Add a button to download df as csv """
-    csv = df.to_csv(index=True)
-    
-    # Create download buttons
-    st.download_button(
-        label=label,
-        data=csv,
-        file_name=file_name,
-        mime='text/csv'
-    )
-
-def _formatting_metadata_df(df, source_prefix="docDB"):
-    """Formatting metadata dataframe
-    Given a dataframe with a column of "name" that contains nwb names
-    1. parse the nwb names into subject_id, session_date, nwb_suffix.
-    2. remove invalid subject_id
-    3. handle multiple sessions per day
-    """
-
-    df.rename(columns={col: f"{source_prefix}_{col}" for col in df.columns}, inplace=True)
-    new_name_field = f"{source_prefix}_name"
-
-    # Create index of subject_id, session_date, nwb_suffix by parsing nwb_name
-    df[["subject_id", "session_date", "nwb_suffix"]] = df[new_name_field].apply(
-        lambda x: pd.Series(split_nwb_name(x))
-    )
-    df["session_date"] = pd.to_datetime(df["session_date"])
-    df = df.set_index(["subject_id", "session_date", "nwb_suffix"]).sort_index(
-        level=["session_date", "subject_id", "nwb_suffix"],
-        ascending=[False, False, False],
-    )
-
-    # Remove invalid subject_id
-    df = df[(df.index.get_level_values("subject_id").astype(int) > 300000) 
-            & (df.index.get_level_values("subject_id").astype(int) < 999999)]
-
-    # --- Handle multiple sessions per day ---
-    # Build a dataframe with unique mouse-dates.
-    # If multiple sessions per day, combine them into a list of 'name'
-    df_unique_mouse_date = (
-        df.reset_index()
-        .groupby(["subject_id", "session_date"])
-        .agg({new_name_field: list, **{col: "first" for col in df.columns if col != new_name_field}})
-    ).sort_index(
-        level=["session_date", "subject_id"], # Restore order 
-        ascending=[False, False],
-    )
-    # Add a new column to indicate multiple sessions per day
-    df_unique_mouse_date[f"{source_prefix}_multiple_sessions_per_day"] = df_unique_mouse_date[
-        new_name_field
-    ].apply(lambda x: len(x) > 1)
-
-    # Also return the dataframe with multiple sessions per day
-    df_multi_sessions_per_day = df_unique_mouse_date[
-        df_unique_mouse_date[f"{source_prefix}_multiple_sessions_per_day"]
-    ]
-
-    # Create a new column to mark duplicates in the original df
-    df.loc[
-        df.index.droplevel("nwb_suffix").isin(df_multi_sessions_per_day.index), 
-        f"{source_prefix}_multiple_sessions_per_day"
-    ] = True
-
-    return df, df_unique_mouse_date, df_multi_sessions_per_day
-
-@st.cache_data(ttl=3600*24)
-def fetch_single_query(query, pagination, paginate_batch_size):
-    """ Fetch a query from docDB and process the result into dataframe
-    
-    Return: 
-    - df: dataframe of the original returned records
-    - df_multi_sessions_per_day: the dataframe with multiple sessions per day
-    - df_unique_mouse_date: the dataframe with multiple sessions per day combined
-    """
-
-    # --- Fetch data from docDB ---
-    results = client.retrieve_docdb_records(
-        filter_query=query["filter"],
-        projection={
-            "_id": 0,
-            "name": 1,
-            "rig.rig_id": 1,
-            "session.experimenter_full_name": 1,
-        },
-        paginate=pagination,
-        paginate_batch_size=paginate_batch_size,
-    )
-    print(f"Done querying {query['alias']}!")
-
-    # --- Process data into dataframe ---
-    df = pd.json_normalize(results)
-    
-    # Formatting dataframe
-    df, df_unique_mouse_date, df_multi_sessions_per_day = _formatting_metadata_df(df)
-    df_unique_mouse_date[query["alias"]] = True  # Add a column to prepare for merging
-
-    return df, df_unique_mouse_date, df_multi_sessions_per_day
-
-
-def fetch_queries_from_docDB(queries_to_merge, pagination=False, paginate_batch_size=5000):  
-    """ Get merged queries from selected queries """
-    
-    dfs = {}
-
-    # Fetch data in serial
-    p_bar = st.progress(0, text="Querying docDB in serial...")
-    for i, query in enumerate(queries_to_merge):
-        df, df_unique_mouse_date, df_multi_sessions_per_day = fetch_single_query(
-            query, pagination=pagination, paginate_batch_size=paginate_batch_size
-        )
-        dfs[query["alias"]] = {
-            "df": df,
-            "df_unique_mouse_date": df_unique_mouse_date,
-            "df_multi_sessions_per_day": df_multi_sessions_per_day,
-        }
-        p_bar.progress((i+1) / len(queries_to_merge), text=f"Querying docDB... ({i+1}/{len(queries_to_merge)})")
-
-    if not dfs:
-        st.warning("Querying docDB error! Refresh the page to try again or ask Han.")
-        return None
-    
-    return dfs
-
-@st.cache_data(ttl=3600*12)
-def fetch_queries_from_docDB_parallel(queries_to_merge, pagination=False, paginate_batch_size=5000):
-    """ Get merged queries from selected queries """
-
-    dfs = {}
-
-    # Fetch data in parallel
-    with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
-        future_to_query = {
-            executor.submit(
-                fetch_single_query,
-                key,
-                pagination=pagination,
-                paginate_batch_size=paginate_batch_size,
-            ): key
-            for key in queries_to_merge
-        }
-        for i, future in enumerate(as_completed(future_to_query), 1):
-            key = future_to_query[future]
-            try:
-                df, df_unique_mouse_date, df_multi_sessions_per_day = future.result()
-                dfs[key["alias"]] = { 
-                    "df": df,
-                    "df_unique_mouse_date": df_unique_mouse_date,
-                    "df_multi_sessions_per_day": df_multi_sessions_per_day,
-                }
-            except Exception as e:
-                print(f"Error querying {key}: {e}")
-                    
-    return dfs
-
 @st.cache_data(ttl=3600*12)
 def merge_queried_dfs(dfs, queries_to_merge):
     # Combine queried dfs using df_unique_mouse_date (on index "subject_id", "session_date" only)
@@ -534,7 +380,7 @@ def app():
         df_raw_sessions_on_VAST, 
         df_raw_sessions_on_VAST_unique_mouse_date,
         df_raw_sessions_on_VAST_multi_sessions_per_day
-    ) = _formatting_metadata_df(df_raw_sessions_on_VAST, source_prefix="VAST")
+    ) = formatting_metadata_df(df_raw_sessions_on_VAST, source_prefix="VAST")
 
     dfs_raw_on_VAST = {
         "df": df_raw_sessions_on_VAST,
@@ -618,7 +464,7 @@ def app():
 
             # Join in other extra columns
             df_this_preset = df_this_preset.join(
-                df_merged[[col for col in df_merged.columns if col not in meta_columns]], how="left"
+                df_merged[[col for col in df_merged.columns if col not in META_COLUMNS]], how="left"
             )
 
             with cols[0]:
diff --git a/code/util/fetch_data_docDB.py b/code/util/fetch_data_docDB.py
index d063ed4..a49893e 100644
--- a/code/util/fetch_data_docDB.py
+++ b/code/util/fetch_data_docDB.py
@@ -13,6 +13,7 @@
 
 from aind_data_access_api.document_db import MetadataDbClient
 
+from util.reformat import formatting_metadata_df
 
 @st.cache_data(ttl=3600*12) # Cache the df_docDB up to 12 hours
 def load_data_from_docDB():
@@ -221,3 +222,95 @@ def find_result(x, lookup):
         if result_name.startswith(x):
             return result
     return {}
+
+
+# --------- Helper functions for Data Inventory (Han) ----------
+@st.cache_data(ttl=3600*24)
+def fetch_single_query(query, pagination, paginate_batch_size):
+    """ Fetch a query from docDB and process the result into dataframe
+    
+    Return: 
+    - df: dataframe of the original returned records
+    - df_multi_sessions_per_day: the dataframe with multiple sessions per day
+    - df_unique_mouse_date: the dataframe with multiple sessions per day combined
+    """
+
+    # --- Fetch data from docDB ---
+    client = load_client()
+    results = client.retrieve_docdb_records(
+        filter_query=query["filter"],
+        projection={
+            "_id": 0,
+            "name": 1,
+            "rig.rig_id": 1,
+            "session.experimenter_full_name": 1,
+        },
+        paginate=pagination,
+        paginate_batch_size=paginate_batch_size,
+    )
+    print(f"Done querying {query['alias']}!")
+
+    # --- Process data into dataframe ---
+    df = pd.json_normalize(results)
+    
+    # Formatting dataframe
+    df, df_unique_mouse_date, df_multi_sessions_per_day = formatting_metadata_df(df)
+    df_unique_mouse_date[query["alias"]] = True  # Add a column to prepare for merging
+
+    return df, df_unique_mouse_date, df_multi_sessions_per_day
+
+# Don't cache this function otherwise the progress bar won't work
+def fetch_queries_from_docDB(queries_to_merge, pagination=False, paginate_batch_size=5000):  
+    """ Get merged queries from selected queries """
+    
+    dfs = {}
+
+    # Fetch data in serial
+    p_bar = st.progress(0, text="Querying docDB in serial...")
+    for i, query in enumerate(queries_to_merge):
+        df, df_unique_mouse_date, df_multi_sessions_per_day = fetch_single_query(
+            query, pagination=pagination, paginate_batch_size=paginate_batch_size
+        )
+        dfs[query["alias"]] = {
+            "df": df,
+            "df_unique_mouse_date": df_unique_mouse_date,
+            "df_multi_sessions_per_day": df_multi_sessions_per_day,
+        }
+        p_bar.progress((i+1) / len(queries_to_merge), text=f"Querying docDB... ({i+1}/{len(queries_to_merge)})")
+
+    if not dfs:
+        st.warning("Querying docDB error! Refresh the page to try again or ask Han.")
+        return None
+    
+    return dfs
+
+@st.cache_data(ttl=3600*12)
+def fetch_queries_from_docDB_parallel(queries_to_merge, pagination=False, paginate_batch_size=5000):
+    """ Get merged queries from selected queries """
+
+    dfs = {}
+
+    # Fetch data in parallel
+    with ThreadPoolExecutor(max_workers=len(queries_to_merge)) as executor:
+        future_to_query = {
+            executor.submit(
+                fetch_single_query,
+                key,
+                pagination=pagination,
+                paginate_batch_size=paginate_batch_size,
+            ): key
+            for key in queries_to_merge
+        }
+        for i, future in enumerate(as_completed(future_to_query), 1):
+            key = future_to_query[future]
+            try:
+                df, df_unique_mouse_date, df_multi_sessions_per_day = future.result()
+                dfs[key["alias"]] = { 
+                    "df": df,
+                    "df_unique_mouse_date": df_unique_mouse_date,
+                    "df_multi_sessions_per_day": df_multi_sessions_per_day,
+                }
+            except Exception as e:
+                print(f"Error querying {key}: {e}")
+                    
+    return dfs
\ No newline at end of file
diff --git a/code/util/reformat.py b/code/util/reformat.py
index efa8dcd..dd3da68 100644
--- a/code/util/reformat.py
+++ b/code/util/reformat.py
@@ -2,7 +2,7 @@
 """ Helper functions to reformat the data
 """
 import re
-
+import pandas as pd
 
 # Function to split the `nwb_name` column
 def split_nwb_name(nwb_name):
@@ -40,4 +40,59 @@ def split_nwb_name(nwb_name):
     else:
         nwb_suffix = 0
 
-    return subject_id, session_date, nwb_suffix
\ No newline at end of file
+    return subject_id, session_date, nwb_suffix
+
+
+def formatting_metadata_df(df, source_prefix="docDB"):
+    """Formatting metadata dataframe
+    Given a dataframe with a column of "name" that contains nwb names
+    1. parse the nwb names into subject_id, session_date, nwb_suffix.
+    2. remove invalid subject_id
+    3. handle multiple sessions per day
+    """
+
+    df.rename(columns={col: f"{source_prefix}_{col}" for col in df.columns}, inplace=True)
+    new_name_field = f"{source_prefix}_name"
+
+    # Create index of subject_id, session_date, nwb_suffix by parsing nwb_name
+    df[["subject_id", "session_date", "nwb_suffix"]] = df[new_name_field].apply(
+        lambda x: pd.Series(split_nwb_name(x))
+    )
+    df["session_date"] = pd.to_datetime(df["session_date"])
+    df = df.set_index(["subject_id", "session_date", "nwb_suffix"]).sort_index(
+        level=["session_date", "subject_id", "nwb_suffix"],
+        ascending=[False, False, False],
+    )
+
+    # Remove invalid subject_id
+    df = df[(df.index.get_level_values("subject_id").astype(int) > 300000) 
+            & (df.index.get_level_values("subject_id").astype(int) < 999999)]
+
+    # --- Handle multiple sessions per day ---
+    # Build a dataframe with unique mouse-dates.
+    # If multiple sessions per day, combine them into a list of 'name'
+    df_unique_mouse_date = (
+        df.reset_index()
+        .groupby(["subject_id", "session_date"])
+        .agg({new_name_field: list, **{col: "first" for col in df.columns if col != new_name_field}})
+    ).sort_index(
+        level=["session_date", "subject_id"], # Restore order 
+        ascending=[False, False],
+    )
+    # Add a new column to indicate multiple sessions per day
+    df_unique_mouse_date[f"{source_prefix}_multiple_sessions_per_day"] = df_unique_mouse_date[
+        new_name_field
+    ].apply(lambda x: len(x) > 1)
+
+    # Also return the dataframe with multiple sessions per day
+    df_multi_sessions_per_day = df_unique_mouse_date[
+        df_unique_mouse_date[f"{source_prefix}_multiple_sessions_per_day"]
+    ]
+
+    # Create a new column to mark duplicates in the original df
+    df.loc[
+        df.index.droplevel("nwb_suffix").isin(df_multi_sessions_per_day.index), 
+        f"{source_prefix}_multiple_sessions_per_day"
+    ] = True
+
+    return df, df_unique_mouse_date, df_multi_sessions_per_day
\ No newline at end of file
diff --git a/code/util/streamlit.py b/code/util/streamlit.py
index aac2342..db42758 100644
--- a/code/util/streamlit.py
+++ b/code/util/streamlit.py
@@ -1157,3 +1157,16 @@ def add_footnote():
     st.markdown('---')
     st.markdown(f'#### Han Hou @ 2024 {__ver__}')
     st.markdown('[bug report / feature request](https://github.com/AllenNeuralDynamics/foraging-behavior-browser/issues)')
+
+
+def download_df(df, label="Download filtered df as CSV", file_name="df.csv"):
+    """ Add a button to download df as csv """
+    csv = df.to_csv(index=True)
+    
+    # Create download buttons
+    st.download_button(
+        label=label,
+        data=csv,
+        file_name=file_name,
+        mime='text/csv'
+    )
\ No newline at end of file

From 57f23e6c51a898781352af50a10a51b01c791b47 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 07:59:30 +0000
Subject: [PATCH 62/68] don't paginate by default

---
 code/pages/0_Data inventory.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index fd1cdcf..da0e194 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -304,7 +304,7 @@ def app():
         with st.expander("MetadataDbClient settings"):
             with st.form("MetadataDbClient settings"):
                 parallel = st.checkbox("Parallel fetching", value=False)
-                pagination = st.checkbox("Pagination", value=True)
+                pagination = st.checkbox("Pagination", value=False)
                 paginate_batch_size = st.number_input("Pagination batch size", value=5000, disabled=not pagination)
                 st.form_submit_button("OK")
 

From e4930e913852ba1f81b5a60a8bf47cc2ef68aa52 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 08:01:30 +0000
Subject: [PATCH 63/68] minor

---
 code/pages/0_Data inventory.py | 2 +-
 code/util/fetch_data_docDB.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index da0e194..8d002a2 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -328,7 +328,7 @@ def app():
                 st.rerun()
 
     if df_merged is None:
-        st.cache_data.clear()
+        st.cache_data.clear() # Fetch failed, re-fetch
         return
 
     # --- 2. Merge in the master df in the Home page (Han's temporary pipeline) ---
diff --git a/code/util/fetch_data_docDB.py b/code/util/fetch_data_docDB.py
index a49893e..f360257 100644
--- a/code/util/fetch_data_docDB.py
+++ b/code/util/fetch_data_docDB.py
@@ -279,7 +279,7 @@ def fetch_queries_from_docDB(queries_to_merge, pagination=False, paginate_batch_
         p_bar.progress((i+1) / len(queries_to_merge), text=f"Querying docDB... ({i+1}/{len(queries_to_merge)})")
 
     if not dfs:
-        st.warning("Querying docDB error! Refresh the page to try again or ask Han.")
+        st.warning("Querying docDB error! Try \"Pagination\" in MetadataDbClient settings or ask Han.")
         return None
     
     return dfs

From 1fe7f62405555d744e402b5816840606bfc8b51e Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 08:03:36 +0000
Subject: [PATCH 64/68] fix import

---
 code/pages/0_Data inventory.py | 2 --
 code/util/fetch_data_docDB.py  | 2 ++
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 8d002a2..583a4a8 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -1,7 +1,5 @@
 import logging
 import re
-from concurrent.futures import ThreadPoolExecutor, as_completed
-import threading
 import json
 
 from matplotlib_venn import venn2, venn3, venn2_circles, venn3_circles
diff --git a/code/util/fetch_data_docDB.py b/code/util/fetch_data_docDB.py
index f360257..9b867be 100644
--- a/code/util/fetch_data_docDB.py
+++ b/code/util/fetch_data_docDB.py
@@ -3,6 +3,8 @@
 
 import logging
 import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import threading
 
 import pandas as pd
 import semver

From 52b82582259118d95291ac9ba0ee855f922a6b33 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 08:07:41 +0000
Subject: [PATCH 65/68] bump version

---
 code/__init__.py               | 2 +-
 code/pages/0_Data inventory.py | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/code/__init__.py b/code/__init__.py
index 005fd5f..fe3e3b8 100644
--- a/code/__init__.py
+++ b/code/__init__.py
@@ -1 +1 @@
-__ver__ = 'v2.5.6'
+__ver__ = 'v2.6.0'
diff --git a/code/pages/0_Data inventory.py b/code/pages/0_Data inventory.py
index 583a4a8..cd07753 100644
--- a/code/pages/0_Data inventory.py	
+++ b/code/pages/0_Data inventory.py	
@@ -15,7 +15,7 @@
 import time
 import streamlit_nested_layout
 
-from util.streamlit import (aggrid_interactive_table_basic, download_df)
+from util.streamlit import aggrid_interactive_table_basic, download_df, add_footnote
 from util.fetch_data_docDB import (
     fetch_queries_from_docDB,
     fetch_queries_from_docDB_parallel,
@@ -206,6 +206,8 @@ def add_sidebar(df_merged, dfs_docDB, df_Han_pipeline, dfs_raw_on_VAST, docDB_re
                 
         st.markdown('''## 3. From VAST /scratch: existing raw data''')
         _show_records_on_sidebar(dfs_raw_on_VAST, file_name_prefix="raw_on_VAST", source_str="VAST /scratch")
+        
+        add_footnote()
 
 @st.cache_data(ttl=3600*24)
 def plot_histogram_over_time(df, venn_preset, time_period="Daily", if_sync_y_limits=True, if_separate_plots=False):

From bcf9cd89552a0725b14cf209673f2e0695c256e4 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 08:09:03 +0000
Subject: [PATCH 66/68] bump year

---
 code/util/streamlit.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/code/util/streamlit.py b/code/util/streamlit.py
index db42758..6c3a0ab 100644
--- a/code/util/streamlit.py
+++ b/code/util/streamlit.py
@@ -1155,7 +1155,7 @@ def _add_agg(df_this, x_name, y_name, group, aggr_method, if_use_x_quantile, q_q
 
 def add_footnote():
     st.markdown('---')
-    st.markdown(f'#### Han Hou @ 2024 {__ver__}')
+    st.markdown(f'#### AIND @ 2025 {__ver__}')
     st.markdown('[bug report / feature request](https://github.com/AllenNeuralDynamics/foraging-behavior-browser/issues)')
 
 

From 0ba04018b6665e9c4b1437bca9c29faf9afdf2a0 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 08:09:28 +0000
Subject: [PATCH 67/68] minor

---
 code/util/streamlit.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/code/util/streamlit.py b/code/util/streamlit.py
index 6c3a0ab..941a3f4 100644
--- a/code/util/streamlit.py
+++ b/code/util/streamlit.py
@@ -1155,7 +1155,7 @@ def _add_agg(df_this, x_name, y_name, group, aggr_method, if_use_x_quantile, q_q
 
 def add_footnote():
     st.markdown('---')
-    st.markdown(f'#### AIND @ 2025 {__ver__}')
+    st.markdown(f'#### AIND Behavior Team @ 2025 {__ver__}')
     st.markdown('[bug report / feature request](https://github.com/AllenNeuralDynamics/foraging-behavior-browser/issues)')
 
 

From 16d5bc090a52226a0ee05957803dd90a3cf3cf58 Mon Sep 17 00:00:00 2001
From: "houhan@gmail.com" <han.hou@alleninstitute.org>
Date: Tue, 7 Jan 2025 17:37:07 +0000
Subject: [PATCH 68/68] update requirements

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 6159d97..bcb790a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -20,6 +20,7 @@ s3fs==2024.6.1
 pillow==10.4.0
 tabulate==0.9.0
 aind-data-access-api[docdb]==0.13.0
+matplotlib_venn==1.1.1
 git+https://github.com/AllenNeuralDynamics/aind-foraging-behavior-bonsai-automatic-training.git@main
 git+https://github.com/AllenNeuralDynamics/aind-dynamic-foraging-models.git@develop
 git+https://github.com/AllenNeuralDynamics/aind-behavior-gym.git@develop
\ No newline at end of file