diff --git a/.gitignore b/.gitignore index e19484278..97a4df3bd 100644 Binary files a/.gitignore and b/.gitignore differ diff --git a/App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py b/App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py index 0ea424d79..a17387980 100644 --- a/App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py +++ b/App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py @@ -259,7 +259,7 @@ def run_geval(transcript: str, summary: str, api_key: str, api_name: str = None, def create_geval_tab(): - with gr.Tab("G-Eval"): + with gr.Tab("G-Eval", id="g-eval"): gr.Markdown("# G-Eval Summarization Evaluation") with gr.Row(): with gr.Column(): diff --git a/App_Function_Libraries/DB/RAG_QA_Chat_DB.py b/App_Function_Libraries/DB/RAG_QA_Chat_DB.py new file mode 100644 index 000000000..bec2ec2c5 --- /dev/null +++ b/App_Function_Libraries/DB/RAG_QA_Chat_DB.py @@ -0,0 +1,722 @@ +# RAG_QA_Chat_DB.py +# Description: This file contains the database operations for the RAG QA Chat + Notes system. +# +# Imports +import configparser +import logging +import re +import sqlite3 +import uuid +from contextlib import contextmanager +from datetime import datetime + +from App_Function_Libraries.Utils.Utils import get_project_relative_path, get_database_path + +# +# External Imports +# (No external imports) +# +# Local Imports +# (No additional local imports) +# +######################################################################################################################## +# +# Functions: + +# Construct the path to the config file +config_path = get_project_relative_path('Config_Files/config.txt') + +# Read the config file +config = configparser.ConfigParser() +config.read(config_path) + +# Get the SQLite path from the config, or use the default if not specified +if config.has_section('Database') and config.has_option('Database', 'rag_qa_db_path'): + rag_qa_db_path = config.get('Database', 'rag_qa_db_path') +else: + rag_qa_db_path = get_database_path('RAG_QA_Chat.db') + +print(f"RAG QA Chat Database path: {rag_qa_db_path}") + +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# Database schema +SCHEMA_SQL = ''' +-- Table for storing chat messages +CREATE TABLE IF NOT EXISTS rag_qa_chats ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + conversation_id TEXT NOT NULL, + timestamp DATETIME NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL +); + +-- Table for storing conversation metadata +CREATE TABLE IF NOT EXISTS conversation_metadata ( + conversation_id TEXT PRIMARY KEY, + created_at DATETIME NOT NULL, + last_updated DATETIME NOT NULL, + title TEXT NOT NULL +); + +-- Table for storing keywords +CREATE TABLE IF NOT EXISTS rag_qa_keywords ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + keyword TEXT NOT NULL UNIQUE +); + +-- Table for linking keywords to conversations +CREATE TABLE IF NOT EXISTS rag_qa_conversation_keywords ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + conversation_id TEXT NOT NULL, + keyword_id INTEGER NOT NULL, + FOREIGN KEY (conversation_id) REFERENCES conversation_metadata(conversation_id), + FOREIGN KEY (keyword_id) REFERENCES rag_qa_keywords(id) +); + +-- Table for storing keyword collections +CREATE TABLE IF NOT EXISTS rag_qa_keyword_collections ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE, + parent_id INTEGER, + FOREIGN KEY (parent_id) REFERENCES rag_qa_keyword_collections(id) +); + +-- Table for linking keywords to collections +CREATE TABLE IF NOT EXISTS rag_qa_collection_keywords ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + collection_id INTEGER NOT NULL, + keyword_id INTEGER NOT NULL, + FOREIGN KEY (collection_id) REFERENCES rag_qa_keyword_collections(id), + FOREIGN KEY (keyword_id) REFERENCES rag_qa_keywords(id) +); + +-- Table for storing notes +CREATE TABLE IF NOT EXISTS rag_qa_notes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + conversation_id TEXT NOT NULL, + title TEXT NOT NULL, + content TEXT NOT NULL, + timestamp DATETIME NOT NULL, + FOREIGN KEY (conversation_id) REFERENCES conversation_metadata(conversation_id) +); + +-- Table for linking notes to keywords +CREATE TABLE IF NOT EXISTS rag_qa_note_keywords ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + note_id INTEGER NOT NULL, + keyword_id INTEGER NOT NULL, + FOREIGN KEY (note_id) REFERENCES rag_qa_notes(id), + FOREIGN KEY (keyword_id) REFERENCES rag_qa_keywords(id) +); + +-- Indexes for improved query performance +CREATE INDEX IF NOT EXISTS idx_rag_qa_chats_conversation_id ON rag_qa_chats(conversation_id); +CREATE INDEX IF NOT EXISTS idx_rag_qa_chats_timestamp ON rag_qa_chats(timestamp); +CREATE INDEX IF NOT EXISTS idx_rag_qa_keywords_keyword ON rag_qa_keywords(keyword); +CREATE INDEX IF NOT EXISTS idx_rag_qa_conversation_keywords_conversation_id ON rag_qa_conversation_keywords(conversation_id); +CREATE INDEX IF NOT EXISTS idx_rag_qa_conversation_keywords_keyword_id ON rag_qa_conversation_keywords(keyword_id); +CREATE INDEX IF NOT EXISTS idx_rag_qa_keyword_collections_parent_id ON rag_qa_keyword_collections(parent_id); +CREATE INDEX IF NOT EXISTS idx_rag_qa_collection_keywords_collection_id ON rag_qa_collection_keywords(collection_id); +CREATE INDEX IF NOT EXISTS idx_rag_qa_collection_keywords_keyword_id ON rag_qa_collection_keywords(keyword_id); + +-- Full-text search virtual table for chat content +CREATE VIRTUAL TABLE IF NOT EXISTS rag_qa_chats_fts USING fts5(conversation_id, timestamp, role, content); + +-- Trigger to keep the FTS table up to date +CREATE TRIGGER IF NOT EXISTS rag_qa_chats_ai AFTER INSERT ON rag_qa_chats BEGIN + INSERT INTO rag_qa_chats_fts(conversation_id, timestamp, role, content) VALUES (new.conversation_id, new.timestamp, new.role, new.content); +END; +''' + +# Database connection management +@contextmanager +def get_db_connection(): + conn = sqlite3.connect(rag_qa_db_path) + try: + yield conn + finally: + conn.close() + +@contextmanager +def transaction(): + with get_db_connection() as conn: + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + +def execute_query(query, params=None, conn=None): + if conn: + cursor = conn.cursor() + if params: + cursor.execute(query, params) + else: + cursor.execute(query) + return cursor.fetchall() + else: + with get_db_connection() as conn: + cursor = conn.cursor() + if params: + cursor.execute(query, params) + else: + cursor.execute(query) + conn.commit() + return cursor.fetchall() + +def create_tables(): + with get_db_connection() as conn: + conn.executescript(SCHEMA_SQL) + logger.info("All RAG QA Chat tables created successfully") + +# Initialize the database +create_tables() + +# +# End of Setup +############################################################ + + +############################################################ +# +# Keyword-related functions + +# Input validation +def validate_keyword(keyword): + if not isinstance(keyword, str): + raise ValueError("Keyword must be a string") + if not keyword.strip(): + raise ValueError("Keyword cannot be empty or just whitespace") + if len(keyword) > 100: + raise ValueError("Keyword is too long (max 100 characters)") + if not re.match(r'^[a-zA-Z0-9\s\-_]+$', keyword): + raise ValueError("Keyword contains invalid characters") + return keyword.strip() + +def validate_collection_name(name): + if not isinstance(name, str): + raise ValueError("Collection name must be a string") + if not name.strip(): + raise ValueError("Collection name cannot be empty or just whitespace") + if len(name) > 100: + raise ValueError("Collection name is too long (max 100 characters)") + if not re.match(r'^[a-zA-Z0-9\s\-_]+$', name): + raise ValueError("Collection name contains invalid characters") + return name.strip() + +# Core functions +def add_keyword(keyword, conn=None): + try: + validated_keyword = validate_keyword(keyword) + query = "INSERT OR IGNORE INTO rag_qa_keywords (keyword) VALUES (?)" + execute_query(query, (validated_keyword,), conn) + logger.info(f"Keyword '{validated_keyword}' added successfully") + except ValueError as e: + logger.error(f"Invalid keyword: {e}") + raise + except Exception as e: + logger.error(f"Error adding keyword '{keyword}': {e}") + raise + +def create_keyword_collection(name, parent_id=None): + try: + validated_name = validate_collection_name(name) + query = "INSERT INTO rag_qa_keyword_collections (name, parent_id) VALUES (?, ?)" + execute_query(query, (validated_name, parent_id)) + logger.info(f"Keyword collection '{validated_name}' created successfully") + except ValueError as e: + logger.error(f"Invalid collection name: {e}") + raise + except Exception as e: + logger.error(f"Error creating keyword collection '{name}': {e}") + raise + +def add_keyword_to_collection(collection_name, keyword): + try: + validated_collection_name = validate_collection_name(collection_name) + validated_keyword = validate_keyword(keyword) + + with transaction() as conn: + add_keyword(validated_keyword, conn) + + query = ''' + INSERT INTO rag_qa_collection_keywords (collection_id, keyword_id) + SELECT c.id, k.id + FROM rag_qa_keyword_collections c, rag_qa_keywords k + WHERE c.name = ? AND k.keyword = ? + ''' + execute_query(query, (validated_collection_name, validated_keyword), conn) + + logger.info(f"Keyword '{validated_keyword}' added to collection '{validated_collection_name}' successfully") + except ValueError as e: + logger.error(f"Invalid input: {e}") + raise + except Exception as e: + logger.error(f"Error adding keyword '{keyword}' to collection '{collection_name}': {e}") + raise + +def add_keywords_to_conversation(conversation_id, keywords): + if not isinstance(keywords, (list, tuple)): + raise ValueError("Keywords must be a list or tuple") + try: + with transaction() as conn: + for keyword in keywords: + validated_keyword = validate_keyword(keyword) + add_keyword(validated_keyword, conn) + + query = ''' + INSERT INTO rag_qa_conversation_keywords (conversation_id, keyword_id) + SELECT ?, id FROM rag_qa_keywords WHERE keyword = ? + ''' + execute_query(query, (conversation_id, validated_keyword), conn) + + logger.info(f"Keywords added to conversation '{conversation_id}' successfully") + except ValueError as e: + logger.error(f"Invalid keyword: {e}") + raise + except Exception as e: + logger.error(f"Error adding keywords to conversation '{conversation_id}': {e}") + raise + +def get_keywords_for_conversation(conversation_id): + try: + query = ''' + SELECT k.keyword + FROM rag_qa_keywords k + JOIN rag_qa_conversation_keywords ck ON k.id = ck.keyword_id + WHERE ck.conversation_id = ? + ''' + result = execute_query(query, (conversation_id,)) + keywords = [row[0] for row in result] + logger.info(f"Retrieved {len(keywords)} keywords for conversation '{conversation_id}'") + return keywords + except Exception as e: + logger.error(f"Error getting keywords for conversation '{conversation_id}': {e}") + raise + +def get_keywords_for_collection(collection_name): + try: + query = ''' + SELECT k.keyword + FROM rag_qa_keywords k + JOIN rag_qa_collection_keywords ck ON k.id = ck.keyword_id + JOIN rag_qa_keyword_collections c ON ck.collection_id = c.id + WHERE c.name = ? + ''' + result = execute_query(query, (collection_name,)) + keywords = [row[0] for row in result] + logger.info(f"Retrieved {len(keywords)} keywords for collection '{collection_name}'") + return keywords + except Exception as e: + logger.error(f"Error getting keywords for collection '{collection_name}': {e}") + raise + +# +# End of Keyword-related functions +################################################### + + +################################################### +# +# Notes and chat-related functions + +def save_notes(conversation_id, title, content): + """Save notes to the database.""" + try: + query = "INSERT INTO rag_qa_notes (conversation_id, title, content, timestamp) VALUES (?, ?, ?, ?)" + timestamp = datetime.now().isoformat() + with transaction() as conn: + cursor = conn.cursor() + cursor.execute(query, (conversation_id, title, content, timestamp)) + note_id = cursor.lastrowid + logger.info(f"Notes saved for conversation '{conversation_id}', note ID '{note_id}'") + return note_id + except Exception as e: + logger.error(f"Error saving notes for conversation '{conversation_id}': {e}") + raise + +def update_note(note_id, title, content): + try: + query = "UPDATE rag_qa_notes SET title = ?, content = ?, timestamp = ? WHERE id = ?" + timestamp = datetime.now().isoformat() + execute_query(query, (title, content, timestamp, note_id)) + logger.info(f"Note ID '{note_id}' updated successfully") + except Exception as e: + logger.error(f"Error updating note ID '{note_id}': {e}") + raise + +def get_notes(conversation_id): + """Retrieve notes for a given conversation.""" + try: + query = "SELECT content FROM rag_qa_notes WHERE conversation_id = ?" + result = execute_query(query, (conversation_id,)) + notes = [row[0] for row in result] + logger.info(f"Retrieved {len(notes)} notes for conversation '{conversation_id}'") + return notes + except Exception as e: + logger.error(f"Error getting notes for conversation '{conversation_id}': {e}") + raise + +def get_note_by_id(note_id): + try: + query = "SELECT id, title, content FROM rag_qa_notes WHERE id = ?" + result = execute_query(query, (note_id,)) + return result + except Exception as e: + logger.error(f"Error getting note by ID '{note_id}': {e}") + raise + +def get_notes_by_keywords(keywords, page=1, page_size=20): + try: + placeholders = ','.join(['?'] * len(keywords)) + query = f''' + SELECT n.id, n.title, n.content, n.timestamp + FROM rag_qa_notes n + JOIN rag_qa_note_keywords nk ON n.id = nk.note_id + JOIN rag_qa_keywords k ON nk.keyword_id = k.id + WHERE k.keyword IN ({placeholders}) + ORDER BY n.timestamp DESC + ''' + results, total_pages, total_count = get_paginated_results(query, tuple(keywords), page, page_size) + logger.info(f"Retrieved {len(results)} notes matching keywords: {', '.join(keywords)} (page {page} of {total_pages})") + notes = [(row[0], row[1], row[2], row[3]) for row in results] + return notes, total_pages, total_count + except Exception as e: + logger.error(f"Error getting notes by keywords: {e}") + raise + +def get_notes_by_keyword_collection(collection_name, page=1, page_size=20): + try: + query = ''' + SELECT n.id, n.title, n.content, n.timestamp + FROM rag_qa_notes n + JOIN rag_qa_note_keywords nk ON n.id = nk.note_id + JOIN rag_qa_keywords k ON nk.keyword_id = k.id + JOIN rag_qa_collection_keywords ck ON k.id = ck.keyword_id + JOIN rag_qa_keyword_collections c ON ck.collection_id = c.id + WHERE c.name = ? + ORDER BY n.timestamp DESC + ''' + results, total_pages, total_count = get_paginated_results(query, (collection_name,), page, page_size) + logger.info(f"Retrieved {len(results)} notes for collection '{collection_name}' (page {page} of {total_pages})") + notes = [(row[0], row[1], row[2], row[3]) for row in results] + return notes, total_pages, total_count + except Exception as e: + logger.error(f"Error getting notes by keyword collection '{collection_name}': {e}") + raise + +def clear_notes(conversation_id): + """Clear all notes for a given conversation.""" + try: + query = "DELETE FROM rag_qa_notes WHERE conversation_id = ?" + execute_query(query, (conversation_id,)) + logger.info(f"Cleared notes for conversation '{conversation_id}'") + except Exception as e: + logger.error(f"Error clearing notes for conversation '{conversation_id}': {e}") + raise + +def add_keywords_to_note(note_id, keywords): + """Associate keywords with a note.""" + try: + with transaction() as conn: + for keyword in keywords: + validated_keyword = validate_keyword(keyword) + add_keyword(validated_keyword, conn) + + # Retrieve the keyword ID + query = "SELECT id FROM rag_qa_keywords WHERE keyword = ?" + result = execute_query(query, (validated_keyword,), conn) + if result: + keyword_id = result[0][0] + else: + raise Exception(f"Keyword '{validated_keyword}' not found after insertion") + + # Link the note and keyword + query = "INSERT INTO rag_qa_note_keywords (note_id, keyword_id) VALUES (?, ?)" + execute_query(query, (note_id, keyword_id), conn) + + logger.info(f"Keywords added to note ID '{note_id}' successfully") + except Exception as e: + logger.error(f"Error adding keywords to note ID '{note_id}': {e}") + raise + +def get_keywords_for_note(note_id): + """Retrieve keywords associated with a given note.""" + try: + query = ''' + SELECT k.keyword + FROM rag_qa_keywords k + JOIN rag_qa_note_keywords nk ON k.id = nk.keyword_id + WHERE nk.note_id = ? + ''' + result = execute_query(query, (note_id,)) + keywords = [row[0] for row in result] + logger.info(f"Retrieved {len(keywords)} keywords for note ID '{note_id}'") + return keywords + except Exception as e: + logger.error(f"Error getting keywords for note ID '{note_id}': {e}") + raise + +def clear_keywords_from_note(note_id): + """Clear all keywords from a given note.""" + try: + query = "DELETE FROM rag_qa_note_keywords WHERE note_id = ?" + execute_query(query, (note_id,)) + logger.info(f"Cleared keywords for note ID '{note_id}'") + except Exception as e: + logger.error(f"Error clearing keywords for note ID '{note_id}': {e}") + raise + +def delete_note_by_id(note_id, conn=None): + """Delete a note and its associated keywords.""" + try: + # Delete note keywords + execute_query("DELETE FROM rag_qa_note_keywords WHERE note_id = ?", (note_id,), conn) + # Delete the note + execute_query("DELETE FROM rag_qa_notes WHERE id = ?", (note_id,), conn) + logging.info(f"Note ID '{note_id}' deleted successfully.") + except Exception as e: + logger.error(f"Error deleting note ID '{note_id}': {e}") + raise + +def delete_note(note_id): + """Delete a note by ID.""" + try: + with transaction() as conn: + delete_note_by_id(note_id, conn) + except Exception as e: + logger.error(f"Error deleting note ID '{note_id}': {e}") + raise + +# +# End of Notes related functions +################################################### + + +################################################### +# +# Chat-related functions + +def save_message(conversation_id, role, content): + try: + timestamp = datetime.now().isoformat() + query = "INSERT INTO rag_qa_chats (conversation_id, timestamp, role, content) VALUES (?, ?, ?, ?)" + execute_query(query, (conversation_id, timestamp, role, content)) + + # Update last_updated in conversation_metadata + update_query = "UPDATE conversation_metadata SET last_updated = ? WHERE conversation_id = ?" + execute_query(update_query, (timestamp, conversation_id)) + + logger.info(f"Message saved for conversation '{conversation_id}'") + except Exception as e: + logger.error(f"Error saving message for conversation '{conversation_id}': {e}") + raise + +def start_new_conversation(title="Untitled Conversation"): + try: + conversation_id = str(uuid.uuid4()) + query = "INSERT INTO conversation_metadata (conversation_id, created_at, last_updated, title) VALUES (?, ?, ?, ?)" + now = datetime.now().isoformat() + execute_query(query, (conversation_id, now, now, title)) + logger.info(f"New conversation '{conversation_id}' started with title '{title}'") + return conversation_id + except Exception as e: + logger.error(f"Error starting new conversation: {e}") + raise + +def get_all_conversations(page=1, page_size=20): + try: + query = "SELECT conversation_id, title FROM conversation_metadata ORDER BY last_updated DESC" + results, total_pages, total_count = get_paginated_results(query, page=page, page_size=page_size) + conversations = [(row[0], row[1]) for row in results] + logger.info(f"Retrieved {len(conversations)} conversations (page {page} of {total_pages})") + return conversations, total_pages, total_count + except Exception as e: + logger.error(f"Error getting conversations: {e}") + raise + +# Pagination helper function +def get_paginated_results(query, params=None, page=1, page_size=20): + try: + offset = (page - 1) * page_size + paginated_query = f"{query} LIMIT ? OFFSET ?" + if params: + paginated_params = params + (page_size, offset) + else: + paginated_params = (page_size, offset) + + result = execute_query(paginated_query, paginated_params) + + count_query = f"SELECT COUNT(*) FROM ({query}) AS total" + count_params = params if params else () + + total_count = execute_query(count_query, count_params)[0][0] + + total_pages = (total_count + page_size - 1) // page_size + + logger.info(f"Retrieved page {page} of {total_pages} (total items: {total_count})") + return result, total_pages, total_count + except Exception as e: + logger.error(f"Error retrieving paginated results: {e}") + raise + +def get_all_collections(page=1, page_size=20): + try: + query = "SELECT name FROM rag_qa_keyword_collections" + results, total_pages, total_count = get_paginated_results(query, page=page, page_size=page_size) + collections = [row[0] for row in results] + logger.info(f"Retrieved {len(collections)} keyword collections (page {page} of {total_pages})") + return collections, total_pages, total_count + except Exception as e: + logger.error(f"Error getting collections: {e}") + raise + +def search_conversations_by_keywords(keywords, page=1, page_size=20): + try: + placeholders = ','.join(['?' for _ in keywords]) + query = f''' + SELECT DISTINCT cm.conversation_id, cm.title + FROM conversation_metadata cm + JOIN rag_qa_conversation_keywords ck ON cm.conversation_id = ck.conversation_id + JOIN rag_qa_keywords k ON ck.keyword_id = k.id + WHERE k.keyword IN ({placeholders}) + ''' + results, total_pages, total_count = get_paginated_results(query, tuple(keywords), page, page_size) + logger.info( + f"Found {total_count} conversations matching keywords: {', '.join(keywords)} (page {page} of {total_pages})") + return results, total_pages, total_count + except Exception as e: + logger.error(f"Error searching conversations by keywords {keywords}: {e}") + raise + +def load_chat_history(conversation_id, page=1, page_size=50): + try: + query = "SELECT role, content FROM rag_qa_chats WHERE conversation_id = ? ORDER BY timestamp" + results, total_pages, total_count = get_paginated_results(query, (conversation_id,), page, page_size) + logger.info( + f"Loaded {len(results)} messages for conversation '{conversation_id}' (page {page} of {total_pages})") + return results, total_pages, total_count + except Exception as e: + logger.error(f"Error loading chat history for conversation '{conversation_id}': {e}") + raise + +def update_conversation_title(conversation_id, new_title): + """Update the title of a conversation.""" + try: + query = "UPDATE conversation_metadata SET title = ? WHERE conversation_id = ?" + execute_query(query, (new_title, conversation_id)) + logger.info(f"Conversation '{conversation_id}' title updated to '{new_title}'") + except Exception as e: + logger.error(f"Error updating conversation title: {e}") + raise + +def delete_conversation(conversation_id): + """Delete a conversation and its associated messages and notes.""" + try: + with transaction() as conn: + # Delete messages + execute_query("DELETE FROM rag_qa_chats WHERE conversation_id = ?", (conversation_id,), conn) + # Delete conversation metadata + execute_query("DELETE FROM conversation_metadata WHERE conversation_id = ?", (conversation_id,), conn) + # Delete conversation keywords + execute_query("DELETE FROM rag_qa_conversation_keywords WHERE conversation_id = ?", (conversation_id,), conn) + # Delete notes associated with the conversation + note_ids = execute_query("SELECT id FROM rag_qa_notes WHERE conversation_id = ?", (conversation_id,), conn) + for (note_id,) in note_ids: + delete_note_by_id(note_id, conn) + logging.info(f"Conversation '{conversation_id}' deleted successfully.") + except Exception as e: + logger.error(f"Error deleting conversation '{conversation_id}': {e}") + raise + +# +# End of Chat-related functions +################################################### + + +################################################### +# +# Functions to export DB data + +def fetch_all_conversations(): + try: + # Fetch all conversation IDs and titles + query = "SELECT conversation_id, title FROM conversation_metadata ORDER BY last_updated DESC" + results = execute_query(query) + conversations = [] + for row in results: + conversation_id, title = row + # Fetch all messages for this conversation + messages = load_all_chat_history(conversation_id) + conversations.append((conversation_id, title, messages)) + logger.info(f"Fetched all conversations: {len(conversations)} found.") + return conversations + except Exception as e: + logger.error(f"Error fetching all conversations: {e}") + raise + +def load_all_chat_history(conversation_id): + try: + query = "SELECT role, content FROM rag_qa_chats WHERE conversation_id = ? ORDER BY timestamp" + results = execute_query(query, (conversation_id,)) + messages = [(row[0], row[1]) for row in results] + return messages + except Exception as e: + logger.error(f"Error loading chat history for conversation '{conversation_id}': {e}") + raise + +def fetch_all_notes(): + try: + query = "SELECT id, title, content FROM rag_qa_notes ORDER BY timestamp DESC" + results = execute_query(query) + notes = [(row[0], row[1], row[2]) for row in results] + logger.info(f"Fetched all notes: {len(notes)} found.") + return notes + except Exception as e: + logger.error(f"Error fetching all notes: {e}") + raise + +def fetch_conversations_by_ids(conversation_ids): + try: + if not conversation_ids: + return [] + placeholders = ','.join(['?'] * len(conversation_ids)) + query = f"SELECT conversation_id, title FROM conversation_metadata WHERE conversation_id IN ({placeholders})" + results = execute_query(query, conversation_ids) + conversations = [] + for row in results: + conversation_id, title = row + # Fetch all messages for this conversation + messages = load_all_chat_history(conversation_id) + conversations.append((conversation_id, title, messages)) + logger.info(f"Fetched {len(conversations)} conversations by IDs.") + return conversations + except Exception as e: + logger.error(f"Error fetching conversations by IDs: {e}") + raise + +def fetch_notes_by_ids(note_ids): + try: + if not note_ids: + return [] + placeholders = ','.join(['?'] * len(note_ids)) + query = f"SELECT id, title, content FROM rag_qa_notes WHERE id IN ({placeholders})" + results = execute_query(query, note_ids) + notes = [(row[0], row[1], row[2]) for row in results] + logger.info(f"Fetched {len(notes)} notes by IDs.") + return notes + except Exception as e: + logger.error(f"Error fetching notes by IDs: {e}") + raise + +# +# End of Export functions +################################################### + +# +# End of RAG_QA_Chat_DB.py +#################################################################################################### diff --git a/App_Function_Libraries/Gradio_Related.py b/App_Function_Libraries/Gradio_Related.py index 91ec99d21..56303dccb 100644 --- a/App_Function_Libraries/Gradio_Related.py +++ b/App_Function_Libraries/Gradio_Related.py @@ -19,8 +19,7 @@ from App_Function_Libraries.Gradio_UI.Arxiv_tab import create_arxiv_tab from App_Function_Libraries.Gradio_UI.Audio_ingestion_tab import create_audio_processing_tab from App_Function_Libraries.Gradio_UI.Book_Ingestion_tab import create_import_book_tab -from App_Function_Libraries.Gradio_UI.Character_Chat_tab import create_character_card_interaction_tab, \ - create_character_card_interaction_tab, create_character_chat_mgmt_tab, create_custom_character_card_tab, \ +from App_Function_Libraries.Gradio_UI.Character_Chat_tab import create_character_card_interaction_tab, create_character_chat_mgmt_tab, create_custom_character_card_tab, \ create_character_card_validation_tab, create_export_characters_tab from App_Function_Libraries.Gradio_UI.Character_interaction_tab import create_narrator_controlled_conversation_tab, \ create_multiple_character_chat_tab @@ -46,7 +45,8 @@ from App_Function_Libraries.Gradio_UI.Plaintext_tab_import import create_plain_text_import_tab from App_Function_Libraries.Gradio_UI.Podcast_tab import create_podcast_tab from App_Function_Libraries.Gradio_UI.Prompt_Suggestion_tab import create_prompt_suggestion_tab -from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_tab import create_rag_qa_chat_tab +from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_tab import create_rag_qa_chat_tab, create_rag_qa_notes_management_tab, \ + create_rag_qa_chat_management_tab from App_Function_Libraries.Gradio_UI.Re_summarize_tab import create_resummary_tab from App_Function_Libraries.Gradio_UI.Search_Tab import create_prompt_search_tab, \ create_search_summaries_tab, create_search_tab @@ -68,7 +68,6 @@ from App_Function_Libraries.Gradio_UI.Evaluations_Benchmarks_tab import create_geval_tab, create_infinite_bench_tab #from App_Function_Libraries.Local_LLM.Local_LLM_huggingface import create_huggingface_tab from App_Function_Libraries.Local_LLM.Local_LLM_ollama import create_ollama_tab - # ####################################################################################################################### # Function Definitions @@ -270,7 +269,7 @@ def launch_ui(share_public=None, server_mode=False): gr.Markdown(f"# tl/dw: Your LLM-powered Research Multi-tool") gr.Markdown(f"(Using {db_type.capitalize()} Database)") with gr.Tabs(): - with gr.TabItem("Transcription / Summarization / Ingestion"): + with gr.TabItem("Transcription / Summarization / Ingestion", id="ingestion-grouping", visible=True): with gr.Tabs(): create_video_transcription_tab() create_audio_processing_tab() @@ -285,17 +284,17 @@ def launch_ui(share_public=None, server_mode=False): create_live_recording_tab() create_arxiv_tab() - with gr.TabItem("Text Search "): + with gr.TabItem("Text Search", id="text search", visible=True): create_search_tab() create_search_summaries_tab() - - with gr.TabItem("RAG Search"): + with gr.TabItem("RAG Chat/Search", id="RAG Chat Notes group", visible=True): create_rag_tab() create_rag_qa_chat_tab() + create_rag_qa_notes_management_tab() + create_rag_qa_chat_management_tab() - - with gr.TabItem("Chat with an LLM"): + with gr.TabItem("Chat with an LLM", id="LLM Chat group", visible=True): create_chat_interface() create_chat_interface_stacked() create_chat_interface_multi_api() @@ -305,18 +304,17 @@ def launch_ui(share_public=None, server_mode=False): chat_workflows_tab() - with gr.TabItem("Character Chat"): - with gr.Tabs(): - create_character_card_interaction_tab() - create_character_chat_mgmt_tab() - create_custom_character_card_tab() - create_character_card_validation_tab() - create_multiple_character_chat_tab() - create_narrator_controlled_conversation_tab() - create_export_characters_tab() + with gr.TabItem("Character Chat", id="character chat group", visible=True): + create_character_card_interaction_tab() + create_character_chat_mgmt_tab() + create_custom_character_card_tab() + create_character_card_validation_tab() + create_multiple_character_chat_tab() + create_narrator_controlled_conversation_tab() + create_export_characters_tab() - with gr.TabItem("View DB Items"): + with gr.TabItem("View DB Items", id="view db items group", visible=True): # This one works create_view_all_with_versions_tab() # This one is WIP @@ -324,7 +322,7 @@ def launch_ui(share_public=None, server_mode=False): create_prompt_view_tab() - with gr.TabItem("Prompts"): + with gr.TabItem("Prompts", id='view prompts group', visible=True): create_prompt_view_tab() create_prompt_search_tab() create_prompt_edit_tab() @@ -332,7 +330,7 @@ def launch_ui(share_public=None, server_mode=False): create_prompt_suggestion_tab() - with gr.TabItem("Manage / Edit Existing Items"): + with gr.TabItem("Manage / Edit Existing Items", id="manage group", visible=True): create_media_edit_tab() create_manage_items_tab() create_media_edit_and_clone_tab() @@ -340,32 +338,31 @@ def launch_ui(share_public=None, server_mode=False): #create_compare_transcripts_tab() - with gr.TabItem("Embeddings Management"): + with gr.TabItem("Embeddings Management", id="embeddings group", visible=True): create_embeddings_tab() create_view_embeddings_tab() create_purge_embeddings_tab() - with gr.TabItem("Writing Tools"): - with gr.Tabs(): - from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab - create_document_feedback_tab() - from App_Function_Libraries.Gradio_UI.Writing_tab import create_grammar_style_check_tab - create_grammar_style_check_tab() - from App_Function_Libraries.Gradio_UI.Writing_tab import create_tone_adjustment_tab - create_tone_adjustment_tab() - from App_Function_Libraries.Gradio_UI.Writing_tab import create_creative_writing_tab - create_creative_writing_tab() - from App_Function_Libraries.Gradio_UI.Writing_tab import create_mikupad_tab - create_mikupad_tab() - - - with gr.TabItem("Keywords"): + with gr.TabItem("Writing Tools", id="writing_tools group", visible=True): + from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab + create_document_feedback_tab() + from App_Function_Libraries.Gradio_UI.Writing_tab import create_grammar_style_check_tab + create_grammar_style_check_tab() + from App_Function_Libraries.Gradio_UI.Writing_tab import create_tone_adjustment_tab + create_tone_adjustment_tab() + from App_Function_Libraries.Gradio_UI.Writing_tab import create_creative_writing_tab + create_creative_writing_tab() + from App_Function_Libraries.Gradio_UI.Writing_tab import create_mikupad_tab + create_mikupad_tab() + + + with gr.TabItem("Keywords", id="keywords group", visible=True): create_view_keywords_tab() create_add_keyword_tab() create_delete_keyword_tab() create_export_keywords_tab() - with gr.TabItem("Import"): + with gr.TabItem("Import", id="import group", visible=True): create_import_item_tab() create_import_obsidian_vault_tab() create_import_single_prompt_tab() @@ -373,40 +370,40 @@ def launch_ui(share_public=None, server_mode=False): create_mediawiki_import_tab() create_mediawiki_config_tab() - with gr.TabItem("Export"): + with gr.TabItem("Export", id="export group", visible=True): create_export_tab() - with gr.TabItem("Backup Management"): + with gr.TabItem("Backup Management", id="backup group", visible=True): create_backup_tab() create_view_backups_tab() create_restore_backup_tab() - with gr.TabItem("Utilities"): + with gr.TabItem("Utilities", id="util group", visible=True): create_utilities_yt_video_tab() create_utilities_yt_audio_tab() create_utilities_yt_timestamp_tab() - with gr.TabItem("Local LLM"): + with gr.TabItem("Local LLM", id="local llm group", visible=True): create_chat_with_llamafile_tab() create_ollama_tab() #create_huggingface_tab() - with gr.TabItem("Trashcan"): + with gr.TabItem("Trashcan", id="trashcan group", visible=True): create_search_and_mark_trash_tab() create_view_trash_tab() create_delete_trash_tab() create_empty_trash_tab() - with gr.TabItem("Evaluations"): + with gr.TabItem("Evaluations", id="eval", visible=True): create_geval_tab() create_infinite_bench_tab() # FIXME #create_mmlu_pro_tab() - with gr.TabItem("Introduction/Help"): + with gr.TabItem("Introduction/Help", id="introduction group", visible=True): create_introduction_tab() - with gr.TabItem("Config Editor"): + with gr.TabItem("Config Editor", id="config group"): create_config_editor_tab() # Launch the interface diff --git a/App_Function_Libraries/Gradio_UI/Arxiv_tab.py b/App_Function_Libraries/Gradio_UI/Arxiv_tab.py index 3c052e8d1..530edefce 100644 --- a/App_Function_Libraries/Gradio_UI/Arxiv_tab.py +++ b/App_Function_Libraries/Gradio_UI/Arxiv_tab.py @@ -20,7 +20,7 @@ # Functions: def create_arxiv_tab(): - with gr.TabItem("Arxiv Search & Ingest"): + with gr.TabItem("Arxiv Search & Ingest", visible=True): gr.Markdown("# arXiv Search, Browse, Download, and Ingest") gr.Markdown("#### Thank you to arXiv for use of its open access interoperability.") with gr.Row(): diff --git a/App_Function_Libraries/Gradio_UI/Audio_ingestion_tab.py b/App_Function_Libraries/Gradio_UI/Audio_ingestion_tab.py index c2f646639..0a785631f 100644 --- a/App_Function_Libraries/Gradio_UI/Audio_ingestion_tab.py +++ b/App_Function_Libraries/Gradio_UI/Audio_ingestion_tab.py @@ -20,7 +20,7 @@ # Functions: def create_audio_processing_tab(): - with gr.TabItem("Audio File Transcription + Summarization"): + with gr.TabItem("Audio File Transcription + Summarization", visible=True): gr.Markdown("# Transcribe & Summarize Audio Files from URLs or Local Files!") with gr.Row(): with gr.Column(): diff --git a/App_Function_Libraries/Gradio_UI/Backup_Functionality.py b/App_Function_Libraries/Gradio_UI/Backup_Functionality.py index 6400dd792..672975da2 100644 --- a/App_Function_Libraries/Gradio_UI/Backup_Functionality.py +++ b/App_Function_Libraries/Gradio_UI/Backup_Functionality.py @@ -34,7 +34,7 @@ def restore_backup(backup_name: str) -> str: def create_backup_tab(): - with gr.Tab("Create Backup"): + with gr.Tab("Create Backup", visible=True): gr.Markdown("# Create a backup of the database") gr.Markdown("This will create a backup of the database in the backup directory(the default backup directory is `/tldw_DB_Backups/')") with gr.Row(): @@ -46,7 +46,7 @@ def create_backup_tab(): def create_view_backups_tab(): - with gr.TabItem("View Backups"): + with gr.TabItem("View Backups", visible=True): gr.Markdown("# Browse available backups") with gr.Row(): with gr.Column(): @@ -57,7 +57,7 @@ def create_view_backups_tab(): def create_restore_backup_tab(): - with gr.TabItem("Restore Backup"): + with gr.TabItem("Restore Backup", visible=True): gr.Markdown("# Restore a backup of the database") with gr.Column(): backup_input = gr.Textbox(label="Backup Filename") diff --git a/App_Function_Libraries/Gradio_UI/Book_Ingestion_tab.py b/App_Function_Libraries/Gradio_UI/Book_Ingestion_tab.py index d4e1b2989..28e60b09a 100644 --- a/App_Function_Libraries/Gradio_UI/Book_Ingestion_tab.py +++ b/App_Function_Libraries/Gradio_UI/Book_Ingestion_tab.py @@ -22,7 +22,7 @@ def create_import_book_tab(): - with gr.TabItem("Ebook(epub) Files"): + with gr.TabItem("Ebook(epub) Files", visible=True): with gr.Row(): with gr.Column(): gr.Markdown("# Import .epub files") diff --git a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py index 882de6dbe..f9723fdd0 100644 --- a/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/Character_Chat_tab.py @@ -2,7 +2,6 @@ # Description: Library for character card import functions # # Imports -import os import re import tempfile import uuid @@ -16,7 +15,6 @@ # # External Imports from PIL import Image -from PIL.PngImagePlugin import PngInfo import gradio as gr # # Local Imports @@ -254,7 +252,7 @@ def export_all_characters(): # Gradio tabs def create_character_card_interaction_tab(): - with gr.TabItem("Chat with a Character Card"): + with gr.TabItem("Chat with a Character Card", visible=True): gr.Markdown("# Chat with a Character Card") with gr.Row(): with gr.Column(scale=1): @@ -1025,7 +1023,7 @@ def answer_for_me( def create_character_chat_mgmt_tab(): - with gr.TabItem("Character and Chat Management"): + with gr.TabItem("Character and Chat Management", visible=True): gr.Markdown("# Character and Chat Management") with gr.Row(): @@ -1063,12 +1061,12 @@ def create_character_chat_mgmt_tab(): conversation_mapping = gr.State({}) with gr.Tabs(): - with gr.TabItem("Edit"): + with gr.TabItem("Edit", visible=True): chat_content = gr.TextArea(label="Chat/Character Content (JSON)", lines=20, max_lines=50) save_button = gr.Button("Save Changes") delete_button = gr.Button("Delete Conversation/Character", variant="stop") - with gr.TabItem("Preview"): + with gr.TabItem("Preview", visible=True): chat_preview = gr.HTML(label="Chat/Character Preview") result_message = gr.Markdown("") @@ -1380,7 +1378,7 @@ def import_multiple_characters(files): ) def create_custom_character_card_tab(): - with gr.TabItem("Create a New Character Card"): + with gr.TabItem("Create a New Character Card", visible=True): gr.Markdown("# Create a New Character Card (v2)") with gr.Row(): @@ -1628,9 +1626,9 @@ def download_character_card_as_image( outputs=[download_image_output, save_status] ) -#v1 + def create_character_card_validation_tab(): - with gr.TabItem("Validate Character Card"): + with gr.TabItem("Validate Character Card", visible=True): gr.Markdown("# Validate Character Card (v2)") gr.Markdown("Upload a character card (PNG, WEBP, or JSON) to validate whether it conforms to the Character Card V2 specification.") @@ -1786,7 +1784,7 @@ def validate_v2_card(card_data): def create_export_characters_tab(): - with gr.TabItem("Export Characters"): + with gr.TabItem("Export Characters", visible=True): gr.Markdown("# Export Characters") gr.Markdown("Export character cards individually as JSON files or all together as a ZIP file.") @@ -1808,19 +1806,20 @@ def create_export_characters_tab(): export_output = gr.File(label="Exported Character(s)", interactive=False) export_status = gr.Markdown("") +# FIXME def export_single_character_wrapper(character_selection): file_path, status_message = export_single_character(character_selection) if file_path: - return gr.File.update(value=file_path), status_message + return gr.update(value=file_path), status_message else: - return gr.File.update(value=None), status_message + return gr.update(value=None), status_message def export_all_characters_wrapper(): zip_path = export_all_characters_as_zip() characters = get_character_cards() exported_characters = [char['name'] for char in characters] status_message = f"Exported {len(exported_characters)} characters successfully:\n" + "\n".join(exported_characters) - return gr.File.update(value=zip_path), status_message + return gr.update(value=zip_path), status_message # Event listeners load_characters_button.click( diff --git a/App_Function_Libraries/Gradio_UI/Character_interaction_tab.py b/App_Function_Libraries/Gradio_UI/Character_interaction_tab.py index 1d2b3a576..0e629def1 100644 --- a/App_Function_Libraries/Gradio_UI/Character_interaction_tab.py +++ b/App_Function_Libraries/Gradio_UI/Character_interaction_tab.py @@ -253,7 +253,7 @@ def character_interaction(character1: str, character2: str, api_endpoint: str, a def create_multiple_character_chat_tab(): - with gr.TabItem("Multi-Character Chat"): + with gr.TabItem("Multi-Character Chat", visible=True): characters, conversation, current_character, other_character = character_interaction_setup() with gr.Blocks() as character_interaction: @@ -393,7 +393,7 @@ def take_turn_with_error_handling(conversation, current_index, char1, char2, cha # From `Fuzzlewumper` on Reddit. def create_narrator_controlled_conversation_tab(): - with gr.TabItem("Narrator-Controlled Conversation"): + with gr.TabItem("Narrator-Controlled Conversation", visible=True): gr.Markdown("# Narrator-Controlled Conversation") with gr.Row(): diff --git a/App_Function_Libraries/Gradio_UI/Chat_Workflows.py b/App_Function_Libraries/Gradio_UI/Chat_Workflows.py index 198acc0a3..8ad2ad22e 100644 --- a/App_Function_Libraries/Gradio_UI/Chat_Workflows.py +++ b/App_Function_Libraries/Gradio_UI/Chat_Workflows.py @@ -24,7 +24,7 @@ def chat_workflows_tab(): - with gr.TabItem("Chat Workflows"): + with gr.TabItem("Chat Workflows", visible=True): gr.Markdown("# Workflows using LLMs") chat_history = gr.State([]) media_content = gr.State({}) diff --git a/App_Function_Libraries/Gradio_UI/Chat_ui.py b/App_Function_Libraries/Gradio_UI/Chat_ui.py index fed68304e..1ccd74263 100644 --- a/App_Function_Libraries/Gradio_UI/Chat_ui.py +++ b/App_Function_Libraries/Gradio_UI/Chat_ui.py @@ -206,7 +206,7 @@ def create_chat_interface(): font-size: 14px !important; } """ - with gr.TabItem("Remote LLM Chat (Horizontal)"): + with gr.TabItem("Remote LLM Chat (Horizontal)", visible=True): gr.Markdown("# Chat with a designated LLM Endpoint, using your selected item as starting context") chat_history = gr.State([]) media_content = gr.State({}) @@ -417,7 +417,7 @@ def create_chat_interface_stacked(): font-size: 14px !important; } """ - with gr.TabItem("Remote LLM Chat - Stacked"): + with gr.TabItem("Remote LLM Chat - Stacked", visible=True): gr.Markdown("# Stacked Chat") chat_history = gr.State([]) media_content = gr.State({}) @@ -580,7 +580,7 @@ def create_chat_interface_multi_api(): overflow-y: auto; } """ - with gr.TabItem("One Prompt - Multiple APIs"): + with gr.TabItem("One Prompt - Multiple APIs", visible=True): gr.Markdown("# One Prompt but Multiple APIs Chat Interface") with gr.Row(): @@ -759,7 +759,7 @@ def create_chat_interface_four(): } """ - with gr.TabItem("Four Independent API Chats"): + with gr.TabItem("Four Independent API Chats", visible=True): gr.Markdown("# Four Independent API Chat Interfaces") with gr.Row(): @@ -956,7 +956,7 @@ def chat_wrapper_single(message, chat_history, chatbot, api_endpoint, api_key, t # FIXME - Finish implementing functions + testing/valdidation def create_chat_management_tab(): - with gr.TabItem("Chat Management"): + with gr.TabItem("Chat Management", visible=True): gr.Markdown("# Chat Management") with gr.Row(): @@ -967,12 +967,12 @@ def create_chat_management_tab(): conversation_mapping = gr.State({}) with gr.Tabs(): - with gr.TabItem("Edit"): + with gr.TabItem("Edit", visible=True): chat_content = gr.TextArea(label="Chat Content (JSON)", lines=20, max_lines=50) save_button = gr.Button("Save Changes") delete_button = gr.Button("Delete Conversation", variant="stop") - with gr.TabItem("Preview"): + with gr.TabItem("Preview", visible=True): chat_preview = gr.HTML(label="Chat Preview") result_message = gr.Markdown("") diff --git a/App_Function_Libraries/Gradio_UI/Config_tab.py b/App_Function_Libraries/Gradio_UI/Config_tab.py index 39a3c618f..cf6631e53 100644 --- a/App_Function_Libraries/Gradio_UI/Config_tab.py +++ b/App_Function_Libraries/Gradio_UI/Config_tab.py @@ -25,7 +25,7 @@ def save_config_from_text(text): def create_config_editor_tab(): - with gr.TabItem("Edit Config"): + with gr.TabItem("Edit Config", visible=True): gr.Markdown("# Edit Configuration File") with gr.Row(): diff --git a/App_Function_Libraries/Gradio_UI/Embeddings_tab.py b/App_Function_Libraries/Gradio_UI/Embeddings_tab.py index 7f5a3bf09..a12e10320 100644 --- a/App_Function_Libraries/Gradio_UI/Embeddings_tab.py +++ b/App_Function_Libraries/Gradio_UI/Embeddings_tab.py @@ -22,7 +22,7 @@ # Functions: def create_embeddings_tab(): - with gr.TabItem("Create Embeddings"): + with gr.TabItem("Create Embeddings", visible=True): gr.Markdown("# Create Embeddings for All Content") with gr.Row(): @@ -185,7 +185,7 @@ def create_all_embeddings(provider, hf_model, openai_model, custom_model, api_ur def create_view_embeddings_tab(): - with gr.TabItem("View/Update Embeddings"): + with gr.TabItem("View/Update Embeddings", visible=True): gr.Markdown("# View and Update Embeddings") item_mapping = gr.State({}) with gr.Row(): @@ -475,7 +475,7 @@ def create_new_embedding_for_item(selected_item, provider, hf_model, openai_mode def create_purge_embeddings_tab(): - with gr.TabItem("Purge Embeddings"): + with gr.TabItem("Purge Embeddings", visible=True): gr.Markdown("# Purge Embeddings") with gr.Row(): diff --git a/App_Function_Libraries/Gradio_UI/Evaluations_Benchmarks_tab.py b/App_Function_Libraries/Gradio_UI/Evaluations_Benchmarks_tab.py index d7783a630..4e27d784f 100644 --- a/App_Function_Libraries/Gradio_UI/Evaluations_Benchmarks_tab.py +++ b/App_Function_Libraries/Gradio_UI/Evaluations_Benchmarks_tab.py @@ -6,7 +6,7 @@ from App_Function_Libraries.Benchmarks_Evaluations.ms_g_eval import run_geval def create_geval_tab(): - with gr.Tab("G-Eval"): + with gr.Tab("G-Eval", visible=True): gr.Markdown("# G-Eval Summarization Evaluation") with gr.Row(): with gr.Column(): @@ -31,7 +31,7 @@ def create_geval_tab(): def create_infinite_bench_tab(): - with gr.Tab("Infinite Bench"): + with gr.Tab("Infinite Bench", visible=True): gr.Markdown("# Infinite Bench Evaluation (Coming Soon)") with gr.Row(): with gr.Column(): diff --git a/App_Function_Libraries/Gradio_UI/Explain_summarize_tab.py b/App_Function_Libraries/Gradio_UI/Explain_summarize_tab.py index 4fd8c4a03..6f549b6b9 100644 --- a/App_Function_Libraries/Gradio_UI/Explain_summarize_tab.py +++ b/App_Function_Libraries/Gradio_UI/Explain_summarize_tab.py @@ -24,7 +24,7 @@ # Functions: def create_summarize_explain_tab(): - with gr.TabItem("Analyze Text"): + with gr.TabItem("Analyze Text", visible=True): gr.Markdown("# Analyze / Explain / Summarize Text without ingesting it into the DB") with gr.Row(): with gr.Column(): diff --git a/App_Function_Libraries/Gradio_UI/Import_Functionality.py b/App_Function_Libraries/Gradio_UI/Import_Functionality.py index a25c3fcf9..b73d974c4 100644 --- a/App_Function_Libraries/Gradio_UI/Import_Functionality.py +++ b/App_Function_Libraries/Gradio_UI/Import_Functionality.py @@ -159,7 +159,7 @@ def parse_obsidian_note(file_path): } def create_import_single_prompt_tab(): - with gr.TabItem("Import a Prompt"): + with gr.TabItem("Import a Prompt", visible=True): gr.Markdown("# Import a prompt into the database") with gr.Row(): @@ -213,7 +213,7 @@ def update_prompt_dropdown(): ) def create_import_item_tab(): - with gr.TabItem("Import Markdown/Text Files"): + with gr.TabItem("Import Markdown/Text Files", visible=True): gr.Markdown("# Import a markdown file or text file into the database") gr.Markdown("...and have it tagged + summarized") with gr.Row(): @@ -246,7 +246,7 @@ def create_import_item_tab(): def create_import_multiple_prompts_tab(): - with gr.TabItem("Import Multiple Prompts"): + with gr.TabItem("Import Multiple Prompts", visible=True): gr.Markdown("# Import multiple prompts into the database") gr.Markdown("Upload a zip file containing multiple prompt files (txt or md)") @@ -326,7 +326,7 @@ def update_prompt_dropdown(): def create_import_obsidian_vault_tab(): - with gr.TabItem("Import Obsidian Vault"): + with gr.TabItem("Import Obsidian Vault", visible=True): gr.Markdown("## Import Obsidian Vault") with gr.Row(): with gr.Column(): diff --git a/App_Function_Libraries/Gradio_UI/Introduction_tab.py b/App_Function_Libraries/Gradio_UI/Introduction_tab.py index 5e8310096..9daa53cd0 100644 --- a/App_Function_Libraries/Gradio_UI/Introduction_tab.py +++ b/App_Function_Libraries/Gradio_UI/Introduction_tab.py @@ -16,7 +16,7 @@ def create_introduction_tab(): - with (gr.TabItem("Introduction")): + with gr.TabItem("Introduction", visible=True): db_config = get_db_config() db_type = db_config['type'] gr.Markdown(f"# tldw: Your LLM-powered Research Multi-tool (Using {db_type.capitalize()} Database)") diff --git a/App_Function_Libraries/Gradio_UI/Keywords.py b/App_Function_Libraries/Gradio_UI/Keywords.py index 3a6579e87..b2c7a213b 100644 --- a/App_Function_Libraries/Gradio_UI/Keywords.py +++ b/App_Function_Libraries/Gradio_UI/Keywords.py @@ -19,7 +19,7 @@ def create_export_keywords_tab(): - with gr.Tab("Export Keywords"): + with gr.TabItem("Export Keywords", visible=True): with gr.Row(): with gr.Column(): export_keywords_button = gr.Button("Export Keywords") @@ -33,7 +33,7 @@ def create_export_keywords_tab(): ) def create_view_keywords_tab(): - with gr.TabItem("View Keywords"): + with gr.TabItem("View Keywords", visible=True): gr.Markdown("# Browse Keywords") with gr.Column(): browse_output = gr.Markdown() @@ -42,7 +42,7 @@ def create_view_keywords_tab(): def create_add_keyword_tab(): - with gr.TabItem("Add Keywords"): + with gr.TabItem("Add Keywords", visible=True): with gr.Row(): with gr.Column(): gr.Markdown("# Add Keywords to the Database") @@ -54,7 +54,7 @@ def create_add_keyword_tab(): def create_delete_keyword_tab(): - with gr.Tab("Delete Keywords"): + with gr.Tab("Delete Keywords", visible=True): with gr.Row(): with gr.Column(): gr.Markdown("# Delete Keywords from the Database") diff --git a/App_Function_Libraries/Gradio_UI/Live_Recording.py b/App_Function_Libraries/Gradio_UI/Live_Recording.py index 94f2c4c82..b19c3664d 100644 --- a/App_Function_Libraries/Gradio_UI/Live_Recording.py +++ b/App_Function_Libraries/Gradio_UI/Live_Recording.py @@ -22,7 +22,7 @@ "distil-large-v2", "distil-medium.en", "distil-small.en"] def create_live_recording_tab(): - with gr.Tab("Live Recording and Transcription"): + with gr.Tab("Live Recording and Transcription", visible=True): gr.Markdown("# Live Audio Recording and Transcription") with gr.Row(): with gr.Column(): diff --git a/App_Function_Libraries/Gradio_UI/Llamafile_tab.py b/App_Function_Libraries/Gradio_UI/Llamafile_tab.py index 0b1164d12..d8b256fa1 100644 --- a/App_Function_Libraries/Gradio_UI/Llamafile_tab.py +++ b/App_Function_Libraries/Gradio_UI/Llamafile_tab.py @@ -74,7 +74,7 @@ def download_preset_model(selected_model: str) -> Tuple[str, str]: logging.error(f"Error downloading model: {e}") return f"Failed to download model: {e}", "" - with gr.TabItem("Local LLM with Llamafile"): + with gr.TabItem("Local LLM with Llamafile", visible=True): gr.Markdown("# Settings for Llamafile") with gr.Row(): diff --git a/App_Function_Libraries/Gradio_UI/MMLU_Pro_tab.py b/App_Function_Libraries/Gradio_UI/MMLU_Pro_tab.py index e7a1c293c..345e860a5 100644 --- a/App_Function_Libraries/Gradio_UI/MMLU_Pro_tab.py +++ b/App_Function_Libraries/Gradio_UI/MMLU_Pro_tab.py @@ -78,7 +78,7 @@ def run_benchmark_from_ui(url, api_key, model, timeout, category, parallel, verb def create_mmlu_pro_tab(): """Create the Gradio UI tab for MMLU-Pro Benchmark.""" - with gr.Tab("MMLU-Pro Benchmark"): + with gr.TabItem("MMLU-Pro Benchmark", visible=True): gr.Markdown("## Run MMLU-Pro Benchmark") with gr.Row(): diff --git a/App_Function_Libraries/Gradio_UI/Media_edit.py b/App_Function_Libraries/Gradio_UI/Media_edit.py index 0473e8246..0912736b4 100644 --- a/App_Function_Libraries/Gradio_UI/Media_edit.py +++ b/App_Function_Libraries/Gradio_UI/Media_edit.py @@ -16,7 +16,7 @@ def create_media_edit_tab(): - with gr.TabItem("Edit Existing Items"): + with gr.TabItem("Edit Existing Items", visible=True): gr.Markdown("# Search and Edit Media Items") with gr.Row(): @@ -89,7 +89,7 @@ def update_media_with_keywords(selected_item, item_mapping, content, prompt, sum def create_media_edit_and_clone_tab(): - with gr.TabItem("Clone and Edit Existing Items"): + with gr.TabItem("Clone and Edit Existing Items", visible=True): gr.Markdown("# Search, Edit, and Clone Existing Items") with gr.Row(): @@ -199,7 +199,7 @@ def save_cloned_item(selected_item, item_mapping, content, prompt, summary, new_ def create_prompt_edit_tab(): - with gr.TabItem("Add & Edit Prompts"): + with gr.TabItem("Add & Edit Prompts", visible=True): with gr.Row(): with gr.Column(): prompt_dropdown = gr.Dropdown( @@ -239,7 +239,7 @@ def create_prompt_edit_tab(): def create_prompt_clone_tab(): - with gr.TabItem("Clone and Edit Prompts"): + with gr.TabItem("Clone and Edit Prompts", visible=True): with gr.Row(): with gr.Column(): gr.Markdown("# Clone and Edit Prompts") diff --git a/App_Function_Libraries/Gradio_UI/Media_wiki_tab.py b/App_Function_Libraries/Gradio_UI/Media_wiki_tab.py index dba547ab2..9a1aeb93a 100644 --- a/App_Function_Libraries/Gradio_UI/Media_wiki_tab.py +++ b/App_Function_Libraries/Gradio_UI/Media_wiki_tab.py @@ -229,7 +229,7 @@ def save_config(updated_config): def create_mediawiki_config_tab(): - with gr.TabItem("MediaWiki Import Configuration"): + with gr.TabItem("MediaWiki Import Configuration", visible=True): gr.Markdown("# MediaWiki Import Configuration (Broken currently/doesn't work)") with gr.Row(): with gr.Column(): diff --git a/App_Function_Libraries/Gradio_UI/PDF_ingestion_tab.py b/App_Function_Libraries/Gradio_UI/PDF_ingestion_tab.py index 0b9ae318c..25c5ba6ec 100644 --- a/App_Function_Libraries/Gradio_UI/PDF_ingestion_tab.py +++ b/App_Function_Libraries/Gradio_UI/PDF_ingestion_tab.py @@ -21,7 +21,7 @@ # Functions: def create_pdf_ingestion_tab(): - with gr.TabItem("PDF Ingestion"): + with gr.TabItem("PDF Ingestion", visible=True): # TODO - Add functionality to extract metadata from pdf as part of conversion process in marker gr.Markdown("# Ingest PDF Files and Extract Metadata") with gr.Row(): @@ -136,7 +136,7 @@ def test_pdf_ingestion(pdf_file): return f"Error ingesting PDF: {str(e)}", "" def create_pdf_ingestion_test_tab(): - with gr.TabItem("Test PDF Ingestion"): + with gr.TabItem("Test PDF Ingestion", visible=True): with gr.Row(): with gr.Column(): pdf_file_input = gr.File(label="Upload PDF for testing") diff --git a/App_Function_Libraries/Gradio_UI/Plaintext_tab_import.py b/App_Function_Libraries/Gradio_UI/Plaintext_tab_import.py index c4796998c..f16c12f8f 100644 --- a/App_Function_Libraries/Gradio_UI/Plaintext_tab_import.py +++ b/App_Function_Libraries/Gradio_UI/Plaintext_tab_import.py @@ -23,7 +23,7 @@ # Functions: def create_plain_text_import_tab(): - with gr.TabItem("Import Plain text & .docx Files"): + with gr.TabItem("Import Plain text & .docx Files", visible=True): with gr.Row(): with gr.Column(): gr.Markdown("# Import Markdown(`.md`)/Text(`.txt`)/rtf & `.docx` Files") diff --git a/App_Function_Libraries/Gradio_UI/Podcast_tab.py b/App_Function_Libraries/Gradio_UI/Podcast_tab.py index 19f89cae2..d31051f82 100644 --- a/App_Function_Libraries/Gradio_UI/Podcast_tab.py +++ b/App_Function_Libraries/Gradio_UI/Podcast_tab.py @@ -17,8 +17,8 @@ def create_podcast_tab(): - with gr.TabItem("Podcast"): - gr.Markdown("# Podcast Transcription and Ingestion") + with gr.TabItem("Podcast", visible=True): + gr.Markdown("# Podcast Transcription and Ingestion", visible=True) with gr.Row(): with gr.Column(): podcast_url_input = gr.Textbox(label="Podcast URL", placeholder="Enter the podcast URL here") diff --git a/App_Function_Libraries/Gradio_UI/Prompt_Suggestion_tab.py b/App_Function_Libraries/Gradio_UI/Prompt_Suggestion_tab.py index ce3810c27..98861d403 100644 --- a/App_Function_Libraries/Gradio_UI/Prompt_Suggestion_tab.py +++ b/App_Function_Libraries/Gradio_UI/Prompt_Suggestion_tab.py @@ -18,7 +18,7 @@ # Gradio tab for prompt suggestion and testing def create_prompt_suggestion_tab(): - with gr.TabItem("Prompt Suggestion/Creation"): + with gr.TabItem("Prompt Suggestion/Creation", visible=True): gr.Markdown("# Generate and Test AI Prompts with the Metaprompt Approach") with gr.Row(): diff --git a/App_Function_Libraries/Gradio_UI/RAG_Chat_tab.py b/App_Function_Libraries/Gradio_UI/RAG_Chat_tab.py index c8b643469..4e4ba46ce 100644 --- a/App_Function_Libraries/Gradio_UI/RAG_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/RAG_Chat_tab.py @@ -16,7 +16,7 @@ # Functions: def create_rag_tab(): - with gr.TabItem("RAG Search"): + with gr.TabItem("RAG Search", visible=True): gr.Markdown("# Retrieval-Augmented Generation (RAG) Search") with gr.Row(): diff --git a/App_Function_Libraries/Gradio_UI/RAG_QA_Chat_tab.py b/App_Function_Libraries/Gradio_UI/RAG_QA_Chat_tab.py index e682d2734..4b9f4cee8 100644 --- a/App_Function_Libraries/Gradio_UI/RAG_QA_Chat_tab.py +++ b/App_Function_Libraries/Gradio_UI/RAG_QA_Chat_tab.py @@ -11,23 +11,52 @@ # External Imports import docx2txt import gradio as gr +# # Local Imports from App_Function_Libraries.Books.Book_Ingestion_Lib import read_epub from App_Function_Libraries.DB.DB_Manager import DatabaseError, get_paginated_files, add_media_with_keywords +from App_Function_Libraries.DB.RAG_QA_Chat_DB import ( + save_notes, + add_keywords_to_note, + start_new_conversation, + save_message, + search_conversations_by_keywords, + load_chat_history, + get_all_conversations, + get_note_by_id, + get_notes_by_keywords, + get_notes_by_keyword_collection, + update_note, + clear_keywords_from_note, get_notes, get_keywords_for_note, delete_conversation, delete_note, execute_query, + add_keywords_to_conversation, fetch_all_notes, fetch_all_conversations, fetch_conversations_by_ids, + fetch_notes_by_ids, +) from App_Function_Libraries.PDF.PDF_Ingestion_Lib import extract_text_and_format_from_pdf from App_Function_Libraries.RAG.RAG_Library_2 import generate_answer, enhanced_rag_pipeline from App_Function_Libraries.RAG.RAG_QA_Chat import search_database, rag_qa_chat -# Eventually... FIXME -from App_Function_Libraries.RAG.RAG_QA_Chat import load_chat_history, save_chat_history # ######################################################################################################################## # # Functions: def create_rag_qa_chat_tab(): - with gr.TabItem("RAG QA Chat"): + with gr.TabItem("RAG QA Chat", visible=True): gr.Markdown("# RAG QA Chat") + state = gr.State({ + "page": 1, + "context_source": "Entire Media Database", + "conversation_messages": [], + }) + + note_state = gr.State({"note_id": None}) + + # Update the conversation list function + def update_conversation_list(): + conversations, total_pages, total_count = get_all_conversations() + choices = [f"{title} (ID: {conversation_id})" for conversation_id, title in conversations] + return choices + with gr.Row(): with gr.Column(scale=1): context_source = gr.Radio( @@ -41,32 +70,251 @@ def create_rag_qa_chat_tab(): prev_page_btn = gr.Button("Previous Page") next_page_btn = gr.Button("Next Page") page_info = gr.HTML("Page 1") + top_k_input = gr.Number(value=10, label="Maximum amount of results to use (Default: 10)", minimum=1, maximum=50, step=1, precision=0, interactive=True) + keywords_input = gr.Textbox(label="Keywords (comma-separated) to filter results by)", visible=True) + use_query_rewriting = gr.Checkbox(label="Use Query Rewriting", value=True) + use_re_ranking = gr.Checkbox(label="Use Re-ranking", value=True) + # with gr.Row(): + # page_number = gr.Number(value=1, label="Page", precision=0) + # page_size = gr.Number(value=20, label="Items per page", precision=0) + # total_pages = gr.Number(label="Total Pages", interactive=False) + search_query = gr.Textbox(label="Search Query", visible=False) search_button = gr.Button("Search", visible=False) search_results = gr.Dropdown(label="Search Results", choices=[], visible=False) + # FIXME - Add pages for search results handling file_upload = gr.File( label="Upload File", visible=False, - file_types=["txt", "pdf", "epub", "md", "rtf", "json", "csv"] + file_types=["txt", "pdf", "epub", "md", "rtf", "json", "csv", "docx"] ) convert_to_text = gr.Checkbox(label="Convert to plain text", visible=False) - keywords = gr.Textbox(label="Keywords (comma-separated)", visible=False) + + with gr.Column(scale=1): + load_conversation = gr.Dropdown( + label="Load Conversation", + choices=update_conversation_list() + ) + new_conversation = gr.Button("New Conversation") + save_conversation_button = gr.Button("Save Conversation") + conversation_title = gr.Textbox( + label="Conversation Title", placeholder="Enter a title for the new conversation" + ) + keywords = gr.Textbox(label="Keywords (comma-separated)", visible=True) api_choice = gr.Dropdown( - choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"], + choices=[ + "Local-LLM", + "OpenAI", + "Anthropic", + "Cohere", + "Groq", + "DeepSeek", + "Mistral", + "OpenRouter", + "Llama.cpp", + "Kobold", + "Ooba", + "Tabbyapi", + "VLLM", + "ollama", + "HuggingFace", + ], label="Select API for RAG", - value="OpenAI" + value="OpenAI", ) - use_query_rewriting = gr.Checkbox(label="Use Query Rewriting", value=True) + with gr.Row(): with gr.Column(scale=2): - chatbot = gr.Chatbot(height=500) + chatbot = gr.Chatbot(height=700) msg = gr.Textbox(label="Enter your message") - submit = gr.Button("Submit (Might take a few seconds/turns blue while processing...)") + submit = gr.Button("Submit") clear_chat = gr.Button("Clear Chat History") - loading_indicator = gr.HTML(visible=False) + with gr.Column(scale=1): + # Adding UI elements for notes + note_title = gr.Textbox(label="Note Title", placeholder="Enter a title for the note") + notes = gr.TextArea(label="Notes", placeholder="Enter your notes here...", lines=25) + keywords_for_notes = gr.Textbox( + label="Keywords for Notes (comma-separated)", + placeholder="Enter keywords for the note", + visible=True, + ) + save_notes_btn = gr.Button("Save Note") + clear_notes_btn = gr.Button("Clear Current Note text") + + new_note_btn = gr.Button("New Note") + search_notes_by_keyword = gr.Textbox(label="Search Notes by Keyword") + search_notes_button = gr.Button("Search Notes") + note_results = gr.Dropdown(label="Notes", choices=[]) + load_note = gr.Dropdown(label="Load Note", choices=[]) + + loading_indicator = gr.HTML("Loading...", visible=False) + status_message = gr.HTML() + + # Function Definitions + + def update_state(state, **kwargs): + new_state = state.copy() + new_state.update(kwargs) + return new_state + + def create_new_note(): + return gr.update(value='un-named note'), gr.update(value=''), {"note_id": None} + + new_note_btn.click( + create_new_note, + outputs=[note_title, notes, note_state] + ) + + def search_notes(keywords): + if keywords: + keywords_list = [kw.strip() for kw in keywords.split(',')] + notes_data, total_pages, total_count = get_notes_by_keywords(keywords_list) + choices = [f"Note {note_id} ({timestamp})" for note_id, title, content, timestamp in notes_data] + return gr.update(choices=choices) + else: + return gr.update(choices=[]) + + search_notes_button.click( + search_notes, + inputs=[search_notes_by_keyword], + outputs=[note_results] + ) + + def load_selected_note(note_selection): + if note_selection: + note_id = int(note_selection.split(' ')[1]) + note_data = get_note_by_id(note_id) + if note_data: + note_id, title, content = note_data[0] + updated_note_state = {"note_id": note_id} + return gr.update(value=title), gr.update(value=content), updated_note_state + return gr.update(value=''), gr.update(value=''), {"note_id": None} + + note_results.change( + load_selected_note, + inputs=[note_results], + outputs=[note_title, notes, note_state] + ) + + def save_notes_function(note_title_text, notes_content, keywords_content, note_state_value, state_value): + """Save the notes and associated keywords to the database.""" + conversation_id = state_value.get("conversation_id") + note_id = note_state_value["note_id"] + if conversation_id and notes_content: + if note_id: + # Update existing note + update_note(note_id, note_title_text, notes_content) + else: + # Save new note + note_id = save_notes(conversation_id, note_title_text, notes_content) + note_state_value["note_id"] = note_id + if keywords_content: + # Clear existing keywords and add new ones + clear_keywords_from_note(note_id) + add_keywords_to_note(note_id, [kw.strip() for kw in keywords_content.split(',')]) + + logging.info("Notes and keywords saved successfully!") + return notes_content, note_state_value + else: + logging.warning("No conversation ID or notes to save.") + return "", note_state_value + + save_notes_btn.click( + save_notes_function, + inputs=[note_title, notes, keywords_for_notes, note_state, state], + outputs=[notes, note_state] + ) + + def clear_notes_function(): + """Clear notes for the current note.""" + return gr.update(value=''), {"note_id": None} + + clear_notes_btn.click( + clear_notes_function, + outputs=[notes, note_state] + ) + + def update_conversation_list(): + conversations, total_pages, total_count = get_all_conversations() + choices = [f"{title} (ID: {conversation_id})" for conversation_id, title in conversations] + return choices + + # Initialize the conversation list + load_conversation.choices = update_conversation_list() + + def load_conversation_history(selected_conversation, state_value): + if selected_conversation: + conversation_id = selected_conversation.split('(ID: ')[1][:-1] + chat_data, total_pages_val, _ = load_chat_history(conversation_id, 1, 50) + # Convert chat data to list of tuples (user_message, assistant_response) + history = [] + for role, content in chat_data: + if role == 'user': + history.append((content, '')) + else: + if history: + history[-1] = (history[-1][0], content) + else: + history.append(('', content)) + # Retrieve notes + notes_content = get_notes(conversation_id) + updated_state = update_state(state_value, conversation_id=conversation_id, page=1, + conversation_messages=[]) + return history, updated_state, "\n".join(notes_content) + return [], state_value, "" + + load_conversation.change( + load_conversation_history, + inputs=[load_conversation, state], + outputs=[chatbot, state, notes] + ) + + # Modify save_conversation_function to use gr.update() + def save_conversation_function(conversation_title_text, keywords_text, state_value): + conversation_messages = state_value.get("conversation_messages", []) + if not conversation_messages: + return gr.update( + value="

No conversation to save.

" + ), state_value, gr.update() + # Start a new conversation in the database + new_conversation_id = start_new_conversation( + conversation_title_text if conversation_title_text else "Untitled Conversation" + ) + # Save the messages + for role, content in conversation_messages: + save_message(new_conversation_id, role, content) + # Save keywords if provided + if keywords_text: + add_keywords_to_conversation(new_conversation_id, [kw.strip() for kw in keywords_text.split(',')]) + # Update state + updated_state = update_state(state_value, conversation_id=new_conversation_id) + # Update the conversation list + conversation_choices = update_conversation_list() + return gr.update( + value="

Conversation saved successfully.

" + ), updated_state, gr.update(choices=conversation_choices) + + save_conversation_button.click( + save_conversation_function, + inputs=[conversation_title, keywords, state], + outputs=[status_message, state, load_conversation] + ) + + def start_new_conversation_wrapper(title, state_value): + # Reset the state with no conversation_id + updated_state = update_state(state_value, conversation_id=None, page=1, + conversation_messages=[]) + # Clear the chat history + return [], updated_state + + new_conversation.click( + start_new_conversation_wrapper, + inputs=[conversation_title, state], + outputs=[chatbot, state] + ) def update_file_list(page): files, total_pages, current_page = get_paginated_files(page) @@ -100,13 +348,45 @@ def update_context_source(choice): next_page_btn.click(next_page_fn, inputs=[file_page], outputs=[existing_file, page_info, file_page]) prev_page_btn.click(prev_page_fn, inputs=[file_page], outputs=[existing_file, page_info, file_page]) - # Initialize the file list - context_source.change(lambda: update_file_list(1), outputs=[existing_file, page_info, file_page]) + # Initialize the file list when context source is changed to "Existing File" + context_source.change(lambda choice: update_file_list(1) if choice == "Existing File" else (gr.update(), gr.update(), 1), + inputs=[context_source], outputs=[existing_file, page_info, file_page]) - loading_indicator = gr.HTML(visible=False) + def perform_search(query): + try: + results = search_database(query) + return gr.update(choices=results) + except Exception as e: + gr.Error(f"Error performing search: {str(e)}") + return gr.update(choices=[]) + + search_button.click( + perform_search, + inputs=[search_query], + outputs=[search_results] + ) + + def rephrase_question(history, latest_question, api_choice): + logging.info("RAG QnA: Rephrasing question") + conversation_history = "\n".join([f"User: {h[0]}\nAssistant: {h[1]}" for h in history[:-1]]) + prompt = f"""You are a helpful assistant. Given the conversation history and the latest question, resolve any ambiguous references in the latest question. + +Conversation History: +{conversation_history} + +Latest Question: +{latest_question} + +Rewritten Question:""" + + # Use the selected API to generate the rephrased question + rephrased_question = generate_answer(api_choice, prompt, "") + logging.info(f"Rephrased question: {rephrased_question}") + return rephrased_question.strip() def rag_qa_chat_wrapper(message, history, context_source, existing_file, search_results, file_upload, - convert_to_text, keywords, api_choice, use_query_rewriting): + convert_to_text, keywords, api_choice, use_query_rewriting, state_value, + keywords_input, top_k_input, use_re_ranking): try: logging.info(f"Starting rag_qa_chat_wrapper with message: {message}") logging.info(f"Context source: {context_source}") @@ -114,7 +394,18 @@ def rag_qa_chat_wrapper(message, history, context_source, existing_file, search_ logging.info(f"Query rewriting: {'enabled' if use_query_rewriting else 'disabled'}") # Show loading indicator - yield history, "", gr.update(visible=True) + yield history, "", gr.update(visible=True), state_value + + conversation_id = state_value.get("conversation_id") + conversation_messages = state_value.get("conversation_messages", []) + + # Save the user's message + if conversation_id: + save_message(conversation_id, "user", message) + else: + # Append to in-memory messages + conversation_messages.append(("user", message)) + state_value["conversation_messages"] = conversation_messages # Ensure api_choice is a string api_choice = api_choice.value if isinstance(api_choice, gr.components.Dropdown) else api_choice @@ -131,7 +422,8 @@ def rag_qa_chat_wrapper(message, history, context_source, existing_file, search_ if context_source == "All Files in the Database": # Use the enhanced_rag_pipeline to search the entire database - context = enhanced_rag_pipeline(rephrased_question, api_choice) + context = enhanced_rag_pipeline(rephrased_question, api_choice, keywords_input, top_k_input, + use_re_ranking) logging.info(f"Using enhanced_rag_pipeline for database search") elif context_source == "Search Database": context = f"media_id:{search_results.split('(ID: ')[1][:-1]}" @@ -188,74 +480,572 @@ def rag_qa_chat_wrapper(message, history, context_source, existing_file, search_ logging.info("Calling rag_qa_chat function") new_history, response = rag_qa_chat(rephrased_question, history, context, api_choice) # Log first 100 chars of response - logging.info( - f"Response received from rag_qa_chat: {response[:100]}...") + logging.info(f"Response received from rag_qa_chat: {response[:100]}...") + + # Save assistant's response + if conversation_id: + save_message(conversation_id, "assistant", response) + else: + conversation_messages.append(("assistant", response)) + state_value["conversation_messages"] = conversation_messages + + # Update the state + state_value["conversation_messages"] = conversation_messages # Safely update history if new_history: - new_history[-1] = (message, new_history[-1][1]) + new_history[-1] = (message, response) else: new_history = [(message, response)] gr.Info("Response generated successfully") logging.info("rag_qa_chat_wrapper completed successfully") - yield new_history, "", gr.update(visible=False) + yield new_history, "", gr.update(visible=False), state_value # Include state_value in outputs except ValueError as e: logging.error(f"Input error in rag_qa_chat_wrapper: {str(e)}") gr.Error(f"Input error: {str(e)}") - yield history, "", gr.update(visible=False) + yield history, "", gr.update(visible=False), state_value except DatabaseError as e: logging.error(f"Database error in rag_qa_chat_wrapper: {str(e)}") gr.Error(f"Database error: {str(e)}") - yield history, "", gr.update(visible=False) + yield history, "", gr.update(visible=False), state_value except Exception as e: logging.error(f"Unexpected error in rag_qa_chat_wrapper: {e}", exc_info=True) gr.Error("An unexpected error occurred. Please try again later.") - yield history, "", gr.update(visible=False) + yield history, "", gr.update(visible=False), state_value - def rephrase_question(history, latest_question, api_choice): - # Thank you https://www.reddit.com/r/LocalLLaMA/comments/1fi1kex/multi_turn_conversation_and_rag/ - logging.info("RAG QnA: Rephrasing question") - conversation_history = "\n".join([f"User: {h[0]}\nAssistant: {h[1]}" for h in history[:-1]]) - prompt = f"""You are a helpful assistant. Given the conversation history and the latest question, resolve any ambiguous references in the latest question. + def clear_chat_history(): + return [], "" + + submit.click( + rag_qa_chat_wrapper, + inputs=[ + msg, + chatbot, + context_source, + existing_file, + search_results, + file_upload, + convert_to_text, + keywords, + api_choice, + use_query_rewriting, + state, + keywords_input, + top_k_input + ], + outputs=[chatbot, msg, loading_indicator, state], + ) - Conversation History: - {conversation_history} + clear_chat.click( + clear_chat_history, + outputs=[chatbot, msg] + ) - Latest Question: - {latest_question} + return ( + context_source, + existing_file, + search_query, + search_button, + search_results, + file_upload, + convert_to_text, + keywords, + api_choice, + use_query_rewriting, + chatbot, + msg, + submit, + clear_chat, + ) - Rewritten Question:""" - # Use the selected API to generate the rephrased question - rephrased_question = generate_answer(api_choice, prompt, "") - logging.info(f"Rephrased question: {rephrased_question}") - return rephrased_question.strip() - def perform_search(query): +def create_rag_qa_notes_management_tab(): + # New Management Tab + with gr.TabItem("Notes Management", visible=True): + gr.Markdown("# RAG QA Notes Management") + + management_state = gr.State({ + "selected_conversation_id": None, + "selected_note_id": None, + }) + + with gr.Row(): + with gr.Column(scale=1): + # Search Notes + search_notes_input = gr.Textbox(label="Search Notes by Keywords") + search_notes_button = gr.Button("Search Notes") + notes_list = gr.Dropdown(label="Notes", choices=[]) + + # Manage Notes + load_note_button = gr.Button("Load Note") + delete_note_button = gr.Button("Delete Note") + note_title_input = gr.Textbox(label="Note Title") + note_content_input = gr.TextArea(label="Note Content", lines=20) + note_keywords_input = gr.Textbox(label="Note Keywords (comma-separated)") + save_note_button = gr.Button("Save Note") + create_new_note_button = gr.Button("Create New Note") + status_message = gr.HTML() + + # Function Definitions + def search_notes(keywords): + if keywords: + keywords_list = [kw.strip() for kw in keywords.split(',')] + notes_data, total_pages, total_count = get_notes_by_keywords(keywords_list) + choices = [f"{title} (ID: {note_id})" for note_id, title, content, timestamp in notes_data] + return gr.update(choices=choices) + else: + return gr.update(choices=[]) + + search_notes_button.click( + search_notes, + inputs=[search_notes_input], + outputs=[notes_list] + ) + + def load_selected_note(selected_note, state_value): + if selected_note: + note_id = int(selected_note.split('(ID: ')[1][:-1]) + note_data = get_note_by_id(note_id) + if note_data: + note_id, title, content = note_data[0] + state_value["selected_note_id"] = note_id + # Get keywords for the note + keywords = get_keywords_for_note(note_id) + keywords_str = ', '.join(keywords) + return ( + gr.update(value=title), + gr.update(value=content), + gr.update(value=keywords_str), + state_value + ) + return gr.update(value=''), gr.update(value=''), gr.update(value=''), state_value + + load_note_button.click( + load_selected_note, + inputs=[notes_list, management_state], + outputs=[note_title_input, note_content_input, note_keywords_input, management_state] + ) + + def save_note_function(title, content, keywords_str, state_value): + note_id = state_value["selected_note_id"] + if note_id: + update_note(note_id, title, content) + if keywords_str: + # Clear existing keywords and add new ones + clear_keywords_from_note(note_id) + keywords_list = [kw.strip() for kw in keywords_str.split(',')] + add_keywords_to_note(note_id, keywords_list) + return gr.Info("Note updated successfully.") + else: + # Create new note + conversation_id = state_value.get("selected_conversation_id") + if conversation_id: + note_id = save_notes(conversation_id, title, content) + state_value["selected_note_id"] = note_id + if keywords_str: + keywords_list = [kw.strip() for kw in keywords_str.split(',')] + add_keywords_to_note(note_id, keywords_list) + return gr.Info("New note created successfully.") + else: + return gr.Error("No conversation selected. Cannot create a new note.") + + save_note_button.click( + save_note_function, + inputs=[note_title_input, note_content_input, note_keywords_input, management_state], + outputs=[] + ) + + def delete_selected_note(state_value): + note_id = state_value["selected_note_id"] + if note_id: + delete_note(note_id) + # Reset state + state_value["selected_note_id"] = None + # Update notes list + updated_notes = search_notes("") + return updated_notes, gr.update(value="Note deleted successfully."), state_value + else: + return gr.update(), gr.update(value="No note selected."), state_value + + delete_note_button.click( + delete_selected_note, + inputs=[management_state], + outputs=[notes_list, status_message, management_state] + ) + + def create_new_note_function(state_value): + state_value["selected_note_id"] = None + return gr.update(value=''), gr.update(value=''), gr.update(value=''), state_value + + create_new_note_button.click( + create_new_note_function, + inputs=[management_state], + outputs=[note_title_input, note_content_input, note_keywords_input, management_state] + ) + + +def create_rag_qa_chat_management_tab(): + # New Management Tab + with gr.TabItem("Chat Management", visible=True): + gr.Markdown("# RAG QA Chat Conversation Management") + + management_state = gr.State({ + "selected_conversation_id": None, + "selected_note_id": None, + }) + + # State to store the mapping between titles and IDs + conversation_mapping = gr.State({}) + + with gr.Row(): + with gr.Column(scale=1): + # Search Conversations + search_conversations_input = gr.Textbox(label="Search Conversations by Keywords") + search_conversations_button = gr.Button("Search Conversations") + conversations_list = gr.Dropdown(label="Conversations", choices=[]) + new_conversation_button = gr.Button("New Conversation") + + # Manage Conversations + load_conversation_button = gr.Button("Load Conversation") + delete_conversation_button = gr.Button("Delete Conversation") + conversation_title_input = gr.Textbox(label="Conversation Title") + conversation_content_input = gr.TextArea(label="Conversation Content", lines=20) + save_conversation_button = gr.Button("Save Conversation") + status_message = gr.HTML() + + # Function Definitions + def search_conversations(keywords): + if keywords: + keywords_list = [kw.strip() for kw in keywords.split(',')] + conversations, total_pages, total_count = search_conversations_by_keywords(keywords_list) + else: + conversations, total_pages, total_count = get_all_conversations() + + # Build choices as list of titles (ensure uniqueness) + choices = [] + mapping = {} + for conversation_id, title in conversations: + display_title = f"{title} (ID: {conversation_id[:8]})" + choices.append(display_title) + mapping[display_title] = conversation_id + + return gr.update(choices=choices), mapping + + search_conversations_button.click( + search_conversations, + inputs=[search_conversations_input], + outputs=[conversations_list, conversation_mapping] + ) + + def load_selected_conversation(selected_title, state_value, mapping): + conversation_id = mapping.get(selected_title) + if conversation_id: + # Load conversation title + conversation_title = get_conversation_title(conversation_id) + # Load conversation messages + messages, total_pages, total_count = load_chat_history(conversation_id) + # Concatenate messages into a single string + conversation_content = "" + for role, content in messages: + conversation_content += f"{role}: {content}\n\n" + # Update state + new_state = state_value.copy() + new_state["selected_conversation_id"] = conversation_id + return ( + gr.update(value=conversation_title), + gr.update(value=conversation_content.strip()), + new_state + ) + return gr.update(value=''), gr.update(value=''), state_value + + load_conversation_button.click( + load_selected_conversation, + inputs=[conversations_list, management_state, conversation_mapping], + outputs=[conversation_title_input, conversation_content_input, management_state] + ) + + def save_conversation(title, content, state_value): + conversation_id = state_value["selected_conversation_id"] + if conversation_id: + # Update conversation title + update_conversation_title(conversation_id, title) + + # Clear existing messages + delete_messages_in_conversation(conversation_id) + + # Parse the content back into messages + messages = [] + for line in content.strip().split('\n\n'): + if ': ' in line: + role, message_content = line.split(': ', 1) + messages.append((role.strip(), message_content.strip())) + else: + # If the format is incorrect, skip or handle accordingly + continue + + # Save new messages + for role, message_content in messages: + save_message(conversation_id, role, message_content) + + return ( + gr.HTML("

Conversation updated successfully.

"), + gr.update(value=title), + gr.update(value=content), + state_value + ) + else: + return ( + gr.HTML("

No conversation selected to save.

"), + gr.update(value=title), + gr.update(value=content), + state_value + ) + + save_conversation_button.click( + save_conversation, + inputs=[conversation_title_input, conversation_content_input, management_state], + outputs=[status_message, conversation_title_input, conversation_content_input, management_state] + ) + + def delete_selected_conversation(state_value, mapping): + conversation_id = state_value["selected_conversation_id"] + if conversation_id: + delete_conversation(conversation_id) + # Reset state + new_state = state_value.copy() + new_state["selected_conversation_id"] = None + # Update conversations list and mapping + conversations, _, _ = get_all_conversations() + choices = [] + new_mapping = {} + for conv_id, title in conversations: + display_title = f"{title} (ID: {conv_id[:8]})" + choices.append(display_title) + new_mapping[display_title] = conv_id + return ( + gr.update(choices=choices, value=None), + gr.HTML("

Conversation deleted successfully.

"), + new_state, + gr.update(value=''), + gr.update(value=''), + new_mapping + ) + else: + return ( + gr.update(), + gr.HTML("

No conversation selected.

"), + state_value, + gr.update(), + gr.update(), + mapping + ) + + delete_conversation_button.click( + delete_selected_conversation, + inputs=[management_state, conversation_mapping], + outputs=[ + conversations_list, + status_message, + management_state, + conversation_title_input, + conversation_content_input, + conversation_mapping + ] + ) + + def create_new_conversation(state_value, mapping): + conversation_id = start_new_conversation() + # Update state + new_state = state_value.copy() + new_state["selected_conversation_id"] = conversation_id + # Update conversations list and mapping + conversations, _, _ = get_all_conversations() + choices = [] + new_mapping = {} + for conv_id, title in conversations: + display_title = f"{title} (ID: {conv_id[:8]})" + choices.append(display_title) + new_mapping[display_title] = conv_id + # Set the new conversation as selected + selected_title = f"Untitled Conversation (ID: {conversation_id[:8]})" + return ( + gr.update(choices=choices, value=selected_title), + gr.update(value='Untitled Conversation'), + gr.update(value=''), + gr.HTML("

New conversation created.

"), + new_state, + new_mapping + ) + + new_conversation_button.click( + create_new_conversation, + inputs=[management_state, conversation_mapping], + outputs=[ + conversations_list, + conversation_title_input, + conversation_content_input, + status_message, + management_state, + conversation_mapping + ] + ) + + def delete_messages_in_conversation(conversation_id): + """Helper function to delete all messages in a conversation.""" try: - results = search_database(query) - return gr.update(choices=results) + execute_query("DELETE FROM rag_qa_chats WHERE conversation_id = ?", (conversation_id,)) + logging.info(f"Messages in conversation '{conversation_id}' deleted successfully.") except Exception as e: - gr.Error(f"Error performing search: {str(e)}") - return gr.update(choices=[]) + logging.error(f"Error deleting messages in conversation '{conversation_id}': {e}") + raise - def clear_chat_history(): - return [], "" + def get_conversation_title(conversation_id): + """Helper function to get the conversation title.""" + query = "SELECT title FROM conversation_metadata WHERE conversation_id = ?" + result = execute_query(query, (conversation_id,)) + if result: + return result[0][0] + else: + return "Untitled Conversation" - search_button.click(perform_search, inputs=[search_query], outputs=[search_results]) - submit.click( - rag_qa_chat_wrapper, - inputs=[msg, chatbot, context_source, existing_file, search_results, file_upload, - convert_to_text, keywords, api_choice, use_query_rewriting], - outputs=[chatbot, msg, loading_indicator] + +def create_export_data_tab(): + with gr.TabItem("Export Data"): + gr.Markdown("# Export Data") + + export_option = gr.Radio( + ["Export All", "Export Selected"], + label="Export Option", + value="Export All" + ) + + conversations_checklist = gr.CheckboxGroup( + choices=[], + label="Select Conversations", + visible=False + ) + + notes_checklist = gr.CheckboxGroup( + choices=[], + label="Select Notes", + visible=False + ) + + export_button = gr.Button("Export") + download_link = gr.File(label="Download Exported Data", visible=False) + status_message = gr.HTML() + + # Function to update visibility and populate checklists + def update_visibility(export_option_value): + if export_option_value == "Export Selected": + # Fetch conversations and notes to populate the checklists + conversations = fetch_all_conversations() + notes = fetch_all_notes() + + conversation_choices = [f"{title} (ID: {conversation_id})" for conversation_id, title, _ in conversations] + note_choices = [f"{title} (ID: {note_id})" for note_id, title, _ in notes] + + return ( + gr.update(visible=True, choices=conversation_choices), + gr.update(visible=True, choices=note_choices) + ) + else: + return ( + gr.update(visible=False), + gr.update(visible=False) + ) + + export_option.change( + update_visibility, + inputs=[export_option], + outputs=[conversations_checklist, notes_checklist] + ) + + import zipfile + import io + def update_visibility(export_option_value): + if export_option_value == "Export Selected": + # Fetch conversations and notes to populate the checklists + conversations = fetch_all_conversations() + notes = fetch_all_notes() + + conversation_choices = [f"{title} (ID: {conversation_id})" for conversation_id, title, _ in + conversations] + note_choices = [f"{title} (ID: {note_id})" for note_id, title, _ in notes] + + return ( + gr.update(visible=True, choices=conversation_choices), + gr.update(visible=True, choices=note_choices) + ) + else: + return ( + gr.update(visible=False), + gr.update(visible=False) + ) + + export_option.change( + update_visibility, + inputs=[export_option], + outputs=[conversations_checklist, notes_checklist] + ) + + def export_data_function(export_option, selected_conversations, selected_notes): + try: + zip_buffer = io.BytesIO() + + with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file: + if export_option == "Export All": + # Fetch all conversations and notes + conversations = fetch_all_conversations() + notes = fetch_all_notes() + else: + # Fetch selected conversations and notes + conversation_ids = [int(item.split(' (ID: ')[1][:-1]) for item in selected_conversations] + note_ids = [int(item.split(' (ID: ')[1][:-1]) for item in selected_notes] + conversations = fetch_conversations_by_ids(conversation_ids) + notes = fetch_notes_by_ids(note_ids) + + # Export conversations + for conversation in conversations: + conversation_id, title, _ = conversation + filename = f"conversation_{conversation_id}_{title.replace(' ', '_')}.md" + zip_file.writestr(filename, conversation) + + # Export notes + for note in notes: + note_id, title, _ = note + filename = f"note_{note_id}_{title.replace(' ', '_')}.md" + zip_file.writestr(filename, note) + + zip_buffer.seek(0) + return zip_buffer, gr.update(visible=True), gr.update( + value="

Export successful!

") + except Exception as e: + logging.error(f"Error exporting data: {str(e)}") + return None, gr.update(visible=False), gr.update(value=f"

Error: {str(e)}

") + + export_button.click( + export_data_function, + inputs=[export_option, conversations_checklist, notes_checklist], + outputs=[download_link, download_link, status_message] ) - clear_chat.click(clear_chat_history, outputs=[chatbot, msg]) - return (context_source, existing_file, search_query, search_button, search_results, file_upload, - convert_to_text, keywords, api_choice, use_query_rewriting, chatbot, msg, submit, clear_chat) + + +def update_conversation_title(conversation_id, new_title): + """Update the title of a conversation.""" + try: + query = "UPDATE conversation_metadata SET title = ? WHERE conversation_id = ?" + execute_query(query, (new_title, conversation_id)) + logging.info(f"Conversation '{conversation_id}' title updated to '{new_title}'") + except Exception as e: + logging.error(f"Error updating conversation title: {e}") + raise + def convert_file_to_text(file_path): """Convert various file types to plain text.""" @@ -275,6 +1065,7 @@ def convert_file_to_text(file_path): else: raise ValueError(f"Unsupported file type: {file_extension}") + def read_structured_file(file_path): """Read and convert JSON or CSV files to text.""" file_extension = os.path.splitext(file_path)[1].lower() @@ -295,4 +1086,3 @@ def read_structured_file(file_path): # # End of RAG_QA_Chat_tab.py ######################################################################################################################## -# \ No newline at end of file diff --git a/App_Function_Libraries/Gradio_UI/Re_summarize_tab.py b/App_Function_Libraries/Gradio_UI/Re_summarize_tab.py index 5a231f4b2..2a51b0b85 100644 --- a/App_Function_Libraries/Gradio_UI/Re_summarize_tab.py +++ b/App_Function_Libraries/Gradio_UI/Re_summarize_tab.py @@ -23,7 +23,7 @@ # Functions: def create_resummary_tab(): - with gr.TabItem("Re-Summarize"): + with gr.TabItem("Re-Summarize", visible=True): gr.Markdown("# Re-Summarize Existing Content") with gr.Row(): with gr.Column(): diff --git a/App_Function_Libraries/Gradio_UI/Search_Tab.py b/App_Function_Libraries/Gradio_UI/Search_Tab.py index 39885b27b..2ad075b81 100644 --- a/App_Function_Libraries/Gradio_UI/Search_Tab.py +++ b/App_Function_Libraries/Gradio_UI/Search_Tab.py @@ -80,7 +80,7 @@ def format_as_html(content, title): """ def create_search_tab(): - with gr.TabItem("Search / Detailed View"): + with gr.TabItem("Search / Detailed View", visible=True): gr.Markdown("# Search across all ingested items in the Database") with gr.Row(): with gr.Column(scale=1): @@ -150,7 +150,7 @@ def display_search_results(query): def create_search_summaries_tab(): - with gr.TabItem("Search/View Title+Summary "): + with gr.TabItem("Search/View Title+Summary", visible=True): gr.Markdown("# Search across all ingested items in the Database and review their summaries") gr.Markdown("Search by Title / URL / Keyword / or Content via SQLite Full-Text-Search") with gr.Row(): @@ -207,7 +207,7 @@ def go_to_previous_search_page(query, search_type, current_page, entries_per_pag def create_prompt_search_tab(): - with gr.TabItem("Search Prompts"): + with gr.TabItem("Search Prompts", visible=True): gr.Markdown("# Search and View Prompt Details") gr.Markdown("Currently has all of the https://github.com/danielmiessler/fabric prompts already available") with gr.Row(): diff --git a/App_Function_Libraries/Gradio_UI/Transcript_comparison.py b/App_Function_Libraries/Gradio_UI/Transcript_comparison.py index 46333fefc..3e8625aef 100644 --- a/App_Function_Libraries/Gradio_UI/Transcript_comparison.py +++ b/App_Function_Libraries/Gradio_UI/Transcript_comparison.py @@ -46,7 +46,7 @@ def compare_transcripts(media_id, transcript1_id, transcript2_id): def create_compare_transcripts_tab(): - with gr.TabItem("Compare Transcripts"): + with gr.TabItem("Compare Transcripts", visible=True): gr.Markdown("# Compare Transcripts") with gr.Row(): diff --git a/App_Function_Libraries/Gradio_UI/Trash.py b/App_Function_Libraries/Gradio_UI/Trash.py index 625f3a82e..aa9581c2e 100644 --- a/App_Function_Libraries/Gradio_UI/Trash.py +++ b/App_Function_Libraries/Gradio_UI/Trash.py @@ -69,7 +69,7 @@ def mark_item_as_trash(media_id: int) -> str: def create_search_and_mark_trash_tab(): - with gr.TabItem("Search and Mark as Trash"): + with gr.TabItem("Search and Mark as Trash", visible=True): gr.Markdown("# Search for Items and Mark as Trash") search_input = gr.Textbox(label="Search Query") @@ -105,14 +105,14 @@ def mark_selected_as_trash(selected_item): def create_view_trash_tab(): - with gr.TabItem("View Trash"): + with gr.TabItem("View Trash", visible=True): view_button = gr.Button("View Trash") trash_list = gr.Textbox(label="Trashed Items") view_button.click(list_trash, inputs=[], outputs=trash_list) def create_delete_trash_tab(): - with gr.TabItem("Delete DB Item"): + with gr.TabItem("Delete DB Item", visible=True): gr.Markdown("# Delete Items from Databases") media_id_input = gr.Number(label="Media ID") @@ -128,7 +128,7 @@ def create_delete_trash_tab(): def create_empty_trash_tab(): - with gr.TabItem("Empty Trash"): + with gr.TabItem("Empty Trash", visible=True): days_input = gr.Slider(minimum=15, maximum=90, step=5, label="Delete items older than (days)") empty_button = gr.Button("Empty Trash") empty_output = gr.Textbox(label="Result") diff --git a/App_Function_Libraries/Gradio_UI/Utilities.py b/App_Function_Libraries/Gradio_UI/Utilities.py index 4a4ba3bd4..02020718f 100644 --- a/App_Function_Libraries/Gradio_UI/Utilities.py +++ b/App_Function_Libraries/Gradio_UI/Utilities.py @@ -10,7 +10,7 @@ def create_utilities_yt_video_tab(): - with gr.Tab("YouTube Video Downloader"): + with gr.TabItem("YouTube Video Downloader", id='youtube_dl', visible=True): with gr.Row(): with gr.Column(): gr.Markdown( @@ -28,7 +28,7 @@ def create_utilities_yt_video_tab(): ) def create_utilities_yt_audio_tab(): - with gr.Tab("YouTube Audio Downloader"): + with gr.TabItem("YouTube Audio Downloader", id="youtube audio downloader", visible=True): with gr.Row(): with gr.Column(): gr.Markdown( @@ -48,7 +48,7 @@ def create_utilities_yt_audio_tab(): ) def create_utilities_yt_timestamp_tab(): - with gr.Tab("YouTube Timestamp URL Generator"): + with gr.TabItem("YouTube Timestamp URL Generator", id="timestamp-gen", visible=True): gr.Markdown("## Generate YouTube URL with Timestamp") with gr.Row(): with gr.Column(): diff --git a/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py b/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py index ef5323f7a..548cf821b 100644 --- a/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py +++ b/App_Function_Libraries/Gradio_UI/Video_transcription_tab.py @@ -32,7 +32,7 @@ # Functions: def create_video_transcription_tab(): - with ((gr.TabItem("Video Transcription + Summarization"))): + with gr.TabItem("Video Transcription + Summarization", visible=True): gr.Markdown("# Transcribe & Summarize Videos from URLs") with gr.Row(): gr.Markdown("""Follow this project at [tldw - GitHub](https://github.com/rmusser01/tldw)""") diff --git a/App_Function_Libraries/Gradio_UI/View_DB_Items_tab.py b/App_Function_Libraries/Gradio_UI/View_DB_Items_tab.py index 870b1d598..5c577ea60 100644 --- a/App_Function_Libraries/Gradio_UI/View_DB_Items_tab.py +++ b/App_Function_Libraries/Gradio_UI/View_DB_Items_tab.py @@ -18,7 +18,7 @@ # Functions def create_prompt_view_tab(): - with gr.TabItem("View Prompt Database"): + with gr.TabItem("View Prompt Database", visible=True): gr.Markdown("# View Prompt Database Entries") with gr.Row(): with gr.Column(): @@ -150,7 +150,7 @@ def extract_prompt_and_summary(content: str): def create_view_all_with_versions_tab(): - with gr.TabItem("View All Items"): + with gr.TabItem("View All Items", visible=True): gr.Markdown("# View All Database Entries with Version Selection") with gr.Row(): with gr.Column(scale=1): @@ -281,7 +281,7 @@ def update_version_content(selected_item, item_mapping, selected_version): def create_viewing_tab(): - with gr.TabItem("View Database Entries"): + with gr.TabItem("View Database Entries", visible=True): gr.Markdown("# View Database Entries") with gr.Row(): with gr.Column(): diff --git a/App_Function_Libraries/Gradio_UI/View_tab.py b/App_Function_Libraries/Gradio_UI/View_tab.py index 816bf6fd4..d701a9952 100644 --- a/App_Function_Libraries/Gradio_UI/View_tab.py +++ b/App_Function_Libraries/Gradio_UI/View_tab.py @@ -22,7 +22,7 @@ # FIXME - Doesn't work. also need ot merge this tab wtih Edit Existing Items tab.... def create_manage_items_tab(): - with gr.TabItem("Edit/Manage DB Items"): + with gr.TabItem("Edit/Manage DB Items", visible=True): search_input = gr.Textbox(label="Search for Media (title or ID)") search_button = gr.Button("Search") media_selector = gr.Dropdown(label="Select Media", choices=[], interactive=True) diff --git a/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py b/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py index 6d6e5d044..5a7914336 100644 --- a/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py +++ b/App_Function_Libraries/Gradio_UI/Website_scraping_tab.py @@ -249,8 +249,8 @@ async def scrape_with_retry(url: str, max_retries: int = 3, retry_delay: float = def create_website_scraping_tab(): - with gr.TabItem("Website Scraping"): - gr.Markdown("# Scrape Websites & Summarize Articles") + with gr.TabItem("Website Scraping", visible=True): + gr.Markdown("# Scrape Websites & Summarize Articles") with gr.Row(): with gr.Column(): scrape_method = gr.Radio( diff --git a/App_Function_Libraries/Gradio_UI/Writing_tab.py b/App_Function_Libraries/Gradio_UI/Writing_tab.py index d52789f84..e62206821 100644 --- a/App_Function_Libraries/Gradio_UI/Writing_tab.py +++ b/App_Function_Libraries/Gradio_UI/Writing_tab.py @@ -41,7 +41,7 @@ def grammar_style_check(input_text, custom_prompt, api_name, api_key, system_pro def create_grammar_style_check_tab(): - with gr.TabItem("Grammar and Style Check"): + with gr.TabItem("Grammar and Style Check", visible=True): with gr.Row(): with gr.Column(): gr.Markdown("# Grammar and Style Check") @@ -98,7 +98,7 @@ def create_grammar_style_check_tab(): def create_tone_adjustment_tab(): - with gr.TabItem("Tone Analyzer & Editor"): + with gr.TabItem("Tone Analyzer & Editor", visible=True): with gr.Row(): with gr.Column(): input_text = gr.Textbox(label="Input Text", lines=10) @@ -174,7 +174,7 @@ def generate_feedback_history_html(history): # FIXME def create_document_feedback_tab(): - with gr.TabItem("Writing Feedback"): + with gr.TabItem("Writing Feedback", visible=True): with gr.Row(): with gr.Column(scale=2): input_text = gr.Textbox(label="Your Writing", lines=10) @@ -364,13 +364,13 @@ def compare_feedback(text, selected_personas, api_name, api_key): def create_creative_writing_tab(): - with gr.TabItem("Creative Writing Assistant"): + with gr.TabItem("Creative Writing Assistant", visible=True): gr.Markdown("# Utility to be added...") def create_mikupad_tab(): - with gr.TabItem("Mikupad"): + with gr.TabItem("Mikupad", visible=True): gr.Markdown("I Wish. Gradio won't embed it successfully...") # diff --git a/App_Function_Libraries/Local_LLM/Local_LLM_Inference_Engine_Lib.py b/App_Function_Libraries/Local_LLM/Local_LLM_Inference_Engine_Lib.py index c5533625f..0546903b7 100644 --- a/App_Function_Libraries/Local_LLM/Local_LLM_Inference_Engine_Lib.py +++ b/App_Function_Libraries/Local_LLM/Local_LLM_Inference_Engine_Lib.py @@ -39,25 +39,25 @@ # LLM models information llm_models = { - "1": { + "Mistral-7B-Instruct-v0.2-Q8.llamafile": { "name": "Mistral-7B-Instruct-v0.2-Q8.llamafile", "url": "https://huggingface.co/Mozilla/Mistral-7B-Instruct-v0.2-llamafile/resolve/main/mistral-7b-instruct-v0.2.Q8_0.llamafile?download=true", "filename": "mistral-7b-instruct-v0.2.Q8_0.llamafile", "hash": "1ee6114517d2f770425c880e5abc443da36b193c82abec8e2885dd7ce3b9bfa6" }, - "2": { + "Samantha-Mistral-Instruct-7B-Bulleted-Notes-Q8.gguf": { "name": "Samantha-Mistral-Instruct-7B-Bulleted-Notes-Q8.gguf", "url": "https://huggingface.co/cognitivetech/samantha-mistral-instruct-7b-bulleted-notes-GGUF/resolve/main/samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf?download=true", "filename": "samantha-mistral-instruct-7b-bulleted-notes.Q8_0.gguf", "hash": "6334c1ab56c565afd86535271fab52b03e67a5e31376946bce7bf5c144e847e4" }, - "3": { + "Phi-3-mini-128k-instruct-Q8_0.gguf": { "name": "Phi-3-mini-128k-instruct-Q8_0.gguf", "url": "https://huggingface.co/gaianet/Phi-3-mini-128k-instruct-GGUF/resolve/main/Phi-3-mini-128k-instruct-Q8_0.gguf?download=true", "filename": "Phi-3-mini-128k-instruct-Q8_0.gguf", "hash": "6817b66d1c3c59ab06822e9732f0e594eea44e64cae2110906eac9d17f75d193" }, - "4": { + "Meta-Llama-3-8B-Instruct.Q8_0.llamafile": { "name": "Meta-Llama-3-8B-Instruct.Q8_0.llamafile", "url": "https://huggingface.co/Mozilla/Meta-Llama-3-8B-Instruct-llamafile/resolve/main/Meta-Llama-3-8B-Instruct.Q8_0.llamafile?download=true", "filename": "Meta-Llama-3-8B-Instruct.Q8_0.llamafile", @@ -286,9 +286,6 @@ def start_llamafile( if numa_checked: command.append('--numa') - if server_timeout_value is not None: - command.extend(['--to', str(server_timeout_value)]) - if host_checked and host_value: command.extend(['--host', host_value]) diff --git a/App_Function_Libraries/RAG/RAG_Library_2.py b/App_Function_Libraries/RAG/RAG_Library_2.py index 95751939a..10c8c5dfc 100644 --- a/App_Function_Libraries/RAG/RAG_Library_2.py +++ b/App_Function_Libraries/RAG/RAG_Library_2.py @@ -115,9 +115,9 @@ # return {"error": "An unexpected error occurred", "details": str(e)} - # RAG Search with keyword filtering -def enhanced_rag_pipeline(query: str, api_choice: str, keywords: str = None) -> Dict[str, Any]: +# FIXME - Update each called function to support modifiable top-k results +def enhanced_rag_pipeline(query: str, api_choice: str, keywords: str = None, top_k=10, apply_re_ranking=True) -> Dict[str, Any]: log_counter("enhanced_rag_pipeline_attempt", labels={"api_choice": api_choice}) start_time = time.time() try: @@ -150,33 +150,33 @@ def enhanced_rag_pipeline(query: str, api_choice: str, keywords: str = None) -> # Combine results all_results = vector_results + fts_results - apply_re_ranking = True if apply_re_ranking: logging.debug(f"\nenhanced_rag_pipeline - Applying Re-Ranking") # FIXME - add option to use re-ranking at call time # FIXME - specify model + add param to modify at call time # FIXME - add option to set a custom top X results # You can specify a model if necessary, e.g., model_name="ms-marco-MiniLM-L-12-v2" - ranker = Ranker() + if all_results: + ranker = Ranker() - # Prepare passages for re-ranking - passages = [{"id": i, "text": result['content']} for i, result in enumerate(all_results)] - rerank_request = RerankRequest(query=query, passages=passages) + # Prepare passages for re-ranking + passages = [{"id": i, "text": result['content']} for i, result in enumerate(all_results)] + rerank_request = RerankRequest(query=query, passages=passages) - # Rerank the results - reranked_results = ranker.rerank(rerank_request) + # Rerank the results + reranked_results = ranker.rerank(rerank_request) - # Sort results based on the re-ranking score - reranked_results = sorted(reranked_results, key=lambda x: x['score'], reverse=True) + # Sort results based on the re-ranking score + reranked_results = sorted(reranked_results, key=lambda x: x['score'], reverse=True) - # Log reranked results - logging.debug(f"\n\nenhanced_rag_pipeline - Reranked results: {reranked_results}") + # Log reranked results + logging.debug(f"\n\nenhanced_rag_pipeline - Reranked results: {reranked_results}") - # Update all_results based on reranking - all_results = [all_results[result['id']] for result in reranked_results] + # Update all_results based on reranking + all_results = [all_results[result['id']] for result in reranked_results] - # Extract content from results (top 10) - context = "\n".join([result['content'] for result in all_results[:10]]) # Limit to top 10 results + # Extract content from results (top 10 by default) + context = "\n".join([result['content'] for result in all_results[:top_k]]) logging.debug(f"Context length: {len(context)}") logging.debug(f"Context: {context[:200]}") @@ -336,14 +336,14 @@ def generate_answer(api_choice: str, context: str, query: str) -> str: logging.error(f"Error in generate_answer: {str(e)}") return "An error occurred while generating the answer." -def perform_vector_search(query: str, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]: +def perform_vector_search(query: str, relevant_media_ids: List[str] = None, top_k=10) -> List[Dict[str, Any]]: log_counter("perform_vector_search_attempt") start_time = time.time() all_collections = chroma_client.list_collections() vector_results = [] try: for collection in all_collections: - collection_results = vector_search(collection.name, query, k=5) + collection_results = vector_search(collection.name, query, k=top_k) filtered_results = [ result for result in collection_results if relevant_media_ids is None or result['metadata'].get('media_id') in relevant_media_ids @@ -358,11 +358,11 @@ def perform_vector_search(query: str, relevant_media_ids: List[str] = None) -> L logging.error(f"Error in perform_vector_search: {str(e)}") raise -def perform_full_text_search(query: str, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]: +def perform_full_text_search(query: str, relevant_media_ids: List[str] = None, fts_top_k=None) -> List[Dict[str, Any]]: log_counter("perform_full_text_search_attempt") start_time = time.time() try: - fts_results = search_db(query, ["content"], "", page=1, results_per_page=5) + fts_results = search_db(query, ["content"], "", page=1, results_per_page=fts_top_k or 10) filtered_fts_results = [ { "content": result['content'], @@ -381,7 +381,7 @@ def perform_full_text_search(query: str, relevant_media_ids: List[str] = None) - raise -def fetch_relevant_media_ids(keywords: List[str]) -> List[int]: +def fetch_relevant_media_ids(keywords: List[str], top_k=10) -> List[int]: log_counter("fetch_relevant_media_ids_attempt", labels={"keyword_count": len(keywords)}) start_time = time.time() relevant_ids = set() diff --git a/App_Function_Libraries/RAG/RAG_QA_Chat.py b/App_Function_Libraries/RAG/RAG_QA_Chat.py index 24c60e523..5ec3af076 100644 --- a/App_Function_Libraries/RAG/RAG_QA_Chat.py +++ b/App_Function_Libraries/RAG/RAG_QA_Chat.py @@ -1,5 +1,5 @@ -# Podcast_tab.py -# Description: Gradio UI for ingesting podcasts into the database +# RAG_QA_Chat.py +# Description: Functions supporting the RAG QA Chat functionality # # Imports # @@ -13,56 +13,42 @@ # # Local Imports from App_Function_Libraries.DB.DB_Manager import db, search_db, DatabaseError, get_media_content -from App_Function_Libraries.RAG.RAG_Library_2 import generate_answer +from App_Function_Libraries.RAG.RAG_Library_2 import generate_answer, enhanced_rag_pipeline from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram # ######################################################################################################################## # # Functions: -def rag_qa_chat(message: str, history: List[Tuple[str, str]], context: Union[str, IO[str]], api_choice: str) -> Tuple[List[Tuple[str, str]], str]: +def rag_qa_chat(query, history, context, api_choice, keywords=None, apply_re_ranking=False): log_counter("rag_qa_chat_attempt", labels={"api_choice": api_choice}) start_time = time.time() + try: - # Prepare the context based on the selected source - if hasattr(context, 'read'): - # Handle uploaded file - context_text = context.read() - if isinstance(context_text, bytes): - context_text = context_text.decode('utf-8') - log_counter("rag_qa_chat_uploaded_file") - elif isinstance(context, str) and context.startswith("media_id:"): - # Handle existing file or search result - media_id = int(context.split(":")[1]) - context_text = get_media_content(media_id) - log_counter("rag_qa_chat_existing_media", labels={"media_id": media_id}) - else: - context_text = str(context) + if isinstance(context, str): log_counter("rag_qa_chat_string_context") - - # Prepare the full context including chat history - full_context = "\n".join([f"Human: {h[0]}\nAI: {h[1]}" for h in history]) - full_context += f"\n\nContext: {context_text}\n\nHuman: {message}\nAI:" - - # Generate response using the selected API - response = generate_answer(api_choice, full_context, message) + # Use the answer and context directly from enhanced_rag_pipeline + result = enhanced_rag_pipeline(query, api_choice, keywords, apply_re_ranking) + answer = result['answer'] + else: + log_counter("rag_qa_chat_no_context") + # If no context is provided, call generate_answer directly + answer = generate_answer(api_choice, "", query) # Update history - history.append((message, response)) + new_history = history + [(query, answer)] - chat_duration = time.time() - start_time - log_histogram("rag_qa_chat_duration", chat_duration, labels={"api_choice": api_choice}) + # Metrics + duration = time.time() - start_time + log_histogram("rag_qa_chat_duration", duration, labels={"api_choice": api_choice}) log_counter("rag_qa_chat_success", labels={"api_choice": api_choice}) - return history, "" - except DatabaseError as e: - log_counter("rag_qa_chat_database_error", labels={"error": str(e)}) - logging.error(f"Database error in rag_qa_chat: {str(e)}") - return history, f"An error occurred while accessing the database: {str(e)}" + return new_history, answer except Exception as e: - log_counter("rag_qa_chat_unexpected_error", labels={"error": str(e)}) - logging.error(f"Unexpected error in rag_qa_chat: {str(e)}") - return history, f"An unexpected error occurred: {str(e)}" + log_counter("rag_qa_chat_error", labels={"api_choice": api_choice, "error": str(e)}) + logging.error(f"Error in rag_qa_chat: {str(e)}") + return history + [(query, "An error occurred while processing your request.")], "An error occurred while processing your request." + @@ -132,6 +118,12 @@ def get_existing_files() -> List[Tuple[int, str]]: logging.error(f"Error fetching existing files: {str(e)}") raise +###################################################### +# +# Notes + + + # # End of RAG_QA_Chat.py ######################################################################################################################## diff --git a/Config_Files/config.txt b/Config_Files/config.txt index 46d24c7cf..a0b079af5 100644 --- a/Config_Files/config.txt +++ b/Config_Files/config.txt @@ -8,7 +8,7 @@ groq_model = llama3-70b-8192 openai_api_key = openai_model = gpt-4o huggingface_api_key = -huggingface_model = CohereForAI/c4ai-command-r-plus +huggingface_model = mistralai/Mistral-Nemo-Instruct-2407 openrouter_api_key = openrouter_model = mistralai/mistral-7b-instruct:free deepseek_api_key = @@ -20,20 +20,20 @@ custom_openai_api_ip = [Local-API] kobold_api_IP = http://127.0.0.1:5001/api/v1/generate -kobold_api_key = +kobold_api_key = llama_api_IP = http://127.0.0.1:8080/completion -llama_api_key = -ooba_api_key = +llama_api_key = +ooba_api_key = ooba_api_IP = http://127.0.0.1:5000/v1/chat/completions tabby_api_IP = http://127.0.0.1:5000/v1/chat/completions -tabby_api_key = +tabby_api_key = vllm_api_IP = http://127.0.0.1:8000/v1/chat/completions -vllm_model = -ollama_api_IP = http://127.0.0.1:11434/api/generate -ollama_api_key = -ollama_model = +vllm_model = +ollama_api_IP = http://127.0.0.1:11434/v1/chat/completions +ollama_api_key = +ollama_model = llama3 aphrodite_api_IP = http://127.0.0.1:8080/completion -aphrodite_api_key = +aphrodite_api_key = [Processing] processing_choice = cuda @@ -56,10 +56,13 @@ elasticsearch_port = 9200 # Additionally you can use elasticsearch as the database type, just replace `sqlite` with `elasticsearch` for `type` and provide the `elasticsearch_host` and `elasticsearch_port` of your configured ES instance. chroma_db_path = Databases/chroma_db prompts_db_path = Databases/prompts.db +rag_qa_db_path = Databases/rag_qa.db [Embeddings] embedding_provider = openai embedding_model = text-embedding-3-small +onnx_model_path = ./App_Function_Libraries/models/onnx_models/ +model_dir = ./App_Function_Libraries/models/embedding_models embedding_api_url = http://localhost:8080/v1/embeddings embedding_api_key = your_api_key_here chunk_size = 400 @@ -78,6 +81,14 @@ adaptive = false multi_level = false language = english +[Metrics] +log_file_path = +#os.getenv("tldw_LOG_FILE_PATH", "tldw_app_logs.json") +max_bytes = +#int(os.getenv("tldw_LOG_MAX_BYTES", 10 * 1024 * 1024)) # 10 MB +backup_count = 5 +#int(os.getenv("tldw_LOG_BACKUP_COUNT", 5)) + #[Comments] #OpenAI Models: @@ -98,7 +109,4 @@ language = english # open-mistral-7b # open-mixtral-8x7b # open-mixtral-8x22b -# open-codestral-mamba - - - +# open-codestral-mamba \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e9ad42d6f..10b50c958 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,9 @@ RUN apt-get update && apt-get install -y \ libsqlite3-dev \ build-essential \ git \ + libpq-dev \ python3 \ + python3-dev \ python3-pyaudio \ portaudio19-dev \ python3-pip \ diff --git a/Docs/Issues/ISSUES.md b/Docs/Issues/ISSUES.md index c2290be7b..2fee461e3 100644 --- a/Docs/Issues/ISSUES.md +++ b/Docs/Issues/ISSUES.md @@ -19,3 +19,10 @@ Update model suggestions for RAG vs Chatting/General use Whisper pipeline https://huggingface.co/spaces/aadnk/faster-whisper-webui https://huggingface.co/spaces/zhang082799/openai-whisper-large-v3-turbo + + +Create Documentation for how this can help + https://stevenberlinjohnson.com/how-to-use-notebooklm-as-a-research-tool-6ad5c3a227cc?gi=9a0b63820ff0 + +Create a blog post + tldwproject.com \ No newline at end of file diff --git a/HF/Dockerfile b/HF/Dockerfile deleted file mode 100644 index 698dde4a7..000000000 --- a/HF/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -FROM nvidia/cuda:12.5.1-cudnn-devel-ubuntu22.04 - -ARG DEBIAN_FRONTEND=noninteractive - -ENV PYTHONUNBUFFERED=1 - -RUN apt-get update && apt-get install -y \ - build-essential \ - python3.9 \ - python3-pip \ - git \ - ffmpeg \ - libcudnn8 \ - libcudnn8-dev \ - sudo - -# Set up a new user named "user" with user ID 1099 -RUN useradd -m -u 1099 user - -WORKDIR /app - -COPY ./requirements.txt /app/requirements.txt - -# Set up CUDA libraries -RUN export LD_LIBRARY_PATH=`python3 -c 'import os; import nvidia.cublas.lib; import nvidia.cudnn.lib; import torch; print(os.path.dirname(nvidia.cublas.lib.__file__) + ":" + os.path.dirname(nvidia.cudnn.lib.__file__) + ":" + os.path.dirname(torch.__file__) +"/lib")'`:$LD_LIBRARY_PATH - -RUN pip3 install --no-cache-dir --upgrade -r /app/requirements.txt - -# Copy the current directory contents into the container at /app -COPY . /app - -# Create necessary directories and set permissions -RUN mkdir -p /app/Results/Audio_Processing /app/Results/Video_Downloads && \ - chown -R user:user /app - -# Switch to the "user" user -USER user - -# Set environment variables -ENV PYTHONUNBUFFERED=1 \ - GRADIO_ALLOW_FLAGGING=never \ - GRADIO_NUM_PORTS=1 \ - GRADIO_SERVER_NAME=0.0.0.0 \ - GRADIO_THEME=huggingface \ - SYSTEM=spaces - -# Command to run the application -CMD ["python3", "app.py"] diff --git a/HF/HF_Dockerfile b/HF/HF_Dockerfile new file mode 100644 index 000000000..784e7700b --- /dev/null +++ b/HF/HF_Dockerfile @@ -0,0 +1,78 @@ +# Use Nvidia CUDA runtime as the base image +FROM nvidia/cuda:12.6.1-cudnn-runtime-ubuntu24.04 + +# Set build arguments for repository configuration +ARG REPO_URL=https://github.com/rmusser01/tldw.git +ARG BRANCH=main +ARG GPU_SUPPORT=cpu + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + ffmpeg \ + libsqlite3-dev \ + build-essential \ + git \ + python3 \ + python-is-python3 \ # Fixed trailing space + python3-pyaudio \ + portaudio19-dev \ + python3-pip \ + python3-venv \ + libpq-dev \ + python3-dev \ + && rm -rf /var/lib/apt/lists/* + +# Create a new user named "user9" with user ID 1009 +RUN useradd -m -u 1009 user9 + +# Switch to the "user9" user +USER user9 + +# Set environment variables for the user's home directory and PATH +ENV HOME=/home/user9 \ + PATH=/home/user9/.local/bin:$PATH + +# Set the working directory to the user's app directory +WORKDIR $HOME/app + +# Clone the repository into the working directory +RUN git clone -b ${BRANCH} ${REPO_URL} . + +# Set correct ownership and permissions for the app directory +RUN chmod -R u+rwX,go+rX,go-w /home/user9/app + +# Create and activate a virtual environment +RUN python3 -m venv venv +ENV PATH="$HOME/app/venv/bin:$PATH" + +# Upgrade pip and install wheel as the non-root user +RUN pip install --no-cache-dir --upgrade pip wheel + +# Install CUDA libraries +RUN pip install --no-cache-dir nvidia-cublas-cu12 nvidia-cudnn-cu12 + +# Install PyTorch based on GPU support +RUN if [ "$GPU_SUPPORT" = "cuda" ]; then \ + pip install torch==2.2.2 torchvision==0.17.2 torchaudio==2.2.2 --index-url https://download.pytorch.org/whl/cu123; \ + elif [ "$GPU_SUPPORT" = "amd" ]; then \ + pip install torch-directml; \ + else \ + pip install torch==2.2.2 torchvision==0.17.2 torchaudio==2.2.2 --index-url https://download.pytorch.org/whl/cpu; \ + fi + +# Install other Python dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Update config.txt for CPU if needed +RUN if [ "$GPU_SUPPORT" = "cpu" ]; then \ + sed -i 's/cuda/cpu/' ./Config_Files/config.txt; \ + fi + +# Expose port 7860 to the outside world +EXPOSE 7860 + +# Set environment variable for Gradio to listen on all interfaces +ENV GRADIO_SERVER_NAME="0.0.0.0" + +# Define the default command to run the application +CMD ["python", "summarize.py", "-gui"] \ No newline at end of file diff --git a/HF/requirements.txt b/HF/requirements.txt deleted file mode 100644 index 93e8a6633..000000000 --- a/HF/requirements.txt +++ /dev/null @@ -1,61 +0,0 @@ -arxiv -beautifulsoup4 -bert_score -chardet -chromadb -docx2txt -EbookLib -elasticsearch -fastapi -faster_whisper -fire -FlashRank -fugashi -gradio -jieba -Jinja2 -joblib -langchain -langdetect -mwparserfromhell -mwxml -nltk -numpy -onnxruntime -openai -pandas -Pillow -playwright -psycopg2 -pyannote.audio -PyAudio -pydub -pymupdf -pypandoc -pypandoc_binary -pytest -python-dotenv -PyYAML -Requests -rouge_score -sacrebleu -scikit_learn -sentence_transformers -SQLAlchemy -streamlit -tenacity -textstat -tiktoken -toml -tqdm -trafilatura -transformers -urllib3 -yt_dlp -datasets -tqdm ---index-url https://download.pytorch.org/whl/cu124 ---extra-index-url https://pypi.org/simple -torch -torchaudio -torchvision diff --git a/Helper_Scripts/Installer_Scripts/Windows_Install_Update.bat b/Helper_Scripts/Installer_Scripts/Windows_Install_Update.bat index 3d545d27b..ae02bc0c6 100644 --- a/Helper_Scripts/Installer_Scripts/Windows_Install_Update.bat +++ b/Helper_Scripts/Installer_Scripts/Windows_Install_Update.bat @@ -155,6 +155,8 @@ powershell -Command "Expand-Archive -Path 'ffmpeg.zip' -DestinationPath 'ffmpeg' move ffmpeg\ffmpeg-master-latest-win64-gpl\bin\ffmpeg.exe . rmdir /s /q ffmpeg del ffmpeg.zip +mkdir .\Bin +move ffmpeg .\Bin\ goto :eof :cleanup diff --git a/README.md b/README.md index 5223f342b..8ecbd3b31 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ## All Automated. All Local. All Yours. -### [Public Demo on HuggingFace Spaces](https://huggingface.co/spaces/oceansweep/Vid-Summarizer/?__theme=dark) +### [Public Demo on HuggingFace Spaces](https://huggingface.co/spaces/oceansweep/Vid-Summarizer/?__theme=dark) - Demo is broken due to a bug in Huggingface spaces/Gradio - **Please Note:** YouTube blocks requests from the demo. You have to provide a logged-in session cookie to bypass it :frowning_face: - Placeholder content is included for the demo. HuggingFace API is also setup in it, so you can select that as your API.) @@ -29,7 +29,7 @@ - Build up a personal knowledge archive, then turn around and use the LLM to help you learn it at a pace your comfortable with. - Also writing tools! Grammar/Style checker, Tone Analyzer, Writing editor(feedback), and more. - Full Character Chat Support - Create/Edit & Import/Export Character Cards, and chat with them. -#### The original scripts by `the-crypt-keeper` are available here: [scripts here](https://github.com/the-crypt-keeper/tldw/tree/main/tldw-original-scripts) +#### The original scripts by `the-crypt-keeper` for transcribing and summarizing youtube videos are available here: [scripts here](https://github.com/the-crypt-keeper/tldw/tree/main/tldw-original-scripts) ---------- @@ -52,13 +52,16 @@ - **Windows:** `wget https://raw.githubusercontent.com/rmusser01/tldw/main/Helper_Scripts/Installer_Scripts/Windows_Install_Update.bat && wget https://raw.githubusercontent.com/rmusser01/tldw/main/Helper_Scripts/Installer_Scripts/Windows_Run_tldw.bat` - Then double-click the downloaded batch file `Windows_Install_Update.bat` to install it, and `Windows_Run_tldw.bat` to run it. - You should now have a web browser tab opened to `http://127.0.0.1:7860/` with the GUI for the app. + - If you don't have CUDA installed on your system and available in your system path, go here: https://github.com/Purfview/whisper-standalone-win/releases/download/Faster-Whisper-XXL/Faster-Whisper-XXL_r192.3.4_windows.7z + - Extract the two files named `cudnn_ops_infer64_8.dll` and `cudnn_cnn_infer64_8.dll` from the 7z file to the `tldw` directory, and then run the `Windows_Run_tldw.bat` file. + - This will allow you to use the faster whisper models with the app. Otherwise, you won't be able to perform transcription. - **Docker:** - There's a docker build for GPU use(Needs Nvidia CUDA Controller(?): https://github.com/rmusser01/tldw/blob/main/Helper_Scripts/Dockerfiles/tldw-nvidia_amd64_Dockerfile - and plain CPU use: https://github.com/rmusser01/tldw/blob/main/Helper_Scripts/Dockerfiles/tldw_Debian_cpu-Dockerfile - the `Dockerfile` in the main directory is the Nvidia base-image-based one. So you can use your GPU if you want with it. -### What is tl/dw? +### Overview of what tl/dw currenlty is
What is this? - Click-Here @@ -142,7 +145,7 @@ All features are designed to run **locally** on your device, ensuring privacy an ---------- -### What is this (tl/dw)? +### More Detailed explanation of this project (tl/dw)
**What is this Project? (Extended) - Click-Here** diff --git a/Tests/RAG/test_RAG_Library_2.py b/Tests/RAG/test_RAG_Library_2.py index d5c26491d..d0dc7d057 100644 --- a/Tests/RAG/test_RAG_Library_2.py +++ b/Tests/RAG/test_RAG_Library_2.py @@ -115,7 +115,7 @@ def test_perform_vector_search_with_relevant_media_ids(self, mock_chroma_client, mock_chroma_client.list_collections.assert_called_once() # Assert vector_search was called with correct arguments - mock_vector_search.assert_called_once_with('collection1', query, k=5) + mock_vector_search.assert_called_once_with('collection1', query, k=10) @patch('App_Function_Libraries.RAG.RAG_Library_2.vector_search') @patch('App_Function_Libraries.RAG.RAG_Library_2.chroma_client') @@ -152,7 +152,7 @@ def test_perform_vector_search_without_relevant_media_ids(self, mock_chroma_clie mock_chroma_client.list_collections.assert_called_once() # Assert vector_search was called with correct arguments - mock_vector_search.assert_called_once_with('collection1', query, k=5) + mock_vector_search.assert_called_once_with('collection1', query, k=10) @patch('App_Function_Libraries.RAG.RAG_Library_2.search_db') def test_perform_full_text_search_with_relevant_media_ids(self, mock_search_db): @@ -171,7 +171,7 @@ def test_perform_full_text_search_with_relevant_media_ids(self, mock_search_db): relevant_media_ids = [1, 3] # Call the function - result = perform_full_text_search(query, relevant_media_ids) + result = perform_full_text_search(query, relevant_media_ids, fts_top_k=10) # Expected to filter out id 2 expected = [ @@ -182,7 +182,7 @@ def test_perform_full_text_search_with_relevant_media_ids(self, mock_search_db): # Assert search_db was called with correct arguments mock_search_db.assert_called_once_with( - query, ['content'], '', page=1, results_per_page=5) + query, ['content'], '', page=1, results_per_page=10) @patch('App_Function_Libraries.RAG.RAG_Library_2.search_db') def test_perform_full_text_search_without_relevant_media_ids(self, mock_search_db): @@ -211,7 +211,7 @@ def test_perform_full_text_search_without_relevant_media_ids(self, mock_search_d # Assert search_db was called with correct arguments mock_search_db.assert_called_once_with( - query, ['content'], '', page=1, results_per_page=5) + query, ['content'], '', page=1, results_per_page=10) @patch('App_Function_Libraries.RAG.RAG_Library_2.search_db') def test_perform_full_text_search_empty_results(self, mock_search_db): @@ -234,7 +234,7 @@ def test_perform_full_text_search_empty_results(self, mock_search_db): # Assert search_db was called with correct arguments mock_search_db.assert_called_once_with( - query, ['content'], '', page=1, results_per_page=5) + query, ['content'], '', page=1, results_per_page=10) @patch('App_Function_Libraries.RAG.RAG_Library_2.fetch_keywords_for_media') @patch('App_Function_Libraries.RAG.RAG_Library_2.logging') @@ -344,7 +344,7 @@ def test_perform_full_text_search_case_insensitive_filtering(self, mock_search_d # Assert search_db was called with correct arguments mock_search_db.assert_called_once_with( - query, ['content'], '', page=1, results_per_page=5) + query, ['content'], '', page=1, results_per_page=10) @patch('App_Function_Libraries.RAG.RAG_Library_2.search_db') def test_perform_full_text_search_multiple_pages(self, mock_search_db): @@ -380,7 +380,7 @@ def test_perform_full_text_search_multiple_pages(self, mock_search_db): # Assert search_db was called with correct arguments mock_search_db.assert_called_once_with( - query, ['content'], '', page=1, results_per_page=5) + query, ['content'], '', page=1, results_per_page=10) @patch('App_Function_Libraries.RAG.RAG_Library_2.chroma_client') @patch('App_Function_Libraries.RAG.RAG_Library_2.vector_search') @@ -429,8 +429,8 @@ def vector_search_side_effect(collection_name, query, k): mock_chroma_client.list_collections.assert_called_once() # Assert vector_search was called twice with correct arguments - mock_vector_search.assert_any_call('collection1', query, k=5) - mock_vector_search.assert_any_call('collection2', query, k=5) + mock_vector_search.assert_any_call('collection1', query, k=10) + mock_vector_search.assert_any_call('collection2', query, k=10) self.assertEqual(mock_vector_search.call_count, 2) @patch('App_Function_Libraries.RAG.RAG_Library_2.search_db') @@ -462,7 +462,7 @@ def test_perform_full_text_search_partial_matches(self, mock_search_db): # Assert search_db was called with correct arguments mock_search_db.assert_called_once_with( - query, ['content'], '', page=1, results_per_page=5) + query, ['content'], '', page=1, results_per_page=10) if __name__ == '__main__': diff --git a/requirements.txt b/requirements.txt index caae64a1d..200dcba05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,8 @@ faster_whisper fire FlashRank fugashi -gradio +# well fuck gradio. again. +gradio==4.44.1 jieba Jinja2 joblib @@ -28,6 +29,8 @@ pandas Pillow playwright psycopg2 +psycopg2-binary +psutil #psycopg2-binary pyannote.audio PyAudio