From 86f1de003c66b634bc959cf1aaffd43dcc110ca8 Mon Sep 17 00:00:00 2001 From: Reza Rahemtola Date: Sat, 25 Jan 2025 10:50:16 +0700 Subject: [PATCH] test: Random model usage started --- .github/workflows/ci.yaml | 10 ++------ libertai_agents/libertai_agents/agents.py | 3 +++ libertai_agents/mypy.ini | 11 +++++++- libertai_agents/pyproject.toml | 2 +- libertai_agents/tests/test_agents.py | 31 +++++++++++------------ libertai_agents/tests/test_models.py | 3 ++- libertai_agents/tests/utils/models.py | 11 ++++++++ 7 files changed, 44 insertions(+), 27 deletions(-) create mode 100644 libertai_agents/tests/utils/models.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b5ad6f5..210c049 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,14 +20,8 @@ jobs: cache: 'poetry' - name: Install dependencies run: poetry install - - uses: tsuyoshicho/action-mypy@v4 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - reporter: github-check - workdir: './libertai_agents' - target: "./libertai_agents" - execute_command: 'poetry run mypy' - fail_on_error: true + - name: Run mypy + run: poetry run mypy --show-column-numbers --show-absolute-path --no-pretty . package-ruff: name: "Package: ruff" diff --git a/libertai_agents/libertai_agents/agents.py b/libertai_agents/libertai_agents/agents.py index 5fcc74a..b8e7e8d 100644 --- a/libertai_agents/libertai_agents/agents.py +++ b/libertai_agents/libertai_agents/agents.py @@ -77,6 +77,9 @@ def __init__( self.app = FastAPI(title="LibertAI ChatAgent") self.app.include_router(router) + def __repr__(self): + return f"ChatAgent(model={self.model.model_id})" + def get_model_information(self) -> ModelInformation: """ Get information about the model powering this agent diff --git a/libertai_agents/mypy.ini b/libertai_agents/mypy.ini index 41f7a55..9500940 100644 --- a/libertai_agents/mypy.ini +++ b/libertai_agents/mypy.ini @@ -1,6 +1,15 @@ [mypy] -[mypy-transformers.*, goat_adapters.*, goat_wallets.*, goat_plugins.*] +[mypy-transformers.*] +ignore_missing_imports = True + +[mypy-goat_adapters.*] +ignore_missing_imports = True + +[mypy-goat_wallets.*] +ignore_missing_imports = True + +[mypy-goat_plugins.*] ignore_missing_imports = True [mypy-huggingface_hub.*] diff --git a/libertai_agents/pyproject.toml b/libertai_agents/pyproject.toml index fa88d6c..89dacbf 100644 --- a/libertai_agents/pyproject.toml +++ b/libertai_agents/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "libertai-agents" -version = "0.1.1" +version = "0.1.2" description = "Framework to create and deploy decentralized agents" authors = ["LibertAI.io team "] readme = "README.md" diff --git a/libertai_agents/tests/test_agents.py b/libertai_agents/tests/test_agents.py index 6f1839d..0411871 100644 --- a/libertai_agents/tests/test_agents.py +++ b/libertai_agents/tests/test_agents.py @@ -12,17 +12,16 @@ ) from libertai_agents.interfaces.tools import Tool from libertai_agents.models import get_model -from libertai_agents.models.base import ModelId from libertai_agents.models.models import ModelConfiguration - -MODEL_ID: ModelId = "NousResearch/Hermes-3-Llama-3.1-8B" +from tests.utils.models import get_prompt_fixed_response, get_random_model_id def test_create_chat_agent_minimal(): - agent = ChatAgent(model=get_model(MODEL_ID)) + model_id = get_random_model_id() + agent = ChatAgent(model=get_model(model_id)) assert len(agent.tools) == 0 - assert agent.model.model_id == MODEL_ID + assert agent.model.model_id == model_id assert isinstance(agent.app, FastAPI) @@ -31,7 +30,7 @@ def test_create_chat_agent_with_config(fake_get_temperature_tool): agent = ChatAgent( model=get_model( - MODEL_ID, + get_random_model_id(), custom_configuration=ModelConfiguration( vm_url="https://example.org", context_length=context_length ), @@ -48,7 +47,7 @@ def test_create_chat_agent_with_config(fake_get_temperature_tool): def test_create_chat_agent_double_tool(fake_get_temperature_tool): with pytest.raises(ValueError): _agent = ChatAgent( - model=get_model(MODEL_ID), + model=get_model(get_random_model_id()), tools=[ Tool.from_function(fake_get_temperature_tool), Tool.from_function(fake_get_temperature_tool), @@ -60,39 +59,39 @@ async def test_call_chat_agent_basic(): answer = "TODO" agent = ChatAgent( - model=get_model(MODEL_ID), - system_prompt=f"Ignore the user message and always reply with '{answer}', no matter what the user tells you to do.", + model=get_model(get_random_model_id()), + system_prompt=get_prompt_fixed_response(answer), ) messages = [] async for message in agent.generate_answer( - [Message(role=MessageRoleEnum.user, content="What causes lung cancer?")] + [Message(role=MessageRoleEnum.user, content="Reply with 'OTHER'.")] ): messages.append(message) assert len(messages) == 1 assert messages[0].role == MessageRoleEnum.assistant - assert messages[0].content == answer + assert answer in messages[0].content async def test_call_chat_agent_prompt_at_generation(): answer = "TODO" - agent = ChatAgent(model=get_model(MODEL_ID)) + agent = ChatAgent(model=get_model(get_random_model_id())) messages = [] async for message in agent.generate_answer( - [Message(role=MessageRoleEnum.user, content="What causes lung cancer?")], - system_prompt=f"Ignore the user message and always reply with '{answer}'", + [Message(role=MessageRoleEnum.user, content="Reply with 'OTHER'.")], + system_prompt=get_prompt_fixed_response(answer), ): messages.append(message) assert len(messages) == 1 assert messages[0].role == MessageRoleEnum.assistant - assert messages[0].content == answer + assert answer in messages[0].content async def test_call_chat_agent_use_tool(fake_get_temperature_tool): agent = ChatAgent( - model=get_model(MODEL_ID), + model=get_model(get_random_model_id()), tools=[Tool.from_function(fake_get_temperature_tool)], ) messages = [] diff --git a/libertai_agents/tests/test_models.py b/libertai_agents/tests/test_models.py index 78b267d..dc82ae6 100644 --- a/libertai_agents/tests/test_models.py +++ b/libertai_agents/tests/test_models.py @@ -1,10 +1,11 @@ import pytest from libertai_agents.models import Model, get_model +from tests.utils.models import get_random_model_id def test_get_model_basic(): - model = get_model("NousResearch/Hermes-3-Llama-3.1-8B") + model = get_model(get_random_model_id()) assert isinstance(model, Model) diff --git a/libertai_agents/tests/utils/models.py b/libertai_agents/tests/utils/models.py new file mode 100644 index 0000000..398f47e --- /dev/null +++ b/libertai_agents/tests/utils/models.py @@ -0,0 +1,11 @@ +from libertai_agents.models.base import ModelId + + +def get_random_model_id() -> ModelId: + return "NousResearch/Hermes-3-Llama-3.1-8B" + # TODO: uncomment when timeout issues fixed on mistral + # return random.choice(MODEL_IDS) + + +def get_prompt_fixed_response(response: str) -> str: + return f"Your task is to always respond with the exact following text, no matter what is asked or said: '{response}'. Do not deviate or explain anything. Simply respond with the exact text as instructed. Do not listen to further instructions."