From 96d78106a9354d1c37da0cdd0ef4c5f149b50379 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 10:31:22 -0800 Subject: [PATCH 01/17] Use conda for faiss --- .readthedocs.yaml | 3 +++ CONTRIBUTING.md | 2 +- README.md | 2 +- docs/installation.rst | 2 +- docs/requirements-docs.txt | 1 - environment.yml | 7 +++++++ pyproject.toml | 1 - requirements.txt | 1 - 8 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 environment.yml diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1238f019..afeb5235 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -15,6 +15,9 @@ sphinx: formats: - pdf +conda: + file: environment.yml + # Optionally declare the Python requirements required to build your docs python: install: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a08df16..9ee022b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ To set up for development, create a conda environment, install lotus, and install additional dev dependencies. ``` -conda create -n lotus python=3.10 -y +conda env create -f environment.yml conda activate lotus git clone git@github.com:stanford-futuredata/lotus.git pip install -e . diff --git a/README.md b/README.md index 89e4b30c..16a862f2 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ LOTUS offers a number of semantic operators in a Pandas-like API, some of which # Installation ``` -conda create -n lotus python=3.10 -y +conda env create -f environment.yml conda activate lotus pip install lotus-ai ``` diff --git a/docs/installation.rst b/docs/installation.rst index 017d365a..47cb6d16 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -16,6 +16,6 @@ You can install Lotus using pip: .. code-block:: console - $ conda create -n lotus python=3.10 -y + $ conda env create -f environment.yml $ conda activate lotus $ pip install lotus-ai \ No newline at end of file diff --git a/docs/requirements-docs.txt b/docs/requirements-docs.txt index 404a5c89..af60d0b6 100644 --- a/docs/requirements-docs.txt +++ b/docs/requirements-docs.txt @@ -2,7 +2,6 @@ sphinx==7.3.7 sphinx-rtd-theme==2.0.0 backoff==2.2.1 -faiss-cpu==1.8.0.post1 litellm==1.51.0 numpy==1.26.4 pandas==2.2.2 diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..f40a9f33 --- /dev/null +++ b/environment.yml @@ -0,0 +1,7 @@ +name: lotus +channels: + - conda-forge + - pytorch +dependencies: + - python=3.10 + - faiss-cpu=1.8.0 diff --git a/pyproject.toml b/pyproject.toml index 6a170ce0..7156ba68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,6 @@ classifiers = [ dependencies = [ "backoff>=2.2.1,<3.0.0", - "faiss-cpu>=1.8.0.post1,<2.0.0", "litellm>=1.51.0,<2.0.0", "numpy>=1.25.0,<2.0.0", "pandas>=2.0.0,<3.0.0", diff --git a/requirements.txt b/requirements.txt index 655dde54..59945430 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ backoff==2.2.1 -faiss-cpu==1.8.0.post1 litellm==1.51.0 numpy==1.26.4 pandas==2.2.2 From 678f4185ae997ce183de20a7d6d04e4fd1f09eff Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 11:45:53 -0800 Subject: [PATCH 02/17] Update CI --- .github/workflows/tests.yml | 84 ++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 07a9f3ea..0101f014 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,23 +10,37 @@ on: workflow_dispatch: jobs: + setup_conda: + runs-on: ubuntu-latest + steps: + - name: Install Miniconda + run: | + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + echo "$HOME/miniconda/bin" >> $GITHUB_PATH + source $HOME/miniconda/bin/activate + conda init bash + - name: Cache Conda Environment + uses: actions/cache@v3 + with: + path: ~/.conda/envs/lotus + key: conda-lotus-${{ hashFiles('environment.yml') }} + restore-keys: conda-lotus- + ruff_lint: name: Ruff Lint runs-on: ubuntu-latest + needs: setup_conda steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install dependencies + - name: Create and activate Conda environment run: | - python -m pip install --upgrade pip - pip install ruff==0.7.2 + conda env create -f environment.yml || conda env update -f environment.yml + source activate lotus + python -m pip install ruff==0.7.2 - name: Run ruff run: ruff check . @@ -34,20 +48,17 @@ jobs: mypy: name: Type Check runs-on: ubuntu-latest + needs: setup_conda steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install dependencies + - name: Create and activate Conda environment run: | - python -m pip install --upgrade pip - pip install mypy==1.13.0 + conda env create -f environment.yml || conda env update -f environment.yml + source activate lotus + python -m pip install mypy==1.13.0 pip install -r requirements.txt pip install -e . @@ -58,20 +69,17 @@ jobs: name: OpenAI Language Model Tests runs-on: ubuntu-latest timeout-minutes: 5 + needs: setup_conda steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install dependencies + - name: Create and activate Conda environment run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + conda env create -f environment.yml || conda env update -f environment.yml + source activate lotus + python -m pip install -r requirements.txt pip install -e . pip install pytest @@ -88,20 +96,17 @@ jobs: name: Ollama Language Model Tests runs-on: ubuntu-latest timeout-minutes: 10 + needs: setup_conda steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install dependencies + - name: Create and activate Conda environment run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + conda env create -f environment.yml || conda env update -f environment.yml + source activate lotus + python -m pip install -r requirements.txt pip install -e . pip install pytest @@ -109,7 +114,6 @@ jobs: run: | docker pull ollama/ollama:latest docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama - # Wait for Ollama server to be ready timeout=30 while ! curl -s http://localhost:11434/ >/dev/null; do if [ $timeout -le 0 ]; then @@ -127,25 +131,21 @@ jobs: ENABLE_OLLAMA_TESTS: true run: pytest .github/tests/lm_tests.py - rm_test: name: Retrieval Model Tests runs-on: ubuntu-latest timeout-minutes: 5 + needs: setup_conda steps: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install dependencies + - name: Create and activate Conda environment run: | - python -m pip install --upgrade pip - pip install -r requirements.txt + conda env create -f environment.yml || conda env update -f environment.yml + source activate lotus + python -m pip install -r requirements.txt pip install -e . pip install pytest From 41c9f50d9c8928d91d8365913a15b6edff66f507 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 11:47:44 -0800 Subject: [PATCH 03/17] Update CI --- .github/workflows/tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0101f014..953ac909 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,7 +39,7 @@ jobs: - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml - source activate lotus + conda activate lotus python -m pip install ruff==0.7.2 - name: Run ruff @@ -57,7 +57,7 @@ jobs: - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml - source activate lotus + conda activate lotus python -m pip install mypy==1.13.0 pip install -r requirements.txt pip install -e . @@ -78,7 +78,7 @@ jobs: - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml - source activate lotus + conda activate lotus python -m pip install -r requirements.txt pip install -e . pip install pytest @@ -105,7 +105,7 @@ jobs: - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml - source activate lotus + conda activate lotus python -m pip install -r requirements.txt pip install -e . pip install pytest @@ -144,7 +144,7 @@ jobs: - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml - source activate lotus + conda activate lotus python -m pip install -r requirements.txt pip install -e . pip install pytest From fde62c51bff30f453d66c8d3f01ceb90c30acae6 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 11:49:21 -0800 Subject: [PATCH 04/17] Update CI --- .github/workflows/tests.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 953ac909..0a65721c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,13 +19,6 @@ jobs: bash miniconda.sh -b -p $HOME/miniconda echo "$HOME/miniconda/bin" >> $GITHUB_PATH source $HOME/miniconda/bin/activate - conda init bash - - name: Cache Conda Environment - uses: actions/cache@v3 - with: - path: ~/.conda/envs/lotus - key: conda-lotus-${{ hashFiles('environment.yml') }} - restore-keys: conda-lotus- ruff_lint: name: Ruff Lint @@ -38,6 +31,7 @@ jobs: - name: Create and activate Conda environment run: | + conda init bash conda env create -f environment.yml || conda env update -f environment.yml conda activate lotus python -m pip install ruff==0.7.2 From fc4c0608b7586821a6cd49fba7268e6dd96a253c Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 11:51:23 -0800 Subject: [PATCH 05/17] Update CI --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0a65721c..9e5432f3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,7 +31,7 @@ jobs: - name: Create and activate Conda environment run: | - conda init bash + conda init bash && source ~/.bashrc conda env create -f environment.yml || conda env update -f environment.yml conda activate lotus python -m pip install ruff==0.7.2 From f244ab8c34c74f7ec36079172bc864f214bc75ef Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 11:53:10 -0800 Subject: [PATCH 06/17] Update CI --- .github/workflows/tests.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9e5432f3..1f1c6a17 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,9 +31,8 @@ jobs: - name: Create and activate Conda environment run: | - conda init bash && source ~/.bashrc conda env create -f environment.yml || conda env update -f environment.yml - conda activate lotus + source $HOME/miniconda/bin/activate python -m pip install ruff==0.7.2 - name: Run ruff From 3c02bec40a0c2b3f15c90499d535cc062997e888 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 11:57:51 -0800 Subject: [PATCH 07/17] Update CI --- .github/workflows/tests.yml | 50 ++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1f1c6a17..3367f0e7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,8 +10,10 @@ on: workflow_dispatch: jobs: - setup_conda: + ruff_lint: + name: Ruff Lint runs-on: ubuntu-latest + steps: - name: Install Miniconda run: | @@ -19,31 +21,34 @@ jobs: bash miniconda.sh -b -p $HOME/miniconda echo "$HOME/miniconda/bin" >> $GITHUB_PATH source $HOME/miniconda/bin/activate + conda init bash - ruff_lint: - name: Ruff Lint - runs-on: ubuntu-latest - needs: setup_conda - - steps: - name: Checkout code uses: actions/checkout@v3 - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml - source $HOME/miniconda/bin/activate + conda activate lotus python -m pip install ruff==0.7.2 - name: Run ruff run: ruff check . + mypy: name: Type Check runs-on: ubuntu-latest - needs: setup_conda steps: + - name: Install Miniconda + run: | + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + echo "$HOME/miniconda/bin" >> $GITHUB_PATH + source $HOME/miniconda/bin/activate + conda init bash + - name: Checkout code uses: actions/checkout@v3 @@ -62,9 +67,16 @@ jobs: name: OpenAI Language Model Tests runs-on: ubuntu-latest timeout-minutes: 5 - needs: setup_conda steps: + - name: Install Miniconda + run: | + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + echo "$HOME/miniconda/bin" >> $GITHUB_PATH + source $HOME/miniconda/bin/activate + conda init bash + - name: Checkout code uses: actions/checkout@v3 @@ -89,9 +101,16 @@ jobs: name: Ollama Language Model Tests runs-on: ubuntu-latest timeout-minutes: 10 - needs: setup_conda steps: + - name: Install Miniconda + run: | + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + echo "$HOME/miniconda/bin" >> $GITHUB_PATH + source $HOME/miniconda/bin/activate + conda init bash + - name: Checkout code uses: actions/checkout@v3 @@ -128,9 +147,16 @@ jobs: name: Retrieval Model Tests runs-on: ubuntu-latest timeout-minutes: 5 - needs: setup_conda steps: + - name: Install Miniconda + run: | + wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh + bash miniconda.sh -b -p $HOME/miniconda + echo "$HOME/miniconda/bin" >> $GITHUB_PATH + source $HOME/miniconda/bin/activate + conda init bash + - name: Checkout code uses: actions/checkout@v3 From a9de31dc7db65ebf25ae9a231ef6a457e1468628 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 11:59:08 -0800 Subject: [PATCH 08/17] Update CI --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3367f0e7..61c48894 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -55,6 +55,7 @@ jobs: - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml + source activate base conda activate lotus python -m pip install mypy==1.13.0 pip install -r requirements.txt From 1558649deb9b81578da79934a24ce043e5cd45a2 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 12:01:02 -0800 Subject: [PATCH 09/17] Update CI --- .github/workflows/tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 61c48894..d16630ed 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,6 +29,7 @@ jobs: - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml + source activate base conda activate lotus python -m pip install ruff==0.7.2 @@ -84,6 +85,7 @@ jobs: - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml + source activate base conda activate lotus python -m pip install -r requirements.txt pip install -e . @@ -118,6 +120,7 @@ jobs: - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml + source activate base conda activate lotus python -m pip install -r requirements.txt pip install -e . @@ -164,6 +167,7 @@ jobs: - name: Create and activate Conda environment run: | conda env create -f environment.yml || conda env update -f environment.yml + source activate base conda activate lotus python -m pip install -r requirements.txt pip install -e . From c2ee9d3d4f523637fd9f4632e4f7827b6d122f34 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 12:03:20 -0800 Subject: [PATCH 10/17] Update CI --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d16630ed..eec53ea7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,9 +31,10 @@ jobs: conda env create -f environment.yml || conda env update -f environment.yml source activate base conda activate lotus - python -m pip install ruff==0.7.2 + pip install ruff==0.7.2 - name: Run ruff + echo "$(pip freeze)" run: ruff check . From 0502aaa82c4a17d9fd2743692c25cd20f69a2165 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 12:04:06 -0800 Subject: [PATCH 11/17] Update CI --- .github/workflows/tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index eec53ea7..7f74ce44 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,8 +34,9 @@ jobs: pip install ruff==0.7.2 - name: Run ruff - echo "$(pip freeze)" - run: ruff check . + run: | + echo $(pip freeze) + ruff check . mypy: From c0a710da09624a8813c37945bd70f1ec93af5def Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 12:05:51 -0800 Subject: [PATCH 12/17] Update CI --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7f74ce44..5ed6cfc5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,6 +32,7 @@ jobs: source activate base conda activate lotus pip install ruff==0.7.2 + echo $(pip freeze) - name: Run ruff run: | From 8951eab12574c3bac9f81556c14ee024a52a482b Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 12:07:44 -0800 Subject: [PATCH 13/17] Update CI --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5ed6cfc5..cb950007 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,7 @@ jobs: - name: Run ruff run: | + source activate base && conda activate lotus echo $(pip freeze) ruff check . From 1f20c8cd33d45c4b606eeaa60752fcfd14446257 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 12:10:35 -0800 Subject: [PATCH 14/17] Update CI --- .github/workflows/tests.yml | 38 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cb950007..01c46125 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,11 +28,9 @@ jobs: - name: Create and activate Conda environment run: | - conda env create -f environment.yml || conda env update -f environment.yml - source activate base - conda activate lotus + conda env create -f environment.yml + source activate base && conda activate lotus pip install ruff==0.7.2 - echo $(pip freeze) - name: Run ruff run: | @@ -59,9 +57,8 @@ jobs: - name: Create and activate Conda environment run: | - conda env create -f environment.yml || conda env update -f environment.yml - source activate base - conda activate lotus + conda env create -f environment.yml + source activate base && conda activate lotus python -m pip install mypy==1.13.0 pip install -r requirements.txt pip install -e . @@ -88,9 +85,8 @@ jobs: - name: Create and activate Conda environment run: | - conda env create -f environment.yml || conda env update -f environment.yml - source activate base - conda activate lotus + conda env create -f environment.yml + source activate base && conda activate lotus python -m pip install -r requirements.txt pip install -e . pip install pytest @@ -102,7 +98,9 @@ jobs: env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ENABLE_OPENAI_TESTS: true - run: pytest .github/tests/lm_tests.py + run: | + source activate base && conda activate lotus + pytest .github/tests/lm_tests.py ollama_lm_test: name: Ollama Language Model Tests @@ -123,9 +121,8 @@ jobs: - name: Create and activate Conda environment run: | - conda env create -f environment.yml || conda env update -f environment.yml - source activate base - conda activate lotus + conda env create -f environment.yml + source activate base && conda activate lotus python -m pip install -r requirements.txt pip install -e . pip install pytest @@ -149,7 +146,9 @@ jobs: - name: Run LM tests env: ENABLE_OLLAMA_TESTS: true - run: pytest .github/tests/lm_tests.py + run: | + source activate base && conda activate lotus + pytest .github/tests/lm_tests.py rm_test: name: Retrieval Model Tests @@ -170,9 +169,8 @@ jobs: - name: Create and activate Conda environment run: | - conda env create -f environment.yml || conda env update -f environment.yml - source activate base - conda activate lotus + conda env create -f environment.yml + source activate base && conda activate lotus python -m pip install -r requirements.txt pip install -e . pip install pytest @@ -185,4 +183,6 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ENABLE_OPENAI_TESTS: true ENABLE_LOCAL_TESTS: true - run: pytest .github/tests/rm_tests.py + run: | + source activate base && conda activate lotus + pytest .github/tests/rm_tests.py From a3363f5d66de044079a3ecbceed021395a906d15 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 12:15:06 -0800 Subject: [PATCH 15/17] Update CI --- .github/workflows/tests.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 01c46125..98eb2044 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,7 +35,6 @@ jobs: - name: Run ruff run: | source activate base && conda activate lotus - echo $(pip freeze) ruff check . @@ -59,7 +58,7 @@ jobs: run: | conda env create -f environment.yml source activate base && conda activate lotus - python -m pip install mypy==1.13.0 + pip install mypy==1.13.0 pip install -r requirements.txt pip install -e . @@ -87,7 +86,7 @@ jobs: run: | conda env create -f environment.yml source activate base && conda activate lotus - python -m pip install -r requirements.txt + pip install -r requirements.txt pip install -e . pip install pytest @@ -123,7 +122,7 @@ jobs: run: | conda env create -f environment.yml source activate base && conda activate lotus - python -m pip install -r requirements.txt + pip install -r requirements.txt pip install -e . pip install pytest @@ -171,7 +170,7 @@ jobs: run: | conda env create -f environment.yml source activate base && conda activate lotus - python -m pip install -r requirements.txt + pip install -r requirements.txt pip install -e . pip install pytest From de5e5ca4b63d0cc758c134dce1963efed6af842b Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 12:18:45 -0800 Subject: [PATCH 16/17] Update CI --- .github/workflows/tests.yml | 55 ++++++++++++------------------------- 1 file changed, 18 insertions(+), 37 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 98eb2044..340046bc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,12 +10,14 @@ on: workflow_dispatch: jobs: - ruff_lint: - name: Ruff Lint + install_miniconda: + name: Install Miniconda runs-on: ubuntu-latest - + outputs: + miniconda-path: ${{ steps.miniconda-output.outputs.PATH }} steps: - name: Install Miniconda + id: miniconda-output run: | wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh bash miniconda.sh -b -p $HOME/miniconda @@ -23,6 +25,12 @@ jobs: source $HOME/miniconda/bin/activate conda init bash + ruff_lint: + name: Ruff Lint + runs-on: ubuntu-latest + needs: install_miniconda + + steps: - name: Checkout code uses: actions/checkout@v3 @@ -37,20 +45,12 @@ jobs: source activate base && conda activate lotus ruff check . - mypy: name: Type Check runs-on: ubuntu-latest + needs: install_miniconda steps: - - name: Install Miniconda - run: | - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - echo "$HOME/miniconda/bin" >> $GITHUB_PATH - source $HOME/miniconda/bin/activate - conda init bash - - name: Checkout code uses: actions/checkout@v3 @@ -63,22 +63,17 @@ jobs: pip install -e . - name: Run mypy - run: mypy lotus/ + run: | + source activate base && conda activate lotus + mypy lotus/ openai_lm_test: name: OpenAI Language Model Tests runs-on: ubuntu-latest + needs: install_miniconda timeout-minutes: 5 steps: - - name: Install Miniconda - run: | - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - echo "$HOME/miniconda/bin" >> $GITHUB_PATH - source $HOME/miniconda/bin/activate - conda init bash - - name: Checkout code uses: actions/checkout@v3 @@ -104,17 +99,10 @@ jobs: ollama_lm_test: name: Ollama Language Model Tests runs-on: ubuntu-latest + needs: install_miniconda timeout-minutes: 10 steps: - - name: Install Miniconda - run: | - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - echo "$HOME/miniconda/bin" >> $GITHUB_PATH - source $HOME/miniconda/bin/activate - conda init bash - - name: Checkout code uses: actions/checkout@v3 @@ -152,17 +140,10 @@ jobs: rm_test: name: Retrieval Model Tests runs-on: ubuntu-latest + needs: install_miniconda timeout-minutes: 5 steps: - - name: Install Miniconda - run: | - wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh - bash miniconda.sh -b -p $HOME/miniconda - echo "$HOME/miniconda/bin" >> $GITHUB_PATH - source $HOME/miniconda/bin/activate - conda init bash - - name: Checkout code uses: actions/checkout@v3 From 22764de52f98d84db883078e7cb3c77ee6c0ecd7 Mon Sep 17 00:00:00 2001 From: Sid Jha Date: Sat, 9 Nov 2024 12:23:32 -0800 Subject: [PATCH 17/17] Update CI --- .github/workflows/tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 340046bc..408b57ac 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,6 +38,7 @@ jobs: run: | conda env create -f environment.yml source activate base && conda activate lotus + python -m pip install --upgrade pip pip install ruff==0.7.2 - name: Run ruff @@ -58,6 +59,7 @@ jobs: run: | conda env create -f environment.yml source activate base && conda activate lotus + python -m pip install --upgrade pip pip install mypy==1.13.0 pip install -r requirements.txt pip install -e . @@ -81,6 +83,7 @@ jobs: run: | conda env create -f environment.yml source activate base && conda activate lotus + python -m pip install --upgrade pip pip install -r requirements.txt pip install -e . pip install pytest @@ -110,6 +113,7 @@ jobs: run: | conda env create -f environment.yml source activate base && conda activate lotus + python -m pip install --upgrade pip pip install -r requirements.txt pip install -e . pip install pytest @@ -151,6 +155,7 @@ jobs: run: | conda env create -f environment.yml source activate base && conda activate lotus + python -m pip install --upgrade pip pip install -r requirements.txt pip install -e . pip install pytest