Use conda for faiss install

.github/workflows/tests.yml

@@ -10,66 +10,79 @@ on:
   workflow_dispatch:
 
 jobs:
+  install_miniconda:
+    name: Install Miniconda
+    runs-on: ubuntu-latest
+    outputs:
+      miniconda-path: ${{ steps.miniconda-output.outputs.PATH }}
+    steps:
+      - name: Install Miniconda
+        id: miniconda-output
+        run: |
+          wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+          bash miniconda.sh -b -p $HOME/miniconda
+          echo "$HOME/miniconda/bin" >> $GITHUB_PATH
+          source $HOME/miniconda/bin/activate
+          conda init bash
+
   ruff_lint:
     name: Ruff Lint
     runs-on: ubuntu-latest
+    needs: install_miniconda
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
+      - name: Create and activate Conda environment
         run: |
+          conda env create -f environment.yml
+          source activate base && conda activate lotus
           python -m pip install --upgrade pip
           pip install ruff==0.7.2
 
       - name: Run ruff
-        run: ruff check .
+        run: |
+          source activate base && conda activate lotus
+          ruff check .
 
   mypy:
     name: Type Check
     runs-on: ubuntu-latest
+    needs: install_miniconda
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
+      - name: Create and activate Conda environment
         run: |
+          conda env create -f environment.yml
+          source activate base && conda activate lotus
           python -m pip install --upgrade pip
           pip install mypy==1.13.0
           pip install -r requirements.txt
           pip install -e .
 
       - name: Run mypy
-        run: mypy lotus/
+        run: |
+          source activate base && conda activate lotus
+          mypy lotus/
 
   openai_lm_test:
     name: OpenAI Language Model Tests
     runs-on: ubuntu-latest
+    needs: install_miniconda
     timeout-minutes: 5
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
+      - name: Create and activate Conda environment
         run: |
+          conda env create -f environment.yml
+          source activate base && conda activate lotus
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -e .
@@ -82,24 +95,24 @@ jobs:
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ENABLE_OPENAI_TESTS: true
-        run: pytest .github/tests/lm_tests.py
+        run: |
+          source activate base && conda activate lotus
+          pytest .github/tests/lm_tests.py
 
   ollama_lm_test:
     name: Ollama Language Model Tests
     runs-on: ubuntu-latest
+    needs: install_miniconda
     timeout-minutes: 10
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
+      - name: Create and activate Conda environment
         run: |
+          conda env create -f environment.yml
+          source activate base && conda activate lotus
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -e .
@@ -109,7 +122,6 @@ jobs:
         run: |
           docker pull ollama/ollama:latest
           docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
-          # Wait for Ollama server to be ready
           timeout=30
           while ! curl -s http://localhost:11434/ >/dev/null; do
             if [ $timeout -le 0 ]; then
@@ -125,25 +137,24 @@ jobs:
       - name: Run LM tests
         env:
           ENABLE_OLLAMA_TESTS: true
-        run: pytest .github/tests/lm_tests.py
-
+        run: |
+          source activate base && conda activate lotus
+          pytest .github/tests/lm_tests.py
 
   rm_test:
     name: Retrieval Model Tests
     runs-on: ubuntu-latest
+    needs: install_miniconda
     timeout-minutes: 5
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
+      - name: Create and activate Conda environment
         run: |
+          conda env create -f environment.yml
+          source activate base && conda activate lotus
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -e .
@@ -157,4 +168,6 @@ jobs:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ENABLE_OPENAI_TESTS: true
           ENABLE_LOCAL_TESTS: true
-        run: pytest .github/tests/rm_tests.py
+        run: |
+          source activate base && conda activate lotus
+          pytest .github/tests/rm_tests.py

.readthedocs.yaml

@@ -15,6 +15,9 @@ sphinx:
 formats:
    - pdf
 
+conda:
+   file: environment.yml
+
 # Optionally declare the Python requirements required to build your docs
 python:
    install:

CONTRIBUTING.md

@@ -4,7 +4,7 @@
 
 To set up for development, create a conda environment, install lotus, and install additional dev dependencies.
 ```
-conda create -n lotus python=3.10 -y
+conda env create -f environment.yml
 conda activate lotus
 git clone [email protected]:stanford-futuredata/lotus.git
 pip install -e .

README.md

@@ -35,7 +35,7 @@ LOTUS offers a number of semantic operators in a Pandas-like API, some of which
 
 # Installation
 ```
-conda create -n lotus python=3.10 -y
+conda env create -f environment.yml
 conda activate lotus
 pip install lotus-ai
 ```

docs/installation.rst

@@ -16,6 +16,6 @@ You can install Lotus using pip:
 
 .. code-block:: console
 
-    $ conda create -n lotus python=3.10 -y
+    $ conda env create -f environment.yml
     $ conda activate lotus
     $ pip install lotus-ai

docs/requirements-docs.txt

@@ -2,7 +2,6 @@ sphinx==7.3.7
 sphinx-rtd-theme==2.0.0
 
 backoff==2.2.1
-faiss-cpu==1.8.0.post1
 litellm==1.51.0
 numpy==1.26.4
 pandas==2.2.2

environment.yml

@@ -0,0 +1,7 @@
+name: lotus
+channels:
+  - conda-forge
+  - pytorch
+dependencies:
+  - python=3.10
+  - faiss-cpu=1.8.0

pyproject.toml

@@ -23,7 +23,6 @@ classifiers = [
 
 dependencies = [
     "backoff>=2.2.1,<3.0.0",
-    "faiss-cpu>=1.8.0.post1,<2.0.0",
     "litellm>=1.51.0,<2.0.0",
     "numpy>=1.25.0,<2.0.0",
     "pandas>=2.0.0,<3.0.0",

requirements.txt

@@ -1,5 +1,4 @@
 backoff==2.2.1
-faiss-cpu==1.8.0.post1
 litellm==1.51.0
 numpy==1.26.4
 pandas==2.2.2