restructure examples

examples/op_examples/multimodal_ops/filter.py

@@ -0,0 +1,21 @@
+import os
+
+import pandas as pd
+
+import lotus
+from lotus.dtype_extensions import ImageArray
+from lotus.models import LM
+
+lotus.settings.configure(lm=LM(model="gpt-4o-mini"))
+
+# The images folder contain images representing digits taken from MNIST dataset
+image_file_names = os.listdir("images")  # get all file in the folder
+
+# file names are the same as the digit represented by image
+labels = [os.path.splitext(image)[0] for image in image_file_names]
+image_paths = [os.path.join("images", image) for image in image_file_names]
+
+df = pd.DataFrame({"image": ImageArray(image_paths), "label": labels, "image_path": image_paths})
+
+df = df.sem_filter("{image} represents number 1")
+print(df)

examples/op_examples/multimodal_ops/join.py

@@ -0,0 +1,22 @@
+import os
+
+import pandas as pd
+
+import lotus
+from lotus.dtype_extensions import ImageArray
+from lotus.models import LM
+
+lotus.settings.configure(lm=LM(model="gpt-4o-mini"))
+
+# The images folder contain images representing digits taken from MNIST dataset
+image_file_names = os.listdir("images")  # get all file in the folder
+
+# file names are the same as the digit represented by image
+image_paths = [os.path.join("images", image) for image in image_file_names]
+
+image_df = pd.DataFrame({"image": ImageArray(image_paths), "image_path": image_paths})
+labels_df = pd.DataFrame({"label": [0, 1]})
+
+df = image_df.sem_join(labels_df, "{image} represents the number {label}", strategy="zs-cot")
+
+print(df)

examples/op_examples/multimodal_ops/map.py

@@ -0,0 +1,21 @@
+import os
+
+import pandas as pd
+
+import lotus
+from lotus.dtype_extensions import ImageArray
+from lotus.models import LM
+
+lotus.settings.configure(lm=LM(model="gpt-4o-mini"))
+
+# The images folder contain images representing digits taken from MNIST dataset
+image_file_names = os.listdir("images")  # get all file in the folder
+
+# file names are the same as the digit represented by image
+labels = [os.path.splitext(image)[0] for image in image_file_names]
+image_paths = [os.path.join("images", image) for image in image_file_names]
+
+df = pd.DataFrame({"image": ImageArray(image_paths), "label": labels, "image_path": image_paths})
+
+df = df.sem_map("convert {image} to the number it represents")
+print(df)

lotus/sem_ops/sem_topk.py

@@ -6,6 +6,7 @@
 import pandas as pd
 
 import lotus
+from lotus.dtype_extensions import ImageDtype
 from lotus.templates import task_instructions
 from lotus.types import LMOutput, SemanticTopKOutput
 
@@ -374,6 +375,9 @@ def __call__(
 
         if method == "quick-sem":
             assert len(col_li) == 1, "Only one column can be used for embedding optimization"
+            assert not isinstance(
+                self._obj[col_li[0]].dtype, ImageDtype
+            ), "Image columns are not supported for embedding optimization"
             col_name = col_li[0]
             # Sort the dataframe by the column to be used for embedding optimization
             self._obj = self._obj.sem_index(col_name, f"{col_name}_lotus_index").sem_search(

lotus/templates/task_instructions.py

@@ -39,13 +39,9 @@ def user_message_formatter(
         }
     return {
         "role": "user",
-        "content": [
-            {
-                "type": "text",
-                "text": f"{user_instruction_with_tag}\n\nContext:\n{text}",
-            },
-        ]
-        + image_inputs,
+        "content": [{"type": "text", "text": f"Context:\n{text}"}]
+        + image_inputs
+        + [{"type": "text", "text": f"\n\n{user_instruction_with_tag}"}],
     }