From 60cccf370017b85a9c0eb1c08a6e2fb7a7d98ae0 Mon Sep 17 00:00:00 2001 From: Harshit Gupta Date: Mon, 18 Nov 2024 00:53:14 -0800 Subject: [PATCH] restructure examples --- examples/multimodal_data/filter.py | 19 --------------- examples/multimodal_data/join.py | 22 ------------------ examples/multimodal_data/map.py | 19 --------------- examples/op_examples/multimodal_ops/filter.py | 21 +++++++++++++++++ .../op_examples/multimodal_ops/images/0.png | Bin 0 -> 553 bytes .../op_examples/multimodal_ops/images/1.png | Bin 0 -> 353 bytes .../op_examples/multimodal_ops/images/4.png | Bin 0 -> 463 bytes .../op_examples/multimodal_ops/images/5.png | Bin 0 -> 549 bytes .../op_examples/multimodal_ops/images/9.png | Bin 0 -> 460 bytes examples/op_examples/multimodal_ops/join.py | 22 ++++++++++++++++++ examples/op_examples/multimodal_ops/map.py | 21 +++++++++++++++++ lotus/sem_ops/sem_topk.py | 4 ++++ lotus/templates/task_instructions.py | 10 +++----- 13 files changed, 71 insertions(+), 67 deletions(-) delete mode 100644 examples/multimodal_data/filter.py delete mode 100644 examples/multimodal_data/join.py delete mode 100644 examples/multimodal_data/map.py create mode 100644 examples/op_examples/multimodal_ops/filter.py create mode 100644 examples/op_examples/multimodal_ops/images/0.png create mode 100644 examples/op_examples/multimodal_ops/images/1.png create mode 100644 examples/op_examples/multimodal_ops/images/4.png create mode 100644 examples/op_examples/multimodal_ops/images/5.png create mode 100644 examples/op_examples/multimodal_ops/images/9.png create mode 100644 examples/op_examples/multimodal_ops/join.py create mode 100644 examples/op_examples/multimodal_ops/map.py diff --git a/examples/multimodal_data/filter.py b/examples/multimodal_data/filter.py deleted file mode 100644 index a1b4ab85..00000000 --- a/examples/multimodal_data/filter.py +++ /dev/null @@ -1,19 +0,0 @@ -import pandas as pd -from torchvision import datasets - -import lotus -from lotus.dtype_extensions import ImageArray -from lotus.models import LM - -lm = LM(model="gpt-4o-mini") -lotus.settings.configure(lm=lm) - -mnist_data = datasets.MNIST(root="mnist_data", train=True, download=True, transform=None) - -images = [image for image, _ in mnist_data] -labels = [label for _, label in mnist_data] - -df = pd.DataFrame({"image": ImageArray(images), "label": labels}) - -df = df.sem_filter("{image} represents number 1") -print(df) diff --git a/examples/multimodal_data/join.py b/examples/multimodal_data/join.py deleted file mode 100644 index 6de5d1fb..00000000 --- a/examples/multimodal_data/join.py +++ /dev/null @@ -1,22 +0,0 @@ -import pandas as pd -from torchvision import datasets - -import lotus -from lotus.dtype_extensions import ImageArray -from lotus.models import LM - -lm = LM(model="gpt-4o-mini") -lotus.settings.configure(lm=lm) - -mnist_data = datasets.MNIST(root="mnist_data", train=True, download=True, transform=None) - -images = [image for image, _ in mnist_data] -labels = [label for _, label in mnist_data] - -df = pd.DataFrame({"image": ImageArray(images[:5]), "label": labels[:5]}) - -df2 = pd.DataFrame({"image": ImageArray(images[5:10]), "label": labels[5:10]}) - -df = df.sem_join(df2, "{image:left} represents the same number as {image:right}", strategy="zs-cot") - -print(df) diff --git a/examples/multimodal_data/map.py b/examples/multimodal_data/map.py deleted file mode 100644 index d8794835..00000000 --- a/examples/multimodal_data/map.py +++ /dev/null @@ -1,19 +0,0 @@ -import pandas as pd -from torchvision import datasets - -import lotus -from lotus.dtype_extensions import ImageArray -from lotus.models import LM - -lm = LM(model="gpt-4o-mini") -lotus.settings.configure(lm=lm) - -mnist_data = datasets.MNIST(root="mnist_data", train=True, download=True, transform=None) - -images = [image for image, _ in mnist_data] -labels = [label for _, label in mnist_data] - -df = pd.DataFrame({"image": ImageArray(images[:5]), "label": labels[:5]}) - -df = df.sem_map("convert {image} to the number it represents") -print(df) diff --git a/examples/op_examples/multimodal_ops/filter.py b/examples/op_examples/multimodal_ops/filter.py new file mode 100644 index 00000000..3fbb0fdb --- /dev/null +++ b/examples/op_examples/multimodal_ops/filter.py @@ -0,0 +1,21 @@ +import os + +import pandas as pd + +import lotus +from lotus.dtype_extensions import ImageArray +from lotus.models import LM + +lotus.settings.configure(lm=LM(model="gpt-4o-mini")) + +# The images folder contain images representing digits taken from MNIST dataset +image_file_names = os.listdir("images") # get all file in the folder + +# file names are the same as the digit represented by image +labels = [os.path.splitext(image)[0] for image in image_file_names] +image_paths = [os.path.join("images", image) for image in image_file_names] + +df = pd.DataFrame({"image": ImageArray(image_paths), "label": labels, "image_path": image_paths}) + +df = df.sem_filter("{image} represents number 1") +print(df) diff --git a/examples/op_examples/multimodal_ops/images/0.png b/examples/op_examples/multimodal_ops/images/0.png new file mode 100644 index 0000000000000000000000000000000000000000..789ddac91a1a9cd091367b1914a1e7f3af10e4cd GIT binary patch literal 553 zcmV+^0@nSBP)fXnG7OYmUVrU&1QGI9U?XwjVHuQr>E0NQ52SC z!{M+j%et;xmW7D(`MlX|CK8FCpl(*H)oeDqEDggbl}gWXZWx9ckH@a!K5qc9ZTq_^ rKblY|R4f)RXti36L?SPVzj-n5{6$L5jYax400000NkvXXu0mjfi~s{F literal 0 HcmV?d00001 diff --git a/examples/op_examples/multimodal_ops/images/1.png b/examples/op_examples/multimodal_ops/images/1.png new file mode 100644 index 0000000000000000000000000000000000000000..e44e0c9cc7b83424faf62d8375a297655a5aad17 GIT binary patch literal 353 zcmV-n0iOPeP)#tAggiiPOo;45lsrLvL`7a84^R^kC~;9S5s6#XMnn`U zhG9=Eg@aSq&Y5z*-u-r9e+J+`pPb|INJLue?RLu}CP{L?-w~0BeBb|hVDSPfgjg<@ zZ_oBEVHi%+gos_&2_aUi)w`nSc}ginM8r6bIfeu{olZnVL{(Msh_-DL5hC7hw;%`@ zL@8z4wgyepltBc5!{J~$bUvRg%VH4s`~5u6h*;OP<2VeWl#-q)ih@A|fIQFfnayUy zAiAzQj-wIFvSbhe;Cj6p@p8GW*J~csbO;fnDB=-|qR?8KFA_p9h_34@rHJV9cw||| zus8r00000NkvXXu0mjf7%7|! literal 0 HcmV?d00001 diff --git a/examples/op_examples/multimodal_ops/images/4.png b/examples/op_examples/multimodal_ops/images/4.png new file mode 100644 index 0000000000000000000000000000000000000000..7d87808b56c627b668d4d9db7eb6875c8b12069c GIT binary patch literal 463 zcmV;=0WkiFP)MA5m69?P$)zQVeOmG=dD)DCK?C?Ow(*On{PTr#K#@4(r7dQU^bhvP*D^q zrS?&e#{&TUexHSAvsstRWmBY-3W5Lt>-Cz2o+o26nb<*VwHl>#I-TxzyT`ZO-Fh?{ zF)X*+oylYhg+e-=27pSXVi?AI)aUbEo}TEDT7He+zciQWl(n@3Z)DN1Ck7c zmFrKpE606@+xO*tKkpwhec$(+PCd{0b)KH{JOKRX??EgU6Te6#66tihTCHle+ES^+ z^E}J4$z<|2U-Fqquh&bZQisFgbUJYy_xXIc(bZ~|&1NBlhr?kw9Jbr-+bDL|Xti3K z&8AYRyc@ku$oQv;QX+i^cEv+iW(8L;?U9 zkH?)(=lQ1@jmBIqw_dM*kL>w;rfK>~om#Dq#bWdM{B{ZS`+cw1`;1PdQn}sk(P;Ex zr$7jw87+b!%H{HhE!FLINs^SwWDo1?c6+5#SuU5iy5sSfN~MGq^+VouA3_Kr6wZ^$ nq*kjTgff}TZnt~*f&VaXoA4WNq4hz900000NkvXXu0mjf6C(kC literal 0 HcmV?d00001 diff --git a/examples/op_examples/multimodal_ops/images/9.png b/examples/op_examples/multimodal_ops/images/9.png new file mode 100644 index 0000000000000000000000000000000000000000..405b2f66aa1f69755a8b66eabac052fd60577f2c GIT binary patch literal 460 zcmV;-0W35oBt1E%(iXM^AHgc8;yo(nzn5-#y%8H(>$F{ z0Dy?D>w2CC05lv9hr(Pq9M0$S*XtDk=JR>6Sd2s>i9}*DnGo7+Higkztwv2vr&C2y zs93F5skqm1g*y(f-5dixA{!5f3X|vf7TB%gTM9Z=e@qWKMjuVT;s2Gh#%jNRz zc0gQxulFJPH-$ps z`FtW`y