Skip to content

Commit

Permalink
add a button for re-processing the last 5 sketches (in prompt view)
Browse files Browse the repository at this point in the history
also adjust actions: get prompt as part of process action
  • Loading branch information
benswift committed Nov 12, 2024
1 parent 4513eb0 commit 8e0c3d1
Show file tree
Hide file tree
Showing 5 changed files with 35 additions and 13 deletions.
16 changes: 7 additions & 9 deletions lib/imaginative_restoration/ai/pipeline.ex
Original file line number Diff line number Diff line change
Expand Up @@ -10,30 +10,25 @@ defmodule ImaginativeRestoration.AI.Pipeline do
def init(opts) do
stage = Keyword.get(opts, :stage)

if stage in [:crop_and_set_prompt, :process] do
if stage in [:crop_and_label, :process] do
{:ok, opts}
else
{:error, "stage must be either :crop_and_set_prompt or :process"}
{:error, "stage must be either :crop_and_label or :process"}
end
end

@impl true
def change(changeset, opts, _context) do
case Keyword.fetch!(opts, :stage) do
:crop_and_set_prompt ->
:crop_and_label ->
raw = changeset.data.raw

case Replicate.invoke("lucataco/florence-2-large", raw) do
{:ok, {label, [x, y, w, h]}} ->
# latest prompt (TODO fail gracefully if none exist)
%Prompt{template: template} = ImaginativeRestoration.Sketches.latest_prompt!()
prompt = String.replace(template, "LABEL", label)

cropped = raw |> Utils.crop!(x, y, w, h) |> Utils.to_dataurl!()

changeset
|> Ash.Changeset.force_change_attribute(:label, label)
|> Ash.Changeset.force_change_attribute(:prompt, prompt)
|> Ash.Changeset.force_change_attribute(:cropped, cropped)

_ ->
Expand All @@ -43,7 +38,10 @@ defmodule ImaginativeRestoration.AI.Pipeline do
:process ->
cropped = changeset.data.cropped
model = changeset.data.model
prompt = changeset.data.prompt

# latest prompt (TODO fail gracefully if none exist)
%Prompt{template: template} = ImaginativeRestoration.Sketches.latest_prompt!()
prompt = String.replace(template, "LABEL", changeset.data.label)

with {:ok, ai_image} <- Replicate.invoke(model, cropped, prompt),
{:ok, final_image_url} <- Replicate.invoke("lucataco/remove-bg", ai_image) do
Expand Down
2 changes: 1 addition & 1 deletion lib/imaginative_restoration/sketches.ex
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ defmodule ImaginativeRestoration.Sketches do
resource ImaginativeRestoration.Sketches.Sketch do
define :init, args: [:raw]
define :init_with_model, args: [:raw, :model], action: :init
define :crop_and_set_prompt
define :crop_and_label
define :process
end

Expand Down
4 changes: 2 additions & 2 deletions lib/imaginative_restoration/sketches/sketch.ex
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ defmodule ImaginativeRestoration.Sketches.Sketch do
change set_attribute(:model, arg(:model))
end

update :crop_and_set_prompt do
update :crop_and_label do
# No attributes needed - will process the sketch's existing raw image

# Validate that we have an raw image to work with
Expand All @@ -64,7 +64,7 @@ defmodule ImaginativeRestoration.Sketches.Sketch do
end
end

change {Pipeline, stage: :crop_and_set_prompt}
change {Pipeline, stage: :crop_and_label}
end

update :process do
Expand Down
2 changes: 1 addition & 1 deletion lib/imaginative_restoration_web/live/app_live.ex
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ defmodule ImaginativeRestorationWeb.AppLive do
Task.start(fn ->
dataurl
|> ImaginativeRestoration.Sketches.init!()
|> ImaginativeRestoration.Sketches.crop_and_set_prompt!()
|> ImaginativeRestoration.Sketches.crop_and_label!()
|> ImaginativeRestoration.Sketches.process!()
end)
end
Expand Down
24 changes: 24 additions & 0 deletions lib/imaginative_restoration_web/live/prompt_live.ex
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ defmodule ImaginativeRestorationWeb.PromptLive do
</.simple_form>
<section class="grid grid-cols-1 gap-4">
<h2 class="text-lg font-semibold">Last 5 captures</h2>
<div class="mb-4">
<.button phx-click="process_recent">Process Recent Sketches</.button>
</div>
<div id="sketches" phx-update="stream">
<.sketch :for={{dom_id, sketch} <- @streams.sketches} sketch={sketch} id={dom_id} />
</div>
Expand Down Expand Up @@ -66,6 +69,27 @@ defmodule ImaginativeRestorationWeb.PromptLive do
{:noreply, assign(socket, form: form)}
end

@impl true
def handle_event("process_recent", _params, socket) do
sketches =
Sketch
|> Ash.Query.for_read(:read)
|> Ash.Query.sort(inserted_at: :desc)
|> Ash.Query.limit(5)
|> Ash.read!()

sketches
|> Task.async_stream(
fn sketch ->
ImaginativeRestoration.Sketches.process!(sketch)
end,
timeout: :infinity
)
|> Stream.run()

{:noreply, put_flash(socket, :info, "Processing recent sketches...")}
end

@impl true
def handle_event("save", %{"form" => params}, socket) do
case AshPhoenix.Form.submit(socket.assigns.form, params: params) do
Expand Down

0 comments on commit 8e0c3d1

Please sign in to comment.