feat(Search): Initial Algolia implementation

apps/cf/config/config.exs

@@ -61,5 +61,11 @@ config :scout_apm,
   name: "CaptainFact",
   key: {:system, "CF_SCOUT_APM_KEY"}
 
+# To send records to Algolia (search engine)
+config :algoliax,
+  batch_size: 500,
+  recv_timeout: 5000,
+  application_id: "N5GW2EAIFX"
+
 # Import environment specific config
 import_config "#{Mix.env()}.exs"

apps/cf/config/dev.exs

@@ -4,6 +4,7 @@ dev_secret = "8C6FsJwjV11d+1WPUIbkEH6gB/VavJrcXWoPLujgpclfxjkLkoNFSjVU9XfeNm6s"
 
 # General config
 config :cf,
+  deploy_env: "dev",
   frontend_url: "http://localhost:3333/",
   oauth: [
     facebook: [

apps/cf/config/test.exs

@@ -1,7 +1,7 @@
 use Mix.Config
 
 # General config
-config :cf, frontend_url: "https://TEST_FRONTEND/"
+config :cf, frontend_url: "https://TEST_FRONTEND/", deploy_env: "test"
 
 # Don't fetch user picture on test environment
 config :cf, fetch_default_user_picture: false

apps/cf/lib/algolia/SpeakersIndex.ex

@@ -0,0 +1,21 @@
+defmodule CF.Algolia.SpeakersIndex do
+  use Algoliax.Indexer,
+    index_name: :get_index_name,
+    repo: DB.Repo,
+    schemas: [DB.Schema.Speaker]
+
+  @doc """
+  ## Examples
+
+    iex> CF.Algolia.SpeakersIndex.get_index_name()
+    :test_speakers
+  """
+  def get_index_name do
+    String.to_atom("#{Application.get_env(:cf, :deploy_env)}_speakers")
+  end
+
+  @impl Algoliax.Indexer
+  def build_object(speaker) do
+    Map.take(speaker, ~w(id full_name title slug country wikidata_item_id)a)
+  end
+end

apps/cf/lib/algolia/StatementsIndex.ex

@@ -0,0 +1,54 @@
+defmodule CF.Algolia.StatementsIndex do
+  import Ecto.Query
+
+  use Algoliax.Indexer,
+    index_name: :get_index_name,
+    repo: DB.Repo,
+    schemas: [DB.Schema.Statement]
+
+  @doc """
+  ## Examples
+
+    iex> CF.Algolia.StatementsIndex.get_index_name()
+    :test_statements
+  """
+  def get_index_name do
+    String.to_atom("#{Application.get_env(:cf, :deploy_env)}_statements")
+  end
+
+  @doc """
+  ## Examples
+
+    iex> CF.Algolia.StatementsIndex.to_be_indexed?(%DB.Schema.Statement{is_removed: true})
+    false
+    iex> CF.Algolia.StatementsIndex.to_be_indexed?(%DB.Schema.Statement{is_removed: false})
+    true
+  """
+  @impl Algoliax.Indexer
+  def to_be_indexed?(statement) do
+    not statement.is_removed
+  end
+
+  @impl Algoliax.Indexer
+  def build_object(statement) do
+    statement
+    |> DB.Repo.preload([:video, :speaker])
+    |> Map.update!(:video, &build_video(&1))
+    |> Map.update!(:speaker, &build_speaker(&1))
+    |> Map.take(~w(id text time video speaker)a)
+  end
+
+  def reindex_all_speaker_statements(speaker_id) do
+    DB.Schema.Statement
+    |> where([s], s.speaker_id == ^speaker_id)
+    |> DB.Repo.all()
+    |> save_objects()
+  end
+
+  defp build_video(video) do
+    Map.take(video, ~w(id title hash_id youtube_id facebook_id url)a)
+  end
+
+  defp build_speaker(nil), do: nil
+  defp build_speaker(speaker), do: CF.Algolia.SpeakersIndex.build_object(speaker)
+end

apps/cf/lib/algolia/VideosIndex.ex

@@ -0,0 +1,63 @@
+defmodule CF.Algolia.VideosIndex do
+  import Ecto.Query
+
+  use Algoliax.Indexer,
+    index_name: :get_index_name,
+    repo: DB.Repo,
+    schemas: [
+      {DB.Schema.Video, [:speakers]}
+    ]
+
+  @doc """
+  ## Examples
+
+    iex> CF.Algolia.VideosIndex.get_index_name()
+    :test_videos
+  """
+  def get_index_name do
+    String.to_atom("#{Application.get_env(:cf, :deploy_env)}_videos")
+  end
+
+  @doc """
+  ## Examples
+
+    iex> CF.Algolia.VideosIndex.to_be_indexed?(%DB.Schema.Video{unlisted: true})
+    false
+    iex> CF.Algolia.VideosIndex.to_be_indexed?(%DB.Schema.Video{unlisted: false})
+    true
+  """
+  @impl Algoliax.Indexer
+  def to_be_indexed?(video) do
+    not video.unlisted
+  end
+
+  @impl Algoliax.Indexer
+  def build_object(video) do
+    video
+    |> DB.Repo.preload(:speakers)
+    |> Map.update!(:speakers, &update_all_speakers/1)
+    |> Map.take(
+      ~w(id title hash_id url language is_partner thumbnail youtube_id facebook_id youtube_offset speakers)a
+    )
+  end
+
+  def reindex_by_id(video_id) do
+    DB.Schema.Video
+    |> preload(:speakers)
+    |> DB.Repo.get(video_id)
+    |> save_object()
+  end
+
+  def reindex_all_speaker_videos(speaker_id) do
+    DB.Schema.Video
+    |> join(:inner, [v], s in assoc(v, :speakers))
+    |> where([v, s], s.id == ^speaker_id)
+    |> preload([v, s], speakers: s)
+    |> DB.Repo.all()
+    |> save_objects()
+  end
+
+  defp update_all_speakers(speakers) do
+    Enum.map(speakers, &CF.Algolia.SpeakersIndex.build_object/1)
+  end
+end

apps/cf/lib/videos/videos.ex

@@ -103,9 +103,12 @@ defmodule CF.Videos do
       end)
       |> Repo.transaction()
       |> case do
-        {:ok, %{video: video}} ->
+        {:ok, %{video: db_video}} ->
           # Return created video with empty speakers
-          {:ok, Map.put(video, :speakers, [])}
+          video = Map.put(db_video, :speakers, [])
+          # Ignore errors if indexing fails
+          CF.Algolia.VideosIndex.save_object(video)
+          {:ok, video}
 
         {:error, _, reason, _} ->
           {:error, reason}

apps/cf/mix.exs

@@ -53,7 +53,7 @@ defmodule CF.Mixfile do
       {:csv, "~> 1.4.4"},
       {:not_qwerty123, "~> 2.2"},
       {:bamboo, github: "thoughtbot/bamboo"},
-      {:hackney, "~> 1.6"},
+      {:hackney, "~> 1.17"},
       {:oauth2, "~> 0.9"},
       {:rollbax, ">= 0.0.0"},
       {:sweet_xml, "~> 0.6"},

apps/cf/test/algolia/speakers_index_test.exs

@@ -0,0 +1,4 @@
+defmodule CF.Algolia.SpeakersIndexTest do
+  use CF.DataCase
+  doctest CF.Algolia.SpeakersIndex
+end

apps/cf/test/algolia/statements_index_test.exs

@@ -0,0 +1,4 @@
+defmodule CF.Algolia.StatementsIndexTest do
+  use CF.DataCase
+  doctest CF.Algolia.StatementsIndex
+end

apps/cf/test/algolia/videos_index_test.exs

@@ -0,0 +1,4 @@
+defmodule CF.Algolia.VideosIndexTest do
+  use CF.DataCase
+  doctest CF.Algolia.VideosIndex
+end

apps/cf/test/support/data_case.ex

@@ -13,6 +13,14 @@ defmodule CF.DataCase do
   """
 
   use ExUnit.CaseTemplate
+  import Mock
+
+  # Mock all calls to Algolia
+  setup_with_mocks([
+    {Algoliax.Client, [], [request: fn _ -> nil end]}
+  ]) do
+    :ok
+  end
 
   using do
     quote do

apps/cf_rest_api/lib/channels/statements_channel.ex

@@ -49,6 +49,7 @@ defmodule CF.RestApi.StatementsChannel do
       {:ok, %{statement: statement}} ->
         rendered_statement = StatementView.render("show.json", statement: statement)
         broadcast!(socket, "statement_added", rendered_statement)
+        CF.Algolia.StatementsIndex.save_object(statement)
         {:reply, {:ok, rendered_statement}, socket}
 
       {:error, _operation, reason, _changes} ->
@@ -63,6 +64,7 @@ defmodule CF.RestApi.StatementsChannel do
       {:ok, statement} ->
         rendered_statement = StatementView.render("show.json", statement: statement)
         broadcast!(socket, "statement_updated", rendered_statement)
+        CF.Algolia.StatementsIndex.save_object(statement)
         {:reply, :ok, socket}
 
       {:error, reason} ->
@@ -82,6 +84,7 @@ defmodule CF.RestApi.StatementsChannel do
     |> case do
       {:ok, _} ->
         broadcast!(socket, "statement_removed", %{id: id})
+        CF.Algolia.StatementsIndex.delete_object(statement)
         {:reply, :ok, socket}
 
       {:error, _, _reason, _} ->

apps/cf_rest_api/lib/channels/video_debate_channel.ex

@@ -109,6 +109,7 @@ defmodule CF.RestApi.VideoDebateChannel do
       {:ok, %{}} ->
         rendered_speaker = SpeakerView.render("show.json", speaker: speaker)
         broadcast!(socket, "speaker_added", rendered_speaker)
+        CF.Algolia.VideosIndex.reindex_by_id(video_id)
         {:reply, :ok, socket}
 
       {:error, _, %{errors: errors}, _} ->
@@ -146,6 +147,8 @@ defmodule CF.RestApi.VideoDebateChannel do
         # Broadcast the speaker
         rendered_speaker = SpeakerView.render("show.json", speaker: speaker)
         broadcast!(socket, "speaker_added", rendered_speaker)
+        CF.Algolia.VideosIndex.reindex_by_id(video_id)
+        CF.Algolia.SpeakersIndex.save_object(speaker)
         {:reply, :ok, socket}
 
       {:error, :speaker, changeset, %{}} ->
@@ -179,6 +182,9 @@ defmodule CF.RestApi.VideoDebateChannel do
 
             rendered_speaker = View.render_one(speaker, SpeakerView, "speaker.json")
             broadcast!(socket, "speaker_updated", rendered_speaker)
+            CF.Algolia.SpeakersIndex.save_object(speaker)
+            CF.Algolia.VideosIndex.reindex_all_speaker_videos(speaker.id)
+            CF.Algolia.StatementsIndex.reindex_all_speaker_statements(speaker.id)
             {:reply, :ok, socket}
 
           {:error, :speaker, changeset = %Ecto.Changeset{}, _} ->
@@ -194,10 +200,12 @@ defmodule CF.RestApi.VideoDebateChannel do
   def handle_in_authenticated!("remove_speaker", %{"id" => id}, socket) do
     speaker = Repo.get!(Speaker, id)
     do_remove_speaker(socket, speaker)
+    CF.Algolia.VideosIndex.reindex_by_id(socket.assigns.video_id)
     broadcast!(socket, "speaker_removed", %{id: id})
     {:reply, :ok, socket}
   end
 
+  # TODO: This should be a GraphQL query rather than a socket event
   @max_speakers_search_results 5
   def handle_in_authenticated!("search_speaker", params, socket) do
     query = "%#{params["query"]}%"

apps/cf_rest_api/lib/channels/video_debate_history_channel.ex

@@ -80,6 +80,7 @@ defmodule CF.RestApi.VideoDebateHistoryChannel do
           StatementView.render("show.json", statement: statement)
         )
 
+        CF.Algolia.StatementsIndex.save_object(statement)
         {:reply, :ok, socket}
 
       {:error, _, reason, _} ->
@@ -116,6 +117,7 @@ defmodule CF.RestApi.VideoDebateHistoryChannel do
           SpeakerView.render("show.json", speaker: speaker)
         )
 
+        CF.Algolia.VideosIndex.reindex_by_id(video_id)
         {:reply, :ok, socket}
 
       {:error, _reason} ->

apps/cf_rest_api/test/support/channel_case.ex

@@ -14,6 +14,14 @@ defmodule CF.RestApi.ChannelCase do
   """
 
   use ExUnit.CaseTemplate
+  import Mock
+
+  # Mock all calls to Algolia
+  setup_with_mocks([
+    {Algoliax.Client, [], [request: fn _ -> nil end]}
+  ]) do
+    :ok
+  end
 
   using do
     quote do

apps/db/mix.exs

@@ -56,6 +56,7 @@ defmodule DB.Mixfile do
       {:mime, "~> 1.2"},
       {:scrivener_ecto, "~> 2.0"},
       {:scout_apm, "~> 1.0.6"},
+      {:algoliax, "~> 0.5.0"},
 
       # Dev only
       {:exsync, "~> 0.2", only: :dev},

config/config.exs

@@ -1,3 +1,4 @@
 use Mix.Config
 
 import_config "../apps/*/config/config.exs"
+import_config "./*.secret.exs" # TODO should filter by env

config/releases.exs

@@ -82,10 +82,15 @@ config :ex_aws,
 config :arc,
   bucket: load_secret.("s3_bucket")
 
+config :algoliax,
+  application_id: load_secret("algolia_app_id"),
+  api_key: load_secret("algolia_api_key")
+
 # ---- [APP CONFIG] :cf ----
 
 config :cf,
   frontend_url: frontend_url,
+  deploy_env: load_secret.("deploy_env"),
   soft_limitations_period: load_int.({"soft_limitations_period", 15 * 60}),
   hard_limitations_period: load_int.({"hard_limitations_period", 3 * 60 * 60}),
   invitation_system: load_bool.({"invitation_system", "false"}),