Merge pull request #3165 from nulib/deploy/staging

Deploy v6.1.2 to production
nulib · Sep 9, 2022 · 7d70003 · 7d70003
2 parents da68340 + b2c743b
commit 7d70003
Show file tree

Hide file tree

Showing 19 changed files with 146 additions and 32 deletions.
diff --git a/app/config/config.exs b/app/config/config.exs
@@ -251,6 +251,11 @@ config :meadow, :extra_mime_types, %{
   "xml" => "application/xml"
 }
 
+config :meadow,
+  dc_api: [
+    v2: aws_secret("meadow", dig: ["dc_api", "v2"], default: "http://localhost:3000")
+  ]
+
 config :hush,
   transformers_override: true,
   transformers: [

diff --git a/app/config/test.exs b/app/config/test.exs
@@ -107,6 +107,13 @@ config :meadow, Meadow.Repo,
   queue_target: 5000,
   pool_size: 50
 
+config :meadow,
+  dc_api: [
+    v2: %{
+      "base_url" => "http://dcapi-test.northwestern.edu"
+    }
+  ]
+
 if System.get_env("AWS_DEV_ENVIRONMENT") |> is_nil() do
   [:mediaconvert, :s3, :secretsmanager, :sns, :sqs]
   |> Enum.each(fn service ->

diff --git a/app/lib/meadow/file_set_delete_listener.ex b/app/lib/meadow/file_set_delete_listener.ex
@@ -29,10 +29,11 @@ defmodule Meadow.FilesetDeleteListener do
 
   @impl GenServer
   def init(initial_state) do
-    Logger.info("#{__MODULE__}: Listening for delete notifications on 'file_sets'")
-    Meadow.Repo.listen(@message)
-
-    {:ok, initial_state}
+    with repo <- Keyword.get(initial_state, :repo, Meadow.Repo) do
+      Logger.info("#{__MODULE__}: Listening for delete notifications on 'file_sets'")
+      repo.listen(@message)
+      {:ok, Keyword.put(initial_state, :ingest_bucket, Meadow.Config.ingest_bucket())}
+    end
   end
 
   def handle_info({:notification, _pid, _ref, @message, payload}, state) do
@@ -90,13 +91,19 @@ defmodule Meadow.FilesetDeleteListener do
   defp clean_derivative!(_, _), do: :ok
 
   defp clean_preservation_file!(file_set_data, state) do
-    with location <- file_set_data.location,
-         repo <- Keyword.get(state, :repo, Meadow.Repo) do
+    clean_preservation_file!(file_set_data, state, in_ingest_bucket(file_set_data, state))
+    file_set_data
+  end
+
+  defp clean_preservation_file!(%{location: location}, _state, true) do
+    Logger.warn("Leaving #{location} intact in the ingest bucket")
+  end
+
+  defp clean_preservation_file!(%{id: id, location: location}, state, _) do
+    with repo <- Keyword.get(state, :repo, Meadow.Repo) do
       refs =
         from(f in FileSet,
-          where:
-            fragment("core_metadata ->> 'location' = ?", ^location) and
-              f.id != ^file_set_data.id
+          where: fragment("core_metadata ->> 'location' = ?", ^location) and f.id != ^id
         )
         |> repo.aggregate(:count)
 
@@ -108,8 +115,10 @@ defmodule Meadow.FilesetDeleteListener do
         Logger.warn("Leaving #{location} intact: #{refs} additional #{references}")
       end
     end
+  end
 
-    file_set_data
+  defp in_ingest_bucket(%{location: location}, state) do
+    URI.parse(location) |> Map.get(:host) == Keyword.get(state, :ingest_bucket)
   end
 
   defp delete_s3_uri(uri, recursive \\ false)

diff --git a/app/lib/meadow/indexing/work.ex b/app/lib/meadow/indexing/work.ex
@@ -29,6 +29,7 @@ defimpl Elasticsearch.Document, for: Meadow.Data.Schemas.Work do
             extractedMetadata: FileSetDocument.extracted_metadata(file_set.extracted_metadata),
             label: file_set.core_metadata.label,
             mime_type: file_set.core_metadata.mime_type,
+            original_filename: file_set.core_metadata.original_filename,
             posterOffset: file_set.poster_offset,
             rank: file_set.rank,
             representativeImageUrl: FileSets.representative_image_url_for(file_set),

diff --git a/app/lib/meadow/utils/mime.ex b/app/lib/meadow/utils/mime.ex
@@ -39,7 +39,9 @@ defmodule Meadow.Utils.MIME do
   "application/octet-stream"
   """
   def type(ext) do
-    Application.get_env(:meadow, :extra_mime_types)
-    |> Map.get(to_string(ext), MimeTypes.type(ext))
+    with normalized_ext <- ext |> to_string() |> String.downcase() do
+      Application.get_env(:meadow, :extra_mime_types)
+      |> Map.get(normalized_ext, MimeTypes.type(normalized_ext))
+    end
   end
 end
diff --git a/app/lib/mix/tasks/elasticsearch.ex b/app/lib/mix/tasks/elasticsearch.ex
@@ -54,7 +54,8 @@ defmodule Mix.Tasks.Meadow.Elasticsearch.Pipelines do
   end
 
   defp create(path, ".json") do
-    [filename | [pipeline_prefix | _]] = Path.split(path) |> Enum.reverse()
+    [filename | [pipeline_prefix | ["pipelines" | [version | _]]]] =
+      Path.split(path) |> Enum.reverse()
 
     environment_prefix =
       [Application.get_env(:meadow, :environment_prefix), pipeline_prefix]
@@ -68,6 +69,7 @@ defmodule Mix.Tasks.Meadow.Elasticsearch.Pipelines do
 
     body =
       File.read!(path)
+      |> interpolate_api_variables(version)
       |> String.replace(pipeline_prefix, environment_prefix)
 
     url =
@@ -142,6 +144,17 @@ defmodule Mix.Tasks.Meadow.Elasticsearch.Pipelines do
 
     Elastix.HTTP.put(url, body, [{"Content-Type", "application/json"}], [])
   end
+
+  defp interpolate_api_variables(content, version) when is_binary(version),
+    do: interpolate_api_variables(content, String.to_existing_atom(version))
+
+  defp interpolate_api_variables(content, version) do
+    Application.get_env(:meadow, :dc_api)
+    |> Keyword.get(version, [])
+    |> Enum.reduce(content, fn {variable, value}, result ->
+      result |> String.replace("$#{variable}$", value)
+    end)
+  end
 end
 
 defmodule Mix.Tasks.Meadow.Elasticsearch.Teardown do

diff --git a/app/mix.exs b/app/mix.exs
@@ -3,7 +3,7 @@ Code.require_file("lib/env.ex")
 defmodule Meadow.MixProject do
   use Mix.Project
 
-  @app_version "6.1.1"
+  @app_version "6.1.2"
 
   def project do
     [

diff --git a/app/priv/elasticsearch/v2/pipelines/v1-to-v2/collection.json b/app/priv/elasticsearch/v2/pipelines/v1-to-v2/collection.json
@@ -10,7 +10,7 @@
     {
       "set": {
         "field": "api_link",
-        "value": "[PLACEHOLDER]/{{id}}"
+        "value": "$base_url$/collections/{{id}}"
       }
     },
     {

diff --git a/app/priv/elasticsearch/v2/pipelines/v1-to-v2/file_set.json b/app/priv/elasticsearch/v2/pipelines/v1-to-v2/file_set.json
@@ -7,6 +7,12 @@
         "value": "FileSet"
       }
     },
+    {
+      "set": {
+        "field": "api_link",
+        "value": "$base_url$/file-sets/{{id}}"
+      }
+    },
     {
       "rename": {
         "field": "accessionNumber",

diff --git a/app/priv/elasticsearch/v2/pipelines/v1-to-v2/work-file-sets.groovy b/app/priv/elasticsearch/v2/pipelines/v1-to-v2/work-file-sets.groovy
@@ -6,6 +6,7 @@ ctx.file_sets = ctx.original.fileSets
     result.id = fs.id;
     result.label = fs.label;
     result.mime_type = fs.mime_type;
+    result.original_filename = fs.original_filename;
     result.poster_offset = fs.posterOffset;
     result.rank = fs.rank;
     result.representative_image_url = fs.representativeImageUrl;

diff --git a/app/priv/elasticsearch/v2/pipelines/v1-to-v2/work.json b/app/priv/elasticsearch/v2/pipelines/v1-to-v2/work.json
@@ -33,7 +33,13 @@
     {
       "set": {
         "field": "api_link",
-        "value": "[PLACEHOLDER]/{{original.id}}"
+        "value": "$base_url$/works/{{original.id}}"
+      }
+    },
+    {
+      "set": {
+        "field": "thumbnail",
+        "value": "$base_url$/works/{{original.id}}/thumbnail"
       }
     },
     {
@@ -42,6 +48,12 @@
         "target_field": "ark"
       }
     },
+    {
+      "rename": {
+        "field": "original.batches",
+        "target_field": "batch_ids"
+      }
+    },
     {
       "rename": {
         "field": "original.descriptiveMetadata.boxName",
@@ -82,7 +94,8 @@
               "id": "{{_ingest._value.term.id}}",
               "label": "{{_ingest._value.term.label}}",
               "role": "{{_ingest._value.role.label}}",
-              "label_with_role": "{{_ingest._value.term.label}} ({{_ingest._value.role.label}})"
+              "label_with_role": "{{_ingest._value.term.label}} ({{_ingest._value.role.label}})",
+              "facet": "{{_ingest._value.facet}}"
             }
           }
         }
@@ -102,7 +115,8 @@
             "field": "creator",
             "value": {
               "id": "{{_ingest._value.term.id}}",
-              "label": "{{_ingest._value.term.label}}"
+              "label": "{{_ingest._value.term.label}}",
+              "facet": "{{_ingest._value.facet}}"
             }
           }
         }
@@ -113,6 +127,18 @@
         "id": "v1-to-v2-work-file-sets"
       }
     },
+    {
+      "rename": {
+        "field": "original.descriptiveMetadata.dateCreated",
+        "target_field": "date_created"
+      }
+    },
+    {
+      "rename": {
+        "field": "original.descriptiveMetadata.description",
+        "target_field": "description"
+      }
+    },
     {
       "rename": {
         "field": "original.descriptiveMetadata.folderName",
@@ -133,7 +159,8 @@
             "field": "genre",
             "value": {
               "id": "{{_ingest._value.term.id}}",
-              "label": "{{_ingest._value.term.label}}"
+              "label": "{{_ingest._value.term.label}}",
+              "facet": "{{_ingest._value.facet}}"
             }
           }
         }
@@ -201,6 +228,12 @@
         "ignore_failure": true
       }
     },
+    {
+      "rename": {
+        "field": "original.metadataUpdateJobs",
+        "target_field": "csv_metadata_update_jobs"
+      }
+    },
     {
       "rename": {
         "field": "original.modifiedDate",
@@ -351,7 +384,8 @@
               "id": "{{_ingest._value.term.id}}",
               "label": "{{_ingest._value.term.label}}",
               "role": "{{_ingest._value.role.label}}",
-              "label_with_role": "{{_ingest._value.term.label}} ({{_ingest._value.role.label}})"
+              "label_with_role": "{{_ingest._value.term.label}} ({{_ingest._value.role.label}})",
+              "facet": "{{_ingest._value.facet}}"
             }
           }
         }
@@ -365,7 +399,8 @@
             "field": "style_period",
             "value": {
               "id": "{{_ingest._value.term.id}}",
-              "label": "{{_ingest._value.term.label}}"
+              "label": "{{_ingest._value.term.label}}",
+              "facet": "{{_ingest._value.facet}}"
             }
           }
         }
@@ -385,7 +420,8 @@
             "field": "technique",
             "value": {
               "id": "{{_ingest._value.term.id}}",
-              "label": "{{_ingest._value.term.label}}"
+              "label": "{{_ingest._value.term.label}}",
+              "facet": "{{_ingest._value.facet}}"
             }
           }
         }
@@ -397,12 +433,6 @@
         "target_field": "terms_of_use"
       }
     },
-    {
-      "rename": {
-        "field": "original.thumbnail",
-        "target_field": "thumbnail"
-      }
-    },
     {
       "rename": {
         "field": "original.descriptiveMetadata.title",

diff --git a/app/priv/elasticsearch/v2/settings/work.json b/app/priv/elasticsearch/v2/settings/work.json
@@ -53,7 +53,7 @@
     "dynamic_templates": [
       {
         "text_fields": {
-          "match": "^abstract$|^alternate_title$|^caption$|^collection.title$|^cultural_context$|^description$|^file_sets.label$|^file_sets.description$|^notes.text$|^table_of_contents$|^title$",
+          "match": "^abstract$|^alternate_title$|^caption$|^collection.title$|^cultural_context$|^description$|^file_sets.label$|^file_sets.description$|^file_sets.original_filename$|^notes.text$|^table_of_contents$|^title$",
           "match_pattern": "regex",
           "mapping": {
             "type": "text",

diff --git a/app/test/meadow/data/indexer_test.exs b/app/test/meadow/data/indexer_test.exs
@@ -271,7 +271,7 @@ defmodule Meadow.Data.IndexerTest do
       assert doc |> get_in(["model", "application"]) == "Meadow"
       assert doc |> get_in(["model", "name"]) == "Work"
       assert doc |> get_in(["fileSets"]) |> length == 2
-      assert doc |> get_in(["fileSets"]) |> List.first() |> map_size() == 10
+      assert doc |> get_in(["fileSets"]) |> List.first() |> map_size() == 11
 
       with metadata <- subject.descriptive_metadata do
         assert doc |> get_in(["descriptiveMetadata", "title"]) ==

diff --git a/app/test/meadow/file_set_delete_listener_test.exs b/app/test/meadow/file_set_delete_listener_test.exs
@@ -9,6 +9,8 @@ defmodule Meadow.FileSetDeleteListenerTest do
   import Ecto.Changeset
   import ExUnit.CaptureLog
 
+  @ingest_key "waiting/to/ingest.tif"
+  @ingest_location "s3://#{@ingest_bucket}/#{@ingest_key}"
   @pairtree "44/87/89/28/-c/4d/e-/40/76/-b/61/3-/15/35/66/5f/fa/e2"
   @poster_key "posters/#{@pairtree}-poster.tif"
   @preservation_key "#{@pairtree}/preservation-file"
@@ -43,6 +45,11 @@ defmodule Meadow.FileSetDeleteListenerTest do
 
   describe "clean_up!/1" do
     @describetag s3: [
+                   %{
+                     bucket: @ingest_bucket,
+                     key: @ingest_key,
+                     content: File.read!("test/fixtures/coffee.tif")
+                   },
                    %{
                      bucket: @preservation_bucket,
                      key: @preservation_key,
@@ -114,6 +121,18 @@ defmodule Meadow.FileSetDeleteListenerTest do
       end
     end
 
+    test "file retained if it's in the ingest bucket", %{file_set: file_set} do
+      with %{id: file_set_id} <- file_set do
+        file_set
+        |> FileSet.update_changeset(%{core_metadata: %{location: @ingest_location}})
+        |> Repo.update!()
+        |> Repo.delete()
+
+        assert_receive {"cleaned", ^file_set_id}, @timeout
+        assert object_exists?(@ingest_bucket, @ingest_key)
+      end
+    end
+
     test "bulk delete", %{file_set: file_set} do
       file_sets = [file_set | Enum.map(2..10, &file_set_notification_fixture/1)]
       SQL.query!(Repo, "DELETE FROM file_sets")

diff --git a/app/test/meadow/utils/mime_test.exs b/app/test/meadow/utils/mime_test.exs
@@ -19,4 +19,8 @@ defmodule Meadow.Utils.MIMETest do
     assert MIME.from_path("/path/to/unknown.blorb") == "application/octet-stream"
     assert MIME.type("blorb") == "application/octet-stream"
   end
+
+  test "is case insensitive" do
+    assert MIME.from_path("/path/to/image.jpg") == MIME.from_path("/path/to/image.JPG")
+  end
 end
diff --git a/app/test/support/unsandboxed_data_case.ex b/app/test/support/unsandboxed_data_case.ex
@@ -14,6 +14,8 @@ defmodule Meadow.UnsandboxedDataCase do
   defmodule Repo do
     @moduledoc false
     use Ecto.Repo, adapter: Ecto.Adapters.Postgres, otp_app: :meadow
+
+    def listen(event_name), do: Meadow.Repo.listen(event_name)
   end
 
   setup_all do