Merge pull request #3405 from rebeccacremona/check-resolved-url

Check `targetUrlResolved` against `PRIVATE_BY_POLICY_DOMAINS`
harvard-lil · Oct 11, 2023 · da0f003 · da0f003
2 parents 0b46a30 + fa938e9
commit da0f003
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 11 deletions.
diff --git a/docker-compose.override.yml b/docker-compose.override.yml
@@ -49,7 +49,7 @@ services:
       - scoop_rest_api_internal
 
   scoop-rest-api:
-    image: registry.lil.tools/harvardlil/scoop-rest-api:13-7077f40b8607621df9827f31bba4e7d4
+    image: registry.lil.tools/harvardlil/scoop-rest-api:15-36faf86818fcb2094e47cfe37c443515
     init: true
     tty: true
     depends_on:

diff --git a/perma_web/api/urls.py b/perma_web/api/urls.py
@@ -101,8 +101,8 @@ def handler500(request):
     urlpatterns += [re_path(r'^__debug__/', include(debug_toolbar.urls))]
 
 # views that only load when running our tests:
-# if settings.TESTING:
-from .tests import views as test_views
-urlpatterns += [
-    re_path(r'tests/redirect-to-file$', test_views.redirect_to_file, name='redirect_to_file')
-]
+if settings.TESTING:
+    from .tests import views as test_views
+    urlpatterns += [
+        re_path(r'tests/redirect-to-file$', test_views.redirect_to_file, name='redirect_to_file')
+    ]
diff --git a/perma_web/perma/celery_tasks.py b/perma_web/perma/celery_tasks.py
@@ -847,11 +847,11 @@ def save_scoop_capture(link, capture_job, data):
     ])
 
     # Make this link private by policy, if the captured domain is on the list.
-    # For now, just check the primary URL, since Scoop does not yet expose the URL
-    # that the capture request resolved to.
-    content_url = data['scoop_capture_summary']['exchangeUrls'][0]
-    if any(domain in content_url for domain in settings.PRIVATE_BY_POLICY_DOMAINS):
-        safe_save_fields(link, is_private=True, private_reason='domain')
+    target_url = data['scoop_capture_summary']['targetUrl']
+    content_url = data['scoop_capture_summary']['targetUrlResolved']
+    for url in [target_url, content_url]:
+        if any(domain in url for domain in settings.PRIVATE_BY_POLICY_DOMAINS):
+            safe_save_fields(link, is_private=True, private_reason='domain')
 
     # See if the primary URL has been munged in any way since we last saw it.
     if link.primary_capture.url != data['scoop_capture_summary']['exchangeUrls'][0]: