Release 0.61.2

RELEASE.rst

@@ -1,6 +1,12 @@
 Release Notes
 =============
 
+Version 0.61.2
+--------------
+
+- Customizable URLs for studio (#1316)
+- improve legacy shortcode handling (#1349)
+
 Version 0.61.1 (Released May 17, 2022)
 --------------
 

content_sync/api.py

@@ -80,12 +80,6 @@ def create_website_backend(website: Website):
     tasks.create_website_backend.delay(website.name)
 
 
-@is_publish_pipeline_enabled
-def create_website_publishing_pipeline(website: Website):
-    """ Create the publish pipeline for a website"""
-    tasks.upsert_website_publishing_pipeline.delay(website.name)
-
-
 @is_sync_enabled
 def update_website_backend(website: Website):
     """ Update the backend content for a website"""
@@ -136,6 +130,10 @@ def publish_website(  # pylint: disable=too-many-arguments
 
         if trigger_pipeline and settings.CONTENT_SYNC_PIPELINE_BACKEND:
             pipeline = get_sync_pipeline(website, api=pipeline_api)
+            # Always upsert pipeline in case the site url path changed,
+            # unless it's been published to production (url path permanent).
+            if not website.publish_date:
+                pipeline.upsert_pipeline()
             pipeline.unpause_pipeline(version)
             build_id = pipeline.trigger_pipeline_build(version)
             update_kwargs = {

content_sync/api_test.py

@@ -194,34 +194,13 @@ def test_get_sync_pipeline(settings, mocker, pipeline_api):
     import_string_mock.assert_any_call(website, api=pipeline_api)
 
 
-def test_create_website_publishing_pipeline(settings, mocker):
-    """upsert_website_publishing_pipeline task should be called if pipelines are enabled"""
-    settings.CONTENT_SYNC_PIPELINE_BACKEND = "concourse"
-    mock_task = mocker.patch(
-        "content_sync.api.tasks.upsert_website_publishing_pipeline.delay"
-    )
-    website = WebsiteFactory.create()
-    api.create_website_publishing_pipeline(website)
-    mock_task.assert_called_once_with(website.name)
-
-
-def test_create_website_publishing_pipeline_disabled(settings, mocker):
-    """upsert_website_publishing_pipeline task should not be called if pipelines are disabled"""
-    settings.CONTENT_SYNC_PIPELINE_BACKEND = None
-    mock_task = mocker.patch(
-        "content_sync.api.tasks.upsert_website_publishing_pipeline.delay"
-    )
-    website = WebsiteFactory.create()
-    api.create_website_publishing_pipeline(website)
-    mock_task.assert_not_called()
-
-
 @pytest.mark.parametrize("prepublish", [True, False])
 @pytest.mark.parametrize("prepublish_actions", [[], ["some.Action"]])
 @pytest.mark.parametrize("has_api", [True, False])
 @pytest.mark.parametrize("version", [VERSION_LIVE, VERSION_DRAFT])
 @pytest.mark.parametrize("status", [None, PUBLISH_STATUS_NOT_STARTED])
 @pytest.mark.parametrize("trigger", [True, False])
+@pytest.mark.parametrize("publish_date", [None, now_in_utc()])
 def test_publish_website(  # pylint:disable=redefined-outer-name,too-many-arguments
     settings,
     mocker,
@@ -232,10 +211,11 @@ def test_publish_website(  # pylint:disable=redefined-outer-name,too-many-argume
     version,
     status,
     trigger,
+    publish_date,
 ):
     """Verify that the appropriate backend calls are made by the publish_website function"""
     settings.PREPUBLISH_ACTIONS = prepublish_actions
-    website = WebsiteFactory.create()
+    website = WebsiteFactory.create(publish_date=publish_date)
     setattr(website, f"{version}_publish_status", status)
     if status:
         setattr(website, f"{version}_publish_status_updated_on", now_in_utc())
@@ -263,6 +243,7 @@ def test_publish_website(  # pylint:disable=redefined-outer-name,too-many-argume
         mock_api_funcs.mock_get_pipeline.assert_called_once_with(
             website, api=pipeline_api
         )
+        assert pipeline.upsert_pipeline.call_count == (0 if publish_date else 1)
         pipeline.trigger_pipeline_build.assert_called_once_with(version)
         pipeline.unpause_pipeline.assert_called_once_with(version)
         assert getattr(website, f"latest_build_id_{version}") == build_id

content_sync/pipelines/concourse.py

@@ -30,7 +30,6 @@
 from content_sync.utils import check_mandatory_settings
 from websites.constants import OCW_HUGO_THEMES_GIT, STARTER_SOURCE_GITHUB
 from websites.models import Website
-from websites.site_config_api import SiteConfig
 
 
 log = logging.getLogger(__name__)
@@ -273,14 +272,12 @@ def upsert_pipeline(self):  # pylint:disable=too-many-locals
             # Invalid github url, so skip
             return
 
-        site_config = SiteConfig(self.website.starter.config)
-        site_url = f"{site_config.root_url_path}/{self.website.name}".strip("/")
         if self.website.name == settings.ROOT_WEBSITE_NAME:
             base_url = ""
             theme_created_trigger = "true"
             theme_deployed_trigger = "false"
         else:
-            base_url = site_url
+            base_url = self.website.get_url_path()
             theme_created_trigger = "false"
             theme_deployed_trigger = "true"
         hugo_projects_url = urljoin(
@@ -342,8 +339,9 @@ def upsert_pipeline(self):  # pylint:disable=too-many-locals
                     .replace("((ocw-site-repo))", self.website.short_id)
                     .replace("((ocw-site-repo-branch))", branch)
                     .replace("((config-slug))", self.website.starter.slug)
+                    .replace("((s3-path))", self.website.s3_path)
                     .replace("((base-url))", base_url)
-                    .replace("((site-url))", site_url)
+                    .replace("((site-url))", self.website.get_url_path())
                     .replace("((site-name))", self.website.name)
                     .replace("((purge-url))", f"purge/{self.website.name}")
                     .replace("((purge_header))", purge_header)

content_sync/pipelines/concourse_test.py

@@ -141,10 +141,13 @@ def test_upsert_website_pipelines(
     if home_page:
         name = settings.ROOT_WEBSITE_NAME
         starter.config["root-url-path"] = ""
+        site_path = None
     else:
         name = "standard-course"
         starter.config["root-url-path"] = "courses"
-    website = WebsiteFactory.create(starter=starter, name=name)
+        site_path = "courses/my-site-fall-2020"
+
+    website = WebsiteFactory.create(starter=starter, name=name, url_path=site_path)
 
     instance_vars = f"%7B%22site%22%3A%20%22{website.name}%22%7D"
     url_path = f"/api/v1/teams/{settings.CONCOURSE_TEAM}/pipelines/{version}/config?vars={instance_vars}"
@@ -188,6 +191,7 @@ def test_upsert_website_pipelines(
     assert settings.OCW_GTM_ACCOUNT_ID in config_str
     assert settings.OCW_IMPORT_STARTER_SLUG in config_str
     assert api_url in config_str
+
     if home_page:
         assert (
             f"s3 sync s3://{settings.AWS_STORAGE_BUCKET_NAME}/{website.name} s3://{bucket}/{website.name}"
@@ -196,11 +200,11 @@ def test_upsert_website_pipelines(
         assert f"aws s3 sync course-markdown/public s3://{bucket}/" in config_str
     else:
         assert (
-            f"s3 sync s3://{settings.AWS_STORAGE_BUCKET_NAME}/courses/{website.name} s3://{bucket}/courses/{website.name}"
+            f"s3 sync s3://{settings.AWS_STORAGE_BUCKET_NAME}/courses/{website.name} s3://{bucket}/{website.url_path}"
             in config_str
         )
         assert (
-            f"aws s3 sync course-markdown/public s3://{bucket}/courses/{website.name}"
+            f"aws s3 sync course-markdown/public s3://{bucket}/{website.url_path}"
             in config_str
         )
     assert f"purge/{website.name}" in config_str
@@ -392,10 +396,10 @@ def test_upsert_pipeline(settings, mocker, mock_auth, pipeline_exists):
 
 @pytest.mark.parametrize("pipeline_exists", [True, False])
 @pytest.mark.parametrize("version", [VERSION_DRAFT, VERSION_LIVE])
-def test_upsert_mass_publish_pipeline(
+def test_upsert_mass_build_pipeline(
     settings, pipeline_settings, mocker, mock_auth, pipeline_exists, version
 ):  # pylint:disable=too-many-locals,too-many-arguments
-    """The mass publish pipeline should have expected configuration"""
+    """The mass build pipeline should have expected configuration"""
     hugo_projects_path = "https://github.com/org/repo"
     WebsiteFactory.create(
         starter=WebsiteStarterFactory.create(
@@ -437,6 +441,10 @@ def test_upsert_mass_publish_pipeline(
         api_url = settings.OCW_STUDIO_LIVE_URL
     config_str = json.dumps(kwargs)
     assert settings.OCW_GTM_ACCOUNT_ID in config_str
+    assert (
+        f"s3://{settings.AWS_STORAGE_BUCKET_NAME}/$S3_PATH s3://{bucket}/$SITE_URL"
+        in config_str
+    )
     assert bucket in config_str
     assert version in config_str
     assert f"{hugo_projects_path}.git" in config_str

content_sync/pipelines/definitions/concourse/mass-build-sites.yml

@@ -105,14 +105,19 @@ jobs:
                 NAME=$(echo $1 | jq -c '.name' | tr -d '"')
                 SHORT_ID=$(echo $1 | jq -c '.short_id' | tr -d '"')
                 STARTER_SLUG=$(echo $1| jq -c '.starter_slug' | tr -d '"')
+                S3_PATH=$(echo $1 | jq -c '.s3_path' | tr -d '"')
                 SITE_URL=$(echo $1 | jq -c '.site_url' | tr -d '"')
                 BASE_URL=$(echo $1 | jq -c '.base_url' | tr -d '"')
+                if [[ "$SITE_URL" == null || "$S3_PATH" == null ]]
+                then
+                  return 1
+                fi
                 git -c core.sshCommand="ssh $GITKEYSSH -o StrictHostKeyChecking=no" clone -b ((ocw-site-repo-branch)) ((markdown-uri))/$SHORT_ID.git || return 1
                 cd $CURDIR/$SHORT_ID
                 cp ../webpack-json/webpack.json ../ocw-hugo-themes/base-theme/data
                 hugo --config ../ocw-hugo-projects/$STARTER_SLUG/config.yaml --baseUrl /$BASE_URL --themesDir ../ocw-hugo-themes/ ((build-drafts))  || return 1
                 cd $CURDIR
-                aws s3 sync s3://((ocw-studio-bucket))/$SITE_URL s3://((ocw-bucket))/$SITE_URL --metadata site-id=$NAME || return 1            
+                aws s3 sync s3://((ocw-studio-bucket))/$S3_PATH s3://((ocw-bucket))/$SITE_URL --metadata site-id=$NAME || return 1            
                 aws s3 sync $SHORT_ID/public s3://((ocw-bucket))/$BASE_URL --metadata site-id=$NAME || return 1
                 curl -X POST -H 'Content-Type: application/json' --data '{"webhook_key":"((open-webhook-key))","prefix":"'"$SITE_URL"/'","version":"((version))"}' ((open-discussions-url))/api/v0/ocw_next_webhook/
                 curl -X POST -H 'Content-Type: application/json' -H 'Authorization: Bearer ((api-token))' --data '{"version":"((version))","status":"succeeded"}' ((ocw-studio-url))/api/websites/$NAME/pipeline_status/

content_sync/pipelines/definitions/concourse/site-pipeline.yml

@@ -178,7 +178,7 @@ jobs:
             args:
             - -exc
             - |
-              aws s3 sync s3://((ocw-studio-bucket))/((site-url)) s3://((ocw-bucket))/((site-url)) --metadata site-id=((site-name))              
+              aws s3 sync s3://((ocw-studio-bucket))/((s3-path)) s3://((ocw-bucket))/((site-url)) --metadata site-id=((site-name))              
               aws s3 sync course-markdown/public s3://((ocw-bucket))/((base-url)) --metadata site-id=((site-name))
         on_failure:
           try:

content_sync/serializers.py

@@ -113,7 +113,11 @@ def deserialize(  # pylint:disable=too-many-locals
                 omitted_keys.append(file_field["name"])
                 file_url = front_matter_data.get(file_field["name"], None)
                 if file_url is not None:
+                    s3_path = website.s3_path
+                    url_path = website.url_path
                     file_url = urlparse(file_url).path.lstrip("/")
+                    if url_path and s3_path != url_path:
+                        file_url = file_url.replace(url_path, s3_path, 1)
 
         base_defaults = {
             "metadata": dict_without_keys(front_matter_data, *omitted_keys),

content_sync/serializers_test.py

@@ -4,6 +4,7 @@
 
 import pytest
 import yaml
+from django.core.files.uploadedfile import SimpleUploadedFile
 from moto import mock_s3
 
 from content_sync.serializers import (
@@ -16,6 +17,7 @@
     deserialize_file_to_website_content,
     serialize_content_to_file,
 )
+from websites.constants import WEBSITE_CONFIG_ROOT_URL_PATH_KEY
 from websites.factories import (
     WebsiteContentFactory,
     WebsiteFactory,
@@ -43,7 +45,7 @@
 title: Example File
 content_type: resource
 uid: abcdefg
-image: https://test.edu/courses/website_name/image.png
+image: https://test.edu/courses/website_name-fall-2025/image.png
 ---
 # My markdown
 - abc
@@ -91,21 +93,26 @@ def get_example_menu_data():
     ]
 
 
+@mock_s3
 @pytest.mark.django_db
 @pytest.mark.parametrize(
     "markdown, exp_sections", [["# Some markdown...\n- and\n- a\n- list", 2], [None, 1]]
 )
-def test_hugo_file_serialize(markdown, exp_sections):
+def test_hugo_file_serialize(settings, markdown, exp_sections):
     """HugoMarkdownFileSerializer.serialize should create the expected file contents"""
+    settings.OCW_STUDIO_USE_S3 = True
     metadata = {"metadata1": "dummy value 1", "metadata2": "dummy value 2"}
     content = WebsiteContentFactory.create(
         text_id="abcdefg",
         title="Content Title",
-        type="sometype",
+        type="resource",
         markdown=markdown,
         metadata=metadata,
+        file=SimpleUploadedFile("mysite/test.pdf", b"content"),
+        website=WebsiteFactory.create(name="mysite", url_path="sites/mysite-fall-2025"),
     )
     site_config = SiteConfig(content.website.starter.config)
+
     file_content = HugoMarkdownFileSerializer(site_config).serialize(
         website_content=content
     )
@@ -127,6 +134,10 @@ def test_hugo_file_serialize(markdown, exp_sections):
         ]
         + [f"{k}: {v}" for k, v in metadata.items()]
     )
+    assert (
+        f"image: /media/{content.website.url_path}/{content.file.name.split('/')[-1]}"
+        in front_matter_lines
+    )
     if exp_sections > 1:
         assert md_file_sections[1] == markdown
 
@@ -222,19 +233,20 @@ def test_hugo_menu_yaml_deserialize(omnibus_config):
 def test_hugo_file_deserialize_with_file(settings):
     """HugoMarkdownFileSerializer.deserialize should create the expected content object from some file contents"""
     settings.DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
-    website = WebsiteFactory.create()
+    website = WebsiteFactory.create(url_path="courses/website_name-fall-2025")
+    website.starter.config[WEBSITE_CONFIG_ROOT_URL_PATH_KEY] = "courses"
     site_config = SiteConfig(website.starter.config)
     website_content = HugoMarkdownFileSerializer(site_config).deserialize(
         website=website,
         filepath="/test/file.md",
         file_contents=EXAMPLE_HUGO_MARKDOWN_WITH_FILE,
     )
-    path = "courses/website_name/image.png"
+    expected_path = f"courses/{website.name}/image.png"
     assert "image" not in website_content.metadata.keys()
-    assert website_content.file == path
+    assert website_content.file == expected_path
     assert (
         website_content.file.url
-        == f"https://s3.amazonaws.com/{settings.AWS_STORAGE_BUCKET_NAME}/{path}"
+        == f"https://s3.amazonaws.com/{settings.AWS_STORAGE_BUCKET_NAME}/{expected_path}"
     )
 
 

localdev/configs/ocw-course-site-config.yml

@@ -1,5 +1,6 @@
 ---
 root-url-path: courses
+site-url-format: "[sitemetadata:primary_course_number]-[sitemetadata:course_title]-[sitemetadata:term]-[sitemetadata:year]"
 collections:
   - category: Content
     folder: content/pages

main/settings.py

@@ -24,7 +24,7 @@
 
 # pylint: disable=too-many-lines
 
-VERSION = "0.61.1"
+VERSION = "0.61.2"
 
 SITE_ID = get_int(
     name="OCW_STUDIO_SITE_ID",
@@ -558,7 +558,6 @@
     default="course_title",
     description="The site metadata field for title",
 )
-
 # YouTube OCW metadata fields
 YT_FIELD_CAPTIONS = get_string(
     name="YT_FIELD_CAPTIONS",