e2e test pipeline cache clearing (#2078)

* enable Fastly cache clearing outside of dev for the end to end test pipeline

* add skip_search_index_update optional argument to SitePipelineOnlineTasks, utilize it in the end to end test pipeline and add cache clearing

* need array wrapper for catalog steps in extend

* dev / not dev testing of cdn_cache_clear steps in end to end test pipeline

content_sync/pipelines/definitions/concourse/e2e_test_site_pipeline.py

@@ -249,8 +249,9 @@ def __init__(self, themes_branch: str, projects_branch: str, **kwargs):
         site_tasks.extend(
             SitePipelineOnlineTasks(
                 pipeline_vars=site_pipeline_vars,
-                fastly_var=version,
-                skip_cache_clear=True,
+                fastly_var="test",
+                skip_cache_clear=is_dev(),
+                skip_search_index_update=True,
             )
         )
 

content_sync/pipelines/definitions/concourse/e2e_test_site_pipeline_test.py

@@ -193,7 +193,10 @@ def test_generate_e2e_test_site_pipeline_definition(  # noqa: PLR0913 PLR0915
         for step in across_step_build_steps
         if step.get("task") == CLEAR_CDN_CACHE_IDENTIFIER
     ]
-    assert len(cdn_cache_clear_steps) == 0
+    if is_dev:
+        assert len(cdn_cache_clear_steps) == 0
+    else:
+        assert len(cdn_cache_clear_steps) == 1
     fetch_built_content_steps = [
         step
         for step in e2e_test_tasks

content_sync/pipelines/definitions/concourse/site_pipeline.py

@@ -443,6 +443,7 @@ class SitePipelineOnlineTasks(list[StepModifierMixin]):
         destructive_sync(bool): (Optional) A boolean override for the delete flag used in AWS syncs
         filter_videos(bool): (Optional) A boolean override for filtering videos out of AWS syncs
         skip_cache_clear(bool): (Optional) A boolean override for skipping the CDN cache clear step
+        skip_search_index_update(bool): (Optional) A boolean override for skipping the search index update
     """  # noqa: E501
 
     def __init__(  # noqa: PLR0913
@@ -453,6 +454,7 @@ def __init__(  # noqa: PLR0913
         destructive_sync: bool = True,
         filter_videos: bool = False,
         skip_cache_clear: bool = False,
+        skip_search_index_update: bool = False,
     ):
         delete_flag = pipeline_vars["delete_flag"] if destructive_sync else ""
         static_resources_task_step = StaticResourcesTaskStep(
@@ -564,23 +566,28 @@ def __init__(  # noqa: PLR0913
             short_id=pipeline_vars["short_id"],
             instance_vars=pipeline_vars["instance_vars"],
         )
-        clear_cdn_cache_online_step.on_success = TryStep(
-            try_=DoStep(
-                do=[
+        clear_cdn_cache_online_on_success_steps = []
+        if not skip_search_index_update:
+            clear_cdn_cache_online_on_success_steps.extend(
+                [
                     *[
                         OpenCatalogWebhookStep(
                             site_url=pipeline_vars["url_path"],
                             pipeline_name=pipeline_vars["pipeline_name"],
                             open_catalog_url=open_catalog_url,
                         )
                         for open_catalog_url in settings.OPEN_CATALOG_URLS
-                    ],
-                    OcwStudioWebhookStep(
-                        pipeline_name=pipeline_vars["pipeline_name"],
-                        status="succeeded",
-                    ),
+                    ]
                 ]
             )
+        clear_cdn_cache_online_on_success_steps.append(
+            OcwStudioWebhookStep(
+                pipeline_name=pipeline_vars["pipeline_name"],
+                status="succeeded",
+            )
+        )
+        clear_cdn_cache_online_step.on_success = TryStep(
+            try_=DoStep(do=clear_cdn_cache_online_on_success_steps)
         )
         self.extend(
             [