Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history 
for more information, see https://pre-commit.ci
  • Loading branch information
@pre-commit-ci
pre-commit-ci[bot] committed Nov 20, 2024
1 parent e8e68b5 commit 704cd76
Show file tree
Hide file tree
Showing 10 changed files with 91 additions and 53 deletions.
17 changes: 13 additions & 4 deletions exodus_gw/aws/dynamodb.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,21 @@ def definitions(self):
self._definitions = self.query_definitions()
return self._definitions

def _aliases(self, alias_types: list[str]) -> list[tuple[str, str, list[str]]]:
def _aliases(
self, alias_types: list[str]
) -> list[tuple[str, str, list[str]]]:
out: list[tuple[str, str, list[str]]] = []

for k, v in self.definitions.items():
if k in alias_types:
for alias in v:
out.append((alias["src"], alias["dest"],
alias.get("exclude_paths") or []))
out.append(
(
alias["src"],
alias["dest"],
alias.get("exclude_paths") or [],
)
)

return out

Expand Down Expand Up @@ -129,7 +136,9 @@ def aliases_for_flush(self) -> list[tuple[str, str, list[str]]]:
# this alias. If we don't traverse the alias from dest => src then we
# will miss the fact that /content/dist/rhel8/rhui paths should also
# have cache flushed.
out = out + [(dest, src, exclusions) for (src, dest, exclusions) in out]
out = out + [
(dest, src, exclusions) for (src, dest, exclusions) in out
]

return out

Expand Down
17 changes: 10 additions & 7 deletions exodus_gw/aws/util.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,9 @@ def get_reader(cls, request):
return cls(request)


def uri_alias(uri: str, aliases: list[tuple[str, str, list[str]]]) -> list[str]:
def uri_alias(
uri: str, aliases: list[tuple[str, str, list[str]]]
) -> list[str]:
# Resolve every alias between paths within the uri (e.g.
# allow RHUI paths to be aliased to non-RHUI).
#
Expand Down Expand Up @@ -225,12 +227,13 @@ def add_out(new_uri: str) -> bool:
# Used to replicate old NetStorage-compatible behaviour. This will
# typically match non-rpm paths, such as /images/ or /isos/
if any([re.search(exclusion, uri) for exclusion in exclude_paths]):
LOG.debug("Aliasing for %s was not applied as it matches one "
"of the following exclusion paths: %s.",
uri,
",".join(exclude_paths),
extra={"event": "publish", "success": True},
)
LOG.debug(
"Aliasing for %s was not applied as it matches one "
"of the following exclusion paths: %s.",
uri,
",".join(exclude_paths),
extra={"event": "publish", "success": True},
)
continue

new_uri = uri.replace(src, dest, 1)
Expand Down
13 changes: 6 additions & 7 deletions exodus_gw/routers/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@
"type": "array",
"items": {"type": "string"},
"description": "Paths for which alias will not be resolved, "
"treated as an unanchored regex."
}
"treated as an unanchored regex.",
},
},
},
"uniqueItems": True,
Expand Down Expand Up @@ -143,27 +143,26 @@ def config_post(
{
"src": "/content/origin",
"dest": "/origin",
"exclude_paths": []
"exclude_paths": [],
},
{
"src": "/origin/rpm",
"dest": "/origin/rpms",
"exclude_paths": ["/iso/"]
"exclude_paths": ["/iso/"],
},
],
"releasever_alias": [
{
"dest": "/content/dist/rhel8/8.5",
"src": "/content/dist/rhel8/8",
"exclude_paths": ["/files/", "/images/", "/iso/"]
"exclude_paths": ["/files/", "/images/", "/iso/"],
},

],
"rhui_alias": [
{
"dest": "/content/dist/rhel8",
"src": "/content/dist/rhel8/rhui",
"exclude_paths": ["/files/", "/images/", "/iso/"]
"exclude_paths": ["/files/", "/images/", "/iso/"],
},
],
}
Expand Down
9 changes: 4 additions & 5 deletions exodus_gw/routers/deploy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,27 +59,26 @@ def deploy_config(
{
"src": "/content/origin",
"dest": "/origin",
"exclude_paths": []
"exclude_paths": [],
},
{
"src": "/origin/rpm",
"dest": "/origin/rpms",
"exclude_paths": ["/iso/"]
"exclude_paths": ["/iso/"],
},
],
"releasever_alias": [
{
"dest": "/content/dist/rhel8/8.5",
"src": "/content/dist/rhel8/8",
"exclude_paths": ["/files/", "/images/", "/iso/"]
"exclude_paths": ["/files/", "/images/", "/iso/"],
},

],
"rhui_alias": [
{
"dest": "/content/dist/rhel8",
"src": "/content/dist/rhel8/rhui",
"exclude_paths": ["/files/", "/images/", "/iso/"]
"exclude_paths": ["/files/", "/images/", "/iso/"],
},
],
}
Expand Down
5 changes: 3 additions & 2 deletions exodus_gw/schemas.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,8 +320,9 @@ class Alias(BaseModel):
..., description="Target of the alias, relative to CDN root."
)
exclude_paths: list[str] | None = Field(
[], description="Paths for which alias will not be resolved, "
"treated as an unanchored regex."
[],
description="Paths for which alias will not be resolved, "
"treated as an unanchored regex.",
)


Expand Down
10 changes: 7 additions & 3 deletions exodus_gw/worker/deploy.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import logging
from typing import Any
import re
from typing import Any

import dramatiq
from dramatiq.middleware import CurrentMessage
Expand Down Expand Up @@ -153,8 +153,12 @@ def deploy_config(
# If any original exclusion matches the uri, the uri wouldn't
# have been treated as an alias, thus cache flushing would be
# unnecessary.
if any([re.search(exclusion, published_path.web_uri)
for exclusion in original_exclusions.get(src, [])]):
if any(
[
re.search(exclusion, published_path.web_uri)
for exclusion in original_exclusions.get(src, [])
]
):
continue
LOG.info(
"Updated alias %s will flush cache for %s",
Expand Down
52 changes: 35 additions & 17 deletions tests/aws/test_uri_alias.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,34 +145,45 @@ def test_uri_alias_limit(caplog: pytest.LogCaptureFixture):
(
"/content/dist/rhel9/9/x86_64/baseos/iso/PULP_MANIFEST",
[
("/content/dist/rhel9/9", "/content/dist/rhel9/9.5", ["/iso/"]),
(
"/content/dist/rhel9/9",
"/content/dist/rhel9/9.5",
["/iso/"],
),
],
# just returns the original
["/content/dist/rhel9/9/x86_64/baseos/iso/PULP_MANIFEST"],
"Aliasing for /content/dist/rhel9/9/x86_64/baseos/iso/PULP_MANIFEST "
"was not applied as it matches one of the following exclusion paths: /iso/."
"was not applied as it matches one of the following exclusion paths: /iso/.",
),
(
"/some/path/with/file/in/it",
[
("/some/path", "/another/different/path", ["/none/", "/here/"]),
("/another/different/path", "/this/wont/alias", ["/file/"])
(
"/some/path",
"/another/different/path",
["/none/", "/here/"],
),
("/another/different/path", "/this/wont/alias", ["/file/"]),
],
[
"/another/different/path/with/file/in/it",
"/some/path/with/file/in/it",
],
["/another/different/path/with/file/in/it",
"/some/path/with/file/in/it"],
"Aliasing for /another/different/path/with/file/in/it was not "
"applied as it matches one of the following exclusion paths: /file/."
"applied as it matches one of the following exclusion paths: /file/.",
),
(
"/my/base/content/path/cool_iso_tool.rpm",
[
("/my/base", "/your/own", ["/iso/"]),
],
["/your/own/content/path/cool_iso_tool.rpm",
"/my/base/content/path/cool_iso_tool.rpm"],
[
"/your/own/content/path/cool_iso_tool.rpm",
"/my/base/content/path/cool_iso_tool.rpm",
],
"Resolved alias:\\n\\tsrc: /my/base/content/path/cool_iso_tool.rpm"
"\\n\\tdest: /your/own/content/path/cool_iso_tool.rpm"
"\\n\\tdest: /your/own/content/path/cool_iso_tool.rpm",
),
(
"/content/dist/rhel9/9.5/x86_64/baseos/iso/PULP_MANIFEST",
Expand All @@ -182,21 +193,28 @@ def test_uri_alias_limit(caplog: pytest.LogCaptureFixture):
["/content/dist/rhel9/9.5/x86_64/baseos/iso/PULP_MANIFEST"],
"Aliasing for /content/dist/rhel9/9.5/x86_64/baseos/iso/PULP_MANIFEST "
"was not applied as it matches one of the following exclusion "
"paths: /rhel[89]/."
"paths: /rhel[89]/.",
),
(
"/content/dist/rhel7/7.5/x86_64/baseos/iso/PULP_MANIFEST",
[
("/content/dist", "/alias/path", ["/rhel[89]/"]),
],
["/alias/path/rhel7/7.5/x86_64/baseos/iso/PULP_MANIFEST",
"/content/dist/rhel7/7.5/x86_64/baseos/iso/PULP_MANIFEST",],
[
"/alias/path/rhel7/7.5/x86_64/baseos/iso/PULP_MANIFEST",
"/content/dist/rhel7/7.5/x86_64/baseos/iso/PULP_MANIFEST",
],
"Resolved alias:\\n\\tsrc: /content/dist/rhel7/7.5/x86_64/baseos/iso/PULP_MANIFEST"
"\\n\\tdest: /alias/path/rhel7/7.5/x86_64/baseos/iso/PULP_MANIFEST"
"\\n\\tdest: /alias/path/rhel7/7.5/x86_64/baseos/iso/PULP_MANIFEST",
),
],
ids=["basic", "transitive", "filename", "pattern_include", "pattern_exclude"],
ids=[
"basic",
"transitive",
"filename",
"pattern_include",
"pattern_exclude",
],
)
def test_uri_alias_exclusions(input, aliases, output, log_message, caplog):
caplog.set_level(DEBUG, logger="exodus-gw")
Expand Down
1 change: 0 additions & 1 deletion tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

from .async_utils import BlockDetector


DEFAULT_EXCLUDE_PATHS = ["/files/", "/images/", "/iso/"]


Expand Down
16 changes: 11 additions & 5 deletions tests/worker/test_cdn_cache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,17 +212,23 @@ def test_flush_cdn_cache_typical(
{
"src": "/path/one",
"dest": "/path/one-dest",
"exclude_paths": ["/files/", "/images/",
"/iso/"],
"exclude_paths": [
"/files/",
"/images/",
"/iso/",
],
},
],
"rhui_alias": [
{
"src": "/path/rhui/two",
"dest": "/path/two",
"exclude_paths": ["/files/", "/images/",
"/iso/"],
},
"exclude_paths": [
"/files/",
"/images/",
"/iso/",
],
},
],
}
)
Expand Down
4 changes: 2 additions & 2 deletions tests/worker/test_deploy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ def test_deploy_config_with_flush(
{
"dest": "/content/testproduct/1.2.0",
"src": "/content/testproduct/1",
"exclude_paths": ["/newExclusion/"]
"exclude_paths": ["/newExclusion/"],
},
]
worker.deploy_config(updated_config, "test", NOW_UTC)
Expand Down Expand Up @@ -222,7 +222,7 @@ def test_deploy_config_with_flush(
"/content/dist/rhel/server/listing",
"/content/testproduct/1/file1",
"/content/testproduct/1/file2",
"/content/testproduct/1/newExclusion/file5"
"/content/testproduct/1/newExclusion/file5",
]

# And actor call should have been delayed by this long.
Expand Down

0 comments on commit 704cd76

Please sign in to comment.