From 8a0804e34a35316442fb9b974d5af3355e6542ff Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Thu, 2 Jan 2025 16:34:26 -0500 Subject: [PATCH 01/10] make _downloader function in FlyteFile/Directory pickleable Signed-off-by: Niels Bantilan --- flytekit/types/directory/types.py | 8 ++++++-- flytekit/types/file/file.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index 3bcbf77c97..7bc71ec03e 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -6,6 +6,7 @@ import random import typing from dataclasses import dataclass, field +from functools import partial from pathlib import Path from typing import Any, Dict, Generator, Tuple from uuid import UUID @@ -665,8 +666,7 @@ async def async_to_python_value( batch_size = get_batch_size(expected_python_type) - def _downloader(): - return ctx.file_access.get_data(uri, local_folder, is_multipart=True, batch_size=batch_size) + _downloader = partial(_flyte_directory_downloader, ctx, uri, local_folder, batch_size) expected_format = self.get_format(expected_python_type) @@ -683,4 +683,8 @@ def guess_python_type(self, literal_type: LiteralType) -> typing.Type[FlyteDirec raise ValueError(f"Transformer {self} cannot reverse {literal_type}") +def _flyte_directory_downloader(ctx: FlyteContext, uri: str, local_folder: str, batch_size: int): + return ctx.file_access.get_data(uri, local_folder, is_multipart=True, batch_size=batch_size) + + TypeEngine.register(FlyteDirToMultipartBlobTransformer()) diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 16ec949264..b833b00354 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -7,6 +7,7 @@ import typing from contextlib import contextmanager from dataclasses import dataclass, field +from functools import partial from typing import Dict, cast from urllib.parse import unquote @@ -694,8 +695,7 @@ async def async_to_python_value( # For the remote case, return an FlyteFile object that can download local_path = ctx.file_access.get_random_local_path(uri) - def _downloader(): - return ctx.file_access.get_data(uri, local_path, is_multipart=False) + _downloader = partial(_flyte_file_downloader, ctx, uri, local_path) expected_format = FlyteFilePathTransformer.get_format(expected_python_type) ff = FlyteFile.__class_getitem__(expected_format)(local_path, _downloader) @@ -714,4 +714,8 @@ def guess_python_type(self, literal_type: LiteralType) -> typing.Type[FlyteFile[ raise ValueError(f"Transformer {self} cannot reverse {literal_type}") +def _flyte_file_downloader(ctx: FlyteContext, uri: str, local_path: str): + return ctx.file_access.get_data(uri, local_path, is_multipart=False) + + TypeEngine.register(FlyteFilePathTransformer(), additional_types=[os.PathLike]) From 206096293700851a3640e1a21bfb2cae1610d849 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Fri, 3 Jan 2025 17:21:56 -0500 Subject: [PATCH 02/10] make FlyteFile and Directory pickleable Signed-off-by: Niels Bantilan --- flytekit/types/directory/types.py | 7 +++-- flytekit/types/file/file.py | 8 ++--- .../unit/core/test_flyte_directory.py | 30 +++++++++++++++++-- tests/flytekit/unit/core/test_flyte_file.py | 27 ++++++++++++++++- 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index 7bc71ec03e..53ca15dd4e 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -23,6 +23,7 @@ from flytekit.core.constants import MESSAGEPACK from flytekit.core.context_manager import FlyteContext, FlyteContextManager from flytekit.core.type_engine import AsyncTypeTransformer, TypeEngine, TypeTransformerFailedError, get_batch_size +from flytekit.core.data_persistence import FileAccessProvider from flytekit.exceptions.user import FlyteAssertion from flytekit.extras.pydantic_transformer.decorator import model_serializer, model_validator from flytekit.models import types as _type_models @@ -666,7 +667,7 @@ async def async_to_python_value( batch_size = get_batch_size(expected_python_type) - _downloader = partial(_flyte_directory_downloader, ctx, uri, local_folder, batch_size) + _downloader = partial(_flyte_directory_downloader, ctx.file_access, uri, local_folder, batch_size) expected_format = self.get_format(expected_python_type) @@ -683,8 +684,8 @@ def guess_python_type(self, literal_type: LiteralType) -> typing.Type[FlyteDirec raise ValueError(f"Transformer {self} cannot reverse {literal_type}") -def _flyte_directory_downloader(ctx: FlyteContext, uri: str, local_folder: str, batch_size: int): - return ctx.file_access.get_data(uri, local_folder, is_multipart=True, batch_size=batch_size) +def _flyte_directory_downloader(file_access_provider: FileAccessProvider, uri: str, local_folder: str, batch_size: int): + return file_access_provider.get_data(uri, local_folder, is_multipart=True, batch_size=batch_size) TypeEngine.register(FlyteDirToMultipartBlobTransformer()) diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index b833b00354..b5891b2155 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -21,6 +21,7 @@ from flytekit.core.constants import MESSAGEPACK from flytekit.core.context_manager import FlyteContext, FlyteContextManager +from flytekit.core.data_persistence import FileAccessProvider from flytekit.core.type_engine import ( AsyncTypeTransformer, TypeEngine, @@ -695,12 +696,11 @@ async def async_to_python_value( # For the remote case, return an FlyteFile object that can download local_path = ctx.file_access.get_random_local_path(uri) - _downloader = partial(_flyte_file_downloader, ctx, uri, local_path) + _downloader = partial(_flyte_file_downloader, ctx.file_access, uri, local_path) expected_format = FlyteFilePathTransformer.get_format(expected_python_type) ff = FlyteFile.__class_getitem__(expected_format)(local_path, _downloader) ff._remote_source = uri - return ff def guess_python_type(self, literal_type: LiteralType) -> typing.Type[FlyteFile[typing.Any]]: @@ -714,8 +714,8 @@ def guess_python_type(self, literal_type: LiteralType) -> typing.Type[FlyteFile[ raise ValueError(f"Transformer {self} cannot reverse {literal_type}") -def _flyte_file_downloader(ctx: FlyteContext, uri: str, local_path: str): - return ctx.file_access.get_data(uri, local_path, is_multipart=False) +def _flyte_file_downloader(file_access_provider: FileAccessProvider, uri: str, local_path: str): + return file_access_provider.get_data(uri, local_path, is_multipart=False) TypeEngine.register(FlyteFilePathTransformer(), additional_types=[os.PathLike]) diff --git a/tests/flytekit/unit/core/test_flyte_directory.py b/tests/flytekit/unit/core/test_flyte_directory.py index be61388fa5..fdb12e1dae 100644 --- a/tests/flytekit/unit/core/test_flyte_directory.py +++ b/tests/flytekit/unit/core/test_flyte_directory.py @@ -1,5 +1,6 @@ import os import pathlib +import pickle import shutil import tempfile import typing @@ -20,7 +21,7 @@ from flytekit.core.workflow import workflow from flytekit.exceptions.user import FlyteAssertion from flytekit.models.core.types import BlobType -from flytekit.models.literals import LiteralMap +from flytekit.models.literals import LiteralMap, Blob, BlobMetadata from flytekit.types.directory.types import FlyteDirectory, FlyteDirToMultipartBlobTransformer from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct @@ -407,8 +408,7 @@ def my_wf(path: SvgDirectory) -> DC: assert dc1 == dc2 -def test_input_from_flyte_console_attribute_access_flytefile( - local_dummy_directory): +def test_input_from_flyte_console_attribute_access_flytefile(local_dummy_directory): # Flyte Console will send the input data as protobuf Struct dict_obj = {"path": local_dummy_directory} @@ -422,3 +422,27 @@ def test_input_from_flyte_console_attribute_access_flytefile( FlyteContextManager.current_context(), upstream_output, FlyteDirectory) assert isinstance(downstream_input, FlyteDirectory) assert downstream_input == FlyteDirectory(local_dummy_directory) + + +def test_flyte_directory_is_pickleable(): + upstream_output = Literal( + scalar=Scalar( + blob=Blob( + uri="s3://sample-path/directory", + metadata=BlobMetadata( + type=BlobType( + dimensionality=BlobType.BlobDimensionality.MULTIPART, + format="" + ) + ) + ) + ) + ) + downstream_input = TypeEngine.to_python_value( + FlyteContextManager.current_context(), upstream_output, FlyteDirectory + ) + + # test round trip pickling + pickled_input = pickle.dumps(downstream_input) + unpickled_input = pickle.loads(pickled_input) + assert downstream_input == unpickled_input diff --git a/tests/flytekit/unit/core/test_flyte_file.py b/tests/flytekit/unit/core/test_flyte_file.py index 1b5d32b1f2..fb0903c567 100644 --- a/tests/flytekit/unit/core/test_flyte_file.py +++ b/tests/flytekit/unit/core/test_flyte_file.py @@ -1,6 +1,7 @@ import json import os import pathlib +import pickle import tempfile import typing from unittest.mock import MagicMock, patch @@ -19,7 +20,7 @@ from flytekit.core.type_engine import TypeEngine from flytekit.core.workflow import workflow from flytekit.models.core.types import BlobType -from flytekit.models.literals import LiteralMap +from flytekit.models.literals import LiteralMap, Blob, BlobMetadata from flytekit.types.file.file import FlyteFile, FlyteFilePathTransformer from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct @@ -782,3 +783,27 @@ def test_input_from_flyte_console_attribute_access_flytefile(local_dummy_file): downstream_input = TypeEngine.to_python_value( FlyteContextManager.current_context(), upstream_output, FlyteFile) assert downstream_input == FlyteFile(local_dummy_file) + + +def test_flyte_file_is_pickleable(): + upstream_output = Literal( + scalar=Scalar( + blob=Blob( + uri="s3://sample-path/file", + metadata=BlobMetadata( + type=BlobType( + dimensionality=BlobType.BlobDimensionality.SINGLE, + format="txt" + ) + ) + ) + ) + ) + downstream_input = TypeEngine.to_python_value( + FlyteContextManager.current_context(), upstream_output, FlyteFile + ) + + # test round trip pickling + pickled_input = pickle.dumps(downstream_input) + unpickled_input = pickle.loads(pickled_input) + assert downstream_input == unpickled_input From dddde88a9d6414feb691414e6210cf3c611c6392 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Mon, 6 Jan 2025 15:02:19 -0500 Subject: [PATCH 03/10] remove unnecessary helper functions Signed-off-by: Niels Bantilan --- flytekit/types/directory/types.py | 2 +- flytekit/types/file/file.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index 53ca15dd4e..190495d4da 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -667,7 +667,7 @@ async def async_to_python_value( batch_size = get_batch_size(expected_python_type) - _downloader = partial(_flyte_directory_downloader, ctx.file_access, uri, local_folder, batch_size) + _downloader = partial(ctx.file_access.get_data, uri, local_folder, is_multipart=True, batch_size=batch_size) expected_format = self.get_format(expected_python_type) diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index b5891b2155..a098260353 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -696,7 +696,7 @@ async def async_to_python_value( # For the remote case, return an FlyteFile object that can download local_path = ctx.file_access.get_random_local_path(uri) - _downloader = partial(_flyte_file_downloader, ctx.file_access, uri, local_path) + _downloader = partial(ctx.file_access.get_data, uri, local_path, is_multipart=False) expected_format = FlyteFilePathTransformer.get_format(expected_python_type) ff = FlyteFile.__class_getitem__(expected_format)(local_path, _downloader) @@ -714,8 +714,4 @@ def guess_python_type(self, literal_type: LiteralType) -> typing.Type[FlyteFile[ raise ValueError(f"Transformer {self} cannot reverse {literal_type}") -def _flyte_file_downloader(file_access_provider: FileAccessProvider, uri: str, local_path: str): - return file_access_provider.get_data(uri, local_path, is_multipart=False) - - TypeEngine.register(FlyteFilePathTransformer(), additional_types=[os.PathLike]) From 6adf14b3d0b97e44786a617d4dba1fdffec7ff9d Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Mon, 6 Jan 2025 15:41:13 -0500 Subject: [PATCH 04/10] fix lint Signed-off-by: Niels Bantilan --- flytekit/types/directory/types.py | 2 +- flytekit/types/file/file.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index 190495d4da..dd6e8ac337 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -22,8 +22,8 @@ from flytekit.core.constants import MESSAGEPACK from flytekit.core.context_manager import FlyteContext, FlyteContextManager -from flytekit.core.type_engine import AsyncTypeTransformer, TypeEngine, TypeTransformerFailedError, get_batch_size from flytekit.core.data_persistence import FileAccessProvider +from flytekit.core.type_engine import AsyncTypeTransformer, TypeEngine, TypeTransformerFailedError, get_batch_size from flytekit.exceptions.user import FlyteAssertion from flytekit.extras.pydantic_transformer.decorator import model_serializer, model_validator from flytekit.models import types as _type_models diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 4098d412ad..7c2d2cb7dd 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -21,7 +21,6 @@ from flytekit.core.constants import MESSAGEPACK from flytekit.core.context_manager import FlyteContext, FlyteContextManager -from flytekit.core.data_persistence import FileAccessProvider from flytekit.core.type_engine import ( AsyncTypeTransformer, TypeEngine, @@ -736,7 +735,7 @@ async def async_to_python_value( local_path = ctx.file_access.get_random_local_path(uri) _downloader = partial(ctx.file_access.get_data, uri, local_path, is_multipart=False) - + expected_format = FlyteFilePathTransformer.get_format(expected_python_type) ff = FlyteFile.__class_getitem__(expected_format)( path=local_path, downloader=lambda: self.downloader(ctx=ctx, remote_path=uri, local_path=local_path) From 37f61e2ff3b5ae1c084a3c489815eff7fa9ded5c Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Mon, 6 Jan 2025 16:42:06 -0500 Subject: [PATCH 05/10] use partials instead of lambda Signed-off-by: Niels Bantilan --- flytekit/types/directory/types.py | 7 ++----- flytekit/types/file/file.py | 7 +++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index dd6e8ac337..1a52a847f9 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -376,23 +376,20 @@ def listdir(cls, directory: FlyteDirectory) -> typing.List[typing.Union[FlyteDir paths.append(FlyteDirectory(joined_path)) return paths - def create_downloader(_remote_path: str, _local_path: str, is_multipart: bool): - return lambda: file_access.get_data(_remote_path, _local_path, is_multipart=is_multipart) - fs = file_access.get_filesystem_for_path(final_path) for key in fs.listdir(final_path): remote_path = os.path.join(final_path, key["name"].split(os.sep)[-1]) if key["type"] == "file": local_path = file_access.get_random_local_path() os.makedirs(pathlib.Path(local_path).parent, exist_ok=True) - downloader = create_downloader(remote_path, local_path, is_multipart=False) + downloader = partial(file_access.get_data, remote_path, local_path, is_multipart=False) flyte_file: FlyteFile = FlyteFile(local_path, downloader=downloader) flyte_file._remote_source = remote_path paths.append(flyte_file) else: local_folder = file_access.get_random_local_directory() - downloader = create_downloader(remote_path, local_folder, is_multipart=True) + downloader = partial(file_access.get_data, remote_path, local_folder, is_multipart=True) flyte_directory: FlyteDirectory = FlyteDirectory(path=local_folder, downloader=downloader) flyte_directory._remote_source = remote_path diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 7c2d2cb7dd..6508232a91 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -308,7 +308,8 @@ def __init__( if ctx.file_access.is_remote(self.path): self._remote_source = self.path self._local_path = ctx.file_access.get_random_local_path(self._remote_source) - self._downloader = lambda: FlyteFilePathTransformer.downloader( + self._downloader = partial( + FlyteFilePathTransformer.downloader, ctx=ctx, remote_path=self._remote_source, # type: ignore local_path=self._local_path, @@ -734,11 +735,9 @@ async def async_to_python_value( # For the remote case, return an FlyteFile object that can download local_path = ctx.file_access.get_random_local_path(uri) - _downloader = partial(ctx.file_access.get_data, uri, local_path, is_multipart=False) - expected_format = FlyteFilePathTransformer.get_format(expected_python_type) ff = FlyteFile.__class_getitem__(expected_format)( - path=local_path, downloader=lambda: self.downloader(ctx=ctx, remote_path=uri, local_path=local_path) + path=local_path, downloader=partial(self.downloader, ctx=ctx, remote_path=uri, local_path=local_path), ) ff._remote_source = uri return ff From 6152ce4bc56c02b920d2942a238c3ba49977ef9b Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Tue, 7 Jan 2025 10:03:31 -0500 Subject: [PATCH 06/10] fix lint Signed-off-by: Niels Bantilan --- flytekit/types/file/file.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 6508232a91..8cb4c17107 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -737,7 +737,8 @@ async def async_to_python_value( expected_format = FlyteFilePathTransformer.get_format(expected_python_type) ff = FlyteFile.__class_getitem__(expected_format)( - path=local_path, downloader=partial(self.downloader, ctx=ctx, remote_path=uri, local_path=local_path), + path=local_path, + downloader=partial(self.downloader, ctx=ctx, remote_path=uri, local_path=local_path), ) ff._remote_source = uri return ff From 3aaf9ac429f37d23419a79188c434c68418017d5 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Tue, 7 Jan 2025 10:10:59 -0500 Subject: [PATCH 07/10] remove unneeded helper function Signed-off-by: Niels Bantilan --- flytekit/types/directory/types.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index 1a52a847f9..82f35fd336 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -22,7 +22,6 @@ from flytekit.core.constants import MESSAGEPACK from flytekit.core.context_manager import FlyteContext, FlyteContextManager -from flytekit.core.data_persistence import FileAccessProvider from flytekit.core.type_engine import AsyncTypeTransformer, TypeEngine, TypeTransformerFailedError, get_batch_size from flytekit.exceptions.user import FlyteAssertion from flytekit.extras.pydantic_transformer.decorator import model_serializer, model_validator @@ -681,8 +680,4 @@ def guess_python_type(self, literal_type: LiteralType) -> typing.Type[FlyteDirec raise ValueError(f"Transformer {self} cannot reverse {literal_type}") -def _flyte_directory_downloader(file_access_provider: FileAccessProvider, uri: str, local_folder: str, batch_size: int): - return file_access_provider.get_data(uri, local_folder, is_multipart=True, batch_size=batch_size) - - TypeEngine.register(FlyteDirToMultipartBlobTransformer()) From a60450bc70f980803c6df096fac69c5eaefb3ed4 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Tue, 7 Jan 2025 10:21:15 -0500 Subject: [PATCH 08/10] update FlyteFilePathTransformer.downloader method Signed-off-by: Niels Bantilan --- flytekit/types/file/file.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 8cb4c17107..4025b2f17a 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -21,6 +21,7 @@ from flytekit.core.constants import MESSAGEPACK from flytekit.core.context_manager import FlyteContext, FlyteContextManager +from flytekit.core.data_persistence import FileAccessProvider from flytekit.core.type_engine import ( AsyncTypeTransformer, TypeEngine, @@ -738,14 +739,16 @@ async def async_to_python_value( expected_format = FlyteFilePathTransformer.get_format(expected_python_type) ff = FlyteFile.__class_getitem__(expected_format)( path=local_path, - downloader=partial(self.downloader, ctx=ctx, remote_path=uri, local_path=local_path), + downloader=partial(self.downloader, ctx.file_access, remote_path=uri, local_path=local_path), ) ff._remote_source = uri return ff @staticmethod def downloader( - ctx: FlyteContext, remote_path: typing.Union[str, os.PathLike], local_path: typing.Union[str, os.PathLike] + file_access_provider: FileAccessProvider, + remote_path: typing.Union[str, os.PathLike], + local_path: typing.Union[str, os.PathLike], ) -> None: """ Download data from remote_path to local_path. @@ -753,7 +756,7 @@ def downloader( We design the downloader as a static method because its behavior is logically related to this class but don't need to interact with class or instance data. """ - ctx.file_access.get_data(remote_path, local_path, is_multipart=False) + file_access_provider.get_data(remote_path, local_path, is_multipart=False) def guess_python_type(self, literal_type: LiteralType) -> typing.Type[FlyteFile[typing.Any]]: if ( From 2e2e5d911624dd156e37645d79f65988bb2f0d99 Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Wed, 8 Jan 2025 16:03:44 -0500 Subject: [PATCH 09/10] remove downloader staticmethod Signed-off-by: Niels Bantilan --- flytekit/types/file/file.py | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 4025b2f17a..0bd4390a15 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -310,7 +310,7 @@ def __init__( self._remote_source = self.path self._local_path = ctx.file_access.get_random_local_path(self._remote_source) self._downloader = partial( - FlyteFilePathTransformer.downloader, + ctx.file_access.get_data, ctx=ctx, remote_path=self._remote_source, # type: ignore local_path=self._local_path, @@ -736,28 +736,13 @@ async def async_to_python_value( # For the remote case, return an FlyteFile object that can download local_path = ctx.file_access.get_random_local_path(uri) + _downloader = partial(ctx.file_access.get_data, remote_path=uri, local_path=local_path, is_multipart=False) + expected_format = FlyteFilePathTransformer.get_format(expected_python_type) - ff = FlyteFile.__class_getitem__(expected_format)( - path=local_path, - downloader=partial(self.downloader, ctx.file_access, remote_path=uri, local_path=local_path), - ) + ff = FlyteFile.__class_getitem__(expected_format)(path=local_path, downloader=_downloader) ff._remote_source = uri return ff - @staticmethod - def downloader( - file_access_provider: FileAccessProvider, - remote_path: typing.Union[str, os.PathLike], - local_path: typing.Union[str, os.PathLike], - ) -> None: - """ - Download data from remote_path to local_path. - - We design the downloader as a static method because its behavior is logically - related to this class but don't need to interact with class or instance data. - """ - file_access_provider.get_data(remote_path, local_path, is_multipart=False) - def guess_python_type(self, literal_type: LiteralType) -> typing.Type[FlyteFile[typing.Any]]: if ( literal_type.blob is not None From d0b1d3c1281ee1a2f6a26fe4cd24555c9133ea4a Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Wed, 8 Jan 2025 16:11:40 -0500 Subject: [PATCH 10/10] fix lint Signed-off-by: Niels Bantilan --- flytekit/types/file/file.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 0bd4390a15..b66c48443c 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -21,7 +21,6 @@ from flytekit.core.constants import MESSAGEPACK from flytekit.core.context_manager import FlyteContext, FlyteContextManager -from flytekit.core.data_persistence import FileAccessProvider from flytekit.core.type_engine import ( AsyncTypeTransformer, TypeEngine,