From 6d9b973814f7fc8f9986c944ca8f562a2e4e3113 Mon Sep 17 00:00:00 2001 From: Thomas Newton Date: Sun, 29 Oct 2023 22:25:10 +0000 Subject: [PATCH] Working directory detection for flat namespace accounts --- cpp/src/arrow/filesystem/azurefs.cc | 37 ++++++++++++++--------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/cpp/src/arrow/filesystem/azurefs.cc b/cpp/src/arrow/filesystem/azurefs.cc index d8b8660849540..25c78be5261b7 100644 --- a/cpp/src/arrow/filesystem/azurefs.cc +++ b/cpp/src/arrow/filesystem/azurefs.cc @@ -79,18 +79,17 @@ struct AzurePath { "Expected an Azure object path of the form 'container/path...', got a URI: '", s, "'"); } - const auto src = internal::RemoveTrailingSlash(s); - auto first_sep = src.find_first_of(internal::kSep); + auto first_sep = s.find_first_of(internal::kSep); if (first_sep == 0) { return Status::Invalid("Path cannot start with a separator ('", s, "')"); } if (first_sep == std::string::npos) { - return AzurePath{std::string(src), std::string(src), "", {}}; + return AzurePath{std::string(s), std::string(s), "", {}}; } AzurePath path; - path.full_path = std::string(src); - path.container = std::string(src.substr(0, first_sep)); - path.path_to_file = std::string(src.substr(first_sep + 1)); + path.full_path = std::string(s); + path.container = std::string(s.substr(0, first_sep)); + path.path_to_file = std::string(s.substr(first_sep + 1)); path.path_to_file_parts = internal::SplitAbstractPath(path.path_to_file); RETURN_NOT_OK(Validate(path)); return path; @@ -340,13 +339,14 @@ class AzureFileSystem::Impl { const AzureOptions& options() const { return options_; } - public: + public: const bool get_is_hierarchical_namespace_enabled(const std::string& container) { if (!is_hierarchical_namespace_enabled_.has_value()) { auto path_client = datalake_service_client_->GetFileSystemClient(container); // try { - path_client.GetAccessPolicy(); - is_hierarchical_namespace_enabled_ = true; + path_client.GetAccessPolicy(); + is_hierarchical_namespace_enabled_ = false; + // is_hierarchical_namespace_enabled_ = true; // } catch () { // is_hierarchical_namespace_enabled_ = false; // } @@ -412,23 +412,22 @@ class AzureFileSystem::Impl { // We need to detect implied directories by listing. Azure::Storage::Blobs::ListBlobsOptions list_blob_options; - // AzurePath::FromString() ensures that path_to_file does not end with a slash. - // If listing the prefix `path.path_to_file + "/"` returns at least one result - // then path refers to an implied directory. - ARROW_RETURN_NOT_OK(internal::AssertNoTrailingSlash(path.path_to_file)); - list_blob_options.Prefix = path.path_to_file + "/"; + // If listing the prefix `path.path_to_file` with trailing slash returns at least + // one result then path refers to an implied directory. + list_blob_options.Prefix = internal::EnsureTrailingSlash(path.path_to_file); // We only need to know if there is at least one result, so minimise page size // for efficiency. list_blob_options.PageSizeHint = 1; // TODO: Confirm this will only fetch a single page, for efficiency - // TODO: Return appropriate status if there is an exception. - if (blob_service_client_->GetBlobContainerClient(path.container) - .ListBlobs(list_blob_options) - .HasPage()) { + // TODO: Return appropriate status if there is an exception. + auto paged_list_result = + blob_service_client_->GetBlobContainerClient(path.container) + .ListBlobs(list_blob_options); + if (paged_list_result.Blobs.size() > 0) { return FileInfo(path.full_path, FileType::Directory); } else { - return FileInfo(path.full_path, FileType::NotFound); + return FileInfo(path.full_path, FileType::NotFound); } } return ErrorToStatus(