Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce p-ranav/glob library for file globbing #50

Merged
merged 2 commits into from
Aug 5, 2022
Merged

Introduce p-ranav/glob library for file globbing #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
72bdfac
Add p-ranav/glob submodule
timwoj Aug 2, 2022
1551652
Use glob library to replace code in files.windows.cc
timwoj Aug 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitmodules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,6 @@
[submodule "3rdparty/vcpkg"]
path = 3rdparty/vcpkg
url = https://github.com/microsoft/vcpkg
[submodule "3rdparty/glob"]
path = 3rdparty/glob
url = https://github.com/p-ranav/glob.git
1 change: 1 addition & 0 deletions 3rdparty/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ add_subdirectory(json EXCLUDE_FROM_ALL)
add_subdirectory(stduuid EXCLUDE_FROM_ALL)
add_subdirectory(tomlplusplus EXCLUDE_FROM_ALL)
add_subdirectory(out_ptr EXCLUDE_FROM_ALL)
add_subdirectory(glob EXCLUDE_FROM_ALL)

set(SPDLOG_FMT_EXTERNAL ON)
add_subdirectory(spdlog EXCLUDE_FROM_ALL)
Expand Down
26 changes: 26 additions & 0 deletions 3rdparty/LICENSE.3rdparty
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,32 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

================================================================================
glob
================================================================================

MIT License

Copyright (c) 2019 Pranav

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

================================================================================
nlohmann::json
================================================================================
Expand Down
1 change: 1 addition & 0 deletions 3rdparty/glob
Open in desktop
Submodule glob added at a5f327
1 change: 1 addition & 0 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ target_link_libraries(zeek-agent PRIVATE sqlite::sqlite)
target_link_libraries(zeek-agent PRIVATE stduuid)
target_link_libraries(zeek-agent PRIVATE tomlplusplus::tomlplusplus)
target_link_libraries(zeek-agent PRIVATE ztd::out_ptr)
target_link_libraries(zeek-agent PRIVATE Glob)

if ( NOT HAVE_GETOPT_LONG )
target_link_libraries(zeek-agent PRIVATE 3rdparty:3rdparty)
Expand Down
292 changes: 98 additions & 194 deletions src/tables/files/files.windows.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,112 +47,65 @@ database::RegisterTable<FilesLinesWindows> _2;
database::RegisterTable<FilesColumnsWindows> _3;
} // namespace

static std::string get_full_path(const std::string& pattern, const char* filename) {
char ret[MAX_PATH];
char* tmp = new char[pattern.size() + 1];
strncpy(tmp, pattern.c_str(), pattern.size());
tmp[pattern.size()] = '\0';

PathRemoveFileSpecA(tmp);
PathCombineA(ret, tmp, filename);
delete[] tmp;
return ret;
}

std::pair<std::string, std::vector<filesystem::path>> FilesBase::expandPaths(const std::vector<table::Argument>& args) {
logger()->warn("FilesBase::expandPaths is not implemented on Windows");
return {};
}

std::vector<Value> FilesListWindows::buildFileRow(const std::string& pattern, const WIN32_FIND_DATAA& data) const {
std::vector<Value> row;
std::string full_path = get_full_path(pattern, data.cFileName);

Value path = full_path;
Value type;

uint16_t temp_mode = 0;
if ( data.dwFileAttributes & FILE_ATTRIBUTE_READONLY )
temp_mode = 0444;
else
temp_mode = 0666;

if ( data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) {
// TODO: not sure this addition to the mode is correct
temp_mode += 0111;
type = "dir";
}
else {
// The data from FindFirstFile doesn't include a file type and the handle that's
// returned isn't a file handle (it's a search handle). Open the file separately
// and call GetFileType(), and then close it again.
DWORD file_type = FILE_TYPE_UNKNOWN;
HandlePtr file_handle{CreateFileA(full_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, 0, NULL)};

if ( file_handle.get() == INVALID_HANDLE_VALUE ) {
std::error_condition cond =
std::system_category().default_error_condition(static_cast<int>(GetLastError()));
logger()->warn(
frmt("Couldn't open file {} to check for type: {} ({})", full_path, cond.message(), cond.value()));
}
else {
file_type = GetFileType(file_handle.get());
}
std::pair<std::string, std::vector<filesystem::path>> result;

// A few fields here don't match the counterparts on Linux. Added is 'remote',
// and missing is 'block' and 'socket'.
switch ( file_type ) {
case FILE_TYPE_CHAR: type = "char"; break;
// FILE_TYPE_DISK is just your average everyday file on a drive.
case FILE_TYPE_DISK: type = "file"; break;
// fifo on Linux is roughly the same thing as a Windows pipe.
case FILE_TYPE_PIPE: type = "fifo"; break;
// FILE_TYPE_REMOTE is marked as unused in the documentation.
case FILE_TYPE_REMOTE: type = "remote"; break;
case FILE_TYPE_UNKNOWN:
default: type = "other"; break;
}
}
auto glob = Table::getArgument<std::string>(args, "_pattern");
result.first = glob;

Value mode = frmt("{:o}", temp_mode);
Value size = combineHighLow(data.nFileSizeHigh, data.nFileSizeLow);
Value mtime = to_time(convertFiletime(data.ftLastWriteTime));
for ( auto p : platform::glob(glob) )
result.second.push_back(std::move(p));

return {pattern, path, type, {}, {}, mode, mtime, size};
return result;
}

std::vector<std::vector<Value>> FilesListWindows::snapshot(const std::vector<table::Argument>& args) {
std::vector<std::vector<Value>> rows;

for ( const auto& a : args ) {
if ( a.column == "_pattern" ) {
auto& pattern = std::get<std::string>(a.expression);
auto [pattern, paths] = expandPaths(args);

// TODO: how much of the linux file-globbing pattern API do we actually
// support here? Windows definitely doesn't support the whole gamut,
// like **.

WIN32_FIND_DATAA find_data{};
FindHandlePtr handle{FindFirstFileA(pattern.c_str(), &find_data)};
if ( handle.get() == INVALID_HANDLE_VALUE )
continue;
for ( const auto& p : paths ) {
Value path = p.string();
Value type;
Value mode;
Value mtime;
Value size;

int find_ret = NO_ERROR;
do {
// Ignore the . and .. directories returned by FindFirstFile
if ( strcmp(find_data.cFileName, ".") != 0 && strcmp(find_data.cFileName, "..") != 0 )
rows.emplace_back(buildFileRow(pattern, find_data));

find_ret = FindNextFile(handle.get(), &find_data);
std::error_code ec;
auto status = filesystem::status(p, ec);
if ( ec ) {
ZEEK_AGENT_DEBUG("FilesListWindows", "Failed to get file status: {}", ec.message());
continue;
}

} while ( find_ret != 0 );
mode = frmt("{:o}", static_cast<int64_t>(status.permissions()));
switch ( status.type() ) {
case filesystem::file_type::none:
case filesystem::file_type::not_found:
case filesystem::file_type::symlink:
case filesystem::file_type::unknown: type = "other"; break;
case filesystem::file_type::regular:
type = "file";
size = static_cast<int64_t>(filesystem::file_size(p, ec));
if ( ec ) {
ZEEK_AGENT_DEBUG("FilesListWindows", "Failed to get file size: {}", ec.message());
continue;
}
break;
case filesystem::file_type::directory: type = "dir"; break;
case filesystem::file_type::block: type = "block"; break;
case filesystem::file_type::character: type = "char"; break;
case filesystem::file_type::fifo: type = "fifo"; break;
case filesystem::file_type::socket: type = "socket"; break;
}

if ( GetLastError() != ERROR_NO_MORE_FILES ) {
std::error_condition cond =
std::system_category().default_error_condition(static_cast<int>(GetLastError()));
logger()->warn(frmt("Failed to find next file: {} ({})", cond.message(), cond.value()));
}
mtime = filesystem::last_write_time(p, ec);
if ( ec ) {
ZEEK_AGENT_DEBUG("FilesListWindows", "Failed to get file mtime: {}", ec.message());
continue;
}

rows.push_back({pattern, path, type, {}, {}, mode, mtime, size});
}

return rows;
Expand All @@ -161,52 +114,29 @@ std::vector<std::vector<Value>> FilesListWindows::snapshot(const std::vector<tab
std::vector<std::vector<Value>> FilesLinesWindows::snapshot(const std::vector<table::Argument>& args) {
std::vector<std::vector<Value>> rows;

for ( const auto& a : args ) {
if ( a.column == "_pattern" ) {
auto& pattern = std::get<std::string>(a.expression);

// TODO: how much of the linux file-globbing pattern API do we actually
// support here? Windows definitely doesn't support the whole gamut,
// like **. I could support recursion, but it doesn't at the moment.

WIN32_FIND_DATAA find_data{};
FindHandlePtr handle{FindFirstFileA(pattern.c_str(), &find_data)};
if ( handle.get() == INVALID_HANDLE_VALUE )
continue;

int find_ret = NO_ERROR;
do {
if ( (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0 ) {
std::string full_path = get_full_path(pattern, find_data.cFileName);
std::ifstream in(full_path);
if ( in.fail() ) {
// If file simply doesn't exist, we silently ignore the error.
// Otherwise we add one row with `line` unset as an error indicator.
if ( filesystem::exists(find_data.cFileName) )
rows.push_back({find_data.cFileName, {}, "<failed to open file>"});

continue;
}

int64_t number = 0;
std::string content;
// TODO: We should use a version of getline() that can abort at a given max-size.
while ( std::getline(in, content) )
rows.push_back({pattern, find_data.cFileName, ++number, trim(content)});

in.close();
}
auto [pattern, paths] = expandPaths(args);
for ( const auto& p : paths ) {
std::ifstream in(p);
if ( in.fail() ) {
// If file simply doesn't exist, we silently ignore the error.
// Otherwise we add one row with `line` unset as an error indicator.
if ( filesystem::exists(p) && ! filesystem::is_directory(p) )
// TODO: this error doesn't actually work right. i copied it from the posix file
// but i'm not sure it's being tested there either. the table requires 4 columns
// but we're only inserting 3.
rows.push_back({p.string(), {}, "<failed to open file>"});

continue;
}

find_ret = FindNextFile(handle.get(), &find_data);
int64_t number = 0;
std::string content;

} while ( find_ret != 0 );
// TODO: should use a version of getline() that can abort at a given max-size.
while ( std::getline(in, content) )
rows.push_back({pattern, p.string(), ++number, trim(content)});

if ( GetLastError() != ERROR_NO_MORE_FILES ) {
std::error_condition cond =
std::system_category().default_error_condition(static_cast<int>(GetLastError()));
logger()->warn(frmt("Failed to find next file: {} ({})", cond.message(), cond.value()));
}
}
in.close();
}

return rows;
Expand Down Expand Up @@ -238,71 +168,45 @@ std::vector<std::vector<Value>> FilesColumnsWindows::snapshot(const std::vector<
}
}

for ( const auto& a : args ) {
if ( a.column == "_pattern" ) {
auto& pattern = std::get<std::string>(a.expression);

// TODO: how much of the linux file-globbing pattern API do we actually
// support here? Windows definitely doesn't support the whole gamut,
// like **. I could support recursion, but it doesn't at the moment.

WIN32_FIND_DATAA find_data{};
FindHandlePtr handle{FindFirstFileA(pattern.c_str(), &find_data)};
if ( handle.get() == INVALID_HANDLE_VALUE )
continue;
auto [pattern, paths] = expandPaths(args);

int find_ret = NO_ERROR;
do {
if ( (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0 ) {
std::string full_path = get_full_path(pattern, find_data.cFileName);
std::ifstream in(full_path);
if ( in.fail() )
// We silently ignore any errors. If the file doesn't exist, we
// assume that's legitimate. For other errors, we don't have good
// way to record them.
continue;

int64_t number = 0;
std::string line;

// TODO: should use a version of getline() that can abort at a given max-size.
while ( std::getline(in, line) ) {
std::smatch match;
if ( ignore_regex && std::regex_search(line, match, ignore_regex.value()) )
continue;

std::vector<std::string> m;
if ( ! separator.empty() )
m = split(line, separator);
else
m = split(line);

Record value;
for ( const auto& [nr, type] : *columns ) {
if ( nr == 0 )
value.emplace_back(stringToValue(line, type));
else if ( nr >= 1 && nr <= m.size() )
value.emplace_back(stringToValue(m[nr - 1], type));
else
value.emplace_back(std::monostate(), value::Type::Null);
}

rows.push_back({pattern, spec, separator, ignore, full_path, ++number, std::move(value)});
}

in.close();
}
for ( const auto& p : paths ) {
std::ifstream in(p);
if ( in.fail() )
// We silently ignore any errors. If the file doesn't exist, we
// assume that's legitimate. For other errors, we don't have good
// way to record them.
continue;

find_ret = FindNextFile(handle.get(), &find_data);
int64_t number = 0;
std::string line;

} while ( find_ret != 0 );
// TODO: should use a version of getline() that can abort at a given max-size.
while ( std::getline(in, line) ) {
std::smatch match;
if ( ignore_regex && std::regex_search(line, match, ignore_regex.value()) )
continue;

if ( GetLastError() != ERROR_NO_MORE_FILES ) {
std::error_condition cond =
std::system_category().default_error_condition(static_cast<int>(GetLastError()));
logger()->warn(frmt("Failed to find next file: {} ({})", cond.message(), cond.value()));
std::vector<std::string> m;
if ( ! separator.empty() )
m = split(line, separator);
else
m = split(line);

Record value;
for ( const auto& [nr, type] : *columns ) {
if ( nr == 0 )
value.emplace_back(stringToValue(line, type));
else if ( nr >= 1 && nr <= m.size() )
value.emplace_back(stringToValue(m[nr - 1], type));
else
value.emplace_back(std::monostate(), value::Type::Null);
}

rows.push_back({pattern, spec, separator, ignore, p.string(), ++number, std::move(value)});
}

in.close();
}

return rows;
Expand Down
Loading