diff --git a/.gitmodules b/.gitmodules index 12d66391..ef300b9c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -46,3 +46,6 @@ [submodule "3rdparty/vcpkg"] path = 3rdparty/vcpkg url = https://github.com/microsoft/vcpkg +[submodule "3rdparty/glob"] + path = 3rdparty/glob + url = https://github.com/p-ranav/glob.git diff --git a/3rdparty/CMakeLists.txt b/3rdparty/CMakeLists.txt index 40959f78..168cdfbb 100644 --- a/3rdparty/CMakeLists.txt +++ b/3rdparty/CMakeLists.txt @@ -33,6 +33,7 @@ add_subdirectory(json EXCLUDE_FROM_ALL) add_subdirectory(stduuid EXCLUDE_FROM_ALL) add_subdirectory(tomlplusplus EXCLUDE_FROM_ALL) add_subdirectory(out_ptr EXCLUDE_FROM_ALL) +add_subdirectory(glob EXCLUDE_FROM_ALL) set(SPDLOG_FMT_EXTERNAL ON) add_subdirectory(spdlog EXCLUDE_FROM_ALL) diff --git a/3rdparty/LICENSE.3rdparty b/3rdparty/LICENSE.3rdparty index 73892352..47526c11 100644 --- a/3rdparty/LICENSE.3rdparty +++ b/3rdparty/LICENSE.3rdparty @@ -120,6 +120,32 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +================================================================================ +glob +================================================================================ + +MIT License + +Copyright (c) 2019 Pranav + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + ================================================================================ nlohmann::json ================================================================================ diff --git a/3rdparty/glob b/3rdparty/glob new file mode 160000 index 00000000..a5f32776 --- /dev/null +++ b/3rdparty/glob @@ -0,0 +1 @@ +Subproject commit a5f32776f93aaf827c8deb5ecd2857c19b9ba4a7 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9e9424c2..b74b123e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -100,6 +100,7 @@ target_link_libraries(zeek-agent PRIVATE sqlite::sqlite) target_link_libraries(zeek-agent PRIVATE stduuid) target_link_libraries(zeek-agent PRIVATE tomlplusplus::tomlplusplus) target_link_libraries(zeek-agent PRIVATE ztd::out_ptr) +target_link_libraries(zeek-agent PRIVATE Glob) if ( NOT HAVE_GETOPT_LONG ) target_link_libraries(zeek-agent PRIVATE 3rdparty:3rdparty) diff --git a/src/tables/files/files.windows.cc b/src/tables/files/files.windows.cc index 36557547..575b2d4b 100644 --- a/src/tables/files/files.windows.cc +++ b/src/tables/files/files.windows.cc @@ -47,112 +47,65 @@ database::RegisterTable _2; database::RegisterTable _3; } // namespace -static std::string get_full_path(const std::string& pattern, const char* filename) { - char ret[MAX_PATH]; - char* tmp = new char[pattern.size() + 1]; - strncpy(tmp, pattern.c_str(), pattern.size()); - tmp[pattern.size()] = '\0'; - - PathRemoveFileSpecA(tmp); - PathCombineA(ret, tmp, filename); - delete[] tmp; - return ret; -} - std::pair> FilesBase::expandPaths(const std::vector& args) { - logger()->warn("FilesBase::expandPaths is not implemented on Windows"); - return {}; -} - -std::vector FilesListWindows::buildFileRow(const std::string& pattern, const WIN32_FIND_DATAA& data) const { - std::vector row; - std::string full_path = get_full_path(pattern, data.cFileName); - - Value path = full_path; - Value type; - - uint16_t temp_mode = 0; - if ( data.dwFileAttributes & FILE_ATTRIBUTE_READONLY ) - temp_mode = 0444; - else - temp_mode = 0666; - - if ( data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY ) { - // TODO: not sure this addition to the mode is correct - temp_mode += 0111; - type = "dir"; - } - else { - // The data from FindFirstFile doesn't include a file type and the handle that's - // returned isn't a file handle (it's a search handle). Open the file separately - // and call GetFileType(), and then close it again. - DWORD file_type = FILE_TYPE_UNKNOWN; - HandlePtr file_handle{CreateFileA(full_path.c_str(), GENERIC_READ, 0, NULL, OPEN_EXISTING, 0, NULL)}; - - if ( file_handle.get() == INVALID_HANDLE_VALUE ) { - std::error_condition cond = - std::system_category().default_error_condition(static_cast(GetLastError())); - logger()->warn( - frmt("Couldn't open file {} to check for type: {} ({})", full_path, cond.message(), cond.value())); - } - else { - file_type = GetFileType(file_handle.get()); - } + std::pair> result; - // A few fields here don't match the counterparts on Linux. Added is 'remote', - // and missing is 'block' and 'socket'. - switch ( file_type ) { - case FILE_TYPE_CHAR: type = "char"; break; - // FILE_TYPE_DISK is just your average everyday file on a drive. - case FILE_TYPE_DISK: type = "file"; break; - // fifo on Linux is roughly the same thing as a Windows pipe. - case FILE_TYPE_PIPE: type = "fifo"; break; - // FILE_TYPE_REMOTE is marked as unused in the documentation. - case FILE_TYPE_REMOTE: type = "remote"; break; - case FILE_TYPE_UNKNOWN: - default: type = "other"; break; - } - } + auto glob = Table::getArgument(args, "_pattern"); + result.first = glob; - Value mode = frmt("{:o}", temp_mode); - Value size = combineHighLow(data.nFileSizeHigh, data.nFileSizeLow); - Value mtime = to_time(convertFiletime(data.ftLastWriteTime)); + for ( auto p : platform::glob(glob) ) + result.second.push_back(std::move(p)); - return {pattern, path, type, {}, {}, mode, mtime, size}; + return result; } std::vector> FilesListWindows::snapshot(const std::vector& args) { std::vector> rows; - for ( const auto& a : args ) { - if ( a.column == "_pattern" ) { - auto& pattern = std::get(a.expression); + auto [pattern, paths] = expandPaths(args); - // TODO: how much of the linux file-globbing pattern API do we actually - // support here? Windows definitely doesn't support the whole gamut, - // like **. - - WIN32_FIND_DATAA find_data{}; - FindHandlePtr handle{FindFirstFileA(pattern.c_str(), &find_data)}; - if ( handle.get() == INVALID_HANDLE_VALUE ) - continue; + for ( const auto& p : paths ) { + Value path = p.string(); + Value type; + Value mode; + Value mtime; + Value size; - int find_ret = NO_ERROR; - do { - // Ignore the . and .. directories returned by FindFirstFile - if ( strcmp(find_data.cFileName, ".") != 0 && strcmp(find_data.cFileName, "..") != 0 ) - rows.emplace_back(buildFileRow(pattern, find_data)); - - find_ret = FindNextFile(handle.get(), &find_data); + std::error_code ec; + auto status = filesystem::status(p, ec); + if ( ec ) { + ZEEK_AGENT_DEBUG("FilesListWindows", "Failed to get file status: {}", ec.message()); + continue; + } - } while ( find_ret != 0 ); + mode = frmt("{:o}", static_cast(status.permissions())); + switch ( status.type() ) { + case filesystem::file_type::none: + case filesystem::file_type::not_found: + case filesystem::file_type::symlink: + case filesystem::file_type::unknown: type = "other"; break; + case filesystem::file_type::regular: + type = "file"; + size = static_cast(filesystem::file_size(p, ec)); + if ( ec ) { + ZEEK_AGENT_DEBUG("FilesListWindows", "Failed to get file size: {}", ec.message()); + continue; + } + break; + case filesystem::file_type::directory: type = "dir"; break; + case filesystem::file_type::block: type = "block"; break; + case filesystem::file_type::character: type = "char"; break; + case filesystem::file_type::fifo: type = "fifo"; break; + case filesystem::file_type::socket: type = "socket"; break; + } - if ( GetLastError() != ERROR_NO_MORE_FILES ) { - std::error_condition cond = - std::system_category().default_error_condition(static_cast(GetLastError())); - logger()->warn(frmt("Failed to find next file: {} ({})", cond.message(), cond.value())); - } + mtime = filesystem::last_write_time(p, ec); + if ( ec ) { + ZEEK_AGENT_DEBUG("FilesListWindows", "Failed to get file mtime: {}", ec.message()); + continue; } + + rows.push_back({pattern, path, type, {}, {}, mode, mtime, size}); } return rows; @@ -161,52 +114,29 @@ std::vector> FilesListWindows::snapshot(const std::vector> FilesLinesWindows::snapshot(const std::vector& args) { std::vector> rows; - for ( const auto& a : args ) { - if ( a.column == "_pattern" ) { - auto& pattern = std::get(a.expression); - - // TODO: how much of the linux file-globbing pattern API do we actually - // support here? Windows definitely doesn't support the whole gamut, - // like **. I could support recursion, but it doesn't at the moment. - - WIN32_FIND_DATAA find_data{}; - FindHandlePtr handle{FindFirstFileA(pattern.c_str(), &find_data)}; - if ( handle.get() == INVALID_HANDLE_VALUE ) - continue; - - int find_ret = NO_ERROR; - do { - if ( (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0 ) { - std::string full_path = get_full_path(pattern, find_data.cFileName); - std::ifstream in(full_path); - if ( in.fail() ) { - // If file simply doesn't exist, we silently ignore the error. - // Otherwise we add one row with `line` unset as an error indicator. - if ( filesystem::exists(find_data.cFileName) ) - rows.push_back({find_data.cFileName, {}, ""}); - - continue; - } - - int64_t number = 0; - std::string content; - // TODO: We should use a version of getline() that can abort at a given max-size. - while ( std::getline(in, content) ) - rows.push_back({pattern, find_data.cFileName, ++number, trim(content)}); - - in.close(); - } + auto [pattern, paths] = expandPaths(args); + for ( const auto& p : paths ) { + std::ifstream in(p); + if ( in.fail() ) { + // If file simply doesn't exist, we silently ignore the error. + // Otherwise we add one row with `line` unset as an error indicator. + if ( filesystem::exists(p) && ! filesystem::is_directory(p) ) + // TODO: this error doesn't actually work right. i copied it from the posix file + // but i'm not sure it's being tested there either. the table requires 4 columns + // but we're only inserting 3. + rows.push_back({p.string(), {}, ""}); + + continue; + } - find_ret = FindNextFile(handle.get(), &find_data); + int64_t number = 0; + std::string content; - } while ( find_ret != 0 ); + // TODO: should use a version of getline() that can abort at a given max-size. + while ( std::getline(in, content) ) + rows.push_back({pattern, p.string(), ++number, trim(content)}); - if ( GetLastError() != ERROR_NO_MORE_FILES ) { - std::error_condition cond = - std::system_category().default_error_condition(static_cast(GetLastError())); - logger()->warn(frmt("Failed to find next file: {} ({})", cond.message(), cond.value())); - } - } + in.close(); } return rows; @@ -238,71 +168,45 @@ std::vector> FilesColumnsWindows::snapshot(const std::vector< } } - for ( const auto& a : args ) { - if ( a.column == "_pattern" ) { - auto& pattern = std::get(a.expression); - - // TODO: how much of the linux file-globbing pattern API do we actually - // support here? Windows definitely doesn't support the whole gamut, - // like **. I could support recursion, but it doesn't at the moment. - - WIN32_FIND_DATAA find_data{}; - FindHandlePtr handle{FindFirstFileA(pattern.c_str(), &find_data)}; - if ( handle.get() == INVALID_HANDLE_VALUE ) - continue; + auto [pattern, paths] = expandPaths(args); - int find_ret = NO_ERROR; - do { - if ( (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0 ) { - std::string full_path = get_full_path(pattern, find_data.cFileName); - std::ifstream in(full_path); - if ( in.fail() ) - // We silently ignore any errors. If the file doesn't exist, we - // assume that's legitimate. For other errors, we don't have good - // way to record them. - continue; - - int64_t number = 0; - std::string line; - - // TODO: should use a version of getline() that can abort at a given max-size. - while ( std::getline(in, line) ) { - std::smatch match; - if ( ignore_regex && std::regex_search(line, match, ignore_regex.value()) ) - continue; - - std::vector m; - if ( ! separator.empty() ) - m = split(line, separator); - else - m = split(line); - - Record value; - for ( const auto& [nr, type] : *columns ) { - if ( nr == 0 ) - value.emplace_back(stringToValue(line, type)); - else if ( nr >= 1 && nr <= m.size() ) - value.emplace_back(stringToValue(m[nr - 1], type)); - else - value.emplace_back(std::monostate(), value::Type::Null); - } - - rows.push_back({pattern, spec, separator, ignore, full_path, ++number, std::move(value)}); - } - - in.close(); - } + for ( const auto& p : paths ) { + std::ifstream in(p); + if ( in.fail() ) + // We silently ignore any errors. If the file doesn't exist, we + // assume that's legitimate. For other errors, we don't have good + // way to record them. + continue; - find_ret = FindNextFile(handle.get(), &find_data); + int64_t number = 0; + std::string line; - } while ( find_ret != 0 ); + // TODO: should use a version of getline() that can abort at a given max-size. + while ( std::getline(in, line) ) { + std::smatch match; + if ( ignore_regex && std::regex_search(line, match, ignore_regex.value()) ) + continue; - if ( GetLastError() != ERROR_NO_MORE_FILES ) { - std::error_condition cond = - std::system_category().default_error_condition(static_cast(GetLastError())); - logger()->warn(frmt("Failed to find next file: {} ({})", cond.message(), cond.value())); + std::vector m; + if ( ! separator.empty() ) + m = split(line, separator); + else + m = split(line); + + Record value; + for ( const auto& [nr, type] : *columns ) { + if ( nr == 0 ) + value.emplace_back(stringToValue(line, type)); + else if ( nr >= 1 && nr <= m.size() ) + value.emplace_back(stringToValue(m[nr - 1], type)); + else + value.emplace_back(std::monostate(), value::Type::Null); } + + rows.push_back({pattern, spec, separator, ignore, p.string(), ++number, std::move(value)}); } + + in.close(); } return rows; diff --git a/src/util/platform.windows.cc b/src/util/platform.windows.cc index 42b291e0..c6f445fc 100644 --- a/src/util/platform.windows.cc +++ b/src/util/platform.windows.cc @@ -14,6 +14,7 @@ #include +#include #include using namespace zeek::agent; @@ -54,8 +55,17 @@ std::optional platform::dataDirectory() { bool platform::isTTY() { return true; } std::vector platform::glob(const filesystem::path& pattern, size_t max) { - logger()->warn("platform::glob is not implemented on Windows"); - return {}; + // glob::glob returns std::filesystem::path, but we're using ghc::filesystem for compatibility + // reasons. this means we need to copy the paths from one vector type to another here. + auto paths = glob::glob(pattern.string()); + if ( paths.size() > max ) + paths.resize(max); + + std::vector ret; + for ( const auto& path : paths ) { + ret.emplace_back(path.string()); + } + return ret; } int platform::setenv(const char* name, const char* value, int overwrite) {