From c5fa470ad82fd9f58991a3a25326677007035e4c Mon Sep 17 00:00:00 2001 From: Shawn Hoffman Date: Tue, 20 Jun 2017 21:24:26 -0700 Subject: [PATCH] replace DoFileSearch with optimized version --- Source/Core/Common/FileSearch.cpp | 85 +++++++++++++++++-- Source/Core/Common/FileSearch.h | 8 +- .../Core/HW/GCMemcard/GCMemcardDirectory.cpp | 2 +- Source/Core/DolphinQt2/Settings.cpp | 2 +- .../DolphinQt2/Settings/InterfacePane.cpp | 5 +- .../DolphinWX/Config/InterfaceConfigPane.cpp | 5 +- Source/Core/DolphinWX/FrameTools.cpp | 2 +- Source/Core/DolphinWX/GameListCtrl.cpp | 11 +-- Source/Core/DolphinWX/GameListCtrl.h | 2 + .../Core/DolphinWX/Input/InputConfigDiag.cpp | 2 +- Source/Core/VideoCommon/HiresTextures.cpp | 2 +- Source/Core/VideoCommon/PostProcessing.cpp | 5 +- 12 files changed, 97 insertions(+), 34 deletions(-) diff --git a/Source/Core/Common/FileSearch.cpp b/Source/Core/Common/FileSearch.cpp index 29b8624b15..783c22d864 100644 --- a/Source/Core/Common/FileSearch.cpp +++ b/Source/Core/Common/FileSearch.cpp @@ -7,10 +7,19 @@ #include "Common/CommonPaths.h" #include "Common/FileSearch.h" + +#ifdef _MSC_VER +#include +namespace fs = std::experimental::filesystem; +#define HAS_STD_FILESYSTEM +#else #include "Common/FileUtil.h" +#endif namespace Common { +#ifndef HAS_STD_FILESYSTEM + static std::vector FileSearchWithTest(const std::vector& directories, bool recursive, std::function callback) @@ -36,10 +45,10 @@ FileSearchWithTest(const std::vector& directories, bool recursive, return result; } -std::vector DoFileSearch(const std::vector& exts, - const std::vector& directories, bool recursive) +std::vector DoFileSearchNoSTL(const std::vector& directories, + const std::vector& exts, bool recursive) { - bool accept_all = std::find(exts.begin(), exts.end(), "") != exts.end(); + bool accept_all = exts.empty(); return FileSearchWithTest(directories, recursive, [&](const File::FSTEntry& entry) { if (accept_all) return true; @@ -52,11 +61,71 @@ std::vector DoFileSearch(const std::vector& exts, }); } -// Result includes the passed directories themselves as well as their subdirectories. -std::vector FindSubdirectories(const std::vector& directories, - bool recursive) +std::vector DoFileSearch(const std::vector& directories, + const std::vector& exts, bool recursive) { - return FileSearchWithTest(directories, true, - [&](const File::FSTEntry& entry) { return entry.isDirectory; }); + return DoFileSearchNoSTL(directories, exts, recursive); } + +#else + +std::vector DoFileSearch(const std::vector& directories, + const std::vector& exts, bool recursive) +{ + bool accept_all = exts.empty(); + + std::vector native_exts; + for (const auto& ext : exts) + native_exts.push_back(ext); + + // N.B. This avoids doing any copies + auto ext_matches = [&native_exts](const fs::path& path) { + const auto& native_path = path.native(); + return std::any_of(native_exts.cbegin(), native_exts.cend(), [&native_path](const auto& ext) { + // TODO provide cross-platform compat for the comparison function, once more platforms + // support std::filesystem + return native_path.length() >= ext.native().length() && + _wcsicmp(&native_path.c_str()[native_path.length() - ext.native().length()], + ext.c_str()) == 0; + }); + }; + + std::vector result; + auto add_filtered = [&](const fs::directory_entry& entry) { + auto& path = entry.path(); + if (accept_all || (ext_matches(path) && !fs::is_directory(path))) + result.emplace_back(path.u8string()); + }; + for (const auto& directory : directories) + { + if (recursive) + { + // TODO use fs::directory_options::follow_directory_symlink ? + for (auto& entry : fs::recursive_directory_iterator(fs::path(directory.c_str()))) + add_filtered(entry); + } + else + { + for (auto& entry : fs::directory_iterator(fs::path(directory.c_str()))) + add_filtered(entry); + } + } + + // Remove duplicates (occurring because caller gave e.g. duplicate or overlapping directories - + // not because std::filesystem returns duplicates). Also note that this pathname-based uniqueness + // isn't as thorough as std::filesystem::equivalent. + std::sort(result.begin(), result.end()); + result.erase(std::unique(result.begin(), result.end()), result.end()); + + // Dolphin expects to be able to use "/" (DIR_SEP) everywhere. std::filesystem uses the OS + // separator. + if (fs::path::preferred_separator != DIR_SEP_CHR) + for (auto& path : result) + std::replace(path.begin(), path.end(), '\\', DIR_SEP_CHR); + + return result; +} + +#endif + } // namespace Common diff --git a/Source/Core/Common/FileSearch.h b/Source/Core/Common/FileSearch.h index dc2f7f6b62..dc38db945a 100644 --- a/Source/Core/Common/FileSearch.h +++ b/Source/Core/Common/FileSearch.h @@ -9,9 +9,9 @@ namespace Common { -std::vector DoFileSearch(const std::vector& exts, - const std::vector& directories, +// Callers can pass empty "exts" to indicate they want all files + directories in results +// Otherwise, only files matching the extensions are returned +std::vector DoFileSearch(const std::vector& directories, + const std::vector& exts = {}, bool recursive = false); -std::vector FindSubdirectories(const std::vector& directories, - bool recursive); } // namespace Common diff --git a/Source/Core/Core/HW/GCMemcard/GCMemcardDirectory.cpp b/Source/Core/Core/HW/GCMemcard/GCMemcardDirectory.cpp index f12751c8be..3dcdb64d11 100644 --- a/Source/Core/Core/HW/GCMemcard/GCMemcardDirectory.cpp +++ b/Source/Core/Core/HW/GCMemcard/GCMemcardDirectory.cpp @@ -148,7 +148,7 @@ GCMemcardDirectory::GCMemcardDirectory(const std::string& directory, int slot, u hdr_file.ReadBytes(&m_hdr, BLOCK_SIZE); } - std::vector filenames = Common::DoFileSearch({".gci"}, {m_save_directory}); + std::vector filenames = Common::DoFileSearch({m_save_directory}, {".gci"}); if (filenames.size() > 112) { diff --git a/Source/Core/DolphinQt2/Settings.cpp b/Source/Core/DolphinQt2/Settings.cpp index 346b802237..bac3a4143f 100644 --- a/Source/Core/DolphinQt2/Settings.cpp +++ b/Source/Core/DolphinQt2/Settings.cpp @@ -365,7 +365,7 @@ QVector Settings::GetProfiles(const InputConfig* config) const const std::string path = GetProfilesDir().toStdString() + config->GetProfileName(); QVector vec; - for (const auto& file : Common::DoFileSearch({".ini"}, {path})) + for (const auto& file : Common::DoFileSearch({path}, {".ini"})) { std::string basename; SplitPath(file, nullptr, &basename, nullptr); diff --git a/Source/Core/DolphinQt2/Settings/InterfacePane.cpp b/Source/Core/DolphinQt2/Settings/InterfacePane.cpp index b34f66f1f6..84998ea13d 100644 --- a/Source/Core/DolphinQt2/Settings/InterfacePane.cpp +++ b/Source/Core/DolphinQt2/Settings/InterfacePane.cpp @@ -59,9 +59,8 @@ void InterfacePane::CreateUI() combobox_layout->addRow(tr("&Theme:"), m_combobox_theme); // List avalable themes - auto file_search_results = Common::DoFileSearch( - {""}, {File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR}, - /*recursive*/ false); + auto file_search_results = + Common::DoFileSearch({File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR}); for (const std::string& filename : file_search_results) { std::string name, ext; diff --git a/Source/Core/DolphinWX/Config/InterfaceConfigPane.cpp b/Source/Core/DolphinWX/Config/InterfaceConfigPane.cpp index 278e1697c3..d3308bc22e 100644 --- a/Source/Core/DolphinWX/Config/InterfaceConfigPane.cpp +++ b/Source/Core/DolphinWX/Config/InterfaceConfigPane.cpp @@ -196,9 +196,8 @@ void InterfaceConfigPane::LoadGUIValues() void InterfaceConfigPane::LoadThemes() { - auto sv = Common::DoFileSearch( - {""}, {File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR}, - /*recursive*/ false); + auto sv = + Common::DoFileSearch({File::GetUserPath(D_THEMES_IDX), File::GetSysDirectory() + THEMES_DIR}); for (const std::string& filename : sv) { std::string name, ext; diff --git a/Source/Core/DolphinWX/FrameTools.cpp b/Source/Core/DolphinWX/FrameTools.cpp index 72addd999b..ddb5f99b1e 100644 --- a/Source/Core/DolphinWX/FrameTools.cpp +++ b/Source/Core/DolphinWX/FrameTools.cpp @@ -1687,7 +1687,7 @@ void CFrame::GameListChanged(wxCommandEvent& event) break; case IDM_PURGE_GAME_LIST_CACHE: std::vector filenames = - Common::DoFileSearch({".cache"}, {File::GetUserPath(D_CACHE_IDX)}); + Common::DoFileSearch({File::GetUserPath(D_CACHE_IDX)}, {".cache"}); for (const std::string& filename : filenames) { diff --git a/Source/Core/DolphinWX/GameListCtrl.cpp b/Source/Core/DolphinWX/GameListCtrl.cpp index 7482caadef..b79919b539 100644 --- a/Source/Core/DolphinWX/GameListCtrl.cpp +++ b/Source/Core/DolphinWX/GameListCtrl.cpp @@ -743,10 +743,9 @@ void GameListCtrl::RescanList() const std::vector search_extensions = {".gcm", ".tgc", ".iso", ".ciso", ".gcz", ".wbfs", ".wad", ".dol", ".elf"}; // TODO This could process paths iteratively as they are found - auto search_results = Common::DoFileSearch(search_extensions, SConfig::GetInstance().m_ISOFolder, + auto search_results = Common::DoFileSearch(SConfig::GetInstance().m_ISOFolder, search_extensions, SConfig::GetInstance().m_RecursiveISOFolder); - // TODO rethink some good algorithms to use here std::vector cached_paths; for (const auto& file : m_cached_files) cached_paths.emplace_back(file->GetFileName()); @@ -761,14 +760,8 @@ void GameListCtrl::RescanList() cached_paths.cend(), std::back_inserter(new_paths)); const Core::TitleDatabase title_database; - // TODO we could store all paths and modification times to judge if file needs to be rescanned. - // If we cached paths that turned out to be invalid, this would save failing on them each time - // refresh is done. - // However if people e.g. set dolphin to recursively scan the root of their drive(s), then we - // would cache way too much data. Maybe just use an upper bound of invalid paths to cache? // For now, only scan new_paths. This could cause false negatives (file actively being written), - // but otherwise - // should be fine. + // but otherwise should be fine. for (const auto& path : removed_paths) { auto it = std::find_if( diff --git a/Source/Core/DolphinWX/GameListCtrl.h b/Source/Core/DolphinWX/GameListCtrl.h index 6092018b16..f8bdfce19f 100644 --- a/Source/Core/DolphinWX/GameListCtrl.h +++ b/Source/Core/DolphinWX/GameListCtrl.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -14,6 +15,7 @@ #include "Common/ChunkFile.h" #include "Common/Event.h" +#include "Common/Flag.h" #include "DolphinWX/ISOFile.h" class wxEmuStateTip : public wxTipWindow diff --git a/Source/Core/DolphinWX/Input/InputConfigDiag.cpp b/Source/Core/DolphinWX/Input/InputConfigDiag.cpp index f1e4e7e2f9..8aaa51ccb0 100644 --- a/Source/Core/DolphinWX/Input/InputConfigDiag.cpp +++ b/Source/Core/DolphinWX/Input/InputConfigDiag.cpp @@ -264,7 +264,7 @@ void InputConfigDialog::UpdateProfileComboBox() pname += PROFILES_PATH; pname += m_config.GetProfileName(); - std::vector sv = Common::DoFileSearch({".ini"}, {pname}); + std::vector sv = Common::DoFileSearch({pname}, {".ini"}); wxArrayString strs; for (const std::string& filename : sv) diff --git a/Source/Core/VideoCommon/HiresTextures.cpp b/Source/Core/VideoCommon/HiresTextures.cpp index 7b4a1d387c..36b170ef10 100644 --- a/Source/Core/VideoCommon/HiresTextures.cpp +++ b/Source/Core/VideoCommon/HiresTextures.cpp @@ -97,7 +97,7 @@ void HiresTexture::Update() }; std::vector filenames = - Common::DoFileSearch(extensions, {texture_directory}, /*recursive*/ true); + Common::DoFileSearch({texture_directory}, extensions, /*recursive*/ true); const std::string code = game_id + "_"; diff --git a/Source/Core/VideoCommon/PostProcessing.cpp b/Source/Core/VideoCommon/PostProcessing.cpp index 71f38732e7..0575402dd1 100644 --- a/Source/Core/VideoCommon/PostProcessing.cpp +++ b/Source/Core/VideoCommon/PostProcessing.cpp @@ -31,8 +31,9 @@ PostProcessingShaderImplementation::~PostProcessingShaderImplementation() static std::vector GetShaders(const std::string& sub_dir = "") { std::vector paths = - Common::DoFileSearch({".glsl"}, {File::GetUserPath(D_SHADERS_IDX) + sub_dir, - File::GetSysDirectory() + SHADERS_DIR DIR_SEP + sub_dir}); + Common::DoFileSearch({File::GetUserPath(D_SHADERS_IDX) + sub_dir, + File::GetSysDirectory() + SHADERS_DIR DIR_SEP + sub_dir}, + {".glsl"}); std::vector result; for (std::string path : paths) {