duckdb::LocalGlobResult Struct Reference

Inheritance diagram for duckdb::LocalGlobResult:

Collaboration diagram for duckdb::LocalGlobResult:

Public Member Functions
	LocalGlobResult (LocalFileSystem &fs, const string &path, FileGlobOptions options, optional_ptr< FileOpener > opener)
Public Member Functions inherited from duckdb::LazyMultiFileList
	LazyMultiFileList (optional_ptr< ClientContext > context)
vector< OpenFileInfo >	GetAllFiles () const override
FileExpandResult	GetExpandResult () const override
idx_t	GetTotalFileCount () const override
	Get the total file count - forces all files to be expanded / known so the exact count can be computed.
MultiFileCount	GetFileCount (idx_t min_exact_count=0) const override
Public Member Functions inherited from duckdb::MultiFileList
MultiFileListIterationHelper	Files () const
	Get Iterator over the files for pretty for loops.
void	InitializeScan (MultiFileListScanData &iterator) const
	Initialize a sequential scan over a file list.
bool	Scan (MultiFileListScanData &iterator, OpenFileInfo &result_file) const
	Scan the next file into result_file, returns false when out of files.
OpenFileInfo	GetFirstFile () const
	Returns the first file or an empty string if GetTotalFileCount() == 0.
bool	IsEmpty () const
	Syntactic sugar for GetExpandResult() == FileExpandResult::NO_FILES.
virtual unique_ptr< MultiFileList >	ComplexFilterPushdown (ClientContext &context, const MultiFileOptions &options, MultiFilePushdownInfo &info, vector< unique_ptr< Expression > > &filters) const
	Virtual functions for subclasses.
virtual unique_ptr< MultiFileList >	DynamicFilterPushdown (ClientContext &context, const MultiFileOptions &options, const vector< string > &names, const vector< LogicalType > &types, const vector< column_t > &column_ids, TableFilterSet &filters) const
virtual vector< OpenFileInfo >	GetDisplayFileList (optional_idx max_files=optional_idx()) const
virtual unique_ptr< NodeStatistics >	GetCardinality (ClientContext &context) const
virtual unique_ptr< MultiFileList >	Copy () const
template<class TARGET >
TARGET &	Cast ()
template<class TARGET >
const TARGET &	Cast () const

Protected Member Functions
bool	ExpandNextPath () const override
	Grabs the next path and expands it into Expanded paths: returns false if no more files to expand.
Protected Member Functions inherited from duckdb::LazyMultiFileList
bool	FileIsAvailable (idx_t i) const override
	Whether or not the file at the index is available instantly - or if this requires additional I/O.
OpenFileInfo	GetFile (idx_t i) const override
	Get the i-th expanded file.

Private Attributes
LocalFileSystem &	fs
string	path
optional_ptr< FileOpener >	opener
vector< PathSplit >	splits
bool	absolute_path = false
std::priority_queue< ExpandDirectory >	expand_directories
bool	finished = false

Additional Inherited Members
Protected Attributes inherited from duckdb::LazyMultiFileList
mutex	lock
vector< OpenFileInfo >	expanded_files
	The expanded files.
bool	all_files_expanded = false
	Whether or not all files have been expanded.
optional_ptr< ClientContext >	context

Constructor & Destructor Documentation

LocalGlobResult()

duckdb::LocalGlobResult::LocalGlobResult	(	LocalFileSystem &	fs,
		const string &	path,
		FileGlobOptions	options,
		optional_ptr< FileOpener >	opener
	)

    : LazyMultiFileList(FileOpener::TryGetClientContext(opener)), fs(fs), path(fs.ExpandPath(path_p, opener)),
      opener(opener) {
    if (path.empty()) {
        finished = true;
        return;
    }
    // split up the path into separate chunks
    idx_t last_pos = 0;
    for (idx_t i = 0; i < path.size(); i++) {
        if (path[i] == '\\' || path[i] == '/') {
            if (i == last_pos) {
                // empty: skip this position
                last_pos = i + 1;
                continue;
            }
            if (splits.empty()) {
                splits.emplace_back(fs, path.substr(0, i));
            } else {
                splits.emplace_back(fs, path.substr(last_pos, i - last_pos));
            }
            last_pos = i + 1;
        }
    }
    splits.emplace_back(fs, path.substr(last_pos, path.size() - last_pos));
    // handle absolute paths
    absolute_path = false;
    if (fs.IsPathAbsolute(path)) {
        // first character is a slash -  unix absolute path
        absolute_path = true;
    } else if (StringUtil::Contains(splits[0].path,
                                    ":")) { // TODO: this is weird? shouldn't IsPathAbsolute handle this?
        // first split has a colon -  windows absolute path
        absolute_path = true;
    } else if (splits[0].path == "~") {
        // starts with home directory
        auto home_directory = fs.GetHomeDirectory(opener);
        if (!home_directory.empty()) {
            absolute_path = true;
            splits[0].path = home_directory;
            D_ASSERT(path[0] == '~');
            if (!fs.HasGlob(path)) {
                expanded_files = fs.FetchFileWithoutGlob(home_directory + path.substr(1), opener, absolute_path);
                finished = true;
                return;
            }
        }
    }
    // Check if the path has a glob at all
    if (!fs.HasGlob(path)) {
        // no glob: return only the file (if it exists or is a pipe)
        expanded_files = fs.FetchFileWithoutGlob(path, opener, absolute_path);
        finished = true;
        return;
    }
    if (absolute_path) {
        // for absolute paths, we don't start by scanning the current directory
        // FIXME: we don't support /[GLOB]/.. - i.e. globs in the first level of an absolute path
        if (splits.size() > 1) {
            expand_directories.emplace(splits[0].path, 1);
        }
    } else {
        // If file_search_path is set, use those paths as the first glob elements
        Value value;
        if (opener && opener->TryGetCurrentSetting("file_search_path", value)) {
            auto search_paths_str = value.ToString();
            auto search_paths = StringUtil::Split(search_paths_str, ',');
            for (const auto &search_path : search_paths) {
                expand_directories.emplace(search_path, 0);
            }
        }
        if (expand_directories.empty()) {
            expand_directories.emplace(".", 0, true);
        }
    }
 
    if (HasMultipleCrawl(splits)) {
        throw IOException("Cannot use multiple \'**\' in one path");
    }
}

Member Function Documentation

ExpandNextPath()

bool duckdb::LocalGlobResult::ExpandNextPath ( ) const

overrideprotectedvirtual

Grabs the next path and expands it into Expanded paths: returns false if no more files to expand.

Implements duckdb::LazyMultiFileList.

                                           {
    if (finished) {
        return false;
    }
    if (expand_directories.empty()) {
        if (expanded_files.empty()) {
            // no result found that matches the glob
            // last ditch effort: search the path as a string literal
            expanded_files = fs.FetchFileWithoutGlob(path, opener, absolute_path);
        }
        finished = true;
        return false;
    }
 
    auto next_dir = expand_directories.top();
    auto is_empty = next_dir.is_empty;
    auto split_index = next_dir.split_index;
    auto &current_path = next_dir.path;
    expand_directories.pop();
 
    auto &next_split = splits[split_index];
    bool is_last_component = split_index + 1 == splits.size();
    auto &next_component = next_split.path;
    bool has_glob = next_split.has_glob;
    // if it's the last chunk we need to find files, otherwise we find directories
    // not the last chunk: gather a list of all directories that match the glob pattern
    if (!has_glob) {
        // no glob, just append as-is
        if (is_empty) {
            if (is_last_component) {
                throw InternalException("No glob in only component - but entire split has globs?");
            }
            // no path yet - just append
            expand_directories.emplace(next_component, split_index + 1);
        } else {
            if (is_last_component) {
                // last component - we are emitting a result here
                auto filename = fs.JoinPath(current_path, next_component);
                if (fs.FileExists(filename, opener) || fs.DirectoryExists(filename, opener)) {
                    expanded_files.emplace_back(std::move(filename));
                }
            } else {
                // not the last component - add the next directory as "to-be-expanded"
                expand_directories.emplace(fs.JoinPath(current_path, next_component), split_index + 1);
            }
        }
    } else {
        // glob - need to resolve the glob
        if (IsCrawl(next_component)) {
            if (is_last_component) {
                // the crawl is the last component - we are looking for files in this directory
                // any directories we encounter are added to the expand directories
                CrawlDirectoryLevel(fs, current_path, expanded_files, expand_directories, split_index);
            } else {
                // not the last crawl
                // ** also matches the current directory (i.e. dir/**/file.parquet also matches dir/file.parquet)
                expand_directories.emplace(current_path, split_index + 1);
                // now crawl the contents of this directory - but don't add any files we find
                CrawlDirectoryLevel(fs, current_path, nullptr, expand_directories, split_index);
            }
        } else {
            // glob this directory according to the next component
            if (is_last_component) {
                // last component - match files and place them in the result
                GlobFilesInternal(fs, current_path, next_component, false, expanded_files);
            } else {
                // not the last component - match directories and add to expansion list
                vector<OpenFileInfo> child_directories;
                GlobFilesInternal(fs, current_path, next_component, true, child_directories);
                for (auto &file : child_directories) {
                    expand_directories.emplace(std::move(file.path), split_index + 1);
                }
            }
        }
    }
    return true;
}

Here is the call graph for this function:

---

The documentation for this struct was generated from the following file:

• external/duckdb/duckdb.cpp