duckdb::FileSystem Class Reference

Inheritance diagram for duckdb::FileSystem:

Public Member Functions
virtual DUCKDB_API unique_ptr< FileHandle >	OpenFile (const string &path, FileOpenFlags flags, optional_ptr< FileOpener > opener=nullptr)
DUCKDB_API unique_ptr< FileHandle >	OpenFile (const OpenFileInfo &path, FileOpenFlags flags, optional_ptr< FileOpener > opener=nullptr)
virtual DUCKDB_API void	Read (FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location)
virtual DUCKDB_API void	Write (FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location)
virtual DUCKDB_API int64_t	Read (FileHandle &handle, void *buffer, int64_t nr_bytes)
virtual DUCKDB_API int64_t	Write (FileHandle &handle, void *buffer, int64_t nr_bytes)
	Write nr_bytes from the buffer into the file, moving the file pointer forward by nr_bytes.
virtual DUCKDB_API bool	Trim (FileHandle &handle, idx_t offset_bytes, idx_t length_bytes)
virtual DUCKDB_API int64_t	GetFileSize (FileHandle &handle)
	Returns the file size of a file handle, returns -1 on error.
virtual DUCKDB_API timestamp_t	GetLastModifiedTime (FileHandle &handle)
	Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error.
virtual DUCKDB_API string	GetVersionTag (FileHandle &handle)
virtual DUCKDB_API FileType	GetFileType (FileHandle &handle)
	Returns the file type of the attached handle.
virtual DUCKDB_API FileMetadata	Stats (FileHandle &handle)
	Returns the file stats of the attached handle.
virtual DUCKDB_API void	Truncate (FileHandle &handle, int64_t new_size)
virtual DUCKDB_API bool	DirectoryExists (const string &directory, optional_ptr< FileOpener > opener=nullptr)
	Check if a directory exists.
virtual DUCKDB_API void	CreateDirectory (const string &directory, optional_ptr< FileOpener > opener=nullptr)
	Create a directory if it does not exist.
virtual DUCKDB_API void	CreateDirectoriesRecursive (const string &path, optional_ptr< FileOpener > opener=nullptr)
	Helper function that uses DirectoryExists and CreateDirectory to ensure all directories in path are created.
virtual DUCKDB_API void	RemoveDirectory (const string &directory, optional_ptr< FileOpener > opener=nullptr)
	Recursively remove a directory and all files in it.
virtual DUCKDB_API bool	ListFiles (const string &directory, const std::function< void(const string &, bool)> &callback, FileOpener *opener=nullptr)
	List files in a directory, invoking the callback method for each one with (filename, is_dir)
DUCKDB_API bool	ListFiles (const string &directory, const std::function< void(OpenFileInfo &info)> &callback, optional_ptr< FileOpener > opener=nullptr)
virtual DUCKDB_API void	MoveFile (const string &source, const string &target, optional_ptr< FileOpener > opener=nullptr)
virtual DUCKDB_API bool	FileExists (const string &filename, optional_ptr< FileOpener > opener=nullptr)
	Check if a file exists.
virtual DUCKDB_API bool	IsPipe (const string &filename, optional_ptr< FileOpener > opener=nullptr)
	Check if path is pipe.
virtual DUCKDB_API void	RemoveFile (const string &filename, optional_ptr< FileOpener > opener=nullptr)
	Remove a file from disk.
virtual DUCKDB_API bool	TryRemoveFile (const string &filename, optional_ptr< FileOpener > opener=nullptr)
	Remvoe a file from disk if it exists - if it does not exist, return false.
virtual DUCKDB_API void	RemoveFiles (const vector< string > &filenames, optional_ptr< FileOpener > opener=nullptr)
	Remove multiple files from disk - does not error if any file does not exist.
virtual DUCKDB_API void	FileSync (FileHandle &handle)
	Sync a file handle to disk.
virtual DUCKDB_API string	GetHomeDirectory ()
	Gets the users home directory.
virtual DUCKDB_API string	ExpandPath (const string &path)
	Expands a given path, including e.g. expanding the home directory of the user.
virtual DUCKDB_API string	PathSeparator (const string &path)
	Path separator for path.
virtual DUCKDB_API bool	IsPathAbsolute (const string &path)
	Checks if path is is an absolute path.
DUCKDB_API string	JoinPath (const string &a, const string &path)
	Join two paths together.
template<typename... ARGS>
string	JoinPath (const string &a, const string &b, ARGS... args)
DUCKDB_API string	ConvertSeparators (const string &path)
	Convert separators in a path to the local separators (e.g. convert "/" into \ on windows)
DUCKDB_API string	ExtractBaseName (const string &path)
	Extract the base name of a file (e.g. if the input is lib/example.dll the base name is 'example')
DUCKDB_API string	ExtractExtension (const string &path)
	Extract the extension of a file (e.g. if the input is lib/example.dll the extension is 'dll')
DUCKDB_API string	ExtractName (const string &path)
	Extract the name of a file (e.g if the input is lib/example.dll the name is 'example.dll')
virtual DUCKDB_API vector< OpenFileInfo >	Glob (const string &path, FileOpener *opener=nullptr)
	Runs a glob on the file system, returning a list of matching files.
DUCKDB_API unique_ptr< MultiFileList >	Glob (const string &path, const FileGlobInput &input, optional_ptr< FileOpener > opener)
DUCKDB_API unique_ptr< MultiFileList >	GlobFileList (const string &path, const FileGlobInput &input=FileGlobOptions::DISALLOW_EMPTY)
DUCKDB_API vector< OpenFileInfo >	GlobFiles (const string &pattern, const FileGlobInput &input=FileGlobOptions::DISALLOW_EMPTY)
virtual DUCKDB_API void	RegisterSubSystem (unique_ptr< FileSystem > sub_fs)
	registers a sub-file system to handle certain file name prefixes, e.g. http:// etc.
virtual DUCKDB_API void	RegisterSubSystem (FileCompressionType compression_type, unique_ptr< FileSystem > fs)
virtual DUCKDB_API void	UnregisterSubSystem (const string &name)
	Unregister a sub-filesystem by name.
virtual DUCKDB_API unique_ptr< FileSystem >	ExtractSubSystem (const string &name)
virtual DUCKDB_API vector< string >	ListSubSystems ()
	List registered sub-filesystems, including builtin ones.
virtual DUCKDB_API bool	CanHandleFile (const string &fpath)
	Whether or not a sub-system can handle a specific file path.
virtual DUCKDB_API void	Seek (FileHandle &handle, idx_t location)
	Set the file pointer of a file handle to a specified location. Reads and writes will happen from this location.
virtual DUCKDB_API void	Reset (FileHandle &handle)
	Reset a file to the beginning (equivalent to Seek(handle, 0) for simple files)
virtual DUCKDB_API idx_t	SeekPosition (FileHandle &handle)
virtual DUCKDB_API bool	IsManuallySet ()
	If FS was manually set by the user.
virtual DUCKDB_API bool	CanSeek ()
	Whether or not we can seek into the file.
virtual DUCKDB_API bool	OnDiskFile (FileHandle &handle)
virtual DUCKDB_API unique_ptr< FileHandle >	OpenCompressedFile (QueryContext context, unique_ptr< FileHandle > handle, bool write)
virtual DUCKDB_API std::string	GetName () const =0
	Return the name of the filesytem. Used for forming diagnosis messages.
virtual DUCKDB_API void	SetDisabledFileSystems (const vector< string > &names)
virtual DUCKDB_API bool	SubSystemIsDisabled (const string &name)
virtual DUCKDB_API bool	IsDisabledForPath (const string &path)
	Check if the filesystem that would handle this path is disabled.
virtual DUCKDB_API string	CanonicalizePath (const string &path, optional_ptr< FileOpener > opener=nullptr)
	Canonicalize a path.
template<class TARGET >
TARGET &	Cast ()
template<class TARGET >
const TARGET &	Cast () const

Static Public Member Functions
static DUCKDB_API FileSystem &	GetFileSystem (ClientContext &context)
static DUCKDB_API FileSystem &	GetFileSystem (DatabaseInstance &db)
static DUCKDB_API FileSystem &	Get (AttachedDatabase &db)
static DUCKDB_API void	SetWorkingDirectory (const string &path)
	Sets the working directory.
static DUCKDB_API string	GetWorkingDirectory ()
	Gets the working directory.
static DUCKDB_API string	GetHomeDirectory (optional_ptr< FileOpener > opener)
	Gets the users home directory.
static DUCKDB_API string	ExpandPath (const string &path, optional_ptr< FileOpener > opener)
	Expands a given path, including e.g. expanding the home directory of the user.
static DUCKDB_API optional_idx	GetAvailableMemory ()
	Returns the system-available memory in bytes. Returns DConstants::INVALID_INDEX if the system function fails.
static DUCKDB_API optional_idx	GetAvailableDiskSpace (const string &path)
	Returns the space available on the disk. Returns DConstants::INVALID_INDEX if the information was not available.
static DUCKDB_API string	GetEnvVariable (const string &name)
	Returns the value of an environment variable - or the empty string if it is not set.
static DUCKDB_API bool	HasGlob (const string &str)
	Whether there is a glob in the string.
static DUCKDB_API unique_ptr< FileSystem >	CreateLocal ()
	Create a LocalFileSystem.
static DUCKDB_API bool	IsRemoteFile (const string &path)
	Whether or not a file is remote or local, based only on file path.
static DUCKDB_API bool	IsRemoteFile (const string &path, string &extension)
static DUCKDB_API bool	IsDirectory (const OpenFileInfo &info)

Protected Member Functions
virtual DUCKDB_API unique_ptr< FileHandle >	OpenFileExtended (const OpenFileInfo &path, FileOpenFlags flags, optional_ptr< FileOpener > opener)
virtual DUCKDB_API bool	SupportsOpenFileExtended () const
virtual DUCKDB_API bool	ListFilesExtended (const string &directory, const std::function< void(OpenFileInfo &info)> &callback, optional_ptr< FileOpener > opener)
virtual DUCKDB_API bool	SupportsListFilesExtended () const
virtual DUCKDB_API unique_ptr< MultiFileList >	GlobFilesExtended (const string &path, const FileGlobInput &input, optional_ptr< FileOpener > opener=nullptr)
virtual DUCKDB_API bool	SupportsGlobExtended () const

Constructor & Destructor Documentation

~FileSystem()

duckdb::FileSystem::~FileSystem ( )

virtual

                        {
}

Member Function Documentation

GetFileSystem()

FileSystem & duckdb::FileSystem::GetFileSystem ( ClientContext &  context )

static

                                                            {
    auto &client_data = ClientData::Get(context);
    return *client_data.client_file_system;
}

OpenFile() [1/2]

unique_ptr< FileHandle > duckdb::FileSystem::OpenFile ( const string &  path, FileOpenFlags  flags, optional_ptr< FileOpener >  opener = nullptr  )

virtual

                                                                                                                    {
    if (SupportsOpenFileExtended()) {
        return OpenFileExtended(OpenFileInfo(path), flags, opener);
    }
    throw NotImplementedException("%s: OpenFile is not implemented!", GetName());
}

OpenFile() [2/2]

unique_ptr< FileHandle > duckdb::FileSystem::OpenFile	(	const OpenFileInfo &	path,
		FileOpenFlags	flags,
		optional_ptr< FileOpener >	opener = nullptr
	)

                                                                             {
    if (SupportsOpenFileExtended()) {
        return OpenFileExtended(file, flags, opener);
    } else {
        return OpenFile(file.path, flags, opener);
    }
}

Read() [1/2]

void duckdb::FileSystem::Read ( FileHandle &  handle, void *  buffer, int64_t  nr_bytes, idx_t  location  )

virtual

Read exactly nr_bytes from the specified location in the file. Fails if nr_bytes could not be read. This is equivalent to calling SetFilePointer(location) followed by calling Read().

Reimplemented in duckdb::LocalFileSystem.

                                                                                        {
    throw NotImplementedException("%s: Read (with location) is not implemented!", GetName());
}

Here is the call graph for this function:

Write() [1/2]

void duckdb::FileSystem::Write ( FileHandle &  handle, void *  buffer, int64_t  nr_bytes, idx_t  location  )

virtual

Write exactly nr_bytes to the specified location in the file. Fails if nr_bytes could not be written. This is equivalent to calling SetFilePointer(location) followed by calling Write().

Reimplemented in duckdb::LocalFileSystem.

                                                                                         {
    throw NotImplementedException("%s: Write (with location) is not implemented!", GetName());
}

Here is the call graph for this function:

Read() [2/2]

int64_t duckdb::FileSystem::Read ( FileHandle &  handle, void *  buffer, int64_t  nr_bytes  )

virtual

Read nr_bytes from the specified file into the buffer, moving the file pointer forward by nr_bytes. Returns the amount of bytes read.

Reimplemented in duckdb::CompressedFileSystem, and duckdb::LocalFileSystem.

                                                                           {
    throw NotImplementedException("%s: Read is not implemented!", GetName());
}

Here is the call graph for this function:

Write() [2/2]

int64_t duckdb::FileSystem::Write ( FileHandle &  handle, void *  buffer, int64_t  nr_bytes  )

virtual

Write nr_bytes from the buffer into the file, moving the file pointer forward by nr_bytes.

Reimplemented in duckdb::CompressedFileSystem, and duckdb::LocalFileSystem.

                                                                            {
    throw NotImplementedException("%s: Write is not implemented!", GetName());
}

Here is the call graph for this function:

Trim()

bool duckdb::FileSystem::Trim ( FileHandle &  handle, idx_t  offset_bytes, idx_t  length_bytes  )

virtual

Excise a range of the file. The OS can drop pages from the page-cache, and the file-system is free to deallocate this range (sparse file support). Reads to the range will succeed but will return undefined data.

Reimplemented in duckdb::LocalFileSystem.

                                                                                {
    // This is not a required method. Derived FileSystems may optionally override/implement.
    return false;
}

GetFileSize()

int64_t duckdb::FileSystem::GetFileSize ( FileHandle &  handle )

virtual

Returns the file size of a file handle, returns -1 on error.

Reimplemented in duckdb::CompressedFileSystem, and duckdb::LocalFileSystem.

                                                  {
    throw NotImplementedException("%s: GetFileSize is not implemented!", GetName());
}

Here is the call graph for this function:

GetLastModifiedTime()

timestamp_t duckdb::FileSystem::GetLastModifiedTime ( FileHandle &  handle )

virtual

Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error.

Reimplemented in duckdb::LocalFileSystem.

                                                              {
    throw NotImplementedException("%s: GetLastModifiedTime is not implemented!", GetName());
}

Here is the call graph for this function:

GetVersionTag()

string duckdb::FileSystem::GetVersionTag ( FileHandle &  handle )

virtual

Returns a tag that uniquely identifies the version of the file, used for checking cache invalidation for CachingFileSystem httpfs files

Reimplemented in duckdb::LocalFileSystem.

                                                   {
    // Used to check cache invalidation for httpfs files with an ETag in CachingFileSystem
    return "";
}

GetFileType()

FileType duckdb::FileSystem::GetFileType ( FileHandle &  handle )

virtual

Returns the file type of the attached handle.

Reimplemented in duckdb::LocalFileSystem.

                                                   {
    return FileType::FILE_TYPE_INVALID;
}

Stats()

FileMetadata duckdb::FileSystem::Stats ( FileHandle &  handle )

virtual

Returns the file stats of the attached handle.

Reimplemented in duckdb::LocalFileSystem.

                                                 {
    throw NotImplementedException("%s: Stats is not implemented!", GetName());
}

Here is the call graph for this function:

Truncate()

void duckdb::FileSystem::Truncate ( FileHandle &  handle, int64_t  new_size  )

virtual

Truncate a file to a maximum size of new_size, new_size should be smaller than or equal to the current size of the file

Reimplemented in duckdb::LocalFileSystem.

                                                              {
    throw NotImplementedException("%s: Truncate is not implemented!", GetName());
}

Here is the call graph for this function:

DirectoryExists()

bool duckdb::FileSystem::DirectoryExists ( const string &  directory, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Check if a directory exists.

Reimplemented in duckdb::LocalFileSystem.

                                                                                         {
    throw NotImplementedException("%s: DirectoryExists is not implemented!", GetName());
}

Here is the call graph for this function:

Here is the caller graph for this function:

CreateDirectory()

void duckdb::FileSystem::CreateDirectory ( const string &  directory, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Create a directory if it does not exist.

Reimplemented in duckdb::LocalFileSystem.

                                                                                         {
    throw NotImplementedException("%s: CreateDirectory is not implemented!", GetName());
}

Here is the call graph for this function:

Here is the caller graph for this function:

CreateDirectoriesRecursive()

void duckdb::FileSystem::CreateDirectoriesRecursive ( const string &  path, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Helper function that uses DirectoryExists and CreateDirectory to ensure all directories in path are created.

                                                                                               {
    // To avoid hitting directories we have no permission for when using allowed_directories + enable_external_access,
    // we construct the list of directories to be created depth-first. This avoids calling DirectoryExists on a parent
    // dir that is not in the allowed_directories list
 
    // Walk up from the full path until we find a prefix that already exists, collecting
    // non-existing ancestors along the way. Stop descending when segments are exhausted
    // (to avoid Path::Parent() generating ".." paths past the root/base).
    auto parsed = Path::FromString(path);
    vector<string> dirs_to_create;
    while (!parsed.GetPathSegments().empty() && !DirectoryExists(parsed.ToString())) {
        dirs_to_create.push_back(parsed.ToString());
        parsed = parsed.Parent();
    }
    // If all segments were non-existing, also check the base itself (e.g. "C:/" on an unknown drive)
    if (!parsed.GetPathSegments().empty()) {
        // broke because DirectoryExists returned true — nothing more needed
    } else if (!DirectoryExists(parsed.ToString())) {
        dirs_to_create.push_back(parsed.ToString());
    }
 
    // Create directories shallowest to deepest
    for (auto riter = dirs_to_create.rbegin(); riter != dirs_to_create.rend(); ++riter) {
        CreateDirectory(*riter);
    }
}

Here is the call graph for this function:

RemoveDirectory()

void duckdb::FileSystem::RemoveDirectory ( const string &  directory, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Recursively remove a directory and all files in it.

Reimplemented in duckdb::LocalFileSystem.

                                                                                         {
    throw NotImplementedException("%s: RemoveDirectory is not implemented!", GetName());
}

Here is the call graph for this function:

ListFiles() [1/2]

bool duckdb::FileSystem::ListFiles ( const string &  directory, const std::function< void(const string &, bool)> &  callback, FileOpener *  opener = nullptr  )

virtual

List files in a directory, invoking the callback method for each one with (filename, is_dir)

                                               {
    if (SupportsListFilesExtended()) {
        return ListFilesExtended(
            directory,
            [&](const OpenFileInfo &info) {
                bool is_dir = IsDirectory(info);
                callback(info.path, is_dir);
            },
            opener);
    }
    throw NotImplementedException("%s: ListFiles is not implemented!", GetName());
}

Here is the call graph for this function:

ListFiles() [2/2]

bool duckdb::FileSystem::ListFiles	(	const string &	directory,
		const std::function< void(OpenFileInfo &info)> &	callback,
		optional_ptr< FileOpener >	opener = nullptr
	)

                                                            {
    if (SupportsListFilesExtended()) {
        return ListFilesExtended(directory, callback, opener);
    } else {
        return ListFiles(
            directory,
            [&](const string &path, bool is_dir) {
                OpenFileInfo info(path);
                if (is_dir) {
                    info.extended_info = make_shared_ptr<ExtendedOpenFileInfo>();
                    info.extended_info->options["type"] = "directory";
                }
                callback(info);
            },
            opener.get());
    }
}

MoveFile()

void duckdb::FileSystem::MoveFile ( const string &  source, const string &  target, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Move a file from source path to the target, StorageManager relies on this being an atomic action for ACID properties

Reimplemented in duckdb::LocalFileSystem.

                                                                                                     {
    throw NotImplementedException("%s: MoveFile is not implemented!", GetName());
}

Here is the call graph for this function:

FileExists()

bool duckdb::FileSystem::FileExists ( const string &  filename, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Check if a file exists.

Reimplemented in duckdb::LocalFileSystem.

                                                                                   {
    throw NotImplementedException("%s: FileExists is not implemented!", GetName());
}

Here is the call graph for this function:

Here is the caller graph for this function:

IsPipe()

bool duckdb::FileSystem::IsPipe ( const string &  filename, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Check if path is pipe.

Reimplemented in duckdb::LocalFileSystem.

                                                                               {
    return false;
}

RemoveFile()

void duckdb::FileSystem::RemoveFile ( const string &  filename, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Remove a file from disk.

Reimplemented in duckdb::LocalFileSystem.

                                                                                   {
    throw NotImplementedException("%s: RemoveFile is not implemented!", GetName());
}

Here is the call graph for this function:

Here is the caller graph for this function:

TryRemoveFile()

bool duckdb::FileSystem::TryRemoveFile ( const string &  filename, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Remvoe a file from disk if it exists - if it does not exist, return false.

                                                                                      {
    if (FileExists(filename, opener)) {
        RemoveFile(filename, opener);
        return true;
    }
    return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

RemoveFiles()

void duckdb::FileSystem::RemoveFiles ( const vector< string > &  filenames, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Remove multiple files from disk - does not error if any file does not exist.

                                                                                             {
    for (const auto &filename : filenames) {
        TryRemoveFile(filename, opener);
    }
}

Here is the call graph for this function:

FileSync()

void duckdb::FileSystem::FileSync ( FileHandle &  handle )

virtual

Sync a file handle to disk.

Reimplemented in duckdb::LocalFileSystem.

                                            {
    throw NotImplementedException("%s: FileSync is not implemented!", GetName());
}

Here is the call graph for this function:

SetWorkingDirectory()

void duckdb::FileSystem::SetWorkingDirectory ( const string &  path )

static

Sets the working directory.

                                                       {
    if (chdir(path.c_str()) != 0) {
        throw IOException("Could not change working directory!");
    }
}

GetWorkingDirectory()

string duckdb::FileSystem::GetWorkingDirectory ( )

static

Gets the working directory.

                                       {
    auto buffer = make_unsafe_uniq_array<char>(PATH_MAX);
    char *ret = getcwd(buffer.get(), PATH_MAX);
    if (!ret) {
        throw IOException("Could not get working directory!");
    }
    return string(buffer.get());
}

Here is the call graph for this function:

GetHomeDirectory() [1/2]

string duckdb::FileSystem::GetHomeDirectory ( optional_ptr< FileOpener >  opener )

static

Gets the users home directory.

                                                                   {
    // read the home_directory setting first, if it is set
    if (opener) {
        Value result;
        if (opener->TryGetCurrentSetting("home_directory", result)) {
            if (!result.IsNull() && !result.ToString().empty()) {
                return result.ToString();
            }
        }
    }
    // fallback to the default home directories for the specified system
#ifdef DUCKDB_WINDOWS
    return FileSystem::GetEnvVariable("USERPROFILE");
#else
    return FileSystem::GetEnvVariable("HOME");
#endif
}

Here is the call graph for this function:

GetHomeDirectory() [2/2]

string duckdb::FileSystem::GetHomeDirectory ( )

virtual

Gets the users home directory.

                                    {
    return GetHomeDirectory(nullptr);
}

Here is the call graph for this function:

Here is the caller graph for this function:

ExpandPath() [1/2]

string duckdb::FileSystem::ExpandPath ( const string &  path, optional_ptr< FileOpener >  opener  )

static

Expands a given path, including e.g. expanding the home directory of the user.

                                                                                 {
    if (path.empty()) {
        return path;
    }
    if (path[0] == '~') {
        return GetHomeDirectory(opener) + path.substr(1);
    }
    // handle file URIs
    auto file_offset = GetFileUrlOffset(path);
    if (file_offset > 0) {
        return path.substr(file_offset);
    }
    return path;
}

Here is the call graph for this function:

Here is the caller graph for this function:

ExpandPath() [2/2]

string duckdb::FileSystem::ExpandPath ( const string &  path )

virtual

Expands a given path, including e.g. expanding the home directory of the user.

                                                {
    return FileSystem::ExpandPath(path, nullptr);
}

Here is the call graph for this function:

GetAvailableMemory()

optional_idx duckdb::FileSystem::GetAvailableMemory ( )

static

Returns the system-available memory in bytes. Returns DConstants::INVALID_INDEX if the system function fails.

                                            {
    errno = 0;
 
#ifdef __MVS__
    struct rlimit limit;
    int rlim_rc = getrlimit(RLIMIT_AS, &limit);
    idx_t max_memory = MinValue<idx_t>(limit.rlim_max, UINTPTR_MAX);
#else
    idx_t max_memory = MinValue<idx_t>((idx_t)sysconf(_SC_PHYS_PAGES) * (idx_t)sysconf(_SC_PAGESIZE), UINTPTR_MAX);
#endif
    if (errno != 0) {
        return optional_idx();
    }
    return max_memory;
}

GetAvailableDiskSpace()

optional_idx duckdb::FileSystem::GetAvailableDiskSpace ( const string &  path )

static

Returns the space available on the disk. Returns DConstants::INVALID_INDEX if the information was not available.

                                                                 {
    struct statvfs vfs;
 
    auto ret = statvfs(path.c_str(), &vfs);
    if (ret == -1) {
        return optional_idx();
    }
    auto block_size = vfs.f_frsize;
    // These are the blocks available for creating new files or extending existing ones
    auto available_blocks = vfs.f_bfree;
    idx_t available_disk_space = DConstants::INVALID_INDEX;
    if (!TryMultiplyOperator::Operation(static_cast<idx_t>(block_size), static_cast<idx_t>(available_blocks),
                                        available_disk_space)) {
        return optional_idx();
    }
    return available_disk_space;
}

PathSeparator()

string duckdb::FileSystem::PathSeparator ( const string &  path )

virtual

Path separator for path.

                                                   {
    return "/";
}

Here is the caller graph for this function:

IsPathAbsolute()

bool duckdb::FileSystem::IsPathAbsolute ( const string &  path )

virtual

Checks if path is is an absolute path.

Reimplemented in duckdb::LocalFileSystem.

                                                  {
    auto path_separator = PathSeparator(path);
    return PathMatched(path, path_separator) || StringUtil::StartsWith(path, "file:/");
}

Here is the call graph for this function:

Here is the caller graph for this function:

JoinPath() [1/2]

string duckdb::FileSystem::JoinPath	(	const string &	a,
		const string &	path
	)

Join two paths together.

                                                            {
    // FIXME: sanitize paths
    return a.empty() ? b : a + PathSeparator(a) + b;
}

Here is the call graph for this function:

Here is the caller graph for this function:

JoinPath() [2/2]

template<typename... ARGS>

string duckdb::FileSystem::JoinPath ( const string &  a, const string &  b, ARGS...  args  )

inline

                                                                    {
        return JoinPath(JoinPath(a, b), args...);
    }

ConvertSeparators()

string duckdb::FileSystem::ConvertSeparators

(

const string &

path

)

Convert separators in a path to the local separators (e.g. convert "/" into \ on windows)

                                                       {
    auto separator_str = PathSeparator(path);
    char separator = separator_str[0];
    if (separator == '/') {
        // on unix-based systems we only accept / as a separator
        return path;
    }
    // on windows-based systems we accept both
    return StringUtil::Replace(path, "/", separator_str);
}

Here is the call graph for this function:

Here is the caller graph for this function:

ExtractBaseName()

string duckdb::FileSystem::ExtractBaseName

(

const string &

path

)

Extract the base name of a file (e.g. if the input is lib/example.dll the base name is 'example')

                                                     {
    if (path.empty()) {
        return string();
    }
    auto vec = StringUtil::Split(ExtractName(path), ".");
    D_ASSERT(!vec.empty());
    return vec[0];
}

Here is the call graph for this function:

ExtractExtension()

string duckdb::FileSystem::ExtractExtension

(

const string &

path

)

Extract the extension of a file (e.g. if the input is lib/example.dll the extension is 'dll')

                                                      {
    if (path.empty()) {
        return string();
    }
    auto vec = StringUtil::Split(ExtractName(path), ".");
    if (vec.size() < 2) {
        return string();
    }
    return vec.back();
}

Here is the call graph for this function:

ExtractName()

string duckdb::FileSystem::ExtractName

(

const string &

path

)

Extract the name of a file (e.g if the input is lib/example.dll the name is 'example.dll')

                                                 {
    if (path.empty()) {
        return string();
    }
    auto normalized_path = ConvertSeparators(path);
    auto sep = PathSeparator(path);
    auto splits = StringUtil::Split(normalized_path, sep);
    D_ASSERT(!splits.empty());
    return splits.back();
}

Here is the call graph for this function:

Here is the caller graph for this function:

GetEnvVariable()

string duckdb::FileSystem::GetEnvVariable ( const string &  name )

static

Returns the value of an environment variable - or the empty string if it is not set.

                                                    {
    const char *env = getenv(name.c_str());
    if (!env) {
        return string();
    }
    return env;
}

Here is the caller graph for this function:

HasGlob()

bool duckdb::FileSystem::HasGlob ( const string &  str )

static

Whether there is a glob in the string.

                                          {
    for (idx_t i = 0; i < str.size(); i++) {
        switch (str[i]) {
        case '*':
        case '?':
        case '[':
            return true;
        default:
            break;
        }
    }
    return false;
}

Glob() [1/2]

vector< OpenFileInfo > duckdb::FileSystem::Glob ( const string &  path, FileOpener *  opener = nullptr  )

virtual

Runs a glob on the file system, returning a list of matching files.

                                                                            {
    if (SupportsGlobExtended()) {
        auto result = GlobFilesExtended(path, FileGlobOptions::ALLOW_EMPTY, opener);
        return result->GetAllFiles();
    }
    throw NotImplementedException("%s: Glob is not implemented!", GetName());
}

Here is the call graph for this function:

Glob() [2/2]

unique_ptr< MultiFileList > duckdb::FileSystem::Glob	(	const string &	path,
		const FileGlobInput &	input,
		optional_ptr< FileOpener >	opener
	)

                                                                            {
    if (!SupportsGlobExtended()) {
        auto result = Glob(path, opener.get());
        return make_uniq<SimpleMultiFileList>(std::move(result));
    } else {
        return GlobFilesExtended(path, input, opener);
    }
}

GlobFileList()

unique_ptr< MultiFileList > duckdb::FileSystem::GlobFileList	(	const string &	path,
		const FileGlobInput &	input = FileGlobOptions::DISALLOW_EMPTY
	)

                                                                                                    {
    auto result = GlobFilesExtended(pattern, input);
    if (result->IsEmpty()) {
        if (input.behavior == FileGlobOptions::FALLBACK_GLOB && !HasGlob(pattern)) {
            // if we have no glob in the pattern and we have an extension, we try to glob
            if (!HasGlob(pattern)) {
                if (input.extension.empty()) {
                    throw InternalException("FALLBACK_GLOB requires an extension to be specified");
                }
                string new_pattern = JoinPath(JoinPath(pattern, "**"), "*." + input.extension);
                result = GlobFileList(new_pattern, FileGlobOptions::ALLOW_EMPTY);
                if (!result->IsEmpty()) {
                    // we found files by globbing the target as if it was a directory - return them
                    return result;
                }
            }
        }
        if (input.behavior == FileGlobOptions::FALLBACK_GLOB || input.behavior == FileGlobOptions::DISALLOW_EMPTY) {
            throw IOException("No files found that match the pattern \"%s\"", pattern);
        }
    }
    return result;
}

GlobFiles()

vector< OpenFileInfo > duckdb::FileSystem::GlobFiles	(	const string &	pattern,
		const FileGlobInput &	input = FileGlobOptions::DISALLOW_EMPTY
	)

                                                                                            {
    auto file_list = GlobFileList(pattern, input);
    return file_list->GetAllFiles();
}

RegisterSubSystem() [1/2]

void duckdb::FileSystem::RegisterSubSystem ( unique_ptr< FileSystem >  sub_fs )

virtual

registers a sub-file system to handle certain file name prefixes, e.g. http:// etc.

                                                                {
    throw NotImplementedException("%s: Can't register a sub system on a non-virtual file system", GetName());
}

Here is the call graph for this function:

RegisterSubSystem() [2/2]

void duckdb::FileSystem::RegisterSubSystem ( FileCompressionType  compression_type, unique_ptr< FileSystem >  fs  )

virtual

                                                                                                      {
    throw NotImplementedException("%s: Can't register a sub system on a non-virtual file system", GetName());
}

UnregisterSubSystem()

void duckdb::FileSystem::UnregisterSubSystem ( const string &  name )

virtual

Unregister a sub-filesystem by name.

                                                       {
    throw NotImplementedException("%s: Can't unregister a sub system on a non-virtual file system", GetName());
}

Here is the call graph for this function:

ExtractSubSystem()

unique_ptr< FileSystem > duckdb::FileSystem::ExtractSubSystem ( const string &  name )

virtual

                                                                      {
    throw NotImplementedException("%s: Can't extract a sub system on a non-virtual file system", GetName());
}

ListSubSystems()

vector< string > duckdb::FileSystem::ListSubSystems ( )

virtual

List registered sub-filesystems, including builtin ones.

                                          {
    throw NotImplementedException("%s: Can't list sub systems on a non-virtual file system", GetName());
}

Here is the call graph for this function:

CanHandleFile()

bool duckdb::FileSystem::CanHandleFile ( const string &  fpath )

virtual

Whether or not a sub-system can handle a specific file path.

Reimplemented in duckdb::LocalFileSystem.

                                                  {
    throw NotImplementedException("%s: CanHandleFile is not implemented!", GetName());
}

Here is the call graph for this function:

Seek()

void duckdb::FileSystem::Seek ( FileHandle &  handle, idx_t  location  )

virtual

Set the file pointer of a file handle to a specified location. Reads and writes will happen from this location.

Reimplemented in duckdb::LocalFileSystem.

                                                        {
    throw NotImplementedException("%s: Seek is not implemented!", GetName());
}

Here is the call graph for this function:

Reset()

void duckdb::FileSystem::Reset ( FileHandle &  handle )

virtual

Reset a file to the beginning (equivalent to Seek(handle, 0) for simple files)

Reimplemented in duckdb::CompressedFileSystem.

                                         {
    handle.Seek(0);
}

SeekPosition()

idx_t duckdb::FileSystem::SeekPosition ( FileHandle &  handle )

virtual

Reimplemented in duckdb::LocalFileSystem.

                                                 {
    throw NotImplementedException("%s: SeekPosition is not implemented!", GetName());
}

IsManuallySet()

bool duckdb::FileSystem::IsManuallySet ( )

virtual

If FS was manually set by the user.

                               {
    return false;
}

CanSeek()

bool duckdb::FileSystem::CanSeek ( )

virtual

Whether or not we can seek into the file.

Reimplemented in duckdb::CompressedFileSystem, and duckdb::LocalFileSystem.

                         {
    throw NotImplementedException("%s: CanSeek is not implemented!", GetName());
}

Here is the call graph for this function:

OnDiskFile()

bool duckdb::FileSystem::OnDiskFile ( FileHandle &  handle )

virtual

Whether or not the FS handles plain files on disk. This is relevant for certain optimizations, as random reads in a file on-disk are much cheaper than e.g. random reads in a file over the network

Reimplemented in duckdb::CompressedFileSystem, and duckdb::LocalFileSystem.

                                              {
    throw NotImplementedException("%s: OnDiskFile is not implemented!", GetName());
}

Here is the call graph for this function:

OpenCompressedFile()

unique_ptr< FileHandle > duckdb::FileSystem::OpenCompressedFile ( QueryContext  context, unique_ptr< FileHandle >  handle, bool  write  )

virtual

                                                                                                                     {
    throw NotImplementedException("%s: OpenCompressedFile is not implemented!", GetName());
}

CreateLocal()

unique_ptr< FileSystem > duckdb::FileSystem::CreateLocal ( )

static

Create a LocalFileSystem.

                                               {
    return make_uniq<LocalFileSystem>();
}

GetName()

virtual DUCKDB_API std::string duckdb::FileSystem::GetName ( ) const

pure virtual

Return the name of the filesytem. Used for forming diagnosis messages.

Implemented in duckdb::GZipFileSystem, and duckdb::LocalFileSystem.

Here is the caller graph for this function:

IsRemoteFile() [1/2]

bool duckdb::FileSystem::IsRemoteFile ( const string &  path )

static

Whether or not a file is remote or local, based only on file path.

                                                {
    string extension = "";
    return IsRemoteFile(path, extension);
}

Here is the call graph for this function:

Here is the caller graph for this function:

IsRemoteFile() [2/2]

bool duckdb::FileSystem::IsRemoteFile ( const string &  path, string &  extension  )

static

                                                                   {
    for (const auto &entry : EXTENSION_FILE_PREFIXES) {
        if (StringUtil::StartsWith(path, entry.name)) {
            extension = entry.extension;
            return true;
        }
    }
    return false;
}

SetDisabledFileSystems()

void duckdb::FileSystem::SetDisabledFileSystems ( const vector< string > &  names )

virtual

                                                                   {
    throw NotImplementedException("%s: Can't disable file systems on a non-virtual file system", GetName());
}

SubSystemIsDisabled()

bool duckdb::FileSystem::SubSystemIsDisabled ( const string &  name )

virtual

                                                       {
    throw NotImplementedException("%s: Non-virtual file system does not have subsystems", GetName());
}

IsDisabledForPath()

bool duckdb::FileSystem::IsDisabledForPath ( const string &  path )

virtual

Check if the filesystem that would handle this path is disabled.

                                                     {
    throw NotImplementedException("%s: Non-virtual file system does not have subsystems", GetName());
}

Here is the call graph for this function:

IsDirectory()

bool duckdb::FileSystem::IsDirectory ( const OpenFileInfo &  info )

static

                                                     {
    if (!info.extended_info) {
        return false;
    }
    auto entry = info.extended_info->options.find("type");
    if (entry == info.extended_info->options.end()) {
        return false;
    }
    return StringValue::Get(entry->second) == "directory";
}

CanonicalizePath()

string duckdb::FileSystem::CanonicalizePath ( const string &  path, optional_ptr< FileOpener >  opener = nullptr  )

virtual

Canonicalize a path.

Reimplemented in duckdb::LocalFileSystem.

                                                                                         {
    if (IsRemoteFile(path_p)) {
        // don't canonicalize remote paths
        return path_p;
    }
    auto path_sep = PathSeparator(path_p);
    auto elements = StringUtil::Split(path_p, path_sep);
 
    deque<string> path_stack;
    string result;
    for (idx_t i = 0; i < elements.size(); i++) {
        if (elements[i].empty() || elements[i] == ".") {
            // we ignore empty and `.`
            continue;
        }
        if (elements[i] == "..") {
            // .. pops from stack if possible, if already at root its ignored
            if (!path_stack.empty()) {
                path_stack.pop_back();
            }
        } else {
            path_stack.push_back(elements[i]);
        }
    }
    // we lost the leading / in the split/loop so lets put it back
    if (path_p[0] == '/') {
        result = "/";
    }
    while (!path_stack.empty()) {
        result += path_stack.front();
        path_stack.pop_front();
        if (!path_stack.empty()) {
            result += path_sep;
        }
    }
 
    return result;
}

Here is the call graph for this function:

Here is the caller graph for this function:

OpenFileExtended()

unique_ptr< FileHandle > duckdb::FileSystem::OpenFileExtended ( const OpenFileInfo &  path, FileOpenFlags  flags, optional_ptr< FileOpener >  opener  )

protectedvirtual

                                                                                     {
    throw NotImplementedException("%s: OpenFileExtended is not implemented!", GetName());
}

SupportsOpenFileExtended()

bool duckdb::FileSystem::SupportsOpenFileExtended ( ) const

protectedvirtual

                                                {
    return false;
}

ListFilesExtended()

bool duckdb::FileSystem::ListFilesExtended ( const string &  directory, const std::function< void(OpenFileInfo &info)> &  callback, optional_ptr< FileOpener >  opener  )

protectedvirtual

                                                                    {
    throw NotImplementedException("%s: ListFilesExtended is not implemented!", GetName());
}

SupportsListFilesExtended()

bool duckdb::FileSystem::SupportsListFilesExtended ( ) const

protectedvirtual

                                                 {
    return false;
}

GlobFilesExtended()

unique_ptr< MultiFileList > duckdb::FileSystem::GlobFilesExtended ( const string &  path, const FileGlobInput &  input, optional_ptr< FileOpener >  opener = nullptr  )

protectedvirtual

                                                                                         {
    throw NotImplementedException("%s: GlobFilesExtended is not implemented!", GetName());
}

SupportsGlobExtended()

bool duckdb::FileSystem::SupportsGlobExtended ( ) const

protectedvirtual

                                            {
    return false;
}

Cast() [1/2]

template<class TARGET >

TARGET & duckdb::FileSystem::Cast ( )

inline

                   {
        DynamicCastCheck<TARGET>(this);
        return reinterpret_cast<TARGET &>(*this);
    }

Cast() [2/2]

template<class TARGET >

const TARGET & duckdb::FileSystem::Cast ( ) const

inline

                               {
        DynamicCastCheck<TARGET>(this);
        return reinterpret_cast<const TARGET &>(*this);
    }

---

The documentation for this class was generated from the following files:

• external/duckdb/duckdb.hpp
• external/duckdb/duckdb.cpp