duckdb::LocalFileSystem Class Reference

Inheritance diagram for duckdb::LocalFileSystem:

Collaboration diagram for duckdb::LocalFileSystem:

Public Member Functions
unique_ptr< FileHandle >	OpenFile (const string &path, FileOpenFlags flags, optional_ptr< FileOpener > opener=nullptr) override
void	Read (FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override
void	Write (FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location) override
int64_t	Read (FileHandle &handle, void *buffer, int64_t nr_bytes) override
int64_t	Write (FileHandle &handle, void *buffer, int64_t nr_bytes) override
	Write nr_bytes from the buffer into the file, moving the file pointer forward by nr_bytes.
bool	Trim (FileHandle &handle, idx_t offset_bytes, idx_t length_bytes) override
int64_t	GetFileSize (FileHandle &handle) override
	Returns the file size of a file handle, returns -1 on error.
timestamp_t	GetLastModifiedTime (FileHandle &handle) override
	Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error.
string	GetVersionTag (FileHandle &handle) override
	Returns a tag that uniquely identifies the version of the file.
FileType	GetFileType (FileHandle &handle) override
	Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error.
FileMetadata	Stats (FileHandle &handle) override
	Returns the file stats of the attached handle.
void	Truncate (FileHandle &handle, int64_t new_size) override
bool	DirectoryExists (const string &directory, optional_ptr< FileOpener > opener=nullptr) override
	Check if a directory exists.
void	CreateDirectory (const string &directory, optional_ptr< FileOpener > opener=nullptr) override
	Create a directory if it does not exist.
void	RemoveDirectory (const string &directory, optional_ptr< FileOpener > opener=nullptr) override
	Recursively remove a directory and all files in it.
void	MoveFile (const string &source, const string &target, optional_ptr< FileOpener > opener=nullptr) override
bool	FileExists (const string &filename, optional_ptr< FileOpener > opener=nullptr) override
	Check if a file exists.
bool	IsPipe (const string &filename, optional_ptr< FileOpener > opener=nullptr) override
	Check if path is a pipe.
void	RemoveFile (const string &filename, optional_ptr< FileOpener > opener=nullptr) override
	Remove a file from disk.
void	FileSync (FileHandle &handle) override
	Sync a file handle to disk.
bool	IsPathAbsolute (const string &path) override
	Checks if path is is an absolute path.
string	MakePathAbsolute (const string &input, optional_ptr< FileOpener > opener)
bool	PathStartsWithDrive (const string &path)
bool	CanHandleFile (const string &fpath) override
	Whether or not a sub-system can handle a specific file path.
void	Seek (FileHandle &handle, idx_t location) override
	Set the file pointer of a file handle to a specified location. Reads and writes will happen from this location.
idx_t	SeekPosition (FileHandle &handle) override
	Return the current seek posiiton in the file.
bool	CanSeek () override
	Whether or not we can seek into the file.
bool	OnDiskFile (FileHandle &handle) override
std::string	GetName () const override
	Return the name of the filesytem. Used for forming diagnosis messages.
vector< OpenFileInfo >	FetchFileWithoutGlob (const string &path, optional_ptr< FileOpener > opener, bool absolute_path)
string	CanonicalizePath (const string &path_p, optional_ptr< FileOpener > opener) override
	Canonicalize a path.
Public Member Functions inherited from duckdb::FileSystem
DUCKDB_API unique_ptr< FileHandle >	OpenFile (const OpenFileInfo &path, FileOpenFlags flags, optional_ptr< FileOpener > opener=nullptr)
virtual DUCKDB_API void	CreateDirectoriesRecursive (const string &path, optional_ptr< FileOpener > opener=nullptr)
	Helper function that uses DirectoryExists and CreateDirectory to ensure all directories in path are created.
virtual DUCKDB_API bool	ListFiles (const string &directory, const std::function< void(const string &, bool)> &callback, FileOpener *opener=nullptr)
	List files in a directory, invoking the callback method for each one with (filename, is_dir)
DUCKDB_API bool	ListFiles (const string &directory, const std::function< void(OpenFileInfo &info)> &callback, optional_ptr< FileOpener > opener=nullptr)
virtual DUCKDB_API bool	TryRemoveFile (const string &filename, optional_ptr< FileOpener > opener=nullptr)
	Remvoe a file from disk if it exists - if it does not exist, return false.
virtual DUCKDB_API void	RemoveFiles (const vector< string > &filenames, optional_ptr< FileOpener > opener=nullptr)
	Remove multiple files from disk - does not error if any file does not exist.
virtual DUCKDB_API string	GetHomeDirectory ()
	Gets the users home directory.
virtual DUCKDB_API string	ExpandPath (const string &path)
	Expands a given path, including e.g. expanding the home directory of the user.
virtual DUCKDB_API string	PathSeparator (const string &path)
	Path separator for path.
DUCKDB_API string	JoinPath (const string &a, const string &path)
	Join two paths together.
template<typename... ARGS>
string	JoinPath (const string &a, const string &b, ARGS... args)
DUCKDB_API string	ConvertSeparators (const string &path)
	Convert separators in a path to the local separators (e.g. convert "/" into \ on windows)
DUCKDB_API string	ExtractBaseName (const string &path)
	Extract the base name of a file (e.g. if the input is lib/example.dll the base name is 'example')
DUCKDB_API string	ExtractExtension (const string &path)
	Extract the extension of a file (e.g. if the input is lib/example.dll the extension is 'dll')
DUCKDB_API string	ExtractName (const string &path)
	Extract the name of a file (e.g if the input is lib/example.dll the name is 'example.dll')
virtual DUCKDB_API vector< OpenFileInfo >	Glob (const string &path, FileOpener *opener=nullptr)
	Runs a glob on the file system, returning a list of matching files.
DUCKDB_API unique_ptr< MultiFileList >	Glob (const string &path, const FileGlobInput &input, optional_ptr< FileOpener > opener)
DUCKDB_API unique_ptr< MultiFileList >	GlobFileList (const string &path, const FileGlobInput &input=FileGlobOptions::DISALLOW_EMPTY)
DUCKDB_API vector< OpenFileInfo >	GlobFiles (const string &pattern, const FileGlobInput &input=FileGlobOptions::DISALLOW_EMPTY)
virtual DUCKDB_API void	RegisterSubSystem (unique_ptr< FileSystem > sub_fs)
	registers a sub-file system to handle certain file name prefixes, e.g. http:// etc.
virtual DUCKDB_API void	RegisterSubSystem (FileCompressionType compression_type, unique_ptr< FileSystem > fs)
virtual DUCKDB_API void	UnregisterSubSystem (const string &name)
	Unregister a sub-filesystem by name.
virtual DUCKDB_API unique_ptr< FileSystem >	ExtractSubSystem (const string &name)
virtual DUCKDB_API vector< string >	ListSubSystems ()
	List registered sub-filesystems, including builtin ones.
virtual DUCKDB_API void	Reset (FileHandle &handle)
	Reset a file to the beginning (equivalent to Seek(handle, 0) for simple files)
virtual DUCKDB_API bool	IsManuallySet ()
	If FS was manually set by the user.
virtual DUCKDB_API unique_ptr< FileHandle >	OpenCompressedFile (QueryContext context, unique_ptr< FileHandle > handle, bool write)
virtual DUCKDB_API void	SetDisabledFileSystems (const vector< string > &names)
virtual DUCKDB_API bool	SubSystemIsDisabled (const string &name)
virtual DUCKDB_API bool	IsDisabledForPath (const string &path)
	Check if the filesystem that would handle this path is disabled.
template<class TARGET >
TARGET &	Cast ()
template<class TARGET >
const TARGET &	Cast () const

Static Public Member Functions
static std::string	GetLastErrorAsString ()
static bool	IsPrivateFile (const string &path_p, FileOpener *opener)
	Checks a file is private (checks for 600 on linux/macos, TODO: currently always returns true on windows)
Static Public Member Functions inherited from duckdb::FileSystem
static DUCKDB_API FileSystem &	GetFileSystem (ClientContext &context)
static DUCKDB_API FileSystem &	GetFileSystem (DatabaseInstance &db)
static DUCKDB_API FileSystem &	Get (AttachedDatabase &db)
static DUCKDB_API void	SetWorkingDirectory (const string &path)
	Sets the working directory.
static DUCKDB_API string	GetWorkingDirectory ()
	Gets the working directory.
static DUCKDB_API string	GetHomeDirectory (optional_ptr< FileOpener > opener)
	Gets the users home directory.
static DUCKDB_API string	ExpandPath (const string &path, optional_ptr< FileOpener > opener)
	Expands a given path, including e.g. expanding the home directory of the user.
static DUCKDB_API optional_idx	GetAvailableMemory ()
	Returns the system-available memory in bytes. Returns DConstants::INVALID_INDEX if the system function fails.
static DUCKDB_API optional_idx	GetAvailableDiskSpace (const string &path)
	Returns the space available on the disk. Returns DConstants::INVALID_INDEX if the information was not available.
static DUCKDB_API string	GetEnvVariable (const string &name)
	Returns the value of an environment variable - or the empty string if it is not set.
static DUCKDB_API bool	HasGlob (const string &str)
	Whether there is a glob in the string.
static DUCKDB_API unique_ptr< FileSystem >	CreateLocal ()
	Create a LocalFileSystem.
static DUCKDB_API bool	IsRemoteFile (const string &path)
	Whether or not a file is remote or local, based only on file path.
static DUCKDB_API bool	IsRemoteFile (const string &path, string &extension)
static DUCKDB_API bool	IsDirectory (const OpenFileInfo &info)

Protected Member Functions
bool	ListFilesExtended (const string &directory, const std::function< void(OpenFileInfo &info)> &callback, optional_ptr< FileOpener > opener) override
bool	SupportsListFilesExtended () const override
unique_ptr< MultiFileList >	GlobFilesExtended (const string &path, const FileGlobInput &input, optional_ptr< FileOpener > opener) override
bool	SupportsGlobExtended () const override
bool	TryCanonicalizeExistingPath (string &path_p)
Protected Member Functions inherited from duckdb::FileSystem
virtual DUCKDB_API unique_ptr< FileHandle >	OpenFileExtended (const OpenFileInfo &path, FileOpenFlags flags, optional_ptr< FileOpener > opener)
virtual DUCKDB_API bool	SupportsOpenFileExtended () const

Private Member Functions
void	SetFilePointer (FileHandle &handle, idx_t location)
	Set the file pointer of a file handle to a specified location. Reads and writes will happen from this location.
idx_t	GetFilePointer (FileHandle &handle)

Member Function Documentation

OpenFile()

unique_ptr< FileHandle > duckdb::LocalFileSystem::OpenFile ( const string &  path, FileOpenFlags  flags, optional_ptr< FileOpener >  opener = nullptr  )

overridevirtual

Reimplemented from duckdb::FileSystem.

                                                                                  {
    auto path = ExpandPath(path_p, opener);
    if (flags.Compression() != FileCompressionType::UNCOMPRESSED) {
        throw NotImplementedException("Unsupported compression type for default file system");
    }
 
    flags.Verify();
 
    int open_flags = 0;
    int rc;
    bool open_read = flags.OpenForReading();
    bool open_write = flags.OpenForWriting();
    if (open_read && open_write) {
        open_flags = O_RDWR;
    } else if (open_read) {
        open_flags = O_RDONLY;
    } else if (open_write) {
        open_flags = O_WRONLY;
    } else {
        throw InternalException("READ, WRITE or both should be specified when opening a file");
    }
    if (open_write) {
        // need Read or Write
        D_ASSERT(flags.OpenForWriting());
        open_flags |= O_CLOEXEC;
        if (flags.CreateFileIfNotExists()) {
            open_flags |= O_CREAT;
        } else if (flags.OverwriteExistingFile()) {
            open_flags |= O_CREAT | O_TRUNC;
        }
        if (flags.OpenForAppending()) {
            open_flags |= O_APPEND;
        }
    }
    if (flags.DirectIO()) {
#if defined(__sun) && defined(__SVR4)
        throw InvalidInputException("DIRECT_IO not supported on Solaris");
#endif
#if defined(__DARWIN__) || defined(__APPLE__) || defined(__OpenBSD__)
        // OSX does not have O_DIRECT, instead we need to use fcntl afterwards to support direct IO
#else
        open_flags |= O_DIRECT;
#endif
    }
 
    // Determine permissions
    mode_t filesec;
    if (flags.CreatePrivateFile()) {
        open_flags |= O_EXCL; // Ensure we error on existing files or the permissions may not set
        filesec = 0600;
    } else {
        filesec = 0666;
    }
 
    if (flags.ExclusiveCreate()) {
        open_flags |= O_EXCL;
    }
 
    // Open the file
    int fd = open(path.c_str(), open_flags, filesec);
 
    if (fd == -1) {
        if (flags.ReturnNullIfNotExists() && errno == ENOENT) {
            return nullptr;
        }
        if (flags.ReturnNullIfExists() && errno == EEXIST) {
            return nullptr;
        }
        throw IOException({{"errno", std::to_string(errno)}}, "Cannot open file \"%s\": %s", path, strerror(errno));
    }
 
#if defined(__DARWIN__) || defined(__APPLE__)
    if (flags.DirectIO()) {
        // OSX requires fcntl for Direct IO
        rc = fcntl(fd, F_NOCACHE, 1);
        if (rc == -1) {
            throw IOException("Could not enable direct IO for file \"%s\": %s", path, strerror(errno));
        }
    }
#endif
 
    if (flags.Lock() != FileLockType::NO_LOCK) {
        // set lock on file
        // but only if it is not an input/output stream
        auto file_type = StatsInternal(fd, path_p).file_type;
        if (file_type != FileType::FILE_TYPE_FIFO && file_type != FileType::FILE_TYPE_SOCKET) {
            struct flock fl;
            memset(&fl, 0, sizeof fl);
            fl.l_type = flags.Lock() == FileLockType::READ_LOCK ? F_RDLCK : F_WRLCK;
            fl.l_whence = SEEK_SET;
            fl.l_start = 0;
            fl.l_len = 0;
            rc = fcntl(fd, F_SETLK, &fl);
            // Retain the original error.
            int retained_errno = errno;
            bool has_error = rc == -1;
            string extended_error;
            if (has_error) {
                if (retained_errno == ENOTSUP) {
                    // file lock not supported for this file system
                    if (flags.Lock() == FileLockType::READ_LOCK) {
                        // for read-only, we ignore not-supported errors
                        has_error = false;
                        errno = 0;
                    } else {
                        extended_error = "File locks are not supported for this file system, cannot open the file in "
                                         "read-write mode. Try opening the file in read-only mode";
                    }
                }
            }
            if (has_error) {
                if (extended_error.empty()) {
                    // try to find out who is holding the lock using F_GETLK
                    rc = fcntl(fd, F_GETLK, &fl);
                    if (rc == -1) { // fnctl does not want to help us
                        extended_error = strerror(errno);
                    } else {
                        extended_error = AdditionalProcessInfo(*this, fl.l_pid);
                    }
                    if (flags.Lock() == FileLockType::WRITE_LOCK) {
                        // maybe we can get a read lock instead and tell this to the user.
                        fl.l_type = F_RDLCK;
                        rc = fcntl(fd, F_SETLK, &fl);
                        if (rc != -1) { // success!
                            extended_error +=
                                ". However, you would be able to open this database in read-only mode, e.g. by "
                                "using the -readonly parameter in the CLI";
                        }
                    }
                }
                rc = close(fd);
                if (rc == -1) {
                    extended_error += ". Also, failed closing file";
                }
                extended_error += ". See also https://duckdb.org/docs/stable/connect/concurrency";
                throw IOException({{"errno", std::to_string(retained_errno)}}, "Could not set lock on file \"%s\": %s",
                                  path, extended_error);
            }
        }
    }
 
    auto file_handle = make_uniq<UnixFileHandle>(*this, path, fd, flags);
    if (opener) {
        file_handle->TryAddLogger(*opener);
        DUCKDB_LOG_FILE_SYSTEM_OPEN((*file_handle));
    }
    return std::move(file_handle);
}

Read() [1/2]

void duckdb::LocalFileSystem::Read ( FileHandle &  handle, void *  buffer, int64_t  nr_bytes, idx_t  location  )

overridevirtual

Read exactly nr_bytes from the specified location in the file. Fails if nr_bytes could not be read. This is equivalent to calling SetFilePointer(location) followed by calling Read().

Reimplemented from duckdb::FileSystem.

                                                                                             {
    auto bytes_to_read = nr_bytes;
    int fd = handle.Cast<UnixFileHandle>().fd;
    auto read_buffer = char_ptr_cast(buffer);
    while (nr_bytes > 0) {
        int64_t bytes_read =
            pread(fd, read_buffer, UnsafeNumericCast<size_t>(nr_bytes), UnsafeNumericCast<off_t>(location));
        if (bytes_read == -1) {
            throw IOException({{"errno", std::to_string(errno)}}, "Could not read from file \"%s\": %s", handle.path,
                              strerror(errno));
        }
        if (bytes_read == 0) {
            throw IOException(
                "Could not read enough bytes from file \"%s\": attempted to read %llu bytes from location %llu",
                handle.path, nr_bytes, location);
        }
        read_buffer += bytes_read;
        nr_bytes -= bytes_read;
        location += UnsafeNumericCast<idx_t>(bytes_read);
    }
 
    DUCKDB_LOG_FILE_SYSTEM_READ(handle, bytes_to_read, location - UnsafeNumericCast<idx_t>(bytes_to_read));
}

Write() [1/2]

void duckdb::LocalFileSystem::Write ( FileHandle &  handle, void *  buffer, int64_t  nr_bytes, idx_t  location  )

overridevirtual

Write exactly nr_bytes to the specified location in the file. Fails if nr_bytes could not be written. This is equivalent to calling SetFilePointer(location) followed by calling Write().

Reimplemented from duckdb::FileSystem.

                                                                                              {
    int fd = handle.Cast<UnixFileHandle>().fd;
    auto write_buffer = char_ptr_cast(buffer);
 
    auto bytes_to_write = nr_bytes;
    auto current_location = location;
 
    while (bytes_to_write > 0) {
        int64_t bytes_written = pwrite(fd, write_buffer, UnsafeNumericCast<size_t>(bytes_to_write),
                                       UnsafeNumericCast<off_t>(current_location));
        if (bytes_written < 0) {
            throw IOException({{"errno", std::to_string(errno)}}, "Could not write file \"%s\": %s", handle.path,
                              strerror(errno));
        }
        if (bytes_written == 0) {
            throw IOException({{"errno", std::to_string(errno)}},
                              "Could not write to file \"%s\" - attempted to write 0 bytes: %s", handle.path,
                              strerror(errno));
        }
        write_buffer += bytes_written;
        bytes_to_write -= bytes_written;
        current_location += UnsafeNumericCast<idx_t>(bytes_written);
    }
 
    DUCKDB_LOG_FILE_SYSTEM_WRITE(handle, nr_bytes, location);
}

Read() [2/2]

int64_t duckdb::LocalFileSystem::Read ( FileHandle &  handle, void *  buffer, int64_t  nr_bytes  )

overridevirtual

Read nr_bytes from the specified file into the buffer, moving the file pointer forward by nr_bytes. Returns the amount of bytes read.

Reimplemented from duckdb::FileSystem.

                                                                                {
    auto &unix_handle = handle.Cast<UnixFileHandle>();
    int fd = unix_handle.fd;
    int64_t bytes_read = read(fd, buffer, UnsafeNumericCast<size_t>(nr_bytes));
    if (bytes_read == -1) {
        throw IOException({{"errno", std::to_string(errno)}}, "Could not read from file \"%s\": %s", handle.path,
                          strerror(errno));
    }
 
    DUCKDB_LOG_FILE_SYSTEM_READ(handle, bytes_read, unix_handle.current_pos);
    unix_handle.current_pos += UnsafeNumericCast<idx_t>(bytes_read);
 
    return bytes_read;
}

Here is the call graph for this function:

Write() [2/2]

int64_t duckdb::LocalFileSystem::Write ( FileHandle &  handle, void *  buffer, int64_t  nr_bytes  )

overridevirtual

Write nr_bytes from the buffer into the file, moving the file pointer forward by nr_bytes.

Reimplemented from duckdb::FileSystem.

                                                                                 {
    auto &unix_handle = handle.Cast<UnixFileHandle>();
    int fd = unix_handle.fd;
 
    auto bytes_to_write = nr_bytes;
    while (bytes_to_write > 0) {
        auto bytes_to_write_this_call =
            MinValue<idx_t>(idx_t(NumericLimits<int32_t>::Maximum()), idx_t(bytes_to_write));
        int64_t current_bytes_written = write(fd, buffer, bytes_to_write_this_call);
        if (current_bytes_written <= 0) {
            throw IOException({{"errno", std::to_string(errno)}}, "Could not write file \"%s\": %s", handle.path,
                              strerror(errno));
        }
        buffer = (void *)(data_ptr_cast(buffer) + current_bytes_written);
        bytes_to_write -= current_bytes_written;
    }
 
    DUCKDB_LOG_FILE_SYSTEM_WRITE(handle, nr_bytes, unix_handle.current_pos);
    unix_handle.current_pos += UnsafeNumericCast<idx_t>(nr_bytes);
 
    return nr_bytes;
}

Here is the call graph for this function:

Trim()

bool duckdb::LocalFileSystem::Trim ( FileHandle &  handle, idx_t  offset_bytes, idx_t  length_bytes  )

overridevirtual

Excise a range of the file. The file-system is free to deallocate this range (sparse file support). Reads to the range will succeed but will return undefined data.

Reimplemented from duckdb::FileSystem.

                                                                                     {
#if defined(__linux__)
    // FALLOC_FL_PUNCH_HOLE requires glibc 2.18 or up
#if __GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ < 18)
    return false;
#else
    int fd = handle.Cast<UnixFileHandle>().fd;
    int res = fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, UnsafeNumericCast<int64_t>(offset_bytes),
                        UnsafeNumericCast<int64_t>(length_bytes));
    return res == 0;
#endif
#else
    return false;
#endif
}

GetFileSize()

int64_t duckdb::LocalFileSystem::GetFileSize ( FileHandle &  handle )

overridevirtual

Returns the file size of a file handle, returns -1 on error.

Reimplemented from duckdb::FileSystem.

                                                       {
    const auto file_metadata = Stats(handle);
    return file_metadata.file_size;
}

Here is the call graph for this function:

GetLastModifiedTime()

timestamp_t duckdb::LocalFileSystem::GetLastModifiedTime ( FileHandle &  handle )

overridevirtual

Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error.

Reimplemented from duckdb::FileSystem.

                                                                   {
    const auto file_metadata = Stats(handle);
    return file_metadata.last_modification_time;
}

Here is the call graph for this function:

GetVersionTag()

string duckdb::LocalFileSystem::GetVersionTag ( FileHandle &  handle )

overridevirtual

Returns a tag that uniquely identifies the version of the file.

Reimplemented from duckdb::FileSystem.

                                                        {
    // TODO: Fix using FileSystem::Stats for v1.5, which should also fix it for Windows
#ifdef _WIN32
    return "";
#else
    int fd = handle.Cast<UnixFileHandle>().fd;
    struct stat s;
    if (fstat(fd, &s) == -1) {
        throw IOException("Failed to get file size for file \"%s\": %s", handle.path, strerror(errno));
    }
    return GetPosixVersionTag(s);
#endif
}

GetFileType()

FileType duckdb::LocalFileSystem::GetFileType ( FileHandle &  handle )

overridevirtual

Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error.

Reimplemented from duckdb::FileSystem.

                                                        {
    const auto file_metadata = Stats(handle);
    return file_metadata.file_type;
}

Here is the call graph for this function:

Stats()

FileMetadata duckdb::LocalFileSystem::Stats ( FileHandle &  handle )

overridevirtual

Returns the file stats of the attached handle.

Reimplemented from duckdb::FileSystem.

                                                      {
    int fd = handle.Cast<UnixFileHandle>().fd;
    auto file_metadata = StatsInternal(fd, handle.GetPath());
    return file_metadata;
}

Here is the caller graph for this function:

Truncate()

void duckdb::LocalFileSystem::Truncate ( FileHandle &  handle, int64_t  new_size  )

overridevirtual

Truncate a file to a maximum size of new_size, new_size should be smaller than or equal to the current size of the file

Reimplemented from duckdb::FileSystem.

                                                                   {
    int fd = handle.Cast<UnixFileHandle>().fd;
    if (ftruncate(fd, new_size) != 0) {
        throw IOException({{"errno", std::to_string(errno)}}, "Could not truncate file \"%s\": %s", handle.path,
                          strerror(errno));
    }
}

DirectoryExists()

bool duckdb::LocalFileSystem::DirectoryExists ( const string &  directory, optional_ptr< FileOpener >  opener = nullptr  )

overridevirtual

Check if a directory exists.

Reimplemented from duckdb::FileSystem.

                                                                                              {
    if (!directory.empty()) {
        auto normalized_dir = ExpandPath(directory, opener);
        if (access(normalized_dir.c_str(), 0) == 0) {
            struct stat status;
            stat(normalized_dir.c_str(), &status);
            if (S_ISDIR(status.st_mode)) {
                return true;
            }
        }
    }
    // if any condition fails
    return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

CreateDirectory()

void duckdb::LocalFileSystem::CreateDirectory ( const string &  directory, optional_ptr< FileOpener >  opener = nullptr  )

overridevirtual

Create a directory if it does not exist.

Reimplemented from duckdb::FileSystem.

                                                                                              {
    struct stat st;
 
    auto normalized_dir = ExpandPath(directory, opener);
    if (stat(normalized_dir.c_str(), &st) != 0) {
        /* Directory does not exist. EEXIST for race condition */
        if (mkdir(normalized_dir.c_str(), 0755) != 0 && errno != EEXIST) {
            throw IOException({{"errno", std::to_string(errno)}}, "Failed to create directory \"%s\": %s", directory,
                              strerror(errno));
        }
    } else if (!S_ISDIR(st.st_mode)) {
        throw IOException({{"errno", std::to_string(errno)}},
                          "Failed to create directory \"%s\": path exists but is not a directory!", directory);
    }
}

Here is the call graph for this function:

RemoveDirectory()

void duckdb::LocalFileSystem::RemoveDirectory ( const string &  directory, optional_ptr< FileOpener >  opener = nullptr  )

overridevirtual

Recursively remove a directory and all files in it.

Reimplemented from duckdb::FileSystem.

                                                                                              {
    auto normalized_dir = ExpandPath(directory, opener);
    RemoveDirectoryRecursive(normalized_dir.c_str());
}

Here is the call graph for this function:

MoveFile()

void duckdb::LocalFileSystem::MoveFile ( const string &  source, const string &  target, optional_ptr< FileOpener >  opener = nullptr  )

overridevirtual

Move a file from source path to the target, StorageManager relies on this being an atomic action for ACID properties

FIXME: rename does not guarantee atomicity or overwriting target file if it exists

Reimplemented from duckdb::FileSystem.

                                                                                                          {
    auto normalized_source = ExpandPath(source, opener);
    auto normalized_target = ExpandPath(target, opener);
    if (rename(normalized_source.c_str(), normalized_target.c_str()) != 0) {
        throw IOException({{"errno", to_string(errno)}}, "Could not rename file \"%s\" to \"%s\": %s", source, target,
                          strerror(errno));
    }
}

Here is the call graph for this function:

FileExists()

bool duckdb::LocalFileSystem::FileExists ( const string &  filename, optional_ptr< FileOpener >  opener = nullptr  )

overridevirtual

Check if a file exists.

Reimplemented from duckdb::FileSystem.

                                                                                        {
    if (!filename.empty()) {
        auto normalized_file = ExpandPath(filename, opener);
        if (access(normalized_file.c_str(), 0) == 0) {
            struct stat status;
            stat(normalized_file.c_str(), &status);
            if (S_ISREG(status.st_mode)) {
                return true;
            }
        }
    }
    // if any condition fails
    return false;
}

Here is the call graph for this function:

Here is the caller graph for this function:

IsPipe()

bool duckdb::LocalFileSystem::IsPipe ( const string &  filename, optional_ptr< FileOpener >  opener = nullptr  )

overridevirtual

Check if path is a pipe.

Reimplemented from duckdb::FileSystem.

                                                                                    {
    if (!filename.empty()) {
        auto normalized_file = ExpandPath(filename, opener);
        if (access(normalized_file.c_str(), 0) == 0) {
            struct stat status;
            stat(normalized_file.c_str(), &status);
            if (S_ISFIFO(status.st_mode)) {
                return true;
            }
        }
    }
    // if any condition fails
    return false;
}

Here is the call graph for this function:

RemoveFile()

void duckdb::LocalFileSystem::RemoveFile ( const string &  filename, optional_ptr< FileOpener >  opener = nullptr  )

overridevirtual

Remove a file from disk.

Reimplemented from duckdb::FileSystem.

                                                                                        {
    auto normalized_file = ExpandPath(filename, opener);
    if (std::remove(normalized_file.c_str()) != 0) {
        throw IOException({{"errno", std::to_string(errno)}}, "Could not remove file \"%s\": %s", filename,
                          strerror(errno));
    }
}

Here is the call graph for this function:

FileSync()

void duckdb::LocalFileSystem::FileSync ( FileHandle &  handle )

overridevirtual

Sync a file handle to disk.

Reimplemented from duckdb::FileSystem.

                                                 {
    int fd = handle.Cast<UnixFileHandle>().fd;
 
#if HAVE_FULLFSYNC
    // On macOS and iOS, fsync() doesn't guarantee durability past power failures. fcntl(F_FULLFSYNC) is required for
    // that purpose. Some filesystems don't support fcntl(F_FULLFSYNC), and require a fallback to fsync().
    if (::fcntl(fd, F_FULLFSYNC) == 0) {
        return;
    }
#endif // HAVE_FULLFSYNC
 
#if HAVE_FDATASYNC
    bool sync_success = ::fdatasync(fd) == 0;
#else
    bool sync_success = ::fsync(fd) == 0;
#endif // HAVE_FDATASYNC
 
    if (sync_success) {
        return;
    }
 
    // Use fatal exception to handle fsyncgate issue: `fsync` only reports EIO for once, which makes it unretriable and
    // data loss unrecoverable.
    if (errno == EIO) {
        throw FatalException("fsync failed!");
    }
 
    // For other types of errors, throw normal IO exception.
    throw IOException("Could not fsync file \"%s\": %s", handle.GetPath(), strerror(errno));
}

IsPathAbsolute()

bool duckdb::LocalFileSystem::IsPathAbsolute ( const string &  path )

overridevirtual

Checks if path is is an absolute path.

Reimplemented from duckdb::FileSystem.

                                                       {
    return FileSystem::IsPathAbsolute(path);
}

Here is the call graph for this function:

MakePathAbsolute()

string duckdb::LocalFileSystem::MakePathAbsolute	(	const string &	input,
		optional_ptr< FileOpener >	opener
	)

                                                                                              {
    auto path = ExpandPath(path_p, opener);
    if (!IsPathAbsolute(path)) {
        // path is not absolute - join with working directory
        return JoinPath(GetWorkingDirectory(), path);
    } else {
        // already absolute
        return path;
    }
}

PathStartsWithDrive()

bool duckdb::LocalFileSystem::PathStartsWithDrive

(

const string &

path

)

                                                            {
    return false;
}

CanHandleFile()

bool duckdb::LocalFileSystem::CanHandleFile ( const string &  fpath )

inlineoverridevirtual

Whether or not a sub-system can handle a specific file path.

Whether or not a sub-system can handle a specific file path

Reimplemented from duckdb::FileSystem.

                                                     {
        return false;
    }

Seek()

void duckdb::LocalFileSystem::Seek ( FileHandle &  handle, idx_t  location  )

overridevirtual

Set the file pointer of a file handle to a specified location. Reads and writes will happen from this location.

Reimplemented from duckdb::FileSystem.

                                                             {
    if (!CanSeek()) {
        throw IOException("Cannot seek in files of this type");
    }
    SetFilePointer(handle, location);
}

Here is the call graph for this function:

SeekPosition()

idx_t duckdb::LocalFileSystem::SeekPosition ( FileHandle &  handle )

overridevirtual

Return the current seek posiiton in the file.

Reimplemented from duckdb::FileSystem.

                                                      {
    if (!CanSeek()) {
        throw IOException("Cannot seek in files of this type");
    }
    return GetFilePointer(handle);
}

Here is the call graph for this function:

CanSeek()

bool duckdb::LocalFileSystem::CanSeek ( )

overridevirtual

Whether or not we can seek into the file.

Reimplemented from duckdb::FileSystem.

                              {
    return true;
}

Here is the caller graph for this function:

OnDiskFile()

bool duckdb::LocalFileSystem::OnDiskFile ( FileHandle &  handle )

overridevirtual

Whether or not the FS handles plain files on disk. This is relevant for certain optimizations, as random reads in a file on-disk are much cheaper than e.g. random reads in a file over the network

Reimplemented from duckdb::FileSystem.

                                                   {
    return true;
}

GetName()

std::string duckdb::LocalFileSystem::GetName ( ) const

inlineoverridevirtual

Return the name of the filesytem. Used for forming diagnosis messages.

Implements duckdb::FileSystem.

                                       {
        return "LocalFileSystem";
    }

GetLastErrorAsString()

std::string duckdb::LocalFileSystem::GetLastErrorAsString ( )

static

Returns the last Win32 error, in string format. Returns an empty string if there is no error, or on non-Windows systems.

                                                {
    return string();
}

IsPrivateFile()

bool duckdb::LocalFileSystem::IsPrivateFile ( const string &  path_p, FileOpener *  opener  )

static

Checks a file is private (checks for 600 on linux/macos, TODO: currently always returns true on windows)

                                                                            {
    auto path = FileSystem::ExpandPath(path_p, opener);
 
    struct stat st;
 
    if (lstat(path.c_str(), &st) != 0) {
        throw IOException(
            "Failed to stat '%s' when checking file permissions, file may be missing or have incorrect permissions",
            path_p.c_str());
    }
 
    // If group or other have any permission, the file is not private
    if (st.st_mode & (S_IRGRP | S_IWGRP | S_IXGRP | S_IROTH | S_IWOTH | S_IXOTH)) {
        return false;
    }
 
    return true;
}

Here is the call graph for this function:

FetchFileWithoutGlob()

vector< OpenFileInfo > duckdb::LocalFileSystem::FetchFileWithoutGlob	(	const string &	path,
		optional_ptr< FileOpener >	opener,
		bool	absolute_path
	)

                                                                               {
    vector<OpenFileInfo> result;
    if (FileExists(path, opener) || IsPipe(path, opener)) {
        result.emplace_back(path);
    } else if (!absolute_path) {
        Value value;
        if (opener && opener->TryGetCurrentSetting("file_search_path", value)) {
            auto search_paths_str = value.ToString();
            vector<std::string> search_paths = StringUtil::Split(search_paths_str, ',');
            for (const auto &search_path : search_paths) {
                auto joined_path = JoinPath(search_path, path);
                if (FileExists(joined_path, opener) || IsPipe(joined_path, opener)) {
                    result.emplace_back(joined_path);
                }
            }
        }
    }
    return result;
}

CanonicalizePath()

string duckdb::LocalFileSystem::CanonicalizePath ( const string &  path, optional_ptr< FileOpener >  opener  )

overridevirtual

Canonicalize a path.

Reimplemented from duckdb::FileSystem.

                                                                                             {
    auto path_sep = PathSeparator(input);
    if (path_sep.size() != 1) {
        throw InternalException("path separator can only be a single byte for local file systems");
    }
    // make the path absolute
    string path = MakePathAbsolute(input, opener);
 
    string current = path;
    string remainder;
    idx_t dot_dot_count = 0;
    bool is_drive = false;
    while (!current.empty()) {
        if (dot_dot_count == 0 && TryCanonicalizeExistingPath(current)) {
            // successfully canonicalized "current" - add remainder if we have any
            if (remainder.empty()) {
                return current;
            }
            if (StringUtil::EndsWith(current, path_sep)) {
                return current + remainder;
            }
            return current + path_sep + remainder;
        }
        if (is_drive) {
            // this is a drive only (e.g. C:\‍)
            // if we reach this, this is an unknown drive letter
            // use fallback canonicalize for the remainder
            return current + FileSystem::CanonicalizePath(remainder);
        }
        // move up one directory
        optional_idx sep_idx;
        for (idx_t i = current.size(); i > 0; i--) {
            // on windows we accept both separators (\ and /, and also :)
            if (current[i - 1] == path_sep[0] || current[i - 1] == '/') {
                sep_idx = i - 1;
                break;
            }
        }
        if (!sep_idx.IsValid()) {
            // exhausted the full path and nothing exists - break out
            current = string();
            break;
        }
        auto sep = sep_idx.GetIndex();
        auto component = current.substr(sep + 1);
        if (component == "..") {
            // dot dot - we need to move up a level
            // increment the count
            dot_dot_count++;
        } else if (!component.empty() && component != ".") {
            if (dot_dot_count > 0) {
                // just clear this directory
                dot_dot_count--;
            } else {
                // add component to remainder - unless it's dot or empty
                if (remainder.empty()) {
                    remainder = component;
                } else {
                    remainder = component + path_sep + remainder;
                }
            }
        }
        // continue with remainder
        current = current.substr(0, sep);
        if (current.size() == 2 && PathStartsWithDrive(current)) {
            // Windows only
            // C: and C:\\ mean different things (C: is relative, C:\\ is absolute)
            // we should have already normalized to C:\\ earlier on in this function
            // so turn this base drive letter into C:\\ for the final lookup
            is_drive = true;
            current += "\\";
        }
    }
    // failed to canonicalize path - fallback to generic canonicalization
    return FileSystem::CanonicalizePath(path);
}

Here is the call graph for this function:

ListFilesExtended()

bool duckdb::LocalFileSystem::ListFilesExtended ( const string &  directory, const std::function< void(OpenFileInfo &info)> &  callback, optional_ptr< FileOpener >  opener  )

overrideprotectedvirtual

Reimplemented from duckdb::FileSystem.

                                                                         {
    auto normalized_dir = ExpandPath(directory, opener);
    auto dir = opendir(normalized_dir.c_str());
    if (!dir) {
        return false;
    }
 
    // RAII wrapper around DIR to automatically free on exceptions in callback
    duckdb::unique_ptr<DIR, std::function<void(DIR *)>> dir_unique_ptr(dir, [](DIR *d) { closedir(d); });
 
    struct dirent *ent;
    // loop over all files in the directory
    while ((ent = readdir(dir)) != nullptr) {
        OpenFileInfo info(ent->d_name);
        auto &name = info.path;
        // skip . .. and empty files
        if (name.empty() || name == "." || name == "..") {
            continue;
        }
        // now stat the file to figure out if it is a regular file or directory
        string full_path = JoinPath(normalized_dir, name);
        struct stat status;
        auto res = stat(full_path.c_str(), &status);
        if (res != 0) {
            continue;
        }
        if (!S_ISREG(status.st_mode) && !S_ISDIR(status.st_mode)) {
            // not a file or directory: skip
            continue;
        }
        // create extended info
        info.extended_info = make_shared_ptr<ExtendedOpenFileInfo>();
        auto &options = info.extended_info->options;
        // file type
        Value file_type(S_ISDIR(status.st_mode) ? "directory" : "file");
        options.emplace("type", std::move(file_type));
        // file size
        options.emplace("file_size", Value::BIGINT(UnsafeNumericCast<int64_t>(status.st_size)));
        // last modified time
        options.emplace("last_modified", Value::TIMESTAMP(Timestamp::FromTimeT(status.st_mtime)));
        // version tag
        options.emplace("etag", Value::BLOB_RAW(GetPosixVersionTag(status)));
 
        // invoke callback
        callback(info);
    }
    return true;
}

SupportsListFilesExtended()

bool duckdb::LocalFileSystem::SupportsListFilesExtended ( ) const

inlineoverrideprotectedvirtual

Reimplemented from duckdb::FileSystem.

                                                    {
        return true;
    }

GlobFilesExtended()

unique_ptr< MultiFileList > duckdb::LocalFileSystem::GlobFilesExtended ( const string &  path, const FileGlobInput &  input, optional_ptr< FileOpener >  opener  )

overrideprotectedvirtual

Reimplemented from duckdb::FileSystem.

                                                                                              {
    return make_uniq<LocalGlobResult>(*this, path, FileGlobOptions::ALLOW_EMPTY, opener);
}

SupportsGlobExtended()

bool duckdb::LocalFileSystem::SupportsGlobExtended ( ) const

inlineoverrideprotectedvirtual

Reimplemented from duckdb::FileSystem.

                                               {
        return true;
    }

TryCanonicalizeExistingPath()

bool duckdb::LocalFileSystem::TryCanonicalizeExistingPath ( string &  path_p )

protected

                                                               {
    char resolved[PATH_MAX];
    if (!realpath(input.c_str(), resolved)) {
        return false;
    }
    input = resolved;
    return true;
}

SetFilePointer()

void duckdb::LocalFileSystem::SetFilePointer ( FileHandle &  handle, idx_t  location  )

private

Set the file pointer of a file handle to a specified location. Reads and writes will happen from this location.

                                                                       {
    int fd = handle.Cast<UnixFileHandle>().fd;
    off_t offset = lseek(fd, UnsafeNumericCast<off_t>(location), SEEK_SET);
    if (offset == (off_t)-1) {
        throw IOException({{"errno", std::to_string(errno)}}, "Could not seek to location %lld for file \"%s\": %s",
                          location, handle.path, strerror(errno));
    }
}

Here is the caller graph for this function:

GetFilePointer()

idx_t duckdb::LocalFileSystem::GetFilePointer ( FileHandle &  handle )

private

                                                        {
    int fd = handle.Cast<UnixFileHandle>().fd;
    off_t position = lseek(fd, 0, SEEK_CUR);
    if (position == (off_t)-1) {
        throw IOException({{"errno", std::to_string(errno)}}, "Could not get file position file \"%s\": %s",
                          handle.path, strerror(errno));
    }
    return UnsafeNumericCast<idx_t>(position);
}

---

The documentation for this class was generated from the following file:

• external/duckdb/duckdb.cpp