duckdb::GZipFileSystem Class Reference

Inheritance diagram for duckdb::GZipFileSystem:

Collaboration diagram for duckdb::GZipFileSystem:

Public Member Functions
unique_ptr< FileHandle >	OpenCompressedFile (QueryContext context, unique_ptr< FileHandle > handle, bool write) override
std::string	GetName () const override
	Return the name of the filesytem. Used for forming diagnosis messages.
unique_ptr< StreamWrapper >	CreateStream () override
idx_t	InBufferSize () override
idx_t	OutBufferSize () override
Public Member Functions inherited from duckdb::CompressedFileSystem
DUCKDB_API int64_t	Read (FileHandle &handle, void *buffer, int64_t nr_bytes) override
DUCKDB_API int64_t	Write (FileHandle &handle, void *buffer, int64_t nr_bytes) override
	Write nr_bytes from the buffer into the file, moving the file pointer forward by nr_bytes.
DUCKDB_API void	Reset (FileHandle &handle) override
	Reset a file to the beginning (equivalent to Seek(handle, 0) for simple files)
DUCKDB_API int64_t	GetFileSize (FileHandle &handle) override
	Returns the file size of a file handle, returns -1 on error.
DUCKDB_API bool	OnDiskFile (FileHandle &handle) override
DUCKDB_API bool	CanSeek () override
	Whether or not we can seek into the file.
Public Member Functions inherited from duckdb::FileSystem
virtual DUCKDB_API unique_ptr< FileHandle >	OpenFile (const string &path, FileOpenFlags flags, optional_ptr< FileOpener > opener=nullptr)
DUCKDB_API unique_ptr< FileHandle >	OpenFile (const OpenFileInfo &path, FileOpenFlags flags, optional_ptr< FileOpener > opener=nullptr)
virtual DUCKDB_API void	Read (FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location)
virtual DUCKDB_API void	Write (FileHandle &handle, void *buffer, int64_t nr_bytes, idx_t location)
virtual DUCKDB_API bool	Trim (FileHandle &handle, idx_t offset_bytes, idx_t length_bytes)
virtual DUCKDB_API timestamp_t	GetLastModifiedTime (FileHandle &handle)
	Returns the file last modified time of a file handle, returns timespec with zero on all attributes on error.
virtual DUCKDB_API string	GetVersionTag (FileHandle &handle)
virtual DUCKDB_API FileType	GetFileType (FileHandle &handle)
	Returns the file type of the attached handle.
virtual DUCKDB_API FileMetadata	Stats (FileHandle &handle)
	Returns the file stats of the attached handle.
virtual DUCKDB_API void	Truncate (FileHandle &handle, int64_t new_size)
virtual DUCKDB_API bool	DirectoryExists (const string &directory, optional_ptr< FileOpener > opener=nullptr)
	Check if a directory exists.
virtual DUCKDB_API void	CreateDirectory (const string &directory, optional_ptr< FileOpener > opener=nullptr)
	Create a directory if it does not exist.
virtual DUCKDB_API void	CreateDirectoriesRecursive (const string &path, optional_ptr< FileOpener > opener=nullptr)
	Helper function that uses DirectoryExists and CreateDirectory to ensure all directories in path are created.
virtual DUCKDB_API void	RemoveDirectory (const string &directory, optional_ptr< FileOpener > opener=nullptr)
	Recursively remove a directory and all files in it.
virtual DUCKDB_API bool	ListFiles (const string &directory, const std::function< void(const string &, bool)> &callback, FileOpener *opener=nullptr)
	List files in a directory, invoking the callback method for each one with (filename, is_dir)
DUCKDB_API bool	ListFiles (const string &directory, const std::function< void(OpenFileInfo &info)> &callback, optional_ptr< FileOpener > opener=nullptr)
virtual DUCKDB_API void	MoveFile (const string &source, const string &target, optional_ptr< FileOpener > opener=nullptr)
virtual DUCKDB_API bool	FileExists (const string &filename, optional_ptr< FileOpener > opener=nullptr)
	Check if a file exists.
virtual DUCKDB_API bool	IsPipe (const string &filename, optional_ptr< FileOpener > opener=nullptr)
	Check if path is pipe.
virtual DUCKDB_API void	RemoveFile (const string &filename, optional_ptr< FileOpener > opener=nullptr)
	Remove a file from disk.
virtual DUCKDB_API bool	TryRemoveFile (const string &filename, optional_ptr< FileOpener > opener=nullptr)
	Remvoe a file from disk if it exists - if it does not exist, return false.
virtual DUCKDB_API void	RemoveFiles (const vector< string > &filenames, optional_ptr< FileOpener > opener=nullptr)
	Remove multiple files from disk - does not error if any file does not exist.
virtual DUCKDB_API void	FileSync (FileHandle &handle)
	Sync a file handle to disk.
virtual DUCKDB_API string	GetHomeDirectory ()
	Gets the users home directory.
virtual DUCKDB_API string	ExpandPath (const string &path)
	Expands a given path, including e.g. expanding the home directory of the user.
virtual DUCKDB_API string	PathSeparator (const string &path)
	Path separator for path.
virtual DUCKDB_API bool	IsPathAbsolute (const string &path)
	Checks if path is is an absolute path.
DUCKDB_API string	JoinPath (const string &a, const string &path)
	Join two paths together.
template<typename... ARGS>
string	JoinPath (const string &a, const string &b, ARGS... args)
DUCKDB_API string	ConvertSeparators (const string &path)
	Convert separators in a path to the local separators (e.g. convert "/" into \ on windows)
DUCKDB_API string	ExtractBaseName (const string &path)
	Extract the base name of a file (e.g. if the input is lib/example.dll the base name is 'example')
DUCKDB_API string	ExtractExtension (const string &path)
	Extract the extension of a file (e.g. if the input is lib/example.dll the extension is 'dll')
DUCKDB_API string	ExtractName (const string &path)
	Extract the name of a file (e.g if the input is lib/example.dll the name is 'example.dll')
virtual DUCKDB_API vector< OpenFileInfo >	Glob (const string &path, FileOpener *opener=nullptr)
	Runs a glob on the file system, returning a list of matching files.
DUCKDB_API unique_ptr< MultiFileList >	Glob (const string &path, const FileGlobInput &input, optional_ptr< FileOpener > opener)
DUCKDB_API unique_ptr< MultiFileList >	GlobFileList (const string &path, const FileGlobInput &input=FileGlobOptions::DISALLOW_EMPTY)
DUCKDB_API vector< OpenFileInfo >	GlobFiles (const string &pattern, const FileGlobInput &input=FileGlobOptions::DISALLOW_EMPTY)
virtual DUCKDB_API void	RegisterSubSystem (unique_ptr< FileSystem > sub_fs)
	registers a sub-file system to handle certain file name prefixes, e.g. http:// etc.
virtual DUCKDB_API void	RegisterSubSystem (FileCompressionType compression_type, unique_ptr< FileSystem > fs)
virtual DUCKDB_API void	UnregisterSubSystem (const string &name)
	Unregister a sub-filesystem by name.
virtual DUCKDB_API unique_ptr< FileSystem >	ExtractSubSystem (const string &name)
virtual DUCKDB_API vector< string >	ListSubSystems ()
	List registered sub-filesystems, including builtin ones.
virtual DUCKDB_API bool	CanHandleFile (const string &fpath)
	Whether or not a sub-system can handle a specific file path.
virtual DUCKDB_API void	Seek (FileHandle &handle, idx_t location)
	Set the file pointer of a file handle to a specified location. Reads and writes will happen from this location.
virtual DUCKDB_API idx_t	SeekPosition (FileHandle &handle)
virtual DUCKDB_API bool	IsManuallySet ()
	If FS was manually set by the user.
virtual DUCKDB_API void	SetDisabledFileSystems (const vector< string > &names)
virtual DUCKDB_API bool	SubSystemIsDisabled (const string &name)
virtual DUCKDB_API bool	IsDisabledForPath (const string &path)
	Check if the filesystem that would handle this path is disabled.
virtual DUCKDB_API string	CanonicalizePath (const string &path, optional_ptr< FileOpener > opener=nullptr)
	Canonicalize a path.
template<class TARGET >
TARGET &	Cast ()
template<class TARGET >
const TARGET &	Cast () const

Static Public Member Functions
static void	VerifyGZIPHeader (uint8_t gzip_hdr[], idx_t read_count, optional_ptr< CompressedFile > source_file)
	Verifies that a buffer contains a valid GZIP header.
static bool	CheckIsZip (const char *length, idx_t size)
static string	UncompressGZIPString (const string &in)
	Consumes a byte stream as a gzip string, returning the decompressed string.
static string	UncompressGZIPString (const char *length, idx_t size)
Static Public Member Functions inherited from duckdb::FileSystem
static DUCKDB_API FileSystem &	GetFileSystem (ClientContext &context)
static DUCKDB_API FileSystem &	GetFileSystem (DatabaseInstance &db)
static DUCKDB_API FileSystem &	Get (AttachedDatabase &db)
static DUCKDB_API void	SetWorkingDirectory (const string &path)
	Sets the working directory.
static DUCKDB_API string	GetWorkingDirectory ()
	Gets the working directory.
static DUCKDB_API string	GetHomeDirectory (optional_ptr< FileOpener > opener)
	Gets the users home directory.
static DUCKDB_API string	ExpandPath (const string &path, optional_ptr< FileOpener > opener)
	Expands a given path, including e.g. expanding the home directory of the user.
static DUCKDB_API optional_idx	GetAvailableMemory ()
	Returns the system-available memory in bytes. Returns DConstants::INVALID_INDEX if the system function fails.
static DUCKDB_API optional_idx	GetAvailableDiskSpace (const string &path)
	Returns the space available on the disk. Returns DConstants::INVALID_INDEX if the information was not available.
static DUCKDB_API string	GetEnvVariable (const string &name)
	Returns the value of an environment variable - or the empty string if it is not set.
static DUCKDB_API bool	HasGlob (const string &str)
	Whether there is a glob in the string.
static DUCKDB_API unique_ptr< FileSystem >	CreateLocal ()
	Create a LocalFileSystem.
static DUCKDB_API bool	IsRemoteFile (const string &path)
	Whether or not a file is remote or local, based only on file path.
static DUCKDB_API bool	IsRemoteFile (const string &path, string &extension)
static DUCKDB_API bool	IsDirectory (const OpenFileInfo &info)

Static Private Attributes
static constexpr const idx_t	BUFFER_SIZE = 1u << 15

Additional Inherited Members
Protected Member Functions inherited from duckdb::FileSystem
virtual DUCKDB_API unique_ptr< FileHandle >	OpenFileExtended (const OpenFileInfo &path, FileOpenFlags flags, optional_ptr< FileOpener > opener)
virtual DUCKDB_API bool	SupportsOpenFileExtended () const
virtual DUCKDB_API bool	ListFilesExtended (const string &directory, const std::function< void(OpenFileInfo &info)> &callback, optional_ptr< FileOpener > opener)
virtual DUCKDB_API bool	SupportsListFilesExtended () const
virtual DUCKDB_API unique_ptr< MultiFileList >	GlobFilesExtended (const string &path, const FileGlobInput &input, optional_ptr< FileOpener > opener=nullptr)
virtual DUCKDB_API bool	SupportsGlobExtended () const

Member Function Documentation

OpenCompressedFile()

unique_ptr< FileHandle > duckdb::GZipFileSystem::OpenCompressedFile ( QueryContext  context, unique_ptr< FileHandle >  handle, bool  write  )

overridevirtual

Reimplemented from duckdb::FileSystem.

                                                                      {
    auto path = handle->path;
    return make_uniq<GZipFile>(context, std::move(handle), path, write);
}

GetName()

std::string duckdb::GZipFileSystem::GetName ( ) const

inlineoverridevirtual

Return the name of the filesytem. Used for forming diagnosis messages.

Implements duckdb::FileSystem.

                                       {
        return "GZipFileSystem";
    }

VerifyGZIPHeader()

void duckdb::GZipFileSystem::VerifyGZIPHeader ( uint8_t  gzip_hdr[], idx_t  read_count, optional_ptr< CompressedFile >  source_file  )

static

Verifies that a buffer contains a valid GZIP header.

                                                                                                                    {
    // include the filename in the error message if known
    string file_info = source_file ? ": " + source_file->path : "";
 
    // check for incorrectly formatted files
    if (read_count != GZIP_HEADER_MINSIZE) {
        throw IOException("Input is not a GZIP stream" + file_info);
    }
    if (gzip_hdr[0] != 0x1F || gzip_hdr[1] != 0x8B) { // magic header
        throw IOException("Input is not a GZIP stream" + file_info);
    }
    if (gzip_hdr[2] != GZIP_COMPRESSION_DEFLATE) { // compression method
        throw IOException("Unsupported GZIP compression method" + file_info);
    }
    if (gzip_hdr[3] & GZIP_FLAG_UNSUPPORTED) {
        throw IOException("Unsupported GZIP archive" + file_info);
    }
}

CheckIsZip()

bool duckdb::GZipFileSystem::CheckIsZip ( const char *  length, duckdb::idx_t  size  )

static

                                                                  {
    if (size < GZIP_HEADER_MINSIZE) {
        return false;
    }
 
    auto data_ptr = reinterpret_cast<const uint8_t *>(data);
    if (data_ptr[0] != 0x1F || data_ptr[1] != 0x8B) {
        return false;
    }
 
    if (data_ptr[2] != GZIP_COMPRESSION_DEFLATE) {
        return false;
    }
 
    return true;
}

UncompressGZIPString() [1/2]

string duckdb::GZipFileSystem::UncompressGZIPString ( const string &  in )

static

Consumes a byte stream as a gzip string, returning the decompressed string.

                                                            {
    return UncompressGZIPString(in.data(), in.size());
}

Here is the call graph for this function:

Here is the caller graph for this function:

UncompressGZIPString() [2/2]

string duckdb::GZipFileSystem::UncompressGZIPString ( const char *  length, idx_t  size  )

static

                                                                        {
    // decompress file
    auto body_ptr = data;
 
    auto mz_stream_ptr = make_uniq<duckdb_miniz::mz_stream>();
    memset(mz_stream_ptr.get(), 0, sizeof(duckdb_miniz::mz_stream));
 
    uint8_t gzip_hdr[GZIP_HEADER_MINSIZE];
 
    // check for incorrectly formatted files
 
    // TODO this is mostly the same as gzip_file_system.cpp
    if (size < GZIP_HEADER_MINSIZE) {
        throw IOException("Input is not a GZIP stream");
    }
    memcpy(gzip_hdr, body_ptr, GZIP_HEADER_MINSIZE);
    body_ptr += GZIP_HEADER_MINSIZE;
    GZipFileSystem::VerifyGZIPHeader(gzip_hdr, GZIP_HEADER_MINSIZE, nullptr);
 
    if (gzip_hdr[3] & GZIP_FLAG_EXTRA) {
        throw IOException("Extra field in a GZIP stream unsupported");
    }
 
    if (gzip_hdr[3] & GZIP_FLAG_NAME) {
        char c;
        do {
            c = *body_ptr;
            body_ptr++;
        } while (c != '\0' && static_cast<idx_t>(body_ptr - data) < size);
    }
 
    // stream is now set to beginning of payload data
    auto status = duckdb_miniz::mz_inflateInit2(mz_stream_ptr.get(), -MZ_DEFAULT_WINDOW_BITS);
    if (status != duckdb_miniz::MZ_OK) {
        throw InternalException("Failed to initialize miniz");
    }
 
    auto bytes_remaining = size - NumericCast<idx_t>(body_ptr - data);
    mz_stream_ptr->next_in = const_uchar_ptr_cast(body_ptr);
    mz_stream_ptr->avail_in = NumericCast<unsigned int>(bytes_remaining);
 
    string decompressed;
 
    while (status == duckdb_miniz::MZ_OK) {
        unsigned char decompress_buffer[BUFSIZ];
        mz_stream_ptr->next_out = decompress_buffer;
        mz_stream_ptr->avail_out = sizeof(decompress_buffer);
        status = mz_inflate(mz_stream_ptr.get(), duckdb_miniz::MZ_NO_FLUSH);
        if (status != duckdb_miniz::MZ_STREAM_END && status != duckdb_miniz::MZ_OK) {
            throw IOException("Failed to uncompress");
        }
        decompressed.append(char_ptr_cast(decompress_buffer), mz_stream_ptr->total_out - decompressed.size());
    }
    duckdb_miniz::mz_inflateEnd(mz_stream_ptr.get());
 
    if (decompressed.empty()) {
        throw IOException("Failed to uncompress");
    }
    return decompressed;
}

CreateStream()

unique_ptr< StreamWrapper > duckdb::GZipFileSystem::CreateStream ( )

overridevirtual

Implements duckdb::CompressedFileSystem.

                                                       {
    return make_uniq<MiniZStreamWrapper>();
}

InBufferSize()

idx_t duckdb::GZipFileSystem::InBufferSize ( )

overridevirtual

Implements duckdb::CompressedFileSystem.

                                   {
    return BUFFER_SIZE;
}

OutBufferSize()

idx_t duckdb::GZipFileSystem::OutBufferSize ( )

overridevirtual

Implements duckdb::CompressedFileSystem.

                                    {
    return BUFFER_SIZE;
}

---

The documentation for this class was generated from the following file:

• external/duckdb/duckdb.cpp