duckdb::CompressedFile Class Reference

Inheritance diagram for duckdb::CompressedFile:

Collaboration diagram for duckdb::CompressedFile:

Public Member Functions
DUCKDB_API	CompressedFile (CompressedFileSystem &fs, unique_ptr< FileHandle > child_handle_p, const string &path)
DUCKDB_API idx_t	GetProgress () override
DUCKDB_API void	Initialize (QueryContext context, bool write)
DUCKDB_API int64_t	ReadData (void *buffer, int64_t nr_bytes)
DUCKDB_API int64_t	WriteData (data_ptr_t buffer, int64_t nr_bytes)
DUCKDB_API void	Close () override
	Closes the file handle.
Public Member Functions inherited from duckdb::FileHandle
DUCKDB_API	FileHandle (FileSystem &file_system, string path, FileOpenFlags flags)
	FileHandle (const FileHandle &)=delete
DUCKDB_API int64_t	Read (void *buffer, idx_t nr_bytes)
DUCKDB_API int64_t	Read (QueryContext context, void *buffer, idx_t nr_bytes)
DUCKDB_API int64_t	Write (void *buffer, idx_t nr_bytes)
DUCKDB_API int64_t	Write (QueryContext context, void *buffer, idx_t nr_bytes)
DUCKDB_API void	Read (void *buffer, idx_t nr_bytes, idx_t location)
DUCKDB_API void	Read (QueryContext context, void *buffer, idx_t nr_bytes, idx_t location)
DUCKDB_API void	Write (QueryContext context, void *buffer, idx_t nr_bytes, idx_t location)
DUCKDB_API void	Seek (idx_t location)
DUCKDB_API void	Reset ()
DUCKDB_API idx_t	SeekPosition ()
DUCKDB_API void	Sync ()
DUCKDB_API void	Truncate (int64_t new_size)
DUCKDB_API string	ReadLine ()
DUCKDB_API string	ReadLine (QueryContext context)
DUCKDB_API bool	Trim (idx_t offset_bytes, idx_t length_bytes)
virtual DUCKDB_API FileCompressionType	GetFileCompressionType ()
DUCKDB_API bool	CanSeek ()
DUCKDB_API bool	IsPipe ()
DUCKDB_API bool	OnDiskFile ()
DUCKDB_API idx_t	GetFileSize ()
DUCKDB_API FileType	GetType ()
DUCKDB_API FileMetadata	Stats ()
DUCKDB_API void	TryAddLogger (FileOpener &opener)
string	GetPath () const
FileOpenFlags	GetFlags () const
template<class TARGET >
TARGET &	Cast ()
template<class TARGET >
const TARGET &	Cast () const

Public Attributes
CompressedFileSystem &	compressed_fs
unique_ptr< FileHandle >	child_handle
bool	write = false
	Whether the file is opened for reading or for writing.
StreamData	stream_data
Public Attributes inherited from duckdb::FileHandle
FileSystem &	file_system
string	path
FileOpenFlags	flags
shared_ptr< Logger >	logger

Private Attributes
idx_t	current_position = 0
unique_ptr< StreamWrapper >	stream_wrapper

Constructor & Destructor Documentation

CompressedFile()

duckdb::CompressedFile::CompressedFile	(	CompressedFileSystem &	fs,
		unique_ptr< FileHandle >	child_handle_p,
		const string &	path
	)

    : FileHandle(fs, path, child_handle_p->GetFlags()), compressed_fs(fs), child_handle(std::move(child_handle_p)) {
}

~CompressedFile()

duckdb::CompressedFile::~CompressedFile ( )

override

                                {
    try {
        // stream_wrapper->Close() might throw
        CompressedFile::Close();
    } catch (std::exception &ex) {
        if (child_handle) {
            // FIXME: Make any log context available here.
            ErrorData data(ex);
            try {
                const auto logger = child_handle->logger;
                if (logger) {
                    DUCKDB_LOG_ERROR(logger, "CompressedFile::~CompressedFile()\t\t" + data.Message());
                }
            } catch (...) { // NOLINT
            }
        }
    } catch (...) { // NOLINT
    }
}

Member Function Documentation

GetProgress()

idx_t duckdb::CompressedFile::GetProgress ( )

overridevirtual

Reimplemented from duckdb::FileHandle.

                                  {
    return current_position;
}

Initialize()

void duckdb::CompressedFile::Initialize	(	QueryContext	context,
		bool	write
	)

                                                                {
    Close();
 
    this->write = write;
    stream_data.in_buf_size = compressed_fs.InBufferSize();
    stream_data.out_buf_size = compressed_fs.OutBufferSize();
    stream_data.in_buff = make_unsafe_uniq_array<data_t>(stream_data.in_buf_size);
    stream_data.in_buff_start = stream_data.in_buff.get();
    stream_data.in_buff_end = stream_data.in_buff.get();
    stream_data.out_buff = make_unsafe_uniq_array<data_t>(stream_data.out_buf_size);
    stream_data.out_buff_start = stream_data.out_buff.get();
    stream_data.out_buff_end = stream_data.out_buff.get();
 
    current_position = 0;
 
    stream_wrapper = compressed_fs.CreateStream();
    stream_wrapper->Initialize(context, *this, write);
}

ReadData()

int64_t duckdb::CompressedFile::ReadData	(	void *	buffer,
		int64_t	nr_bytes
	)

                                                                {
    idx_t total_read = 0;
    while (true) {
        // first check if there are input bytes available in the output buffers
        if (stream_data.out_buff_start != stream_data.out_buff_end) {
            // there is! copy it into the output buffer
            auto available =
                MinValue<idx_t>(UnsafeNumericCast<idx_t>(remaining),
                                UnsafeNumericCast<idx_t>(stream_data.out_buff_end - stream_data.out_buff_start));
            memcpy(static_cast<data_ptr_t>(buffer) + total_read, stream_data.out_buff_start, available);
 
            // increment the total read variables as required
            stream_data.out_buff_start += available;
            total_read += available;
            remaining = UnsafeNumericCast<int64_t>(UnsafeNumericCast<idx_t>(remaining) - available);
            if (remaining == 0) {
                // done! read enough
                return UnsafeNumericCast<int64_t>(total_read);
            }
        }
        if (!stream_wrapper) {
            return UnsafeNumericCast<int64_t>(total_read);
        }
        current_position += static_cast<idx_t>(stream_data.in_buff_end - stream_data.in_buff_start);
        // ran out of buffer: read more data from the child stream
        stream_data.out_buff_start = stream_data.out_buff.get();
        stream_data.out_buff_end = stream_data.out_buff.get();
        D_ASSERT(stream_data.in_buff_start <= stream_data.in_buff_end);
        D_ASSERT(stream_data.in_buff_end <= stream_data.in_buff_start + stream_data.in_buf_size);
 
        // read more input when requested and still data in the input stream
        if (stream_data.refresh && (stream_data.in_buff_end == stream_data.in_buff.get() + stream_data.in_buf_size)) {
            auto bufrem = stream_data.in_buff_end - stream_data.in_buff_start;
            // buffer not empty, move remaining bytes to the beginning
            memmove(stream_data.in_buff.get(), stream_data.in_buff_start, UnsafeNumericCast<size_t>(bufrem));
            stream_data.in_buff_start = stream_data.in_buff.get();
            // refill the rest of input buffer
            auto sz = child_handle->Read(QueryContext(), stream_data.in_buff_start + bufrem,
                                         stream_data.in_buf_size - UnsafeNumericCast<idx_t>(bufrem));
            stream_data.in_buff_end = stream_data.in_buff_start + bufrem + sz;
            if (sz <= 0) {
                stream_wrapper.reset();
                break;
            }
        }
 
        // read more input if none available
        if (stream_data.in_buff_start == stream_data.in_buff_end) {
            // empty input buffer: refill from the start
            stream_data.in_buff_start = stream_data.in_buff.get();
            stream_data.in_buff_end = stream_data.in_buff_start;
            auto sz = child_handle->Read(QueryContext(), stream_data.in_buff.get(), stream_data.in_buf_size);
            if (sz <= 0) {
                stream_wrapper.reset();
                break;
            }
            stream_data.in_buff_end = stream_data.in_buff_start + sz;
        }
 
        auto finished = stream_wrapper->Read(stream_data);
        if (finished) {
            stream_wrapper.reset();
        }
    }
    return UnsafeNumericCast<int64_t>(total_read);
}

WriteData()

int64_t duckdb::CompressedFile::WriteData	(	data_ptr_t	buffer,
		int64_t	nr_bytes
	)

                                                                     {
    stream_wrapper->Write(*this, stream_data, buffer, nr_bytes);
    return nr_bytes;
}

Close()

void duckdb::CompressedFile::Close ( )

overridevirtual

Closes the file handle.

Implements duckdb::FileHandle.

                           {
    if (stream_wrapper) {
        stream_wrapper->Close();
        stream_wrapper.reset();
    }
    stream_data.in_buff.reset();
    stream_data.out_buff.reset();
    stream_data.out_buff_start = nullptr;
    stream_data.out_buff_end = nullptr;
    stream_data.in_buff_start = nullptr;
    stream_data.in_buff_end = nullptr;
    stream_data.in_buf_size = 0;
    stream_data.out_buf_size = 0;
    stream_data.refresh = false;
}

---

The documentation for this class was generated from the following file:

• external/duckdb/duckdb.cpp