duckdb::ArrowAppender Class Reference

The ArrowAppender class can be used to incrementally construct an arrow array by appending data chunks into it. More...

Collaboration diagram for duckdb::ArrowAppender:

Public Member Functions
DUCKDB_API	ArrowAppender (vector< LogicalType > types_p, const idx_t initial_capacity, ClientProperties options, unordered_map< idx_t, const shared_ptr< ArrowTypeExtensionData > > extension_type_cast)
DUCKDB_API void	Append (DataChunk &input, idx_t from, idx_t to, idx_t input_size)
	Append a data chunk to the underlying arrow array.
DUCKDB_API ArrowArray	Finalize ()
	Returns the underlying arrow array.
idx_t	RowCount () const

Static Public Member Functions
static void	ReleaseArray (ArrowArray *array)
static ArrowArray *	FinalizeChild (const LogicalType &type, unique_ptr< ArrowAppendData > append_data_p)
static unique_ptr< ArrowAppendData >	InitializeChild (const LogicalType &type, const idx_t capacity, ClientProperties &options, const shared_ptr< ArrowTypeExtensionData > &extension_type=nullptr)
static void	AddChildren (ArrowAppendData &data, const idx_t count)

Private Attributes
vector< LogicalType >	types
	The types of the chunks that will be appended in.
vector< unique_ptr< ArrowAppendData > >	root_data
	The root arrow append data.
idx_t	row_count = 0
	The total row count that has been appended.
ClientProperties	options

Detailed Description

The ArrowAppender class can be used to incrementally construct an arrow array by appending data chunks into it.

Constructor & Destructor Documentation

ArrowAppender()

duckdb::ArrowAppender::ArrowAppender	(	vector< LogicalType >	types_p,
		const idx_t	initial_capacity,
		ClientProperties	options,
		unordered_map< idx_t, const shared_ptr< ArrowTypeExtensionData > >	extension_type_cast
	)

    : types(std::move(types_p)), options(options) {
    for (idx_t i = 0; i < types.size(); i++) {
        unique_ptr<ArrowAppendData> entry;
        bool bitshift_boolean = types[i].id() == LogicalTypeId::BOOLEAN && !options.arrow_lossless_conversion;
        if (extension_type_cast.find(i) != extension_type_cast.end() && !bitshift_boolean) {
            entry = InitializeChild(types[i], initial_capacity, options, extension_type_cast[i]);
        } else {
            entry = InitializeChild(types[i], initial_capacity, options);
        }
        root_data.push_back(std::move(entry));
    }
}

~ArrowAppender()

duckdb::ArrowAppender::~ArrowAppender

(

)

                              {
}

Member Function Documentation

Append()

void duckdb::ArrowAppender::Append	(	DataChunk &	input,
		idx_t	from,
		idx_t	to,
		idx_t	input_size
	)

Append a data chunk to the underlying arrow array.

                                                                                                     {
    D_ASSERT(types == input.GetTypes());
    D_ASSERT(to >= from);
    for (idx_t i = 0; i < input.ColumnCount(); i++) {
        if (root_data[i]->extension_data && root_data[i]->extension_data->duckdb_to_arrow) {
            Vector input_data(root_data[i]->extension_data->GetInternalType());
            root_data[i]->extension_data->duckdb_to_arrow(*options.client_context, input.data[i], input_data,
                                                          input_size);
            root_data[i]->append_vector(*root_data[i], input_data, from, to, input_size);
        } else {
            root_data[i]->append_vector(*root_data[i], input.data[i], from, to, input_size);
        }
    }
    row_count += to - from;
}

Here is the call graph for this function:

Finalize()

ArrowArray duckdb::ArrowAppender::Finalize

(

)

Returns the underlying arrow array.

                                   {
    D_ASSERT(root_data.size() == types.size());
    auto root_holder = make_uniq<ArrowAppendData>(options);
 
    ArrowArray result;
    AddChildren(*root_holder, types.size());
    result.children = root_holder->child_pointers.data();
    result.n_children = NumericCast<int64_t>(types.size());
 
    // Configure root array
    result.length = NumericCast<int64_t>(row_count);
    result.n_buffers = 1;
    result.buffers = root_holder->buffers.data(); // there is no actual buffer there since we don't have NULLs
    result.offset = 0;
    result.null_count = 0; // needs to be 0
    result.dictionary = nullptr;
    root_holder->child_data = std::move(root_data);
 
    for (idx_t i = 0; i < root_holder->child_data.size(); i++) {
        root_holder->child_arrays[i] = *ArrowAppender::FinalizeChild(types[i], std::move(root_holder->child_data[i]));
    }
 
    // Release ownership to caller
    result.private_data = root_holder.release();
    result.release = ArrowAppender::ReleaseArray;
    return result;
}

RowCount()

idx_t duckdb::ArrowAppender::RowCount

(

)

const

                                    {
    return row_count;
}

ReleaseArray()

void duckdb::ArrowAppender::ReleaseArray ( ArrowArray *  array )

static

                                                  {
    if (!array || !array->release) {
        return;
    }
    auto holder = static_cast<ArrowAppendData *>(array->private_data);
    for (int64_t i = 0; i < array->n_children; i++) {
        auto child = array->children[i];
        if (!child->release) {
            // Child was moved out of the array
            continue;
        }
        child->release(child);
        D_ASSERT(!child->release);
    }
    if (array->dictionary && array->dictionary->release) {
        array->dictionary->release(array->dictionary);
    }
    array->release = nullptr;
    delete holder;
}

FinalizeChild()

ArrowArray * duckdb::ArrowAppender::FinalizeChild ( const LogicalType &  type, unique_ptr< ArrowAppendData >  append_data_p  )

static

                                                                                                           {
    auto result = make_uniq<ArrowArray>();
 
    auto &append_data = *append_data_p;
    result->private_data = append_data_p.release();
    result->release = ReleaseArray;
    result->n_children = 0;
    result->null_count = 0;
    result->offset = 0;
    result->dictionary = nullptr;
    result->buffers = append_data.buffers.data();
    result->null_count = NumericCast<int64_t>(append_data.null_count);
    result->length = NumericCast<int64_t>(append_data.row_count);
    result->buffers[0] = append_data.GetValidityBuffer().data();
 
    if (append_data.finalize) {
        append_data.finalize(append_data, type, result.get());
    }
 
    append_data.array = std::move(result);
    return append_data.array.get();
}

InitializeChild()

unique_ptr< ArrowAppendData > duckdb::ArrowAppender::InitializeChild ( const LogicalType &  type, const idx_t  capacity, ClientProperties &  options, const shared_ptr< ArrowTypeExtensionData > &  extension_type = nullptr  )

static

                                                                                                                     {
    auto result = make_uniq<ArrowAppendData>(options);
    LogicalType array_type = type;
    if (extension_type) {
        array_type = extension_type->GetInternalType();
    }
    InitializeFunctionPointers(*result, array_type);
    result->extension_data = extension_type;
 
    const auto byte_count = (capacity + 7) / 8;
    result->GetValidityBuffer().reserve(byte_count);
    result->initialize(*result, array_type, capacity);
    return result;
}

AddChildren()

void duckdb::ArrowAppender::AddChildren ( ArrowAppendData &  data, const idx_t  count  )

static

                                                                        {
    data.child_pointers.resize(count);
    data.child_arrays.resize(count);
    for (idx_t i = 0; i < count; i++) {
        data.child_pointers[i] = &data.child_arrays[i];
    }
}

---

The documentation for this class was generated from the following file:

• external/duckdb/duckdb.cpp