duckdb::ArrowVarcharData< SRC, OP, BUFTYPE > Struct Template Reference

Static Public Member Functions
static void	Initialize (ArrowAppendData &result, const LogicalType &type, idx_t capacity)
template<bool LARGE_STRING>
static void	AppendTemplated (ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size)
static void	Append (ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size)
static void	Finalize (ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result)

Member Function Documentation

Initialize()

template<class SRC = string_t, class OP = ArrowVarcharConverter, class BUFTYPE = int64_t>

static void duckdb::ArrowVarcharData< SRC, OP, BUFTYPE >::Initialize ( ArrowAppendData &  result, const LogicalType &  type, idx_t  capacity  )

inlinestatic

                                                                                             {
        result.GetMainBuffer().reserve((capacity + 1) * sizeof(BUFTYPE));
        result.GetAuxBuffer().reserve(capacity);
    }

AppendTemplated()

template<class SRC = string_t, class OP = ArrowVarcharConverter, class BUFTYPE = int64_t>

template<bool LARGE_STRING>

static void duckdb::ArrowVarcharData< SRC, OP, BUFTYPE >::AppendTemplated ( ArrowAppendData &  append_data, Vector &  input, idx_t  from, idx_t  to, idx_t  input_size  )

inlinestatic

                                                                                                                     {
        idx_t size = to - from;
        UnifiedVectorFormat format;
        input.ToUnifiedFormat(input_size, format);
        auto &main_buffer = append_data.GetMainBuffer();
        auto &validity_buffer = append_data.GetValidityBuffer();
        auto &aux_buffer = append_data.GetAuxBuffer();
 
        // resize the validity mask and set up the validity buffer for iteration
        ArrowAppendData::ResizeValidity(validity_buffer, append_data.row_count + size);
        auto validity_data = (uint8_t *)validity_buffer.data();
 
        // resize the offset buffer - the offset buffer holds the offsets into the child array
        main_buffer.resize(main_buffer.size() + sizeof(BUFTYPE) * (size + 1));
        auto data = UnifiedVectorFormat::GetData<SRC>(format);
        auto offset_data = main_buffer.GetData<BUFTYPE>();
        if (append_data.row_count == 0) {
            // first entry
            offset_data[0] = 0;
        }
        // now append the string data to the auxiliary buffer
        // the auxiliary buffer's length depends on the string lengths, so we resize as required
        auto last_offset = offset_data[append_data.row_count];
        for (idx_t i = from; i < to; i++) {
            auto source_idx = format.sel->get_index(i);
            auto offset_idx = append_data.row_count + i + 1 - from;
 
            if (!format.validity.RowIsValid(source_idx)) {
                uint8_t current_bit;
                idx_t current_byte;
                ArrowAppendData::GetBitPosition(append_data.row_count + i - from, current_byte, current_bit);
                append_data.SetNull(validity_data, current_byte, current_bit);
                offset_data[offset_idx] = last_offset;
                continue;
            }
 
            auto string_length = OP::GetLength(data[source_idx]);
 
            // append the offset data
            auto current_offset = UnsafeNumericCast<idx_t>(last_offset) + string_length;
            if (!LARGE_STRING &&
                UnsafeNumericCast<idx_t>(last_offset) + string_length > NumericLimits<int32_t>::Maximum()) {
                D_ASSERT(append_data.options.arrow_offset_size == ArrowOffsetSize::REGULAR);
                throw InvalidInputException(
                    "Arrow Appender: The maximum total string size for regular string buffers is "
                    "%u but the offset of %lu exceeds this.\n* SET arrow_large_buffer_size=true to use large string "
                    "buffers",
                    NumericLimits<int32_t>::Maximum(), current_offset);
            }
            offset_data[offset_idx] = UnsafeNumericCast<BUFTYPE>(current_offset);
 
            // resize the string buffer if required, and write the string data
            aux_buffer.resize(current_offset);
            OP::WriteData(aux_buffer.data() + last_offset, data[source_idx]);
 
            last_offset = UnsafeNumericCast<BUFTYPE>(current_offset);
        }
        append_data.row_count += size;
    }

Append()

template<class SRC = string_t, class OP = ArrowVarcharConverter, class BUFTYPE = int64_t>

static void duckdb::ArrowVarcharData< SRC, OP, BUFTYPE >::Append ( ArrowAppendData &  append_data, Vector &  input, idx_t  from, idx_t  to, idx_t  input_size  )

inlinestatic

                                                                                                            {
        if (append_data.options.arrow_offset_size == ArrowOffsetSize::REGULAR) {
            // Check if the offset exceeds the max supported value
            AppendTemplated<false>(append_data, input, from, to, input_size);
        } else {
            AppendTemplated<true>(append_data, input, from, to, input_size);
        }
    }

Finalize()

template<class SRC = string_t, class OP = ArrowVarcharConverter, class BUFTYPE = int64_t>

static void duckdb::ArrowVarcharData< SRC, OP, BUFTYPE >::Finalize ( ArrowAppendData &  append_data, const LogicalType &  type, ArrowArray *  result  )

inlinestatic

                                                                                                    {
        result->n_buffers = 3;
        result->buffers[1] = append_data.GetMainBuffer().data();
        result->buffers[2] = append_data.GetAuxBuffer().data();
    }

---

The documentation for this struct was generated from the following file:

• external/duckdb/duckdb.cpp