duckdb::ArrowVarcharToStringViewData Struct Reference

Static Public Member Functions
static void	Initialize (ArrowAppendData &result, const LogicalType &type, idx_t capacity)
static void	Append (ArrowAppendData &append_data, Vector &input, idx_t from, idx_t to, idx_t input_size)
static void	Finalize (ArrowAppendData &append_data, const LogicalType &type, ArrowArray *result)

Member Function Documentation

Initialize()

static void duckdb::ArrowVarcharToStringViewData::Initialize ( ArrowAppendData &  result, const LogicalType &  type, idx_t  capacity  )

inlinestatic

                                                                                             {
        result.GetMainBuffer().reserve((capacity) * sizeof(arrow_string_view_t));
        result.GetAuxBuffer().reserve(capacity);
        result.GetBufferSizeBuffer().reserve(sizeof(int64_t));
    }

Append()

static void duckdb::ArrowVarcharToStringViewData::Append ( ArrowAppendData &  append_data, Vector &  input, idx_t  from, idx_t  to, idx_t  input_size  )

inlinestatic

                                                                                                            {
        idx_t size = to - from;
        UnifiedVectorFormat format;
        input.ToUnifiedFormat(input_size, format);
        auto &main_buffer = append_data.GetMainBuffer();
        auto &validity_buffer = append_data.GetValidityBuffer();
        auto &aux_buffer = append_data.GetAuxBuffer();
        // resize the validity mask and set up the validity buffer for iteration
        ArrowAppendData::ResizeValidity(validity_buffer, append_data.row_count + size);
        auto validity_data = (uint8_t *)validity_buffer.data();
 
        main_buffer.resize(main_buffer.size() + sizeof(arrow_string_view_t) * (size));
        // resize the offset buffer - the offset buffer holds the offsets into the child array
        auto data = UnifiedVectorFormat::GetData<string_t>(format);
        for (idx_t i = from; i < to; i++) {
            auto result_idx = append_data.row_count + i - from;
            auto arrow_data = main_buffer.GetData<arrow_string_view_t>();
            auto source_idx = format.sel->get_index(i);
            if (!format.validity.RowIsValid(source_idx)) {
                // Null value
                uint8_t current_bit;
                idx_t current_byte;
                ArrowAppendData::GetBitPosition(result_idx, current_byte, current_bit);
                append_data.SetNull(validity_data, current_byte, current_bit);
                // We have to set these bytes to 0, for some reason
                arrow_data[result_idx] = arrow_string_view_t(0, "");
                continue;
            }
            // These two are now the same buffer
            idx_t string_length = ArrowVarcharConverter::GetLength(data[source_idx]);
            auto string_data = data[source_idx].GetData();
            if (string_length <= ArrowStringViewConstants::MAX_INLINED_BYTES) {
                //  This string is inlined
                //  | Bytes 0-3  | Bytes 4-15                            |
                //  |------------|---------------------------------------|
                //  | length     | data (padded with 0)                  |
                arrow_data[result_idx] = arrow_string_view_t(UnsafeNumericCast<int32_t>(string_length), string_data);
            } else {
                // This string is not inlined, we have to check a different buffer and offsets
                //  | Bytes 0-3  | Bytes 4-7  | Bytes 8-11 | Bytes 12-15 |
                //  |------------|------------|------------|-------------|
                //  | length     | prefix     | buf. index | offset      |
                arrow_data[result_idx] = arrow_string_view_t(UnsafeNumericCast<int32_t>(string_length), string_data, 0,
                                                             UnsafeNumericCast<int32_t>(append_data.offset));
                auto current_offset = append_data.offset + string_length;
                aux_buffer.resize(current_offset);
                ArrowVarcharConverter::WriteData(aux_buffer.data() + append_data.offset, data[source_idx]);
                append_data.offset = current_offset;
            }
        }
        append_data.row_count += size;
    }

Finalize()

static void duckdb::ArrowVarcharToStringViewData::Finalize ( ArrowAppendData &  append_data, const LogicalType &  type, ArrowArray *  result  )

inlinestatic

                                                                                                    {
        // We output four buffers
        result->n_buffers = 4;
        // Buffer 0 is the validity mask
        // Buffer 1 is our string views (short/long strings)
        result->buffers[1] = append_data.GetMainBuffer().data();
        // Buffer 2 is our only data buffer, could theoretically be more [ buffers ]
        result->buffers[2] = append_data.GetAuxBuffer().data();
        // Buffer 3 is the data-buffer lengths buffer, and we also populate it in to finalize
        reinterpret_cast<int64_t *>(append_data.GetBufferSizeBuffer().data())[0] =
            UnsafeNumericCast<int64_t>(append_data.offset);
        result->buffers[3] = append_data.GetBufferSizeBuffer().data();
    }

---

The documentation for this struct was generated from the following file:

• external/duckdb/duckdb.cpp