duckdb::CSVWriter Class Reference

Collaboration diagram for duckdb::CSVWriter:

Public Member Functions
	CSVWriter (WriteStream &stream, vector< string > name_list, bool shared=true)
	Create a CSVWriter that writes to a (non-owned) WriteStream.
	CSVWriter (CSVReaderOptions &options, FileSystem &fs, const string &file_path, FileCompressionType compression, bool shared=true)
	Create a CSVWriter that writes to a file.
void	Initialize (bool force=false)
	Writes header and prefix if necessary.
void	WriteRawString (const string &data)
	Writes the raw string directly into the output stream.
void	WriteHeader ()
	Writes the header directly into the output stream.
void	WriteRawString (const string &prefix, CSVWriterState &local_state)
	Write the Raw String, using the local_state.
void	WriteChunk (DataChunk &input, CSVWriterState &local_state)
	Write a chunk of VARCHAR vectors to the CSV file (any casts are the responsibility of caller)
void	WriteChunk (DataChunk &input)
	(Non-shared only) variant of WriteChunk
void	Flush (CSVWriterState &local_state)
	Flushes all data in the local write state.
void	Flush ()
	(Non-shared only) variant of Flush
void	Reset (optional_ptr< CSVWriterState > local_state)
	Resets the state of the writer. Warning: the file_writer is not reset.
void	Close ()
	Closes the writer, optionally writes a postfix.
idx_t	BytesWritten ()
idx_t	FileSize ()
	BytesWritten + OriginalSize;.
bool	WrittenAnything ()
void	SetWrittenAnything (bool val)

Public Attributes
vector< unique_ptr< Expression > >	string_casts
CSVReaderOptions	options
CSVWriterOptions	writer_options

Protected Member Functions
void	FlushInternal (CSVWriterState &local_state)
void	ResetInternal (optional_ptr< CSVWriterState > local_state)

Static Protected Member Functions
static void	WriteQuoteOrEscape (WriteStream &writer, char quote_or_escape)
static string	AddEscapes (char to_be_escaped, char escape, const string &val)
static bool	RequiresQuotes (const char *str, idx_t len, const string &null_str, const vector< bool > &requires_quotes)
static void	WriteQuotedString (WriteStream &writer, const char *str, idx_t len, bool force_quote, const string &null_str, const vector< bool > &requires_quotes, char quote, char escape)
static void	WriteQuotedString (WriteStream &writer, const char *str, idx_t len, idx_t col_idx, CSVReaderOptions &options, CSVWriterOptions &writer_options)
static void	WriteChunk (DataChunk &input, MemoryStream &writer, CSVReaderOptions &options, bool &written_anything, CSVWriterOptions &writer_options)
static void	WriteHeader (MemoryStream &stream, CSVReaderOptions &options, CSVWriterOptions &writer_options)

Protected Attributes
bool	written_anything = false
	If we've written any rows yet, allows us to prevent a trailing comma when writing JSON ARRAY.
unique_ptr< BufferedFileWriter >	file_writer
	(optional) The owned file writer of this CSVWriter
WriteStream &	write_stream
	The WriteStream to write the CSV data to.
idx_t	bytes_written = 0
bool	should_initialize
mutex	lock
bool	shared
unique_ptr< CSVWriterState >	global_write_state

Constructor & Destructor Documentation

CSVWriter() [1/2]

duckdb::CSVWriter::CSVWriter	(	WriteStream &	stream,
		vector< string >	name_list,
		bool	shared = true
	)

Create a CSVWriter that writes to a (non-owned) WriteStream.

    : writer_options(options.dialect_options.state_machine_options.delimiter.GetValue(),
                     options.dialect_options.state_machine_options.quote.GetValue(), options.write_newline),
      write_stream(stream), should_initialize(true), shared(shared) {
    auto size = name_list.size();
    options.name_list = std::move(name_list);
    options.force_quote.resize(size, false);
    options.force_quote.resize(size, false);
 
    if (!shared) {
        global_write_state = make_uniq<CSVWriterState>();
    }
}

Here is the call graph for this function:

CSVWriter() [2/2]

duckdb::CSVWriter::CSVWriter	(	CSVReaderOptions &	options,
		FileSystem &	fs,
		const string &	file_path,
		FileCompressionType	compression,
		bool	shared = true
	)

Create a CSVWriter that writes to a file.

    : options(options_p),
      writer_options(options.dialect_options.state_machine_options.delimiter.GetValue(),
                     options.dialect_options.state_machine_options.quote.GetValue(), options.write_newline),
      file_writer(make_uniq<BufferedFileWriter>(fs, file_path,
                                                FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW |
                                                    FileLockType::WRITE_LOCK | compression)),
      write_stream(*file_writer), should_initialize(true), shared(shared) {
    if (!shared) {
        global_write_state = make_uniq<CSVWriterState>();
    }
}

Member Function Documentation

Initialize()

void duckdb::CSVWriter::Initialize

(

bool

force = false

)

Writes header and prefix if necessary.

                                     {
    if (!force && !should_initialize) {
        return;
    }
 
    if (!options.prefix.empty()) {
        WriteRawString(options.prefix);
    }
 
    if (!(options.dialect_options.header.IsSetByUser() && !options.dialect_options.header.GetValue())) {
        WriteHeader();
    }
 
    should_initialize = false;
}

Here is the call graph for this function:

WriteRawString() [1/2]

void duckdb::CSVWriter::WriteRawString

(

const string &

data

)

Writes the raw string directly into the output stream.

                                                       {
    if (shared) {
        lock_guard<mutex> flock(lock);
        bytes_written += raw_string.size();
        write_stream.WriteData(const_data_ptr_cast(raw_string.c_str()), raw_string.size());
    } else {
        bytes_written += raw_string.size();
        write_stream.WriteData(const_data_ptr_cast(raw_string.c_str()), raw_string.size());
    }
}

Here is the caller graph for this function:

WriteHeader() [1/2]

void duckdb::CSVWriter::WriteHeader

(

)

Writes the header directly into the output stream.

                            {
    CSVWriterState state;
    WriteHeader(*state.stream, options, writer_options);
    state.written_anything = true;
    Flush(state);
}

Here is the call graph for this function:

Here is the caller graph for this function:

WriteRawString() [2/2]

void duckdb::CSVWriter::WriteRawString	(	const string &	prefix,
		CSVWriterState &	local_state
	)

Write the Raw String, using the local_state.

                                                                                {
    local_state.stream->WriteData(const_data_ptr_cast(prefix.c_str()), prefix.size());
 
    if (!local_state.require_manual_flush && local_state.stream->GetPosition() >= writer_options.flush_size) {
        Flush(local_state);
    }
}

Here is the call graph for this function:

WriteChunk() [1/3]

void duckdb::CSVWriter::WriteChunk	(	DataChunk &	input,
		CSVWriterState &	local_state
	)

Write a chunk of VARCHAR vectors to the CSV file (any casts are the responsibility of caller)

                                                                        {
    WriteChunk(input, *local_state.stream, options, local_state.written_anything, writer_options);
 
    if (!local_state.require_manual_flush && local_state.stream->GetPosition() >= local_state.flush_size) {
        Flush(local_state);
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

WriteChunk() [2/3]

void duckdb::CSVWriter::WriteChunk

(

DataChunk &

input

)

(Non-shared only) variant of WriteChunk

                                           {
    // Method intended for non-shared use only
    D_ASSERT(!shared);
 
    WriteChunk(input, *global_write_state);
}

Here is the call graph for this function:

Flush() [1/2]

void duckdb::CSVWriter::Flush

(

CSVWriterState &

local_state

)

Flushes all data in the local write state.

                                                 {
    if (shared) {
        lock_guard<mutex> flock(lock);
        FlushInternal(local_state);
    } else {
        FlushInternal(local_state);
    }
}

Flush() [2/2]

void duckdb::CSVWriter::Flush

(

)

(Non-shared only) variant of Flush

                      {
    // Method intended for non-shared use only
    D_ASSERT(!shared);
    FlushInternal(*global_write_state);
}

Here is the caller graph for this function:

Reset()

void duckdb::CSVWriter::Reset

(

optional_ptr< CSVWriterState >

local_state

)

Resets the state of the writer. Warning: the file_writer is not reset.

                                                              {
    if (shared) {
        lock_guard<mutex> flock(lock);
        ResetInternal(local_state);
    } else {
        ResetInternal(local_state);
    }
}

Close()

void duckdb::CSVWriter::Close

(

)

Closes the writer, optionally writes a postfix.

                      {
    if (shared) {
        lock_guard<mutex> flock(lock);
        if (file_writer) {
            file_writer->Close();
        }
    } else {
        if (file_writer) {
            file_writer->Close();
        }
    }
}

BytesWritten()

idx_t duckdb::CSVWriter::BytesWritten

(

)

                              {
    if (shared) {
        lock_guard<mutex> flock(lock);
        return bytes_written;
    }
    return bytes_written;
}

FileSize()

idx_t duckdb::CSVWriter::FileSize

(

)

BytesWritten + OriginalSize;.

                          {
    if (shared) {
        lock_guard<mutex> flock(lock);
        return GetFileSize(file_writer, bytes_written);
    }
    return GetFileSize(file_writer, bytes_written);
}

WrittenAnything()

bool duckdb::CSVWriter::WrittenAnything ( )

inline

                           {
        return written_anything;
    }

SetWrittenAnything()

void duckdb::CSVWriter::SetWrittenAnything ( bool  val )

inline

                                      {
        if (shared) {
            lock_guard<mutex> guard(lock);
            written_anything = val;
        } else {
            written_anything = val;
        }
    }

FlushInternal()

void duckdb::CSVWriter::FlushInternal ( CSVWriterState &  local_state )

protected

                                                         {
    if (!local_state.written_anything) {
        return;
    }
 
    if (!written_anything) {
        written_anything = true;
    } else if (writer_options.newline_writing_mode == CSVNewLineMode::WRITE_BEFORE) {
        write_stream.WriteData(const_data_ptr_cast(writer_options.newline.c_str()), writer_options.newline.size());
    }
 
    written_anything = true;
    bytes_written += local_state.stream->GetPosition();
    write_stream.WriteData(local_state.stream->GetData(), local_state.stream->GetPosition());
 
    local_state.Reset();
}

ResetInternal()

void duckdb::CSVWriter::ResetInternal ( optional_ptr< CSVWriterState >  local_state )

protected

                                                                      {
    if (local_state) {
        local_state->Reset();
    }
 
    written_anything = false;
    bytes_written = 0;
}

WriteQuoteOrEscape()

void duckdb::CSVWriter::WriteQuoteOrEscape ( WriteStream &  writer, char  quote_or_escape  )

staticprotected

                                                                            {
    if (quote_or_escape != '\0') {
        writer.Write(quote_or_escape);
    }
}

AddEscapes()

string duckdb::CSVWriter::AddEscapes ( char  to_be_escaped, char  escape, const string &  val  )

staticprotected

                                                                               {
    if (escape == '\0') {
        return val;
    }
    idx_t i = 0;
    string new_val = "";
    idx_t found = val.find(to_be_escaped);
 
    while (found != string::npos) {
        while (i < found) {
            new_val += val[i];
            i++;
        }
        new_val += escape;
        found = val.find(to_be_escaped, found + 1);
    }
    while (i < val.length()) {
        new_val += val[i];
        i++;
    }
    return new_val;
}

RequiresQuotes()

bool duckdb::CSVWriter::RequiresQuotes ( const char *  str, idx_t  len, const string &  null_str, const vector< bool > &  requires_quotes  )

staticprotected

                                                                    {
    // check if the string is equal to the null string
    if (len == null_str.size() && memcmp(str, null_str.c_str(), len) == 0) {
        return true;
    }
    auto str_data = const_data_ptr_cast(str);
    for (idx_t i = 0; i < len; i++) {
        if (requires_quotes[str_data[i]]) {
            // this byte requires quotes - write a quoted string
            return true;
        }
    }
    // no newline, quote or delimiter in the string
    // no quoting or escaping necessary
    return false;
}

WriteQuotedString() [1/2]

void duckdb::CSVWriter::WriteQuotedString ( WriteStream &  writer, const char *  str, idx_t  len, bool  force_quote, const string &  null_str, const vector< bool > &  requires_quotes, char  quote, char  escape  )

staticprotected

                                               {
    if (!force_quote) {
        // force quote is disabled: check if we need to add quotes anyway
        force_quote = RequiresQuotes(str, len, null_str, requires_quotes);
    }
    // If a quote is set to none (i.e., null-terminator) we skip the quotation
    if (force_quote && quote != '\0') {
        // quoting is enabled: we might need to escape things in the string
        bool requires_escape = false;
        // simple CSV
        // do a single loop to check for a quote or escape value
        for (idx_t i = 0; i < len; i++) {
            if (str[i] == quote || str[i] == escape) {
                requires_escape = true;
                break;
            }
        }
 
        if (!requires_escape) {
            // fast path: no need to escape anything
            WriteQuoteOrEscape(writer, quote);
            writer.WriteData(const_data_ptr_cast(str), len);
            WriteQuoteOrEscape(writer, quote);
            return;
        }
 
        // slow path: need to add escapes
        string new_val(str, len);
        new_val = AddEscapes(escape, escape, new_val);
        if (escape != quote) {
            // need to escape quotes separately
            new_val = AddEscapes(quote, escape, new_val);
        }
        WriteQuoteOrEscape(writer, quote);
        writer.WriteData(const_data_ptr_cast(new_val.c_str()), new_val.size());
        WriteQuoteOrEscape(writer, quote);
    } else {
        writer.WriteData(const_data_ptr_cast(str), len);
    }
}

WriteQuotedString() [2/2]

void duckdb::CSVWriter::WriteQuotedString ( WriteStream &  writer, const char *  str, idx_t  len, idx_t  col_idx, CSVReaderOptions &  options, CSVWriterOptions &  writer_options  )

staticprotected

                                                                                               {
    WriteQuotedString(writer, str, len, options.force_quote[col_idx], options.null_str[0],
                      writer_options.requires_quotes, options.dialect_options.state_machine_options.quote.GetValue(),
                      options.dialect_options.state_machine_options.escape.GetValue());
}

WriteChunk() [3/3]

void duckdb::CSVWriter::WriteChunk ( DataChunk &  input, MemoryStream &  writer, CSVReaderOptions &  options, bool &  written_anything, CSVWriterOptions &  writer_options  )

staticprotected

                                                             {
    // now loop over the vectors and output the values
    for (idx_t row_idx = 0; row_idx < input.size(); row_idx++) {
        if (row_idx == 0 && !written_anything) {
            written_anything = true;
        } else if (writer_options.newline_writing_mode == CSVNewLineMode::WRITE_BEFORE) {
            writer.WriteData(const_data_ptr_cast(writer_options.newline.c_str()), writer_options.newline.size());
        }
        // write values
        D_ASSERT(options.null_str.size() == 1);
        for (idx_t col_idx = 0; col_idx < input.ColumnCount(); col_idx++) {
            if (col_idx != 0) {
                CSVWriter::WriteQuoteOrEscape(writer,
                                              options.dialect_options.state_machine_options.delimiter.GetValue()[0]);
            }
            if (FlatVector::IsNull(input.data[col_idx], row_idx)) {
                // write null value
                writer.WriteData(const_data_ptr_cast(options.null_str[0].c_str()), options.null_str[0].size());
                continue;
            }
 
            // non-null value, fetch the string value from the cast chunk
            auto str_data = FlatVector::GetData<string_t>(input.data[col_idx]);
            // FIXME: we could gain some performance here by checking for certain types if they ever require quotes
            // (e.g. integers only require quotes if the delimiter is a number, decimals only require quotes if the
            // delimiter is a number or "." character)
 
            WriteQuotedString(writer, str_data[row_idx].GetData(), str_data[row_idx].GetSize(), col_idx, options,
                              writer_options);
        }
        if (writer_options.newline_writing_mode == CSVNewLineMode::WRITE_AFTER) {
            writer.WriteData(const_data_ptr_cast(writer_options.newline.c_str()), writer_options.newline.size());
        }
    }
}

WriteHeader() [2/2]

void duckdb::CSVWriter::WriteHeader ( MemoryStream &  stream, CSVReaderOptions &  options, CSVWriterOptions &  writer_options  )

staticprotected

                                                                                                             {
    for (idx_t i = 0; i < options.name_list.size(); i++) {
        if (i != 0) {
            WriteQuoteOrEscape(stream, options.dialect_options.state_machine_options.delimiter.GetValue()[0]);
        }
 
        WriteQuotedString(stream, options.name_list[i].c_str(), options.name_list[i].size(), i, options,
                          writer_options);
    }
}

---

The documentation for this class was generated from the following file:

• external/duckdb/duckdb.cpp