duckdb::HivePartitioning Class Reference

Static Public Member Functions
static DUCKDB_API std::map< string, string >	Parse (const string &filename)
	Parse a filename that follows the hive partitioning scheme.
static DUCKDB_API void	ApplyFiltersToFileList (ClientContext &context, vector< OpenFileInfo > &files, vector< unique_ptr< Expression > > &filters, const HivePartitioningFilterInfo &filter_info, MultiFilePushdownInfo &info)
static DUCKDB_API Value	GetValue (ClientContext &context, const string &key, const string &value, const LogicalType &type)
static DUCKDB_API string	Escape (const string &input)
	Escape a hive partition key or value using URL encoding.
static DUCKDB_API string	Unescape (const string &input)
	Unescape a hive partition key or value encoded using URL encoding.
static DUCKDB_API bool	IsNull (const string &input)
	Whether the column is "NULL"/"__HIVE_DEFAULT_PARTITION".

Member Function Documentation

Parse()

std::map< string, string > duckdb::HivePartitioning::Parse ( const string &  filename )

static

Parse a filename that follows the hive partitioning scheme.

                                                                     {
    idx_t partition_start = 0;
    idx_t equality_sign = 0;
    bool candidate_partition = true;
    std::map<string, string> result;
    for (idx_t c = 0; c < filename.size(); c++) {
        if (filename[c] == '?' || filename[c] == '\n') {
            // get parameter or newline - not a partition
            candidate_partition = false;
        }
        if (filename[c] == '\\' || filename[c] == '/') {
            // separator
            if (candidate_partition && equality_sign > partition_start) {
                // we found a partition with an equality sign
                string key = filename.substr(partition_start, equality_sign - partition_start);
                string value = filename.substr(equality_sign + 1, c - equality_sign - 1);
                result.insert(make_pair(std::move(key), std::move(value)));
            }
            partition_start = c + 1;
            candidate_partition = true;
        } else if (filename[c] == '=') {
            if (equality_sign > partition_start) {
                // multiple equality signs - not a partition
                candidate_partition = false;
            }
            equality_sign = c;
        }
    }
    return result;
}

ApplyFiltersToFileList()

void duckdb::HivePartitioning::ApplyFiltersToFileList ( ClientContext &  context, vector< OpenFileInfo > &  files, vector< unique_ptr< Expression > > &  filters, const HivePartitioningFilterInfo &  filter_info, MultiFilePushdownInfo &  info  )

static

Prunes a list of filenames based on a set of filters, can be used by TableFunctions in the pushdown_complex_filter function to skip files with filename-based filters. Also removes the filters that always evaluate to true.

                                                                           {
    vector<OpenFileInfo> pruned_files;
    vector<bool> have_preserved_filter(filters.size(), false);
    vector<unique_ptr<Expression>> pruned_filters;
    unordered_set<idx_t> filters_applied_to_files;
    auto table_index = info.table_index;
 
    if ((!filter_info.filename_enabled && !filter_info.hive_enabled) || filters.empty()) {
        return;
    }
 
    for (idx_t i = 0; i < files.size(); i++) {
        auto &file = files[i];
        bool should_prune_file = false;
        auto known_values = GetKnownColumnValues(file.path, filter_info);
 
        for (idx_t j = 0; j < filters.size(); j++) {
            auto &filter = filters[j];
            unique_ptr<Expression> filter_copy = filter->Copy();
            ConvertKnownColRefToConstants(context, filter_copy, known_values, table_index);
            // Evaluate the filter, if it can be evaluated here, we can not prune this filter
            Value result_value;
 
            if (!filter_copy->IsScalar() || !filter_copy->IsFoldable() ||
                !ExpressionExecutor::TryEvaluateScalar(context, *filter_copy, result_value)) {
                // can not be evaluated only with the filename/hive columns added, we can not prune this filter
                if (!have_preserved_filter[j]) {
                    pruned_filters.emplace_back(filter->Copy());
                    have_preserved_filter[j] = true;
                }
            } else if (result_value.IsNull() || !result_value.GetValue<bool>()) {
                // filter evaluates to false
                should_prune_file = true;
                // convert the filter to a table filter.
                if (filters_applied_to_files.find(j) == filters_applied_to_files.end()) {
                    info.extra_info.file_filters += filter->ToString();
                    filters_applied_to_files.insert(j);
                }
            }
        }
 
        if (!should_prune_file) {
            pruned_files.push_back(file);
        }
    }
 
    D_ASSERT(filters.size() >= pruned_filters.size());
 
    info.extra_info.total_files = files.size();
    info.extra_info.filtered_files = pruned_files.size();
 
    filters = std::move(pruned_filters);
    files = std::move(pruned_files);
}

Here is the call graph for this function:

GetValue()

Value duckdb::HivePartitioning::GetValue ( ClientContext &  context, const string &  key, const string &  value, const LogicalType &  type  )

static

                                                          {
    // Handle nulls
    if (IsNull(str_val)) {
        return Value(type);
    }
    if (type.id() == LogicalTypeId::VARCHAR) {
        // for string values we can directly return the type
        return Value(Unescape(str_val));
    }
    if (str_val.empty()) {
        // empty strings are NULL for non-string types
        return Value(type);
    }
 
    // cast to the target type
    Value value(Unescape(str_val));
    if (!value.TryCastAs(context, type)) {
        throw InvalidInputException("Unable to cast '%s' (from hive partition column '%s') to: '%s'", value.ToString(),
                                    StringUtil::Upper(key), type.ToString());
    }
    return value;
}

Escape()

string duckdb::HivePartitioning::Escape ( const string &  input )

static

Escape a hive partition key or value using URL encoding.

                                                   {
    return StringUtil::URLEncode(input);
}

Here is the call graph for this function:

Unescape()

string duckdb::HivePartitioning::Unescape ( const string &  input )

static

Unescape a hive partition key or value encoded using URL encoding.

                                                     {
    return StringUtil::URLDecode(input);
}

Here is the call graph for this function:

IsNull()

bool duckdb::HivePartitioning::IsNull ( const string &  input )

static

Whether the column is "NULL"/"__HIVE_DEFAULT_PARTITION".

                                                 {
    return StringUtil::CIEquals(input, "NULL") || input == "__HIVE_DEFAULT_PARTITION__";
}

Here is the call graph for this function:

---

The documentation for this class was generated from the following files:

• external/duckdb/duckdb.hpp
• external/duckdb/duckdb.cpp