59871 {
59872 if (lower >= upper) {
59873 return;
59874 }
59875
59876 D_ASSERT(upper <= tree[0].first.size());
59877
59878 using IdxRange = std::pair<idx_t, idx_t>;
59879
59880
59881 IdxRange run_idx(lower, upper - 1);
59882 idx_t level_width = 1;
59883 idx_t level = 0;
59884 IdxRange prev_run_idx;
59885 IdxRange curr;
59886 if (run_idx.first == run_idx.second) {
59887 curr.first = curr.second = run_idx.first;
59888 } else {
59889 do {
59890 prev_run_idx.second = run_idx.second;
59891 run_idx.first /= FANOUT;
59892 run_idx.second /= FANOUT;
59893 level_width *= FANOUT;
59894 ++level;
59895 } while (run_idx.first != run_idx.second);
59896 curr.second = prev_run_idx.second * level_width / FANOUT;
59897 curr.first = curr.second;
59898 }
59899
59900
59901 if (level > LowestCascadingLevel()) {
59902 IdxRange cascading_idx;
59903
59904 {
59905 IdxRange entry;
59906 entry.first = run_idx.first * level_width;
59907 entry.second = std::min(entry.first + level_width, static_cast<idx_t>(tree[0].first.size()));
59908 auto *level_data = tree[level].first.data();
59909 auto entry_idx = NumericCast<idx_t>(
59910 std::lower_bound(level_data + entry.first, level_data + entry.second, needle) - level_data);
59911 cascading_idx.first = cascading_idx.second =
59912 (entry_idx / CASCADING + 2 * (entry.first / level_width)) * FANOUT;
59913
59914
59915
59916 auto correction = (prev_run_idx.second - run_idx.second * FANOUT);
59917 cascading_idx.first -= (FANOUT - correction);
59918 cascading_idx.second += correction;
59919 }
59920
59921
59922
59923
59924 do {
59925 --level;
59926 level_width /= FANOUT;
59927 auto *level_data = tree[level].first.data();
59928 auto &cascading_idcs = tree[level + 1].second;
59929
59930
59931 cascading_idx.first += FANOUT - 1;
59932 while (curr.first - lower >= level_width) {
59933
59934 const auto *search_begin = level_data + cascading_idcs[cascading_idx.first];
59935 const auto *search_end = level_data + cascading_idcs[cascading_idx.first + FANOUT];
59936 const auto run_pos = std::lower_bound(search_begin, search_end, needle, cmp.cmp) - level_data;
59937
59938 const auto run_begin = curr.first - level_width;
59939 aggregate(level, run_begin, NumericCast<idx_t>(run_pos));
59940
59941 curr.first -= level_width;
59942 --cascading_idx.first;
59943 }
59944
59945 if (curr.first != lower) {
59946 const auto *search_begin = level_data + cascading_idcs[cascading_idx.first];
59947 const auto *search_end = level_data + cascading_idcs[cascading_idx.first + FANOUT];
59948 auto idx = NumericCast<idx_t>(std::lower_bound(search_begin, search_end, needle, cmp.cmp) - level_data);
59949 cascading_idx.first = (idx / CASCADING + 2 * (lower / level_width)) * FANOUT;
59950 }
59951
59952
59953
59954 while (upper - curr.second >= level_width) {
59955
59956 const auto *search_begin = level_data + cascading_idcs[cascading_idx.second];
59957 const auto *search_end = level_data + cascading_idcs[cascading_idx.second + FANOUT];
59958 const auto run_pos = std::lower_bound(search_begin, search_end, needle, cmp.cmp) - level_data;
59959
59960 const auto run_begin = curr.second;
59961 aggregate(level, run_begin, NumericCast<idx_t>(run_pos));
59962
59963 curr.second += level_width;
59964 ++cascading_idx.second;
59965 }
59966
59967 if (curr.second != upper) {
59968 const auto *search_begin = level_data + cascading_idcs[cascading_idx.second];
59969 const auto *search_end = level_data + cascading_idcs[cascading_idx.second + FANOUT];
59970 auto idx = NumericCast<idx_t>(std::lower_bound(search_begin, search_end, needle, cmp.cmp) - level_data);
59971 cascading_idx.second = (idx / CASCADING + 2 * (upper / level_width)) * FANOUT;
59972 }
59973 } while (level >= LowestCascadingLevel());
59974 }
59975
59976
59977 if (level) {
59978 while (--level) {
59979 level_width /= FANOUT;
59980 auto *level_data = tree[level].first.data();
59981
59982 while (curr.first - lower >= level_width) {
59983 const auto *search_end = level_data + curr.first;
59984 const auto *search_begin = search_end - level_width;
59985 const auto run_pos =
59986 NumericCast<idx_t>(std::lower_bound(search_begin, search_end, needle, cmp.cmp) - level_data);
59987 const auto run_begin = NumericCast<idx_t>(search_begin - level_data);
59988 aggregate(level, run_begin, run_pos);
59989 curr.first -= level_width;
59990 }
59991
59992 while (upper - curr.second >= level_width) {
59993 const auto *search_begin = level_data + curr.second;
59994 const auto *search_end = search_begin + level_width;
59995 const auto run_pos =
59996 NumericCast<idx_t>(std::lower_bound(search_begin, search_end, needle, cmp.cmp) - level_data);
59997 const auto run_begin = NumericCast<idx_t>(search_begin - level_data);
59998 aggregate(level, run_begin, run_pos);
59999 curr.second += level_width;
60000 }
60001 }
60002 }
60003
60004
60005 {
60006 auto *level_data = tree[0].first.data();
60007
60008 auto lower_it = lower;
60009 while (lower_it != curr.first) {
60010 const auto *search_begin = level_data + lower_it;
60011 const auto run_begin = lower_it;
60012 const auto run_pos = run_begin + cmp.cmp(*search_begin, needle);
60013 aggregate(level, run_begin, run_pos);
60014 ++lower_it;
60015 }
60016
60017 while (curr.second != upper) {
60018 const auto *search_begin = level_data + curr.second;
60019 const auto run_begin = curr.second;
60020 const auto run_pos = run_begin + cmp.cmp(*search_begin, needle);
60021 aggregate(level, run_begin, run_pos);
60022 ++curr.second;
60023 }
60024 }
60025}