Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create a balanced union tree, also for export #13956

Merged
merged 1 commit into from
Sep 16, 2024
Merged

Create a balanced union tree, also for export #13956

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Create a balanced union tree, also for export
  • Loading branch information
@Mytherin
Mytherin committed Sep 16, 2024
commit c01dcbdc21779b3b891f893963bde1dce630fdd6
2 changes: 2 additions & 0 deletions src/include/duckdb/planner/binder.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,8 @@ class Binder : public enable_shared_from_this<Binder> {
unique_ptr<BoundTableRef> BindShowTable(ShowRef &ref);
unique_ptr<BoundTableRef> BindSummarize(ShowRef &ref);

unique_ptr<LogicalOperator> UnionOperators(vector<unique_ptr<LogicalOperator>> nodes);

private:
Binder(ClientContext &context, shared_ptr<Binder> parent, BinderType binder_type);
};
Expand Down
16 changes: 1 addition & 15 deletions src/planner/binder/statement/bind_copy_database.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,21 +96,7 @@ unique_ptr<LogicalOperator> Binder::BindCopyDatabaseData(Catalog &source_catalog
result->children.push_back(make_uniq<LogicalDummyScan>(GenerateTableIndex()));
} else {
// use UNION ALL to combine the individual copy statements into a single node
while (insert_nodes.size() > 1) {
vector<unique_ptr<LogicalOperator>> new_nodes;
for (idx_t i = 0; i < insert_nodes.size(); i += 2) {
if (i + 1 == insert_nodes.size()) {
new_nodes.push_back(std::move(insert_nodes[i]));
} else {
auto copy_union = make_uniq<LogicalSetOperation>(
GenerateTableIndex(), 1U, std::move(insert_nodes[i]), std::move(insert_nodes[i + 1]),
LogicalOperatorType::LOGICAL_UNION, true, false);
new_nodes.push_back(std::move(copy_union));
}
}
insert_nodes = std::move(new_nodes);
}
result = std::move(insert_nodes[0]);
result = UnionOperators(std::move(insert_nodes));
}
return result;
}
Expand Down
33 changes: 24 additions & 9 deletions src/planner/binder/statement/bind_export.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,27 @@ static unique_ptr<QueryNode> CreateSelectStatement(CopyStatement &stmt, child_li
return std::move(statement);
}

unique_ptr<LogicalOperator> Binder::UnionOperators(vector<unique_ptr<LogicalOperator>> nodes) {
if (nodes.empty()) {
return nullptr;
}
while (nodes.size() > 1) {
vector<unique_ptr<LogicalOperator>> new_nodes;
for (idx_t i = 0; i < nodes.size(); i += 2) {
if (i + 1 == nodes.size()) {
new_nodes.push_back(std::move(nodes[i]));
} else {
auto copy_union = make_uniq<LogicalSetOperation>(GenerateTableIndex(), 1U, std::move(nodes[i]),
std::move(nodes[i + 1]),
LogicalOperatorType::LOGICAL_UNION, true, false);
new_nodes.push_back(std::move(copy_union));
}
}
nodes = std::move(new_nodes);
}
return std::move(nodes[0]);
}

BoundStatement Binder::Bind(ExportStatement &stmt) {
// COPY TO a file
auto &config = DBConfig::GetConfig(context);
Expand Down Expand Up @@ -170,11 +191,11 @@ BoundStatement Binder::Bind(ExportStatement &stmt) {

// now generate the COPY statements for each of the tables
auto &fs = FileSystem::GetFileSystem(context);
unique_ptr<LogicalOperator> child_operator;

BoundExportData exported_tables;

unordered_set<string> table_name_index;
vector<unique_ptr<LogicalOperator>> export_nodes;
for (auto &t : tables) {
auto &table = t.get().Cast<TableCatalogEntry>();
auto info = make_uniq<CopyInfo>();
Expand Down Expand Up @@ -237,15 +258,9 @@ BoundStatement Binder::Bind(ExportStatement &stmt) {

auto plan = std::move(bound_statement.plan);

if (child_operator) {
// use UNION ALL to combine the individual copy statements into a single node
auto copy_union = make_uniq<LogicalSetOperation>(GenerateTableIndex(), 1U, std::move(child_operator),
std::move(plan), LogicalOperatorType::LOGICAL_UNION, true);
child_operator = std::move(copy_union);
} else {
child_operator = std::move(plan);
}
export_nodes.push_back(std::move(plan));
}
auto child_operator = UnionOperators(std::move(export_nodes));

// try to create the directory, if it doesn't exist yet
// a bit hacky to do it here, but we need to create the directory BEFORE the copy statements run
Expand Down
Loading