Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[enhance](mtmv)MTMV support history partition #46569

Draft
wants to merge 36 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
1
  • Loading branch information
zddr committed Jan 7, 2025
commit 03a207ad5c3188086ad942fb96f58aaebc4ca2c3
Original file line number Diff line number Diff line change
Expand Up @@ -478,18 +478,19 @@ private MTMVTaskRefreshMode generateRefreshMode(List<String> needRefreshPartitio

public List<String> calculateNeedRefreshPartitions(MTMVRefreshContext context)
throws AnalysisException {

// check whether the user manually triggers it
if (taskContext.getTriggerMode() == MTMVTaskTriggerMode.MANUAL) {
if (taskContext.isComplete()) {
return Lists.newArrayList(mtmv.getPartitionNames());
return getHasRelatedPartitions(context);
} else if (!CollectionUtils
.isEmpty(taskContext.getPartitions())) {
return taskContext.getPartitions();
}
}
// if refreshMethod is COMPLETE, we must FULL refresh, avoid external table MTMV always not refresh
if (mtmv.getRefreshInfo().getRefreshMethod() == RefreshMethod.COMPLETE) {
return Lists.newArrayList(mtmv.getPartitionNames());
return getHasRelatedPartitions(context);
}
// check if data is fresh
// We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation()
Expand All @@ -508,6 +509,17 @@ public List<String> calculateNeedRefreshPartitions(MTMVRefreshContext context)
return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(context, relation.getBaseTablesOneLevel());
}

private List<String> getHasRelatedPartitions(MTMVRefreshContext context) {
List<String> res = Lists.newArrayList();
Map<String, Set<String>> partitionMappings = context.getPartitionMappings();
for (Entry<String, Set<String>> entry : partitionMappings.entrySet()) {
if (CollectionUtils.isNotEmpty(entry.getValue())) {
res.add(entry.getKey());
}
}
return res;
}

public MTMVTaskContext getTaskContext() {
return taskContext;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,9 +109,8 @@ public static boolean isMTMVPartitionSync(MTMVRefreshContext refreshContext, Str
// if follow base table, not need compare with related table, only should compare with related partition
excludedTriggerTables.add(relatedTable.getName());
if (CollectionUtils.isEmpty(relatedPartitionNames)) {
LOG.warn("can not found related partition, partitionId: {}, mtmvName: {}, relatedTableName: {}",
partitionName, mtmv.getName(), relatedTable.getName());
return false;
// history partition not have relatedPartitionNames, we think it is sync
return true;
}
isSyncWithPartition = isSyncWithPartitions(refreshContext, partitionName, relatedPartitionNames);
}
Expand Down
Loading