Skip to content

Commit

Permalink
SONAR-9028 add PurgeDao.deleteNonRootComponents
Browse files Browse the repository at this point in the history
  • Loading branch information
@sns-seb
sns-seb committed Jun 5, 2017
1 parent 6ce959d commit 326c71a
Show file tree
Hide file tree
Showing 6 changed files with 608 additions and 26 deletions.
68 changes: 61 additions & 7 deletions server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeCommands.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ void deleteAnalyses(PurgeSnapshotQuery... queries) {
}

@VisibleForTesting
protected void deleteAnalyses(List<IdUuidPair> analysisIdUuids) {
void deleteAnalyses(List<IdUuidPair> analysisIdUuids) {
List<List<String>> analysisUuidsPartitions = Lists.partition(IdUuidPairs.uuids(analysisIdUuids), MAX_SNAPSHOTS_PER_QUERY);

deleteAnalysisDuplications(analysisUuidsPartitions);
Expand All @@ -107,7 +107,7 @@ protected void deleteAnalyses(List<IdUuidPair> analysisIdUuids) {
profiler.stop();
}

public void purgeAnalyses(List<IdUuidPair> analysisUuids) {
void purgeAnalyses(List<IdUuidPair> analysisUuids) {
List<List<String>> analysisUuidsPartitions = Lists.partition(IdUuidPairs.uuids(analysisUuids), MAX_SNAPSHOTS_PER_QUERY);

deleteAnalysisDuplications(analysisUuidsPartitions);
Expand Down Expand Up @@ -156,6 +156,24 @@ void deleteIssues(String rootUuid) {
profiler.stop();
}

void deleteIssues(List<String> componentUuids) {
if (componentUuids.isEmpty()) {
return;
}

List<List<String>> uuidPartitions = Lists.partition(componentUuids, MAX_RESOURCES_PER_QUERY);

profiler.start("deleteIssues (issue_changes)");
uuidPartitions.forEach(purgeMapper::deleteIssueChangesByComponentUuids);
session.commit();
profiler.stop();

profiler.start("deleteIssues (issues)");
uuidPartitions.forEach(purgeMapper::deleteIssuesByComponentUuids);
session.commit();
profiler.stop();
}

void deleteLinks(String rootUuid) {
profiler.start("deleteLinks (project_links)");
purgeMapper.deleteProjectLinksByComponentUuid(rootUuid);
Expand All @@ -164,6 +182,9 @@ void deleteLinks(String rootUuid) {
}

void deleteByRootAndModulesOrSubviews(List<IdUuidPair> rootAndModulesOrSubviewsIds) {
if (rootAndModulesOrSubviewsIds.isEmpty()) {
return;
}
List<List<Long>> idPartitions = Lists.partition(IdUuidPairs.ids(rootAndModulesOrSubviewsIds), MAX_RESOURCES_PER_QUERY);
List<List<String>> uuidsPartitions = Lists.partition(IdUuidPairs.uuids(rootAndModulesOrSubviewsIds), MAX_RESOURCES_PER_QUERY);

Expand All @@ -185,7 +206,29 @@ void deleteComponents(String rootUuid) {
profiler.stop();
}

public void deleteComponentMeasures(List<String> analysisUuids, List<String> componentUuids) {
void deleteComponents(List<String> componentUuids) {
if (componentUuids.isEmpty()) {
return;
}

profiler.start("deleteComponents (projects)");
Lists.partition(componentUuids, MAX_RESOURCES_PER_QUERY).forEach(purgeMapper::deleteComponentsByUuids);
session.commit();
profiler.stop();
}

void deleteComponentMeasures(List<String> componentUuids) {
if (componentUuids.isEmpty()) {
return;
}

profiler.start("deleteComponentMeasures (project_measures)");
Lists.partition(componentUuids, MAX_RESOURCES_PER_QUERY).forEach(purgeMapper::fullDeleteComponentMeasures);
session.commit();
profiler.stop();
}

void deleteComponentMeasures(List<String> analysisUuids, List<String> componentUuids) {
if (analysisUuids.isEmpty() || componentUuids.isEmpty()) {
return;
}
Expand All @@ -203,28 +246,39 @@ public void deleteComponentMeasures(List<String> analysisUuids, List<String> com
profiler.stop();
}

public void deleteFileSources(String rootUuid) {
void deleteFileSources(List<String> componentUuids) {
if (componentUuids.isEmpty()) {
return;
}

profiler.start("deleteFileSources (file_sources)");
Lists.partition(componentUuids, MAX_RESOURCES_PER_QUERY).forEach(purgeMapper::deleteFileSourcesByFileUuid);
session.commit();
profiler.stop();
}

void deleteFileSources(String rootUuid) {
profiler.start("deleteFileSources (file_sources)");
purgeMapper.deleteFileSourcesByProjectUuid(rootUuid);
session.commit();
profiler.stop();
}

public void deleteCeActivity(String rootUuid) {
void deleteCeActivity(String rootUuid) {
profiler.start("deleteCeActivity (ce_activity)");
purgeMapper.deleteCeActivityByProjectUuid(rootUuid);
session.commit();
profiler.stop();
}

public void deleteCeQueue(String rootUuid) {
void deleteCeQueue(String rootUuid) {
profiler.start("deleteCeQueue (ce_queue)");
purgeMapper.deleteCeQueueByProjectUuid(rootUuid);
session.commit();
profiler.stop();
}

public void deleteWebhookDeliveries(String rootUuid) {
void deleteWebhookDeliveries(String rootUuid) {
profiler.start("deleteWebhookDeliveries (webhook_deliveries)");
purgeMapper.deleteWebhookDeliveriesByProjectUuid(rootUuid);
session.commit();
Expand Down
64 changes: 61 additions & 3 deletions server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,14 @@
*/
package org.sonar.db.purge;

import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
import org.sonar.api.utils.System2;
import org.sonar.api.utils.log.Logger;
import org.sonar.api.utils.log.Loggers;
Expand All @@ -45,6 +48,12 @@
public class PurgeDao implements Dao {
private static final Logger LOG = Loggers.get(PurgeDao.class);
private static final String[] UNPROCESSED_STATUS = new String[] {"U"};
private static final ImmutableSet<String> QUALIFIERS_PROJECT_VIEW = ImmutableSet.of("TRK", "VW");
private static final ImmutableSet<String> QUALIFIERS_MODULE_SUBVIEW = ImmutableSet.of("BRC", "SVW");
private static final String SCOPE_PROJECT = "PRJ";
private static final String SCOPE_FILE = "FIL";
private static final String QUALIFIER_FILE = "FIL";
private static final String QUALIFIER_UNIT_TEST = "UTS";

private final ComponentDao componentDao;
private final System2 system2;
Expand Down Expand Up @@ -145,14 +154,14 @@ public List<PurgeableAnalysisDto> selectPurgeableAnalyses(String componentUuid,
return result;
}

public PurgeDao deleteProject(DbSession session, String uuid) {
public PurgeDao deleteRootComponent(DbSession session, String uuid) {
PurgeProfiler profiler = new PurgeProfiler();
PurgeCommands purgeCommands = new PurgeCommands(session, profiler);
deleteProject(uuid, mapper(session), purgeCommands);
deleteRootComponent(uuid, mapper(session), purgeCommands);
return this;
}

private static void deleteProject(String rootUuid, PurgeMapper mapper, PurgeCommands commands) {
private static void deleteRootComponent(String rootUuid, PurgeMapper mapper, PurgeCommands commands) {
List<IdUuidPair> rootAndModulesOrSubviews = mapper.selectRootAndModulesOrSubviewsByProjectUuid(rootUuid);
long rootId = rootAndModulesOrSubviews.stream()
.filter(pair -> pair.getUuid().equals(rootUuid))
Expand All @@ -171,6 +180,55 @@ private static void deleteProject(String rootUuid, PurgeMapper mapper, PurgeComm
commands.deleteWebhookDeliveries(rootUuid);
}

/**
* Delete the non root components (ie. neither project nor view) from the specified collection of {@link ComponentDto}
* and data from their child tables.
* <p>
* This method has no effect when passed an empty collection or only root components.
* </p>
*/
public PurgeDao deleteNonRootComponents(DbSession dbSession, Collection<ComponentDto> components) {
Set<ComponentDto> nonRootComponents = components.stream().filter(PurgeDao::isNotRoot).collect(MoreCollectors.toSet());
if (nonRootComponents.isEmpty()) {
return this;
}

PurgeProfiler profiler = new PurgeProfiler();
PurgeCommands purgeCommands = new PurgeCommands(dbSession, profiler);
deleteNonRootComponents(nonRootComponents, purgeCommands);

return this;
}

private static void deleteNonRootComponents(Set<ComponentDto> nonRootComponents, PurgeCommands purgeCommands) {
List<IdUuidPair> modulesOrSubviews = nonRootComponents.stream()
.filter(PurgeDao::isModuleOrSubview)
.map(PurgeDao::toIdUuidPair)
.collect(MoreCollectors.toList());
purgeCommands.deleteByRootAndModulesOrSubviews(modulesOrSubviews);
List<String> nonRootComponentUuids = nonRootComponents.stream().map(ComponentDto::uuid).collect(MoreCollectors.toList(nonRootComponents.size()));
purgeCommands.deleteComponentMeasures(nonRootComponentUuids);
purgeCommands.deleteFileSources(nonRootComponents.stream().filter(PurgeDao::isFile).map(ComponentDto::uuid).collect(MoreCollectors.toList()));
purgeCommands.deleteIssues(nonRootComponentUuids);
purgeCommands.deleteComponents(nonRootComponentUuids);
}

private static boolean isNotRoot(ComponentDto dto) {
return !(SCOPE_PROJECT.equals(dto.scope()) && QUALIFIERS_PROJECT_VIEW.contains(dto.qualifier()));
}

private static boolean isModuleOrSubview(ComponentDto dto) {
return SCOPE_PROJECT.equals(dto.scope()) && QUALIFIERS_MODULE_SUBVIEW.contains(dto.qualifier());
}

private static boolean isFile(ComponentDto dto) {
return SCOPE_FILE.equals(dto.qualifier()) && ImmutableSet.of(QUALIFIER_FILE, QUALIFIER_UNIT_TEST).contains(dto.qualifier());
}

private static IdUuidPair toIdUuidPair(ComponentDto dto) {
return new IdUuidPair(dto.getId(), dto.uuid());
}

public void deleteAnalyses(DbSession session, PurgeProfiler profiler, List<IdUuidPair> analysisIdUuids) {
new PurgeCommands(session, profiler).deleteAnalyses(analysisIdUuids);
}
Expand Down
8 changes: 8 additions & 0 deletions server/sonar-db-dao/src/main/java/org/sonar/db/purge/PurgeMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ public interface PurgeMapper {

void deleteAnalysisMeasures(@Param("analysisUuids") List<String> analysisUuids);

void fullDeleteComponentMeasures(@Param("componentUuids") List<String> componentUuids);

void deleteComponentMeasures(@Param("analysisUuids") List<String> analysisUuids, @Param("componentUuids") List<String> componentUuids);

List<Long> selectMetricIdsWithoutHistoricalData();
Expand All @@ -56,6 +58,8 @@ public interface PurgeMapper {

void deleteComponentsByProjectUuid(@Param("rootUuid") String rootUuid);

void deleteComponentsByUuids(@Param("componentUuids") List<String> componentUuids);

void deleteGroupRolesByComponentId(@Param("rootId") long rootId);

void deleteUserRolesByComponentId(@Param("rootId") long rootId);
Expand All @@ -72,6 +76,10 @@ public interface PurgeMapper {

void deleteIssuesByProjectUuid(@Param("projectUuid") String projectUuid);

void deleteIssueChangesByComponentUuids(@Param("componentUuids") List<String> componentUuids);

void deleteIssuesByComponentUuids(@Param("componentUuids") List<String> componentUuids);

List<String> selectOldClosedIssueKeys(@Param("projectUuid") String projectUuid, @Nullable @Param("toDate") Long toDate);

void deleteIssuesFromKeys(@Param("keys") List<String> keys);
Expand Down
66 changes: 66 additions & 0 deletions server/sonar-db-dao/src/main/resources/org/sonar/db/purge/PurgeMapper.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,15 @@
</foreach>
</delete>

<delete id="fullDeleteComponentMeasures" parameterType="map">
delete from project_measures
where
component_uuid in
<foreach collection="componentUuids" open="(" close=")" item="componentUuid" separator=",">
#{componentUuid,jdbcType=VARCHAR}
</foreach>
</delete>

<delete id="deleteComponentMeasures" parameterType="map">
delete from project_measures
where
Expand Down Expand Up @@ -184,6 +193,15 @@
project_uuid = #{rootUuid,jdbcType=VARCHAR}
</delete>

<delete id="deleteComponentsByUuids" parameterType="map">
delete from projects
where
uuid in
<foreach collection="componentUuids" open="(" close=")" item="componentUuid" separator=",">
#{componentUuid,jdbcType=VARCHAR}
</foreach>
</delete>

<delete id="deleteGroupRolesByComponentId" parameterType="map">
delete from group_roles
where
Expand Down Expand Up @@ -237,6 +255,54 @@
where project_uuid = #{projectUuid,jdbcType=VARCHAR}
</delete>

<delete id="deleteIssueChangesByComponentUuids" parameterType="map">
delete from issue_changes ic
where
exists (
select
1
from issues i
where
i.kee=ic.issue_key
and i.component_uuid in
<foreach collection="componentUuids" open="(" close=")" item="componentUuid" separator=",">
#{componentUuid,jdbcType=VARCHAR}
</foreach>
)
</delete>

<!-- Mssql -->
<delete id="deleteIssueChangesByComponentUuids" databaseId="mssql" parameterType="map">
delete issue_changes from issue_changes
inner join issues on
issue_changes.issue_key=issues.kee
where
issues.project_uuid in
<foreach collection="componentUuids" open="(" close=")" item="componentUuid" separator=",">
#{componentUuid,jdbcType=VARCHAR}
</foreach>
</delete>

<!-- Mysql -->
<delete id="deleteIssueChangesByComponentUuids" databaseId="mysql" parameterType="map">
delete ic from issue_changes as ic, issues as i
where
ic.issue_key=i.kee
and i.component_uuid in
<foreach collection="componentUuids" open="(" close=")" item="componentUuid" separator=",">
#{componentUuid,jdbcType=VARCHAR}
</foreach>
</delete>

<delete id="deleteIssuesByComponentUuids" parameterType="map">
delete from issues
where
project_uuid in
<foreach collection="componentUuids" open="(" close=")" item="componentUuid" separator=",">
#{componentUuid,jdbcType=VARCHAR}
</foreach>
</delete>

<delete id="deleteFileSourcesByProjectUuid">
delete from file_sources where project_uuid=#{rootProjectUuid,jdbcType=VARCHAR}
</delete>
Expand Down
Loading

0 comments on commit 326c71a

Please sign in to comment.