Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add upgrade() to Admin API #1204

Merged
merged 5 commits into from
Oct 23, 2023
Merged

Add upgrade() to Admin API #1204

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
59c9033
Add upgrade() to Admin API
Torch3333 Oct 18, 2023
0c30f4d
Fix typo in comment
Torch3333 Oct 19, 2023
05c36f8
Fix typos
Torch3333 Oct 19, 2023
c370cd6
Apply spotless
Torch3333 Oct 19, 2023
5879099
Add missing `@Override` annotation on `getAdminUtils()` methods
Torch3333 Oct 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .../integration-test/java/com/scalar/db/storage/cassandra/CassandraAdminIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.scalar.db.storage.cassandra;

import com.scalar.db.api.DistributedStorageAdminIntegrationTestBase;
import com.scalar.db.util.AdminTestUtils;
import java.util.Collections;
import java.util.Map;
import java.util.Properties;
Expand All @@ -15,4 +16,9 @@ protected Properties getProperties(String testName) {
protected Map<String, String> getCreationOptions() {
return Collections.singletonMap(CassandraAdmin.REPLICATION_FACTOR, "1");
}

@Override
protected AdminTestUtils getAdminTestUtils(String testName) {
return new CassandraAdminTestUtils(getProperties(testName));
}
}
5 changes: 5 additions & 0 deletions ...ava/com/scalar/db/storage/cassandra/ConsensusCommitAdminIntegrationTestWithCassandra.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.scalar.db.storage.cassandra;

import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
import com.scalar.db.util.AdminTestUtils;
import java.util.Collections;
import java.util.Map;
import java.util.Properties;
Expand All @@ -16,4 +17,8 @@ protected Properties getProps(String testName) {
protected Map<String, String> getCreationOptions() {
return Collections.singletonMap(CassandraAdmin.REPLICATION_FACTOR, "1");
}

protected AdminTestUtils getAdminTestUtils(String testName) {
return new CassandraAdminTestUtils(getProperties(testName));
}
}
5 changes: 5 additions & 0 deletions ...test/java/com/scalar/db/storage/cosmos/ConsensusCommitAdminIntegrationTestWithCosmos.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.scalar.db.storage.cosmos;

import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
import com.scalar.db.util.AdminTestUtils;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
Expand Down Expand Up @@ -37,4 +38,8 @@ private String getNamespace(String namespace) {
protected Map<String, String> getCreationOptions() {
return CosmosEnv.getCreationOptions();
}

protected AdminTestUtils getAdminTestUtils(String testName) {
return new CosmosAdminTestUtils(getProperties(testName));
}
}
5 changes: 5 additions & 0 deletions core/src/integration-test/java/com/scalar/db/storage/cosmos/CosmosAdminIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.scalar.db.storage.cosmos;

import com.scalar.db.api.DistributedStorageAdminIntegrationTestBase;
import com.scalar.db.util.AdminTestUtils;
import java.util.Map;
import java.util.Optional;
import java.util.Properties;
Expand Down Expand Up @@ -36,4 +37,8 @@ private String getNamespace(String namespace) {
protected Map<String, String> getCreationOptions() {
return CosmosEnv.getCreationOptions();
}

protected AdminTestUtils getAdminTestUtils(String testName) {
return new CosmosAdminTestUtils(getProperties(testName));
}
}
5 changes: 5 additions & 0 deletions ...test/java/com/scalar/db/storage/dynamo/ConsensusCommitAdminIntegrationTestWithDynamo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.scalar.db.storage.dynamo;

import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
import com.scalar.db.util.AdminTestUtils;
import java.util.Map;
import java.util.Properties;

Expand All @@ -21,4 +22,8 @@ protected Map<String, String> getCreationOptions() {
protected boolean isIndexOnBooleanColumnSupported() {
return false;
}

protected AdminTestUtils getAdminTestUtils(String testName) {
return new DynamoAdminTestUtils(getProperties(testName));
}
}
5 changes: 5 additions & 0 deletions core/src/integration-test/java/com/scalar/db/storage/dynamo/DynamoAdminIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.scalar.db.storage.dynamo;

import com.scalar.db.api.DistributedStorageAdminIntegrationTestBase;
import com.scalar.db.util.AdminTestUtils;
import java.util.Map;
import java.util.Properties;

Expand All @@ -20,4 +21,8 @@ protected Map<String, String> getCreationOptions() {
protected boolean isIndexOnBooleanColumnSupported() {
return false;
}

protected AdminTestUtils getAdminTestUtils(String testName) {
return new DynamoAdminTestUtils(getProperties(testName));
}
}
5 changes: 5 additions & 0 deletions .../java/com/scalar/db/storage/jdbc/ConsensusCommitAdminIntegrationTestWithJdbcDatabase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.scalar.db.storage.jdbc;

import com.scalar.db.transaction.consensuscommit.ConsensusCommitAdminIntegrationTestBase;
import com.scalar.db.util.AdminTestUtils;
import java.util.Properties;

public class ConsensusCommitAdminIntegrationTestWithJdbcDatabase
Expand All @@ -10,4 +11,8 @@ public class ConsensusCommitAdminIntegrationTestWithJdbcDatabase
protected Properties getProps(String testName) {
return JdbcEnv.getProperties(testName);
}

protected AdminTestUtils getAdminTestUtils(String testName) {
return new JdbcAdminTestUtils(getProperties(testName));
}
}
5 changes: 5 additions & 0 deletions core/src/integration-test/java/com/scalar/db/storage/jdbc/JdbcAdminIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.scalar.db.storage.jdbc;

import com.scalar.db.api.DistributedStorageAdminIntegrationTestBase;
import com.scalar.db.util.AdminTestUtils;
import java.util.Properties;

public class JdbcAdminIntegrationTest extends DistributedStorageAdminIntegrationTestBase {
Expand All @@ -9,4 +10,8 @@ public class JdbcAdminIntegrationTest extends DistributedStorageAdminIntegration
protected Properties getProperties(String testName) {
return JdbcEnv.getProperties(testName);
}

protected AdminTestUtils getAdminTestUtils(String testName) {
return new JdbcAdminTestUtils(getProperties(testName));
}
}
12 changes: 12 additions & 0 deletions ...gration-test/java/com/scalar/db/transaction/jdbc/JdbcTransactionAdminIntegrationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.scalar.db.config.DatabaseConfig;
import com.scalar.db.exception.storage.ExecutionException;
import com.scalar.db.storage.jdbc.JdbcEnv;
import com.scalar.db.util.AdminTestUtils;
import java.util.Properties;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -71,4 +72,15 @@ public void dropCoordinatorTables_IfExist_CoordinatorTablesDoNotExist_ShouldNotT
throws ExecutionException {
super.dropCoordinatorTables_IfExist_CoordinatorTablesDoNotExist_ShouldNotThrowAnyException();
}

@Test
@Disabled("JDBC Transaction Admin does not support upgrade()")
@Override
public void
upgrade_WhenMetadataTableExistsButNotNamespacesTable_ShouldCreateNamespacesTableAndImportExistingNamespaces() {}

@Override
protected AdminTestUtils getAdminTestUtils(String testName) {
throw new UnsupportedOperationException();
}
}
10 changes: 10 additions & 0 deletions core/src/main/java/com/scalar/db/api/Admin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -431,4 +431,14 @@ void addNewColumnToTable(String namespace, String table, String columnName, Data
* @throws ExecutionException if the operation fails
*/
Set<String> getNamespaceNames() throws ExecutionException;

/**
* Upgrades the ScalarDB environment to support the latest version of the ScalarDB API. Typically,
* you will be requested, as indicated on the release notes, to run this method after updating the
* ScalarDB version of your application environment.
*
* @param options options to upgrade
* @throws ExecutionException if the operation fails
*/
void upgrade(Map<String, String> options) throws ExecutionException;
}
9 changes: 9 additions & 0 deletions core/src/main/java/com/scalar/db/common/CheckedDistributedStorageAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,15 @@ public void addRawColumnToTable(
}
}

@Override
public void upgrade(Map<String, String> options) throws ExecutionException {
try {
admin.upgrade(options);
} catch (ExecutionException e) {
throw new ExecutionException("Upgrading the ScalarDB environment failed", e);
}
}

@Override
public void close() {
admin.close();
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/com/scalar/db/service/AdminService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,11 @@ public void repairNamespace(String namespace, Map<String, String> options)
admin.repairNamespace(namespace, options);
}

@Override
public void upgrade(Map<String, String> options) throws ExecutionException {
admin.upgrade(options);
}

@Override
public void close() {
admin.close();
Expand Down
20 changes: 20 additions & 0 deletions core/src/main/java/com/scalar/db/storage/cassandra/CassandraAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,26 @@ public Set<String> getNamespaceNames() throws ExecutionException {
}
}

@Override
public void upgrade(Map<String, String> options) throws ExecutionException {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The upgrade() implementation of the CassandraAdmin

try {
createKeyspace(metadataKeyspace, options, true);
createNamespacesTableIfNotExists();
// Retrieve user keyspace and filter out system ones. A downside is that this may include
// keyspace not created by ScalarDB.
Set<String> userKeyspaces =
clusterManager.getSession().getCluster().getMetadata().getKeyspaces().stream()
.map(KeyspaceMetadata::getName)
.filter(name -> !name.startsWith("system") && !name.equals(metadataKeyspace))
.collect(Collectors.toSet());
for (String userKeyspace : userKeyspaces) {
upsertIntoNamespacesTable(userKeyspace);
}
} catch (RuntimeException e) {
throw new ExecutionException("Upgrading the ScalarDB environment failed", e);
}
}

private void createNamespacesTableIfNotExists() {
String createTableQuery =
SchemaBuilder.createTable(
Expand Down
27 changes: 27 additions & 0 deletions core/src/main/java/com/scalar/db/storage/cosmos/CosmosAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -647,6 +647,33 @@ public Set<String> getNamespaceNames() throws ExecutionException {
}
}

@Override
public void upgrade(Map<String, String> options) throws ExecutionException {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The upgrade() implementation of the CosmosAdmin

try {
if (!tableMetadataContainerExists()) {
return;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just quick silly questions.

  • It's impossible to create the table metadata container if not exists?
  • If not possible, don't need to throw an exception?

Copy link
Contributor Author

@Torch3333 Torch3333 Oct 19, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There may be a misunderstanding here.
This check verifies if the table metadata container does not exist, if true this means there is no table installed, so there is no namespace to import to the namespaces metadata table.
A limitation is that if there was a namespace that did not contain a ScalarDB-managed table, we wouldn't import it to the namespace table. It's because we assume such namespace are the storage system namespaces or are user-one but not used for ScalarDB.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this answer your question ?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you! Understood 👍

}
createMetadataDatabaseAndNamespaceContainerIfNotExists();

// Upsert namespace of existing tables in the "namespaces" container
getTableMetadataContainer()
.queryItems(
"SELECT container.id FROM container",
new CosmosQueryRequestOptions(),
CosmosTableMetadata.class)
.stream()
.map(
tableMetadata ->
tableMetadata.getId().substring(0, tableMetadata.getId().indexOf('.')))
.distinct()
.forEach(
namespaceName ->
getNamespacesContainer().upsertItem(new CosmosNamespace(namespaceName)));
} catch (RuntimeException e) {
throw new ExecutionException("Upgrading the ScalarDB environemnt failed", e);
}
}

private void createMetadataDatabaseAndNamespaceContainerIfNotExists() {
ThroughputProperties manualThroughput =
ThroughputProperties.createManualThroughput(Integer.parseInt(DEFAULT_REQUEST_UNIT));
Expand Down
44 changes: 44 additions & 0 deletions core/src/main/java/com/scalar/db/storage/dynamo/DynamoAdmin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1363,6 +1363,50 @@ public Set<String> getNamespaceNames() throws ExecutionException {
}
}

@Override
public void upgrade(Map<String, String> options) throws ExecutionException {
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The upgrade() implementation of the DynamoAdmin

if (!metadataTableExists()) {
return;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similar question to the above one

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My answer is the same as #1204 (comment)

}
boolean noBackup = Boolean.parseBoolean(options.getOrDefault(NO_BACKUP, DEFAULT_NO_BACKUP));
createNamespacesTableIfNotExists(noBackup);
try {
for (Namespace namespace : getNamespacesOfExistingTables()) {
upsertIntoNamespacesTable(namespace);
}
} catch (ExecutionException e) {
throw new ExecutionException("Upgrading the ScalarDB environment failed", e);
}
}

private Set<Namespace> getNamespacesOfExistingTables() throws ExecutionException {
Set<Namespace> namespaceNames = new HashSet<>();
Map<String, AttributeValue> lastEvaluatedKey = null;
do {
ScanResponse scanResponse;
try {
scanResponse =
client.scan(
ScanRequest.builder()
.tableName(ScalarDbUtils.getFullTableName(metadataNamespace, METADATA_TABLE))
.exclusiveStartKey(lastEvaluatedKey)
.build());
} catch (RuntimeException e) {
throw new ExecutionException(
"Failed to retrieve the namespaces names of existing tables", e);
}
lastEvaluatedKey = scanResponse.lastEvaluatedKey();

for (Map<String, AttributeValue> tableMetadata : scanResponse.items()) {
String fullTableName = tableMetadata.get(METADATA_ATTR_TABLE).s();
String namespaceName = fullTableName.substring(0, fullTableName.indexOf('.'));
namespaceNames.add(Namespace.of(namespaceName));
}
} while (!lastEvaluatedKey.isEmpty());

return namespaceNames;
}

private void createNamespacesTableIfNotExists(boolean noBackup) throws ExecutionException {
try {
if (!namespacesTableExists()) {
Expand Down
19 changes: 19 additions & 0 deletions core/src/main/java/com/scalar/db/storage/dynamo/Namespace.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package com.scalar.db.storage.dynamo;

import java.util.Objects;

public class Namespace {
private final String prefix;
private final String name;
Expand Down Expand Up @@ -35,4 +37,21 @@ public String nonPrefixed() {
public String toString() {
return prefixed();
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof Namespace)) {
return false;
}
Namespace namespace = (Namespace) o;
return Objects.equals(prefix, namespace.prefix) && Objects.equals(name, namespace.name);
}

@Override
public int hashCode() {
return Objects.hash(prefix, name);
}
}
Loading