Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
@smiklosovic
smiklosovic committed Jun 13, 2023
1 parent ff9b8b6 commit bafbe93
Show file tree
Hide file tree
Showing 24 changed files with 52 additions and 532 deletions.
30 changes: 5 additions & 25 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<url>https://github.com/instaclustr/instaclustr-esop</url>

<properties>
<instaclustr.commons.version>1.5.0</instaclustr.commons.version>
<instaclustr.commons.version>2.0.0</instaclustr.commons.version>
<azure-storage.version>8.6.6</azure-storage.version>
<google-cloud-libraries.version>26.0.0</google-cloud-libraries.version>
<aws-java-sdk.version>2.20.45</aws-java-sdk.version>
Expand All @@ -33,7 +33,7 @@
<maven.gpg.plugin.version>1.6</maven.gpg.plugin.version>
<maven.jar.plugin.version>3.1.1</maven.jar.plugin.version>
<maven.javadoc.plugin.version>3.1.1</maven.javadoc.plugin.version>
<maven.compiler.plugin.version>3.8.1</maven.compiler.plugin.version>
<maven.compiler.plugin.version>3.11.0</maven.compiler.plugin.version>
<maven.surefire.plugin.version>2.22.2</maven.surefire.plugin.version>
<git.command.plugin.version>4.9.10</git.command.plugin.version>
<nexus.staging.maven.plugin.version>1.6.8</nexus.staging.maven.plugin.version>
Expand Down Expand Up @@ -354,8 +354,7 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven.compiler.plugin.version}</version>
<configuration>
<source>${java.source.version}</source>
<target>${java.target.version}</target>
<release>8</release>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
Expand Down Expand Up @@ -493,7 +492,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven.surefire.plugin.version}</version>
<configuration>
<excludedGroups>cloudTest,k8sTest,azureTest,googleTest,s3Test,cephTest</excludedGroups>
<excludedGroups>cloudTest,azureTest,googleTest,s3Test,cephTest</excludedGroups>
</configuration>
</plugin>
</plugins>
Expand Down Expand Up @@ -537,26 +536,7 @@
</plugins>
</build>
</profile>

<profile>
<id>k8sTests</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven.surefire.plugin.version}</version>
<configuration>
<groups>k8sTest</groups>
</configuration>
</plugin>
</plugins>
</build>
</profile>


<profile>
<id>azureTests</id>
<activation>
Expand Down
100 changes: 23 additions & 77 deletions src/main/java/com/instaclustr/esop/azure/AzureModule.java
View file
Original file line number Diff line number Diff line change
@@ -1,33 +1,27 @@
package com.instaclustr.esop.azure;

import java.net.URISyntaxException;
import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.inject.AbstractModule;
import com.google.inject.Provider;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.instaclustr.esop.impl.AbstractOperationRequest;
import com.instaclustr.kubernetes.KubernetesHelper;
import com.instaclustr.kubernetes.SecretReader;
import com.microsoft.azure.storage.CloudStorageAccount;
import com.microsoft.azure.storage.StorageCredentialsAccountAndKey;
import io.kubernetes.client.apis.CoreV1Api;

import static com.google.common.base.Strings.isNullOrEmpty;
import static com.instaclustr.esop.guice.BackupRestoreBindings.installBindings;
import static com.instaclustr.kubernetes.KubernetesHelper.isRunningAsClient;
import static java.lang.String.format;

public class AzureModule extends AbstractModule {
public class AzureModule extends AbstractModule
{

private static final Logger logger = LoggerFactory.getLogger(AzureModule.class);

@Override
protected void configure() {
protected void configure()
{
installBindings(binder(),
"azure",
AzureRestorer.class,
Expand All @@ -37,88 +31,40 @@ protected void configure() {

@Provides
@Singleton
CloudStorageAccountFactory provideCloudStorageAccountFactory(final Provider<CoreV1Api> coreV1ApiProvider) {
return new CloudStorageAccountFactory(coreV1ApiProvider);
CloudStorageAccountFactory provideCloudStorageAccountFactory()
{
return new CloudStorageAccountFactory();
}

public static class CloudStorageAccountFactory {
public static class CloudStorageAccountFactory
{

private final Provider<CoreV1Api> coreV1ApiProvider;

public CloudStorageAccountFactory(final Provider<CoreV1Api> coreV1ApiProvider) {
this.coreV1ApiProvider = coreV1ApiProvider;
}

public CloudStorageAccount build(final AbstractOperationRequest operationRequest) throws AzureModuleException, URISyntaxException {
return new CloudStorageAccount(provideStorageCredentialsAccountAndKey(coreV1ApiProvider, operationRequest), !operationRequest.insecure);
public CloudStorageAccount build(final AbstractOperationRequest operationRequest) throws AzureModuleException, URISyntaxException
{
return new CloudStorageAccount(provideStorageCredentialsAccountAndKey(), !operationRequest.insecure);
}

public boolean isRunningInKubernetes() {
return KubernetesHelper.isRunningInKubernetes() || isRunningAsClient();
private StorageCredentialsAccountAndKey provideStorageCredentialsAccountAndKey() throws AzureModuleException
{
return resolveCredentialsFromEnvProperties();
}

private StorageCredentialsAccountAndKey provideStorageCredentialsAccountAndKey(final Provider<CoreV1Api> coreV1ApiProvider,
final AbstractOperationRequest operationRequest) throws AzureModuleException {
if (isRunningInKubernetes()) {
if (isNullOrEmpty(operationRequest.resolveKubernetesSecretName())) {
logger.warn("Kubernetes secret name for resolving Azure credentials was not specified, going to resolve them from env. properties.");
return resolveCredentialsFromEnvProperties();
}
return resolveCredentialsFromK8S(coreV1ApiProvider, operationRequest);
} else {
return resolveCredentialsFromEnvProperties();
}
}

private StorageCredentialsAccountAndKey resolveCredentialsFromEnvProperties() {
private StorageCredentialsAccountAndKey resolveCredentialsFromEnvProperties()
{
return new StorageCredentialsAccountAndKey(System.getenv("AZURE_STORAGE_ACCOUNT"), System.getenv("AZURE_STORAGE_KEY"));
}

private StorageCredentialsAccountAndKey resolveCredentialsFromK8S(final Provider<CoreV1Api> coreV1ApiProvider,
final AbstractOperationRequest operationrequest) {

final String secretName = operationrequest.resolveKubernetesSecretName();

try {
final String namespace = operationrequest.resolveKubernetesNamespace();
final SecretReader secretReader = new SecretReader(coreV1ApiProvider);

return secretReader.readIntoObject(namespace,
secretName,
secret -> {
final Map<String, byte[]> data = secret.getData();

final byte[] azureStorageAccount = data.get("azurestorageaccount");
final byte[] azureStorageKey = data.get("azurestoragekey");

if (azureStorageAccount == null) {
throw new AzureModuleException(format("Secret %s does not contain any entry with key 'azurestorageaccount'",
secret.getMetadata().getName()));
}

if (azureStorageKey == null) {
throw new AzureModuleException(format("Secret %s does not contain any entry with key 'azurestoragekey'",
secret.getMetadata().getName()));
}

return new StorageCredentialsAccountAndKey(
new String(azureStorageAccount),
new String(azureStorageKey)
);
});
} catch (final Exception ex) {
throw new AzureModuleException("Unable to resolve Azure credentials for backup / restores from Kubernetes secret " + secretName, ex);
}
}
}

public static final class AzureModuleException extends RuntimeException {
public static final class AzureModuleException extends RuntimeException
{

public AzureModuleException(final String message, final Throwable cause) {
public AzureModuleException(final String message, final Throwable cause)
{
super(message, cause);
}

public AzureModuleException(final String message) {
public AzureModuleException(final String message)
{
super(message);
}
}
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/instaclustr/esop/gcp/GCPBackuper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,14 @@ public class GCPBackuper extends Backuper {
public GCPBackuper(final GoogleStorageFactory storageFactory,
@Assisted final BackupOperationRequest backupOperationRequest) {
super(backupOperationRequest);
this.storage = storageFactory.build(backupOperationRequest);
this.storage = storageFactory.build();
}

@AssistedInject
public GCPBackuper(final GoogleStorageFactory storageFactory,
@Assisted final BackupCommitLogsOperationRequest backupOperationRequest) {
super(backupOperationRequest);
this.storage = storageFactory.build(backupOperationRequest);
this.storage = storageFactory.build();
}

@Override
Expand Down
10 changes: 5 additions & 5 deletions src/main/java/com/instaclustr/esop/gcp/GCPBucketService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,31 +29,31 @@ public class GCPBucketService extends BucketService {
@AssistedInject
public GCPBucketService(final GoogleStorageFactory storageFactory,
@Assisted final BackupOperationRequest request) {
this.storage = storageFactory.build(request);
this.storage = storageFactory.build();
}

@AssistedInject
public GCPBucketService(final GoogleStorageFactory storageFactory,
@Assisted final BackupCommitLogsOperationRequest request) {
this.storage = storageFactory.build(request);
this.storage = storageFactory.build();
}

@AssistedInject
public GCPBucketService(final GoogleStorageFactory storageFactory,
@Assisted final RestoreOperationRequest request) {
this.storage = storageFactory.build(request);
this.storage = storageFactory.build();
}

@AssistedInject
public GCPBucketService(final GoogleStorageFactory storageFactory,
@Assisted final RestoreCommitLogsOperationRequest request) {
this.storage = storageFactory.build(request);
this.storage = storageFactory.build();
}

@AssistedInject
public GCPBucketService(final GoogleStorageFactory storageFactory,
@Assisted final ListOperationRequest request) {
this.storage = storageFactory.build(request);
this.storage = storageFactory.build();
}

@Override
Expand Down
65 changes: 6 additions & 59 deletions src/main/java/com/instaclustr/esop/gcp/GCPModule.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,9 @@
package com.instaclustr.esop.gcp;

import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Optional;

import com.google.common.collect.Lists;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -16,15 +12,9 @@
import com.google.cloud.storage.Storage;
import com.google.cloud.storage.StorageOptions;
import com.google.inject.AbstractModule;
import com.google.inject.Provider;
import com.google.inject.Provides;
import com.google.inject.Singleton;
import com.instaclustr.esop.impl.AbstractOperationRequest;
import com.instaclustr.kubernetes.KubernetesHelper;
import com.instaclustr.kubernetes.SecretReader;
import io.kubernetes.client.apis.CoreV1Api;

import static com.google.common.base.Strings.isNullOrEmpty;
import static com.instaclustr.esop.guice.BackupRestoreBindings.installBindings;
import static java.lang.String.format;

Expand All @@ -41,36 +31,16 @@ protected void configure() {

@Provides
@Singleton
GoogleStorageFactory provideGoogleStorageFactory(final Provider<CoreV1Api> coreV1ApiProvider) {
return new GoogleStorageFactory(coreV1ApiProvider);
GoogleStorageFactory provideGoogleStorageFactory() {
return new GoogleStorageFactory();
}

public static class GoogleStorageFactory {

private static final Logger logger = LoggerFactory.getLogger(GoogleStorageFactory.class);

private final Provider<CoreV1Api> coreV1ApiProvider;

public GoogleStorageFactory(final Provider<CoreV1Api> coreV1ApiProvider) {
this.coreV1ApiProvider = coreV1ApiProvider;
}

public Storage build(final AbstractOperationRequest operationRequest) {
if (KubernetesHelper.isRunningInKubernetes() || KubernetesHelper.isRunningAsClient()) {
if (isNullOrEmpty(operationRequest.resolveKubernetesSecretName())) {
logger.warn("Kubernetes secret name for resolving GCP credentials was not specified, going to resolve them from file.");
return resolveStorageFromEnvProperties();
} else {
return resolveStorageFromKubernetesSecret(operationRequest);
}
} else {
return resolveStorageFromEnvProperties();
}
}

private Storage resolveStorageFromKubernetesSecret(final AbstractOperationRequest operationRequest) {
final GoogleCredentials credentials = resolveGoogleCredentials(operationRequest);
return StorageOptions.newBuilder().setCredentials(credentials).build().getService();
public Storage build() {
return resolveStorageFromEnvProperties();
}

private Storage resolveStorageFromEnvProperties() {
Expand All @@ -97,32 +67,9 @@ private Storage resolveStorageFromEnvProperties() {
private GoogleCredentials resolveGoogleCredentialsFromFile(String googleAppCredentialsPath) {
try (InputStream is = new FileInputStream(googleAppCredentialsPath)) {
return GoogleCredentials.fromStream(is);
} catch (Exception ex) {
throw new RuntimeException("Unable to read credentials from " + googleAppCredentialsPath);
}
}

private GoogleCredentials resolveGoogleCredentials(final AbstractOperationRequest operationRequest) {
final String secretName = operationRequest.resolveKubernetesSecretName();
final String dataKey = "gcp";
final String namespace = operationRequest.resolveKubernetesNamespace();

try {
Optional<byte[]> gcpCredentials = new SecretReader(coreV1ApiProvider).read(namespace,
secretName,
dataKey);

if (!gcpCredentials.isPresent()) {
throw new GCPModuleException(format("GCP credentials from Kubernetes namespace %s from secret %s under key %s were not set.",
namespace,
secretName,
dataKey));
}

return GoogleCredentials.fromStream(new ByteArrayInputStream(gcpCredentials.get()))
.createScoped(Lists.newArrayList("https://www.googleapis.com/auth/cloud-platform"));
} catch (final Exception ex) {
throw new GCPModuleException(format("Unable to resolve data for key %s on secret %s", dataKey, secretName), ex);
catch (Exception ex) {
throw new RuntimeException("Unable to read credentials from " + googleAppCredentialsPath);
}
}
}
Expand Down
Loading

0 comments on commit bafbe93

Please sign in to comment.