Skip to content

Commit

Permalink
style: Format source code using mvn fmt:format
Browse files Browse the repository at this point in the history
  • Loading branch information
@fdlk
fdlk committed Sep 3, 2018
1 parent da5f8cf commit 46e5cf2
Show file tree
Hide file tree
Showing 1,752 changed files with 157,618 additions and 165,241 deletions.
118 changes: 62 additions & 56 deletions molgenis-amazon-bucket/src/main/java/org/molgenis/amazon/bucket/AmazonBucketConfig.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.molgenis.amazon.bucket;

import static java.util.Objects.requireNonNull;

import org.molgenis.amazon.bucket.meta.AmazonBucketJobExecution;
import org.molgenis.amazon.bucket.meta.AmazonBucketJobExecutionMetaData;
import org.molgenis.jobs.Job;
Expand All @@ -11,65 +13,69 @@
import org.springframework.context.annotation.Import;
import org.springframework.context.annotation.Lazy;

import static java.util.Objects.requireNonNull;

@Configuration
@Import(AmazonBucketIngester.class)
public class AmazonBucketConfig
{
private final AmazonBucketIngester ingester;
private final ScheduledJobTypeFactory scheduledJobTypeFactory;
private final AmazonBucketJobExecutionMetaData amazonBucketJobExecutionMetaData;
public class AmazonBucketConfig {
private final AmazonBucketIngester ingester;
private final ScheduledJobTypeFactory scheduledJobTypeFactory;
private final AmazonBucketJobExecutionMetaData amazonBucketJobExecutionMetaData;

public AmazonBucketConfig(AmazonBucketIngester ingester, ScheduledJobTypeFactory scheduledJobTypeFactory,
AmazonBucketJobExecutionMetaData amazonBucketJobExecutionMetaData)
{
this.ingester = requireNonNull(ingester);
this.scheduledJobTypeFactory = requireNonNull(scheduledJobTypeFactory);
this.amazonBucketJobExecutionMetaData = requireNonNull(amazonBucketJobExecutionMetaData);
}
public AmazonBucketConfig(
AmazonBucketIngester ingester,
ScheduledJobTypeFactory scheduledJobTypeFactory,
AmazonBucketJobExecutionMetaData amazonBucketJobExecutionMetaData) {
this.ingester = requireNonNull(ingester);
this.scheduledJobTypeFactory = requireNonNull(scheduledJobTypeFactory);
this.amazonBucketJobExecutionMetaData = requireNonNull(amazonBucketJobExecutionMetaData);
}

@Bean
public JobFactory<AmazonBucketJobExecution> amazonBucketJobFactory()
{
return new JobFactory<AmazonBucketJobExecution>()
{
@Override
public Job createJob(AmazonBucketJobExecution amazonBucketJobExecution)
{
final String targetEntityId = amazonBucketJobExecution.getTargetEntityId();
final String jobExecutionID = amazonBucketJobExecution.getIdentifier();
final String bucket = amazonBucketJobExecution.getBucket();
final String key = amazonBucketJobExecution.getKey();
final String extension = amazonBucketJobExecution.getExtension();
final String accessKey = amazonBucketJobExecution.getAccessKey();
final String secretKey = amazonBucketJobExecution.getSecretKey();
final String region = amazonBucketJobExecution.getRegion();
final boolean isExpression = amazonBucketJobExecution.isExpression();
return progress -> ingester.ingest(jobExecutionID, targetEntityId, bucket, key, extension, accessKey,
secretKey, region, isExpression, progress);
}
};
}
@Bean
public JobFactory<AmazonBucketJobExecution> amazonBucketJobFactory() {
return new JobFactory<AmazonBucketJobExecution>() {
@Override
public Job createJob(AmazonBucketJobExecution amazonBucketJobExecution) {
final String targetEntityId = amazonBucketJobExecution.getTargetEntityId();
final String jobExecutionID = amazonBucketJobExecution.getIdentifier();
final String bucket = amazonBucketJobExecution.getBucket();
final String key = amazonBucketJobExecution.getKey();
final String extension = amazonBucketJobExecution.getExtension();
final String accessKey = amazonBucketJobExecution.getAccessKey();
final String secretKey = amazonBucketJobExecution.getSecretKey();
final String region = amazonBucketJobExecution.getRegion();
final boolean isExpression = amazonBucketJobExecution.isExpression();
return progress ->
ingester.ingest(
jobExecutionID,
targetEntityId,
bucket,
key,
extension,
accessKey,
secretKey,
region,
isExpression,
progress);
}
};
}

@Bean
@Lazy
public ScheduledJobType bucketIngest()
{
ScheduledJobType result = scheduledJobTypeFactory.create("BucketIngest");
result.setLabel("Bucket ingest");
result.setDescription("This job downloads a file from a URL and imports it into MOLGENIS.");
result.setSchema(
"{'title': 'Bucket Ingest Job','type': 'object','properties': { 'bucket': {'type': 'string', 'description': 'The name of the bucket.'},"
+ "'key': {'type': 'string', 'description': 'Expression to match the file key'}, 'accessKey': { 'type': 'string', "
+ "'description': 'the access key to be used to login to the amazon bucket'},'secretKey': {'type': 'string', "
+ "'description': 'the secretkey to be used to login to the amazon bucket'},'expression': {'type': 'boolean', "
+ "'description': 'Is the key an expression or an exact match'},'extension': {'type': 'string', "
+ "'description': 'Optional extension of the file, is not part of the key in the bucket'},'region': {'type': 'string', "
+ "'description': 'The region where the amazon bucket is located'},'targetEntityId': {'type': 'string', "
+ "'description': 'Target EntityType ID'}},"
+ "'required': ['bucket','key','accessKey','secretKey','expression','region']}");
result.setJobExecutionType(amazonBucketJobExecutionMetaData);
return result;
}
@Bean
@Lazy
public ScheduledJobType bucketIngest() {
ScheduledJobType result = scheduledJobTypeFactory.create("BucketIngest");
result.setLabel("Bucket ingest");
result.setDescription("This job downloads a file from a URL and imports it into MOLGENIS.");
result.setSchema(
"{'title': 'Bucket Ingest Job','type': 'object','properties': { 'bucket': {'type': 'string', 'description': 'The name of the bucket.'},"
+ "'key': {'type': 'string', 'description': 'Expression to match the file key'}, 'accessKey': { 'type': 'string', "
+ "'description': 'the access key to be used to login to the amazon bucket'},'secretKey': {'type': 'string', "
+ "'description': 'the secretkey to be used to login to the amazon bucket'},'expression': {'type': 'boolean', "
+ "'description': 'Is the key an expression or an exact match'},'extension': {'type': 'string', "
+ "'description': 'Optional extension of the file, is not part of the key in the bucket'},'region': {'type': 'string', "
+ "'description': 'The region where the amazon bucket is located'},'targetEntityId': {'type': 'string', "
+ "'description': 'Target EntityType ID'}},"
+ "'required': ['bucket','key','accessKey','secretKey','expression','region']}");
result.setJobExecutionType(amazonBucketJobExecutionMetaData);
return result;
}
}
171 changes: 92 additions & 79 deletions molgenis-amazon-bucket/src/main/java/org/molgenis/amazon/bucket/AmazonBucketIngester.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,11 @@
package org.molgenis.amazon.bucket;

import static java.util.Objects.requireNonNull;

import com.amazonaws.services.s3.AmazonS3;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import org.molgenis.amazon.bucket.client.AmazonBucketClient;
import org.molgenis.data.DataAction;
import org.molgenis.data.MolgenisDataException;
Expand All @@ -17,86 +22,94 @@
import org.molgenis.jobs.Progress;
import org.springframework.stereotype.Component;

import java.io.File;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;

import static java.util.Objects.requireNonNull;

@Component
public class AmazonBucketIngester
{
private final ImportServiceFactory importServiceFactory;
private final FileRepositoryCollectionFactory fileRepositoryCollectionFactory;
private final FileMetaFactory fileMetaFactory;
private final FileStore fileStore;
private final AmazonBucketClient amazonBucketClient;
public class AmazonBucketIngester {
private final ImportServiceFactory importServiceFactory;
private final FileRepositoryCollectionFactory fileRepositoryCollectionFactory;
private final FileMetaFactory fileMetaFactory;
private final FileStore fileStore;
private final AmazonBucketClient amazonBucketClient;

public AmazonBucketIngester(ImportServiceFactory importServiceFactory,
FileRepositoryCollectionFactory fileRepositoryCollectionFactory, FileMetaFactory fileMetaFactory,
FileStore fileStore, AmazonBucketClient amazonBucketClient)
{
this.importServiceFactory = requireNonNull(importServiceFactory);
this.fileRepositoryCollectionFactory = requireNonNull(fileRepositoryCollectionFactory);
this.fileMetaFactory = requireNonNull(fileMetaFactory);
this.fileStore = requireNonNull(fileStore);
this.amazonBucketClient = requireNonNull(amazonBucketClient);
}
public AmazonBucketIngester(
ImportServiceFactory importServiceFactory,
FileRepositoryCollectionFactory fileRepositoryCollectionFactory,
FileMetaFactory fileMetaFactory,
FileStore fileStore,
AmazonBucketClient amazonBucketClient) {
this.importServiceFactory = requireNonNull(importServiceFactory);
this.fileRepositoryCollectionFactory = requireNonNull(fileRepositoryCollectionFactory);
this.fileMetaFactory = requireNonNull(fileMetaFactory);
this.fileStore = requireNonNull(fileStore);
this.amazonBucketClient = requireNonNull(amazonBucketClient);
}

public FileMeta ingest(String jobExecutionID, String targetEntityTypeName, String bucket, String key,
String extension, String accessKey, String secretKey, String region, boolean isExpression,
Progress progress)
{
FileMeta fileMeta;
try
{
progress.setProgressMax(3);
progress.progress(0, "Connection to Amazon Bucket with accessKey '" + accessKey + "'");
AmazonS3 client = amazonBucketClient.getClient(accessKey, secretKey, region);
progress.progress(1, "downloading...");
File file = amazonBucketClient.downloadFile(client, fileStore, jobExecutionID, bucket, key, extension,
isExpression, targetEntityTypeName);
if (targetEntityTypeName != null && ExcelUtils.isExcelFile(file.getName()))
{
if (ExcelUtils.getNumberOfSheets(file) == 1)
{
ExcelUtils.renameSheet(targetEntityTypeName, file, 0);
}
else
{
throw new MolgenisDataException(
"Amazon Bucket imports to a specified entityType are only possible with CSV files or Excel files with one sheet");
}
}
progress.progress(2, "Importing...");
ImportService importService = importServiceFactory.getImportService(file.getName());
File renamed = new File(
String.format("%s%s%s.%s", file.getParent(), File.separatorChar, targetEntityTypeName, extension));
Files.copy(file.toPath(), renamed.toPath(), StandardCopyOption.REPLACE_EXISTING);
RepositoryCollection repositoryCollection = fileRepositoryCollectionFactory.createFileRepositoryCollection(
renamed);
EntityImportReport report = importService.doImport(repositoryCollection, MetadataAction.UPSERT,
DataAction.ADD_UPDATE_EXISTING,
null);
progress.status("Download and import from Amazon Bucket done.");
progress.progress(3,
"Successfully imported " + report.getNrImportedEntitiesMap().keySet().toString() + " entities.");
fileMeta = createFileMeta(jobExecutionID, file);
}
catch (Exception e)
{
throw new MolgenisDataException(e);
}
return fileMeta;
}
public FileMeta ingest(
String jobExecutionID,
String targetEntityTypeName,
String bucket,
String key,
String extension,
String accessKey,
String secretKey,
String region,
boolean isExpression,
Progress progress) {
FileMeta fileMeta;
try {
progress.setProgressMax(3);
progress.progress(0, "Connection to Amazon Bucket with accessKey '" + accessKey + "'");
AmazonS3 client = amazonBucketClient.getClient(accessKey, secretKey, region);
progress.progress(1, "downloading...");
File file =
amazonBucketClient.downloadFile(
client,
fileStore,
jobExecutionID,
bucket,
key,
extension,
isExpression,
targetEntityTypeName);
if (targetEntityTypeName != null && ExcelUtils.isExcelFile(file.getName())) {
if (ExcelUtils.getNumberOfSheets(file) == 1) {
ExcelUtils.renameSheet(targetEntityTypeName, file, 0);
} else {
throw new MolgenisDataException(
"Amazon Bucket imports to a specified entityType are only possible with CSV files or Excel files with one sheet");
}
}
progress.progress(2, "Importing...");
ImportService importService = importServiceFactory.getImportService(file.getName());
File renamed =
new File(
String.format(
"%s%s%s.%s",
file.getParent(), File.separatorChar, targetEntityTypeName, extension));
Files.copy(file.toPath(), renamed.toPath(), StandardCopyOption.REPLACE_EXISTING);
RepositoryCollection repositoryCollection =
fileRepositoryCollectionFactory.createFileRepositoryCollection(renamed);
EntityImportReport report =
importService.doImport(
repositoryCollection, MetadataAction.UPSERT, DataAction.ADD_UPDATE_EXISTING, null);
progress.status("Download and import from Amazon Bucket done.");
progress.progress(
3,
"Successfully imported "
+ report.getNrImportedEntitiesMap().keySet().toString()
+ " entities.");
fileMeta = createFileMeta(jobExecutionID, file);
} catch (Exception e) {
throw new MolgenisDataException(e);
}
return fileMeta;
}

private FileMeta createFileMeta(String jobExecutionID, File file)
{
FileMeta fileMeta = fileMetaFactory.create(jobExecutionID);
fileMeta.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
fileMeta.setSize(file.length());
fileMeta.setFilename(jobExecutionID + '/' + file.getName());
fileMeta.setUrl("/files/" + jobExecutionID);
return fileMeta;
}
}
private FileMeta createFileMeta(String jobExecutionID, File file) {
FileMeta fileMeta = fileMetaFactory.create(jobExecutionID);
fileMeta.setContentType("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
fileMeta.setSize(file.length());
fileMeta.setFilename(jobExecutionID + '/' + file.getName());
fileMeta.setUrl("/files/" + jobExecutionID);
return fileMeta;
}
}
20 changes: 13 additions & 7 deletions ...nis-amazon-bucket/src/main/java/org/molgenis/amazon/bucket/client/AmazonBucketClient.java
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
package org.molgenis.amazon.bucket.client;

import com.amazonaws.services.s3.AmazonS3;
import org.molgenis.data.file.FileStore;

import java.io.File;
import java.io.IOException;
import org.molgenis.data.file.FileStore;

public interface AmazonBucketClient
{
AmazonS3 getClient(String accessKey, String secretKey, String region);
public interface AmazonBucketClient {
AmazonS3 getClient(String accessKey, String secretKey, String region);

File downloadFile(AmazonS3 s3Client, FileStore fileStore, String jobIdentifier, String bucketName, String keyName,
String extension, boolean isExpression, String targetEntityType) throws IOException;
File downloadFile(
AmazonS3 s3Client,
FileStore fileStore,
String jobIdentifier,
String bucketName,
String keyName,
String extension,
boolean isExpression,
String targetEntityType)
throws IOException;
}
Loading

0 comments on commit 46e5cf2

Please sign in to comment.