Add more FileSystemPipe/Sender exists (frankframework#6740)

lenardvandermaas · May 22, 2024 · 0929856 · 0929856
1 parent a094c94
commit 0929856
Show file tree

Hide file tree

Showing 65 changed files with 1,293 additions and 794 deletions.
diff --git a/akamai/src/test/java/org/frankframework/extensions/akamai/NetStorageSenderTest.java b/akamai/src/test/java/org/frankframework/extensions/akamai/NetStorageSenderTest.java
@@ -21,13 +21,13 @@
 
 import java.io.IOException;
 
-import org.frankframework.extensions.akamai.NetStorageSender.Action;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import org.frankframework.core.PipeLineSession;
 import org.frankframework.core.SenderException;
+import org.frankframework.extensions.akamai.NetStorageSender.Action;
 import org.frankframework.http.HttpResponseHandler;
 import org.frankframework.http.HttpSenderTestBase;
 import org.frankframework.parameters.Parameter;
@@ -46,7 +46,7 @@ public void setUp() throws Exception {
 
 	@Override
 	@AfterEach
-	public void tearDown() throws Exception {
+	public void tearDown() {
 		AppConstants.getInstance().remove("http.headers.messageid");
 	}
 

diff --git a/aspose/src/test/java/org/frankframework/extensions/aspose/pipe/PdfPipeTest.java b/aspose/src/test/java/org/frankframework/extensions/aspose/pipe/PdfPipeTest.java
@@ -38,6 +38,7 @@
 
 import javax.imageio.ImageIO;
 import jakarta.annotation.Nonnull;
+
 import org.apache.commons.io.FileUtils;
 import org.frankframework.configuration.ConfigurationException;
 import org.frankframework.core.PipeForward;
@@ -53,10 +54,11 @@
 import org.frankframework.testutil.TestFileUtils;
 import org.frankframework.util.LogUtil;
 import org.frankframework.util.MessageUtils;
+import org.springframework.util.MimeType;
+
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.springframework.util.MimeType;
 
 import com.testautomationguru.utility.CompareMode;
 import com.testautomationguru.utility.ImageUtil;
@@ -91,11 +93,14 @@ public void setUp() throws Exception {
 	}
 
 	@AfterEach
-	public void tearDown() throws Exception {
+	public void tearDown() {
 		synchronized(pdfOutputLocation) {
-			Files.walk(pdfOutputLocation).forEach(PdfPipeTest::removeFile); //Remove each individual file
-
-			Files.deleteIfExists(pdfOutputLocation); //Remove root folder
+			try {
+				Files.walk(pdfOutputLocation).forEach(PdfPipeTest::removeFile); //Remove each individual file
+				Files.deleteIfExists(pdfOutputLocation); //Remove root folder
+			} catch (IOException e) {
+				log.warn("Error deleting temporary file", e);
+			}
 		}
 
 		super.tearDown();

diff --git a/aws/src/main/java/org/frankframework/filesystem/AmazonS3FileSystem.java b/aws/src/main/java/org/frankframework/filesystem/AmazonS3FileSystem.java
@@ -1,5 +1,5 @@
 /*
-   Copyright 2018-2023 WeAreFrank!
+   Copyright 2018-2024 WeAreFrank!
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -29,19 +29,6 @@
 import java.util.List;
 import java.util.Map;
 
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.io.input.NullInputStream;
-import org.apache.commons.lang3.NotImplementedException;
-import org.apache.commons.lang3.StringUtils;
-import org.frankframework.aws.AwsUtil;
-import org.frankframework.configuration.ConfigurationException;
-import org.frankframework.doc.Mandatory;
-import org.frankframework.stream.Message;
-import org.frankframework.util.CredentialFactory;
-import org.frankframework.util.FileUtils;
-import org.frankframework.util.StreamUtil;
-import org.frankframework.util.StringUtil;
-
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
@@ -63,7 +50,20 @@
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.StorageClass;
 
+import jakarta.annotation.Nullable;
 import lombok.Getter;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.io.input.NullInputStream;
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.commons.lang3.StringUtils;
+import org.frankframework.aws.AwsUtil;
+import org.frankframework.configuration.ConfigurationException;
+import org.frankframework.doc.Mandatory;
+import org.frankframework.stream.Message;
+import org.frankframework.util.CredentialFactory;
+import org.frankframework.util.FileUtils;
+import org.frankframework.util.StreamUtil;
+import org.frankframework.util.StringUtil;
 
 
 public class AmazonS3FileSystem extends FileSystemBase<S3Object> implements IWritableFileSystem<S3Object> {
@@ -158,7 +158,7 @@ public void close() throws FileSystemException {
 	 * This method may be used to upload a file to S3.
 	 */
 	@Override
-	public S3Object toFile(String filename) {
+	public S3Object toFile(@Nullable String filename) {
 		S3Object object = new S3Object();
 		int separatorPos = filename.indexOf(BUCKET_OBJECT_SEPARATOR);
 		if (separatorPos<0) {
@@ -172,7 +172,7 @@ public S3Object toFile(String filename) {
 	}
 
 	@Override
-	public S3Object toFile(String folder, String filename) {
+	public S3Object toFile(@Nullable String folder, @Nullable String filename) {
 		return toFile(StringUtil.concatStrings(folder, FILE_DELIMITER, filename));
 	}
 
@@ -273,7 +273,7 @@ public boolean exists(S3Object f) {
 	public void createFile(S3Object f, InputStream content) throws FileSystemException, IOException {
 		String folder = getParentFolder(f);
 		if (folder != null && !folderExists(folder)) { //AWS Supports the creation of folders, this check is purely here so all FileSystems have the same behavior
-			throw new FileSystemException("folder ["+folder+"] does not exist");
+			throw new FolderNotFoundException("folder ["+folder+"] does not exist");
 		}
 
 		final File file = FileUtils.createTempFile(".s3-upload"); //The lesser evil to allow streaming uploads
@@ -341,7 +341,7 @@ public void deleteFile(S3Object f) throws FileSystemException {
 			DeleteObjectRequest deleteObjectRequest = new DeleteObjectRequest(bucketName, f.getKey());
 			s3Client.deleteObject(deleteObjectRequest);
 		} catch (AmazonServiceException e) {
-			throw new FileSystemException("Could not delete object [" + getCanonicalName(f) + "]: " + e.getMessage());
+			throw new FileSystemException("Could not delete object [" + getCanonicalNameOrErrorMessage(f) + "]: " + e.getMessage());
 		}
 	}
 
@@ -382,31 +382,29 @@ public boolean folderExists(String folder) throws FileSystemException {
 	@Override
 	public void createFolder(String folder) throws FileSystemException {
 		String folderName = folder.endsWith("/") ? folder : folder + "/";
-		if(!folderExists(folder)) {
-			ObjectMetadata metadata = new ObjectMetadata();
-			metadata.setContentLength(0);
-			metadata.setContentType("binary/octet-stream");
-			InputStream emptyContent = NullInputStream.nullInputStream();
-
-			PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, folderName, emptyContent, metadata);
-			s3Client.putObject(putObjectRequest);
-		} else {
-			throw new FileSystemException("Create directory for [" + folderName + "] has failed. Directory already exists.");
+		if (folderExists(folder)) {
+			throw new FolderAlreadyExistsException("Create directory for [" + folderName + "] has failed. Directory already exists.");
 		}
+		ObjectMetadata metadata = new ObjectMetadata();
+		metadata.setContentLength(0);
+		metadata.setContentType("binary/octet-stream");
+		InputStream emptyContent = NullInputStream.nullInputStream();
+
+		PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, folderName, emptyContent, metadata);
+		s3Client.putObject(putObjectRequest);
 	}
 
 	@Override
 	public void removeFolder(String folder, boolean removeNonEmptyFolder) throws FileSystemException {
-		if (folderExists(folder)) {
-			if(!removeNonEmptyFolder && listFiles(folder, true).iterator().hasNext()) { //Check if there are files or folders
-				throw new FileSystemException("Cannot remove folder [" + folder + "]. Directory not empty.");
-			}
-
-			final String absFolder = folder.endsWith("/") ? folder : folder + "/"; //Ensure it's a folder that's being removed
-			s3Client.deleteObject(bucketName, absFolder);
-		} else {
-			throw new FileSystemException("Cannot remove folder [" + folder + "]. Directory does not exist.");
+		if (!folderExists(folder)) {
+			throw new FolderNotFoundException("Cannot remove folder [" + folder + "]. Directory does not exist.");
 		}
+		if(!removeNonEmptyFolder && listFiles(folder, true).iterator().hasNext()) { //Check if there are files or folders
+			throw new FileSystemException("Cannot remove folder [" + folder + "]. Directory not empty.");
+		}
+
+		final String absFolder = folder.endsWith("/") ? folder : folder + "/"; //Ensure it's a folder that's being removed
+		s3Client.deleteObject(bucketName, absFolder);
 	}
 
 	@Override
@@ -420,9 +418,9 @@ public S3Object renameFile(S3Object source, S3Object destination) throws FileSys
 	}
 
 	@Override
-	public S3Object copyFile(S3Object f, String destinationFolder, boolean createFolder, boolean resultantMustBeReturned) throws FileSystemException {
+	public S3Object copyFile(S3Object f, String destinationFolder, boolean createFolder) throws FileSystemException {
 		if (!createFolder && !folderExists(destinationFolder)) {
-			throw new FileSystemException("folder ["+destinationFolder+"] does not exist");
+			throw new FolderNotFoundException("folder ["+destinationFolder+"] does not exist");
 		}
 		String destinationFile = destinationFolder+"/"+getName(f);
 		CopyObjectRequest cor = new CopyObjectRequest(bucketName, f.getKey(), bucketName,destinationFile);
@@ -433,12 +431,13 @@ public S3Object copyFile(S3Object f, String destinationFolder, boolean createFol
 
 	@Override
 	// move is actually implemented via copy and delete
-	public S3Object moveFile(S3Object f, String destinationFolder, boolean createFolder, boolean resultantMustBeReturned) throws FileSystemException {
+	public S3Object moveFile(S3Object f, String destinationFolder, boolean createFolder) throws FileSystemException {
 		return renameFile(f,toFile(destinationFolder,getName(f)));
 	}
 
 
 	@Override
+	@Nullable
 	public Map<String, Object> getAdditionalFileProperties(S3Object f) {
 		Map<String, Object> attributes = new HashMap<>();
 		attributes.put("bucketName", bucketName);

diff --git a/commons/src/main/java/org/frankframework/util/CloseUtils.java b/commons/src/main/java/org/frankframework/util/CloseUtils.java
@@ -0,0 +1,63 @@
+/*
+   Copyright 2024 WeAreFrank!
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+package org.frankframework.util;
+
+import java.util.Collection;
+
+import jakarta.annotation.Nullable;
+import lombok.extern.log4j.Log4j2;
+
+@Log4j2
+public class CloseUtils {
+	private CloseUtils() {}
+
+	/**
+	 * Safely close an {@link AutoCloseable}, logging but ignoring any exceptions thrown.
+	 *
+	 * @param closeable AutoCloseable to close. It is safe to pass {@code null}.
+	 */
+	public static void closeSilently(@Nullable AutoCloseable closeable) {
+		if (closeable != null) {
+			try {
+				closeable.close();
+			} catch (Exception e) {
+				log.warn("Error closing [{}]", closeable, e);
+			}
+		}
+	}
+
+	/**
+	 * Safely close all {@link AutoCloseable}s, logging but ignoring any exceptions thrown.
+	 *
+	 * @param closeables AutoCloseables to close. It is safe to pass {@code null} values.
+	 */
+	public static void closeSilently(AutoCloseable... closeables) {
+		for (AutoCloseable closeable : closeables) {
+			closeSilently(closeable);
+		}
+	}
+
+	/**
+	 * Safely close all {@link AutoCloseable}s, logging but ignoring any exceptions thrown.
+	 *
+	 * @param closeables AutoCloseables to close. It is safe to pass {@code null}, or for the collection to contain {@code null} values.
+	 */
+	public static void closeSilently(@Nullable Collection<AutoCloseable> closeables) {
+		if (closeables != null) {
+			closeables.forEach(CloseUtils::closeSilently);
+		}
+	}
+}
diff --git a/core/src/main/java/org/frankframework/configuration/ApplicationWarningsBase.java b/core/src/main/java/org/frankframework/configuration/ApplicationWarningsBase.java
@@ -21,14 +21,13 @@
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
+import org.frankframework.util.AppConstants;
+import org.frankframework.util.ClassUtils;
 import org.springframework.beans.factory.DisposableBean;
 import org.springframework.beans.factory.InitializingBean;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
 
-import org.frankframework.util.AppConstants;
-import org.frankframework.util.ClassUtils;
-
 public abstract class ApplicationWarningsBase implements ApplicationContextAware, InitializingBean, DisposableBean {
 	private ApplicationContext applicationContext;
 	private AppConstants appConstants;
@@ -44,7 +43,7 @@ protected void addWarnings(List<String> warnings) {
 	}
 
 	@Override
-	public void destroy() throws Exception {
+	public void destroy() {
 		warnings.clear();
 	}
 

diff --git a/core/src/main/java/org/frankframework/core/IListener.java b/core/src/main/java/org/frankframework/core/IListener.java
@@ -20,9 +20,9 @@
 import jakarta.annotation.Nonnull;
 import org.frankframework.configuration.ConfigurationException;
 import org.frankframework.doc.ElementType;
+import org.frankframework.doc.ElementType.ElementTypes;
 import org.frankframework.doc.FrankDocGroup;
 import org.frankframework.doc.FrankDocGroupValue;
-import org.frankframework.doc.ElementType.ElementTypes;
 import org.frankframework.receivers.RawMessageWrapper;
 import org.frankframework.stream.Message;
 

diff --git a/core/src/main/java/org/frankframework/core/PipeLine.java b/core/src/main/java/org/frankframework/core/PipeLine.java
@@ -26,6 +26,10 @@
 
 import jakarta.annotation.Nonnull;
 import jakarta.annotation.Nullable;
+
+import io.micrometer.core.instrument.DistributionSummary;
+import lombok.Getter;
+import lombok.Setter;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.CloseableThreadContext;
 import org.frankframework.cache.ICache;
@@ -46,10 +50,6 @@
 import org.frankframework.util.Misc;
 import org.springframework.context.ApplicationContext;
 
-import io.micrometer.core.instrument.DistributionSummary;
-import lombok.Getter;
-import lombok.Setter;
-
 /**
  * Required in each {@link Adapter} to transform incoming messages. A pipeline
  * is a sequence of pipes. A
@@ -625,7 +625,7 @@ public void addPipe(IPipe pipe) throws ConfigurationException {
 		}
 		pipesByName.put(name, pipe);
 		pipes.add(pipe);
-		log.debug("added pipe [" + pipe + "]");
+		log.debug("added pipe [{}]", pipe);
 	}
 
 	/**

diff --git a/core/src/main/java/org/frankframework/core/PipeRunResult.java b/core/src/main/java/org/frankframework/core/PipeRunResult.java
@@ -15,12 +15,11 @@
 */
 package org.frankframework.core;
 
-import org.apache.commons.lang3.builder.ToStringBuilder;
-
 import lombok.Getter;
 import lombok.Setter;
-import org.frankframework.stream.Message;
+import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.frankframework.pipes.AbstractPipe;
+import org.frankframework.stream.Message;
 
 /**
  * The PipeRunResult is a type to store both the result of the processing of a message
@@ -38,7 +37,7 @@
  * @see AbstractPipe#doPipe
  * @see AbstractPipe#findForward
  */
-public class PipeRunResult {
+public class PipeRunResult implements AutoCloseable {
 
 	private @Getter @Setter PipeForward pipeForward;
 	private Message result;
@@ -71,4 +70,11 @@ public String toString() {
 	public boolean isSuccessful() {
 		return PipeForward.SUCCESS_FORWARD_NAME.equalsIgnoreCase(getPipeForward().getName());
 	}
+
+	@Override
+	public void close() throws Exception {
+		if (result != null) {
+			result.close();
+		}
+	}
 }