Skip to content

Commit

Permalink
Allow create folder when creating/writing to a file (frankframework#5220
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@nielsm5
nielsm5 authored Aug 10, 2023
1 parent 9bf93c2 commit 89726d0
Show file tree
Hide file tree
Showing 43 changed files with 992 additions and 628 deletions.
5 changes: 5 additions & 0 deletions aws/src/main/java/nl/nn/adapterframework/filesystem/AmazonS3FileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,11 @@ public boolean exists(S3Object f) throws FileSystemException {

@Override
public OutputStream createFile(final S3Object f) throws FileSystemException, IOException {
String folder = getParentFolder(f);
if (folder != null && !folderExists(folder)) { //AWS Supports the creation of folders, this check is purely here so all FileSystems have the same behavior
throw new FileSystemException("folder ["+folder+"] does not exist");
}

final File file = FileUtils.createTempFile(".s3-upload");
final FileOutputStream fos = new FileOutputStream(file);
return new BufferedOutputStream(fos) {
Expand Down
6 changes: 0 additions & 6 deletions core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,12 +390,6 @@
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
</dependency>
<dependency>
<groupId>org.mockftpserver</groupId>
<artifactId>MockFtpServer</artifactId>
<version>3.1.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.sshd</groupId>
<artifactId>sshd-sftp</artifactId>
Expand Down
27 changes: 19 additions & 8 deletions core/src/main/java/nl/nn/adapterframework/filesystem/FileSystemActor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@

import org.apache.commons.codec.binary.Base64InputStream;
import org.apache.commons.codec.binary.Base64OutputStream;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.Logger;
import org.xml.sax.SAXException;
Expand Down Expand Up @@ -78,6 +79,7 @@
*/
public class FileSystemActor<F, FS extends IBasicFileSystem<F>> implements IOutputStreamingSupport {
protected Logger log = LogUtil.getLogger(this);
private static final String LINE_SEPARATOR = System.getProperty("line.separator");

public static final String ACTION_CREATE="create";
public static final String ACTION_LIST="list";
Expand Down Expand Up @@ -112,7 +114,7 @@ public class FileSystemActor<F, FS extends IBasicFileSystem<F>> implements IOutp
private @Getter String filename;
private @Getter String destination;
private @Getter String inputFolder; // folder for action=list
private @Getter boolean createFolder; // for action move, rename and list
private @Getter boolean createFolder; // for action create, write, move, rename and list

private @Getter Base64Pipe.Direction base64;
private @Getter int rotateDays=0;
Expand Down Expand Up @@ -156,7 +158,7 @@ public enum FileSystemAction implements DocumentedEnum {
@EnumLabel(ACTION_MKDIR) MKDIR,
/** remove a folder/directory, specified by attribute <code>inputFolder</code>, parameter <code>inputFolder</code> or input message */
@EnumLabel(ACTION_RMDIR) RMDIR,
/** write contents, specified by parameter <code>contents</code> or input message, to a file, specified by attribute <code>filename</code>, parameter <code>filename</code> or input message.
/** Creates file and writes contents, specified by parameter <code>contents</code> or input message, to a file, specified by attribute <code>filename</code>, parameter <code>filename</code> or input message.
* At least one of the parameters must be specified. The missing parameter defaults to the input message. For streaming operation, the parameter <code>filename</code> must be specified. */
@EnumLabel(ACTION_WRITE1) WRITE,
/** replaced by <code>write</code> */
Expand Down Expand Up @@ -219,7 +221,7 @@ public void configure(FS fileSystem, ParameterList parameterList, IConfigurable
throw new ConfigurationException("FileSystem ["+ClassUtils.nameOf(fileSystem)+"] does not support setting attribute 'rotateDays'");
}
}
eolArray = System.getProperty("line.separator").getBytes();
eolArray = LINE_SEPARATOR.getBytes(StreamUtil.DEFAULT_CHARSET);
}

private void checkConfiguration(FileSystemAction action2) throws ConfigurationException {
Expand Down Expand Up @@ -296,10 +298,19 @@ private String determineDestination(ParameterValueList pvl) throws FileSystemExc
throw new FileSystemException("no destination specified");
}

private F getFile(Message input, ParameterValueList pvl) throws FileSystemException {
private F getFile(@Nonnull Message input, ParameterValueList pvl) throws FileSystemException {
return fileSystem.toFile(determineFilename(input, pvl));
}

private F getFileAndCreateFolder(@Nonnull Message input, ParameterValueList pvl) throws FileSystemException {
String filenameWithFolder = determineFilename(input, pvl);
String folder = FilenameUtils.getFullPathNoEndSeparator(filenameWithFolder);
if(StringUtils.isNotBlank(folder) && isCreateFolder() && !fileSystem.folderExists(folder)) {
fileSystem.createFolder(folder);
}
return fileSystem.toFile(filenameWithFolder);
}

private String determineInputFoldername(Message input, ParameterValueList pvl) throws FileSystemException {
if (StringUtils.isNotEmpty(getInputFolder())) {
return getInputFolder();
Expand Down Expand Up @@ -335,10 +346,10 @@ public Message doAction(@Nonnull Message input, ParameterValueList pvl, @Nonnull

switch(action) {
case CREATE:{
F file=getFile(input, pvl);
F file = getFileAndCreateFolder(input, pvl);
if (fileSystem.exists(file)) {
FileSystemUtils.prepareDestination((IWritableFileSystem<F>)fileSystem, file, isOverwrite(), getNumberOfBackups(), FileSystemAction.CREATE);
file=getFile(input, pvl); // reobtain the file, as the object itself may have changed because of the rollover
file = fileSystem.toFile(fileSystem.getCanonicalName(file)); // reobtain the file, as the object itself may have changed because of the rollover
}
//noinspection EmptyTryBlock
try (OutputStream ignored = ((IWritableFileSystem<F>)fileSystem).createFile(file)) {
Expand Down Expand Up @@ -423,7 +434,7 @@ protected void finalize() throws Throwable {
return Message.asMessage(directoryBuilder.toString());
}
case WRITE: {
F file=getFile(input, pvl);
F file = getFileAndCreateFolder(input, pvl);
if (fileSystem.exists(file)) {
FileSystemUtils.prepareDestination((IWritableFileSystem<F>)fileSystem, file, isOverwrite(), getNumberOfBackups(), FileSystemAction.WRITE);
file=getFile(input, pvl); // reobtain the file, as the object itself may have changed because of the rollover
Expand Down Expand Up @@ -679,7 +690,7 @@ public void setInputFolder(String inputFolder) {
}

/**
* If set <code>true</code>, the folder to move or copy to is created if it does not exist
* If set <code>true</code>, the folder to create, write, move or copy the file to is created if it does not exist
* @ff.default false
*/
public void setCreateFolder(boolean createFolder) {
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/nl/nn/adapterframework/filesystem/FtpFileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ public boolean exists(FTPFileRef file) throws FileSystemException {
try {
FTPFile[] files = ftpClient.listFiles(file.getFolder(), f -> f.getName().equals(file.getFileName()));
if(files != null && files.length > 0) {
return FTPFileRef.fromFTPFile(files[0]);
return FTPFileRef.fromFTPFile(files[0], file.getFolder());
}
} catch(IOException e) {
throw new FileSystemException("unable to browse remote directory", e);
Expand Down
9 changes: 1 addition & 8 deletions core/src/main/java/nl/nn/adapterframework/filesystem/Samba1FileSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
import nl.nn.adapterframework.configuration.ConfigurationException;
import nl.nn.adapterframework.configuration.ConfigurationWarning;
import nl.nn.adapterframework.stream.Message;
import nl.nn.adapterframework.stream.MessageContext;
import nl.nn.adapterframework.util.CredentialFactory;
import nl.nn.adapterframework.util.FilenameUtils;

Expand Down Expand Up @@ -138,13 +137,7 @@ public OutputStream appendFile(SmbFile f) throws FileSystemException {

@Override
public Message readFile(SmbFile f, String charset) throws IOException, FileSystemException {
return new Samba1Message(f, FileSystemUtils.getContext(this, f, charset));
}

private class Samba1Message extends Message {
public Samba1Message(SmbFile f, MessageContext context) {
super(() -> new SmbFileInputStream(f), context, f.getClass());
}
return new Message(new SmbFileInputStream(f), FileSystemUtils.getContext(this, f, charset));
}

@Override
Expand Down
34 changes: 34 additions & 0 deletions core/src/test/java/nl/nn/adapterframework/filesystem/FTPFileRefTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package nl.nn.adapterframework.filesystem;

import static org.junit.jupiter.api.Assertions.assertEquals;

import org.junit.jupiter.api.Test;

import nl.nn.adapterframework.ftp.FTPFileRef;

public class FTPFileRefTest {

@Test
public void testFTPFileRefSetRelative() {
assertEquals("test123", new FTPFileRef("test123").getName());
assertEquals("folder/test123", new FTPFileRef("folder/test123").getName());
}

@Test
public void testFTPFileRefSetFolder() {
FTPFileRef ref1 = new FTPFileRef("test123", "folder");
assertEquals("folder/test123", ref1.getName());
}

@Test
public void testFTPFileRefRelativeWithSetFolder() {
FTPFileRef ref2 = new FTPFileRef("folder1/test123", "folder2");
assertEquals("folder2/test123", ref2.getName());
}

@Test
public void testFTPFileRefWindowsSlash() {
FTPFileRef ref2 = new FTPFileRef("folder1\\test123", "folder2");
assertEquals("folder2/test123", ref2.getName());
}
}
129 changes: 125 additions & 4 deletions core/src/test/java/nl/nn/adapterframework/filesystem/FileSystemPipeTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import nl.nn.adapterframework.core.PipeForward;
import nl.nn.adapterframework.core.PipeLineSession;
import nl.nn.adapterframework.core.PipeRunException;
import nl.nn.adapterframework.core.PipeRunResult;
import nl.nn.adapterframework.core.PipeStartException;
import nl.nn.adapterframework.filesystem.FileSystemActor.FileSystemAction;
Expand All @@ -28,6 +29,7 @@
import nl.nn.adapterframework.stream.MessageOutputStream;
import nl.nn.adapterframework.testutil.ParameterBuilder;
import nl.nn.adapterframework.testutil.TestAssertions;
import nl.nn.adapterframework.util.StreamUtil;

public abstract class FileSystemPipeTest<FSP extends FileSystemPipe<F, FS>, F, FS extends IWritableFileSystem<F>> extends HelperedFileSystemTestBase {

Expand Down Expand Up @@ -227,10 +229,10 @@ public void fileSystemPipeMoveActionTest(String folder1, String folder2, boolean
String filename = "sendermove" + FILE1;
String contents = "Tekst om te lezen";

if (folder1!=null) {
if(folder1 != null) {
_createFolder(folder1);
}
if (folderExists && folder2!=null) {
if(folderExists && folder2 != null) {
_createFolder(folder2);
}
createFile(folder1, filename, contents);
Expand All @@ -245,9 +247,9 @@ public void fileSystemPipeMoveActionTest(String folder1, String folder2, boolean
fileSystemPipe.configure();
fileSystemPipe.start();

Message message= new Message(filename);
Message message = new Message(filename);
PipeRunResult prr = fileSystemPipe.doPipe(message, session);
String result=prr.getResult().asString();
String result = prr.getResult().asString();

// test
// result should be name of the moved file
Expand Down Expand Up @@ -282,6 +284,125 @@ public void fileSystemPipeMoveActionTestRootToFolderFailIfolderDoesNotExist() th
// fileSystemPipeMoveActionTest("folder1","folder2");
// }

public void fileSystemPipeCreateFile(String folder, boolean fileAlreadyExists, boolean setCreateFolderAttribute) throws Exception {
String filename = "create" + FILE1;

if(_folderExists(folder)) {
_deleteFolder(folder);
}
waitForActionToFinish();

if(fileAlreadyExists && !_fileExists(folder, filename)) {
_createFile(folder, filename);
}

fileSystemPipe.setAction(FileSystemAction.CREATE);
if (setCreateFolderAttribute) {
fileSystemPipe.setCreateFolder(true);
}
fileSystemPipe.configure();
fileSystemPipe.start();

Message message = new Message(folder + "/" + filename);
PipeRunResult prr = fileSystemPipe.doPipe(message, session);
String result = prr.getResult().asString();

// test
// result should be name of the moved file
assertNotNull(result);

// TODO: result should point to new location of file
// TODO: contents of result should be contents of original file

assertTrue(_fileExists(folder, filename), "file should exist in destination folder ["+folder+"]");
}

@Test
public void fileSystemPipeCreateFileInFolder() throws Exception {
PipeRunException e = assertThrows(PipeRunException.class, () -> fileSystemPipeCreateFile("folder1", false, false));
assertEquals(e.getCause().getClass(), FileSystemException.class);
assertThat(e.getMessage(), containsString("unable to process [CREATE] action for File [folder1/createfile1.txt]"));
}

@Test
public void fileSystemPipeCreateFileAndCreateFolderAttributeEnabled() throws Exception {
fileSystemPipeCreateFile("folder2", false, true);
}

@Test
public void fileSystemPipeCreatingFileThatAlreadyExists() throws Exception {
PipeRunException e = assertThrows(PipeRunException.class, () -> fileSystemPipeCreateFile("folder3", true, false));
assertEquals(e.getCause().getClass(), FileSystemException.class);
assertThat(e.getMessage(), containsString("unable to process [CREATE] action for File [folder3/createfile1.txt]"));
}

@Test
public void fileSystemPipeCreatingFileThatAlreadyExistsAndCreateFolderAttributeEnabled() throws Exception {
PipeRunException e = assertThrows(PipeRunException.class, () -> fileSystemPipeCreateFile("folder4", true, true));
assertEquals(e.getCause().getClass(), FileSystemException.class);
assertThat(e.getMessage(), containsString("unable to process [CREATE] action for File [folder4/createfile1.txt]"));
}

public void fileSystemPipeWriteFile(String folder, boolean fileAlreadyExists, boolean setCreateFolderAttribute) throws Exception {
String filename = "write" + FILE1;

if(_folderExists(folder)) {
_deleteFolder(folder);
}
waitForActionToFinish();

if(fileAlreadyExists && !_fileExists(folder, filename)) {
_createFile(folder, filename);
}

fileSystemPipe.setAction(FileSystemAction.WRITE);
if (setCreateFolderAttribute) {
fileSystemPipe.setCreateFolder(true);
}
fileSystemPipe.addParameter(ParameterBuilder.create("filename", folder + "/" + filename));
fileSystemPipe.configure();
fileSystemPipe.start();

Message message = new Message("dummyText");
PipeRunResult prr = fileSystemPipe.doPipe(message, session);
String result = prr.getResult().asString();

// test
// result should be name of the moved file
assertNotNull(result);

// TODO: result should point to new location of file
// TODO: contents of result should be contents of original file

assertTrue(_fileExists(folder, filename), "file should exist in destination folder ["+folder+"]");
assertEquals("dummyText", StreamUtil.streamToString(_readFile(folder, filename)));
}
@Test
public void fileSystemPipeWriteNewFileInFolder() throws Exception {
PipeRunException e = assertThrows(PipeRunException.class, () -> fileSystemPipeWriteFile("folder1", false, false));
assertEquals(e.getCause().getClass(), FileSystemException.class);
assertThat(e.getMessage(), containsString("unable to process [WRITE] action for File [folder1/writefile1.txt]"));
}

@Test
public void fileSystemPipeWritingFileAndCreateFolderAttributeEnabled() throws Exception {
fileSystemPipeWriteFile("folder2", false, true);
}

@Test
public void fileSystemPipeWritingFileThatAlreadyExists() throws Exception {
PipeRunException e = assertThrows(PipeRunException.class, () -> fileSystemPipeWriteFile("folder3", true, false));
assertEquals(e.getCause().getClass(), FileSystemException.class);
assertThat(e.getMessage(), containsString("unable to process [WRITE] action for File [folder3/writefile1.txt]"));
}

@Test
public void fileSystemPipeWritingFileThatAlreadyExistsAndCreateFolderAttributeEnabled() throws Exception {
PipeRunException e = assertThrows(PipeRunException.class, () -> fileSystemPipeWriteFile("folder3", true, false));
assertEquals(e.getCause().getClass(), FileSystemException.class);
assertThat(e.getMessage(), containsString("unable to process [WRITE] action for File [folder3/writefile1.txt]"));
}

@Test
public void fileSystemPipeMkdirActionTest() throws Exception {
String folder = "mkdir" + DIR1;
Expand Down
Loading

0 comments on commit 89726d0

Please sign in to comment.