Feature: Adds and unifies problem report for deployment files (missin…

…g files and missing entries in files.json)
structr · Feb 28, 2024 · 16cfa7c · 16cfa7c
1 parent 0d985a6
commit 16cfa7c
Show file tree

Hide file tree

Showing 2 changed files with 93 additions and 33 deletions.
diff --git a/structr-base/src/main/java/org/structr/web/maintenance/DeployCommand.java b/structr-base/src/main/java/org/structr/web/maintenance/DeployCommand.java
@@ -87,10 +87,8 @@ public class DeployCommand extends NodeServiceCommand implements MaintenanceComm
 
 	protected static final Set<String> missingPrincipals       = new HashSet<>();
 	protected static final Set<String> missingSchemaFile       = new HashSet<>();
-	protected static final Set<String> missingEntryInFilesJson = new HashSet<>();
 	protected static final Set<String> deferredLogTexts        = new HashSet<>();
 
-
 	protected static final AtomicBoolean deploymentActive      = new AtomicBoolean(false);
 
 	private final static String DEPLOYMENT_DOM_NODE_VISIBILITY_RELATIVE_TO_KEY          = "visibility-flags-relative-to";
@@ -265,7 +263,6 @@ protected void doImport(final Map<String, Object> attributes) throws FrameworkEx
 
 			missingPrincipals.clear();
 			missingSchemaFile.clear();
-			missingEntryInFilesJson.clear();
 			deferredLogTexts.clear();
 
 			final long startTime = System.currentTimeMillis();
@@ -389,7 +386,7 @@ protected void doImport(final Map<String, Object> attributes) throws FrameworkEx
 			importLocalizations(localizationsMetadataFile);
 			importApplicationConfigurationNodes(applicationConfigurationDataMetadataFile);
 			importSchema(schemaFolder, extendExistingApp);
-			importFiles(filesMetadataFile, source, ctx);
+			final FileImportVisitor.FileImportProblems fileImportProblems = importFiles(filesMetadataFile, source, ctx);
 			importModuleData(source);
 			importHTMLContent(app, source, pagesMetadataFile, componentsMetadataFile, templatesMetadataFile, sitesConfFile, extendExistingApp, relativeVisibility, deferredNodesAndTheirProperties);
 			linkDeferredPages(app);
@@ -438,21 +435,16 @@ protected void doImport(final Map<String, Object> attributes) throws FrameworkEx
 				publishWarningMessage(title, text);
 			}
 
-			if (!missingEntryInFilesJson.isEmpty()) {
+			if (fileImportProblems != null && fileImportProblems.hasAnyProblems()) {
 
-				final String title = "Missing entries in files.json";
-				final String text = "The following file paths exist in the files folder, but have no corresponding entry in files.json. Because those entries do not exist, the files were <b>not imported</b>!<br><br>"
-						+ "The most common cause is that files.json was not correctly committed."
-						+ "<ul><li>" + String.join("</li><li>", missingEntryInFilesJson) + "</li></ul>";
+				final String title = "Encountered problems during import of files";
 
 				logger.info("\n###############################################################################\n"
 						+ "\tWarning: " + title + "!\n"
-						+ "\tThe following files paths exist in the files folder, but have no corresponding entry in files.json. Because those entries do not exist, the files were not imported!\n\n"
-						+ "\tThe most common cause is that files.json was not correctly committed.\n\n"
-						+ "\t" + String.join("\n\t", missingEntryInFilesJson)
+						+ fileImportProblems.getProblemsText()
 						+ "\n###############################################################################"
 				);
-				publishWarningMessage(title, text);
+				publishWarningMessage(title, fileImportProblems.getProblemsHtml());
 			}
 
 			final long endTime = System.currentTimeMillis();
@@ -1978,7 +1970,8 @@ private void importLocalizations(final Path localizationsMetadataFile) throws Fr
 
 			final PropertyMap additionalData = new PropertyMap();
 
-			// Question: shouldn't this be true? No, 'imported' is a flag for legacy-localization which
+			// Question: shouldn't this be true?
+			// No! 'imported' is a flag for legacy-localization which
 			// have been imported from a legacy-system which was replaced by structr.
 			// it is a way to differentiate between new and old localization strings
 			additionalData.put(StructrApp.key(Localization.class, "imported"), false);
@@ -2001,33 +1994,36 @@ private void importApplicationConfigurationNodes(final Path applicationConfigura
 		}
 	}
 
-	private void importFiles(final Path filesMetadataFile, final Path source, final SecurityContext ctx) throws FrameworkException {
+	private FileImportVisitor.FileImportProblems importFiles(final Path filesMetadataFile, final Path source, final SecurityContext ctx) throws FrameworkException {
 
 		if (Files.exists(filesMetadataFile)) {
 
-			final Map<String, Object> filesMetadata      = new HashMap<>();
-
 			logger.info("Reading {}", filesMetadataFile);
-			filesMetadata.putAll(readMetadataFileIntoMap(filesMetadataFile));
+			final Map<String, Object> filesMetadata = new HashMap<>(readMetadataFileIntoMap(filesMetadataFile));
 
 			final Path files = source.resolve("files");
 			if (Files.exists(files)) {
 
+				final FileImportVisitor fiv = new FileImportVisitor(ctx, files, filesMetadata);
+
 				try {
 
 					logger.info("Importing files (unchanged files will be skipped)");
 					publishProgressMessage(DEPLOYMENT_IMPORT_STATUS, "Importing files");
 
-					FileImportVisitor fiv = new FileImportVisitor(ctx, files, filesMetadata);
 					Files.walkFileTree(files, fiv);
 
 				} catch (IOException ioex) {
+
 					logger.warn("Exception while importing files", ioex);
 				}
+
+				return fiv.getFileImportProblems();
 			}
 		}
-	}
 
+		return null;
+	}
 
 	private void importModuleData(final Path source) throws FrameworkException {
 
@@ -2721,10 +2717,6 @@ public static void addMissingSchemaFile (final String fileName) {
 		missingSchemaFile.add(fileName);
 	}
 
-	public static void addMissingFileEntryInFilesJson(final String filePath) {
-		missingEntryInFilesJson.add(filePath);
-	}
-
 	// ----- nested helper classes -----
 	protected static class IdFirstComparator implements Comparator<String> {
 

diff --git a/structr-base/src/main/java/org/structr/web/maintenance/deploy/FileImportVisitor.java b/structr-base/src/main/java/org/structr/web/maintenance/deploy/FileImportVisitor.java
@@ -18,6 +18,9 @@
  */
 package org.structr.web.maintenance.deploy;
 
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -51,9 +54,6 @@
 
 import static org.structr.core.graph.NodeInterface.name;
 
-/**
- *
- */
 public class FileImportVisitor implements FileVisitor<Path> {
 
 	private static final Logger logger      = LoggerFactory.getLogger(FileImportVisitor.class.getName());
@@ -63,14 +63,16 @@ public class FileImportVisitor implements FileVisitor<Path> {
 	protected App app                         = null;
 	protected Map<String, Folder> folderCache = null;
 
+	private final ArrayList<String> encounteredPaths               = new ArrayList<>();
+	private final Set<String> encounteredButNotConfiguredFilePaths = new HashSet<>();
+
 	public FileImportVisitor(final SecurityContext securityContext, final Path basePath, final Map<String, Object> metadata) {
 
 		this.securityContext = securityContext;
 		this.basePath        = basePath;
 		this.metadata        = metadata;
 		this.app             = StructrApp.getInstance(this.securityContext);
 		this.folderCache     = new HashMap<>();
-
 	}
 
 	@Override
@@ -107,6 +109,14 @@ public FileVisitResult postVisitDirectory(final Path dir, final IOException exc)
 		return FileVisitResult.CONTINUE;
 	}
 
+	public FileImportProblems getFileImportProblems() {
+
+		final Set configuredButNotEncounteredPaths = new HashSet(metadata.keySet());
+		configuredButNotEncounteredPaths.removeAll(encounteredPaths);
+
+		return new FileImportProblems(configuredButNotEncounteredPaths, encounteredButNotConfiguredFilePaths);
+	}
+
 	// ----- private methods -----
 	protected Folder getExistingFolder(final String path) throws FrameworkException {
 
@@ -133,6 +143,8 @@ protected void createFolder(final Path folderObj) {
 
 		final String folderPath = harmonizeFileSeparators("/", basePath.relativize(folderObj).toString());
 
+		encounteredPaths.add(folderPath);
+
 		try (final Tx tx = app.tx(true, false, false)) {
 
 			tx.disableChangelog();
@@ -184,14 +196,13 @@ protected void createFile(final Path path, final String fileName) throws IOExcep
 			final String fullPath                   = harmonizeFileSeparators("/", basePath.relativize(path).toString());
 			final Map<String, Object> rawProperties = getRawPropertiesForFileOrFolder(fullPath);
 
+			encounteredPaths.add(fullPath);
+
 			if (rawProperties == null) {
 
-				if (!fileName.startsWith(".")) {
+				logger.info("Ignoring {} (not in files.json)", fullPath);
 
-					logger.info("Ignoring {} (not in files.json)", fullPath);
-
-					DeployCommand.addMissingFileEntryInFilesJson(path.toString());
-				}
+				encounteredButNotConfiguredFilePaths.add(path.toString());
 
 			} else {
 
@@ -375,4 +386,61 @@ protected String harmonizeFileSeparators(final String... sources) {
 
 		return buf.toString();
 	}
+
+	public class FileImportProblems {
+
+		final Set configuredButNotEncountered;
+		final Set encounteredButNotConfigured;
+
+		public FileImportProblems(final Set configuredButNotEncountered, final Set encounteredButNotConfigured) {
+
+			this.configuredButNotEncountered = configuredButNotEncountered;
+			this.encounteredButNotConfigured = encounteredButNotConfigured;
+		}
+
+		public boolean hasAnyProblems() {
+
+			return configuredButNotEncountered.size() > 0 || encounteredButNotConfigured.size() > 0;
+		}
+
+		public String getProblemsHtml() {
+
+			final ArrayList<String> problems = new ArrayList();
+
+			if (configuredButNotEncountered.size() > 0) {
+
+				problems.add(
+						"The following entries were configured in files.json, but the <b>expected files were not found</b>. The most common cause is that files.json was not correctly committed."
+						+ "<ul><li>" + String.join("</li><li>", configuredButNotEncountered) + "</li></ul>"
+				);
+			}
+
+			if (encounteredButNotConfigured.size() > 0) {
+
+				problems.add(
+						"The following files were found, but <b>are missing in files.json</b>. The most common cause is that files.json was correctly committed, but the file itself was not added to the repository."
+						+ "<ul><li>" + String.join("</li><li>", encounteredButNotConfigured) + "</li></ul>"
+				);
+			}
+
+			return String.join("<br>", problems);
+		}
+
+		public String getProblemsText() {
+
+			final ArrayList<String> problems = new ArrayList();
+
+			if (configuredButNotEncountered.size() > 0) {
+
+				problems.add("\tThe following entries were configured in files.json, but the expected files were not found. The most common cause is that files.json was not correctly committed.\n\t\t" + String.join("\n\t\t", configuredButNotEncountered));
+			}
+
+			if (encounteredButNotConfigured.size() > 0) {
+
+				problems.add("\tThe following files were found, but are missing in files.json. The most common cause is that files.json was correctly committed, but the file itself was not added to the repository.\n\t\t" + String.join("\n\t\t", encounteredButNotConfigured));
+			}
+
+			return String.join("\n\n", problems);
+		}
+	}
 }