Merge pull request #10 from ghv/compact

Compact Invalidation
ghv · May 16, 2023 · 59b0dab · 59b0dab
2 parents b25b682 + 447f3b0
commit 59b0dab
Show file tree

Hide file tree

Showing 7 changed files with 129 additions and 8 deletions.
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@
 
 /.build
 /.swiftpm
+/.idea
diff --git a/Sources/PrintKit/Constants.swift b/Sources/PrintKit/Constants.swift
@@ -7,7 +7,7 @@
 //
 
 public enum PrintKitConstants {
-    public static let version = "0.0.2"
+    public static let version = "0.0.3"
     public static let configFile = "contents.json"
     public static let timeStampsFile = ".contents-ts.json"
     public static let rootFolderEnvironmentVariable = "PRINTROOT"

diff --git a/Sources/PrintKit/ContentConfiguration.swift b/Sources/PrintKit/ContentConfiguration.swift
@@ -28,6 +28,7 @@ public struct ContentConfiguration: Decodable {
 
     struct ContentFolder: Decodable {
         enum CodingKeys: String, CodingKey {
+            case compactInvalidation
             case folder
             case files
         }
@@ -50,6 +51,9 @@ public struct ContentConfiguration: Decodable {
             }
         }
 
+        /// Invalidate files using "\(folder)/*" rather than individual files under this path.
+        var compactInvalidation: Bool
+
         /// The folder or path that will contiain the files specified (key path prefix in S3)
         var folder: String
 
@@ -59,6 +63,7 @@ public struct ContentConfiguration: Decodable {
         init(from decoder: Decoder) throws {
             let container = try decoder.container(keyedBy: CodingKeys.self)
             folder = try container.decode(String.self, forKey: .folder)
+            compactInvalidation = try container.decodeIfPresent(Bool.self, forKey: .compactInvalidation) ?? false
             let anyFiles = try container.decode([File].self, forKey: .files)
             files = anyFiles.map { $0.value }
         }
@@ -93,6 +98,45 @@ public struct ContentConfiguration: Decodable {
             }
         }
     }
+
+    // Compacts the paths using wildcards if there is more than one change in or under a compactable folder
+    func compactChangedKeysToWildcards(_ changedKeys: [String]) -> [String] {
+        var remainingChangedKeys = changedKeys
+        let wildcardFolders = contents.compactMap {
+            if $0.compactInvalidation {
+                return $0.folder
+            } else {
+                return nil
+            }
+        }.sorted{ $0 > $1 }
+
+        var wildcards: [String] = []
+        for folder in wildcardFolders {
+            let keysInFolder = remainingChangedKeys.filter { $0.starts(with: folder) }
+            if keysInFolder.count > 1 {
+                wildcards.append("\(folder)/*")
+            } else {
+                wildcards.append(contentsOf: keysInFolder)
+            }
+            remainingChangedKeys = remainingChangedKeys.filter { !$0.starts(with: folder) }
+        }
+
+        // Handle the case where there are nested wildcard levels like "/foo/*" and "/foo/bar/*" to
+        // invalidate at the lowest level but combine higher levels to reduce this to one global.
+        var result: [String] = []
+        for folder in wildcardFolders.reversed() {
+            let keysInFolder = wildcards.filter { $0.starts(with: folder) }
+            if keysInFolder.count > 1 {
+                result.append("/\(folder)/*")
+            } else {
+                result.append(contentsOf: keysInFolder.map  { "/\($0)" })
+            }
+            wildcards = wildcards.filter { !$0.starts(with: folder) }
+        }
+        result.append(contentsOf: remainingChangedKeys.map { "/\($0)" })
+        return result
+    }
+
 }
 
 typealias UploadedContents = [String:Double]
diff --git a/Sources/PrintKit/Foundation+Extensions.swift b/Sources/PrintKit/Foundation+Extensions.swift
@@ -12,7 +12,7 @@ import Foundation
 extension Encodable {
     public func endcoded() throws -> Data {
         let encoder = JSONEncoder()
-        encoder.outputFormatting = .prettyPrinted
+        encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
         return try encoder.encode(self)
     }
 }

diff --git a/Sources/PrintKit/S3CloudFrontDeployer.swift b/Sources/PrintKit/S3CloudFrontDeployer.swift
@@ -49,6 +49,7 @@ public class S3CloudFrontDeployer {
         s3.headBucket(S3.HeadBucketRequest(bucket: config.bucket))
     }
 
+    // [(fullLocalPath, cloudFrontFileName, localRelativePath)]
     func buildTouchedFilesList() -> [(String, String, String)] {
         var uploadList = [(String, String, String)]()
         for target in config.contents {
@@ -95,11 +96,18 @@ public class S3CloudFrontDeployer {
     }
 
     func createInvalidationFuture(with changes: [(String, String, String)]) -> EventLoopFuture<CloudFront.CreateInvalidationResult> {
-        let changedKeys = changes.map { (_, cloudFrontPath, _) in
-            "/\(cloudFrontPath)"
+        let allChangedKeys = changes.map{ $0.1 }
+        print("All Changed Keys:")
+        for key in allChangedKeys {
+            print("   \(key)")
         }
-        if changedKeys.count > 0 {
-            let paths = CloudFront.Paths(items: changedKeys, quantity: changedKeys.count)
+        let reducedChangedKeys = config.compactChangedKeysToWildcards(allChangedKeys)
+        if reducedChangedKeys.count > 0 {
+            print("Invalidated Keys:")
+            for key in reducedChangedKeys {
+                print("   \(key)")
+            }
+            let paths = CloudFront.Paths(items: reducedChangedKeys, quantity: reducedChangedKeys.count)
             let batch = CloudFront.InvalidationBatch(callerReference: Date().timeStampID, paths: paths)
             let request = CloudFront.CreateInvalidationRequest(distributionId: config.cloudFront, invalidationBatch: batch)
             return cloudFront.createInvalidation(request)

diff --git a/Tests/PrintKitTests/ContentConfigTests.swift b/Tests/PrintKitTests/ContentConfigTests.swift
@@ -61,6 +61,7 @@ final class ContentConfigTests: XCTestCase {
         XCTAssertEqual(config.originPathFolder, "someOriginPathFolder")
         XCTAssertEqual(config.contents.count, 1)
         XCTAssertEqual(config.contents[0].folder, "someFolder")
+        XCTAssertEqual(config.contents[0].compactInvalidation, false)
         XCTAssertEqual(config.contents[0].files.count, 2)
         XCTAssertEqual(config.contents[0].files[0].count, 2)
         XCTAssertEqual(config.contents[0].files[0][0], "someFile")
@@ -70,6 +71,71 @@ final class ContentConfigTests: XCTestCase {
         XCTAssertEqual(config.contents[0].files[1][1], "alias")
     }
 
+    func testReduceChangedKeys() throws {
+        let data = Data("""
+            {
+              "region": "someRegion",
+              "bucket": "someBucket",
+              "cloudFront": "someCloudFront",
+              "originPathFolder": "someOriginPathFolder",
+              "contents": [
+                {
+                    "folder": "someFolderOne",
+                    "compactInvalidation": true,
+                    "files": [
+                    ]
+                },
+                {
+                    "folder": "someFolderTwo",
+                    "files": [
+                    ]
+                },
+                {
+                    "folder": "someFolderThree",
+                    "compactInvalidation": true,
+                    "files": [
+                    ]
+                },
+                {
+                    "folder": "someFolderFour",
+                    "compactInvalidation": true,
+                    "files": [
+                    ]
+                },
+                {
+                    "folder": "someFolderFour/Five",
+                    "compactInvalidation": true,
+                    "files": [
+                    ]
+                }
+              ]
+            }
+            """.utf8)
+        let config: ContentConfiguration = try data.decoded()
+
+        let changedKeys = [
+            "someFolderOne/Foo",
+            "someFolderTwo/Foo",
+            "someFolderThree/Foo",
+            "someFolderThree/Bar",
+            "someFolderFour/Foo",
+            "someFolderFour/Five/Bar",
+            "someFolderFour/Five/Baz",
+        ].shuffled()
+
+        let expectedKeys = [
+            "/someFolderFour/*",
+            "/someFolderOne/Foo",
+            "/someFolderThree/*",
+            "/someFolderTwo/Foo",
+        ]
+
+        let resultKeys = config.compactChangedKeysToWildcards(changedKeys)
+
+        XCTAssertEqual(resultKeys, expectedKeys)
+    }
+
+
     func testVariableExpanded() throws {
         let someBAR = "BAR"
         let someFOO = "FOO"
@@ -95,7 +161,7 @@ final class ContentConfigTests: XCTestCase {
         XCTAssertEqual(config.contents.count, 0)
     }
 
-    func testVariableNotExpander() throws {
+    func testVariableNotExpanded() throws {
         let data = Data("""
             {
               "region": "$BAT",

diff --git a/readme.markdown b/readme.markdown
@@ -94,7 +94,8 @@ You can use the [JSON Schema Validator](https://www.jsonschemavalidator.net) to
                                 }
                             ]
                         }
-                    }
+                    },
+                    "compactInvalidation": { "type": "boolean" }
                 },
                 "required": ["folder", "files"]
            }   
@@ -123,6 +124,7 @@ Each element in the `contents` array should contain the following:
 | ---------- | ------------- |
 | `folder` | The path in the S3 bucket that will contain the files specified by the `files` property. |
 | `files`  | An array of local file paths to be uploaded in `folder`. Each element in this array can be either a string containing the local file path or an array of two strings where the first element is the local file path and the second is the remote file name. The local file paths are relative to the `contents.json` folder. |
+| `compactInvalidation` | When set to `true`, invalidates everything under this folder rather than invalidating individual files when two or more files have changed. The default value is `false` if the key is omitted. |
 
 ### Sample `contents.json`