Truncate large issues to improve memory usage and log processing speed (

#69) * Add flag to truncate number of issues reported in a task * Make Tag optional Co-authored-by: Erick Camacho <erickc@spotify.com>
spotify · Dec 1, 2021 · ac8ce19 · ac8ce19
1 parent 408540e
commit ac8ce19
Show file tree

Hide file tree

Showing 21 changed files with 74 additions and 446 deletions.
diff --git a/Examples/BasicApp/BasicApp.xcodeproj/xcshareddata/xcschemes/BasicApp.xcscheme b/Examples/BasicApp/BasicApp.xcodeproj/xcshareddata/xcschemes/BasicApp.xcscheme
@@ -11,7 +11,7 @@
             ActionType = "Xcode.IDEStandardExecutionActionsCore.ExecutionActionType.ShellScriptAction">
             <ActionContent
                title = "Run Script"
-               scriptText = "${SRCROOT}/../../XCMetricsLauncher ${SRCROOT}/../../.build/release/XCMetrics --name BasicApp --buildDir ${BUILD_DIR} --isCI false --skipNotes false --serviceURL http://localhost:8080/v1/metrics&#10;&#10;&#10;">
+               scriptText = "${SRCROOT}/../../XCMetricsLauncher ${SRCROOT}/../../.build/release/XCMetrics --name BasicApp --buildDir ${BUILD_DIR} --isCI false --skipNotes false --truncateLargeIssues true --serviceURL http://localhost:8080/v1/metrics&#10;&#10;&#10;">
                <EnvironmentBuildable>
                   <BuildableReference
                      BuildableIdentifier = "primary"

diff --git a/Package.resolved b/Package.resolved
@@ -141,8 +141,8 @@
         "repositoryURL": "https://github.com/kylef/PathKit.git",
         "state": {
           "branch": null,
-          "revision": "73f8e9dca9b7a3078cb79128217dc8f2e585a511",
-          "version": "1.0.0"
+          "revision": "3bfd2737b700b9a36565a8c94f4ad2b050a5e574",
+          "version": "1.0.1"
         }
       },
       {
@@ -240,8 +240,8 @@
         "repositoryURL": "https://github.com/kylef/Spectre.git",
         "state": {
           "branch": null,
-          "revision": "f79d4ecbf8bc4e1579fbd86c3e1d652fb6876c53",
-          "version": "0.9.2"
+          "revision": "26cc5e9ae0947092c7139ef7ba612e34646086c7",
+          "version": "0.10.1"
         }
       },
       {
@@ -384,8 +384,8 @@
         "repositoryURL": "https://github.com/spotify/xclogparser",
         "state": {
           "branch": null,
-          "revision": "e463154a481c2e8162dec482296aae852f474728",
-          "version": "0.2.28"
+          "revision": "946d72266691ed1f5ad91a3ba36c59f65f1c938c",
+          "version": "0.2.31"
         }
       },
       {

diff --git a/Package.swift b/Package.swift
@@ -15,7 +15,7 @@ let package = Package(
         .library(name: "XCMetricsUtils", targets: ["XCMetricsUtils"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/spotify/xclogparser", from: "0.2.28"),
+        .package(url: "https://github.com/spotify/xclogparser", from: "0.2.31"),
         .package(url: "https://github.com/apple/swift-tools-support-core.git", .exact("0.2.3")),
         .package(url: "https://github.com/grpc/grpc-swift.git", .exact("1.0.0-alpha.9")),
         .package(url: "https://github.com/apple/swift-nio.git", from: "2.23.0"),

diff --git a/Sources/XCMetricsBackendLib/UploadMetrics/Jobs/ProcessMetricsJob.swift b/Sources/XCMetricsBackendLib/UploadMetrics/Jobs/ProcessMetricsJob.swift
@@ -73,10 +73,8 @@ class ProcessMetricsJob: Job {
                 promise.fail(error)
                 return
             }
-
-            let metricsWithRequestData = self.addBuildRequest(buildMetrics: buildMetrics, payload: payload)
             logWithTimestamp(context.logger, msg: "[ProcessMetricsJob] log parsed \(payload.logURL)")
-            _ = self.metricsRepository.insertBuildMetrics(metricsWithRequestData, using: eventLoop)
+            _ = self.metricsRepository.insertBuildMetrics(buildMetrics, using: eventLoop)
                 .flatMapAlways { (result) -> EventLoopFuture<Void> in
                     var wasProcessed: Bool = false
                     switch result {
@@ -100,15 +98,6 @@ class ProcessMetricsJob: Job {
         return promise.futureResult
     }
 
-    private func addBuildRequest(buildMetrics: BuildMetrics, payload: UploadMetricsRequest) -> BuildMetrics {
-        guard let buildIdentifier = buildMetrics.build.id else {
-            return buildMetrics
-        }
-        return buildMetrics.withHost(payload.buildHost.withBuildIdentifier(buildIdentifier))
-            .withBuildMetadata(payload.buildMetadata?.withBuildIdentifier(buildIdentifier))
-            .withXcodeVersion(payload.xcodeVersion?.withBuildIdentifier(buildIdentifier))
-    }
-
     private func removeLocalLog(_ log: LogFile,
                                 wasProcessed: Bool,
                                 using eventLoop: EventLoop,

diff --git a/Sources/XCMetricsBackendLib/UploadMetrics/LogProcessing/LogParser.swift b/Sources/XCMetricsBackendLib/UploadMetrics/LogProcessing/LogParser.swift
@@ -52,7 +52,9 @@ struct LogParser {
         let activityLog = try ActivityParser().parseActivityLogInURL(url, redacted: true, withoutBuildSpecificInformation: true)
         let buildSteps = try ParserBuildSteps(machineName: machineName,
                                               omitWarningsDetails: false,
-                                              omitNotesDetails: metricsRequest.extraInfo.skipNotes ?? false)
+                                              omitNotesDetails: metricsRequest.extraInfo.skipNotes ?? false,
+                                              truncLargeIssues: metricsRequest.extraInfo.truncLargeIssues ?? false
+        )
             .parse(activityLog: activityLog)
             .flatten()
         return toBuildMetrics(
@@ -77,7 +79,7 @@ struct LogParser {
         build.projectName = metricsRequest.extraInfo.projectName
         build.userid = userId
         build.userid256 = userIdSHA256
-        build.tag = metricsRequest.extraInfo.tag
+        build.tag = metricsRequest.extraInfo.tag ?? ""
         build.isCi = isCI
 
         if let sleepTime = metricsRequest.extraInfo.sleepTime {