diff --git a/Package.swift b/Package.swift index 076f3b9e6..b029fcbae 100644 --- a/Package.swift +++ b/Package.swift @@ -282,11 +282,12 @@ let package = Package( name: "Deployer", dependencies: [ "EmceeLogging", + .product(name: "FileSystem", package: "CommandLineToolkit"), .product(name: "PathLib", package: "CommandLineToolkit"), - .product(name: "ProcessController", package: "CommandLineToolkit"), "QueueModels", .product(name: "Tmp", package: "CommandLineToolkit"), "UniqueIdentifierGenerator", + "Zip", ], path: "Sources/Deployer" ), @@ -302,12 +303,12 @@ let package = Package( name: "DeployerTests", dependencies: [ "Deployer", + .product(name: "FileSystemTestHelpers", package: "CommandLineToolkit"), .product(name: "PathLib", package: "CommandLineToolkit"), - .product(name: "ProcessController", package: "CommandLineToolkit"), - .product(name: "ProcessControllerTestHelpers", package: "CommandLineToolkit"), .product(name: "TestHelpers", package: "CommandLineToolkit"), .product(name: "Tmp", package: "CommandLineToolkit"), "UniqueIdentifierGeneratorTestHelpers", + "ZipTestHelpers", ], path: "Tests/DeployerTests" ), @@ -352,15 +353,16 @@ let package = Package( dependencies: [ "Deployer", "EmceeLogging", + .product(name: "FileSystem", package: "CommandLineToolkit"), .product(name: "LaunchdUtils", package: "CommandLineToolkit"), .product(name: "PathLib", package: "CommandLineToolkit"), - .product(name: "ProcessController", package: "CommandLineToolkit"), "QueueModels", "SSHDeployer", .product(name: "SocketModels", package: "CommandLineToolkit"), .product(name: "Tmp", package: "CommandLineToolkit"), "TypedResourceLocation", "UniqueIdentifierGenerator", + "Zip", ], path: "Sources/DistDeployer" ), @@ -541,6 +543,7 @@ let package = Package( "QueueServer", "QueueServerPortProvider", "RESTMethods", + "RESTServer", "RemotePortDeterminer", "RequestSender", "ResourceLocation", @@ -554,6 +557,7 @@ let package = Package( "SimulatorPoolModels", .product(name: "SocketModels", package: "CommandLineToolkit"), .product(name: "Statsd", package: "CommandLineToolkit"), + .product(name: "Swifter", package: "Swifter"), .product(name: "SynchronousWaiter", package: "CommandLineToolkit"), "TestArgFile", "TestDiscovery", @@ -565,6 +569,7 @@ let package = Package( "WorkerAlivenessProvider", "WorkerCapabilities", "WorkerCapabilitiesModels", + "Zip", ], path: "Sources/EmceeLib" ), @@ -774,10 +779,10 @@ let package = Package( "DistWorkerModels", "EmceeLogging", "FileLock", + .product(name: "FileSystem", package: "CommandLineToolkit"), "LocalHostDeterminer", "LoggingSetup", "MetricsExtensions", - .product(name: "ProcessController", package: "CommandLineToolkit"), "QueueCommunication", "QueueModels", "QueueServer", @@ -787,6 +792,7 @@ let package = Package( .product(name: "SynchronousWaiter", package: "CommandLineToolkit"), .product(name: "Tmp", package: "CommandLineToolkit"), "UniqueIdentifierGenerator", + "Zip", ], path: "Sources/LocalQueueServerRunner" ), @@ -1262,6 +1268,7 @@ let package = Package( dependencies: [ "AutomaticTermination", "EmceeLogging", + .product(name: "PathLib", package: "CommandLineToolkit"), "QueueModels", "RESTInterfaces", "RESTMethods", @@ -1559,11 +1566,12 @@ let package = Package( dependencies: [ "Deployer", "EmceeLogging", + .product(name: "FileSystem", package: "CommandLineToolkit"), .product(name: "PathLib", package: "CommandLineToolkit"), - .product(name: "ProcessController", package: "CommandLineToolkit"), .product(name: "Shout", package: "Shout"), .product(name: "Tmp", package: "CommandLineToolkit"), "UniqueIdentifierGenerator", + "Zip", ], path: "Sources/SSHDeployer" ), @@ -1571,12 +1579,13 @@ let package = Package( name: "SSHDeployerTests", dependencies: [ "Deployer", + .product(name: "FileSystemTestHelpers", package: "CommandLineToolkit"), .product(name: "PathLib", package: "CommandLineToolkit"), - .product(name: "ProcessControllerTestHelpers", package: "CommandLineToolkit"), "SSHDeployer", .product(name: "TestHelpers", package: "CommandLineToolkit"), .product(name: "Tmp", package: "CommandLineToolkit"), "UniqueIdentifierGeneratorTestHelpers", + "ZipTestHelpers", ], path: "Tests/SSHDeployerTests" ), @@ -2072,5 +2081,32 @@ let package = Package( ], path: "Sources/XcodebuildTestRunnerConstants" ), + .target( + name: "Zip", + dependencies: [ + .product(name: "PathLib", package: "CommandLineToolkit"), + .product(name: "ProcessController", package: "CommandLineToolkit"), + ], + path: "Sources/Zip" + ), + .target( + name: "ZipTestHelpers", + dependencies: [ + .product(name: "PathLib", package: "CommandLineToolkit"), + .product(name: "TestHelpers", package: "CommandLineToolkit"), + "Zip", + ], + path: "Tests/ZipTestHelpers" + ), + .testTarget( + name: "ZipTests", + dependencies: [ + .product(name: "PathLib", package: "CommandLineToolkit"), + .product(name: "ProcessControllerTestHelpers", package: "CommandLineToolkit"), + .product(name: "TestHelpers", package: "CommandLineToolkit"), + "Zip", + ], + path: "Tests/ZipTests" + ), ] ) diff --git a/Sources/Deployer/Deployer.swift b/Sources/Deployer/Deployer.swift index d6d4a9df9..ee850b1f0 100644 --- a/Sources/Deployer/Deployer.swift +++ b/Sources/Deployer/Deployer.swift @@ -1,9 +1,10 @@ +import FileSystem import Foundation import EmceeLogging import PathLib -import ProcessController import Tmp import UniqueIdentifierGenerator +import Zip /** Basic class that defines a logic for deploying a number of DeployableItems. */ open class Deployer { @@ -11,29 +12,32 @@ open class Deployer { public let deployables: [DeployableItem] public let deployableCommands: [DeployableCommand] public let destination: DeploymentDestination + private let fileSystem: FileSystem private let logger: ContextualLogger - private let processControllerProvider: ProcessControllerProvider private let temporaryFolder: TemporaryFolder private let uniqueIdentifierGenerator: UniqueIdentifierGenerator + private let zipCompressor: ZipCompressor public init( deploymentId: String, deployables: [DeployableItem], deployableCommands: [DeployableCommand], destination: DeploymentDestination, + fileSystem: FileSystem, logger: ContextualLogger, - processControllerProvider: ProcessControllerProvider, temporaryFolder: TemporaryFolder, - uniqueIdentifierGenerator: UniqueIdentifierGenerator + uniqueIdentifierGenerator: UniqueIdentifierGenerator, + zipCompressor: ZipCompressor ) throws { self.deploymentId = deploymentId self.deployables = deployables self.deployableCommands = deployableCommands self.destination = destination + self.fileSystem = fileSystem self.logger = logger - self.processControllerProvider = processControllerProvider self.temporaryFolder = temporaryFolder self.uniqueIdentifierGenerator = uniqueIdentifierGenerator + self.zipCompressor = zipCompressor } /** Deploys all the deployable items and invokes deployment commands. */ @@ -51,7 +55,10 @@ open class Deployer { let syncQueue = DispatchQueue(label: "Deployer.syncQueue") var deployablesFailedToPrepare = [DeployableItem]() var pathToDeployable = [AbsolutePath: DeployableItem]() - let packager = Packager(processControllerProvider: processControllerProvider) + let packager = Packager( + fileSystem: fileSystem, + zipCompressor: zipCompressor + ) let queue = DispatchQueue( label: "Deployer.queue", diff --git a/Sources/Deployer/Packager.swift b/Sources/Deployer/Packager.swift index 19e2b809d..f184ea7d3 100644 --- a/Sources/Deployer/Packager.swift +++ b/Sources/Deployer/Packager.swift @@ -1,15 +1,19 @@ import Foundation +import FileSystem import PathLib -import ProcessController -import Tmp +import Zip /** Packs DeployableItem, returns URL to a single file with a package. */ public final class Packager { - private let fileManager = FileManager() - private let processControllerProvider: ProcessControllerProvider + private let fileSystem: FileSystem + private let zipCompressor: ZipCompressor - public init(processControllerProvider: ProcessControllerProvider) { - self.processControllerProvider = processControllerProvider + public init( + fileSystem: FileSystem, + zipCompressor: ZipCompressor + ) { + self.fileSystem = fileSystem + self.zipCompressor = zipCompressor } /** @@ -17,37 +21,32 @@ public final class Packager { * If the DeployableItem has been already packed, it will return URL without re-packing it. */ public func preparePackage(deployable: DeployableItem, packageFolder: AbsolutePath) throws -> AbsolutePath { - let archivePath = deployable.name.components(separatedBy: "/").reduce(packageFolder) { $0.appending($1) } - try fileManager.createDirectory(atPath: archivePath.removingLastComponent) + let archivePath = deployable.name.components(separatedBy: "/") + .reduce(packageFolder) { $0.appending($1) } - if fileManager.fileExists(atPath: archivePath.pathString) { + try fileSystem.createDirectory( + path: archivePath.removingLastComponent, + withIntermediateDirectories: true, + ignoreExisting: true + ) + + if fileSystem.exists(path: archivePath) { return archivePath } - let temporaryFolder = try TemporaryFolder() - for file in deployable.files { - let containerPath = file.destination.removingLastComponent - if !fileManager.fileExists(atPath: containerPath.pathString) { - _ = try temporaryFolder.createDirectory(components: containerPath.components) - } - try fileManager.copyItem( - atPath: file.source.pathString, - toPath: temporaryFolder.absolutePath.appending(relativePath: file.destination).pathString + try fileSystem.copy( + source: file.source, + destination: packageFolder.appending(relativePath: file.destination), + overwrite: false, + ensureDirectoryExists: true ) } - let controller = try processControllerProvider.createProcessController( - subprocess: Subprocess( - arguments: ["/usr/bin/zip", archivePath.pathString, "-r", "."], - workingDirectory: temporaryFolder.absolutePath - ) + return try zipCompressor.createArchive( + archivePath: archivePath, + workingDirectory: packageFolder, + contentsToCompress: "." ) - try controller.startAndListenUntilProcessDies() - if archivePath.extension.isEmpty { - return archivePath.appending(extension: "zip") - } else { - return archivePath - } } } diff --git a/Sources/DistDeployer/DistDeployer.swift b/Sources/DistDeployer/DistDeployer.swift index ab658a87f..fdcdcff4c 100644 --- a/Sources/DistDeployer/DistDeployer.swift +++ b/Sources/DistDeployer/DistDeployer.swift @@ -1,42 +1,45 @@ import Deployer import EmceeLogging +import FileSystem import Foundation -import ProcessController import SSHDeployer import Tmp import UniqueIdentifierGenerator +import Zip /// Class for generic usage: it deploys the provided deployable items to the provided deployment destinations, and /// invokes the provided deployable commands. final class DistDeployer { - private let deploymentId: String private let deploymentDestination: DeploymentDestination private let deployableItems: [DeployableItem] private let deployableCommands: [DeployableCommand] + private let fileSystem: FileSystem private let logger: ContextualLogger - private let processControllerProvider: ProcessControllerProvider private let tempFolder: TemporaryFolder private let uniqueIdentifierGenerator: UniqueIdentifierGenerator + private let zipCompressor: ZipCompressor public init( deploymentId: String, deploymentDestination: DeploymentDestination, deployableItems: [DeployableItem], deployableCommands: [DeployableCommand], + fileSystem: FileSystem, logger: ContextualLogger, - processControllerProvider: ProcessControllerProvider, tempFolder: TemporaryFolder, - uniqueIdentifierGenerator: UniqueIdentifierGenerator + uniqueIdentifierGenerator: UniqueIdentifierGenerator, + zipCompressor: ZipCompressor ) { self.deploymentId = deploymentId self.deploymentDestination = deploymentDestination self.deployableItems = deployableItems self.deployableCommands = deployableCommands + self.fileSystem = fileSystem self.logger = logger - self.processControllerProvider = processControllerProvider self.tempFolder = tempFolder self.uniqueIdentifierGenerator = uniqueIdentifierGenerator + self.zipCompressor = zipCompressor } public func deploy() throws { @@ -46,10 +49,11 @@ final class DistDeployer { deployables: deployableItems, deployableCommands: deployableCommands, destination: deploymentDestination, + fileSystem: fileSystem, logger: logger, - processControllerProvider: processControllerProvider, temporaryFolder: tempFolder, - uniqueIdentifierGenerator: uniqueIdentifierGenerator + uniqueIdentifierGenerator: uniqueIdentifierGenerator, + zipCompressor: zipCompressor ) try deployer.deploy() } diff --git a/Sources/DistDeployer/RemoteQueueStarter.swift b/Sources/DistDeployer/RemoteQueueStarter.swift index 08ec7d1fc..97d10b905 100644 --- a/Sources/DistDeployer/RemoteQueueStarter.swift +++ b/Sources/DistDeployer/RemoteQueueStarter.swift @@ -1,40 +1,44 @@ import Deployer import EmceeLogging +import FileSystem import Foundation import PathLib -import ProcessController import QueueModels import Tmp import UniqueIdentifierGenerator +import Zip public final class RemoteQueueStarter { private let deploymentId: String private let deploymentDestination: DeploymentDestination private let emceeVersion: Version + private let fileSystem: FileSystem private let logger: ContextualLogger - private let processControllerProvider: ProcessControllerProvider private let queueServerConfigurationLocation: QueueServerConfigurationLocation private let tempFolder: TemporaryFolder private let uniqueIdentifierGenerator: UniqueIdentifierGenerator + private let zipCompressor: ZipCompressor public init( deploymentId: String, deploymentDestination: DeploymentDestination, emceeVersion: Version, + fileSystem: FileSystem, logger: ContextualLogger, - processControllerProvider: ProcessControllerProvider, queueServerConfigurationLocation: QueueServerConfigurationLocation, tempFolder: TemporaryFolder, - uniqueIdentifierGenerator: UniqueIdentifierGenerator + uniqueIdentifierGenerator: UniqueIdentifierGenerator, + zipCompressor: ZipCompressor ) { self.deploymentId = deploymentId self.deploymentDestination = deploymentDestination self.emceeVersion = emceeVersion + self.fileSystem = fileSystem self.logger = logger - self.processControllerProvider = processControllerProvider self.queueServerConfigurationLocation = queueServerConfigurationLocation self.tempFolder = tempFolder self.uniqueIdentifierGenerator = uniqueIdentifierGenerator + self.zipCompressor = zipCompressor } public func deployAndStart() throws { @@ -85,10 +89,11 @@ public final class RemoteQueueStarter { launchctlDeployableCommands.forceUnloadFromBackgroundCommand(), launchctlDeployableCommands.forceLoadInBackgroundCommand() ], + fileSystem: fileSystem, logger: logger, - processControllerProvider: processControllerProvider, tempFolder: tempFolder, - uniqueIdentifierGenerator: uniqueIdentifierGenerator + uniqueIdentifierGenerator: uniqueIdentifierGenerator, + zipCompressor: zipCompressor ) try deployer.deploy() } diff --git a/Sources/DistDeployer/RemoteWorkersStarter/DefaultRemoteWorkersStarter.swift b/Sources/DistDeployer/RemoteWorkersStarter/DefaultRemoteWorkersStarter.swift index 20ac4a1d6..9bdfa6b09 100644 --- a/Sources/DistDeployer/RemoteWorkersStarter/DefaultRemoteWorkersStarter.swift +++ b/Sources/DistDeployer/RemoteWorkersStarter/DefaultRemoteWorkersStarter.swift @@ -1,35 +1,39 @@ import Deployer +import FileSystem import Foundation import EmceeLogging import PathLib -import ProcessController import QueueModels import SocketModels import Tmp import UniqueIdentifierGenerator +import Zip public final class DefaultRemoteWorkersStarter: RemoteWorkerStarter { private let deploymentDestination: DeploymentDestination private let emceeVersion: Version + private let fileSystem: FileSystem private let logger: ContextualLogger - private let processControllerProvider: ProcessControllerProvider private let tempFolder: TemporaryFolder private let uniqueIdentifierGenerator: UniqueIdentifierGenerator + private let zipCompressor: ZipCompressor public init( deploymentDestination: DeploymentDestination, emceeVersion: Version, + fileSystem: FileSystem, logger: ContextualLogger, - processControllerProvider: ProcessControllerProvider, tempFolder: TemporaryFolder, - uniqueIdentifierGenerator: UniqueIdentifierGenerator + uniqueIdentifierGenerator: UniqueIdentifierGenerator, + zipCompressor: ZipCompressor ) { self.deploymentDestination = deploymentDestination self.emceeVersion = emceeVersion + self.fileSystem = fileSystem self.logger = logger - self.processControllerProvider = processControllerProvider self.tempFolder = tempFolder self.uniqueIdentifierGenerator = uniqueIdentifierGenerator + self.zipCompressor = zipCompressor } public func deployAndStartWorker( @@ -82,10 +86,11 @@ public final class DefaultRemoteWorkersStarter: RemoteWorkerStarter { ], launchctlDeployableCommands.forceLoadInBackgroundCommand() ], + fileSystem: fileSystem, logger: logger, - processControllerProvider: processControllerProvider, tempFolder: tempFolder, - uniqueIdentifierGenerator: uniqueIdentifierGenerator + uniqueIdentifierGenerator: uniqueIdentifierGenerator, + zipCompressor: zipCompressor ) try deployer.deploy() diff --git a/Sources/DistWorker/DistWorker.swift b/Sources/DistWorker/DistWorker.swift index eeacb400e..36345c4b5 100644 --- a/Sources/DistWorker/DistWorker.swift +++ b/Sources/DistWorker/DistWorker.swift @@ -63,7 +63,7 @@ public final class DistWorker: SchedulerDataSource, SchedulerDelegate { self.httpRestServer = HTTPRESTServer( automaticTerminationController: StayAliveTerminationController(), logger: logger, - portProvider: PortProviderWrapper(provider: { 0 }) + portProvider: AnyAvailablePortProvider() ) self.version = version self.workerId = workerId diff --git a/Sources/EmceeLib/Arguments/ArgumentDescriptions.swift b/Sources/EmceeLib/Arguments/ArgumentDescriptions.swift index 62387f7c2..4a36383a8 100644 --- a/Sources/EmceeLib/Arguments/ArgumentDescriptions.swift +++ b/Sources/EmceeLib/Arguments/ArgumentDescriptions.swift @@ -6,11 +6,11 @@ final class ArgumentDescriptions { static let junit = doubleDashedDescription(dashlessName: "junit", overview: "Path where the combined (for all test destinations) Junit report file should be created") static let output = doubleDashedDescription(dashlessName: "output", overview: "Path to file where to store the output") static let queueServer = doubleDashedDescription(dashlessName: "queue-server", overview: "An address to a server which runs job queues, e.g. 127.0.0.1:1234") - static let queueServerConfigurationLocation = doubleDashedDescription(dashlessName: "queue-server-configuration-location", overview: "JSON file location which describes QueueServerConfiguration, e.g. http://example.com/file.zip#path/to/config.json") + static let queueServerConfigurationLocation = doubleDashedDescription(dashlessName: "queue-server-configuration-location", overview: "JSON file location which describes QueueServerConfiguration, local path or URL. See: https://github.com/avito-tech/Emcee/wiki/URL-Handling") static let remoteCacheConfig = doubleDashedDescription(dashlessName: "remote-cache-config", overview: "JSON file with remote server settings") static let setFeatureStatus = doubleDashedDescription(dashlessName: "set-feature-status", overview: "Enabled/Disabled") static let tempFolder = doubleDashedDescription(dashlessName: "temp-folder", overview: "Where to store temporary stuff, including simulator data") - static let testArgFile = doubleDashedDescription(dashlessName: "test-arg-file", overview: "JSON file with test plan") + static let testArgFile = doubleDashedDescription(dashlessName: "test-arg-file", overview: "JSON file with test plan. See: https://github.com/avito-tech/Emcee/wiki/Test-Arg-File") static let trace = doubleDashedDescription(dashlessName: "trace", overview: "Path where the combined (for all test destinations) Chrome trace file should be created") static let workerId = doubleDashedDescription(dashlessName: "worker-id", overview: "An identifier used to distinguish between workers. Useful to match with deployment destination's identifier") diff --git a/Sources/EmceeLib/Commands/RunTestsOnRemoteQueueCommand.swift b/Sources/EmceeLib/Commands/RunTestsOnRemoteQueueCommand.swift index 8a4f39146..90d297302 100644 --- a/Sources/EmceeLib/Commands/RunTestsOnRemoteQueueCommand.swift +++ b/Sources/EmceeLib/Commands/RunTestsOnRemoteQueueCommand.swift @@ -1,4 +1,5 @@ import ArgLib +import AutomaticTermination import BucketQueue import EmceeDI import DateProvider @@ -20,6 +21,7 @@ import QueueClient import QueueCommunication import QueueModels import QueueServer +import RESTServer import RemotePortDeterminer import RequestSender import ResourceLocationResolver @@ -48,26 +50,68 @@ public final class RunTestsOnRemoteQueueCommand: Command { private let callbackQueue = DispatchQueue(label: "RunTestsOnRemoteQueueCommand.callbackQueue") private let di: DI + private let httpRestServer: HTTPRESTServer public init(di: DI) throws { self.di = di + self.httpRestServer = HTTPRESTServer( + automaticTerminationController: StayAliveTerminationController(), + logger: try di.get(), + portProvider: AnyAvailablePortProvider() + ) } public func run(payload: CommandPayload) throws { + try httpRestServer.start() + let commonReportOutput = ReportOutput( junit: try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.junit.name), tracingReport: try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.trace.name) ) - let queueServerConfigurationLocation: QueueServerConfigurationLocation = try payload.expectedSingleTypedValue(argumentName: ArgumentDescriptions.queueServerConfigurationLocation.name) - let queueServerConfiguration = try ArgumentsReader.queueServerConfiguration( - location: queueServerConfigurationLocation, - resourceLocationResolver: try di.get() - ) - let emceeVersion: Version = try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.emceeVersion.name) ?? EmceeVersion.version let tempFolder = try TemporaryFolder(containerPath: try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.tempFolder.name)) - let testArgFile = try ArgumentsReader.testArgFile(try payload.expectedSingleTypedValue(argumentName: ArgumentDescriptions.testArgFile.name)) + let logger = try di.get(ContextualLogger.self) + + let remoteCacheConfig = try ArgumentsReader.remoteCacheConfig( + try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.remoteCacheConfig.name) + ) + + di.set(tempFolder, for: TemporaryFolder.self) + + di.set( + SwifterRemotelyAccessibleUrlForLocalFileProvider( + server: httpRestServer, + serverRoot: "/build-artifacts/", + uniqueIdentifierGenerator: try di.get() + ), + for: RemotelyAccessibleUrlForLocalFileProvider.self + ) + + let localTypedResourceLocationPreparer = LocalTypedResourceLocationPreparerImpl( + logger: try di.get(), + pathForStoringArchives: try tempFolder.createDirectory( + components: ["job_prepatation"] + ), + remotelyAccessibleUrlForLocalFileProvider: try di.get(), + uniqueIdentifierGenerator: try di.get(), + zipCompressor: try di.get() + ) + di.set(localTypedResourceLocationPreparer, for: LocalTypedResourceLocationPreparer.self) + + let buildArtifactsPreparer = BuildArtifactsPreparerImpl( + localTypedResourceLocationPreparer: try di.get(), + logger: try di.get() + ) + di.set(buildArtifactsPreparer, for: BuildArtifactsPreparer.self) + + let testArgFile = try preprocessTestArgFile( + testArgFile: try ArgumentsReader.testArgFile( + try payload.expectedSingleTypedValue(argumentName: ArgumentDescriptions.testArgFile.name) + ), + buildArtifactsPreparer: buildArtifactsPreparer, + logger: logger + ) if let kibanaConfiguration = testArgFile.prioritizedJob.analyticsConfiguration.kibanaConfiguration { try di.get(LoggingSetup.self).set(kibanaConfiguration: kibanaConfiguration) @@ -80,13 +124,16 @@ public final class RunTestsOnRemoteQueueCommand: Command { analyticsConfiguration: testArgFile.prioritizedJob.analyticsConfiguration ) ) - let logger = try di.get(ContextualLogger.self) - - let remoteCacheConfig = try ArgumentsReader.remoteCacheConfig( - try payload.optionalSingleTypedValue(argumentName: ArgumentDescriptions.remoteCacheConfig.name) - ) - di.set(tempFolder, for: TemporaryFolder.self) + let queueServerConfigurationLocation: QueueServerConfigurationLocation = try localTypedResourceLocationPreparer.generateRemotelyAccessibleTypedResourceLocation( + try payload.expectedSingleTypedValue( + argumentName: ArgumentDescriptions.queueServerConfigurationLocation.name + ) + ) + let queueServerConfiguration = try ArgumentsReader.queueServerConfiguration( + location: queueServerConfigurationLocation, + resourceLocationResolver: try di.get() + ) let runningQueueServerAddress = try detectRemotelyRunningQueueServerPortsOrStartRemoteQueueIfNeeded( emceeVersion: emceeVersion, @@ -98,6 +145,7 @@ public final class RunTestsOnRemoteQueueCommand: Command { let jobResults = try runTestsOnRemotelyRunningQueue( queueServerAddress: runningQueueServerAddress, remoteCacheConfig: remoteCacheConfig, + tempFolder: tempFolder, testArgFile: testArgFile, version: emceeVersion, logger: logger @@ -111,6 +159,27 @@ public final class RunTestsOnRemoteQueueCommand: Command { try resultOutputGenerator.generateOutput() } + private func preprocessTestArgFile( + testArgFile: TestArgFile, + buildArtifactsPreparer: BuildArtifactsPreparer, + logger: ContextualLogger + ) throws -> TestArgFile { + logger.info("Preparing build artifacts to be accessible by workers...") + defer { + logger.info("Build artifacts are now accessible by workers") + } + + return testArgFile.with( + entries: try testArgFile.entries.map { testArgFileEntry in + testArgFileEntry.with( + buildArtifacts: try buildArtifactsPreparer.prepare( + buildArtifacts: testArgFileEntry.buildArtifacts + ) + ) + } + ) + } + private func detectRemotelyRunningQueueServerPortsOrStartRemoteQueueIfNeeded( emceeVersion: Version, queueServerDeploymentDestinations: [DeploymentDestination], @@ -169,11 +238,12 @@ public final class RunTestsOnRemoteQueueCommand: Command { deploymentId: jobId.value, deploymentDestination: queueServerDeploymentDestination, emceeVersion: emceeVersion, + fileSystem: try di.get(), logger: logger, - processControllerProvider: try di.get(), queueServerConfigurationLocation: queueServerConfigurationLocation, tempFolder: try di.get(), - uniqueIdentifierGenerator: try di.get() + uniqueIdentifierGenerator: try di.get(), + zipCompressor: try di.get() ) try remoteQueueStarter.deployAndStart() logger.debug("Started queue on \(queueServerDeploymentDestination.host)") @@ -190,6 +260,7 @@ public final class RunTestsOnRemoteQueueCommand: Command { private func runTestsOnRemotelyRunningQueue( queueServerAddress: SocketAddress, remoteCacheConfig: RuntimeDumpRemoteCacheConfig?, + tempFolder: TemporaryFolder, testArgFile: TestArgFile, version: Version, logger: ContextualLogger @@ -240,7 +311,6 @@ public final class RunTestsOnRemoteQueueCommand: Command { ), for: JobDeleter.self ) - defer { deleteJob(jobId: testArgFile.prioritizedJob.jobId, logger: logger) } diff --git a/Sources/EmceeLib/Commands/StartQueueServerCommand.swift b/Sources/EmceeLib/Commands/StartQueueServerCommand.swift index 04b7f2a48..173764ee2 100644 --- a/Sources/EmceeLib/Commands/StartQueueServerCommand.swift +++ b/Sources/EmceeLib/Commands/StartQueueServerCommand.swift @@ -136,11 +136,12 @@ public final class StartQueueServerCommand: Command { let remoteWorkerStarterProvider = DefaultRemoteWorkerStarterProvider( emceeVersion: emceeVersion, + fileSystem: try di.get(), logger: logger, - processControllerProvider: try di.get(), tempFolder: try TemporaryFolder(), uniqueIdentifierGenerator: try di.get(), - workerDeploymentDestinations: workerDestinations + workerDeploymentDestinations: workerDestinations, + zipCompressor: try di.get() ) let workerConfigurations = try createWorkerConfigurations( queueServerConfiguration: queueServerConfiguration diff --git a/Sources/EmceeLib/InProcessMain.swift b/Sources/EmceeLib/InProcessMain.swift index 91da097bf..702f3b74e 100644 --- a/Sources/EmceeLib/InProcessMain.swift +++ b/Sources/EmceeLib/InProcessMain.swift @@ -25,6 +25,7 @@ import SynchronousWaiter import TestDiscovery import URLResource import UniqueIdentifierGenerator +import Zip public final class InProcessMain { public init() {} @@ -193,6 +194,12 @@ public final class InProcessMain { SynchronousWaiter(), for: Waiter.self ) + di.set( + ZipCompressorImpl( + processControllerProvider: try di.get() + ), + for: ZipCompressor.self + ) let commandInvoker = CommandInvoker( commands: [ diff --git a/Sources/EmceeLib/Utils/JobPreparer/BuildArtifactsPreparer.swift b/Sources/EmceeLib/Utils/JobPreparer/BuildArtifactsPreparer.swift new file mode 100644 index 000000000..be1212f40 --- /dev/null +++ b/Sources/EmceeLib/Utils/JobPreparer/BuildArtifactsPreparer.swift @@ -0,0 +1,6 @@ +import BuildArtifacts +import Foundation + +public protocol BuildArtifactsPreparer { + func prepare(buildArtifacts: BuildArtifacts) throws -> BuildArtifacts +} diff --git a/Sources/EmceeLib/Utils/JobPreparer/BuildArtifactsPreparerImpl.swift b/Sources/EmceeLib/Utils/JobPreparer/BuildArtifactsPreparerImpl.swift new file mode 100644 index 000000000..b963beede --- /dev/null +++ b/Sources/EmceeLib/Utils/JobPreparer/BuildArtifactsPreparerImpl.swift @@ -0,0 +1,53 @@ +import BuildArtifacts +import EmceeLogging +import Foundation +import PathLib +import ResourceLocation +import TypedResourceLocation + +public final class BuildArtifactsPreparerImpl: BuildArtifactsPreparer { + private let localTypedResourceLocationPreparer: LocalTypedResourceLocationPreparer + private let logger: ContextualLogger + + public init( + localTypedResourceLocationPreparer: LocalTypedResourceLocationPreparer, + logger: ContextualLogger + ) { + self.localTypedResourceLocationPreparer = localTypedResourceLocationPreparer + self.logger = logger + } + + public func prepare(buildArtifacts: BuildArtifacts) throws -> BuildArtifacts { + try remotelyAccessibleBuildArtifacts(buildArtifacts: buildArtifacts) + } + + private func remotelyAccessibleBuildArtifacts( + buildArtifacts: BuildArtifacts + ) throws -> BuildArtifacts { + let remotelyAccessibleTestBundle = XcTestBundle( + location: try localTypedResourceLocationPreparer.generateRemotelyAccessibleTypedResourceLocation(buildArtifacts.xcTestBundle.location), + testDiscoveryMode: buildArtifacts.xcTestBundle.testDiscoveryMode + ) + + switch buildArtifacts { + case .iosLogicTests: + return .iosLogicTests( + xcTestBundle: remotelyAccessibleTestBundle + ) + case .iosApplicationTests(_, let appBundle): + return .iosApplicationTests( + xcTestBundle: remotelyAccessibleTestBundle, + appBundle: try localTypedResourceLocationPreparer.generateRemotelyAccessibleTypedResourceLocation(appBundle) + ) + case .iosUiTests(_, let appBundle, let runner, let additionalApplicationBundles): + return .iosUiTests( + xcTestBundle: remotelyAccessibleTestBundle, + appBundle: try localTypedResourceLocationPreparer.generateRemotelyAccessibleTypedResourceLocation(appBundle), + runner: try localTypedResourceLocationPreparer.generateRemotelyAccessibleTypedResourceLocation(runner), + additionalApplicationBundles: try additionalApplicationBundles.map { + try localTypedResourceLocationPreparer.generateRemotelyAccessibleTypedResourceLocation($0) + } + ) + } + } +} diff --git a/Sources/EmceeLib/Utils/JobPreparer/LocalTypedResourceLocationPreparer.swift b/Sources/EmceeLib/Utils/JobPreparer/LocalTypedResourceLocationPreparer.swift new file mode 100644 index 000000000..06b9ceba8 --- /dev/null +++ b/Sources/EmceeLib/Utils/JobPreparer/LocalTypedResourceLocationPreparer.swift @@ -0,0 +1,10 @@ +import Foundation +import TypedResourceLocation + +public protocol LocalTypedResourceLocationPreparer { + + /// If provided `TypedResourceLocation` refers to a local file, it will be translated into a remotely accessible resource location. + func generateRemotelyAccessibleTypedResourceLocation( + _ from: TypedResourceLocation + ) throws -> TypedResourceLocation +} diff --git a/Sources/EmceeLib/Utils/JobPreparer/LocalTypedResourceLocationPreparerImpl.swift b/Sources/EmceeLib/Utils/JobPreparer/LocalTypedResourceLocationPreparerImpl.swift new file mode 100644 index 000000000..4f7b23730 --- /dev/null +++ b/Sources/EmceeLib/Utils/JobPreparer/LocalTypedResourceLocationPreparerImpl.swift @@ -0,0 +1,56 @@ +import EmceeLogging +import Foundation +import TypedResourceLocation +import PathLib +import UniqueIdentifierGenerator +import Zip + +public final class LocalTypedResourceLocationPreparerImpl: LocalTypedResourceLocationPreparer { + private let logger: ContextualLogger + private let pathForStoringArchives: AbsolutePath + private let remotelyAccessibleUrlForLocalFileProvider: RemotelyAccessibleUrlForLocalFileProvider + private let zipCompressor: ZipCompressor + private let uniqueIdentifierGenerator: UniqueIdentifierGenerator + + public init( + logger: ContextualLogger, + pathForStoringArchives: AbsolutePath, + remotelyAccessibleUrlForLocalFileProvider: RemotelyAccessibleUrlForLocalFileProvider, + uniqueIdentifierGenerator: UniqueIdentifierGenerator, + zipCompressor: ZipCompressor + ) { + self.logger = logger + self.pathForStoringArchives = pathForStoringArchives + self.remotelyAccessibleUrlForLocalFileProvider = remotelyAccessibleUrlForLocalFileProvider + self.uniqueIdentifierGenerator = uniqueIdentifierGenerator + self.zipCompressor = zipCompressor + } + + public func generateRemotelyAccessibleTypedResourceLocation( + _ from: TypedResourceLocation + ) throws -> TypedResourceLocation { + return try TypedResourceLocation( + from.resourceLocation.mapLocalFile { value in + let localPath = try AbsolutePath.validating(string: value) + logger.debug("Preparing local file at \(value) to be accessible remotely") + + let archivePath = try zipCompressor.createArchive( + archivePath: pathForStoringArchives + .appending(uniqueIdentifierGenerator.generate()) + .appending(extension: "zip"), + workingDirectory: localPath.removingLastComponent, + contentsToCompress: RelativePath(localPath.lastComponent) + ) + + logger.debug("Generated archive at \(archivePath)") + let url = try remotelyAccessibleUrlForLocalFileProvider.remotelyAccessibleUrlForLocalFile( + archivePath: archivePath, + inArchivePath: RelativePath(localPath.lastComponent) + ) + + logger.debug("Archive should be accessible via URL: \(url)") + return .remoteUrl(url, nil) + } + ) + } +} diff --git a/Sources/EmceeLib/Utils/JobPreparer/RemotelyAccessibleUrlForLocalFileProvider.swift b/Sources/EmceeLib/Utils/JobPreparer/RemotelyAccessibleUrlForLocalFileProvider.swift new file mode 100644 index 000000000..ad9366586 --- /dev/null +++ b/Sources/EmceeLib/Utils/JobPreparer/RemotelyAccessibleUrlForLocalFileProvider.swift @@ -0,0 +1,9 @@ +import Foundation +import PathLib + +public protocol RemotelyAccessibleUrlForLocalFileProvider { + func remotelyAccessibleUrlForLocalFile( + archivePath: AbsolutePath, + inArchivePath: RelativePath + ) throws -> URL +} diff --git a/Sources/EmceeLib/Utils/JobPreparer/SwifterRemotelyAccessibleUrlForLocalFileProvider.swift b/Sources/EmceeLib/Utils/JobPreparer/SwifterRemotelyAccessibleUrlForLocalFileProvider.swift new file mode 100644 index 000000000..59575cc38 --- /dev/null +++ b/Sources/EmceeLib/Utils/JobPreparer/SwifterRemotelyAccessibleUrlForLocalFileProvider.swift @@ -0,0 +1,55 @@ +import Foundation +import LocalHostDeterminer +import PathLib +import RESTServer +import Swifter +import UniqueIdentifierGenerator + +public final class SwifterRemotelyAccessibleUrlForLocalFileProvider: RemotelyAccessibleUrlForLocalFileProvider { + private let server: HTTPRESTServer + private let serverRoot: AbsolutePath + private let uniqueIdentifierGenerator: UniqueIdentifierGenerator + + public init( + server: HTTPRESTServer, + serverRoot: AbsolutePath, + uniqueIdentifierGenerator: UniqueIdentifierGenerator + ) { + self.server = server + self.serverRoot = serverRoot + self.uniqueIdentifierGenerator = uniqueIdentifierGenerator + } + + public struct NoUrlError: Error, CustomStringConvertible { + public let components: URLComponents + public var description: String { + "Failed to generate URL from components \(components)" + } + } + + public func remotelyAccessibleUrlForLocalFile( + archivePath: AbsolutePath, + inArchivePath: RelativePath + ) throws -> URL { + let path = serverRoot.appending( + uniqueIdentifierGenerator.generate(), + archivePath.lastComponent + ) + + server.add( + requestPath: path, + localFilePath: archivePath + ) + + var urlComponents = URLComponents() + urlComponents.scheme = "http" + urlComponents.host = LocalHostDeterminer.currentHostAddress + urlComponents.port = try server.port().value + urlComponents.path = path.pathString + urlComponents.fragment = inArchivePath.pathString + guard let result = urlComponents.url else { + throw NoUrlError(components: urlComponents) + } + return result + } +} diff --git a/Sources/LocalQueueServerRunner/RemoteWorkersStarterProvider/DefaultRemoteWorkerStarterProvider.swift b/Sources/LocalQueueServerRunner/RemoteWorkersStarterProvider/DefaultRemoteWorkerStarterProvider.swift index e03f50c13..6c417c613 100644 --- a/Sources/LocalQueueServerRunner/RemoteWorkersStarterProvider/DefaultRemoteWorkerStarterProvider.swift +++ b/Sources/LocalQueueServerRunner/RemoteWorkersStarterProvider/DefaultRemoteWorkerStarterProvider.swift @@ -1,35 +1,39 @@ import Deployer import DistDeployer import EmceeLogging +import FileSystem import Foundation -import ProcessController import QueueModels import SocketModels import Tmp import UniqueIdentifierGenerator +import Zip public final class DefaultRemoteWorkerStarterProvider: RemoteWorkerStarterProvider { private let emceeVersion: Version + private let fileSystem: FileSystem private let logger: ContextualLogger - private let processControllerProvider: ProcessControllerProvider private let tempFolder: TemporaryFolder private let uniqueIdentifierGenerator: UniqueIdentifierGenerator private let workerDeploymentDestinations: [DeploymentDestination] + private let zipCompressor: ZipCompressor public init( emceeVersion: Version, + fileSystem: FileSystem, logger: ContextualLogger, - processControllerProvider: ProcessControllerProvider, tempFolder: TemporaryFolder, uniqueIdentifierGenerator: UniqueIdentifierGenerator, - workerDeploymentDestinations: [DeploymentDestination] + workerDeploymentDestinations: [DeploymentDestination], + zipCompressor: ZipCompressor ) { self.emceeVersion = emceeVersion + self.fileSystem = fileSystem self.logger = logger - self.processControllerProvider = processControllerProvider self.tempFolder = tempFolder self.uniqueIdentifierGenerator = uniqueIdentifierGenerator self.workerDeploymentDestinations = workerDeploymentDestinations + self.zipCompressor = zipCompressor } public enum DefaultRemoteWorkerStarterProviderError: Error, CustomStringConvertible { @@ -53,10 +57,11 @@ public final class DefaultRemoteWorkerStarterProvider: RemoteWorkerStarterProvid return DefaultRemoteWorkersStarter( deploymentDestination: deploymentDestination, emceeVersion: emceeVersion, + fileSystem: fileSystem, logger: logger, - processControllerProvider: processControllerProvider, tempFolder: tempFolder, - uniqueIdentifierGenerator: uniqueIdentifierGenerator + uniqueIdentifierGenerator: uniqueIdentifierGenerator, + zipCompressor: zipCompressor ) } } diff --git a/Sources/RESTServer/AnyAvailablePortProvider.swift b/Sources/RESTServer/AnyAvailablePortProvider.swift new file mode 100644 index 000000000..80fc2a857 --- /dev/null +++ b/Sources/RESTServer/AnyAvailablePortProvider.swift @@ -0,0 +1,10 @@ +import Foundation +import SocketModels + +public final class AnyAvailablePortProvider: PortProvider { + public init() {} + + public func localPort() throws -> SocketModels.Port { + 0 + } +} diff --git a/Sources/RESTServer/HTTPRESTServer.swift b/Sources/RESTServer/HTTPRESTServer.swift index fe22ecac3..7665fafee 100644 --- a/Sources/RESTServer/HTTPRESTServer.swift +++ b/Sources/RESTServer/HTTPRESTServer.swift @@ -1,6 +1,7 @@ import AutomaticTermination import Foundation import EmceeLogging +import PathLib import RESTMethods import SocketModels import Swifter @@ -29,6 +30,13 @@ public final class HTTPRESTServer { endpoint: handler ) } + + public func add( + requestPath: AbsolutePath, + localFilePath: AbsolutePath + ) { + server[requestPath.pathString] = shareFile(localFilePath.pathString) + } private func processRequest( endpoint: RESTEndpointOf @@ -51,6 +59,7 @@ public final class HTTPRESTServer { } } + @discardableResult public func start() throws -> SocketModels.Port { let port = try portProvider.localPort() try server.start(in_port_t(port.value), forceIPv4: false, priority: .default) diff --git a/Sources/RESTServer/PortProvider.swift b/Sources/RESTServer/PortProvider.swift index 4ccbe191d..8fc4e1884 100644 --- a/Sources/RESTServer/PortProvider.swift +++ b/Sources/RESTServer/PortProvider.swift @@ -3,4 +3,3 @@ import SocketModels public protocol PortProvider { func localPort() throws -> SocketModels.Port } - diff --git a/Sources/RESTServer/PortProviderWrapper.swift b/Sources/RESTServer/PortProviderWrapper.swift deleted file mode 100644 index 4abda722b..000000000 --- a/Sources/RESTServer/PortProviderWrapper.swift +++ /dev/null @@ -1,13 +0,0 @@ -import SocketModels - -public final class PortProviderWrapper: PortProvider { - private let provider: () throws -> SocketModels.Port - - public init(provider: @escaping () throws -> SocketModels.Port) { - self.provider = provider - } - - public func localPort() throws -> SocketModels.Port { - return try provider() - } -} diff --git a/Sources/ResourceLocation/ResourceLocation.swift b/Sources/ResourceLocation/ResourceLocation.swift index 40f4ba9ad..309f19b5f 100644 --- a/Sources/ResourceLocation/ResourceLocation.swift +++ b/Sources/ResourceLocation/ResourceLocation.swift @@ -172,4 +172,13 @@ public enum ResourceLocation: Hashable, CustomStringConvertible, Codable { } } } + + public func mapLocalFile(_ mapper: (String) throws -> Self) rethrows -> Self { + switch self { + case .localFilePath(let string): + return try mapper(string) + case .remoteUrl: + return self + } + } } diff --git a/Sources/SSHDeployer/DefaultSSHClient.swift b/Sources/SSHDeployer/DefaultSSHClient.swift index 485366513..d23df9a10 100644 --- a/Sources/SSHDeployer/DefaultSSHClient.swift +++ b/Sources/SSHDeployer/DefaultSSHClient.swift @@ -1,7 +1,7 @@ +import Deployer import Foundation -import Shout import PathLib -import Deployer +import Shout public final class DefaultSSHClient: SSHClient { private let ssh: SSH @@ -31,7 +31,7 @@ public final class DefaultSSHClient: SSHClient { return try ssh.execute(shellCommand) { _ in } } - public func upload(localUrl: URL, remotePath: String) throws { - try ssh.openSftp().upload(localURL: localUrl, remotePath: remotePath) + public func upload(localPath: AbsolutePath, remotePath: AbsolutePath) throws { + try ssh.openSftp().upload(localURL: localPath.fileUrl, remotePath: remotePath.pathString) } } diff --git a/Sources/SSHDeployer/SSHClient.swift b/Sources/SSHDeployer/SSHClient.swift index dd6f6f1ce..034c724b1 100644 --- a/Sources/SSHDeployer/SSHClient.swift +++ b/Sources/SSHDeployer/SSHClient.swift @@ -1,10 +1,11 @@ -import Foundation import Deployer +import Foundation +import PathLib public protocol SSHClient { init(host: String, port: Int32, username: String, authentication: DeploymentDestinationAuthenticationType) throws func connectAndAuthenticate() throws @discardableResult func execute(_ command: [String]) throws -> Int32 - func upload(localUrl: URL, remotePath: String) throws + func upload(localPath: AbsolutePath, remotePath: AbsolutePath) throws } diff --git a/Sources/SSHDeployer/SSHDeployer.swift b/Sources/SSHDeployer/SSHDeployer.swift index 04e8c0815..f19e8dab7 100644 --- a/Sources/SSHDeployer/SSHDeployer.swift +++ b/Sources/SSHDeployer/SSHDeployer.swift @@ -1,10 +1,11 @@ import Deployer +import FileSystem import Foundation import EmceeLogging import PathLib -import ProcessController import Tmp import UniqueIdentifierGenerator +import Zip public final class SSHDeployer: Deployer { @@ -17,10 +18,11 @@ public final class SSHDeployer: Deployer { deployables: [DeployableItem], deployableCommands: [DeployableCommand], destination: DeploymentDestination, + fileSystem: FileSystem, logger: ContextualLogger, - processControllerProvider: ProcessControllerProvider, temporaryFolder: TemporaryFolder, - uniqueIdentifierGenerator: UniqueIdentifierGenerator + uniqueIdentifierGenerator: UniqueIdentifierGenerator, + zipCompressor: ZipCompressor ) throws { self.sshClientType = sshClientType self.logger = logger @@ -29,10 +31,11 @@ public final class SSHDeployer: Deployer { deployables: deployables, deployableCommands: deployableCommands, destination: destination, + fileSystem: fileSystem, logger: logger, - processControllerProvider: processControllerProvider, temporaryFolder: temporaryFolder, - uniqueIdentifierGenerator: uniqueIdentifierGenerator + uniqueIdentifierGenerator: uniqueIdentifierGenerator, + zipCompressor: zipCompressor ) } @@ -130,7 +133,7 @@ public final class SSHDeployer: Deployer { remoteAbsolutePath: AbsolutePath ) throws { log(destination, "Uploading \(localAbsolutePath) -> \(remoteAbsolutePath)") - try sshClient.upload(localUrl: localAbsolutePath.fileUrl, remotePath: remoteAbsolutePath.pathString) + try sshClient.upload(localPath: localAbsolutePath, remotePath: remoteAbsolutePath) log(destination, "Uploaded \(localAbsolutePath) -> \(remoteAbsolutePath)") } diff --git a/Sources/TestArgFile/TestArgFile.swift b/Sources/TestArgFile/TestArgFile.swift index 9496e4ba2..345967f41 100644 --- a/Sources/TestArgFile/TestArgFile.swift +++ b/Sources/TestArgFile/TestArgFile.swift @@ -5,7 +5,7 @@ import QueueModels /// Represents --test-arg-file file contents which describes test plan. public struct TestArgFile: Codable, Equatable { - public let entries: [TestArgFileEntry] + public private(set) var entries: [TestArgFileEntry] public let prioritizedJob: PrioritizedJob public let testDestinationConfigurations: [TestDestinationConfiguration] @@ -19,6 +19,12 @@ public struct TestArgFile: Codable, Equatable { self.testDestinationConfigurations = testDestinationConfigurations } + public func with(entries: [TestArgFileEntry]) -> Self { + var result = self + result.entries = entries + return result + } + private enum CodingKeys: String, CodingKey { case entries case prioritizedJob diff --git a/Sources/TestArgFile/TestArgFileEntry.swift b/Sources/TestArgFile/TestArgFileEntry.swift index 529e911ca..eaff5c37d 100644 --- a/Sources/TestArgFile/TestArgFileEntry.swift +++ b/Sources/TestArgFile/TestArgFileEntry.swift @@ -9,7 +9,7 @@ import SimulatorPoolModels import WorkerCapabilitiesModels public struct TestArgFileEntry: Codable, Equatable { - public let buildArtifacts: BuildArtifacts + public private(set) var buildArtifacts: BuildArtifacts public let developerDir: DeveloperDir public let environment: [String: String] public let numberOfRetries: UInt @@ -50,6 +50,12 @@ public struct TestArgFileEntry: Codable, Equatable { self.workerCapabilityRequirements = workerCapabilityRequirements } + public func with(buildArtifacts: BuildArtifacts) -> Self { + var result = self + result.buildArtifacts = buildArtifacts + return result + } + public init(from decoder: Decoder) throws { let container = try decoder.container(keyedBy: CodingKeys.self) diff --git a/Sources/Zip/ZipCompressor.swift b/Sources/Zip/ZipCompressor.swift new file mode 100644 index 000000000..0c1731655 --- /dev/null +++ b/Sources/Zip/ZipCompressor.swift @@ -0,0 +1,13 @@ +import Foundation +import PathLib + +public protocol ZipCompressor { + + /// Creates a new archive. It will compress contents of `path` directory recursively. + /// - Returns: Path to created ZIP archive. Returned path may be altered. E.g. if you provide `archivePath` without `zip` extension, it will be added implicitly. + func createArchive( + archivePath: AbsolutePath, + workingDirectory: AbsolutePath, + contentsToCompress: RelativePath + ) throws -> AbsolutePath +} diff --git a/Sources/Zip/ZipCompressorImpl.swift b/Sources/Zip/ZipCompressorImpl.swift new file mode 100644 index 000000000..705dc2c0a --- /dev/null +++ b/Sources/Zip/ZipCompressorImpl.swift @@ -0,0 +1,30 @@ +import Foundation +import PathLib +import ProcessController + +public final class ZipCompressorImpl: ZipCompressor { + private let processControllerProvider: ProcessControllerProvider + + public init(processControllerProvider: ProcessControllerProvider) { + self.processControllerProvider = processControllerProvider + } + + public func createArchive( + archivePath: AbsolutePath, + workingDirectory: AbsolutePath, + contentsToCompress: RelativePath + ) throws -> AbsolutePath { + let controller = try processControllerProvider.createProcessController( + subprocess: Subprocess( + arguments: ["/usr/bin/zip", archivePath.pathString, "-r", contentsToCompress.pathString], + workingDirectory: workingDirectory + ) + ) + try controller.startAndWaitForSuccessfulTermination() + if archivePath.extension.isEmpty { + return archivePath.appending(extension: "zip") + } else { + return archivePath + } + } +} diff --git a/Tests/DeployerTests/DeployerTests.swift b/Tests/DeployerTests/DeployerTests.swift index d7958ef2a..a2dcbcf8e 100644 --- a/Tests/DeployerTests/DeployerTests.swift +++ b/Tests/DeployerTests/DeployerTests.swift @@ -1,12 +1,12 @@ @testable import Deployer import Foundation +import FileSystemTestHelpers import PathLib -import ProcessController -import ProcessControllerTestHelpers import TestHelpers import Tmp import UniqueIdentifierGeneratorTestHelpers import XCTest +import ZipTestHelpers class DeployerTests: XCTestCase { private let uniqueIdentifierGenerator = FixedValueUniqueIdentifierGenerator(value: "fixed") @@ -16,6 +16,8 @@ class DeployerTests: XCTestCase { source: deployableFileSource, destination: RelativePath("remote/file.swift") ) + lazy var fileSystem = FakeFileSystem(rootPath: tempFolder.absolutePath) + lazy var zipCompressor = FakeZipCompressor() func testDeployer() throws { let deployableWithSingleFile = DeployableItem( @@ -23,6 +25,37 @@ class DeployerTests: XCTestCase { files: [deployableFile] ) + let filePropertiesContainer = FakeFilePropertiesContainer() + fileSystem.propertiesProvider = { _ in filePropertiesContainer } + filePropertiesContainer.pathExists = false + + let zipCompressorInvoker = XCTestExpectation() + zipCompressor.handler = { (archivePath: AbsolutePath, + workingDirectory: AbsolutePath, + contentsToCompress: RelativePath) -> AbsolutePath in + assert { + archivePath + } equals: { + self.tempFolder.pathWith(components: ["fixed", "simple_file"]) + } + + assert { + workingDirectory + } equals: { + self.tempFolder.pathWith(components: ["fixed"]) + } + + assert { + contentsToCompress + } equals: { + RelativePath("./") + } + + zipCompressorInvoker.fulfill() + + return archivePath.appending(extension: "fake.zip") + } + let deployer = try FakeDeployer( deploymentId: "ID", deployables: [deployableWithSingleFile], @@ -34,16 +67,11 @@ class DeployerTests: XCTestCase { authentication: .password("pass"), remoteDeploymentPath: "/remote/path" ), + fileSystem: fileSystem, logger: .noOp, - processControllerProvider: FakeProcessControllerProvider { subprocess -> ProcessController in - XCTAssertEqual( - try subprocess.arguments.map { try $0.stringValue() }, - ["/usr/bin/zip", self.tempFolder.pathWith(components: ["fixed", "simple_file"]).pathString, "-r", "."] - ) - return FakeProcessController(subprocess: subprocess) - }, temporaryFolder: tempFolder, - uniqueIdentifierGenerator: uniqueIdentifierGenerator + uniqueIdentifierGenerator: uniqueIdentifierGenerator, + zipCompressor: zipCompressor ) try deployer.deploy() XCTAssertEqual(deployer.pathsAskedToBeDeployed.count, 1) @@ -55,6 +83,8 @@ class DeployerTests: XCTestCase { Set([deployableFile]) ) } + + wait(for: [zipCompressorInvoker], timeout: 3) } func testDeployerDeletesItsTemporaryStuff() throws { @@ -66,6 +96,7 @@ class DeployerTests: XCTestCase { var paths = [AbsolutePath]() let deployerWork = { + self.zipCompressor.handler = { archive, _, _ in archive } let deployer = try FakeDeployer( deploymentId: "ID", deployables: [deployableWithSingleFile], @@ -77,10 +108,11 @@ class DeployerTests: XCTestCase { authentication: .password("pass"), remoteDeploymentPath: "/remote/path" ), + fileSystem: self.fileSystem, logger: .noOp, - processControllerProvider: FakeProcessControllerProvider(), temporaryFolder: self.tempFolder, - uniqueIdentifierGenerator: self.uniqueIdentifierGenerator + uniqueIdentifierGenerator: self.uniqueIdentifierGenerator, + zipCompressor: self.zipCompressor ) try deployer.deploy() paths = Array(deployer.pathsAskedToBeDeployed.keys) diff --git a/Tests/RESTServerTests/HTTPRESTServerTests.swift b/Tests/RESTServerTests/HTTPRESTServerTests.swift index 3c5349d10..930c49314 100644 --- a/Tests/RESTServerTests/HTTPRESTServerTests.swift +++ b/Tests/RESTServerTests/HTTPRESTServerTests.swift @@ -9,7 +9,7 @@ import XCTest final class HTTPRESTServerTests: XCTestCase { lazy var automaticTerminationController = AutomaticTerminationControllerFixture(isTerminationAllowed: false) - lazy var anyPortProvider = PortProviderWrapper { 0 } + lazy var anyPortProvider = AnyAvailablePortProvider() lazy var server = HTTPRESTServer( automaticTerminationController: automaticTerminationController, logger: .noOp, diff --git a/Tests/SSHDeployerTests/FakeSSHClient.swift b/Tests/SSHDeployerTests/FakeSSHClient.swift index a698c2591..2394f351b 100644 --- a/Tests/SSHDeployerTests/FakeSSHClient.swift +++ b/Tests/SSHDeployerTests/FakeSSHClient.swift @@ -1,6 +1,7 @@ import Foundation @testable import SSHDeployer import Deployer +import PathLib class FakeSSHClient: SSHClient { let host: String @@ -10,7 +11,7 @@ class FakeSSHClient: SSHClient { var calledConnectAndAuthenticate = false var executeCommands = [[String]]() - var uploadCommands = [[URL: String]]() + var uploadCommands = [(local: AbsolutePath, remote: AbsolutePath)]() required init(host: String, port: Int32, username: String, authentication: DeploymentDestinationAuthenticationType) throws { self.host = host @@ -31,8 +32,8 @@ class FakeSSHClient: SSHClient { return 0 } - func upload(localUrl: URL, remotePath: String) throws { - uploadCommands.append([localUrl: remotePath]) + func upload(localPath: AbsolutePath, remotePath: AbsolutePath) throws { + uploadCommands.append((local: localPath, remote: remotePath)) } static var lastCreatedInstance: FakeSSHClient? diff --git a/Tests/SSHDeployerTests/SSHDeployerTests.swift b/Tests/SSHDeployerTests/SSHDeployerTests.swift index 291c7b2ca..e87d8d776 100644 --- a/Tests/SSHDeployerTests/SSHDeployerTests.swift +++ b/Tests/SSHDeployerTests/SSHDeployerTests.swift @@ -1,18 +1,27 @@ @testable import Deployer @testable import SSHDeployer +import FileSystemTestHelpers import Foundation import PathLib -import ProcessControllerTestHelpers import Tmp import TestHelpers import UniqueIdentifierGeneratorTestHelpers import XCTest +import ZipTestHelpers class SSHDeployerTests: XCTestCase { private let uniqueIdentifierGenerator = FixedValueUniqueIdentifierGenerator(value: "fixed") private lazy var tempFolder = assertDoesNotThrow { try TemporaryFolder() } + private lazy var fileSystem = FakeFileSystem(rootPath: tempFolder.absolutePath) + private lazy var zipCompressor = FakeZipCompressor { path, _, _ in + path.appending(extension: "fakezip") + } func testForInputCorrectness() throws { + let filePropertiesContainer = FakeFilePropertiesContainer() + fileSystem.propertiesProvider = { _ in filePropertiesContainer } + filePropertiesContainer.pathExists = false + let deploymentId = UUID().uuidString let deployableWithSingleFile = DeployableItem( name: "deployable_name", @@ -24,7 +33,8 @@ class SSHDeployerTests: XCTestCase { port: 1034, username: "user", authentication: .password("pa$$"), - remoteDeploymentPath: "/some/remote/container") + remoteDeploymentPath: "/some/remote/container" + ) let deployer = try SSHDeployer( sshClientType: FakeSSHClient.self, @@ -37,10 +47,11 @@ class SSHDeployerTests: XCTestCase { ] ], destination: destination, + fileSystem: fileSystem, logger: .noOp, - processControllerProvider: FakeProcessControllerProvider(), temporaryFolder: tempFolder, - uniqueIdentifierGenerator: uniqueIdentifierGenerator + uniqueIdentifierGenerator: uniqueIdentifierGenerator, + zipCompressor: zipCompressor ) try deployer.deploy() @@ -66,20 +77,32 @@ class SSHDeployerTests: XCTestCase { XCTAssertEqual(client.uploadCommands.count, 1) let uploadCommand = client.uploadCommands[0] - XCTAssertEqual( - Array(uploadCommand.keys), - [tempFolder.pathWith(components: ["fixed", "deployable_name.zip"]).fileUrl] - ) - XCTAssertEqual( - Array(uploadCommand.values), - ["/some/remote/container/\(deploymentId)/deployable_name/_package.zip"]) + assert { + uploadCommand.local + } equals: { + tempFolder.pathWith(components: ["fixed", "deployable_name.fakezip"]) + } + assert { + uploadCommand.remote + } equals: { + AbsolutePath("/some/remote/container/\(deploymentId)/deployable_name/_package.zip") + } - XCTAssertEqual( - client.executeCommands[2], - ["unzip", "\(destination.remoteDeploymentPath)/\(deploymentId)/deployable_name/_package.zip", - "-d", "\(destination.remoteDeploymentPath)/\(deploymentId)/deployable_name"]) - XCTAssertEqual( - client.executeCommands[3], - ["string_arg", "/some/remote/container/\(deploymentId)/deployable_name/remote/file.swift"]) + assert { + client.executeCommands[2] + } equals: { + [ + "unzip", + "\(destination.remoteDeploymentPath)/\(deploymentId)/deployable_name/_package.zip", + "-d", + "\(destination.remoteDeploymentPath)/\(deploymentId)/deployable_name", + ] + } + + assert { + client.executeCommands[3] + } equals: { + ["string_arg", "/some/remote/container/\(deploymentId)/deployable_name/remote/file.swift"] + } } } diff --git a/Tests/ZipTestHelpers/FakeZipCompressor.swift b/Tests/ZipTestHelpers/FakeZipCompressor.swift new file mode 100644 index 000000000..627707a68 --- /dev/null +++ b/Tests/ZipTestHelpers/FakeZipCompressor.swift @@ -0,0 +1,24 @@ +import Foundation +import PathLib +import TestHelpers +import Zip + +open class FakeZipCompressor: ZipCompressor { + public var handler: (AbsolutePath, AbsolutePath, RelativePath) throws -> AbsolutePath + + public init( + handler: @escaping (AbsolutePath, AbsolutePath, RelativePath) throws -> AbsolutePath = { archivePath, _, _ in + archivePath + } + ) { + self.handler = handler + } + + public func createArchive( + archivePath: AbsolutePath, + workingDirectory: AbsolutePath, + contentsToCompress: RelativePath + ) throws -> AbsolutePath { + try handler(archivePath, workingDirectory, contentsToCompress) + } +} diff --git a/Tests/ZipTests/ZipCompressorTests.swift b/Tests/ZipTests/ZipCompressorTests.swift new file mode 100644 index 000000000..dee980679 --- /dev/null +++ b/Tests/ZipTests/ZipCompressorTests.swift @@ -0,0 +1,40 @@ +import Foundation +import PathLib +import ProcessControllerTestHelpers +import TestHelpers +import Zip +import XCTest + +final class ZipCompressorTests: XCTestCase { + lazy var processControllerProvider = FakeProcessControllerProvider() + lazy var compressor = ZipCompressorImpl( + processControllerProvider: processControllerProvider + ) + + func test() throws { + let validated = XCTestExpectation() + processControllerProvider.creator = { subprocess in + defer { + validated.fulfill() + } + + assert { + try subprocess.arguments.map { try $0.stringValue() } + } equals: { + ["/usr/bin/zip", "/where/to/create/archive.zip", "-r", "what/to/compress/file.or.dir"] + } + + let controller = FakeProcessController(subprocess: subprocess) + controller.overridedProcessStatus = .terminated(exitCode: 0) + return controller + } + + _ = try compressor.createArchive( + archivePath: AbsolutePath("/where/to/create/archive.zip"), + workingDirectory: AbsolutePath("/where/contents/is/located"), + contentsToCompress: RelativePath("what/to/compress/file.or.dir") + ) + + wait(for: [validated], timeout: 15) + } +}