diff --git a/Sources/SWBApplePlatform/Specs/AssetCatalogCompiler.swift b/Sources/SWBApplePlatform/Specs/AssetCatalogCompiler.swift index 86bcb8f2..a94e3ea7 100644 --- a/Sources/SWBApplePlatform/Specs/AssetCatalogCompiler.swift +++ b/Sources/SWBApplePlatform/Specs/AssetCatalogCompiler.swift @@ -299,17 +299,17 @@ public final class ActoolCompilerSpec : GenericCompilerSpec, SpecIdentifierType, (macro == BuiltinMacros.ASSETCATALOG_COMPILER_INPUTS) ? cbc.scope.table.namespace.parseLiteralStringList(assetSymbolInputs.map { $0.path.str }) : lookup(macro) } - let cachingEnabled = cbc.scope.evaluate(BuiltinMacros.ENABLE_GENERIC_TASK_CACHING) let action: (any PlannedTaskAction)? if let deferredAction = delegate.taskActionCreationDelegate.createDeferredExecutionTaskActionIfRequested(userPreferences: cbc.producer.userPreferences) { action = deferredAction - } else if cachingEnabled { + } else if cbc.scope.evaluate(BuiltinMacros.ENABLE_GENERIC_TASK_CACHING), let casOptions = try? CASOptions.create(cbc.scope, nil) { action = delegate.taskActionCreationDelegate.createGenericCachingTaskAction( enableCacheDebuggingRemarks: cbc.scope.evaluate(BuiltinMacros.GENERIC_TASK_CACHE_ENABLE_DIAGNOSTIC_REMARKS), enableTaskSandboxEnforcement: !cbc.scope.evaluate(BuiltinMacros.DISABLE_TASK_SANDBOXING), sandboxDirectory: cbc.scope.evaluate(BuiltinMacros.TEMP_SANDBOX_DIR), extraSandboxSubdirectories: [], - developerDirectory: cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR) + developerDirectory: cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR), + casOptions: casOptions ) } else { action = nil @@ -359,17 +359,17 @@ public final class ActoolCompilerSpec : GenericCompilerSpec, SpecIdentifierType, var ruleInfo = defaultRuleInfo(cbc, delegate, lookup: lookup) ruleInfo[0..<1] = ["CompileAssetCatalogVariant", variant.rawValue] - let cachingEnabled = cbc.scope.evaluate(BuiltinMacros.ENABLE_GENERIC_TASK_CACHING) let action: (any PlannedTaskAction)? if let deferredAction = delegate.taskActionCreationDelegate.createDeferredExecutionTaskActionIfRequested(userPreferences: cbc.producer.userPreferences) { action = deferredAction - } else if cachingEnabled { + } else if cbc.scope.evaluate(BuiltinMacros.ENABLE_GENERIC_TASK_CACHING), let casOptions = try? CASOptions.create(cbc.scope, nil) { action = delegate.taskActionCreationDelegate.createGenericCachingTaskAction( enableCacheDebuggingRemarks: cbc.scope.evaluate(BuiltinMacros.GENERIC_TASK_CACHE_ENABLE_DIAGNOSTIC_REMARKS), enableTaskSandboxEnforcement: !cbc.scope.evaluate(BuiltinMacros.DISABLE_TASK_SANDBOXING), sandboxDirectory: cbc.scope.evaluate(BuiltinMacros.TEMP_SANDBOX_DIR), extraSandboxSubdirectories: [Path("outputs/0/\(overrideDir.basename)")], - developerDirectory: cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR) + developerDirectory: cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR), + casOptions: casOptions ) } else { action = nil diff --git a/Sources/SWBApplePlatform/Specs/InterfaceBuilderCompiler.swift b/Sources/SWBApplePlatform/Specs/InterfaceBuilderCompiler.swift index df8dac9c..67c4c1fa 100644 --- a/Sources/SWBApplePlatform/Specs/InterfaceBuilderCompiler.swift +++ b/Sources/SWBApplePlatform/Specs/InterfaceBuilderCompiler.swift @@ -158,13 +158,14 @@ public final class IbtoolCompilerSpecStoryboard: IbtoolCompilerSpec, SpecIdentif } override public func createTaskAction(_ cbc: CommandBuildContext, _ delegate: any TaskGenerationDelegate) -> (any PlannedTaskAction)? { - if cbc.scope.evaluate(BuiltinMacros.ENABLE_GENERIC_TASK_CACHING) { + if cbc.scope.evaluate(BuiltinMacros.ENABLE_GENERIC_TASK_CACHING), let casOptions = try? CASOptions.create(cbc.scope, nil) { return delegate.taskActionCreationDelegate.createGenericCachingTaskAction( enableCacheDebuggingRemarks: cbc.scope.evaluate(BuiltinMacros.GENERIC_TASK_CACHE_ENABLE_DIAGNOSTIC_REMARKS), enableTaskSandboxEnforcement: !cbc.scope.evaluate(BuiltinMacros.DISABLE_TASK_SANDBOXING), sandboxDirectory: cbc.scope.evaluate(BuiltinMacros.TEMP_SANDBOX_DIR), extraSandboxSubdirectories: [], - developerDirectory: cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR)) + developerDirectory: cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR), + casOptions: casOptions) } else { return nil } diff --git a/Sources/SWBApplePlatform/Specs/RealityAssetsCompilerSpec.swift b/Sources/SWBApplePlatform/Specs/RealityAssetsCompilerSpec.swift index d792a144..f88fd9a2 100644 --- a/Sources/SWBApplePlatform/Specs/RealityAssetsCompilerSpec.swift +++ b/Sources/SWBApplePlatform/Specs/RealityAssetsCompilerSpec.swift @@ -175,7 +175,23 @@ package final class RealityAssetsCompilerSpec: GenericCompilerSpec, SpecIdentifi inputs.append(delegate.createNode(usdaSchemaPath) as (any PlannedNode)) } - let cachingEnabled = cbc.scope.evaluate(BuiltinMacros.ENABLE_GENERIC_TASK_CACHING) + let action: (any PlannedTaskAction)? + let cachingEnabled: Bool + if cbc.scope.evaluate(BuiltinMacros.ENABLE_GENERIC_TASK_CACHING), let casOptions = try? CASOptions.create(cbc.scope, nil) { + action = delegate.taskActionCreationDelegate.createGenericCachingTaskAction( + enableCacheDebuggingRemarks: cbc.scope.evaluate(BuiltinMacros.GENERIC_TASK_CACHE_ENABLE_DIAGNOSTIC_REMARKS), + enableTaskSandboxEnforcement: !cbc.scope.evaluate(BuiltinMacros.DISABLE_TASK_SANDBOXING), + sandboxDirectory: cbc.scope.evaluate(BuiltinMacros.TEMP_SANDBOX_DIR), + extraSandboxSubdirectories: [], + developerDirectory: cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR), + casOptions: casOptions + ) + cachingEnabled = true + } else { + action = nil + cachingEnabled = false + } + let ruleInfo = ["RealityAssetsCompile", cbc.output.str] delegate.createTask(type: self, @@ -190,7 +206,7 @@ package final class RealityAssetsCompilerSpec: GenericCompilerSpec, SpecIdentifi inputs: inputs, outputs: outputs, mustPrecede: [], - action: cachingEnabled ? delegate.taskActionCreationDelegate.createGenericCachingTaskAction(enableCacheDebuggingRemarks: cbc.scope.evaluate(BuiltinMacros.GENERIC_TASK_CACHE_ENABLE_DIAGNOSTIC_REMARKS), enableTaskSandboxEnforcement: !cbc.scope.evaluate(BuiltinMacros.DISABLE_TASK_SANDBOXING), sandboxDirectory: cbc.scope.evaluate(BuiltinMacros.TEMP_SANDBOX_DIR), extraSandboxSubdirectories: [], developerDirectory: cbc.scope.evaluate(BuiltinMacros.DEVELOPER_DIR)) : nil, + action: action, execDescription: "Compile Reality Asset \(rkAssetsPath.basename)", preparesForIndexing: true, enableSandboxing: !cachingEnabled, diff --git a/Sources/SWBBuildSystem/BuildOperation.swift b/Sources/SWBBuildSystem/BuildOperation.swift index 16b0b941..75a60dab 100644 --- a/Sources/SWBBuildSystem/BuildOperation.swift +++ b/Sources/SWBBuildSystem/BuildOperation.swift @@ -1339,15 +1339,23 @@ internal final class OperationSystemAdaptor: SWBLLBuild.BuildSystemDelegate, Act self.buildOutputDelegate = buildOutputDelegate self._progressStatistics = BuildOperation.ProgressStatistics(numCommandsLowerBound: operation.buildDescription.targetTaskCounts.values.reduce(0, { $0 + $1 })) self.core = core - self.dynamicOperationContext = DynamicTaskOperationContext(core: core, definingTargetsByModuleName: operation.buildDescription.definingTargetsByModuleName, cas: Self.setupCAS(core: core, operation: operation)) + let cas: ToolchainCAS? + do { + cas = try Self.setupCAS(core: core, operation: operation) + } catch { + buildOutputDelegate.error(error.localizedDescription) + cas = nil + } + self.dynamicOperationContext = DynamicTaskOperationContext(core: core, definingTargetsByModuleName: operation.buildDescription.definingTargetsByModuleName, cas: cas) self.queue = SWBQueue(label: "SWBBuildSystem.OperationSystemAdaptor.queue", qos: operation.request.qos, autoreleaseFrequency: .workItem) } - private static func setupCAS(core: Core, operation: BuildOperation) -> ToolchainCAS? { + private static func setupCAS(core: Core, operation: BuildOperation) throws -> ToolchainCAS? { let settings = operation.requestContext.getCachedSettings(operation.request.parameters) - let cachePath = Path(settings.globalScope.evaluate(BuiltinMacros.COMPILATION_CACHE_CAS_PATH)).join("swiftbuild") + let casOptions = try CASOptions.create(settings.globalScope, nil) guard let casPlugin = core.lookupCASPlugin() else { return nil } - guard let cas = try? casPlugin.createCAS(path: cachePath, options: [:]) else { return nil } + guard let cas = try? casPlugin.createCAS(path: casOptions.casPath, options: [:]) else { return nil } + return cas } diff --git a/Sources/SWBCAS/Errors.swift b/Sources/SWBCAS/Errors.swift index a9b74fc3..305d099d 100644 --- a/Sources/SWBCAS/Errors.swift +++ b/Sources/SWBCAS/Errors.swift @@ -10,7 +10,7 @@ // //===----------------------------------------------------------------------===// -public enum ToolchainCASPluginError: Error, Sendable { +public enum ToolchainCASPluginError: Error, Sendable, CustomStringConvertible { case missingRequiredSymbol(String) case settingCASOptionFailed(String?) case casCreationFailed(String?) @@ -18,4 +18,35 @@ public enum ToolchainCASPluginError: Error, Sendable { case loadFailed(String?) case cacheInsertionFailed(String?) case cacheLookupFailed(String?) + case casSizeOperationUnsupported + case casSizeOperationFailed(String?) + case casPruneOperationUnsupported + case casPruneOperationFailed(String?) + + public var description: String { + switch self { + case .missingRequiredSymbol(let symbol): + "missing required symbol: \(symbol)" + case .settingCASOptionFailed(let detail): + "setting CAS option failed: \(detail ?? "unknown error")" + case .casCreationFailed(let detail): + "creating CAS failed: \(detail ?? "unknown error")" + case .storeFailed(let detail): + "CAS store failed: \(detail ?? "unknown error")" + case .loadFailed(let detail): + "CAS load failed: \(detail ?? "unknown error")" + case .cacheInsertionFailed(let detail): + "cache insert failed: \(detail ?? "unknown error")" + case .cacheLookupFailed(let detail): + "cache lookup failed: \(detail ?? "unknown error")" + case .casSizeOperationUnsupported: + "CAS does not support size lookup" + case .casSizeOperationFailed(let detail): + "CAS size operation failed: \(detail ?? "unknown error")" + case .casPruneOperationUnsupported: + "CAS does not support pruning" + case .casPruneOperationFailed(let detail): + "CAS prune operation failed: \(detail ?? "unknown error")" + } + } } diff --git a/Sources/SWBCAS/ToolchainCASPlugin.swift b/Sources/SWBCAS/ToolchainCASPlugin.swift index 5f4b27fd..8caee630 100644 --- a/Sources/SWBCAS/ToolchainCASPlugin.swift +++ b/Sources/SWBCAS/ToolchainCASPlugin.swift @@ -172,6 +172,61 @@ public final class ToolchainCAS: @unchecked Sendable, CASProtocol, ActionCachePr }) } + public func getOnDiskSize() throws -> Int64 { + var error: UnsafeMutablePointer? = nil + guard let llcas_cas_get_ondisk_size = api.llcas_cas_get_ondisk_size else { + throw ToolchainCASPluginError.casSizeOperationUnsupported + } + let result = llcas_cas_get_ondisk_size(cCas, &error) + switch result { + case -1: + throw ToolchainCASPluginError.casSizeOperationUnsupported + case -2: + if let error = error { + let detailedError = String(cString: error) + api.llcas_string_dispose(error) + throw ToolchainCASPluginError.casSizeOperationFailed(detailedError) + } + throw ToolchainCASPluginError.casSizeOperationFailed(nil) + default: + return result + } + } + + public func setOnDiskSizeLimit(_ limit: Int64) throws { + var error: UnsafeMutablePointer? = nil + guard let llcas_cas_set_ondisk_size_limit = api.llcas_cas_set_ondisk_size_limit else { + throw ToolchainCASPluginError.casSizeOperationUnsupported + } + if llcas_cas_set_ondisk_size_limit(cCas, limit, &error) { + if let error = error { + let detailedError = String(cString: error) + api.llcas_string_dispose(error) + throw ToolchainCASPluginError.casSizeOperationFailed(detailedError) + } + throw ToolchainCASPluginError.casSizeOperationFailed(nil) + } + } + + public func prune() throws { + var error: UnsafeMutablePointer? = nil + guard let llcas_cas_prune_ondisk_data = api.llcas_cas_prune_ondisk_data else { + throw ToolchainCASPluginError.casPruneOperationUnsupported + } + if llcas_cas_prune_ondisk_data(cCas, &error) { + if let error = error { + let detailedError = String(cString: error) + api.llcas_string_dispose(error) + throw ToolchainCASPluginError.casPruneOperationFailed(detailedError) + } + throw ToolchainCASPluginError.casPruneOperationFailed(nil) + } + } + + public var supportsPruning: Bool { + api.llcas_cas_get_ondisk_size != nil && api.llcas_cas_set_ondisk_size_limit != nil && api.llcas_cas_prune_ondisk_data != nil + } + deinit { api.llcas_cas_dispose(cCas) } diff --git a/Sources/SWBCore/PlannedTaskAction.swift b/Sources/SWBCore/PlannedTaskAction.swift index 9d33d3f2..553eee0b 100644 --- a/Sources/SWBCore/PlannedTaskAction.swift +++ b/Sources/SWBCore/PlannedTaskAction.swift @@ -316,7 +316,7 @@ public protocol TaskActionCreationDelegate func createDeferredExecutionTaskAction() -> any PlannedTaskAction func createEmbedSwiftStdLibTaskAction() -> any PlannedTaskAction func createFileCopyTaskAction(_ context: FileCopyTaskActionContext) -> any PlannedTaskAction - func createGenericCachingTaskAction(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path) -> any PlannedTaskAction + func createGenericCachingTaskAction(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path, casOptions: CASOptions) -> any PlannedTaskAction func createInfoPlistProcessorTaskAction(_ contextPath: Path) -> any PlannedTaskAction func createMergeInfoPlistTaskAction() -> any PlannedTaskAction func createLinkAssetCatalogTaskAction() -> any PlannedTaskAction diff --git a/Sources/SWBCore/Settings/CASOptions.swift b/Sources/SWBCore/Settings/CASOptions.swift index 7a40f7b0..55f7c9f7 100644 --- a/Sources/SWBCore/Settings/CASOptions.swift +++ b/Sources/SWBCore/Settings/CASOptions.swift @@ -12,9 +12,24 @@ import protocol Foundation.LocalizedError public import SWBUtil +public import SWBMacro public struct CASOptions: Hashable, Serializable, Encodable, Sendable { + enum Errors: Error, CustomStringConvertible { + case invalidSizeLimit(sizeLimitString: String, origin: String) + case invalidPercentLimit(percentLimitString: String) + + var description: String { + switch self { + case .invalidSizeLimit(sizeLimitString: let sizeLimitString, origin: let origin): + return "invalid \(origin): '\(sizeLimitString)'" + case .invalidPercentLimit(percentLimitString: let percentLimitString): + return "invalid COMPILATION_CACHE_LIMIT_PERCENT: '\(percentLimitString)'" + } + } + } + public enum SizeLimitingStrategy: Hashable, Serializable, Encodable, Sendable { /// Cache directory is removed after the build is finished. case discarded @@ -155,69 +170,68 @@ public struct CASOptions: Hashable, Serializable, Encodable, Sendable { self.limitingStrategy = try deserializer.deserialize() } - public static func createFromBuildContext( - _ cbc: CommandBuildContext, - _ language: GCCCompatibleLanguageDialect, - _ delegate: any TaskGenerationDelegate - ) -> CASOptions { + public static func create( + _ scope: MacroEvaluationScope, + _ language: GCCCompatibleLanguageDialect? + ) throws -> CASOptions { func isLanguageSupportedForRemoteCaching() -> Bool { - let supportedLangs = cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_REMOTE_SUPPORTED_LANGUAGES) + let supportedLangs = scope.evaluate(BuiltinMacros.COMPILATION_CACHE_REMOTE_SUPPORTED_LANGUAGES) // If no specific list of languages is provided then all languages are supported. guard !supportedLangs.isEmpty else { return true } + // If we're not compiling a specific language, assume support. + guard let language else { return true } return supportedLangs.contains(language.dialectNameForCompilerCommandLineArgument) } let casPath: Path let pluginPath: Path? let remoteServicePath: Path? - let enableIntegratedCacheQueries = cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_ENABLE_INTEGRATED_QUERIES) - let enableDiagnosticRemarks = cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_ENABLE_DIAGNOSTIC_REMARKS) - let enableStrictCASErrors = cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_ENABLE_STRICT_CAS_ERRORS) - let enableDetachedKeyQueries = cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_ENABLE_DETACHED_KEY_QUERIES) - if cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_ENABLE_PLUGIN) { - casPath = Path(cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_CAS_PATH)).join("plugin") - pluginPath = Path(cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_PLUGIN_PATH)) - let remoteServicePathSetting = Path(cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_REMOTE_SERVICE_PATH)) + let enableIntegratedCacheQueries = scope.evaluate(BuiltinMacros.COMPILATION_CACHE_ENABLE_INTEGRATED_QUERIES) + let enableDiagnosticRemarks = scope.evaluate(BuiltinMacros.COMPILATION_CACHE_ENABLE_DIAGNOSTIC_REMARKS) + let enableStrictCASErrors = scope.evaluate(BuiltinMacros.COMPILATION_CACHE_ENABLE_STRICT_CAS_ERRORS) + let enableDetachedKeyQueries = scope.evaluate(BuiltinMacros.COMPILATION_CACHE_ENABLE_DETACHED_KEY_QUERIES) + if scope.evaluate(BuiltinMacros.COMPILATION_CACHE_ENABLE_PLUGIN) { + casPath = Path(scope.evaluate(BuiltinMacros.COMPILATION_CACHE_CAS_PATH)).join("plugin") + pluginPath = Path(scope.evaluate(BuiltinMacros.COMPILATION_CACHE_PLUGIN_PATH)) + let remoteServicePathSetting = Path(scope.evaluate(BuiltinMacros.COMPILATION_CACHE_REMOTE_SERVICE_PATH)) if !remoteServicePathSetting.isEmpty && isLanguageSupportedForRemoteCaching() { remoteServicePath = remoteServicePathSetting } else { remoteServicePath = nil } } else { - casPath = Path(cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_CAS_PATH)).join("builtin") + casPath = Path(scope.evaluate(BuiltinMacros.COMPILATION_CACHE_CAS_PATH)).join("builtin") pluginPath = nil remoteServicePath = nil } - let limitingStrategy: CASOptions.SizeLimitingStrategy = { - guard cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_KEEP_CAS_DIRECTORY) else { + let limitingStrategy: CASOptions.SizeLimitingStrategy = try { + guard scope.evaluate(BuiltinMacros.COMPILATION_CACHE_KEEP_CAS_DIRECTORY) else { return .discarded } - func parseSizeLimit(_ sizeLimitStr: String, origin: String) -> CASOptions.SizeLimitingStrategy { + func parseSizeLimit(_ sizeLimitStr: String, origin: String) throws -> CASOptions.SizeLimitingStrategy { guard let sizeLimit = CASOptions.parseSizeLimit(sizeLimitStr) else { - delegate.error("invalid \(origin): '\(sizeLimitStr)'") - return .default + throw Errors.invalidSizeLimit(sizeLimitString: sizeLimitStr, origin: origin) } return .maxSizeBytes(sizeLimit > 0 ? sizeLimit : nil) } - let sizeLimitStr = cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_LIMIT_SIZE) + let sizeLimitStr = scope.evaluate(BuiltinMacros.COMPILATION_CACHE_LIMIT_SIZE) if !sizeLimitStr.isEmpty { - return parseSizeLimit(sizeLimitStr, origin: "COMPILATION_CACHE_LIMIT_SIZE") + return try parseSizeLimit(sizeLimitStr, origin: "COMPILATION_CACHE_LIMIT_SIZE") } - let percentLimitStr = cbc.scope.evaluate(BuiltinMacros.COMPILATION_CACHE_LIMIT_PERCENT) + let percentLimitStr = scope.evaluate(BuiltinMacros.COMPILATION_CACHE_LIMIT_PERCENT) if !percentLimitStr.isEmpty { guard let percent = Int(percentLimitStr) else { - delegate.error("invalid COMPILATION_CACHE_LIMIT_PERCENT: '\(percentLimitStr)'") - return .default + throw Errors.invalidPercentLimit(percentLimitString: percentLimitStr) } return .maxPercentageOfAvailableSpace(percent) } if let sizeLimitDefault = UserDefaults.compilationCachingDiskSizeLimit { - return parseSizeLimit(sizeLimitDefault, origin: "CompilationCachingDiskSizeLimit default") + return try parseSizeLimit(sizeLimitDefault, origin: "CompilationCachingDiskSizeLimit default") } return .default diff --git a/Sources/SWBCore/Specs/Tools/CCompiler.swift b/Sources/SWBCore/Specs/Tools/CCompiler.swift index 33025a99..31eb80a5 100644 --- a/Sources/SWBCore/Specs/Tools/CCompiler.swift +++ b/Sources/SWBCore/Specs/Tools/CCompiler.swift @@ -954,14 +954,19 @@ public class ClangCompilerSpec : CompilerSpec, SpecIdentifierType, GCCCompatible let casOptions: CASOptions? = { guard cachedBuild else { return nil } - var casOpts = CASOptions.createFromBuildContext(cbc, language, delegate) - if casOpts.enableIntegratedCacheQueries, let clangInfo { - if !clangInfo.toolFeatures.has(.libclangCacheQueries) { - delegate.warning("COMPILATION_CACHE_ENABLE_INTEGRATED_QUERIES ignored because it's not supported by the toolchain") - casOpts.enableIntegratedCacheQueries = false + do { + var casOpts = try CASOptions.create(cbc.scope, language) + if casOpts.enableIntegratedCacheQueries, let clangInfo { + if !clangInfo.toolFeatures.has(.libclangCacheQueries) { + delegate.warning("COMPILATION_CACHE_ENABLE_INTEGRATED_QUERIES ignored because it's not supported by the toolchain") + casOpts.enableIntegratedCacheQueries = false + } } + return casOpts + } catch { + delegate.error(error.localizedDescription) + return nil } - return casOpts }() let explicitModulesPayload = ClangExplicitModulesPayload( uniqueID: String(commandLine.hashValue), diff --git a/Sources/SWBCore/Specs/Tools/SwiftCompiler.swift b/Sources/SWBCore/Specs/Tools/SwiftCompiler.swift index 8d8266e8..70e2014a 100644 --- a/Sources/SWBCore/Specs/Tools/SwiftCompiler.swift +++ b/Sources/SWBCore/Specs/Tools/SwiftCompiler.swift @@ -1796,18 +1796,24 @@ public final class SwiftCompilerSpec : CompilerSpec, SpecIdentifierType, SwiftDi } // Add caching related configurations. - let casOptions = isCachingEnabled ? CASOptions.createFromBuildContext(cbc, .other(dialectName: "swift"), delegate) : nil - if let casOpts = casOptions { - args.append("-cache-compile-job") - args += ["-cas-path", casOpts.casPath.str] - if let pluginPath = casOpts.pluginPath { - args += ["-cas-plugin-path", pluginPath.str] - } - // If the integrated cache queries is enabled, the remote service is handled by build system and no need to pass to compiler - if !casOpts.enableIntegratedCacheQueries && casOpts.hasRemoteCache, - let remoteService = casOpts.remoteServicePath { - args += ["-cas-plugin-option", "remote-service-path=" + remoteService.str] + let casOptions: CASOptions? + do { + casOptions = isCachingEnabled ? (try CASOptions.create(cbc.scope, .other(dialectName: "swift"))) : nil + if let casOpts = casOptions { + args.append("-cache-compile-job") + args += ["-cas-path", casOpts.casPath.str] + if let pluginPath = casOpts.pluginPath { + args += ["-cas-plugin-path", pluginPath.str] + } + // If the integrated cache queries is enabled, the remote service is handled by build system and no need to pass to compiler + if !casOpts.enableIntegratedCacheQueries && casOpts.hasRemoteCache, + let remoteService = casOpts.remoteServicePath { + args += ["-cas-plugin-option", "remote-service-path=" + remoteService.str] + } } + } catch { + delegate.error(error.localizedDescription) + casOptions = nil } // Instruct the compiler to serialize diagnostics. diff --git a/Sources/SWBTaskExecution/BuildDescriptionManager.swift b/Sources/SWBTaskExecution/BuildDescriptionManager.swift index 3a1cc867..d5efd11d 100644 --- a/Sources/SWBTaskExecution/BuildDescriptionManager.swift +++ b/Sources/SWBTaskExecution/BuildDescriptionManager.swift @@ -777,8 +777,8 @@ extension BuildSystemTaskPlanningDelegate: TaskActionCreationDelegate { return FileCopyTaskAction(context) } - func createGenericCachingTaskAction(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path) -> any PlannedTaskAction { - return GenericCachingTaskAction(enableCacheDebuggingRemarks: enableCacheDebuggingRemarks, enableTaskSandboxEnforcement: enableTaskSandboxEnforcement, sandboxDirectory: sandboxDirectory, extraSandboxSubdirectories: extraSandboxSubdirectories, developerDirectory: developerDirectory) + func createGenericCachingTaskAction(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path, casOptions: CASOptions) -> any PlannedTaskAction { + return GenericCachingTaskAction(enableCacheDebuggingRemarks: enableCacheDebuggingRemarks, enableTaskSandboxEnforcement: enableTaskSandboxEnforcement, sandboxDirectory: sandboxDirectory, extraSandboxSubdirectories: extraSandboxSubdirectories, developerDirectory: developerDirectory, casOptions: casOptions) } func createInfoPlistProcessorTaskAction(_ contextPath: Path) -> any PlannedTaskAction { diff --git a/Sources/SWBTaskExecution/DynamicTaskSpecs/CompilationCachingDataPruner.swift b/Sources/SWBTaskExecution/DynamicTaskSpecs/CompilationCachingDataPruner.swift index a61630af..b4180bd7 100644 --- a/Sources/SWBTaskExecution/DynamicTaskSpecs/CompilationCachingDataPruner.swift +++ b/Sources/SWBTaskExecution/DynamicTaskSpecs/CompilationCachingDataPruner.swift @@ -13,31 +13,32 @@ package import SWBCore import SWBProtocol package import SWBUtil +package import SWBCAS package struct ClangCachingPruneDataTaskKey: Hashable, Serializable, CustomDebugStringConvertible, Sendable { - let libclangPath: Path + let path: Path let casOptions: CASOptions - init(libclangPath: Path, casOptions: CASOptions) { - self.libclangPath = libclangPath + init(path: Path, casOptions: CASOptions) { + self.path = path self.casOptions = casOptions } package func serialize(to serializer: T) where T : Serializer { serializer.serializeAggregate(2) { - serializer.serialize(libclangPath) + serializer.serialize(path) serializer.serialize(casOptions) } } package init(from deserializer: any Deserializer) throws { try deserializer.beginAggregate(2) - libclangPath = try deserializer.deserialize() + path = try deserializer.deserialize() casOptions = try deserializer.deserialize() } package var debugDescription: String { - "" + "" } } @@ -95,7 +96,7 @@ package final class CompilationCachingDataPruner: Sendable { let signature = signatureCtx.signature let casPath = casOpts.casPath.str - let libclangPath = key.libclangPath.str + let libclangPath = key.path.str // Avoiding the swift concurrency variant because it may lead to starvation when `waitForCompletion()` // blocks on such tasks. Before using a swift concurrency task here make sure there's no deadlock @@ -140,6 +141,75 @@ package final class CompilationCachingDataPruner: Sendable { } } + package func pruneCAS( + _ toolchainCAS: ToolchainCAS, + key: ClangCachingPruneDataTaskKey, + activityReporter: any ActivityReporter, + fileSystem fs: any FSProxy + ) { + let casOpts = key.casOptions + guard casOpts.limitingStrategy != .discarded else { + return // No need to prune, CAS directory is getting deleted. + } + let inserted = state.withLock { $0.prunedCASes.insert(key).inserted } + guard inserted else { + return // already pruned + } + + startedAction() + let serializer = MsgPackSerializer() + key.serialize(to: serializer) + let signatureCtx = MD5Context() + signatureCtx.add(string: "ClangCachingPruneData") + signatureCtx.add(bytes: serializer.byteString) + let signature = signatureCtx.signature + + let casPath = casOpts.casPath.str + let path = key.path.str + + // Avoiding the swift concurrency variant because it may lead to starvation when `waitForCompletion()` + // blocks on such tasks. Before using a swift concurrency task here make sure there's no deadlock + // when setting `LIBDISPATCH_COOPERATIVE_POOL_STRICT`. + queue.async { + activityReporter.withActivity( + ruleInfo: "ClangCachingPruneData \(casPath) \(path)", + executionDescription: "Pruning \(casPath) using \(path)", + signature: signature, + target: nil, + parentActivity: nil) + { activityID in + let status: BuildOperationTaskEnded.Status + do { + let dbSize = (try? toolchainCAS.getOnDiskSize()).map { Int($0) } + let sizeLimit = try computeCASSizeLimit(casOptions: casOpts, dbSize: dbSize, fileSystem: fs).map { Int64($0) } + if casOpts.enableDiagnosticRemarks, let dbSize, let sizeLimit, sizeLimit < dbSize { + activityReporter.emit( + diagnostic: Diagnostic( + behavior: .remark, + location: .unknown, + data: DiagnosticData("cache size (\(dbSize)) larger than size limit (\(sizeLimit)") + ), + for: activityID, + signature: signature + ) + } + try toolchainCAS.setOnDiskSizeLimit(sizeLimit ?? 0) + try toolchainCAS.prune() + status = .succeeded + } catch { + activityReporter.emit( + diagnostic: Diagnostic(behavior: .error, location: .unknown, data: DiagnosticData(error.localizedDescription)), + for: activityID, + signature: signature + ) + status = .failed + } + return status + } + self.finishedAction() + } + } + package func waitForCompletion() async { await group.wait(queue: .global()) } diff --git a/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift index 8574891e..2e6afccc 100644 --- a/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/ClangCompileTaskAction.swift @@ -234,7 +234,7 @@ public final class ClangCompileTaskAction: TaskAction, BuildValueValidatingTaskA ) let casKey = ClangCachingPruneDataTaskKey( - libclangPath: explicitModulesPayload.libclangPath, + path: explicitModulesPayload.libclangPath, casOptions: casOptions ) dynamicExecutionDelegate.operationContext.compilationCachingDataPruner.pruneCAS( diff --git a/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift b/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift index 759bb965..1d3d7f33 100644 --- a/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift +++ b/Sources/SWBTaskExecution/TaskActions/GenericCachingTaskAction.swift @@ -26,33 +26,37 @@ public final class GenericCachingTaskAction: TaskAction { let sandboxDirectory: Path let extraSandboxSubdirectories: [Path] let developerDirectory: Path + let casOptions: CASOptions - public init(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path) { + public init(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path, casOptions: CASOptions) { self.enableCacheDebuggingRemarks = enableCacheDebuggingRemarks self.enableTaskSandboxEnforcement = enableTaskSandboxEnforcement self.sandboxDirectory = sandboxDirectory self.extraSandboxSubdirectories = extraSandboxSubdirectories self.developerDirectory = developerDirectory + self.casOptions = casOptions super.init() } required init(from deserializer: any Deserializer) throws { - try deserializer.beginAggregate(6) + try deserializer.beginAggregate(7) self.enableCacheDebuggingRemarks = try deserializer.deserialize() self.enableTaskSandboxEnforcement = try deserializer.deserialize() self.sandboxDirectory = try deserializer.deserialize() self.extraSandboxSubdirectories = try deserializer.deserialize() self.developerDirectory = try deserializer.deserialize() + self.casOptions = try deserializer.deserialize() try super.init(from: deserializer) } public override func serialize(to serializer: T) where T : Serializer { - serializer.beginAggregate(6) + serializer.beginAggregate(7) serializer.serialize(enableCacheDebuggingRemarks) serializer.serialize(enableTaskSandboxEnforcement) serializer.serialize(sandboxDirectory) serializer.serialize(extraSandboxSubdirectories) serializer.serialize(developerDirectory) + serializer.serialize(casOptions) super.serialize(to: serializer) serializer.endAggregate() } @@ -68,6 +72,15 @@ public final class GenericCachingTaskAction: TaskAction { return .failed } + defer { + if cas.supportsPruning { + dynamicExecutionDelegate.operationContext.compilationCachingDataPruner.pruneCAS(cas, + key: .init(path: casOptions.casPath, casOptions: casOptions), + activityReporter: dynamicExecutionDelegate, + fileSystem: executionDelegate.fs) + } + } + // Ingest the inputs let cacheKey: CacheKey let keyObjectID: ToolchainDataID diff --git a/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift b/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift index 5fa8877f..516931d9 100644 --- a/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift +++ b/Sources/SWBTestSupport/CapturingTaskGenerationDelegate.swift @@ -156,8 +156,8 @@ extension CapturingTaskGenerationDelegate: TaskActionCreationDelegate { return FileCopyTaskAction(context) } - package func createGenericCachingTaskAction(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path) -> any PlannedTaskAction { - return GenericCachingTaskAction(enableCacheDebuggingRemarks: enableCacheDebuggingRemarks, enableTaskSandboxEnforcement: enableTaskSandboxEnforcement, sandboxDirectory: sandboxDirectory, extraSandboxSubdirectories: extraSandboxSubdirectories, developerDirectory: developerDirectory) + package func createGenericCachingTaskAction(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path, casOptions: CASOptions) -> any PlannedTaskAction { + return GenericCachingTaskAction(enableCacheDebuggingRemarks: enableCacheDebuggingRemarks, enableTaskSandboxEnforcement: enableTaskSandboxEnforcement, sandboxDirectory: sandboxDirectory, extraSandboxSubdirectories: extraSandboxSubdirectories, developerDirectory: developerDirectory, casOptions: casOptions) } package func createInfoPlistProcessorTaskAction(_ contextPath: Path) -> any PlannedTaskAction { diff --git a/Sources/SWBTestSupport/TaskPlanningTestSupport.swift b/Sources/SWBTestSupport/TaskPlanningTestSupport.swift index c4e49c73..806593ef 100644 --- a/Sources/SWBTestSupport/TaskPlanningTestSupport.swift +++ b/Sources/SWBTestSupport/TaskPlanningTestSupport.swift @@ -382,8 +382,8 @@ extension TestTaskPlanningDelegate: TaskActionCreationDelegate { return FileCopyTaskAction(context) } - package func createGenericCachingTaskAction(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path) -> any PlannedTaskAction { - return GenericCachingTaskAction(enableCacheDebuggingRemarks: enableCacheDebuggingRemarks, enableTaskSandboxEnforcement: enableTaskSandboxEnforcement, sandboxDirectory: sandboxDirectory, extraSandboxSubdirectories: extraSandboxSubdirectories, developerDirectory: developerDirectory) + package func createGenericCachingTaskAction(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path, casOptions: CASOptions) -> any PlannedTaskAction { + return GenericCachingTaskAction(enableCacheDebuggingRemarks: enableCacheDebuggingRemarks, enableTaskSandboxEnforcement: enableTaskSandboxEnforcement, sandboxDirectory: sandboxDirectory, extraSandboxSubdirectories: extraSandboxSubdirectories, developerDirectory: developerDirectory, casOptions: casOptions) } package func createInfoPlistProcessorTaskAction(_ contextPath: Path) -> any PlannedTaskAction { diff --git a/Tests/SWBBuildSystemTests/GenericTaskCachingTests.swift b/Tests/SWBBuildSystemTests/GenericTaskCachingTests.swift index 10ba5a92..c7730aa2 100644 --- a/Tests/SWBBuildSystemTests/GenericTaskCachingTests.swift +++ b/Tests/SWBBuildSystemTests/GenericTaskCachingTests.swift @@ -266,4 +266,72 @@ fileprivate struct GenericTaskCachingTests: CoreBasedTests { } } } + + @Test(.requireSDKs(.macOS), .requireHostOS(.macOS)) + func repeatedReplay() async throws { + try await withTemporaryDirectory { tmpDir in + let testProject = TestProject( + "aProject", + sourceRoot: tmpDir, + groupTree: TestGroup( + "SomeFiles", + children: [ + TestFile("Assets.xcassets"), + ]), + buildConfigurations: [ + TestBuildConfiguration( + "Debug", + buildSettings: [ + "SDKROOT": "macosx", + "CODE_SIGNING_ALLOWED": "NO", + "GENERATE_INFOPLIST_FILE": "YES", + "PRODUCT_NAME": "$(TARGET_NAME)", + "VERSIONING_SYSTEM": "apple-generic", + "DSTROOT": tmpDir.join("dstroot").str, + "ENABLE_GENERIC_TASK_CACHING": "YES", + "GENERIC_TASK_CACHE_ENABLE_DIAGNOSTIC_REMARKS": "YES", + "COMPILATION_CACHE_LIMIT_SIZE": "1" + ]), + ], + targets: [ + TestStandardTarget( + "FrameworkTarget", + type: .framework, + buildPhases: [ + TestResourcesBuildPhase([ + TestBuildFile("Assets.xcassets"), + ]), + ] + ), + ]) + + let core = try await getCore() + let tester = try await BuildOperationTester(core, testProject, simulated: false) + let SRCROOT = tester.workspace.projects[0].sourceRoot + try await localFS.writeAssetCatalog(SRCROOT.join("Assets.xcassets")) + + // Repeated replay of the task in incremental builds should successfully place outputs + try await tester.checkBuild(parameters: BuildParameters(configuration: "Debug", overrides: ["CCHROOT": tmpDir.join("cchroot").str]), runDestination: .macOS) { results in + results.checkTaskExists(.matchRuleType("CompileAssetCatalogVariant")) + results.checkRemark(.prefix("caching result")) + results.checkNoErrors() + } + + try tester.fs.touch(SRCROOT.join("Assets.xcassets")) + + try await tester.checkBuild(parameters: BuildParameters(configuration: "Debug", overrides: ["CCHROOT": tmpDir.join("cchroot").str]), runDestination: .macOS) { results in + results.checkTaskExists(.matchRuleType("CompileAssetCatalogVariant")) + results.checkRemark(.prefix("replaying cache hit")) + results.checkNoErrors() + } + + try tester.fs.touch(SRCROOT.join("Assets.xcassets")) + + try await tester.checkBuild(parameters: BuildParameters(configuration: "Debug", overrides: ["CCHROOT": tmpDir.join("cchroot").str]), runDestination: .macOS) { results in + results.checkTaskExists(.matchRuleType("CompileAssetCatalogVariant")) + results.checkRemark(.prefix("replaying cache hit")) + results.checkNoErrors() + } + } + } } diff --git a/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift b/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift index 8a10e929..6a2eae3b 100644 --- a/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift +++ b/Tests/SWBCorePerfTests/CommandLineSpecPerfTests.swift @@ -150,8 +150,8 @@ extension CapturingTaskGenerationDelegate: TaskActionCreationDelegate { return FileCopyTaskAction(context) } - public func createGenericCachingTaskAction(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path) -> any PlannedTaskAction { - return GenericCachingTaskAction(enableCacheDebuggingRemarks: enableCacheDebuggingRemarks, enableTaskSandboxEnforcement: enableTaskSandboxEnforcement, sandboxDirectory: sandboxDirectory, extraSandboxSubdirectories: extraSandboxSubdirectories, developerDirectory: developerDirectory) + public func createGenericCachingTaskAction(enableCacheDebuggingRemarks: Bool, enableTaskSandboxEnforcement: Bool, sandboxDirectory: Path, extraSandboxSubdirectories: [Path], developerDirectory: Path, casOptions: CASOptions) -> any PlannedTaskAction { + return GenericCachingTaskAction(enableCacheDebuggingRemarks: enableCacheDebuggingRemarks, enableTaskSandboxEnforcement: enableTaskSandboxEnforcement, sandboxDirectory: sandboxDirectory, extraSandboxSubdirectories: extraSandboxSubdirectories, developerDirectory: developerDirectory, casOptions: casOptions) } public func createInfoPlistProcessorTaskAction(_ contextPath: Path) -> any PlannedTaskAction {