From 911ead8a42063ed1766aa63bbefc0b6c10e223b1 Mon Sep 17 00:00:00 2001 From: Dianna Date: Fri, 7 Feb 2025 02:22:01 +0000 Subject: [PATCH 01/12] return to a symbolgraph-extract based model, and pass -fmodule-map along with includes --- Package.resolved | 20 ++--- Package.swift | 2 - .../Builds/SSGC.PackageBuild.Flags.swift | 16 +--- .../Builds/SSGC.PackageBuild.swift | 73 +++++++++++++------ .../Builds/SSGC.PackageBuildDirectory.swift | 23 +++++- .../Sources/SSGC.ModuleLayout.swift | 11 +++ .../Sources/SSGC.ModuleLayoutError.swift | 7 ++ .../Sources/SSGC.PackageSources.swift | 2 +- .../SSGC.Toolchain.SymbolDumpParameters.swift | 25 +++++++ .../Toolchains/SSGC.Toolchain.swift | 45 +++++++----- 10 files changed, 156 insertions(+), 68 deletions(-) create mode 100644 Sources/SymbolGraphBuilder/Sources/SSGC.ModuleLayoutError.swift create mode 100644 Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpParameters.swift diff --git a/Package.resolved b/Package.resolved index e1507065..65661e86 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "e55ab1bd27ee2fd5913a1597116db214d4284763763234fedd67e53da4f4a1a7", + "originHash" : "7bb6ebf87d5168c153ffa23c4ba2aaac599d53da4735e0203ed072df49cdaa3c", "pins" : [ { "identity" : "indexstore-db", @@ -7,7 +7,7 @@ "location" : "https://github.com/swiftlang/indexstore-db", "state" : { "branch" : "main", - "revision" : "b610e664675a7ae95276313725e80d3191e46dd1" + "revision" : "e08105badfbea2ff27cc4519c4ebb287b91c3cb8" } }, { @@ -16,7 +16,7 @@ "location" : "https://github.com/apple/swift-argument-parser", "state" : { "branch" : "main", - "revision" : "41d58ffe702cfe7b00b2c4c6a6611d9d25a0adc1" + "revision" : "1248cc4bda29753883de0685ab350267a50f3f18" } }, { @@ -60,8 +60,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/tayloraswift/swift-dom", "state" : { - "revision" : "8d98e89a23f48e58d25b530dfabb3d00d3075da2", - "version" : "1.1.1" + "revision" : "9920611b7ba050912bf6e7f7083067aa14816bc0", + "version" : "1.1.2" } }, { @@ -115,7 +115,7 @@ "location" : "https://github.com/swiftlang/swift-lmdb.git", "state" : { "branch" : "main", - "revision" : "35f243462cbb13f43f277e44ce15a7dbc07831a3" + "revision" : "29e6ac814f4f5138576bcb3d3995ed900aa8714e" } }, { @@ -141,8 +141,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio", "state" : { - "revision" : "ba72f31e11275fc5bf060c966cf6c1f36842a291", - "version" : "2.79.0" + "revision" : "dff45738d84a53dbc8ee899c306b3a7227f54f89", + "version" : "2.80.0" } }, { @@ -159,8 +159,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl", "state" : { - "revision" : "c7e95421334b1068490b5d41314a50e70bab23d1", - "version" : "2.29.0" + "revision" : "0cc3528ff48129d64ab9cab0b1cd621634edfc6b", + "version" : "2.29.3" } }, { diff --git a/Package.swift b/Package.swift index d79f3645..2ec60a04 100644 --- a/Package.swift +++ b/Package.swift @@ -111,8 +111,6 @@ let package:Package = .init( from: "1.33.0")), .package(url: "https://github.com/apple/swift-markdown", .upToNextMinor( from: "0.4.0")), - .package(url: "https://github.com/apple/swift-system", .upToNextMinor( - from: "1.4.0")), .package(url: "https://github.com/apple/swift-syntax", from: "600.0.1"), ], diff --git a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.Flags.swift b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.Flags.swift index d10d38a3..cbde2432 100644 --- a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.Flags.swift +++ b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.Flags.swift @@ -23,19 +23,9 @@ extension SSGC.PackageBuild } extension SSGC.PackageBuild.Flags { - /// This is pretty useless right now, because `swift build -emit-symbol-graph` incorrectly - /// duplicates `@_exported` symbols. But Apple might fix this in the future, which would be - /// a much better way to generate symbol graphs than the current method of using - /// `swift symbolgraph-extract`. - consuming - func dumping(symbols options:SSGC.Toolchain.SymbolDumpOptions, - to output:FilePath.Directory) -> Self - { - self.dump(symbols: options, to: output) - return self - } - - mutating + /// This is kept for historical interest. + @available(*, unavailable) + private mutating func dump(symbols options:SSGC.Toolchain.SymbolDumpOptions, to output:FilePath.Directory) { self.swift.append("-emit-symbol-graph") diff --git a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift index 0b20d1d3..ed20388b 100644 --- a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift +++ b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift @@ -68,35 +68,71 @@ extension SSGC.PackageBuild private func modulesToDump( - among modules:SSGC.ModuleGraph) throws -> [(Symbol.Module, [FilePath.Directory])] + among modules:SSGC.ModuleGraph) throws -> [SSGC.Toolchain.SymbolDumpParameters] { - var modulesToDump:[Symbol.Module: [FilePath.Directory]] = [:] + var moduleCohabitants:[Symbol.Module: [SSGC.ModuleLayout]] = [:] for module:SSGC.ModuleLayout in modules.sinkLayout.cultures { let constituents:[SSGC.ModuleLayout] = try modules.constituents(of: module) - let include:[FilePath.Directory] = constituents.reduce(into: [self.scratch.include]) + for constituent:SSGC.ModuleLayout in constituents where + constituent.module.type.hasSymbols { - $0 += $1.include + { + guard + let modules:[SSGC.ModuleLayout] = $0 + else + { + $0 = constituents + return + } + // This is a shared dependency, but we know it can be built alongside a + // smaller set of cohabitating modules. + if modules.count > constituents.count + { + $0 = constituents + } + } (&moduleCohabitants[constituent.id]) } - for constituent:SSGC.ModuleLayout in constituents + } + + let moduleMaps:[Symbol.Module: FilePath] = moduleCohabitants.values.reduce(into: [:]) + { + for layout:SSGC.ModuleLayout in $1 { - // The Swift compiler won’t generate these automatically, so we need to extract - // the symbols manually. - switch constituent.language + switch layout.module.language { - case .c?: break case .cpp?: break + case .c?: break default: continue } - if constituent.module.type.hasSymbols { - modulesToDump[constituent.id] = include - } + $0 = $0 ?? layout.modulemap ?? self.scratch.modulemap( + target: layout.module.name) + } (&$0[layout.module.id]) } } - return modulesToDump.sorted { $0.key < $1.key } + let modules:[SSGC.Toolchain.SymbolDumpParameters] = moduleCohabitants.map + { + $0.value.reduce(into: .init( + moduleName: $0.key, + includePaths: [self.scratch.modules])) + { + let id:Symbol.Module = $1.id + if id != $0.moduleName + { + $0.allowedReexportedModules.append(id) + } + if let file:FilePath = moduleMaps[id] + { + $0.moduleMaps.append(file) + } + + $0.includePaths += $1.include + } + } + return modules.sorted { $0.moduleName < $1.moduleName } } } extension SSGC.PackageBuild @@ -317,9 +353,7 @@ extension SSGC.PackageBuild do { - try toolchain.build(package: self.root, - using: self.scratch, - flags: self.flags.dumping(symbols: .init(), to: artifacts)) + try toolchain.build(package: self.root, using: self.scratch, flags: self.flags) } catch SystemProcessError.exit(let code, let invocation) { @@ -352,13 +386,10 @@ extension SSGC.PackageBuild let artifactsCached:FilePath.Directory = try toolchain.dump( standardLibrary: standardLibrary, cache: cache) - for (module, include):(Symbol.Module, [FilePath.Directory]) in try self.modulesToDump( + for parameters:SSGC.Toolchain.SymbolDumpParameters in try self.modulesToDump( among: modules) { - try toolchain.dump(module: module, - to: artifacts, - options: .init(), - include: include) + try toolchain.dump(parameters: parameters, options: .init(), to: artifacts) } // This step is considered part of documentation building. diff --git a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuildDirectory.swift b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuildDirectory.swift index be0d144c..3e9ac786 100644 --- a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuildDirectory.swift +++ b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuildDirectory.swift @@ -1,3 +1,4 @@ +import Symbols import SystemIO extension SSGC @@ -26,5 +27,25 @@ extension SSGC } extension SSGC.PackageBuildDirectory { - var include:FilePath.Directory { self.location / "\(self.configuration)" } + var index:FilePath.Directory + { + self.location / "\(self.configuration)" / "index" + } + + var modules:FilePath.Directory + { + self.location / "\(self.configuration)" / "Modules" + } + + func modulemap(target:String) -> FilePath + { + self.build(target: target) / "module.modulemap" + } + + /// This takes an unmangled target name, not a c99 name. + private + func build(target:String) -> FilePath.Directory + { + self.location / "\(self.configuration)" / "\(target).build" + } } diff --git a/Sources/SymbolGraphBuilder/Sources/SSGC.ModuleLayout.swift b/Sources/SymbolGraphBuilder/Sources/SSGC.ModuleLayout.swift index e570557c..0005f875 100644 --- a/Sources/SymbolGraphBuilder/Sources/SSGC.ModuleLayout.swift +++ b/Sources/SymbolGraphBuilder/Sources/SSGC.ModuleLayout.swift @@ -20,6 +20,8 @@ extension SSGC private(set) var include:[FilePath.Directory] + private(set) + var modulemap:FilePath? private(set) var module:SymbolGraph.Module @@ -33,6 +35,7 @@ extension SSGC self.resources = [] self.markdown = [] self.include = [] + self.modulemap = nil self.module = module self.origin = origin } @@ -223,6 +226,14 @@ extension SSGC.ModuleLayout case "modulemap": // But modulemaps do. headers.update(with: $0) + + guard case nil = self.modulemap + else + { + throw SSGC.ModuleLayoutError.foundMultipleModulemapFiles + } + + self.modulemap = file.path fallthrough case "c": diff --git a/Sources/SymbolGraphBuilder/Sources/SSGC.ModuleLayoutError.swift b/Sources/SymbolGraphBuilder/Sources/SSGC.ModuleLayoutError.swift new file mode 100644 index 00000000..9bbfa2b5 --- /dev/null +++ b/Sources/SymbolGraphBuilder/Sources/SSGC.ModuleLayoutError.swift @@ -0,0 +1,7 @@ +extension SSGC +{ + enum ModuleLayoutError:Error + { + case foundMultipleModulemapFiles + } +} diff --git a/Sources/SymbolGraphBuilder/Sources/SSGC.PackageSources.swift b/Sources/SymbolGraphBuilder/Sources/SSGC.PackageSources.swift index f98a75f5..9466fc3a 100644 --- a/Sources/SymbolGraphBuilder/Sources/SSGC.PackageSources.swift +++ b/Sources/SymbolGraphBuilder/Sources/SSGC.PackageSources.swift @@ -106,7 +106,7 @@ extension SSGC.PackageSources:SSGC.DocumentationSources #if canImport(IndexStoreDB) let libIndexStore:IndexStoreLibrary = try swift.libIndexStore() - let indexPath:FilePath = self.scratch.include / "index" + let indexPath:FilePath.Directory = self.scratch.index return try IndexStoreDB.init(storePath: "\(indexPath)/store", databasePath: "\(indexPath)/db", library: libIndexStore, diff --git a/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpParameters.swift b/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpParameters.swift new file mode 100644 index 00000000..68a09ebe --- /dev/null +++ b/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpParameters.swift @@ -0,0 +1,25 @@ +import Symbols +import SystemIO + +extension SSGC.Toolchain +{ + struct SymbolDumpParameters + { + let moduleName:Symbol.Module + /// I don’t know why `clang` spells `modulemap` as two words. + var moduleMaps:[FilePath] + var includePaths:[FilePath.Directory] + var allowedReexportedModules:[Symbol.Module] + + init(moduleName:Symbol.Module, + moduleMaps:[FilePath] = [], + includePaths:[FilePath.Directory] = [], + allowedReexportedModules:[Symbol.Module] = []) + { + self.moduleName = moduleName + self.moduleMaps = moduleMaps + self.includePaths = includePaths + self.allowedReexportedModules = allowedReexportedModules + } + } +} diff --git a/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.swift b/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.swift index 581ea53c..66a0d102 100644 --- a/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.swift +++ b/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.swift @@ -223,10 +223,10 @@ extension SSGC.Toolchain for module:SymbolGraph.Module in standardLibrary.modules { - try self.dump(module: module.id, - to: temporary, + try self.dump( + parameters: .init(moduleName: module.id, allowedReexportedModules: dumped), options: options, - allowedReexportedModules: dumped) + to: temporary) dumped.append(module.id) } @@ -239,19 +239,17 @@ extension SSGC.Toolchain /// Dumps the symbols for the given targets, using the `output` workspace as the /// output directory. - func dump(module id:Symbol.Module, - to output:FilePath.Directory, + func dump(parameters:SymbolDumpParameters, options:SymbolDumpOptions, - include:[FilePath.Directory] = [], - allowedReexportedModules:[Symbol.Module] = []) throws + to output:FilePath.Directory) throws { - print("Dumping symbols for module '\(id)'") + print("Dumping symbols for module '\(parameters.moduleName)'") var arguments:[String] = [ "symbolgraph-extract", - "-module-name", "\(id)", + "-module-name", "\(parameters.moduleName)", "-target", "\(self.splash.triple)", "-output-dir", "\(output.path)", ] @@ -271,10 +269,12 @@ extension SSGC.Toolchain { arguments.append("-skip-inherited-docs") } - if self.splash.swift.version >= .v(6, 0, 0), !allowedReexportedModules.isEmpty + if !parameters.allowedReexportedModules.isEmpty { - let whitelist:String = allowedReexportedModules.lazy.map { "\($0)" }.joined( - separator: ",") + let whitelist:String = parameters.allowedReexportedModules.lazy.map + { + "\($0)" + }.joined(separator: ",") arguments.append(""" -experimental-allowed-reexported-modules=\(whitelist) @@ -299,10 +299,15 @@ extension SSGC.Toolchain { arguments.append("-pretty-print") } - for include:FilePath.Directory in include + for includePath:FilePath.Directory in parameters.includePaths { arguments.append("-I") - arguments.append("\(include)") + arguments.append("\(includePath)") + } + for moduleMap:FilePath in parameters.moduleMaps + { + arguments.append("-Xcc") + arguments.append("-fmodule-map-file=\(moduleMap)") } let environment:SystemProcess.Environment = .inherit @@ -330,23 +335,23 @@ extension SSGC.Toolchain { case 139: print(""" - Failed to dump symbols for module '\(id)' due to SIGSEGV \ - from 'swift symbolgraph-extract'. \ + Failed to dump symbols for module '\(parameters.moduleName)' due to \ + SIGSEGV from 'swift symbolgraph-extract'. \ This is a known bug in the Apple Swift compiler; see \ https://github.com/apple/swift/issues/68767. """) case 134: print(""" - Failed to dump symbols for module '\(id)' due to SIGABRT \ - from 'swift symbolgraph-extract'. \ + Failed to dump symbols for module '\(parameters.moduleName)' due to \ + SIGABRT from 'swift symbolgraph-extract'. \ This is a known bug in the Apple Swift compiler; see \ https://github.com/swiftlang/swift/issues/75318. """) case let code: print(""" - Failed to dump symbols for module '\(id)' due to exit code \(code) \ - from 'swift symbolgraph-extract'. \ + Failed to dump symbols for module '\(parameters.moduleName)' due to exit \ + code \(code) from 'swift symbolgraph-extract'. \ If the output above indicates 'swift symbolgraph-extract' exited \ gracefully, this is most likely because the module.modulemap file declares \ a different module name than we detected from the package manifest. From 4e0189d57d23c9253f04a64dc3388addb9a7eb0c Mon Sep 17 00:00:00 2001 From: Dianna Date: Fri, 7 Feb 2025 02:24:53 +0000 Subject: [PATCH 02/12] =?UTF-8?q?we=20shouldn=E2=80=99t=20actually=20need?= =?UTF-8?q?=20include=20paths=20if=20we=20have=20modulemaps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift | 2 -- 1 file changed, 2 deletions(-) diff --git a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift index ed20388b..edf7c397 100644 --- a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift +++ b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift @@ -128,8 +128,6 @@ extension SSGC.PackageBuild { $0.moduleMaps.append(file) } - - $0.includePaths += $1.include } } return modules.sorted { $0.moduleName < $1.moduleName } From c1c7690ea66e335c1e17202afc7073744aa3e134 Mon Sep 17 00:00:00 2001 From: Dianna Date: Fri, 7 Feb 2025 23:39:57 +0000 Subject: [PATCH 03/12] i think we need --no-parallel here --- .github/workflows/{test.yml => Test.yml} | 11 +++++------ README.md | 2 +- Scripts/Linux/TestAll | 8 ++------ 3 files changed, 8 insertions(+), 13 deletions(-) rename .github/workflows/{test.yml => Test.yml} (90%) diff --git a/.github/workflows/test.yml b/.github/workflows/Test.yml similarity index 90% rename from .github/workflows/test.yml rename to .github/workflows/Test.yml index a88d8d19..64787250 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/Test.yml @@ -57,10 +57,9 @@ jobs: run: | swift --version swift build -c release \ - --build-tests \ + --product ssgc \ --explicit-target-dependency-import-check=error \ - -Xcxx -I$SWIFT_INSTALLATION/lib/swift \ - -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block + --destination Scripts/Linux/x86_64/x86_64.json mkdir $HOME/bin mv .build/release/ssgc $HOME/bin echo "$HOME/bin" >> $GITHUB_PATH @@ -71,10 +70,9 @@ jobs: - name: Validate packages run: | swift test -c release \ - --skip-build \ + --no-parallel \ --filter SymbolGraphValidationTests \ - -Xcxx -I$SWIFT_INSTALLATION/lib/swift \ - -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block + --destination Scripts/Linux/x86_64/x86_64.json macos: runs-on: macos-15 @@ -102,4 +100,5 @@ jobs: - name: Validate packages run: | swift test -c release \ + --no-parallel \ --filter SymbolGraphValidationTests diff --git a/README.md b/README.md index d173a516..97b675b6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ unidoc -[![ci build status](https://github.com/tayloraswift/swift-unidoc/actions/workflows/test.yml/badge.svg)](https://github.com/tayloraswift/swift-unidoc/actions/workflows/test.yml) +[![ci build status](https://github.com/tayloraswift/swift-unidoc/actions/workflows/Test.yml/badge.svg)](https://github.com/tayloraswift/swift-unidoc/actions/workflows/Test.yml) [![ci build status](https://github.com/tayloraswift/swift-unidoc/actions/workflows/docs.yml/badge.svg)](https://github.com/tayloraswift/swift-unidoc/actions/workflows/docs.yml) [![ci build status](https://github.com/tayloraswift/swift-unidoc/actions/workflows/Deploy.yml/badge.svg)](https://github.com/tayloraswift/swift-unidoc/actions/workflows/Deploy.yml) diff --git a/Scripts/Linux/TestAll b/Scripts/Linux/TestAll index c15c748e..1993be46 100755 --- a/Scripts/Linux/TestAll +++ b/Scripts/Linux/TestAll @@ -6,18 +6,14 @@ export UNIDOC_ENABLE_INDEXSTORE=1 swift --version swift build -c release \ --explicit-target-dependency-import-check=error \ - -Xcxx -I$SWIFT_INSTALLATION/lib/swift \ - -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block \ - --build-tests + --destination Scripts/Linux/x86_64/x86_64.json Scripts/Linux/GenerateTestSymbolGraphs swift test -c release \ --explicit-target-dependency-import-check=error \ - -Xcxx -I$SWIFT_INSTALLATION/lib/swift \ - -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block \ + --destination Scripts/Linux/x86_64/x86_64.json \ --no-parallel \ - --skip-build \ --skip SymbolGraphValidationTests \ --disable-testable-imports From 9d11424bc481ff73214bd751c00f11f48e7f7dcb Mon Sep 17 00:00:00 2001 From: Dianna Date: Fri, 7 Feb 2025 23:44:27 +0000 Subject: [PATCH 04/12] which swift? --- .github/workflows/Test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index 64787250..13bfee45 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -55,6 +55,7 @@ jobs: - name: Build release run: | + which swift swift --version swift build -c release \ --product ssgc \ From 3db147c7ba99bd01ca7079672717c3546ba31f56 Mon Sep 17 00:00:00 2001 From: Dianna Date: Fri, 7 Feb 2025 23:54:02 +0000 Subject: [PATCH 05/12] the runner uses a different runtime layout --- .github/workflows/Test.yml | 6 ++++-- Scripts/Linux/TestAll | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index 13bfee45..d04e8c07 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -60,7 +60,8 @@ jobs: swift build -c release \ --product ssgc \ --explicit-target-dependency-import-check=error \ - --destination Scripts/Linux/x86_64/x86_64.json + --Xcxx -I$SWIFT_INSTALLATION/lib/swift \ + --Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block mkdir $HOME/bin mv .build/release/ssgc $HOME/bin echo "$HOME/bin" >> $GITHUB_PATH @@ -73,7 +74,8 @@ jobs: swift test -c release \ --no-parallel \ --filter SymbolGraphValidationTests \ - --destination Scripts/Linux/x86_64/x86_64.json + --Xcxx -I$SWIFT_INSTALLATION/lib/swift \ + --Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block macos: runs-on: macos-15 diff --git a/Scripts/Linux/TestAll b/Scripts/Linux/TestAll index 1993be46..0ee34aa6 100755 --- a/Scripts/Linux/TestAll +++ b/Scripts/Linux/TestAll @@ -6,13 +6,15 @@ export UNIDOC_ENABLE_INDEXSTORE=1 swift --version swift build -c release \ --explicit-target-dependency-import-check=error \ - --destination Scripts/Linux/x86_64/x86_64.json + -Xcxx -I$SWIFT_INSTALLATION/lib/swift \ + -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block Scripts/Linux/GenerateTestSymbolGraphs swift test -c release \ --explicit-target-dependency-import-check=error \ - --destination Scripts/Linux/x86_64/x86_64.json \ + -Xcxx -I$SWIFT_INSTALLATION/lib/swift \ + -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block \ --no-parallel \ --skip SymbolGraphValidationTests \ --disable-testable-imports From 2e8b8f30a4274b7cc4dfe55ad3b0f43d52e677be Mon Sep 17 00:00:00 2001 From: Dianna Date: Fri, 7 Feb 2025 23:58:37 +0000 Subject: [PATCH 06/12] ? --- .github/workflows/Test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index d04e8c07..6b264ab4 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -55,7 +55,6 @@ jobs: - name: Build release run: | - which swift swift --version swift build -c release \ --product ssgc \ From 11dc1ddcec4b300bb9a248d1aa3d10f5af52f22b Mon Sep 17 00:00:00 2001 From: Dianna Date: Sat, 8 Feb 2025 00:04:10 +0000 Subject: [PATCH 07/12] no double dash??? --- .github/workflows/Test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index 6b264ab4..4d283926 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -59,8 +59,8 @@ jobs: swift build -c release \ --product ssgc \ --explicit-target-dependency-import-check=error \ - --Xcxx -I$SWIFT_INSTALLATION/lib/swift \ - --Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block + -Xcxx -I$SWIFT_INSTALLATION/lib/swift \ + -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block mkdir $HOME/bin mv .build/release/ssgc $HOME/bin echo "$HOME/bin" >> $GITHUB_PATH @@ -73,8 +73,8 @@ jobs: swift test -c release \ --no-parallel \ --filter SymbolGraphValidationTests \ - --Xcxx -I$SWIFT_INSTALLATION/lib/swift \ - --Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block + -Xcxx -I$SWIFT_INSTALLATION/lib/swift \ + -Xcxx -I$SWIFT_INSTALLATION/lib/swift/Block macos: runs-on: macos-15 From c23d8851312edd6d08861ff36c50d7445b6157a0 Mon Sep 17 00:00:00 2001 From: Dianna Date: Wed, 12 Feb 2025 02:26:53 +0000 Subject: [PATCH 08/12] centralize build planning logic in ModuleGraph --- .../Packages/PackageNode.Densified.swift | 25 + .../PackageGraphs/Packages/PackageNode.swift | 92 +-- .../Manifests/PackageNode (ext).swift | 58 +- .../Builds/SSGC.PackageBuild.Flags.swift | 29 - .../Builds/SSGC.PackageBuild.swift | 150 ++--- .../PackageNode (ext).swift | 22 - ...rtex.swift => SSGC.ModuleGraph.Node.swift} | 10 +- ... => SSGC.ModuleGraph.NodeIdentifier.swift} | 6 +- .../SymbolGraphBuilder/SSGC.ModuleGraph.swift | 547 +++++++++++++++--- .../SSGC.PackageGraph.swift | 70 ++- .../Sources/SSGC.BookSources.swift | 52 +- .../Sources/SSGC.DocumentationSources.swift | 24 +- .../Sources/SSGC.PackageLayout.swift | 51 -- .../Sources/SSGC.PackageRoot.swift | 97 ++++ .../Sources/SSGC.PackageSources.swift | 74 +-- .../SSGC.StandardLibrary.swift | 253 -------- .../SSGC.StandardLibraryBuild.swift | 10 +- .../SSGC.StandardLibrarySources.swift | 20 +- .../SSGC.Toolchain.SymbolDumpOptions.swift | 22 +- .../SSGC.Toolchain.SymbolDumpParameters.swift | 25 - .../Toolchains/SSGC.Toolchain.swift | 70 +-- Sources/SymbolGraphBuilderTests/Main.swift | 2 +- 22 files changed, 844 insertions(+), 865 deletions(-) create mode 100644 Sources/PackageGraphs/Packages/PackageNode.Densified.swift delete mode 100644 Sources/SymbolGraphBuilder/PackageNode (ext).swift rename Sources/SymbolGraphBuilder/{SSGC.ModuleGraph.Vertex.swift => SSGC.ModuleGraph.Node.swift} (68%) rename Sources/SymbolGraphBuilder/{SSGC.PackageGraph.Vertex.swift => SSGC.ModuleGraph.NodeIdentifier.swift} (74%) delete mode 100644 Sources/SymbolGraphBuilder/Sources/SSGC.PackageLayout.swift delete mode 100644 Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibrary.swift delete mode 100644 Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpParameters.swift diff --git a/Sources/PackageGraphs/Packages/PackageNode.Densified.swift b/Sources/PackageGraphs/Packages/PackageNode.Densified.swift new file mode 100644 index 00000000..8fbe6953 --- /dev/null +++ b/Sources/PackageGraphs/Packages/PackageNode.Densified.swift @@ -0,0 +1,25 @@ +import SymbolGraphs +import Symbols + +extension PackageNode +{ + @frozen public + struct Densified + { + public + let dependencies:[any Identifiable] + public + let products:[SymbolGraph.Product] + public + let modules:[SymbolGraph.Module] + + init(dependencies:[any Identifiable], + products:[SymbolGraph.Product], + modules:[SymbolGraph.Module]) + { + self.dependencies = dependencies + self.products = products + self.modules = modules + } + } +} diff --git a/Sources/PackageGraphs/Packages/PackageNode.swift b/Sources/PackageGraphs/Packages/PackageNode.swift index 7c14e54c..7587fec0 100644 --- a/Sources/PackageGraphs/Packages/PackageNode.swift +++ b/Sources/PackageGraphs/Packages/PackageNode.swift @@ -13,6 +13,9 @@ struct PackageNode:Identifiable public var dependencies:[any Identifiable] + /// The name of the snippets directory. + public + var snippets:String public var products:[SymbolGraph.Product] public @@ -26,6 +29,7 @@ struct PackageNode:Identifiable @inlinable public init(id:Symbol.Package, dependencies:[any Identifiable], + snippets:String, products:[SymbolGraph.Product], modules:[SymbolGraph.Module], exclude:[[String]], @@ -33,6 +37,7 @@ struct PackageNode:Identifiable { self.id = id self.dependencies = dependencies + self.snippets = snippets self.products = products self.modules = modules self.exclude = exclude @@ -42,46 +47,51 @@ struct PackageNode:Identifiable extension PackageNode { public consuming - func flattened(dependencies:[PackageNode]) throws -> Self - { - try self.flatten(dependencies: dependencies) - return self - } - - mutating - func flatten(dependencies:[PackageNode]) throws + func joined(with dependencies:[PackageNode]) throws -> ([Densified], Densified) { - var nodes:DigraphExplorer.Nodes = .init() - for package:PackageNode in dependencies + let nodes:DigraphExplorer.Nodes = try dependencies.reduce(into: .init()) { - for product:SymbolGraph.Product in package.products + for product:SymbolGraph.Product in $1.products { - try nodes.index(.init(id: .init(name: product.name, package: package.id), - predecessors: product.dependencies)) + let id:Symbol.Product = .init(name: product.name, package: $1.id) + try $0.index(.init(id: id, predecessors: product.dependencies)) } } + let densifiedUpstream:[Densified] = try dependencies.map + { + try $0.densified(products: nodes, packages: dependencies) + } + let densifiedSink:Densified = try self.densified( + products: nodes, + packages: dependencies) + + return (densifiedUpstream, densifiedSink) + } + + private consuming + func densified( + products:DigraphExplorer.Nodes, + packages:[PackageNode]) throws -> Densified + { var cache:[Symbol.Product: [Symbol.Product]] = [:] - self.products = try self.products.map + for i:Int in self.products.indices { - .init(name: $0.name, type: $0.type, - dependencies: try nodes.included(by: $0.dependencies, cache: &cache), - cultures: $0.cultures) + try + { + $0 = try products.included(by: $0, cache: &cache) + } (&self.products[i].dependencies) } - self.modules = try self.modules.map + for i:Int in self.modules.indices { - .init(name: $0.name, type: $0.type, dependencies: .init( - products: try nodes.included(by: $0.dependencies.products, - cache: &cache), - modules: $0.dependencies.modules), - location: $0.location) + try + { + $0 = try products.included(by: $0, cache: &cache) + } (&self.modules[i].dependencies.products) } - let declared:[Symbol.Package: any Identifiable] = - self.dependencies.reduce(into: [:]) { $0[$1.id] = $1 } - - let actuallyUsed:Set = cache.values.reduce(into: []) + let dependenciesActuallyUsed:Set = cache.values.reduce(into: []) { for product:Symbol.Product in $1 { @@ -89,28 +99,20 @@ extension PackageNode } } - let directedEdges:[(Symbol.Package, Symbol.Package)] = dependencies.reduce(into: []) + let dependenciesDeclared:[Symbol.Package: any Identifiable] = + self.dependencies.reduce(into: [:]) { $0[$1.id] = $1 } + + let dependenciesBlamed:[any Identifiable] = packages.reduce( + into: []) { - for dependency:any Identifiable in $1.dependencies + if dependenciesActuallyUsed.contains($1.id) { - $0.append((dependency.id, $1.id)) + $0.append(dependenciesDeclared[$1.id] ?? TransitiveDependency.init(id: $1.id)) } } - // FIXME: in Swift 6, it will be legal to have cyclic package dependencies! - guard - let dependenciesOrdered:[PackageNode] = dependencies.sortedTopologically( - by: directedEdges) - else - { - throw DigraphCycleError.init() - } - - self.dependencies = dependenciesOrdered.compactMap - { - actuallyUsed.contains($0.id) - ? declared[$0.id] ?? TransitiveDependency.init(id: $0.id) - : nil - } + return .init(dependencies: dependenciesBlamed, + products: self.products, + modules: self.modules) } } diff --git a/Sources/PackageMetadata/Manifests/PackageNode (ext).swift b/Sources/PackageMetadata/Manifests/PackageNode (ext).swift index 7bc7b3af..2307f07e 100644 --- a/Sources/PackageMetadata/Manifests/PackageNode (ext).swift +++ b/Sources/PackageMetadata/Manifests/PackageNode (ext).swift @@ -4,48 +4,30 @@ import Symbols extension PackageNode { - public static - func all(flattening manifest:borrowing SPM.Manifest, + public + init(from manifest:borrowing SPM.Manifest, on platform:borrowing SymbolGraphMetadata.Platform, - as id:__owned Symbol.Package) throws -> Self + as id:Symbol.Package, + filter:borrowing (SymbolGraph.ProductType) throws -> Bool = { _ in true }) throws { - try .init(as: id, flattening: manifest, platform: platform) { _ in true } - } -} -extension PackageNode -{ - private - init(as id:Symbol.Package, - flattening manifest:borrowing SPM.Manifest, - platform:borrowing SymbolGraphMetadata.Platform, - filter predicate:(SymbolGraph.ProductType) throws -> Bool) throws - { - try self.init(id: id, - predecessors: manifest.dependencies, - platform: platform, - products: try manifest.products.values.filter { try predicate($0.type) }, - targets: try .init(indexing: manifest.targets.values), - root: manifest.root) - } - private - init(id:Symbol.Package, - predecessors:[SPM.Manifest.Dependency], - platform:borrowing SymbolGraphMetadata.Platform, - products:borrowing [SPM.Manifest.Product], - targets:borrowing DigraphExplorer.Nodes, - root:Symbol.FileBase) throws - { - let ordering:[TargetNode] = try targets.included(by: products, + let products:[SPM.Manifest.Product] = try manifest.products.values.filter + { + try filter($0.type) + } + let modules:DigraphExplorer.Nodes = try .init( + indexing: manifest.targets.values) + let modulesInOrder:[TargetNode] = try modules.included(by: products, on: platform) self.init(id: id, - dependencies: predecessors, + dependencies: manifest.dependencies, + snippets: manifest.snippets ?? "Snippets", products: try products.map { - let constituents:Set = try targets.included(by: $0, on: platform) + let constituents:Set = try modules.included(by: $0, on: platform) var (modules, dependencies):([Int], Set) = ([], []) - for (index, constituent):(Int, TargetNode) in ordering.enumerated() + for (index, constituent):(Int, TargetNode) in modulesInOrder.enumerated() where constituents.contains(constituent.name) { dependencies.formUnion(constituent.dependencies.products(on: platform)) @@ -55,12 +37,12 @@ extension PackageNode dependencies: dependencies.sorted(), cultures: modules) }, - modules: try ordering.map + modules: try modulesInOrder.map { - let constituents:Set = try targets.included(by: $0, on: platform) + let constituents:Set = try modules.included(by: $0, on: platform) var (modules, dependencies):([Int], Set) = ([], []) - for (index, constituent):(Int, TargetNode) in ordering.enumerated() + for (index, constituent):(Int, TargetNode) in modulesInOrder.enumerated() where constituents.contains(constituent.name) { dependencies.formUnion(constituent.dependencies.products(on: platform)) @@ -76,7 +58,7 @@ extension PackageNode modules: modules), location: $0.path) }, - exclude: ordering.map(\.exclude), - root: root) + exclude: modulesInOrder.map(\.exclude), + root: manifest.root) } } diff --git a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.Flags.swift b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.Flags.swift index cbde2432..65558bad 100644 --- a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.Flags.swift +++ b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.Flags.swift @@ -21,32 +21,3 @@ extension SSGC.PackageBuild } } } -extension SSGC.PackageBuild.Flags -{ - /// This is kept for historical interest. - @available(*, unavailable) - private mutating - func dump(symbols options:SSGC.Toolchain.SymbolDumpOptions, to output:FilePath.Directory) - { - self.swift.append("-emit-symbol-graph") - - self.swift.append("-emit-symbol-graph-dir") - self.swift.append("\(output.path)") - - self.swift.append("-symbol-graph-minimum-access-level") - self.swift.append("\(options.minimumACL)") - - if options.emitExtensionBlockSymbols - { - self.swift.append("-emit-extension-block-symbols") - } - if options.includeInterfaceSymbols - { - self.swift.append("-include-spi-symbols") - } - if options.skipInheritedDocs - { - self.swift.append("-skip-inherited-docs") - } - } -} diff --git a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift index edf7c397..1ea63359 100644 --- a/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift +++ b/Sources/SymbolGraphBuilder/Builds/SSGC.PackageBuild.swift @@ -65,73 +65,6 @@ extension SSGC.PackageBuild return versions } - - private - func modulesToDump( - among modules:SSGC.ModuleGraph) throws -> [SSGC.Toolchain.SymbolDumpParameters] - { - var moduleCohabitants:[Symbol.Module: [SSGC.ModuleLayout]] = [:] - for module:SSGC.ModuleLayout in modules.sinkLayout.cultures - { - let constituents:[SSGC.ModuleLayout] = try modules.constituents(of: module) - for constituent:SSGC.ModuleLayout in constituents where - constituent.module.type.hasSymbols - { - { - guard - let modules:[SSGC.ModuleLayout] = $0 - else - { - $0 = constituents - return - } - // This is a shared dependency, but we know it can be built alongside a - // smaller set of cohabitating modules. - if modules.count > constituents.count - { - $0 = constituents - } - } (&moduleCohabitants[constituent.id]) - } - } - - let moduleMaps:[Symbol.Module: FilePath] = moduleCohabitants.values.reduce(into: [:]) - { - for layout:SSGC.ModuleLayout in $1 - { - switch layout.module.language - { - case .cpp?: break - case .c?: break - default: continue - } - - { - $0 = $0 ?? layout.modulemap ?? self.scratch.modulemap( - target: layout.module.name) - } (&$0[layout.module.id]) - } - } - - let modules:[SSGC.Toolchain.SymbolDumpParameters] = moduleCohabitants.map - { - $0.value.reduce(into: .init( - moduleName: $0.key, - includePaths: [self.scratch.modules])) - { - let id:Symbol.Module = $1.id - if id != $0.moduleName - { - $0.allowedReexportedModules.append(id) - } - if let file:FilePath = moduleMaps[id] - { - $0.moduleMaps.append(file) - } - } - } - return modules.sorted { $0.moduleName < $1.moduleName } - } } extension SSGC.PackageBuild { @@ -266,16 +199,19 @@ extension SSGC.PackageBuild } // This step is considered part of documentation building. - let sources:SSGC.BookSources + let modules:SSGC.ModuleGraph do { - sources = try .init(scanning: self.root) + modules = try .book(name: self.id.package, root: self.root) } catch let error { throw SSGC.DocumentationBuildError.scanning(error) } + let sources:SSGC.BookSources = .init(modules: modules, + root: .init(self.root.path.string)) + let metadata:SymbolGraphMetadata = .init( package: .init( scope: self.id.pin?.location.owner, @@ -296,7 +232,7 @@ extension SSGC.PackageBuild @_spi(testable) public func compileSwiftPM(updating status:SSGC.StatusStream? = nil, - cache:FilePath.Directory, + cache _:FilePath.Directory, with toolchain:SSGC.Toolchain, clean:Bool = true) throws -> (SymbolGraphMetadata, SSGC.PackageSources) { @@ -319,7 +255,7 @@ extension SSGC.PackageBuild } let manifestVersions:[MinorVersion] = try self.listExtraManifests() - var manifest:SPM.Manifest = try toolchain.manifest(package: self.root, + let manifest:SPM.Manifest = try toolchain.manifest(package: self.root, json: artifacts / "\(self.id.package).package.json", leaf: true) @@ -358,7 +294,7 @@ extension SSGC.PackageBuild throw SSGC.PackageBuildError.swift_build(code, invocation) } - var packages:SSGC.PackageGraph = .init(platform: try toolchain.platform()) + var packageGraph:SSGC.PackageGraph = .init(platform: try toolchain.platform()) for pin:SPM.DependencyPin in pins { @@ -369,56 +305,38 @@ extension SSGC.PackageBuild json: artifacts / "\(pin.identity).package.json", leaf: false) - packages.attach(manifest, as: pin.identity) + packageGraph.attach(manifest, as: pin.identity) } - let standardLibrary:SSGC.StandardLibrary = .init(platform: packages.platform, + let packageNodes:([PackageNode], sink:PackageNode) = try packageGraph.join( + dependencies: pins, + sinkManifest: manifest, + sinkPackage: self.id.package) + + let stdlib:SSGC.ModuleGraph = .stdlib( + platform: packageGraph.platform, version: toolchain.splash.swift.version.minor) - let modules:SSGC.ModuleGraph = try packages.join(dependencies: pins, - standardLibrary: standardLibrary, - with: &manifest, - as: self.id.package) + let modules:SSGC.ModuleGraph = try .package(sink: packageNodes.sink, + dependencies: packageNodes.0, + substrate: stdlib.cultures.map(\.layout), + sparseEdges: packageGraph.sparseEdges) // Dump the standard library’s symbols, unless they’re already cached. - let artifactsCached:FilePath.Directory = try toolchain.dump( - standardLibrary: standardLibrary, - cache: cache) - for parameters:SSGC.Toolchain.SymbolDumpParameters in try self.modulesToDump( - among: modules) - { - try toolchain.dump(parameters: parameters, options: .init(), to: artifacts) - } + let symbolsCached:FilePath.Directory = try toolchain.dump(stdlib: stdlib, + cache: artifacts) - // This step is considered part of documentation building. - var sources:SSGC.PackageSources = .init(scratch: self.scratch, - symbols: [artifacts, artifactsCached], - modules: modules) - do - { - let snippetsDirectory:FilePath.Component - if let customDirectory:String = manifest.snippets - { - guard - let customDirectory:FilePath.Component = .init(customDirectory) - else - { - throw SSGC.SnippetDirectoryError.invalid(customDirectory) - } - - snippetsDirectory = customDirectory - } - else - { - snippetsDirectory = "Snippets" - } + let symbols:FilePath.Directory = artifacts / "symbols" + try symbols.create(clean: false) - try sources.detect(snippets: snippetsDirectory) - } - catch let error - { - throw SSGC.DocumentationBuildError.scanning(error) - } + try toolchain.dump(scratch: self.scratch, modules: modules, to: symbols) + + // This step is considered part of documentation building. + let sources:SSGC.PackageSources = .init( + modules: modules, + symbols: [symbols, symbolsCached], + scratch: self.scratch, + root: packageNodes.sink.root) let metadata:SymbolGraphMetadata = .init( package: .init( @@ -430,8 +348,8 @@ extension SSGC.PackageBuild tools: manifest.format, manifests: manifestVersions, requirements: manifest.requirements, - dependencies: try modules.dependenciesUsed(pins: pins), - products: .init(viewing: modules.sink.products), + dependencies: try modules.dependenciesUsed(among: pins), + products: .init(viewing: modules.products), display: manifest.name, root: sources.prefix) diff --git a/Sources/SymbolGraphBuilder/PackageNode (ext).swift b/Sources/SymbolGraphBuilder/PackageNode (ext).swift deleted file mode 100644 index e08c7b44..00000000 --- a/Sources/SymbolGraphBuilder/PackageNode (ext).swift +++ /dev/null @@ -1,22 +0,0 @@ -import PackageGraphs -import PackageMetadata -import SHA1 -import SymbolGraphs -import Symbols -import SystemIO - -extension PackageNode -{ - func pin(to pins:[SPM.DependencyPin]) throws -> [SymbolGraphMetadata.Dependency] - { - let pins:SPM.DependencyPins = try .init(indexing: pins) - return try self.dependencies.map - { - let pin:SPM.DependencyPin = try pins($0.id) - return .init(package: .init(scope: pin.location.owner, name: $0.id), - requirement: ($0 as? SPM.Manifest.Dependency)?.requirement?.stable, - revision: pin.revision, - version: pin.version) - } - } -} diff --git a/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.Vertex.swift b/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.Node.swift similarity index 68% rename from Sources/SymbolGraphBuilder/SSGC.ModuleGraph.Vertex.swift rename to Sources/SymbolGraphBuilder/SSGC.ModuleGraph.Node.swift index 931ba343..d8bd3d7c 100644 --- a/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.Vertex.swift +++ b/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.Node.swift @@ -3,21 +3,23 @@ import Symbols extension SSGC.ModuleGraph { - struct Vertex:Identifiable + final + class Node:Identifiable { - let id:SSGC.PackageGraph.Vertex + let id:NodeIdentifier let layout:SSGC.ModuleLayout private - init(id:SSGC.PackageGraph.Vertex, layout:SSGC.ModuleLayout) + init(id:NodeIdentifier, layout:SSGC.ModuleLayout) { self.id = id self.layout = layout } } } -extension SSGC.ModuleGraph.Vertex +extension SSGC.ModuleGraph.Node { + convenience init(module layout:SSGC.ModuleLayout, in package:Symbol.Package) { self.init(id: .init(package: package, module: layout.module.id), layout: layout) diff --git a/Sources/SymbolGraphBuilder/SSGC.PackageGraph.Vertex.swift b/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.NodeIdentifier.swift similarity index 74% rename from Sources/SymbolGraphBuilder/SSGC.PackageGraph.Vertex.swift rename to Sources/SymbolGraphBuilder/SSGC.ModuleGraph.NodeIdentifier.swift index 21a7ed7d..390d4a1e 100644 --- a/Sources/SymbolGraphBuilder/SSGC.PackageGraph.Vertex.swift +++ b/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.NodeIdentifier.swift @@ -1,8 +1,8 @@ import Symbols -extension SSGC.PackageGraph +extension SSGC.ModuleGraph { - struct Vertex:Equatable, Hashable + struct NodeIdentifier:Equatable, Hashable { let package:Symbol.Package let module:Symbol.Module @@ -14,7 +14,7 @@ extension SSGC.PackageGraph } } } -extension SSGC.PackageGraph.Vertex:Comparable +extension SSGC.ModuleGraph.NodeIdentifier:Comparable { static func < (a:Self, b:Self) -> Bool { diff --git a/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.swift b/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.swift index d775759f..68b2bdb0 100644 --- a/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.swift +++ b/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.swift @@ -1,117 +1,239 @@ import PackageGraphs import PackageMetadata +import SemanticVersions import SymbolGraphs import Symbols +import SystemIO extension SSGC { + @_spi(testable) public struct ModuleGraph { private - var productPartitions:[Symbol.Product: [ModuleLayout]] = [:] - /// Note: the standard library is not modeled as part of the module graph, as its - /// modules are assumed to be dependencies of every module in the module graph. + let upstreamPackages:[any Identifiable] + /// Individualized dependencies of each module in the graph, including all transitive + /// dependencies, and the module itself, but not including constant ``substrate`` + /// dependencies. private - let standardLibrary:[ModuleLayout] + let constituents:[NodeIdentifier: [Node]] + /// Modules that are assumed to be dependencies of every module in the module graph. private - let sparseEdges:[(PackageGraph.Vertex, PackageGraph.Vertex)] + let substrate:[ModuleLayout] - let sinkLayout:PackageLayout - let sink:PackageNode + let products:[SymbolGraph.Product] + let cultures:[Node] + + @_spi(testable) public + let snippets:[LazyFile] private - init(standardLibrary:[ModuleLayout], - sparseEdges:[(PackageGraph.Vertex, PackageGraph.Vertex)], - sinkLayout:PackageLayout, - sink:PackageNode) + init(upstreamPackages:[any Identifiable], + constituents:[NodeIdentifier: [Node]], + substrate:[ModuleLayout], + products:[SymbolGraph.Product], + cultures:[Node], + snippets:[LazyFile]) { - self.standardLibrary = standardLibrary - self.sparseEdges = sparseEdges - self.sinkLayout = sinkLayout - self.sink = sink + self.upstreamPackages = upstreamPackages + self.constituents = constituents + self.substrate = substrate + self.products = products + self.cultures = cultures + self.snippets = snippets } } } extension SSGC.ModuleGraph { - init(standardLibrary:[SSGC.ModuleLayout], - sparseEdges:[(SSGC.PackageGraph.Vertex, SSGC.PackageGraph.Vertex)], - dependencies:[PackageNode], - sink:PackageNode) throws + static func book(name id:Symbol.Package, root:FilePath.Directory) throws -> Self + { + let root:SSGC.PackageRoot = .init(normalizing: root) + let layouts:[SSGC.ModuleLayout] = try root.chapters() + let modules:[Node] = layouts.map { .init(module: $0, in: id) } + + return .init( + upstreamPackages: [], + constituents: modules.reduce(into: [:]) { $0[$1.id] = [$1] }, + substrate: [], + products: [], + cultures: modules, + snippets: []) + } + + static func stdlib(platform:SymbolGraphMetadata.Platform, version:MinorVersion) -> Self { - let densified:PackageNode = try sink.flattened(dependencies: dependencies) - self.init(standardLibrary: standardLibrary, - sparseEdges: sparseEdges, - sinkLayout: try .init(scanning: densified), - sink: consume densified) + let stdlib:(products:[SymbolGraph.Product], modules:[SymbolGraph.Module]) - for node:PackageNode in dependencies + switch (platform, version) { - let dependencyLayout:SSGC.PackageLayout = try .init(scanning: node) - self.addPartitions(of: dependencyLayout.cultures, in: node) + case (.linux, .v(6, 0)): stdlib = Self.linux_6_0 + case (.linux, _): stdlib = Self.linux_5_10 + case (.macOS, .v(6, 0)): stdlib = Self.macOS_6_0 + case (.macOS, _): stdlib = Self.macOS_5_10 + default: fatalError("Unsupported platform: \(platform)") } - self.addPartitions(of: self.sinkLayout.cultures, in: self.sink) + var constituents:[NodeIdentifier: [Node]] = [:] + let modules:[Node] = stdlib.modules.map + { + .init(module: .init(toolchain: $0), in: .swift) + } + for module:Node in modules + { + constituents[module.id] = module.layout.dependencies.modules.map { modules[$0] } + } + + return .init( + upstreamPackages: [], + constituents: constituents, + substrate: [], + products: stdlib.products, + cultures: modules, + snippets: []) } - private mutating - func addPartitions(of moduleLayouts:[SSGC.ModuleLayout], in package:PackageNode) + static func package(sink:PackageNode, + dependencies:[PackageNode], + substrate:[SSGC.ModuleLayout], + sparseEdges:[(NodeIdentifier, NodeIdentifier)]) throws -> Self { - for product:SymbolGraph.Product in package.products + let directedEdges:[(Symbol.Package, Symbol.Package)] = dependencies.reduce(into: []) { - let id:Symbol.Product = .init(name: product.name, package: package.id) - self.productPartitions[id] = product.cultures.map { moduleLayouts[$0] } + for dependency:any Identifiable in $1.dependencies + { + $0.append((dependency.id, $1.id)) + } } - } -} -extension SSGC.ModuleGraph -{ - /// Returns all dependencies of the given module (including transitive dependencies), - /// sorted in topological dependency order. The list begins with all the standard library - /// modules and ends with the given module. - func constituents(of module:__owned SSGC.ModuleLayout) throws -> [SSGC.ModuleLayout] - { - var dependencies:[Vertex] = module.dependencies.modules.map + + // FIXME: in Swift 6, it will be legal to have cyclic package dependencies! + guard + let dependencies:[PackageNode] = dependencies.sortedTopologically(by: directedEdges) + else { - .init(module: self.sinkLayout.cultures[$0], in: self.sink.id) + throw DigraphCycleError.init() } - for dependency:Symbol.Product in module.dependencies.products + + let dependencyTopologies:[PackageNode.Densified] + let sinkTopology:PackageNode.Densified + + (dependencyTopologies, sinkTopology) = try sink.joined(with: dependencies) + + var constituentsFromHomePackage:[Symbol.Product: [Node]] = [:] + var constituents:[NodeIdentifier: [Node]] = [:] + var nodes:[Node] = [] + + for (dependencyTopology, dependency):(PackageNode.Densified, PackageNode) in zip( + dependencyTopologies, + dependencies) { - guard - let modules:[SSGC.ModuleLayout] = self.productPartitions[dependency] - else + let root:SSGC.PackageRoot = .init(normalizing: dependency.root) + let layouts:[SSGC.ModuleLayout] = try root.layouts( + modules: dependencyTopology.modules, + exclude: dependency.exclude) + + let modules:[Node] = layouts.map { - throw TargetNode.DependencyError.undefinedProduct(dependency) + let node:Node = .init(module: $0, in: dependency.id) + nodes.append(node) + return node } - - for module:SSGC.ModuleLayout in modules + for module:Node in modules { - dependencies.append(.init(module: module, in: dependency.package)) + constituents[module.id] = module.layout.dependencies.modules.map { modules[$0] } + } + for product:SymbolGraph.Product in dependency.products + { + let id:Symbol.Product = .init(name: product.name, package: dependency.id) + constituentsFromHomePackage[id] = product.cultures.map { modules[$0] } } } - guard - let dependencies:[Vertex] = dependencies.sortedTopologically(by: self.sparseEdges) - else + let root:SSGC.PackageRoot = .init(normalizing: sink.root) + let layouts:[SSGC.ModuleLayout] = try root.layouts( + modules: sinkTopology.modules, + exclude: sink.exclude) + let modules:[Node] = layouts.map + { + let node:Node = .init(module: $0, in: sink.id) + nodes.append(node) + return node + } + for module:Node in modules + { + constituents[module.id] = module.layout.dependencies.modules.map { modules[$0] } + } + for product:SymbolGraph.Product in sink.products + { + let id:Symbol.Product = .init(name: product.name, package: sink.id) + constituentsFromHomePackage[id] = product.cultures.map { modules[$0] } + } + + // This needs to happen in a second pass, to account for future cyclic dependencies. + for node:Node in nodes { - throw DigraphCycleError.init() + try + { + for product:Symbol.Product in node.layout.dependencies.products + { + if let modules:[Node] = constituentsFromHomePackage[product] + { + $0 += modules + } + else + { + throw TargetNode.DependencyError.undefinedProduct(product) + } + } + + if let sorted:[Node] = $0.sortedTopologically(by: sparseEdges) + { + $0 = sorted + } + else + { + throw DigraphCycleError.init() + } + + // Every module is a constituent of itself. + $0.append(node) + + } (&constituents[node.id, default: []]) } - var modules:[SSGC.ModuleLayout] = self.standardLibrary - for dependency:Vertex in dependencies + guard + let snippetsDirectory:FilePath.Component = .init(sink.snippets) + else { - modules.append(dependency.layout) + throw SSGC.SnippetDirectoryError.invalid(sink.snippets) } - modules.append(module) - return modules - } + return .init( + upstreamPackages: sinkTopology.dependencies, + constituents: constituents, + substrate: substrate, + products: sinkTopology.products, + cultures: modules, + snippets: try root.snippets(in: snippetsDirectory)) + } +} +extension SSGC.ModuleGraph +{ /// Filters the given dependency pins, returning the dependencies that are actually used by /// at least one package product. - func dependenciesUsed(pins:[SPM.DependencyPin]) throws -> [SymbolGraphMetadata.Dependency] + func dependenciesUsed( + among dependencyPins:[SPM.DependencyPin]) throws -> [SymbolGraphMetadata.Dependency] { - let dependenciesPinned:[SymbolGraphMetadata.Dependency] = try self.sink.pin(to: pins) - let dependenciesUsed:Set = self.sink.products.reduce(into: []) + let dependenciesIndexed:SPM.DependencyPins = try .init(indexing: dependencyPins) + let dependenciesPinned:[SymbolGraphMetadata.Dependency] = try self.upstreamPackages.map + { + let pin:SPM.DependencyPin = try dependenciesIndexed($0.id) + return .init(package: .init(scope: pin.location.owner, name: $0.id), + requirement: ($0 as? SPM.Manifest.Dependency)?.requirement?.stable, + revision: pin.revision, + version: pin.version) + } + let dependenciesUsed:Set = self.products.reduce(into: []) { guard case .library = $1.type @@ -128,3 +250,298 @@ extension SSGC.ModuleGraph return dependenciesPinned.filter { dependenciesUsed.contains($0.package.name) } } } +extension SSGC.ModuleGraph +{ + private + func plan(for module:NodeIdentifier) -> [SSGC.ModuleLayout] + { + self.substrate + self.constituents[module, default: []].map(\.layout) + } + + /// Returns all dependencies of the given module (including transitive dependencies), + /// sorted in topological dependency order. The list begins with all the ``substrate`` + /// modules and ends with the given module. + var plans:[(SSGC.ModuleLayout, [SSGC.ModuleLayout])] + { + self.cultures.map { ($0.layout, self.plan(for: $0.id)) } + } +} +extension SSGC.ModuleGraph +{ + func symbolDumpCommands(scratch:SSGC.PackageBuildDirectory? = nil, + minimumAccessLevel:Symbol.ACL = .internal, + emitExtensionBlockSymbols:Bool = true, + includeInterfaceSymbols:Bool = true, + skipInheritedDocs:Bool = true) throws -> [SSGC.Toolchain.SymbolDumpOptions] + { + /// Compute the union of the non-substrate constituents of all modules being built. + let nodes:[NodeIdentifier: Node] = self.cultures.reduce(into: [:]) + { + for constituent:Node in self.constituents[$1.id, default: []] + { + $0[constituent.id] = constituent + } + } + + var modules:[SSGC.Toolchain.SymbolDumpOptions] = nodes.map + { + let plan:[SSGC.ModuleLayout] = self.plan(for: $0) + return plan.reduce(into: .init( + moduleName: $1.layout.module.id, + includePaths: scratch.map { [$0.modules] } ?? [], + minimumAccessLevel: minimumAccessLevel, + emitExtensionBlockSymbols: emitExtensionBlockSymbols, + includeInterfaceSymbols: includeInterfaceSymbols, + skipInheritedDocs: skipInheritedDocs)) + { + let constituent:Symbol.Module = $1.module.id + if constituent != $0.moduleName + { + $0.allowedReexportedModules.append(constituent) + } + + switch $1.module.language + { + case .cpp?: break + case .c?: break + default: return + } + + $0.includePaths += $1.include + + if let file:FilePath = $1.modulemap ?? + scratch?.modulemap(target: $1.module.name) + { + $0.moduleMaps.append(file) + } + } + } + + modules.sort { $0.moduleName < $1.moduleName } + return modules + } +} +// https://forums.swift.org/t/dependency-graph-of-the-standard-library-modules/59267 +extension SSGC.ModuleGraph +{ + static var macOS_5_10:([SymbolGraph.Product], [SymbolGraph.Module]) + { + ( + [ + .init(name: "__stdlib__", type: .library(.automatic), + dependencies: [], + cultures: [Int].init(0 ... 4)), + .init(name: "__corelibs__", type: .library(.automatic), + dependencies: [], + cultures: [Int].init(0 ... 7)), + ], + [ + // 0: + .toolchain(module: "Swift"), + // 1: + .toolchain(module: "_Concurrency", + dependencies: 0), + // 2: + .toolchain(module: "Distributed", + dependencies: 0, 1), + + // 3: + .toolchain(module: "_StringProcessing", + dependencies: 0), + // 4: + .toolchain(module: "RegexBuilder", + dependencies: 0, 3), + + // 5: + .toolchain(module: "Dispatch", + dependencies: 0), + // 6: + .toolchain(module: "DispatchIntrospection", + dependencies: 0), + // 7: + .toolchain(module: "Foundation", + dependencies: 0, 5), + ] + ) + } + + static var linux_5_10:([SymbolGraph.Product], [SymbolGraph.Module]) + { + ( + [ + .init(name: "__stdlib__", type: .library(.automatic), + dependencies: [], + cultures: [Int].init(0 ... 7)), + .init(name: "__corelibs__", type: .library(.automatic), + dependencies: [], + cultures: [Int].init(0 ... 13)), + ], + [ + // 0: + .toolchain(module: "Swift"), + // 1: + .toolchain(module: "_Concurrency", + dependencies: 0), + // 2: + .toolchain(module: "Distributed", + dependencies: 0, 1), + + // 3: + .toolchain(module: "_Differentiation", + dependencies: 0), + + // 4: + .toolchain(module: "_RegexParser", + dependencies: 0), + // 5: + .toolchain(module: "_StringProcessing", + dependencies: 0, 4), + // 6: + .toolchain(module: "RegexBuilder", + dependencies: 0, 4, 5), + + // 7: + .toolchain(module: "Cxx", + dependencies: 0), + + // 8: + .toolchain(module: "Dispatch", + dependencies: 0), + // 9: + .toolchain(module: "DispatchIntrospection", + dependencies: 0), + // 10: + .toolchain(module: "Foundation", + dependencies: 0, 8), + // 11: + .toolchain(module: "FoundationNetworking", + dependencies: 0, 8, 10), + // 12: + .toolchain(module: "FoundationXML", + dependencies: 0, 8, 10), + // 13: + .toolchain(module: "XCTest", + dependencies: 0), + ] + ) + } +} +extension SSGC.ModuleGraph +{ + static var macOS_6_0:([SymbolGraph.Product], [SymbolGraph.Module]) + { + ( + [ + .init(name: "__stdlib__", type: .library(.automatic), + dependencies: [], + cultures: [Int].init(0 ... 6)), + .init(name: "__corelibs__", type: .library(.automatic), + dependencies: [], + cultures: [Int].init(0 ... 9)), + ], + [ + // 0: + .toolchain(module: "Swift"), + // 1: + .toolchain(module: "_Concurrency", + dependencies: 0), + // 2: + .toolchain(module: "Distributed", + dependencies: 0, 1), + + // 3: + .toolchain(module: "_StringProcessing", + dependencies: 0), + // 4: + .toolchain(module: "RegexBuilder", + dependencies: 0, 3), + // 5: + .toolchain(module: "Synchronization", + dependencies: 0), + // 6: + .toolchain(module: "Cxx", + dependencies: 0), + + // 7: + .toolchain(module: "Dispatch", + dependencies: 0), + // 8: + .toolchain(module: "DispatchIntrospection", + dependencies: 0, 7), + // 9: + .toolchain(module: "Foundation", + dependencies: 0, 7, 8), + ] + ) + } + + static var linux_6_0:([SymbolGraph.Product], [SymbolGraph.Module]) + { + ( + [ + .init(name: "__stdlib__", type: .library(.automatic), + dependencies: [], + cultures: [Int].init(0 ... 8)), + .init(name: "__corelibs__", type: .library(.automatic), + dependencies: [], + cultures: [Int].init(0 ... 16)), + ], + [ + // 0: + .toolchain(module: "Swift"), + // 1: + .toolchain(module: "_Concurrency", + dependencies: 0), + // 2: + .toolchain(module: "Distributed", + dependencies: 0, 1), + + // 3: + .toolchain(module: "_Differentiation", + dependencies: 0), + + // 4: + .toolchain(module: "_RegexParser", + dependencies: 0), + // 5: + .toolchain(module: "_StringProcessing", + dependencies: 0, 4), + // 6: + .toolchain(module: "RegexBuilder", + dependencies: 0, 4, 5), + // 7: + .toolchain(module: "Synchronization", + dependencies: 0), + // 8: + .toolchain(module: "Cxx", + dependencies: 0), + + // 9: + .toolchain(module: "Dispatch", + dependencies: 0), + // 10: + .toolchain(module: "DispatchIntrospection", + dependencies: 0, 9), + // 11: + .toolchain(module: "FoundationEssentials", + dependencies: 0, 4, 5, 9), + // 12: + .toolchain(module: "FoundationInternationalization", + dependencies: 0, 4, 5, 9, 11), + // 13: + .toolchain(module: "Foundation", + dependencies: 0, 4, 5, 9, 11, 12), + // 14: + .toolchain(module: "FoundationNetworking", + dependencies: 0, 4, 5, 9, 11, 12, 13), + // 15: + .toolchain(module: "FoundationXML", + dependencies: 0, 4, 5, 9, 11, 12, 13), + + // 16: + .toolchain(module: "XCTest", + dependencies: 0, 4, 5, 9, 11, 12, 13), + ] + ) + } +} diff --git a/Sources/SymbolGraphBuilder/SSGC.PackageGraph.swift b/Sources/SymbolGraphBuilder/SSGC.PackageGraph.swift index 437be051..a1ac76e7 100644 --- a/Sources/SymbolGraphBuilder/SSGC.PackageGraph.swift +++ b/Sources/SymbolGraphBuilder/SSGC.PackageGraph.swift @@ -15,8 +15,8 @@ extension SSGC var packageManifests:[Symbol.Package: SPM.Manifest] private var packagesUnused:Set - private - var sparseEdges:[(Vertex, Vertex)] + private(set) + var sparseEdges:[(ModuleGraph.NodeIdentifier, ModuleGraph.NodeIdentifier)] let platform:SymbolGraphMetadata.Platform @@ -32,6 +32,24 @@ extension SSGC } } extension SSGC.PackageGraph +{ + private + func diagnoseUnusedPackages() + { + if !self.packagesUnused.isEmpty + { + print(""" + Note: \ + the following packages were never used in any documentation-bearing product: + """) + } + for (i, package):(Int, Symbol.Package) in self.packagesUnused.sorted().enumerated() + { + print("\(i + 1). \(package)") + } + } +} +extension SSGC.PackageGraph { private mutating func indexProducts(in manifest:SPM.Manifest, from id:Symbol.Package) @@ -69,7 +87,7 @@ extension SSGC.PackageGraph } } (&self.packageManifests[id]) - return try .all(flattening: dependencyManifest, on: self.platform, as: id) + return try .init(from: dependencyManifest, on: self.platform, as: id) } private mutating @@ -77,10 +95,12 @@ extension SSGC.PackageGraph { for target:TargetNode in normalizedManifest.targets.values { - let dependent:Vertex = .init(package: id, module: .init(mangling: target.id)) + let dependent:SSGC.ModuleGraph.NodeIdentifier = .init(package: id, + module: .init(mangling: target.id)) for dependency:String in target.dependencies.targets(on: self.platform) { - let dependency:Vertex = .init(package: id, module: .init(mangling: dependency)) + let dependency:SSGC.ModuleGraph.NodeIdentifier = .init(package: id, + module: .init(mangling: dependency)) self.sparseEdges.append((dependency, dependent)) } for dependency:Symbol.Product in target.dependencies.products(on: self.platform) @@ -98,7 +118,7 @@ extension SSGC.PackageGraph for constituent:String in product.targets { - let dependency:Vertex = .init( + let dependency:SSGC.ModuleGraph.NodeIdentifier = .init( package: dependency.package, module: .init(mangling: constituent)) @@ -108,17 +128,17 @@ extension SSGC.PackageGraph } } - consuming - func join(dependencies pins:[SPM.DependencyPin], - standardLibrary:SSGC.StandardLibrary, - with sinkManifest:inout SPM.Manifest, - as id:Symbol.Package) throws -> SSGC.ModuleGraph + mutating + func join( + dependencies dependencyPins:[SPM.DependencyPin], + sinkManifest:SPM.Manifest, + sinkPackage:Symbol.Package) throws -> ([PackageNode], PackageNode) { /// This pass must not make any assumptions about the ordering of the pins. /// /// These nodes are partially flattened, but they are still considered partitioned, as /// we have not yet flattened the product dependencies. - let dependencies:[PackageNode] = try pins.reduce(into: []) + let dependencies:[PackageNode] = try dependencyPins.reduce(into: []) { $0.append(try self.normalizeManifest(for: $1.identity)) } @@ -128,27 +148,17 @@ extension SSGC.PackageGraph self.createEdges(from: dependencyManifest, as: id) } - self.indexProducts(in: sinkManifest, from: id) + self.indexProducts(in: sinkManifest, from: sinkPackage) + var sinkManifest:SPM.Manifest = sinkManifest try sinkManifest.normalizeUnqualifiedDependencies(with: self.packageContainingProduct) - self.createEdges(from: sinkManifest, as: id) + let sink:PackageNode = try .init(from: sinkManifest, + on: self.platform, + as: sinkPackage) - if !self.packagesUnused.isEmpty - { - print(""" - Note: \ - the following packages were never used in any documentation-bearing product: - """) - } - for (i, package):(Int, Symbol.Package) in self.packagesUnused.sorted().enumerated() - { - print("\(i + 1). \(package)") - } + self.createEdges(from: sinkManifest, as: sinkPackage) + self.diagnoseUnusedPackages() - return try .init( - standardLibrary: standardLibrary.modules.map(SSGC.ModuleLayout.init(toolchain:)), - sparseEdges: self.sparseEdges, - dependencies: dependencies, - sink: try .all(flattening: sinkManifest, on: self.platform, as: id)) + return (dependencies, sink) } } diff --git a/Sources/SymbolGraphBuilder/Sources/SSGC.BookSources.swift b/Sources/SymbolGraphBuilder/Sources/SSGC.BookSources.swift index 07e1caea..a7cd336a 100644 --- a/Sources/SymbolGraphBuilder/Sources/SSGC.BookSources.swift +++ b/Sources/SymbolGraphBuilder/Sources/SSGC.BookSources.swift @@ -9,64 +9,24 @@ extension SSGC @_spi(testable) public struct BookSources { - private(set) - var cultures:[ModuleLayout] - let root:PackageRoot - + let modules:ModuleGraph private - init(cultures:[ModuleLayout] = [], root:PackageRoot) - { - self.cultures = cultures - self.root = root - } - } -} -extension SSGC.BookSources -{ - init(scanning location:FilePath.Directory) throws - { - self.init(root: .init(normalizing: location)) - - var bundles:[(FilePath.Directory, FilePath.Component)] = [] - try location.walk - { - switch $1.extension - { - case "docc"?, "unidoc"?: - bundles.append(($0, $1)) - return false + let root:Symbol.FileBase - default: - return ($0 / $1).exists() - } - } - - for (parent, bundle):(FilePath.Directory, FilePath.Component) in bundles + init(modules:ModuleGraph, root:Symbol.FileBase) { - let module:SymbolGraph.Module = .init(name: bundle.stem, type: .book) - self.cultures.append(try .init(package: self.root, - bundle: parent / bundle, - module: module)) + self.modules = modules + self.root = root } } } extension SSGC.BookSources:SSGC.DocumentationSources { - @_spi(testable) public - var snippets:[SSGC.LazyFile] { [] } - @_spi(testable) public var symbols:[FilePath.Directory] { [] } @_spi(testable) public - var prefix:Symbol.FileBase? { .init(self.root.location.path.string) } - - - @_spi(testable) public - func constituents(of module:__owned SSGC.ModuleLayout) throws -> [SSGC.ModuleLayout] - { - [module] - } + var prefix:Symbol.FileBase? { self.root } @_spi(testable) public func indexStore(for swift:SSGC.Toolchain) throws -> (any Markdown.SwiftLanguage.IndexStore)? diff --git a/Sources/SymbolGraphBuilder/Sources/SSGC.DocumentationSources.swift b/Sources/SymbolGraphBuilder/Sources/SSGC.DocumentationSources.swift index ef47a34a..f9d5203d 100644 --- a/Sources/SymbolGraphBuilder/Sources/SSGC.DocumentationSources.swift +++ b/Sources/SymbolGraphBuilder/Sources/SSGC.DocumentationSources.swift @@ -14,16 +14,13 @@ extension SSGC { protocol DocumentationSources { + var modules:ModuleGraph { get } var symbols:[FilePath.Directory] { get } - - var cultures:[ModuleLayout] { get } - var snippets:[LazyFile] { get } - var prefix:Symbol.FileBase? { get } /// Returns all constituents of the given module (including transitive dependencies), /// sorted in topological dependency order. The list ends with the given module. - func constituents(of culture:__owned ModuleLayout) throws -> [ModuleLayout] + // func constituents(of culture:__owned ModuleLayout) throws -> [ModuleLayout] func indexStore( for swift:SSGC.Toolchain) throws -> (any Markdown.SwiftLanguage.IndexStore)? @@ -35,8 +32,8 @@ extension SSGC.DocumentationSources logger:SSGC.Logger, with swift:SSGC.Toolchain) throws -> SymbolGraph { - let moduleLayouts:[SSGC.ModuleLayout] = self.cultures - let snippets:[SSGC.LazyFile] = self.snippets + let plans:[(SSGC.ModuleLayout, [SSGC.ModuleLayout])] = self.modules.plans + let snippets:[SSGC.LazyFile] = self.modules.snippets let prefix:Symbol.FileBase? = self.prefix let moduleIndexes:[SSGC.ModuleIndex] @@ -46,11 +43,10 @@ extension SSGC.DocumentationSources { var symbolCache:SSGC.SymbolCache = .init(symbols: try .collect(from: self.symbols)) - moduleIndexes = try moduleLayouts.map + moduleIndexes = try plans.map { + let constituents:[SSGC.ModuleLayout] = $1.filter(\.module.type.hasSymbols) let id:Symbol.Module = $0.id - let constituents:[SSGC.ModuleLayout] = try self.constituents(of: $0).filter( - \.module.type.hasSymbols) let symbols: ( @@ -111,7 +107,7 @@ extension SSGC.DocumentationSources Compiled documentation! time loading symbols : \(profiler.loadingSymbols) time compiling : \(profiler.compiling) - cultures : \(cultures.count) + cultures : \(plans.count) namespaces : \(moduleIndexes.reduce(0) { $0 + $1.declarations.count }) declarations : \(moduleIndexes.reduce(0) { @@ -147,18 +143,18 @@ extension SSGC.DocumentationSources try .link(projectRoot: prefix, definitions: definitions, plugins: [.swift(index: index)], - modules: moduleLayouts.map(\.module), + modules: plans.map(\.0.module), indexes: moduleIndexes, snippets: snippets, logger: logger) } - for resource:SSGC.LazyFile in moduleLayouts.lazy.map(\.resources).joined() + for resource:SSGC.LazyFile in plans.lazy.map(\.0.resources).joined() { profiler.loadingSources += resource.loadingTime profiler.linking -= resource.loadingTime } - for markdown:SSGC.LazyFile in moduleLayouts.lazy.map(\.markdown).joined() + for markdown:SSGC.LazyFile in plans.lazy.map(\.0.markdown).joined() { profiler.loadingSources += markdown.loadingTime profiler.linking -= markdown.loadingTime diff --git a/Sources/SymbolGraphBuilder/Sources/SSGC.PackageLayout.swift b/Sources/SymbolGraphBuilder/Sources/SSGC.PackageLayout.swift deleted file mode 100644 index 997591d8..00000000 --- a/Sources/SymbolGraphBuilder/Sources/SSGC.PackageLayout.swift +++ /dev/null @@ -1,51 +0,0 @@ -import PackageGraphs -import SymbolGraphs -import Symbols -import SystemIO - -extension SSGC -{ - /// Stores information about the source files for a package. - struct PackageLayout - { - let cultures:[ModuleLayout] - let root:PackageRoot - - private - init(cultures:[ModuleLayout], root:PackageRoot) - { - self.cultures = cultures - self.root = root - } - } -} -extension SSGC.PackageLayout -{ - init(scanning package:__shared PackageNode) throws - { - let root:SSGC.PackageRoot = .init(normalizing: package.root) - let count:[SSGC.ModuleLayout.DefaultDirectory: Int] = package.modules.reduce( - into: [:]) - { - if case nil = $1.location, - let directory:SSGC.ModuleLayout.DefaultDirectory = .init(for: $1.type) - { - $0[directory, default: 0] += 1 - } - } - - var cultures:[SSGC.ModuleLayout] = [] - cultures.reserveCapacity(package.modules.count) - - for i:Int in package.modules.indices - { - cultures.append(try .init( - exclude: package.exclude[i], - package: root, - module: package.modules[i], - count: count)) - } - - self.init(cultures: cultures, root: root) - } -} diff --git a/Sources/SymbolGraphBuilder/Sources/SSGC.PackageRoot.swift b/Sources/SymbolGraphBuilder/Sources/SSGC.PackageRoot.swift index 7ee851ff..bc594974 100644 --- a/Sources/SymbolGraphBuilder/Sources/SSGC.PackageRoot.swift +++ b/Sources/SymbolGraphBuilder/Sources/SSGC.PackageRoot.swift @@ -1,3 +1,4 @@ +import SymbolGraphs import Symbols import SystemIO @@ -42,3 +43,99 @@ extension SSGC.PackageRoot return .init("\(relative)") } } +extension SSGC.PackageRoot +{ + func layouts(modules:[SymbolGraph.Module], exclude:[[String]]) throws -> [SSGC.ModuleLayout] + { + let count:[SSGC.ModuleLayout.DefaultDirectory: Int] = modules.reduce( + into: [:]) + { + if case nil = $1.location, + let directory:SSGC.ModuleLayout.DefaultDirectory = .init(for: $1.type) + { + $0[directory, default: 0] += 1 + } + } + + var layouts:[SSGC.ModuleLayout] = [] + layouts.reserveCapacity(modules.count) + + for i:Int in modules.indices + { + layouts.append(try .init( + exclude: exclude[i], + package: self, + module: modules[i], + count: count)) + } + + return layouts + } + + func chapters() throws -> [SSGC.ModuleLayout] + { + var bundles:[(FilePath.Directory, FilePath.Component)] = [] + try location.walk + { + switch $1.extension + { + case "docc"?, "unidoc"?: + bundles.append(($0, $1)) + return false + + default: + return ($0 / $1).exists() + } + } + + return try bundles.map + { + try .init(package: self, + bundle: $0 / $1, + module: .init(name: $1.stem, type: .book)) + } + } + + func snippets(in snippetsDirectory:FilePath.Component) throws -> [SSGC.LazyFile] + { + let snippetsDirectory:FilePath.Directory = self.location / snippetsDirectory + if !snippetsDirectory.exists() + { + return [] + } + + var snippets:[SSGC.LazyFile] = [] + try snippetsDirectory.walk + { + let file:(path:FilePath, extension:String) + + if let `extension`:String = $1.extension + { + file.extension = `extension` + file.path = $0 / $1 + } + else + { + // directory, or some extensionless file we don’t care about + return true + } + + if file.extension == "swift" + { + // Should we be mangling URL-unsafe characters? + let snippet:SSGC.LazyFile = .init(location: file.path, + path: self.rebase(file.path), + name: $1.stem) + + snippets.append(snippet) + return true + } + else + { + return true + } + } + + return snippets + } +} diff --git a/Sources/SymbolGraphBuilder/Sources/SSGC.PackageSources.swift b/Sources/SymbolGraphBuilder/Sources/SSGC.PackageSources.swift index 9466fc3a..9e1ab818 100644 --- a/Sources/SymbolGraphBuilder/Sources/SSGC.PackageSources.swift +++ b/Sources/SymbolGraphBuilder/Sources/SSGC.PackageSources.swift @@ -20,85 +20,31 @@ extension SSGC @_spi(testable) public struct PackageSources { - @_spi(testable) public internal(set) - var snippets:[LazyFile] - - let scratch:PackageBuildDirectory + @_spi(testable) public + let modules:ModuleGraph let symbols:[FilePath.Directory] private - let modules:ModuleGraph + let scratch:PackageBuildDirectory + private + let root:Symbol.FileBase init( - snippets:[LazyFile] = [], - scratch:PackageBuildDirectory, + modules:ModuleGraph, symbols:[FilePath.Directory], - modules:ModuleGraph) + scratch:PackageBuildDirectory, + root:Symbol.FileBase) { - self.snippets = snippets self.scratch = scratch self.symbols = symbols self.modules = modules - } - } -} -extension SSGC.PackageSources -{ - private - var root:SSGC.PackageRoot { self.modules.sinkLayout.root } -} -extension SSGC.PackageSources -{ - mutating - func detect(snippets snippetsDirectory:FilePath.Component) throws - { - let snippets:FilePath.Directory = self.root.location / snippetsDirectory - if !snippets.exists() - { - return - } - - try snippets.walk - { - let file:(path:FilePath, extension:String) - - if let `extension`:String = $1.extension - { - file.extension = `extension` - file.path = $0 / $1 - } - else - { - // directory, or some extensionless file we don’t care about - return true - } - - if file.extension == "swift" - { - // Should we be mangling URL-unsafe characters? - let snippet:SSGC.LazyFile = .init(location: file.path, - path: self.root.rebase(file.path), - name: $1.stem) - - self.snippets.append(snippet) - return true - } - else - { - return true - } + self.root = root } } } extension SSGC.PackageSources:SSGC.DocumentationSources { - var cultures:[SSGC.ModuleLayout] { self.modules.sinkLayout.cultures } - var prefix:Symbol.FileBase? { .init(self.root.location.path.string) } - - func constituents(of module:__owned SSGC.ModuleLayout) throws -> [SSGC.ModuleLayout] - { - try self.modules.constituents(of: module) - } + var prefix:Symbol.FileBase? { self.root } @_spi(testable) public func indexStore(for swift:SSGC.Toolchain) throws -> (any Markdown.SwiftLanguage.IndexStore)? diff --git a/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibrary.swift b/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibrary.swift deleted file mode 100644 index e9201ef1..00000000 --- a/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibrary.swift +++ /dev/null @@ -1,253 +0,0 @@ -import SemanticVersions -import SymbolGraphs - -extension SSGC -{ - struct StandardLibrary - { - let products:SymbolGraph.Table - let modules:[SymbolGraph.Module] - - private - init(products:SymbolGraph.Table, - modules:[SymbolGraph.Module]) - { - self.products = products - self.modules = modules - } - } -} -extension SSGC.StandardLibrary -{ - init(platform:SymbolGraphMetadata.Platform, version:MinorVersion) - { - switch (platform, version) - { - case (.linux, .v(6, 0)): self = .linux_6_0 - case (.linux, _): self = .linux_5_10 - case (.macOS, .v(6, 0)): self = .macOS_6_0 - case (.macOS, _): self = .macOS_5_10 - default: fatalError("Unsupported platform: \(platform)") - } - } -} -// https://forums.swift.org/t/dependency-graph-of-the-standard-library-modules/59267 -extension SSGC.StandardLibrary -{ - static var macOS_5_10:Self - { - .init( - products: [ - .init(name: "__stdlib__", type: .library(.automatic), - dependencies: [], - cultures: [Int].init(0 ... 4)), - .init(name: "__corelibs__", type: .library(.automatic), - dependencies: [], - cultures: [Int].init(0 ... 7)), - ], - modules: [ - // 0: - .toolchain(module: "Swift"), - // 1: - .toolchain(module: "_Concurrency", - dependencies: 0), - // 2: - .toolchain(module: "Distributed", - dependencies: 0, 1), - - // 3: - .toolchain(module: "_StringProcessing", - dependencies: 0), - // 4: - .toolchain(module: "RegexBuilder", - dependencies: 0, 3), - - // 5: - .toolchain(module: "Dispatch", - dependencies: 0), - // 6: - .toolchain(module: "DispatchIntrospection", - dependencies: 0), - // 7: - .toolchain(module: "Foundation", - dependencies: 0, 5), - ]) - } - - static var linux_5_10:Self - { - .init( - products: [ - .init(name: "__stdlib__", type: .library(.automatic), - dependencies: [], - cultures: [Int].init(0 ... 7)), - .init(name: "__corelibs__", type: .library(.automatic), - dependencies: [], - cultures: [Int].init(0 ... 13)), - ], - modules: [ - // 0: - .toolchain(module: "Swift"), - // 1: - .toolchain(module: "_Concurrency", - dependencies: 0), - // 2: - .toolchain(module: "Distributed", - dependencies: 0, 1), - - // 3: - .toolchain(module: "_Differentiation", - dependencies: 0), - - // 4: - .toolchain(module: "_RegexParser", - dependencies: 0), - // 5: - .toolchain(module: "_StringProcessing", - dependencies: 0, 4), - // 6: - .toolchain(module: "RegexBuilder", - dependencies: 0, 4, 5), - - // 7: - .toolchain(module: "Cxx", - dependencies: 0), - - // 8: - .toolchain(module: "Dispatch", - dependencies: 0), - // 9: - .toolchain(module: "DispatchIntrospection", - dependencies: 0), - // 10: - .toolchain(module: "Foundation", - dependencies: 0, 8), - // 11: - .toolchain(module: "FoundationNetworking", - dependencies: 0, 8, 10), - // 12: - .toolchain(module: "FoundationXML", - dependencies: 0, 8, 10), - // 13: - .toolchain(module: "XCTest", - dependencies: 0), - ]) - } -} -extension SSGC.StandardLibrary -{ - static var macOS_6_0:Self - { - .init( - products: [ - .init(name: "__stdlib__", type: .library(.automatic), - dependencies: [], - cultures: [Int].init(0 ... 6)), - .init(name: "__corelibs__", type: .library(.automatic), - dependencies: [], - cultures: [Int].init(0 ... 9)), - ], - modules: [ - // 0: - .toolchain(module: "Swift"), - // 1: - .toolchain(module: "_Concurrency", - dependencies: 0), - // 2: - .toolchain(module: "Distributed", - dependencies: 0, 1), - - // 3: - .toolchain(module: "_StringProcessing", - dependencies: 0), - // 4: - .toolchain(module: "RegexBuilder", - dependencies: 0, 3), - // 5: - .toolchain(module: "Synchronization", - dependencies: 0), - // 6: - .toolchain(module: "Cxx", - dependencies: 0), - - // 7: - .toolchain(module: "Dispatch", - dependencies: 0), - // 8: - .toolchain(module: "DispatchIntrospection", - dependencies: 0, 7), - // 9: - .toolchain(module: "Foundation", - dependencies: 0, 7, 8), - ]) - } - - static var linux_6_0:Self - { - .init( - products: [ - .init(name: "__stdlib__", type: .library(.automatic), - dependencies: [], - cultures: [Int].init(0 ... 8)), - .init(name: "__corelibs__", type: .library(.automatic), - dependencies: [], - cultures: [Int].init(0 ... 16)), - ], - modules: [ - // 0: - .toolchain(module: "Swift"), - // 1: - .toolchain(module: "_Concurrency", - dependencies: 0), - // 2: - .toolchain(module: "Distributed", - dependencies: 0, 1), - - // 3: - .toolchain(module: "_Differentiation", - dependencies: 0), - - // 4: - .toolchain(module: "_RegexParser", - dependencies: 0), - // 5: - .toolchain(module: "_StringProcessing", - dependencies: 0, 4), - // 6: - .toolchain(module: "RegexBuilder", - dependencies: 0, 4, 5), - // 7: - .toolchain(module: "Synchronization", - dependencies: 0), - // 8: - .toolchain(module: "Cxx", - dependencies: 0), - - // 9: - .toolchain(module: "Dispatch", - dependencies: 0), - // 10: - .toolchain(module: "DispatchIntrospection", - dependencies: 0, 9), - // 11: - .toolchain(module: "FoundationEssentials", - dependencies: 0, 4, 5, 9), - // 12: - .toolchain(module: "FoundationInternationalization", - dependencies: 0, 4, 5, 9, 11), - // 13: - .toolchain(module: "Foundation", - dependencies: 0, 4, 5, 9, 11, 12), - // 14: - .toolchain(module: "FoundationNetworking", - dependencies: 0, 4, 5, 9, 11, 12, 13), - // 15: - .toolchain(module: "FoundationXML", - dependencies: 0, 4, 5, 9, 11, 12, 13), - - // 16: - .toolchain(module: "XCTest", - dependencies: 0, 4, 5, 9, 11, 12, 13), - ]) - } -} diff --git a/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibraryBuild.swift b/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibraryBuild.swift index f9886737..616594f4 100644 --- a/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibraryBuild.swift +++ b/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibraryBuild.swift @@ -24,19 +24,17 @@ extension SSGC.StandardLibraryBuild:SSGC.DocumentationBuild with toolchain:SSGC.Toolchain, clean _:Bool) throws -> (SymbolGraphMetadata, any SSGC.DocumentationSources) { - let standardLibrary:SSGC.StandardLibrary = .init(platform: try toolchain.platform(), + let modules:SSGC.ModuleGraph = .stdlib(platform: try toolchain.platform(), version: toolchain.splash.swift.version.minor) - let artifacts:FilePath.Directory = try toolchain.dump(standardLibrary: standardLibrary, - cache: cache) + let artifacts:FilePath.Directory = try toolchain.dump(stdlib: modules, cache: cache) let metadata:SymbolGraphMetadata = .swift(toolchain.splash.swift, commit: toolchain.splash.commit, triple: toolchain.splash.triple, - products: standardLibrary.products) + products: .init(viewing: modules.products)) - let sources:SSGC.StandardLibrarySources = .init(modules: standardLibrary.modules, - symbols: [artifacts]) + let sources:SSGC.StandardLibrarySources = .init(modules: modules, symbols: [artifacts]) return (metadata, sources) } } diff --git a/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibrarySources.swift b/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibrarySources.swift index a5318f7c..80c56286 100644 --- a/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibrarySources.swift +++ b/Sources/SymbolGraphBuilder/Standard library/SSGC.StandardLibrarySources.swift @@ -8,34 +8,20 @@ extension SSGC /// Stores information about the source files for a package. struct StandardLibrarySources { - let cultures:[ModuleLayout] + let modules:ModuleGraph let symbols:[FilePath.Directory] - private - init(cultures:[ModuleLayout], symbols:[FilePath.Directory]) + init(modules:ModuleGraph, symbols:[FilePath.Directory]) { - self.cultures = cultures + self.modules = modules self.symbols = symbols } } } -extension SSGC.StandardLibrarySources -{ - init(modules:[SymbolGraph.Module], symbols:[FilePath.Directory]) - { - self.init(cultures: modules.map(SSGC.ModuleLayout.init(toolchain:)), symbols: symbols) - } -} extension SSGC.StandardLibrarySources:SSGC.DocumentationSources { - var snippets:[SSGC.LazyFile] { [] } var prefix:Symbol.FileBase? { nil } - func constituents(of layout:__owned SSGC.ModuleLayout) throws -> [SSGC.ModuleLayout] - { - layout.dependencies.modules.map { self.cultures[$0] } + [layout] - } - func indexStore(for swift:SSGC.Toolchain) throws -> (any Markdown.SwiftLanguage.IndexStore)? { nil diff --git a/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpOptions.swift b/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpOptions.swift index c269018b..4d9b6f42 100644 --- a/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpOptions.swift +++ b/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpOptions.swift @@ -1,20 +1,36 @@ import Symbols +import SystemIO extension SSGC.Toolchain { struct SymbolDumpOptions { - var minimumACL:Symbol.ACL + /// Despite the name, this is the *mangled* C99 name, not the file system name. + let moduleName:Symbol.Module + /// I don’t know why `clang` spells `modulemap` as two words. + var moduleMaps:[FilePath] + var includePaths:[FilePath.Directory] + var allowedReexportedModules:[Symbol.Module] + + var minimumAccessLevel:Symbol.ACL var emitExtensionBlockSymbols:Bool var includeInterfaceSymbols:Bool var skipInheritedDocs:Bool - init(minimumACL:Symbol.ACL = .internal, + init(moduleName:Symbol.Module, + moduleMaps:[FilePath] = [], + includePaths:[FilePath.Directory] = [], + allowedReexportedModules:[Symbol.Module] = [], + minimumAccessLevel:Symbol.ACL = .internal, emitExtensionBlockSymbols:Bool = true, includeInterfaceSymbols:Bool = true, skipInheritedDocs:Bool = true) { - self.minimumACL = minimumACL + self.moduleName = moduleName + self.moduleMaps = moduleMaps + self.includePaths = includePaths + self.allowedReexportedModules = allowedReexportedModules + self.minimumAccessLevel = minimumAccessLevel self.emitExtensionBlockSymbols = emitExtensionBlockSymbols self.includeInterfaceSymbols = includeInterfaceSymbols self.skipInheritedDocs = skipInheritedDocs diff --git a/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpParameters.swift b/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpParameters.swift deleted file mode 100644 index 68a09ebe..00000000 --- a/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.SymbolDumpParameters.swift +++ /dev/null @@ -1,25 +0,0 @@ -import Symbols -import SystemIO - -extension SSGC.Toolchain -{ - struct SymbolDumpParameters - { - let moduleName:Symbol.Module - /// I don’t know why `clang` spells `modulemap` as two words. - var moduleMaps:[FilePath] - var includePaths:[FilePath.Directory] - var allowedReexportedModules:[Symbol.Module] - - init(moduleName:Symbol.Module, - moduleMaps:[FilePath] = [], - includePaths:[FilePath.Directory] = [], - allowedReexportedModules:[Symbol.Module] = []) - { - self.moduleName = moduleName - self.moduleMaps = moduleMaps - self.includePaths = includePaths - self.allowedReexportedModules = allowedReexportedModules - } - } -} diff --git a/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.swift b/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.swift index 66a0d102..fecf1fd8 100644 --- a/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.swift +++ b/Sources/SymbolGraphBuilder/Toolchains/SSGC.Toolchain.swift @@ -206,56 +206,60 @@ extension SSGC.Toolchain extension SSGC.Toolchain { /// Dumps the symbols for the standard library. Due to upstream bugs in the Swift compiler, - /// this methods disables extension block symbols by default. - func dump( - standardLibrary:SSGC.StandardLibrary, - options:SymbolDumpOptions = .init(emitExtensionBlockSymbols: false), - cache:FilePath.Directory) throws -> FilePath.Directory + /// this methods disables extension block symbols. + func dump(stdlib:SSGC.ModuleGraph, cache:FilePath.Directory) throws -> FilePath.Directory { let cached:FilePath.Directory = cache / "swift@\(self.splash.swift.version)" - - if !cached.exists() + if cached.exists() { - let temporary:FilePath.Directory = cache / "swift" - try temporary.create(clean: true) - - var dumped:[Symbol.Module] = [] - - for module:SymbolGraph.Module in standardLibrary.modules - { - try self.dump( - parameters: .init(moduleName: module.id, allowedReexportedModules: dumped), - options: options, - to: temporary) + return cached + } - dumped.append(module.id) - } + let temporary:FilePath.Directory = cache / "_batch" + try temporary.create(clean: true) - try temporary.move(replacing: cached) + for options:SymbolDumpOptions in try stdlib.symbolDumpCommands( + emitExtensionBlockSymbols: false) + { + try self.dump(options: options, to: temporary) } + try temporary.move(replacing: cached) + return cached } /// Dumps the symbols for the given targets, using the `output` workspace as the /// output directory. - func dump(parameters:SymbolDumpParameters, - options:SymbolDumpOptions, + func dump( + scratch:SSGC.PackageBuildDirectory, + modules:SSGC.ModuleGraph, + to output:FilePath.Directory) throws + { + for options:SymbolDumpOptions in try modules.symbolDumpCommands( + scratch: scratch) + { + try self.dump(options: options, to: output) + } + } + + private + func dump(options:SymbolDumpOptions, to output:FilePath.Directory) throws { - print("Dumping symbols for module '\(parameters.moduleName)'") + print("Dumping symbols for module '\(options.moduleName)'") var arguments:[String] = [ "symbolgraph-extract", - "-module-name", "\(parameters.moduleName)", + "-module-name", "\(options.moduleName)", "-target", "\(self.splash.triple)", "-output-dir", "\(output.path)", ] arguments.append("-minimum-access-level") - arguments.append("\(options.minimumACL)") + arguments.append("\(options.minimumAccessLevel)") if options.emitExtensionBlockSymbols { @@ -269,9 +273,9 @@ extension SSGC.Toolchain { arguments.append("-skip-inherited-docs") } - if !parameters.allowedReexportedModules.isEmpty + if !options.allowedReexportedModules.isEmpty { - let whitelist:String = parameters.allowedReexportedModules.lazy.map + let whitelist:String = options.allowedReexportedModules.lazy.map { "\($0)" }.joined(separator: ",") @@ -299,12 +303,12 @@ extension SSGC.Toolchain { arguments.append("-pretty-print") } - for includePath:FilePath.Directory in parameters.includePaths + for includePath:FilePath.Directory in options.includePaths { arguments.append("-I") arguments.append("\(includePath)") } - for moduleMap:FilePath in parameters.moduleMaps + for moduleMap:FilePath in options.moduleMaps { arguments.append("-Xcc") arguments.append("-fmodule-map-file=\(moduleMap)") @@ -335,14 +339,14 @@ extension SSGC.Toolchain { case 139: print(""" - Failed to dump symbols for module '\(parameters.moduleName)' due to \ + Failed to dump symbols for module '\(options.moduleName)' due to \ SIGSEGV from 'swift symbolgraph-extract'. \ This is a known bug in the Apple Swift compiler; see \ https://github.com/apple/swift/issues/68767. """) case 134: print(""" - Failed to dump symbols for module '\(parameters.moduleName)' due to \ + Failed to dump symbols for module '\(options.moduleName)' due to \ SIGABRT from 'swift symbolgraph-extract'. \ This is a known bug in the Apple Swift compiler; see \ https://github.com/swiftlang/swift/issues/75318. @@ -350,7 +354,7 @@ extension SSGC.Toolchain case let code: print(""" - Failed to dump symbols for module '\(parameters.moduleName)' due to exit \ + Failed to dump symbols for module '\(options.moduleName)' due to exit \ code \(code) from 'swift symbolgraph-extract'. \ If the output above indicates 'swift symbolgraph-extract' exited \ gracefully, this is most likely because the module.modulemap file declares \ diff --git a/Sources/SymbolGraphBuilderTests/Main.swift b/Sources/SymbolGraphBuilderTests/Main.swift index 4e34a161..6c61f90d 100644 --- a/Sources/SymbolGraphBuilderTests/Main.swift +++ b/Sources/SymbolGraphBuilderTests/Main.swift @@ -89,7 +89,7 @@ enum Main:TestMain, TestBattery guard let snippet:SSGC.LazyFile = tests.expect( - value: sources.snippets.first(where: { $0.name == "UnitTests" })) + value: sources.modules.snippets.first(where: { $0.name == "UnitTests" })) else { return From 991a06f97ec3b2a52049653ab9ad480872e53e81 Mon Sep 17 00:00:00 2001 From: Dianna Date: Wed, 12 Feb 2025 02:58:19 +0000 Subject: [PATCH 09/12] fix failure to dump all stdlib modules --- Scripts/Linux/GenerateTestSymbolGraphs | 61 +++++++++---------- .../SymbolGraphBuilder/SSGC.ModuleGraph.swift | 3 +- TestModules/Package.swift | 2 +- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/Scripts/Linux/GenerateTestSymbolGraphs b/Scripts/Linux/GenerateTestSymbolGraphs index 80797c2e..30b988b6 100755 --- a/Scripts/Linux/GenerateTestSymbolGraphs +++ b/Scripts/Linux/GenerateTestSymbolGraphs @@ -3,42 +3,41 @@ set -e swift --version -swift build \ - --package-path TestModules \ - -Xswiftc -emit-symbol-graph \ - -Xswiftc -emit-symbol-graph-dir -Xswiftc SymbolGraphs \ - -Xswiftc -symbol-graph-minimum-access-level -Xswiftc internal \ - -Xswiftc -emit-extension-block-symbols \ - -Xswiftc -include-spi-symbols \ - -Xswiftc -skip-inherited-docs \ - -Xswiftc -pretty-print \ - -Xswiftc -swift-version -Xswiftc 6 - +swift build --package-path TestModules swift package --package-path TestModules dump-package > TestModules/Package.swift.json -for TARGET in Swift _Concurrency +for module in Swift _Concurrency _Differentiation do - swift symbolgraph-extract \ - -target x86_64-unknown-linux-gnu \ - -minimum-access-level internal \ - -output-dir TestModules/SymbolGraphs \ - -skip-inherited-docs \ - -emit-extension-block-symbols \ - -include-spi-symbols \ - -pretty-print \ - -module-name $TARGET + for output in SymbolGraphs Determinism + do + swift symbolgraph-extract \ + -target x86_64-unknown-linux-gnu \ + -minimum-access-level internal \ + -output-dir TestModules/$output \ + -skip-inherited-docs \ + -emit-extension-block-symbols \ + -include-spi-symbols \ + -pretty-print \ + -module-name $module + done done -swift symbolgraph-extract \ - -target x86_64-unknown-linux-gnu \ - -minimum-access-level internal \ - -output-dir TestModules/Determinism \ - -skip-inherited-docs \ - -emit-extension-block-symbols \ - -include-spi-symbols \ - -pretty-print \ - -module-name Swift - +for include in TestModules/.build/debug/Modules +do + for file in $include/*.swiftmodule + do + swift symbolgraph-extract \ + -target x86_64-unknown-linux-gnu \ + -minimum-access-level internal \ + -output-dir TestModules/SymbolGraphs \ + -skip-inherited-docs \ + -emit-extension-block-symbols \ + -include-spi-symbols \ + -pretty-print \ + -module-name $(basename $file .swiftmodule) \ + -I $include + done +done .build/release/ssgc -u $SWIFT_INSTALLATION \ -n swift \ diff --git a/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.swift b/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.swift index 68b2bdb0..ec1ffb63 100644 --- a/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.swift +++ b/Sources/SymbolGraphBuilder/SSGC.ModuleGraph.swift @@ -81,7 +81,8 @@ extension SSGC.ModuleGraph } for module:Node in modules { - constituents[module.id] = module.layout.dependencies.modules.map { modules[$0] } + let nodes:[Node] = module.layout.dependencies.modules.map { modules[$0] } + constituents[module.id] = nodes + [module] } return .init( diff --git a/TestModules/Package.swift b/TestModules/Package.swift index a8687a5d..ee74c335 100644 --- a/TestModules/Package.swift +++ b/TestModules/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.9 +// swift-tools-version:6.0 import PackageDescription let package:Package = .init(name: "swift-unidoc-testmodules", From cb179b4ab551a02f89f3518345d7f7cf17144160 Mon Sep 17 00:00:00 2001 From: Dianna Date: Wed, 12 Feb 2025 03:12:24 +0000 Subject: [PATCH 10/12] create missing directories --- Scripts/Linux/GenerateTestSymbolGraphs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Scripts/Linux/GenerateTestSymbolGraphs b/Scripts/Linux/GenerateTestSymbolGraphs index 30b988b6..7b6c0ddc 100755 --- a/Scripts/Linux/GenerateTestSymbolGraphs +++ b/Scripts/Linux/GenerateTestSymbolGraphs @@ -6,9 +6,10 @@ swift --version swift build --package-path TestModules swift package --package-path TestModules dump-package > TestModules/Package.swift.json -for module in Swift _Concurrency _Differentiation +for output in SymbolGraphs Determinism do - for output in SymbolGraphs Determinism + mkdir -p TestModules/$output + for module in Swift _Concurrency _Differentiation do swift symbolgraph-extract \ -target x86_64-unknown-linux-gnu \ From c77b4b4ca234916bd15ca4e8a5f1bcc8b355c53e Mon Sep 17 00:00:00 2001 From: Dianna Date: Wed, 12 Feb 2025 23:54:09 +0000 Subject: [PATCH 11/12] port SymbolGraphPartTests to Swift Testing, and update expectations to reflect bugs that have been fixed in the Swift 6.0 compiler --- Package.swift | 3 +- .../SymbolGraphPartTests/AccessControl.swift | 26 + .../DocumentationInheritance.swift | 167 +++++++ ...cumentationInheritanceFromDependency.swift | 76 +++ .../ExternalExtensions.swift | 53 ++ .../InternalExtensions.swift | 38 ++ Sources/SymbolGraphPartTests/Main.swift | 456 ------------------ Sources/SymbolGraphPartTests/Phyla.swift | 60 +++ .../PhylaExtensions.swift | 25 + .../Support/GenericType (ext).swift | 8 + .../Support/SymbolGraphPart (ext).swift | 33 ++ .../SystemProgrammingInterfaces.swift | 24 + Sources/SymbolGraphPartTests/TestGroup.swift | 38 -- 13 files changed, 511 insertions(+), 496 deletions(-) create mode 100644 Sources/SymbolGraphPartTests/AccessControl.swift create mode 100644 Sources/SymbolGraphPartTests/DocumentationInheritance.swift create mode 100644 Sources/SymbolGraphPartTests/DocumentationInheritanceFromDependency.swift create mode 100644 Sources/SymbolGraphPartTests/ExternalExtensions.swift create mode 100644 Sources/SymbolGraphPartTests/InternalExtensions.swift delete mode 100644 Sources/SymbolGraphPartTests/Main.swift create mode 100644 Sources/SymbolGraphPartTests/Phyla.swift create mode 100644 Sources/SymbolGraphPartTests/PhylaExtensions.swift create mode 100644 Sources/SymbolGraphPartTests/Support/GenericType (ext).swift create mode 100644 Sources/SymbolGraphPartTests/Support/SymbolGraphPart (ext).swift create mode 100644 Sources/SymbolGraphPartTests/SystemProgrammingInterfaces.swift delete mode 100644 Sources/SymbolGraphPartTests/TestGroup.swift diff --git a/Package.swift b/Package.swift index 2ec60a04..f2b0368a 100644 --- a/Package.swift +++ b/Package.swift @@ -649,10 +649,9 @@ let package:Package = .init( .target(name: "SymbolGraphLinker"), ]), - .executableTarget(name: "SymbolGraphPartTests", + .testTarget(name: "SymbolGraphPartTests", dependencies: [ .target(name: "SymbolGraphParts"), - .target(name: "Testing_"), .product(name: "SystemIO", package: "swift-io"), ]), diff --git a/Sources/SymbolGraphPartTests/AccessControl.swift b/Sources/SymbolGraphPartTests/AccessControl.swift new file mode 100644 index 00000000..2599903b --- /dev/null +++ b/Sources/SymbolGraphPartTests/AccessControl.swift @@ -0,0 +1,26 @@ +import SymbolGraphParts +import Symbols +import Testing + +@Suite +struct AccessControl +{ + private + let symbols:SymbolGraphPart + + init() throws + { + self.symbols = try .load(part: "TestModules/SymbolGraphs/ACL.symbols.json") + } + + @Test(arguments: [ + (["Public"], .public), + (["Package"], .package), + (["Internal"], .internal), + ] as [([String], Symbol.ACL)]) + func Levels(_ symbol:[String], acl:Symbol.ACL) throws + { + let vertex:SymbolGraphPart.Vertex? = self.symbols.first(named: symbol) + #expect(vertex?.acl == acl) + } +} diff --git a/Sources/SymbolGraphPartTests/DocumentationInheritance.swift b/Sources/SymbolGraphPartTests/DocumentationInheritance.swift new file mode 100644 index 00000000..14f480e6 --- /dev/null +++ b/Sources/SymbolGraphPartTests/DocumentationInheritance.swift @@ -0,0 +1,167 @@ +import SymbolGraphParts +import Symbols +import Testing + +@Suite +struct DocumentationInheritance +{ + private + let symbols:SymbolGraphPart + + init() throws + { + self.symbols = try .load( + part: "TestModules/SymbolGraphs/DocumentationInheritance.symbols.json") + } + + @Test(arguments: [ + ( + ["Protocol", "everywhere"], + "DocumentationInheritance", + "This comment is from the root protocol." + ), + ( + ["Protocol", "protocol"], + "DocumentationInheritance", + "This comment is from the root protocol." + ), + ( + ["Protocol", "refinement"], + nil, + nil + ), + ( + ["Protocol", "conformer"], + nil, + nil + ), + ( + ["Protocol", "nowhere"], + nil, + nil + ), + + ( + ["Refinement", "everywhere"], + "DocumentationInheritance", + "This comment is from the refined protocol." + ), + ( + ["Refinement", "protocol"], + nil, + nil + ), + ( + ["Refinement", "refinement"], + "DocumentationInheritance", + "This comment is from the refined protocol." + ), + ( + ["Refinement", "conformer"], + nil, + nil + ), + ( + ["Refinement", "nowhere"], + nil, + nil + ), + + ( + ["OtherRefinement", "everywhere"], + "DocumentationInheritance", + "This is a default implementation provided by a refined protocol." + ), + ( + ["OtherRefinement", "protocol"], + nil, + nil + ), + + ( + ["Conformer", "everywhere"], + "DocumentationInheritance", + "This comment is from the conforming type." + ), + + // This would inherit (from Protocol) if the part + // were generated without `-skip-inherited-docs`. + ( + ["Conformer", "protocol"], + nil, + nil + ), + // This would inherit (from Refinement) if the part + // were generated without `-skip-inherited-docs`. + ( + ["Conformer", "refinement"], + nil, + nil + ), + ( + ["Conformer", "conformer"], + "DocumentationInheritance", + "This comment is from the conforming type." + ), + ( + ["Conformer", "nowhere"], + nil, + nil + ), + ] as [([String], Symbol.Module?, String?)]) + func Comments(_ symbol:[String], _ culture:Symbol.Module?, _ comment:String?) throws + { + let vertex:SymbolGraphPart.Vertex? = self.symbols.first(named: symbol) + #expect(vertex?.doccomment?.culture == culture) + #expect(vertex?.doccomment?.text == comment) + } + + @Test + func Origins() throws + { + let relationships:Set = self.symbols.relationships.reduce( + into: []) + { + if case _? = $1.origin + { + $0.insert($1) + } + } + + #expect(relationships == [ + .intrinsicWitness(.init( + _ : "s24DocumentationInheritance15OtherRefinementPAAE8protocolytvp", + of: "s24DocumentationInheritance8ProtocolP8protocolytvp", + origin: .init("s24DocumentationInheritance8ProtocolP8protocolytvp"))), + + .member(.init( + _ : "s24DocumentationInheritance9ConformerV7nowhereytvp", + in: .scalar("s24DocumentationInheritance9ConformerV"), + origin: .init("s24DocumentationInheritance10RefinementP7nowhereytvp"))), + + .member(.init( + _ : "s24DocumentationInheritance9ConformerV10refinementytvp", + in: .scalar(.init("s:24DocumentationInheritance9ConformerV")!), + origin: .init(.init( + "s:24DocumentationInheritance10RefinementP10refinementytvp")!))), + + .member(.init( + _ : "s24DocumentationInheritance9ConformerV9conformerytvp", + in: .scalar("s24DocumentationInheritance9ConformerV"), + origin: .init( + "s24DocumentationInheritance10RefinementP9conformerytvp"))), + + .member(.init( + _ : "s24DocumentationInheritance9ConformerV10everywhereytvp", + in: .scalar("s24DocumentationInheritance9ConformerV"), + origin: .init( + "s24DocumentationInheritance10RefinementP10everywhereytvp"))), + + .member(.init( + _ : "s24DocumentationInheritance9ConformerV8protocolytvp", + in: .scalar("s24DocumentationInheritance9ConformerV"), + origin: .init( + "s24DocumentationInheritance8ProtocolP8protocolytvp"))), + ]) + } +} diff --git a/Sources/SymbolGraphPartTests/DocumentationInheritanceFromDependency.swift b/Sources/SymbolGraphPartTests/DocumentationInheritanceFromDependency.swift new file mode 100644 index 00000000..639ec5b4 --- /dev/null +++ b/Sources/SymbolGraphPartTests/DocumentationInheritanceFromDependency.swift @@ -0,0 +1,76 @@ +import SymbolGraphParts +import Symbols +import Testing + +@Suite +struct DocumentationInheritanceFromDependency +{ + private + let symbols:SymbolGraphPart + + init() throws + { + self.symbols = try .load( + part: "TestModules/SymbolGraphs/DocumentationInheritanceFromSwift.symbols.json") + } + + @Test(arguments: [ + ( + ["Documented", "id"], + "DocumentationInheritanceFromSwift", + "A documented id property." + ), + ( + ["Undocumented", "id"], + nil, + nil + ), + ] as [([String], Symbol.Module?, String?)]) + func Comments(_ symbol:[String], _ culture:Symbol.Module?, _ comment:String?) throws + { + let vertex:SymbolGraphPart.Vertex? = self.symbols.first(named: symbol) + #expect(vertex?.doccomment?.culture == culture) + #expect(vertex?.doccomment?.text == comment) + } + + @Test + func Origins() throws + { + let relationships:Set = self.symbols.relationships.reduce( + into: []) + { + if case .member(let membership) = $1, + case _? = membership.origin + { + $0.insert($1) + } + } + /// Concrete type members always get an origin edge pointing back + /// to whatever requirement they fulfill, regardless of whether: + /// + /// - they already have documentation of their own, or + /// - they had no documentation, and inherited no documentation. + /// + /// In both cases, they will get an edge to their immediate origin. + /// If concrete type members do inherit documentation, their origin + /// edges point to the symbols they inherited documenation from, + /// even if there were undocumented symbols in between. (The edges + /// “skip” the undocumented symbols.) + /// + /// Default implementations only get an origin edge if they actually + /// had no documentation of their own, and successfully inherited some. + #expect(relationships == [ + .member(.init(""" + s33DocumentationInheritanceFromSwift12UndocumentedV2ids5NeverOSgvp + """, + in: .scalar("s33DocumentationInheritanceFromSwift12UndocumentedV"), + origin: "ss12IdentifiableP2id2IDQzvp")), + + .member(.init(""" + s33DocumentationInheritanceFromSwift10DocumentedV2ids5NeverOSgvp + """, + in: .scalar("s33DocumentationInheritanceFromSwift10DocumentedV"), + origin: "ss12IdentifiableP2id2IDQzvp")), + ]) + } +} diff --git a/Sources/SymbolGraphPartTests/ExternalExtensions.swift b/Sources/SymbolGraphPartTests/ExternalExtensions.swift new file mode 100644 index 00000000..5ae02336 --- /dev/null +++ b/Sources/SymbolGraphPartTests/ExternalExtensions.swift @@ -0,0 +1,53 @@ +import Signatures +import SymbolGraphParts +import Symbols +import Testing + +@Suite +struct ExternalExtensions +{ + private + let symbols:SymbolGraphPart + + init() throws + { + self.symbols = try .load(part: """ + TestModules/SymbolGraphs/ExternalExtensionsWithConstraints@\ + ExtendableTypesWithConstraints.symbols.json + """) + } + + @Test(arguments: [ + ( + ["Struct"], + [ + .where("T", is: .conformer, to: .Equatable), + .where("T", is: .conformer, to: .Sequence), + ] + ), + ( + ["Struct", "external(_:)"], + [ + .where("T", is: .conformer, to: .Equatable), + .where("T", is: .conformer, to: .Sequence), + ] + ), + ( + ["Protocol"], + [ + .where("Self.T", is: .conformer, to: .Equatable), + ] + ), + ( + ["Protocol", "external(_:)"], + [ + .where("Self.T", is: .conformer, to: .Equatable), + ] + ), + ] as [([String], [GenericConstraint])]) + func Constraints(_ symbol:[String], _ conditions:[GenericConstraint]) throws + { + let vertex:SymbolGraphPart.Vertex? = self.symbols.first(named: symbol) + #expect(vertex?.extension.conditions == conditions) + } +} diff --git a/Sources/SymbolGraphPartTests/InternalExtensions.swift b/Sources/SymbolGraphPartTests/InternalExtensions.swift new file mode 100644 index 00000000..9e98509f --- /dev/null +++ b/Sources/SymbolGraphPartTests/InternalExtensions.swift @@ -0,0 +1,38 @@ +import Signatures +import SymbolGraphParts +import Symbols +import Testing + +@Suite +struct InternalExtensions +{ + private + let symbols:SymbolGraphPart + + init() throws + { + self.symbols = try .load( + part: "TestModules/SymbolGraphs/InternalExtensionsWithConstraints.symbols.json") + } + + @Test(arguments: [ + ( + ["Struct", "internal(_:)"], + [ + .where("T", is: .conformer, to: .Equatable), + .where("T", is: .conformer, to: .Sequence), + ] + ), + ( + ["Protocol", "internal(_:)"], + [ + .where("Self.T", is: .conformer, to: .Equatable), + ] + ), + ] as [([String], [GenericConstraint])]) + func Constraints(_ symbol:[String], _ conditions:[GenericConstraint]) throws + { + let vertex:SymbolGraphPart.Vertex? = self.symbols.first(named: symbol) + #expect(vertex?.extension.conditions == conditions) + } +} diff --git a/Sources/SymbolGraphPartTests/Main.swift b/Sources/SymbolGraphPartTests/Main.swift deleted file mode 100644 index 462871f5..00000000 --- a/Sources/SymbolGraphPartTests/Main.swift +++ /dev/null @@ -1,456 +0,0 @@ -import Signatures -import SymbolGraphParts -import Symbols -import SystemIO -import Testing_ - -@main -enum Main:TestMain, TestBattery -{ - static - func run(tests:TestGroup) - { - if let tests:TestGroup = tests / "Phyla", - let part:SymbolGraphPart = tests.load( - part: "TestModules/SymbolGraphs/Phyla.symbols.json") - { - for (symbol, phylum):([String], Phylum.Decl) in - [ - (["Actor"], .actor), - (["Class"], .class), - (["Enum"], .enum), - (["Enum", "case"], .case), - (["Protocol"], .protocol), - (["Protocol", "AssociatedType"], .associatedtype), - (["Struct"], .struct), - (["Typealias"], .typealias), - (["Var"], .var(nil)), - ] - { - if let name:String = symbol.last?.lowercased(), - let tests:TestGroup = tests / name, - let symbol:SymbolGraphPart.Vertex = tests.expect(symbol: symbol, in: part) - { - tests.expect(symbol.phylum ==? .decl(phylum)) - } - } - for (symbol, phylum, name):([String], Phylum.Decl, String) in - [ - (["Func"], .func(nil), "global-func"), - (["Struct", "instanceMethod"], .func(.instance), "instance-func"), - (["Struct", "staticMethod"], .func(.static), "static-func"), - - (["Struct", "instanceProperty"],.var(.instance), "instance-var"), - (["Struct", "staticProperty"], .var(.static), "static-var"), - - (["Struct", "subscript"], .subscript(.instance), "instance-subscript"), - (["Struct", "subscript(_:)"], .subscript(.static), "static-subscript"), - - (["Class", "classMethod"], .func(.class), "class-func"), - (["Class", "classProperty"], .var(.class), "class-var"), - (["Class", "init"], .initializer, "class-init"), - (["Actor", "init"], .initializer, "actor-init"), - - (["?/(_:)"], .operator, "operator-prefix"), - (["<-(_:_:)"], .operator, "operator-infix"), - (["/?(_:)"], .operator, "operator-suffix"), - - (["Struct", "?/(_:)"], .operator, "type-operator-prefix"), - (["Struct", "<-(_:_:)"], .operator, "type-operator-infix"), - (["Struct", "/?(_:)"], .operator, "type-operator-suffix"), - ] - { - if let tests:TestGroup = tests / name, - let symbol:SymbolGraphPart.Vertex = tests.expect(symbol: symbol, in: part) - { - tests.expect(symbol.phylum ==? .decl(phylum)) - } - } - - if let tests:TestGroup = tests / "deinit" - { - tests.expect(nil: part.vertices.first { $0.phylum == .decl(.deinitializer) }) - } - } - if let tests:TestGroup = tests / "Phyla" / "Extension", - let part:SymbolGraphPart = tests.load( - part: "TestModules/SymbolGraphs/Phyla@Swift.symbols.json") - { - for (symbol, phylum):([String], Phylum) in - [ - (["Int"], .block), - (["Int", "AssociatedType"], .decl(.typealias)), - ] - { - if let symbol:SymbolGraphPart.Vertex = tests.expect(symbol: symbol, in: part) - { - tests.expect(symbol.phylum ==? phylum) - } - } - } - if let tests:TestGroup = tests / "ACLs", - let part:SymbolGraphPart = tests.load( - part: "TestModules/SymbolGraphs/ACL.symbols.json") - { - for (symbol, level):([String], Symbol.ACL?) in - [ - (["Public"], .public), - (["Package"], .package), - (["Internal"], .internal), - ] - { - if let symbol:SymbolGraphPart.Vertex = tests.expect(symbol: symbol, in: part) - { - tests.expect(symbol.acl ==? level) - } - } - } - if let tests:TestGroup = tests / "SPIs", - let part:SymbolGraphPart = tests.load( - part: "TestModules/SymbolGraphs/SPI.symbols.json") - { - for (symbol, interfaces):([String], [String]?) in - [ - (["NoSPI"], nil), - (["SPI"], []), - ] - { - if let symbol:SymbolGraphPart.Vertex = tests.expect(symbol: symbol, in: part) - { - tests.expect(symbol.signature.spis ==? interfaces) - } - } - } - if let tests:TestGroup = tests / "InternalDocumentationInheritance", - let part:SymbolGraphPart = tests.load( - part: "TestModules/SymbolGraphs/DocumentationInheritance.symbols.json") - { - for (symbol, culture, comment):([String], Symbol.Module?, String?) in - [ - ( - ["Protocol", "everywhere"], - "DocumentationInheritance", - "This comment is from the root protocol." - ), - ( - ["Protocol", "protocol"], - "DocumentationInheritance", - "This comment is from the root protocol." - ), - ( - ["Protocol", "refinement"], - nil, - nil - ), - ( - ["Protocol", "conformer"], - nil, - nil - ), - ( - ["Protocol", "nowhere"], - nil, - nil - ), - - ( - ["Refinement", "everywhere"], - "DocumentationInheritance", - "This comment is from the refined protocol." - ), - ( - ["Refinement", "protocol"], - nil, - nil - ), - ( - ["Refinement", "refinement"], - "DocumentationInheritance", - "This comment is from the refined protocol." - ), - ( - ["Refinement", "conformer"], - nil, - nil - ), - ( - ["Refinement", "nowhere"], - nil, - nil - ), - - ( - ["OtherRefinement", "everywhere"], - "DocumentationInheritance", - "This is a default implementation provided by a refined protocol." - ), - ( - ["OtherRefinement", "protocol"], - "DocumentationInheritance", - "This comment is from the root protocol." - ), - - ( - ["Conformer", "everywhere"], - "DocumentationInheritance", - "This comment is from the conforming type." - ), - // NOTE: - // These should not be present if `-skip-inherited-docs` worked correctly, - // but they are, likely due to a compiler bug - - // This would inherit (from Protocol) if the part - // were generated without `-skip-inherited-docs`. - ( - ["Conformer", "protocol"], - "DocumentationInheritance", // nil, - "This comment is from the root protocol." // nil, - ), - // This would inherit (from Refinement) if the part - // were generated without `-skip-inherited-docs`. - ( - ["Conformer", "refinement"], - "DocumentationInheritance", // nil, - "This comment is from the refined protocol." // nil, - ), - ( - ["Conformer", "conformer"], - "DocumentationInheritance", - "This comment is from the conforming type." - ), - ( - ["Conformer", "nowhere"], - nil, - nil - ), - ] - { - if let tests:TestGroup = tests / symbol.joined(separator: "-").lowercased(), - let symbol:SymbolGraphPart.Vertex = tests.expect(symbol: symbol, in: part) - { - tests.expect(symbol.doccomment?.culture ==? culture) - tests.expect(symbol.doccomment?.text ==? comment) - } - } - - if let tests:TestGroup = tests / "origins" - { - /// Concrete type members always get an origin edge pointing back - /// to whatever requirement they fulfill, regardless of whether: - /// - /// - they already have documentation of their own, or - /// - they had no documentation, and inherited no documentation. - /// - /// In both cases, they will get an edge to their immediate origin. - /// If concrete type members do inherit documentation, their origin - /// edges point to the symbols they inherited documenation from, - /// even if there were undocumented symbols in between. (The edges - /// “skip” the undocumented symbols.) - /// - /// Default implementations only get an origin edge if they actually - /// had no documentation of their own, and successfully inherited some. - tests.expect(part.relationships.filter - { - $0.origin != nil - } - **? - [ - .intrinsicWitness(.init( - _ : "s24DocumentationInheritance15OtherRefinementPAAE8protocolytvp", - of: "s24DocumentationInheritance8ProtocolP8protocolytvp", - origin: .init("s24DocumentationInheritance8ProtocolP8protocolytvp"))), - - .member(.init( - _ : "s24DocumentationInheritance9ConformerV7nowhereytvp", - in: .scalar("s24DocumentationInheritance9ConformerV"), - origin: .init("s24DocumentationInheritance10RefinementP7nowhereytvp"))), - - .member(.init( - _ : "s24DocumentationInheritance9ConformerV10refinementytvp", - in: .scalar(.init("s:24DocumentationInheritance9ConformerV")!), - origin: .init(.init( - "s:24DocumentationInheritance10RefinementP10refinementytvp")!))), - - .member(.init( - _ : "s24DocumentationInheritance9ConformerV9conformerytvp", - in: .scalar("s24DocumentationInheritance9ConformerV"), - origin: .init( - "s24DocumentationInheritance10RefinementP9conformerytvp"))), - - .member(.init( - _ : "s24DocumentationInheritance9ConformerV10everywhereytvp", - in: .scalar("s24DocumentationInheritance9ConformerV"), - origin: .init( - "s24DocumentationInheritance10RefinementP10everywhereytvp"))), - - .member(.init( - _ : "s24DocumentationInheritance9ConformerV8protocolytvp", - in: .scalar("s24DocumentationInheritance9ConformerV"), - origin: .init( - "s24DocumentationInheritance8ProtocolP8protocolytvp"))), - ]) - } - } - if let tests:TestGroup = tests / "ExternalDocumentationInheritance", - let part:SymbolGraphPart = tests.load(part: """ - TestModules/SymbolGraphs/DocumentationInheritanceFromSwift.symbols.json - """) - { - for (symbol, culture, comment):([String], Symbol.Module?, String?) in - [ - ( - ["Documented", "id"], - "DocumentationInheritanceFromSwift", - "A documented id property." - ), - ( - ["Undocumented", "id"], - "Swift", - "The stable identity of the entity associated with this instance." - ), - ] - { - if let tests:TestGroup = tests / symbol[0].lowercased(), - let symbol:SymbolGraphPart.Vertex = tests.expect(symbol: symbol, in: part) - { - tests.expect(symbol.doccomment?.culture ==? culture) - tests.expect(symbol.doccomment?.text ==? comment) - } - } - - if let tests:TestGroup = tests / "origins" - { - /// Concrete type members always get an origin edge pointing back - /// to whatever requirement they fulfill, regardless of whether: - /// - /// - they already have documentation of their own, or - /// - they had no documentation, and inherited no documentation. - /// - /// In both cases, they will get an edge to their immediate origin. - /// If concrete type members do inherit documentation, their origin - /// edges point to the symbols they inherited documenation from, - /// even if there were undocumented symbols in between. (The edges - /// “skip” the undocumented symbols.) - /// - /// Default implementations only get an origin edge if they actually - /// had no documentation of their own, and successfully inherited some. - tests.expect(part.relationships.filter - { - if case .member(let membership) = $0, - case _? = membership.origin - { - true - } - else - { - false - } - } - **? - [ - .member(.init(""" - s33DocumentationInheritanceFromSwift12UndocumentedV2ids5NeverOSgvp - """, - in: .scalar("s33DocumentationInheritanceFromSwift12UndocumentedV"), - origin: "ss12IdentifiableP2id2IDQzvp")), - - .member(.init(""" - s33DocumentationInheritanceFromSwift10DocumentedV2ids5NeverOSgvp - """, - in: .scalar("s33DocumentationInheritanceFromSwift10DocumentedV"), - origin: "ss12IdentifiableP2id2IDQzvp")), - ]) - } - } - - if let tests:TestGroup = tests / "InternalExtensionConstraints", - let part:SymbolGraphPart = tests.load(part: """ - TestModules/SymbolGraphs/InternalExtensionsWithConstraints.symbols.json - """) - { - for (symbol, conditions):([String], [GenericConstraint]) in - [ - ( - ["Struct", "internal(_:)"], - [ - .where("T", is: .conformer, to: .init( - spelling: "Equatable", - nominal: "sSQ")), - .where("T", is: .conformer, to: .init( - spelling: "Sequence", - nominal: "sST")), - ] - ), - ( - ["Protocol", "internal(_:)"], - [ - .where("Self.T", is: .conformer, to: .init( - spelling: "Equatable", - nominal: "sSQ")), - ] - ), - ] - { - if let symbol:SymbolGraphPart.Vertex = tests.expect(symbol: symbol, in: part) - { - tests.expect(symbol.extension.conditions ..? conditions) - } - } - } - if let tests:TestGroup = tests / "ExternalExtensionConstraints", - let part:SymbolGraphPart = tests.load(part: """ - TestModules/SymbolGraphs/\ - ExternalExtensionsWithConstraints@\ - ExtendableTypesWithConstraints.symbols.json - """) - { - for (symbol, conditions):([String], [GenericConstraint]) in - [ - ( - ["Struct"], - [ - .where("T", is: .conformer, to: .init( - spelling: "Equatable", - nominal: "sSQ")), - .where("T", is: .conformer, to: .init( - spelling: "Sequence", - nominal: "sST")), - ] - ), - ( - ["Struct", "external(_:)"], - [ - .where("T", is: .conformer, to: .init( - spelling: "Equatable", - nominal: "sSQ")), - .where("T", is: .conformer, to: .init( - spelling: "Sequence", - nominal: "sST")), - ] - ), - ( - ["Protocol"], - [ - .where("Self.T", is: .conformer, to: .init( - spelling: "Equatable", - nominal: "sSQ")), - ] - ), - ( - ["Protocol", "external(_:)"], - [ - .where("Self.T", is: .conformer, to: .init( - spelling: "Equatable", - nominal: "sSQ")), - ] - ), - ] - { - if let symbol:SymbolGraphPart.Vertex = tests.expect(symbol: symbol, in: part) - { - tests.expect(symbol.extension.conditions ..? conditions) - } - } - } - } -} diff --git a/Sources/SymbolGraphPartTests/Phyla.swift b/Sources/SymbolGraphPartTests/Phyla.swift new file mode 100644 index 00000000..561730bd --- /dev/null +++ b/Sources/SymbolGraphPartTests/Phyla.swift @@ -0,0 +1,60 @@ +import SymbolGraphParts +import Symbols +import Testing + +@Suite +struct Phyla +{ + private + let symbols:SymbolGraphPart + + init() throws + { + self.symbols = try .load(part: "TestModules/SymbolGraphs/Phyla.symbols.json") + } + + @Test(arguments: [ + (["Actor"], .actor), + (["Class"], .class), + (["Enum"], .enum), + (["Enum", "case"], .case), + (["Protocol"], .protocol), + (["Protocol", "AssociatedType"], .associatedtype), + (["Struct"], .struct), + (["Typealias"], .typealias), + (["Var"], .var(nil)), + + (["Func"], .func(nil)), + (["Struct", "instanceMethod"], .func(.instance)), + (["Struct", "staticMethod"], .func(.static)), + + (["Struct", "instanceProperty"], .var(.instance)), + (["Struct", "staticProperty"], .var(.static)), + + (["Struct", "subscript"], .subscript(.instance)), + (["Struct", "subscript(_:)"], .subscript(.static)), + + (["Class", "classMethod"], .func(.class)), + (["Class", "classProperty"], .var(.class)), + (["Class", "init"], .initializer), + (["Actor", "init"], .initializer), + + (["?/(_:)"], .operator), + (["<-(_:_:)"], .operator), + (["/?(_:)"], .operator), + + (["Struct", "?/(_:)"], .operator), + (["Struct", "<-(_:_:)"], .operator), + (["Struct", "/?(_:)"], .operator), + ] as [([String], Phylum.Decl)]) + func Decls(_ symbol:[String], phylum:Phylum.Decl) throws + { + let vertex:SymbolGraphPart.Vertex? = self.symbols.first(named: symbol) + #expect(vertex?.phylum == .decl(phylum)) + } + + func Deinit() throws + { + #expect(nil == self.symbols.vertices.first { $0.phylum == .decl(.deinitializer) }) + } +} diff --git a/Sources/SymbolGraphPartTests/PhylaExtensions.swift b/Sources/SymbolGraphPartTests/PhylaExtensions.swift new file mode 100644 index 00000000..e7ad47c5 --- /dev/null +++ b/Sources/SymbolGraphPartTests/PhylaExtensions.swift @@ -0,0 +1,25 @@ +import SymbolGraphParts +import Symbols +import Testing + +@Suite +struct PhylaExtensions +{ + private + let symbols:SymbolGraphPart + + init() throws + { + self.symbols = try .load(part: "TestModules/SymbolGraphs/Phyla@Swift.symbols.json") + } + + @Test(arguments: [ + (["Int"], .block), + (["Int", "AssociatedType"], .decl(.typealias)), + ] as [([String], Phylum)]) + func Blocks(_ symbol:[String], phylum:Phylum) throws + { + let vertex:SymbolGraphPart.Vertex? = self.symbols.first(named: symbol) + #expect(vertex?.phylum == phylum) + } +} diff --git a/Sources/SymbolGraphPartTests/Support/GenericType (ext).swift b/Sources/SymbolGraphPartTests/Support/GenericType (ext).swift new file mode 100644 index 00000000..aef8e549 --- /dev/null +++ b/Sources/SymbolGraphPartTests/Support/GenericType (ext).swift @@ -0,0 +1,8 @@ +import Signatures +import Symbols + +extension GenericType +{ + static var Equatable:Self { .init(spelling: "Equatable", nominal: "sSQ") } + static var Sequence:Self { .init(spelling: "Sequence", nominal: "sST") } +} diff --git a/Sources/SymbolGraphPartTests/Support/SymbolGraphPart (ext).swift b/Sources/SymbolGraphPartTests/Support/SymbolGraphPart (ext).swift new file mode 100644 index 00000000..fd84fc53 --- /dev/null +++ b/Sources/SymbolGraphPartTests/Support/SymbolGraphPart (ext).swift @@ -0,0 +1,33 @@ +import JSONDecoding +import SymbolGraphParts +import SystemIO +import Testing + +extension SymbolGraphPart +{ + static func load(part path:FilePath) throws -> Self + { + let name:FilePath.Component = try #require(path.lastComponent) + let id:SymbolGraphPart.ID = try #require(.init(name.string)) + + let part:SymbolGraphPart = try .init( + json: .init(utf8: try path.read([UInt8].self)[...]), + id: id) + + #expect(part.metadata.version == .v(0, 6, 0)) + + return part + } + + func first(named path:[String]) -> SymbolGraphPart.Vertex? + { + self.vertices.first + { + switch $0.usr + { + case .vector: false + default: $0.path.elementsEqual(path) + } + } + } +} diff --git a/Sources/SymbolGraphPartTests/SystemProgrammingInterfaces.swift b/Sources/SymbolGraphPartTests/SystemProgrammingInterfaces.swift new file mode 100644 index 00000000..e91dab27 --- /dev/null +++ b/Sources/SymbolGraphPartTests/SystemProgrammingInterfaces.swift @@ -0,0 +1,24 @@ +import SymbolGraphParts +import Testing + +@Suite +struct SystemProgrammingInterfaces +{ + private + let symbols:SymbolGraphPart + + init() throws + { + self.symbols = try .load(part: "TestModules/SymbolGraphs/SPI.symbols.json") + } + + @Test(arguments: [ + (["NoSPI"], nil), + (["SPI"], []), + ] as [([String], [String]?)]) + func Existence(_ symbol:[String], _ spis:[String]?) throws + { + let vertex:SymbolGraphPart.Vertex? = self.symbols.first(named: symbol) + #expect(vertex?.signature.spis == spis) + } +} diff --git a/Sources/SymbolGraphPartTests/TestGroup.swift b/Sources/SymbolGraphPartTests/TestGroup.swift deleted file mode 100644 index e4954e35..00000000 --- a/Sources/SymbolGraphPartTests/TestGroup.swift +++ /dev/null @@ -1,38 +0,0 @@ -import JSONDecoding -import SymbolGraphParts -import SystemIO -import Testing_ - -extension TestGroup -{ - func expect(symbol path:[String], in part:SymbolGraphPart) -> SymbolGraphPart.Vertex? - { - self.expect(value: part.vertices.first - { - switch $0.usr - { - case .vector: false - default: $0.path.elementsEqual(path) - } - }) - } - func load(part path:FilePath) -> SymbolGraphPart? - { - guard let id:FilePath.Component = self.expect(value: path.lastComponent), - let id:SymbolGraphPart.ID = .init(id.string) - else - { - return nil - } - return self.do - { - let part:SymbolGraphPart = try .init( - json: .init(utf8: try path.read([UInt8].self)[...]), - id: id) - - self.expect(part.metadata.version ==? .v(0, 6, 0)) - - return part - } - } -} From 843a69512d73b89592fb52c98f12a83e694dcb04 Mon Sep 17 00:00:00 2001 From: Dianna Date: Thu, 13 Feb 2025 00:03:57 +0000 Subject: [PATCH 12/12] add a test case for reexported symbols --- Scripts/Linux/GenerateTestSymbolGraphs | 1 + Sources/SymbolGraphPartTests/Reexports.swift | 21 ++++++++++++++++++++ TestModules/Snippets/Reexports.swift | 1 + 3 files changed, 23 insertions(+) create mode 100644 Sources/SymbolGraphPartTests/Reexports.swift create mode 100644 TestModules/Snippets/Reexports.swift diff --git a/Scripts/Linux/GenerateTestSymbolGraphs b/Scripts/Linux/GenerateTestSymbolGraphs index 7b6c0ddc..3d2a25d4 100755 --- a/Scripts/Linux/GenerateTestSymbolGraphs +++ b/Scripts/Linux/GenerateTestSymbolGraphs @@ -36,6 +36,7 @@ do -include-spi-symbols \ -pretty-print \ -module-name $(basename $file .swiftmodule) \ + -experimental-allowed-reexported-modules=ACL \ -I $include done done diff --git a/Sources/SymbolGraphPartTests/Reexports.swift b/Sources/SymbolGraphPartTests/Reexports.swift new file mode 100644 index 00000000..65fe834d --- /dev/null +++ b/Sources/SymbolGraphPartTests/Reexports.swift @@ -0,0 +1,21 @@ +import SymbolGraphParts +import Symbols +import Testing + +@Suite +struct Reexports +{ + private + let symbols:SymbolGraphPart + + init() throws + { + self.symbols = try .load(part: "TestModules/SymbolGraphs/Reexports.symbols.json") + } + + @Test + func Existence() throws + { + #expect(nil != self.symbols.first(named: ["Public"])) + } +} diff --git a/TestModules/Snippets/Reexports.swift b/TestModules/Snippets/Reexports.swift new file mode 100644 index 00000000..6bd2cf70 --- /dev/null +++ b/TestModules/Snippets/Reexports.swift @@ -0,0 +1 @@ +@_exported import ACL