Add abstractions for end-to-end tests.

Package.swift

@@ -71,6 +71,7 @@ let package = Package(
             name: "EndToEndTests",
             dependencies: [
                 .target(name: "swift-doc"),
+                .product(name: "Markup", package: "Markup"),
             ]
         ),
     ]

Tests/EndToEndTests/Supporting Types for Generate Tests/GenerateTestCase.swift

@@ -0,0 +1,172 @@
+import XCTest
+import Foundation
+
+/// A class that provides abstractions to write tests for the `generate` subcommand.
+///
+/// Create a subclass of this class to write test cases for the `generate` subcommand.
+/// It provides an API to create source files which should be included in the sources.
+/// Then you can generate the documentation.
+/// If there's an error while generating the documentation for any of the formats,
+/// the test automatically fails.
+/// Additionally, it provides APIs to assert validations on the generated documentation.
+///
+/// ``` swift
+/// class TestVisibility: GenerateTestCase {
+///     func testClassesVisibility() {
+///         sourceFile("Example.swift") {
+///             #"""
+///             public class PublicClass {}
+///
+///             class InternalClass {}
+///
+///             private class PrivateClass {}
+///             """#
+///         }
+///
+///         generate(minimumAccessLevel: .internal)
+///
+///         XCTAssertDocumentationContains(.class("PublicClass"))
+///         XCTAssertDocumentationContains(.class("InternalClass"))
+///         XCTAssertDocumentationNotContains(.class("PrivateClass"))
+///     }
+/// }
+/// ```
+///
+/// The tests are end-to-end tests.
+/// They use the command-line tool to build the documentation
+/// and run the assertions
+/// by reading and understanding the created output of the documentation.
+class GenerateTestCase: XCTestCase {
+    private var sourcesDirectory: URL?
+
+    private var outputs: [GeneratedDocumentation] = []
+
+    /// The output formats which should be generated for this test case.
+    /// You can set a new value in `setUp()` if a test should only generate specific formats.
+    var testedOutputFormats: [GeneratedDocumentation.Type] = []
+
+    override func setUpWithError() throws {
+        try super.setUpWithError()
+
+        sourcesDirectory = try createTemporaryDirectory()
+
+        testedOutputFormats = [GeneratedHTMLDocumentation.self, GeneratedCommonMarkDocumentation.self]
+    }
+
+    override func tearDown() {
+        super.tearDown()
+
+        if let sourcesDirectory = self.sourcesDirectory {
+            try? FileManager.default.removeItem(at: sourcesDirectory)
+        }
+        for output in outputs {
+            try? FileManager.default.removeItem(at: output.directory)
+        }
+    }
+
+    func sourceFile(_ fileName: String, contents: () -> String, file: StaticString = #filePath, line: UInt = #line) {
+        guard let sourcesDirectory = self.sourcesDirectory else {
+            return assertionFailure()
+        }
+        do {
+            try contents().write(to: sourcesDirectory.appendingPathComponent(fileName), atomically: true, encoding: .utf8)
+        }
+        catch let error {
+            XCTFail("Could not create source file '\(fileName)' (\(error))", file: file, line: line)
+        }
+    }
+
+    func generate(minimumAccessLevel: MinimumAccessLevel, file: StaticString = #filePath, line: UInt = #line) {
+        for format in testedOutputFormats {
+            do {
+                let outputDirectory = try createTemporaryDirectory()
+                try Process.run(command: swiftDocCommand,
+                                arguments: [
+                                    "generate",
+                                    "--module-name", "SwiftDoc",
+                                    "--format", format.outputFormat,
+                                    "--output", outputDirectory.path,
+                                    "--minimum-access-level", minimumAccessLevel.rawValue,
+                                    sourcesDirectory!.path
+                ]) { result in
+                    if result.terminationStatus != EXIT_SUCCESS {
+                        XCTFail("Generating documentation failed for format \(format.outputFormat)", file: file, line: line)
+                    }
+                }
+
+                outputs.append(format.init(directory: outputDirectory))
+            }
+            catch let error {
+                XCTFail("Could not generate documentation format \(format.outputFormat) (\(error))", file: file, line: line)
+            }
+        }
+    }
+}
+
+
+extension GenerateTestCase {
+    func XCTAssertDocumentationContains(_ symbolType: SymbolType, file: StaticString = #filePath, line: UInt = #line) {
+        for output in outputs {
+            if output.symbol(symbolType) == nil {
+                XCTFail("Output \(type(of: output).outputFormat) is missing \(symbolType)", file: file, line: line)
+            }
+        }
+    }
+
+    func XCTAssertDocumentationNotContains(_ symbolType: SymbolType, file: StaticString = #filePath, line: UInt = #line) {
+        for output in outputs {
+            if output.symbol(symbolType) != nil {
+                XCTFail("Output \(type(of: output).outputFormat) contains \(symbolType) although it should be omitted", file: file, line: line)
+            }
+        }
+    }
+
+    enum SymbolType: CustomStringConvertible {
+        case `class`(String)
+        case `struct`(String)
+        case `enum`(String)
+        case `typealias`(String)
+        case `protocol`(String)
+        case function(String)
+        case variable(String)
+        case `extension`(String)
+
+        var description: String {
+            switch self {
+            case .class(let name):
+                return "class '\(name)'"
+            case .struct(let name):
+                return "struct '\(name)'"
+            case .enum(let name):
+                return "enum '\(name)'"
+            case .typealias(let name):
+                return "typealias '\(name)'"
+            case .protocol(let name):
+                return "protocol '\(name)'"
+            case .function(let name):
+                return "func '\(name)'"
+            case .variable(let name):
+                return "variable '\(name)'"
+            case .extension(let name):
+                return "extension '\(name)'"
+            }
+        }
+    }
+}
+
+
+extension GenerateTestCase {
+
+    enum MinimumAccessLevel: String {
+        case `public`, `internal`, `private`
+    }
+}
+
+
+
+private func createTemporaryDirectory() throws -> URL {
+    let temporaryDirectoryURL = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(UUID().uuidString)
+    try FileManager.default.createDirectory(at: temporaryDirectoryURL, withIntermediateDirectories: true)
+
+    return temporaryDirectoryURL
+}

Tests/EndToEndTests/Supporting Types for Generate Tests/GeneratedDocumentation.swift

@@ -0,0 +1,184 @@
+import Foundation
+import HTML
+import CommonMark
+
+/// A protocol which needs to be implemented by the different documentation generators. It provides an API to operate
+/// on the generated documentation.
+protocol GeneratedDocumentation {
+
+    /// The name of the output format. This needs to be the name name like the value passed to swift-doc's `format` option.
+    static var outputFormat: String { get }
+
+    init(directory: URL)
+
+    var directory: URL { get }
+
+    func symbol(_ symbolType: GenerateTestCase.SymbolType) -> Page?
+}
+
+protocol Page {
+    var type: String? { get }
+
+    var name: String? { get }
+}
+
+
+
+struct GeneratedHTMLDocumentation: GeneratedDocumentation {
+
+    static let outputFormat = "html"
+
+    let directory: URL
+
+    func symbol(_ symbolType: GenerateTestCase.SymbolType) -> Page? {
+        switch symbolType {
+        case .class(let name):
+            return page(for: name, ofType: "Class")
+        case .typealias(let name):
+            return page(for: name, ofType: "Typealias")
+        case .struct(let name):
+            return page(for: name, ofType: "Structure")
+        case .enum(let name):
+            return page(for: name, ofType: "Enumeration")
+        case .protocol(let name):
+            return page(for: name, ofType: "Protocol")
+        case .function(let name):
+            return page(for: name, ofType: "Function")
+        case .variable(let name):
+            return page(for: name, ofType: "Variable")
+        case .extension(let name):
+            return page(for: name, ofType: "Extensions on")
+        }
+    }
+
+    private func page(for symbolName: String, ofType type: String) -> Page? {
+        guard let page = page(named: symbolName) else { return nil }
+        guard page.type == type else { return nil }
+
+        return page
+    }
+
+    private func page(named name: String) -> HtmlPage? {
+        let fileUrl = directory.appendingPathComponent(fileName(forSymbol: name)).appendingPathComponent("index.html")
+        guard
+            FileManager.default.isReadableFile(atPath: fileUrl.path),
+            let contents = try? String(contentsOf: fileUrl),
+            let document = try? HTML.Document(string: contents)
+            else { return nil }
+
+        return HtmlPage(document: document)
+    }
+
+    private func fileName(forSymbol symbolName: String) -> String {
+        symbolName
+            .replacingOccurrences(of: ".", with: "_")
+            .replacingOccurrences(of: " ", with: "-")
+            .components(separatedBy: reservedCharactersInFilenames).joined(separator: "_")
+    }
+
+    private struct HtmlPage: Page {
+        let document: HTML.Document
+
+        var type: String? {
+            let results = document.search(xpath: "//h1/small")
+            assert(results.count == 1)
+            return results.first?.content
+        }
+
+        var name: String? {
+            let results = document.search(xpath: "//h1/code")
+            assert(results.count == 1)
+            return results.first?.content
+        }
+    }
+}
+
+
+struct GeneratedCommonMarkDocumentation: GeneratedDocumentation {
+
+    static let outputFormat = "commonmark"
+
+    let directory: URL
+
+    func symbol(_ symbolType: GenerateTestCase.SymbolType) -> Page? {
+        switch symbolType {
+        case .class(let name):
+            return page(for: name, ofType: "class")
+        case .typealias(let name):
+            return page(for: name, ofType: "typealias")
+        case .struct(let name):
+            return page(for: name, ofType: "struct")
+        case .enum(let name):
+            return page(for: name, ofType: "enum")
+        case .protocol(let name):
+            return page(for: name, ofType: "protocol")
+        case .function(let name):
+            return page(for: name, ofType: "func")
+        case .variable(let name):
+            return page(for: name, ofType: "var") ?? page(for: name, ofType: "let")
+        case .extension(let name):
+            return page(for: name, ofType: "extension")
+        }
+    }
+
+    private func page(for symbolName: String, ofType type: String) -> Page? {
+        guard let page = page(named: symbolName) else { return nil }
+        guard page.type == type else { return nil }
+
+        return page
+    }
+
+    private func page(named name: String) -> CommonMarkPage? {
+        let fileUrl = directory.appendingPathComponent("\(name).md")
+        guard
+            FileManager.default.isReadableFile(atPath: fileUrl.path),
+            let contents = try? String(contentsOf: fileUrl),
+            let document = try? CommonMark.Document(contents)
+            else { return nil }
+
+        return CommonMarkPage(document: document)
+    }
+
+    private func fileName(forSymbol symbolName: String) -> String {
+        symbolName
+            .replacingOccurrences(of: ".", with: "_")
+            .replacingOccurrences(of: " ", with: "-")
+            .components(separatedBy: reservedCharactersInFilenames).joined(separator: "_")
+    }
+
+    private struct CommonMarkPage: Page {
+        let document: CommonMark.Document
+
+        private var headingElement: Heading? {
+            document.children.first(where: { ($0 as? Heading)?.level == 1 }) as? Heading
+        }
+
+        var type: String? {
+            // Our CommonMark pages don't give a hint of the actual type of a documentation page. That's why we extract
+            // it via a regex out of the declaration. Not very nice, but works for now.
+            guard
+                let name = self.name,
+                let code = document.children.first(where: { $0 is CodeBlock}) as? CodeBlock,
+                let codeContents = code.literal,
+                let extractionRegex = try? NSRegularExpression(pattern: "([a-z]+) \(name)")
+            else { return nil }
+
+            guard
+                let match = extractionRegex.firstMatch(in: codeContents, range: NSRange(location: 0, length: codeContents.utf16.count)),
+                match.numberOfRanges > 0,
+                let range = Range(match.range(at: 1), in: codeContents)
+            else { return nil }
+
+            return String(codeContents[range])
+        }
+
+        var name: String? {
+            headingElement?.children.compactMap { ($0 as? Literal)?.literal }.joined()
+        }
+    }
+}
+
+private let reservedCharactersInFilenames: CharacterSet = [
+    // Windows Reserved Characters
+    "<", ">", ":", "\"", "/", "\\", "|", "?", "*",
+]