diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bb460e7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.DS_Store +/.build +/Packages +/*.xcodeproj +xcuserdata/ +DerivedData/ +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata diff --git a/.spi.yml b/.spi.yml new file mode 100644 index 0000000..e07f7a2 --- /dev/null +++ b/.spi.yml @@ -0,0 +1,4 @@ +version: 1 +builder: + configs: + - documentation_targets: [MVTPostgis] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b51348b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Outdooractive AG + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Package.resolved b/Package.resolved new file mode 100644 index 0000000..76b66c0 --- /dev/null +++ b/Package.resolved @@ -0,0 +1,212 @@ +{ + "pins" : [ + { + "identity" : "async-kit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vapor/async-kit.git", + "state" : { + "revision" : "7ece208cd401687641c88367a00e3ea2b04311f1", + "version" : "1.19.0" + } + }, + { + "identity" : "gis-tools", + "kind" : "remoteSourceControl", + "location" : "https://github.com/Outdooractive/gis-tools", + "state" : { + "revision" : "271ffa105096a3c5889b211227c9a31dc59030db", + "version" : "1.2.0" + } + }, + { + "identity" : "gzipswift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/1024jp/GzipSwift.git", + "state" : { + "revision" : "7a7f17761c76a932662ab77028a4329f67d645a4", + "version" : "5.2.0" + } + }, + { + "identity" : "mvt-tools", + "kind" : "remoteSourceControl", + "location" : "https://github.com/Outdooractive/mvt-tools", + "state" : { + "revision" : "f19faad0ddbd8378ad8538909fb150b5fa6f7141", + "version" : "1.3.2" + } + }, + { + "identity" : "postgres-kit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vapor/postgres-kit", + "state" : { + "revision" : "475bf6f04ee1840917a70c32b48e4a724df4ccaf", + "version" : "2.12.3" + } + }, + { + "identity" : "postgres-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vapor/postgres-nio.git", + "state" : { + "revision" : "69ccfdf4c80144d845e3b439961b7ec6cd7ae33f", + "version" : "1.20.2" + } + }, + { + "identity" : "postgresconnectionpool", + "kind" : "remoteSourceControl", + "location" : "https://github.com/Outdooractive/PostgresConnectionPool.git", + "state" : { + "revision" : "e8c65e5b0a987b678be2294a8113c5a21889c86f", + "version" : "0.7.0" + } + }, + { + "identity" : "sql-kit", + "kind" : "remoteSourceControl", + "location" : "https://github.com/vapor/sql-kit.git", + "state" : { + "revision" : "b2f128cb62a3abfbb1e3b2893ff3ee69e70f4f0f", + "version" : "3.28.0" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms.git", + "state" : { + "revision" : "f6919dfc309e7f1b56224378b11e28bab5bccc42", + "version" : "1.2.0" + } + }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser", + "state" : { + "revision" : "c8ed701b513cf5177118a175d85fbbbcd707ab41", + "version" : "1.3.0" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "cd142fd2f64be2100422d658e7411e39489da985", + "version" : "1.2.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections", + "state" : { + "revision" : "94cf62b3ba8d4bed62680a282d4c25f9c63c2efb", + "version" : "1.1.0" + } + }, + { + "identity" : "swift-crypto", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-crypto.git", + "state" : { + "revision" : "cc76b894169a3c86b71bac10c78a4db6beb7a9ad", + "version" : "3.2.0" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "e97a6fcb1ab07462881ac165fdbb37f067e205d5", + "version" : "1.5.4" + } + }, + { + "identity" : "swift-metrics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-metrics.git", + "state" : { + "revision" : "971ba26378ab69c43737ee7ba967a896cb74c0d1", + "version" : "2.4.1" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "635b2589494c97e48c62514bc8b37ced762e0a62", + "version" : "2.63.0" + } + }, + { + "identity" : "swift-nio-ssl", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-ssl.git", + "state" : { + "revision" : "7c381eb6083542b124a6c18fae742f55001dc2b5", + "version" : "2.26.0" + } + }, + { + "identity" : "swift-nio-transport-services", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio-transport-services.git", + "state" : { + "revision" : "6cbe0ed2b394f21ab0d46b9f0c50c6be964968ce", + "version" : "1.20.1" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", + "version" : "1.0.2" + } + }, + { + "identity" : "swift-protobuf", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-protobuf", + "state" : { + "revision" : "65e8f29b2d63c4e38e736b25c27b83e012159be8", + "version" : "1.25.2" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "025bcb1165deab2e20d4eaba79967ce73013f496", + "version" : "1.2.1" + } + }, + { + "identity" : "swiftyxmlparser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/yahoojapan/SwiftyXMLParser", + "state" : { + "revision" : "d7a1d23f04c86c1cd2e8f19247dd15d74e0ea8be", + "version" : "5.6.0" + } + }, + { + "identity" : "yams", + "kind" : "remoteSourceControl", + "location" : "https://github.com/jpsim/Yams", + "state" : { + "revision" : "9ff1cc9327586db4e0c8f46f064b6a82ec1566fa", + "version" : "4.0.6" + } + } + ], + "version" : 2 +} diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000..981856e --- /dev/null +++ b/Package.swift @@ -0,0 +1,37 @@ +// swift-tools-version:5.9 + +import PackageDescription + +let package = Package( + name: "mvt-postgis", + platforms: [ + .macOS(.v13), + .iOS(.v16), + .tvOS(.v16), + ], + products: [ + .library(name: "MVTPostgis", targets: ["MVTPostgis"]), + ], + dependencies: [ + .package(url: "https://github.com/Outdooractive/mvt-tools", from: "1.3.2"), + .package(url: "https://github.com/Outdooractive/gis-tools", from: "1.2.0"), + .package(url: "https://github.com/Outdooractive/PostgresConnectionPool.git", from: "0.7.0"), + .package(url: "https://github.com/vapor/postgres-nio.git", from: "1.20.2"), + .package(url: "https://github.com/apple/swift-collections", from: "1.1.0"), + .package(url: "https://github.com/yahoojapan/SwiftyXMLParser", from: "5.6.0"), + .package(url: "https://github.com/jpsim/Yams", from: "4.0.6"), + ], + targets: [ + .target( + name: "MVTPostgis", + dependencies: [ + .product(name: "GISTools", package: "gis-tools"), + .product(name: "MVTTools", package: "mvt-tools"), + .product(name: "PostgresConnectionPool", package: "PostgresConnectionPool"), + .product(name: "PostgresNIO", package: "postgres-nio"), + .product(name: "Collections", package: "swift-collections"), + .product(name: "SwiftyXMLParser", package: "SwiftyXMLParser"), + .product(name: "Yams", package: "Yams"), + ]), + ] +) diff --git a/README.md b/README.md new file mode 100644 index 0000000..e7b85b9 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +[![](https://img.shields.io/endpoint?url=https%3A%2F%2Fswiftpackageindex.com%2Fapi%2Fpackages%2FOutdooractive%2Fmvt-postgis%2Fbadge%3Ftype%3Dswift-versions)](https://swiftpackageindex.com/Outdooractive/mvt-postgis) +[![](https://img.shields.io/endpoint?url=https%3A%2F%2Fswiftpackageindex.com%2Fapi%2Fpackages%2FOutdooractive%2Fmvt-postgis%2Fbadge%3Ftype%3Dplatforms)](https://swiftpackageindex.com/Outdooractive/mvt-postgis) + +# MVTPostgis + +Creates vector tiles from Postgis databases. + +## Installation with Swift Package Manager + +```swift +dependencies: [ + .package(url: "https://github.com/Outdooractive/mvt-postgis", from: "1.0.0"), +], +targets: [ + .target(name: "MyTarget", dependencies: [ + .product(name: "MVTPostgis", package: "mvt-postgis"), + ]), +] +``` + +## Features + +TODO + +## Usage + +TODO + +## Contributing + +Please create an issue or open a pull request with a fix + +## TODOs and future improvements + +- Restart queries after timeout +- Explore ST_AsMVTGeom +- Define a JSON source format +- Documentation (!) +- Tests + +## Links + +- Libraries + - https://github.com/Outdooractive/gis-tools + - https://github.com/Outdooractive/mvt-tools + +- Mapnik Postgis documentation: + - https://github.com/mapnik/mapnik/wiki/PostGIS + - https://github.com/mapnik/mapnik/wiki/OptimizeRenderingWithPostGIS + +- Mapnik files: + - https://github.com/mapnik/mapnik/blob/master/test/unit/datasource/postgis.cpp + - https://github.com/mapnik/mapnik/blob/master/plugins/input/postgis/postgis_datasource.cpp + +- Other: + - https://github.com/plarson/fluent-postgis + - https://github.com/koher/swift-image + - https://github.com/t-ae/swim + - https://github.com/GEOSwift/GEOSwift + +## License + +MIT + +## Author + +Thomas Rasch, Outdooractive diff --git a/Sources/MVTPostgis/Extensions/ArrayExtensions.swift b/Sources/MVTPostgis/Extensions/ArrayExtensions.swift new file mode 100644 index 0000000..bfec098 --- /dev/null +++ b/Sources/MVTPostgis/Extensions/ArrayExtensions.swift @@ -0,0 +1,24 @@ +import Foundation + +extension Array { + + var isNotEmpty: Bool { + !isEmpty + } + + /// Adds a new element at the end of the array, if it's not nil. + mutating func append(ifNotNil newElement: Element?) { + guard let element = newElement else { return } + append(element) + } + +} + +extension Array where Element: Equatable { + + /// Removes all occurrences of the given object + mutating func remove(_ element: Element) { + self = filter { $0 != element } + } + +} diff --git a/Sources/MVTPostgis/Extensions/DataExtensions.swift b/Sources/MVTPostgis/Extensions/DataExtensions.swift new file mode 100644 index 0000000..7fad6b0 --- /dev/null +++ b/Sources/MVTPostgis/Extensions/DataExtensions.swift @@ -0,0 +1,9 @@ +import Foundation + +extension Data { + + var asUTF8EncodedString: String? { + String(data: self, encoding: .utf8) + } + +} diff --git a/Sources/MVTPostgis/Extensions/DequeExtensions.swift b/Sources/MVTPostgis/Extensions/DequeExtensions.swift new file mode 100644 index 0000000..6618355 --- /dev/null +++ b/Sources/MVTPostgis/Extensions/DequeExtensions.swift @@ -0,0 +1,9 @@ +import Collections + +extension Deque { + + var isNotEmpty: Bool { + !isEmpty + } + +} diff --git a/Sources/MVTPostgis/Extensions/DictionaryExtensions.swift b/Sources/MVTPostgis/Extensions/DictionaryExtensions.swift new file mode 100644 index 0000000..b18ba29 --- /dev/null +++ b/Sources/MVTPostgis/Extensions/DictionaryExtensions.swift @@ -0,0 +1,13 @@ +import Foundation + +extension Dictionary { + + func hasKey(_ key: Key) -> Bool { + return self[key] != nil + } + + var isNotEmpty: Bool { + !isEmpty + } + +} diff --git a/Sources/MVTPostgis/Extensions/DoubleExtensions.swift b/Sources/MVTPostgis/Extensions/DoubleExtensions.swift new file mode 100644 index 0000000..b718286 --- /dev/null +++ b/Sources/MVTPostgis/Extensions/DoubleExtensions.swift @@ -0,0 +1,9 @@ +import Foundation + +extension Double { + + func atLeast(_ minValue: Double) -> Double { + Swift.max(minValue, self) + } + +} diff --git a/Sources/MVTPostgis/Extensions/FloatingPointExtensions.swift b/Sources/MVTPostgis/Extensions/FloatingPointExtensions.swift new file mode 100644 index 0000000..a2085b9 --- /dev/null +++ b/Sources/MVTPostgis/Extensions/FloatingPointExtensions.swift @@ -0,0 +1,22 @@ +import Foundation + +extension FloatingPoint { + + /// Returns rounded FloatingPoint to specified number of places + func rounded(toPlaces places: Int) -> Self { + guard places >= 0 else { return self } + + var divisor: Self = 1 + for _ in 0 ..< places { + divisor *= 10 + } + + return (self * divisor).rounded() / divisor + } + + /// Rounds current FloatingPoint to specified number of places + mutating func round(toPlaces places: Int) { + self = rounded(toPlaces: places) + } + +} diff --git a/Sources/MVTPostgis/Extensions/IntExtensions.swift b/Sources/MVTPostgis/Extensions/IntExtensions.swift new file mode 100644 index 0000000..0c10078 --- /dev/null +++ b/Sources/MVTPostgis/Extensions/IntExtensions.swift @@ -0,0 +1,9 @@ +import Foundation + +extension Int { + + func atLeast(_ minValue: Int) -> Int { + Swift.max(minValue, self) + } + +} diff --git a/Sources/MVTPostgis/Extensions/StringExtensions.swift b/Sources/MVTPostgis/Extensions/StringExtensions.swift new file mode 100644 index 0000000..acbc934 --- /dev/null +++ b/Sources/MVTPostgis/Extensions/StringExtensions.swift @@ -0,0 +1,19 @@ +import Foundation + +extension String { + + var isNotEmpty: Bool { + !isEmpty + } + + /// The string, or nil if it is empty + var nilIfEmpty: String? { + guard isNotEmpty else { return nil } + return self + } + + var toInt: Int? { + Int(self) + } + +} diff --git a/Sources/MVTPostgis/Extensions/TaskExtensions.swift b/Sources/MVTPostgis/Extensions/TaskExtensions.swift new file mode 100644 index 0000000..5f2403b --- /dev/null +++ b/Sources/MVTPostgis/Extensions/TaskExtensions.swift @@ -0,0 +1,30 @@ +import Foundation + +extension Task where Failure == Error { + + @discardableResult + static func after( + seconds: TimeInterval, + priority: TaskPriority? = nil, + operation: @escaping @Sendable () async throws -> Success) + -> Task + { + Task(priority: priority) { + let delay = UInt64(seconds * 1_000_000_000) + try await Task.sleep(nanoseconds: delay) + + return try await operation() + } + } + + @discardableResult + static func background(_ operation: @escaping @Sendable () async throws -> Success) -> Task { + Task(priority: .background, operation: operation) + } + + @discardableResult + static func userInitiated(_ operation: @escaping @Sendable () async throws -> Success) -> Task { + Task(priority: .userInitiated, operation: operation) + } + +} diff --git a/Sources/MVTPostgis/MVTLayerPerformanceData.swift b/Sources/MVTPostgis/MVTLayerPerformanceData.swift new file mode 100644 index 0000000..2dc213f --- /dev/null +++ b/Sources/MVTPostgis/MVTLayerPerformanceData.swift @@ -0,0 +1,27 @@ +import Foundation + +/// Some statistics about query performance of one (Postgis) layer. +public struct MVTLayerPerformanceData { + + /// The total query runtime (Postgis + parsing). + public let runtime: TimeInterval + /// The received WKB geometry bytes from the db server. + public let wkbBytes: Int64 + /// The number of features in a layer. + public let features: Int + /// The number of invslid features in a layer. + public let invalidFeatures: Int + + public init( + runtime: TimeInterval, + wkbBytes: Int64, + features: Int, + invalidFeatures: Int) + { + self.runtime = runtime + self.wkbBytes = wkbBytes + self.features = features + self.invalidFeatures = invalidFeatures + } + +} diff --git a/Sources/MVTPostgis/MVTPostgis.swift b/Sources/MVTPostgis/MVTPostgis.swift new file mode 100644 index 0000000..7e0d902 --- /dev/null +++ b/Sources/MVTPostgis/MVTPostgis.swift @@ -0,0 +1,506 @@ +import Foundation +import GISTools +import MVTTools +import PostgresNIO +import PostgresConnectionPool + +// https://github.com/mapnik/mapnik/wiki/PostGIS +// https://github.com/mapnik/mapnik/wiki/OptimizeRenderingWithPostGIS +// https://github.com/mapnik/mapnik/blob/master/plugins/input/postgis/postgis_datasource.cpp +// https://github.com/mapnik/mapnik/blob/master/test/unit/datasource/postgis.cpp + +/// A tool for creating vector tiles from Postgis datasources. +/// Accepts YML and XML sources (as used by Mapnik and the old Mapbox Studio). +public final class MVTPostgis { + + /// **MUST** be changed before first use. See ``MVTPostgisConfiguration``. + public static var configuration: MVTPostgisConfiguration = MVTPostgisConfiguration() + + private static let postgisDatasourceTypeCode = "postgis" + private static var batchId: Int = 0 + + /// The minimum zoom of the datasource. + public let minZoom: Int + /// The maximum zoom of the datasource. + public let maxZoom: Int + + /// The Mapnik source. + public let source: PostgisSource + /// The source's projection (either EPSG:3857 or EPSG:4326). + public let projection: Projection + + /// Some external name that users can coose to distinguish this + /// instance from other instances. Used e.g. in runtime tracking + /// and some error messages. + public let externalName: String? + + private let logger: Logger + private let poolDistributor: PoolDistributor + + // MARK: - + + /// Initialize a MVT creator from a file or from the network. + public convenience init( + sourceURL: URL, + externalName: String? = nil, + layerWhitelist: [String]? = nil, + logger: Logger? = nil) + throws + { + let source = try PostgisSource.load( + from: sourceURL, + layerWhitelist: layerWhitelist) + + try self.init( + source: source, + externalName: externalName, + logger: logger) + } + + /// Initialize a MVT creator directly from a data object. + public convenience init( + sourceData: Data, + externalName: String? = nil, + layerWhitelist: [String]? = nil, + logger: Logger? = nil) + throws + { + let source = try PostgisSource.load( + from: sourceData, + layerWhitelist: layerWhitelist) + + try self.init( + source: source, + externalName: externalName, + logger: logger) + } + + /// Initialize a MVT creator directly with a parsed source object. + public init( + source: PostgisSource, + externalName: String? = nil, + logger: Logger? = nil) + throws + { + guard source.layers.count > 0 else { throw MVTPostgisError.needLayers } + + guard source.layers.allSatisfy({ $0.datasource.type == MVTPostgis.postgisDatasourceTypeCode }) else { + throw MVTPostgisError.wrongDatasourceType(message: "All datasources must be of type '\(MVTPostgis.postgisDatasourceTypeCode)'") + } + + let layer = source.layers[0] + let datasource = layer.datasource + + let srs = layer.srs + if !srs.isEmpty { + // Supported: + // srs: +proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs + // srs: +proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over + + switch srs.lowercased() { + case "+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs": + projection = .epsg4326 + case "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0.0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs +over": + projection = .epsg3857 + default: + throw MVTPostgisError.unsupportedSRS + } + } + else { + guard let srid = Int(datasource.srid), + let projection = Projection(srid: srid), + projection != .noSRID + else { throw MVTPostgisError.unsupportedSRID } + self.projection = projection + } + + self.source = source + self.externalName = externalName + self.minZoom = source.minZoom + self.maxZoom = source.maxZoom + self.logger = logger ?? { + var logger = Logger(label: "\(MVTPostgis.configuration.applicationName).\(externalName ?? source.name)") + logger.logLevel = .info + return logger + }() + self.poolDistributor = PoolDistributor( + configuration: MVTPostgis.configuration, + logger: self.logger) + } + + /// Close all database connections. + /// + /// **MUST** be called when done with all MVTPostgis instances + public func shutdown() async { + await poolDistributor.shutdown() + } + + /// Forcibly close all idle connections in all pools. + public func closeIdleConnections() async { + await poolDistributor.closeIdleConnections() + } + + /// Information about database pools and open connections. + public func poolInfos() async -> [PoolInfo] { + await poolDistributor.poolInfos() + } + + // MARK: - + + /// Return tile data at the given z/x/y coordinate. + /// + /// - Note: Only `bufferSize` from `options` will be used here. + public func tileData( + tile: MapTile, + options: VectorTileExportOptions) + async throws -> (data: Data?, performance: [String: MVTLayerPerformanceData]?) + { + let tileAndPerformanceData = try await mvtForTile(tile: tile, options: options) + return (tileAndPerformanceData.tile.data(options: options), tileAndPerformanceData.performance) + } + + /// Create a tile at the given z/x/y coordinate. + /// + /// - Note: Only `bufferSize` from `options` will be used here. + public func mvtForTile( + tile: MapTile, + options: VectorTileExportOptions? = nil) + async throws -> (tile: VectorTile, performance: [String: MVTLayerPerformanceData]?) + { + if Task.isCancelled { + throw MVTPostgisError.cancelled + } + + let nextBatchId = MVTPostgis.batchId + MVTPostgis.batchId += 1 + + return try await withThrowingTaskGroup( + of: (String, String, [Feature], MVTLayerPerformanceData).self, + body: { group -> (tile: VectorTile, performance: [String: MVTLayerPerformanceData]?) in + // Note: Geometries loaded from WKB will always be projected to EPSG:4326 + guard var mvt = VectorTile(tile: tile, projection: projection) else { + throw MVTPostgisError.tileOutOfBounds + } + + // https://github.com/mapnik/mapnik/blob/master/src/scale_denominator.cpp + // https://github.com/openstreetmap/mapnik-stylesheets/blob/master/zoom-to-scale.txt + // 0.0293611270703125 ? (in nodejs/wms-client) + let pixelSize = 0.00028 // 0.28mm, in meters + let tileSize = GISTool.tileSideLength // 256px + let scaleDenominator = GISTool.earthCircumference / ((tileSize * pow(2.0, Double(tile.z))) * pixelSize) + let pixelWidth = tile.metersPerPixel + let simplificationTolerance = simplificationTolerance(pixelWidth: pixelWidth, atZoom: tile.z) + + let deadline = Date(timeIntervalSinceNow: MVTPostgis.configuration.tileTimeout) + let expectedTasksCount = source.layers.count + var finishedTasksCount = 0 + + for layer in source.layers { + let bounds = try queryBounds( + tile: tile, + tileSize: options?.tileSize ?? 256, // pixels + bufferSize: layer.bufferSize) // pixels + let envelope = "ST_MakeEnvelope(\(bounds.southWest.longitude), \(bounds.southWest.latitude), \(bounds.northEast.longitude), \(bounds.northEast.latitude), \(bounds.projection.srid))" + + let sql = layer.datasource.sql + .replacingOccurrences(of: "!bbox!", with: envelope) + .replacingOccurrences(of: "!scale_denominator!", with: String(scaleDenominator)) + .replacingOccurrences(of: "!pixel_width!", with: String(pixelWidth)) + + let geometryField = layer.datasource.geometryField.nilIfEmpty ?? "geometry" + let simplificationOption = MVTPostgis.configuration.simplification(tile.z, self.source) + let clippingOption = MVTPostgis.configuration.clipping(tile.z, self.source) + var columns = layer.fields.keys.map({ "\"\($0)\"" }) + var useLocalSimplification = false + + // Assemble the geometry query + var postgisGeometryColumn = "ST_AsBinary(" + switch simplificationOption { + case .postgis, .meters(_, _): postgisGeometryColumn.append("ST_Simplify(") + case .local: useLocalSimplification = true + default: break + } + if clippingOption == .postgis { + postgisGeometryColumn.append("ST_ClipByBox2D(") + } + postgisGeometryColumn.append("\"\(geometryField)\"") + if clippingOption == .postgis { + postgisGeometryColumn.append(",\(envelope))") + } + if case let .postgis(preserveCollapsed) = simplificationOption { + postgisGeometryColumn.append(",\(simplificationTolerance)") + if preserveCollapsed { + postgisGeometryColumn.append(",true") + } + postgisGeometryColumn.append(")") + } + else if case let .meters(meters, preserveCollapsed) = simplificationOption { + postgisGeometryColumn.append(",\(meters)") + if preserveCollapsed { + postgisGeometryColumn.append(",true") + } + postgisGeometryColumn.append(")") + } + postgisGeometryColumn.append(") AS \"\(geometryField)\"") + columns.append(postgisGeometryColumn) + + // The final query + let query = "SELECT \(columns.joined(separator: ",")) FROM \(sql)" + + let clipBounds = (clippingOption == .local ? bounds : nil) + let localSimplificationTolerance = (useLocalSimplification ? simplificationTolerance : nil) + + group.addTask { + let (features, performanceData) = try await self.load( + query: query, + layer: layer, + projection: self.projection, + clipBounds: clipBounds, + simplificationTolerance: localSimplificationTolerance, + batchId: nextBatchId) + return (layer.id, layer.datasource.databaseName, features, performanceData) + } + } + + // Add a timeout for the whole tile/batch + group.addTask { [weak self] in + let interval = deadline.timeIntervalSinceNow + if interval > 0 { + do { + try await Task.sleep(nanoseconds: UInt64(interval * 1_000_000_000)) + } + catch {} + } + guard Task.isCancelled else { + let timedOutQueries = await self?.poolDistributor + .poolInfos(batchId: nextBatchId) + .flatMap({ pool in + pool.connections.compactMap({ connection -> String? in + let query = connection.query ?? "" + let runtime = connection.queryRuntime ?? 0.0 + return "Query runtime: \(Int(runtime))s\n\(query)" + }) + }) ?? [] + + self?.logger.info("\(self?.externalName ?? self?.source.name ?? "n/a"): Batch \(nextBatchId) (\(tile.z)/\(tile.x)/\(tile.y)) timed out after \(MVTPostgis.configuration.tileTimeout) seconds:\n\(timedOutQueries.joined(separator: "\n"))") + throw MVTPostgisError.tileTimedOut(queries: timedOutQueries) + } + return ("", "", [], MVTLayerPerformanceData(runtime: 0.0, wkbBytes: 0, features: 0, invalidFeatures: 0)) + } + + var layerIdToRuntimeMapping: [String: MVTLayerPerformanceData]? + if MVTPostgis.configuration.trackRuntimes { + layerIdToRuntimeMapping = [:] + } + + do { + for try await (layerId, databaseName, features, performanceData) in group { + guard layerId.isNotEmpty else { continue } + + mvt.appendFeatures(features, to: layerId) + + if MVTPostgis.configuration.trackRuntimes { + layerIdToRuntimeMapping?["\(externalName ?? source.name).\(databaseName).\(layerId)"] = performanceData + } + + // The last task to finish is (should be) the timeout task + finishedTasksCount += 1 + if finishedTasksCount == expectedTasksCount { + group.cancelAll() + } + } + } + catch { + group.cancelAll() + await poolDistributor.abortBatch(nextBatchId) + throw error + } + + return (mvt, layerIdToRuntimeMapping) + }) + } + + // MARK: - Private + + private func queryBounds( + tile: MapTile, + tileSize: Int, + bufferSize: Int) // pixels + throws -> BoundingBox + { + var bounds: BoundingBox + + switch projection { + case .noSRID: + throw MVTPostgisError.unsupportedSRID + case .epsg3857: + bounds = tile.boundingBox(projection: .epsg3857) + case .epsg4326: + bounds = tile.boundingBox(projection: .epsg4326) + } + + if bufferSize != 0 { + let sqrt2 = 2.0.squareRoot() + let diagonal = Double(tileSize) * sqrt2 + let bufferDiagonal = Double(bufferSize) * sqrt2 + let factor = bufferDiagonal / diagonal + + let diagonalLength = bounds.southWest.distance(from: bounds.northEast) + let distance = diagonalLength * factor + + bounds = bounds.expanded(byDistance: distance) + } + + return bounds + } + + private func simplificationTolerance( + pixelWidth: Double, + atZoom zoom: Int) + -> Double + { + // pixelWidth at zoom 10 + pixelWidth * ((((0.6 - 1.4) / 20.0) * Double(zoom)) + 1.4) + } + + // The resulting features will always be projected to EPSG:4326 + private func load( + query: String, + layer: PostgisLayer, + projection: Projection, + clipBounds: BoundingBox?, + simplificationTolerance: Double?, + batchId: Int) + async throws -> (features: [Feature], performance: MVTLayerPerformanceData) + { + if Task.isCancelled { + throw MVTPostgisError.cancelled + } + + var features: [Feature] = [] + var runtime: TimeInterval = 0.0 + var wkbBytes: Int64 = 0 + var invalidFeatures: Int = 0 + + let geometryColumn = layer.datasource.geometryField.nilIfEmpty ?? "geometry" + try await poolDistributor.connection( + forLayer: layer, + batchId: batchId, + callback: { connection in + let startTimestamp = Date() + + let rowSequence = try await connection.query(PostgresQuery(stringLiteral: query), logger: logger) + + // Probably a long running connection + guard !connection.isClosed else { + throw MVTPostgisError.connectionFailed + } + + for try await serialRow in rowSequence { + let row = serialRow.makeRandomAccess() + + guard row.contains(geometryColumn) else { + // TODO: Throw error or find a suitable column + logger.warning("\(externalName ?? source.name): Couldn't find geometry column in layer \(layer.id)") + break + } + guard let geometryBytes = row[data: geometryColumn].value else { continue } + + let geometryData = Data(buffer: geometryBytes) + + var properties: [String: Any] = [:] + for field in row { + guard field.columnName != layer.datasource.geometryField else { continue } + + switch field.dataType { + case .bpchar, .varchar, .text: + properties[field.columnName] = row[data: field.columnName].string + case .varcharArray, .textArray: + if let array = row[data: field.columnName].array { + properties[field.columnName] = array.compactMap({ $0.string }) + } + + case .int2, .int4, .int8: + properties[field.columnName] = row[data: field.columnName].int + case .int2Array, .int4Array, .int8Array: + if let array = row[data: field.columnName].array { + properties[field.columnName] = array.map({ $0.int }) + } + + case .float4, .float8, .numeric: + properties[field.columnName] = row[data: field.columnName].double + case .float4Array, .float8Array: + if let array = row[data: field.columnName].array { + properties[field.columnName] = array.map({ $0.double }) + } + + case .bool: + properties[field.columnName] = row[data: field.columnName].bool + case .boolArray: + if let array = row[data: field.columnName].array { + properties[field.columnName] = array.map({ $0.bool }) + } + + default: + // select * from pg_type where oid = ?; + // Please open an issue if you need more + logger.debug("\(externalName ?? source.name): Unknown type OID \(field.dataType.rawValue) for column '\(field.columnName)' in layer '\(layer.id)'") + } + } + + wkbBytes += Int64(geometryData.count) + + // The vector tile spec only allows Int ids + var featureId: Feature.Identifier? + if properties["id"] is Int { + featureId = .init(value: properties.removeValue(forKey: "id")) + } + + let feature = Feature( + wkb: geometryData, + sourceProjection: projection, + targetProjection: projection, + id: featureId, + properties: properties) + + guard let feature else { + invalidFeatures += 1 + continue + } + + if let clipBounds, let simplificationTolerance { + guard let clippedFeature = feature.clipped(to: clipBounds) else { + invalidFeatures += 1 + continue + } + features.append(clippedFeature.simplified(tolerance: simplificationTolerance)) + } + else if let clipBounds { + guard let clippedFeature = feature.clipped(to: clipBounds) else { + invalidFeatures += 1 + continue + } + features.append(clippedFeature) + } + else if let simplificationTolerance { + features.append(feature.simplified(tolerance: simplificationTolerance)) + } + else { + features.append(feature) + } + } + + runtime = fabs(startTimestamp.timeIntervalSinceNow) + }) + + logger.debug("\(externalName ?? source.name).\(layer.datasource.databaseName).\(layer.id): \(features.count) feature(s) (\(invalidFeatures) invalid) in \(runtime.rounded(toPlaces: 3))s (\(wkbBytes) bytes)") + +// if invalidFeatures > 0, logger.logLevel > .debug { +// logger.info("\(externalName ?? source.name).\(layer.datasource.databaseName).\(layer.id): \(invalidFeatures) invalid features") +// } + + // Features will be projected to EPSG:4326 + return (features, MVTLayerPerformanceData(runtime: runtime, wkbBytes: wkbBytes, features: features.count, invalidFeatures: invalidFeatures)) + } + +} diff --git a/Sources/MVTPostgis/MVTPostgisConfiguration.swift b/Sources/MVTPostgis/MVTPostgisConfiguration.swift new file mode 100644 index 0000000..0305440 --- /dev/null +++ b/Sources/MVTPostgis/MVTPostgisConfiguration.swift @@ -0,0 +1,88 @@ +import Foundation + +/// Global configuration for the MVT Postgis adapter. +public struct MVTPostgisConfiguration { + + /// The name used for database connections and the default logger (default: 'MVTPostgis'). + public let applicationName: String + + /// Timeout for opening new connections to the PostgreSQL database, in seconds (default: 5 seconds). + public let connectTimeout: TimeInterval + + /// TImeout for individual database queries, in seconds (default: 10 seconds). + /// Can be disabled by setting to `nil`. + public let queryTimeout: TimeInterval? + + /// Timeout for one tile, i.e. the time in which one tile must be finished, in seconds. + public let tileTimeout: TimeInterval + + /// The pool size, per database. Each database connection is backed by a pool of this size (default: 10). + public let poolSize: Int + + /// The maximum number of idle connections (over a 60 seconds period). + public let maxIdleConnections: Int? + + /// Controls if and where the clipping of features happens. + public let clipping: ((_ zoom: Int, _ source: PostgisSource) -> MVTClippingOption) + + /// Controls if and how much features are simplified. + public let simplification: ((_ zoom: Int, _ source: PostgisSource) -> MVTSimplificationOption) + + /// Track SQL runtimes and return them together with the vector tile. + public let trackRuntimes: Bool + + public init( + applicationName: String = "MVTPostgis", + connectTimeout: TimeInterval = 5.0, + queryTimeout: TimeInterval? = 10.0, + tileTimeout: TimeInterval = 60.0, + poolSize: Int = 10, + maxIdleConnections: Int? = nil, + clipping: @escaping ((_ zoom: Int, _ source: PostgisSource) -> MVTClippingOption) = { _,_ in .postgis }, + simplification: @escaping ((_ zoom: Int, _ source: PostgisSource) -> MVTSimplificationOption) = { _,_ in .none }, + trackRuntimes: Bool = false) + { + self.applicationName = applicationName + self.connectTimeout = connectTimeout.atLeast(1.0) + self.queryTimeout = queryTimeout?.atLeast(1.0) + self.tileTimeout = tileTimeout.atLeast(1.0) + self.poolSize = poolSize.atLeast(1) + self.maxIdleConnections = maxIdleConnections?.atLeast(0) + self.clipping = clipping + self.simplification = simplification + self.trackRuntimes = trackRuntimes + } + +} + +// MARK: - Clipping + +/// Controls if and where the clipping of features happens. +public enum MVTClippingOption { + + /// No clipping, all features are added to the vector tile as they come from + /// the database. Clipping will then be done when serializing the tile. + /// Note: Might lead to memory explosion. + case none + /// Do the clipping in Postgis with `ST_ClipByBox2D`. + case postgis + /// Do the clipping locally, before adding features to the vector tile. + case local + +} + +// MARK: - Simplification + +/// Controls if and how much features are simplified. +public enum MVTSimplificationOption { + + /// No simplification will be done. + case none + /// Do the simplification locally, before adding features to the vector tile. + case local + /// Do the simplification in Postgis with `ST_Simplify`. + case postgis(preserveCollapsed: Bool) + /// Simplification distance in meters, forwarded to `ST_Simplify`. + case meters(Double, preserveCollapsed: Bool) + +} diff --git a/Sources/MVTPostgis/MVTPostgisError.swift b/Sources/MVTPostgis/MVTPostgisError.swift new file mode 100644 index 0000000..8dec1a7 --- /dev/null +++ b/Sources/MVTPostgis/MVTPostgisError.swift @@ -0,0 +1,24 @@ +import Foundation +import PostgresNIO + +/// Possible errors thrown from the *MVTPostgis* library. +public enum MVTPostgisError: Error { + /// The request was cancelled. + case cancelled + /// The connection to the database was unexpectedly closed. + case connectionFailed + /// The source doesn't contain any layers. + case needLayers + /// The z/x/y coordinates of the tile are invalid. + case tileOutOfBounds + /// The tile timed out, i.e. not all queries return in time. + case tileTimedOut(queries: [String]) + /// This library only supports EPSG:3857 and EPSG:4326. + case unsupportedSRID + /// This library only supports EPSG:3857 and EPSG:4326. + case unsupportedSRS + /// All datasources must be of type "postgis". + case wrongDatasourceType(message: String) + /// XML parsing error, see `message` for more details. + case xmlError(message: String) +} diff --git a/Sources/MVTPostgis/Pool/PoolDistributor.swift b/Sources/MVTPostgis/Pool/PoolDistributor.swift new file mode 100644 index 0000000..af61da8 --- /dev/null +++ b/Sources/MVTPostgis/Pool/PoolDistributor.swift @@ -0,0 +1,122 @@ +import Foundation +import PostgresNIO +import PostgresConnectionPool + +actor PoolDistributor { + + private var pools: [String: PostgresConnectionPool] = [:] + + private let logger: Logger + private let configuration: MVTPostgisConfiguration + + init(configuration: MVTPostgisConfiguration, + logger: Logger) + { + self.logger = logger + self.configuration = configuration + } + + func pool(forLayer layer: PostgisLayer) async -> PostgresConnectionPool { + if let pool = pools[layer.uniqueDatabaseKey] { + return pool + } + + let postgresConfiguration = PostgresConnection.Configuration( + host: layer.datasource.host, + port: layer.datasource.port, + username: layer.datasource.user, + password: layer.datasource.password, + database: layer.datasource.databaseName, + tls: .disable) + var poolConfiguration = PoolConfiguration( + applicationName: configuration.applicationName, + postgresConfiguration: postgresConfiguration, + connectTimeout: configuration.connectTimeout, + queryTimeout: configuration.queryTimeout, + poolSize: configuration.poolSize, + maxIdleConnections: configuration.maxIdleConnections) + poolConfiguration.onOpenConnection = { connection, logger in + try await connection.query(PostgresQuery(stringLiteral: "SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY"), logger: logger) + } + + // Note: Do not make PostgresConnectionPool.init async, + // or there will be a race condition here. + let pool = PostgresConnectionPool(configuration: poolConfiguration, logger: logger) + pools[layer.uniqueDatabaseKey] = pool + return pool + } + + func connection( + forLayer layer: PostgisLayer, + batchId: Int, + callback: (PostgresConnectionWrapper) async throws -> Void) + async throws + { + let pool = await pool(forLayer: layer) + + do { + try await pool.connection(batchId: batchId, callback) + + if Task.isCancelled { + await abortBatch(batchId) + throw MVTPostgisError.cancelled + } + } + catch PoolError.cancelled { + await abortBatch(batchId) + throw MVTPostgisError.cancelled + } + catch { + await abortBatch(batchId) + + logger.debug("Layer '\(layer.id)': Failed to get a connection for batchId '\(batchId)': \(error)") + + throw error + } + } + + func abortBatch(_ batchId: Int) async { + for pool in pools.values { + await pool.abortBatch(batchId) + } + } + + /// Forcibly close all idle connections in all pools. + func closeIdleConnections() async { + for pool in pools.values { + await pool.closeIdleConnections() + } + } + + /// It's actually no problem to continue the PoolDistributor after calling shutdown(), + /// `shutdown` will just close all pools. + func shutdown() async { + for pool in pools.values { + await pool.shutdown() + } + pools.removeAll() + } + + func poolInfos(batchId: Int? = nil) async -> [PoolInfo] { + var poolInfos: [PoolInfo] = [] + for pool in pools.values { + let poolInfo = await pool.poolInfo(batchId: batchId) + poolInfos.append(poolInfo) + } + return poolInfos + } + +} + +fileprivate extension PostgisLayer { + + var uniqueDatabaseKey: String { + [ + datasource.host, + String(datasource.port), + datasource.user, + datasource.databaseName + ].joined(separator: ",") + } + +} diff --git a/Sources/MVTPostgis/Postgis/Mapnik/MapnikXMLSource.swift b/Sources/MVTPostgis/Postgis/Mapnik/MapnikXMLSource.swift new file mode 100644 index 0000000..7cc0cb7 --- /dev/null +++ b/Sources/MVTPostgis/Postgis/Mapnik/MapnikXMLSource.swift @@ -0,0 +1,199 @@ +import Foundation +import GISTools +import SwiftyXMLParser + +/// Loads the Postgis configuration from a Mapnik XML source file. +struct MapnikXMLSource { + + static func load( + from url: URL, + layerAllowlist: [String]) + throws -> PostgisSource + { + let data = try Data(contentsOf: url) + return try load(from: data, layerAllowlist: layerAllowlist) + } + + static func load( + from data: Data, + layerAllowlist: [String]) + throws -> PostgisSource + { + let xmlParser = XML.parse(data) + + var name = "" + var description = "" + var attribution = "" + var fields: [String: [String: String]] = [:] + var center = Coordinate3D.zero + var defaultZoom = 13 + var minZoom = 0 + var maxZoom = 20 + + var layers: [PostgisLayer] = [] + + xmlParser.Map.Parameters.Parameter.all?.forEach({ element in + guard let nameAttribute = element.attributes["name"], + let value = element.text ?? element.CDATA?.asUTF8EncodedString + else { return } + + switch nameAttribute { + case "name": name = value + case "description": description = value + case "attribution": attribution = value + case "minzoom": minZoom = Int(value) ?? 0 + case "maxzoom": maxZoom = Int(value) ?? 20 + case "json": fields = MapnikXMLSource.parseFields(from: value) + case "center": + let components = value.split(separator: ",").compactMap({ Double($0) }) + guard components.count == 3 else { return } + center = Coordinate3D(latitude: components[1], longitude: components[0]) + defaultZoom = Int(components[2]) + default: return + } + }) + + guard name.isNotEmpty, fields.isNotEmpty else { + throw MVTPostgisError.xmlError(message: "Missing name parameter") + } + + xmlParser.Map.Layer.all?.forEach({ element in + guard let name = element.attributes["name"], + let srs = element.attributes["srs"], + let bufferSize = element.attributes["buffer-size"]?.toInt + else { + print("Missing attributes for Layer \(element.attributes)") + return + } + + if layerAllowlist.isNotEmpty, + !layerAllowlist.contains(name) + { + return + } + + guard let datasourceElement = element.childElements.first, + datasourceElement.name == "Datasource" + else { + print("Missing Datasource for Layer \(name)") + return + } + + var user = "" + var password = "" + var host = "" + var port = 5432 + + var databaseName = "" + var geometryField = "" + var geometryTable = "" + var keyField = "" + var keyFieldAsAttribute = "" + + var extent = "" + var srid = "" + var type = "" + var maxSize = 0 + var sql = "" + + datasourceElement.childElements.forEach({ element in + guard let nameAttribute = element.attributes["name"], + let value = element.text ?? element.CDATA?.asUTF8EncodedString + else { return } + + switch nameAttribute { + case "user": user = value + case "password": password = value + case "host": host = value + case "port": port = value.toInt ?? 5432 + case "dbname": databaseName = value + case "geometry_field": geometryField = value + case "geometry_table": geometryTable = value + case "key_field": keyField = value + case "key_field_as_attribute": keyFieldAsAttribute = value + case "extent": extent = value + case "srid": srid = value + case "type": type = value + case "max_size": maxSize = value.toInt ?? 512 + case "table": sql = value + default: return + } + }) + + guard user.isNotEmpty, + password.isNotEmpty, + host.isNotEmpty, + databaseName.isNotEmpty, + type.isNotEmpty, + sql.isNotEmpty + else { + print("Missing datasource info in layer \(name)") + return + } + + let layerFields = fields[name] ?? [:] + + let datasource = PostgisDatasource( + user: user, + password: password, + host: host, + port: port, + databaseName: databaseName, + geometryField: geometryField, + geometryTable: geometryTable, + keyField: keyField, + keyFieldAsAttribute: keyFieldAsAttribute, + extent: extent, + srid: srid, + type: type, + maxSize: maxSize, + sql: sql) + let layer = PostgisLayer( + id: name, + description: "", + srs: srs, + fields: layerFields, + datasource: datasource, + bufferSize: bufferSize) + layers.append(layer) + }) + + guard layers.isNotEmpty else { + throw MVTPostgisError.xmlError(message: "Datasource without layers") + } + + let source = PostgisSource( + name: name, + description: description, + attribution: attribution, + center: center, + defaultZoom: defaultZoom, + minZoom: minZoom, + maxZoom: maxZoom, + layers: layers) + + return source + } + + // MARK: - + + private static func parseFields(from jsonString: String) -> [String: [String: String]] { + guard let data = jsonString.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let layers = json["vector_layers"] as? [[String: Any]] + else { return [:] } + + var fields: [String: [String: String]] = [:] + + for layer in layers { + guard let name = layer["id"] as? String, + let layerFields = layer["fields"] as? [String: String] + else { continue } + + fields[name] = layerFields + } + + return fields + } + +} diff --git a/Sources/MVTPostgis/Postgis/Mapnik/MapnikYMLSource.swift b/Sources/MVTPostgis/Postgis/Mapnik/MapnikYMLSource.swift new file mode 100644 index 0000000..a3b95a5 --- /dev/null +++ b/Sources/MVTPostgis/Postgis/Mapnik/MapnikYMLSource.swift @@ -0,0 +1,169 @@ +import Foundation +import GISTools +import Yams + +// MARK: MapnikYMLSource + +/// Loads the Postgis configuration from a Mapnik YML source file. +struct MapnikYMLSource: Decodable { + + static func load( + from url: URL, + layerAllowlist: [String]) + throws -> PostgisSource + { + let data = try Data(contentsOf: url) + return try load(from: data, layerAllowlist: layerAllowlist) + } + + static func load( + from data: Data, + layerAllowlist: [String]) + throws -> PostgisSource + { + let ymlSource = try YAMLDecoder().decode(MapnikYMLSource.self, from: data) + let ymlLayers = ymlSource.layers + .filter({ layer in + guard layerAllowlist.isNotEmpty else { return true } + return layerAllowlist.contains(layer.id) + }) + .map(\.asMapnikLayer) + + return PostgisSource( + name: ymlSource.name, + description: ymlSource.description, + attribution: ymlSource.attribution, + center: ymlSource.center, + defaultZoom: ymlSource.defaultZoom, + minZoom: ymlSource.minZoom, + maxZoom: ymlSource.maxZoom, + layers: ymlLayers) + } + + // MARK: - Private + + private let name: String + private let description: String + private let attribution: String + + private let _center: [Double] + private var center: Coordinate3D { + guard _center.count >= 2 else { return Coordinate3D(latitude: 0.0, longitude: 0.0) } + return Coordinate3D(latitude: _center[1], longitude: _center[0]) + } + private var defaultZoom: Int { + guard _center.count >= 3 else { return 14 } + return Int(_center[2]) + } + private let minZoom: Int + private let maxZoom: Int + + private let layers: [MapnikYMLLayer] + + enum CodingKeys: String, CodingKey { + case name, description, attribution + + case _center = "center" + case minZoom = "minzoom" + case maxZoom = "maxzoom" + + case layers = "Layer" + } + +} + +// MARK: - MapnikYMLLayer + +private struct MapnikYMLLayer: Decodable { + + let id: String + let description: String + let srs: String + let fields: [String: String] + let datasource: MapnikYMLDatasource + + private let properties: MapnikYMLProperties + var bufferSize: Int { + return properties.bufferSize + } + + var asMapnikLayer: PostgisLayer { + PostgisLayer( + id: id, + description: description, + srs: srs, + fields: fields, + datasource: datasource.asMapnikDatasource, + bufferSize: bufferSize) + } + + enum CodingKeys: String, CodingKey { + case id, description, srs, properties, fields + + case datasource = "Datasource" + } + + private struct MapnikYMLProperties: Decodable { + let bufferSize: Int + + enum CodingKeys: String, CodingKey { + case bufferSize = "buffer-size" + } + } + +} + +// MARK: - MapnikYMLDatasource + +private struct MapnikYMLDatasource: Decodable { + + let user: String + let password: String + let host: String + let port: Int + + let databaseName: String + let geometryField: String + let geometryTable: String + let keyField: String + let keyFieldAsAttribute: String + + let extent: String + let srid: String + let type: String + let maxSize: Int + let sql: String + + var asMapnikDatasource: PostgisDatasource { + PostgisDatasource( + user: user, + password: password, + host: host, + port: port, + databaseName: databaseName, + geometryField: geometryField, + geometryTable: geometryTable, + keyField: keyField, + keyFieldAsAttribute: keyFieldAsAttribute, + extent: extent, + srid: srid, + type: type, + maxSize: maxSize, + sql: sql) + } + + enum CodingKeys: String, CodingKey { + case user, password, host, port + + case databaseName = "dbname" + case geometryField = "geometry_field" + case geometryTable = "geometry_table" + case keyField = "key_field" + case keyFieldAsAttribute = "key_field_as_attribute" + + case extent, srid, type + case maxSize = "max_size" + case sql = "table" + } + +} diff --git a/Sources/MVTPostgis/Postgis/PostgisDatasource.swift b/Sources/MVTPostgis/Postgis/PostgisDatasource.swift new file mode 100644 index 0000000..dbada41 --- /dev/null +++ b/Sources/MVTPostgis/Postgis/PostgisDatasource.swift @@ -0,0 +1,23 @@ +import Foundation + +/// A datasource, part of Postgis layers. +public struct PostgisDatasource { + + public let user: String + public let password: String + public let host: String + public let port: Int + + public let databaseName: String + public let geometryField: String + public let geometryTable: String + public let keyField: String + public let keyFieldAsAttribute: String + + public let extent: String + public let srid: String + public let type: String + public let maxSize: Int + public let sql: String + +} diff --git a/Sources/MVTPostgis/Postgis/PostgisLayer.swift b/Sources/MVTPostgis/Postgis/PostgisLayer.swift new file mode 100644 index 0000000..091d904 --- /dev/null +++ b/Sources/MVTPostgis/Postgis/PostgisLayer.swift @@ -0,0 +1,16 @@ +import Foundation + +/// A Layer, part of a Postgis source. +public struct PostgisLayer { + + public let id: String + public let description: String + public let srs: String + public let fields: [String: String] + + public let datasource: PostgisDatasource + + /// The buffer around a tile in pixels. + public let bufferSize: Int + +} diff --git a/Sources/MVTPostgis/Postgis/PostgisSource.swift b/Sources/MVTPostgis/Postgis/PostgisSource.swift new file mode 100644 index 0000000..2c0d05f --- /dev/null +++ b/Sources/MVTPostgis/Postgis/PostgisSource.swift @@ -0,0 +1,43 @@ +import Foundation +import GISTools + +/// A Postgis source as parsed from a Mapnik YML or XML file. +public struct PostgisSource { + + public let name: String + public let description: String + public let attribution: String + + public let center: Coordinate3D + + public let defaultZoom: Int + public let minZoom: Int + public let maxZoom: Int + + public let layers: [PostgisLayer] + + /// Load a source from an URL, can be either Mapnik YML or XML. + public static func load( + from url: URL, + layerWhitelist: [String]?) + throws -> PostgisSource + { + let data = try Data(contentsOf: url) + return try load(from: data, layerWhitelist: layerWhitelist) + } + + /// Load a source from an URL, can be either Mapnik YML or XML. + public static func load( + from data: Data, + layerWhitelist: [String]?) + throws -> PostgisSource + { + if data.starts(with: [0x3C, 0x3F, 0x78, 0x6D, 0x6C, 0x20]) { + return try MapnikXMLSource.load(from: data, layerAllowlist: layerWhitelist ?? []) + } + else { + return try MapnikYMLSource.load(from: data, layerAllowlist: layerWhitelist ?? []) + } + } + +}