diff --git a/Makefile b/Makefile index 3661948bb..1794a6b2e 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ COVERAGE_DIR?=$(shell mktemp -d) GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) -# Uncomment to update test outputs +# Uncomment to update system test gold files # CAPTURE := "--capture" help: ## Print this help @@ -50,11 +50,11 @@ swagger-install: echo "// @version $(VERSION)" >> docs/swagger.conf azurite-start: - azurite & \ + azurite -l /tmp/aptly-azurite & \ echo $$! > ~/.azurite.pid azurite-stop: - kill `cat ~/.azurite.pid` + @kill `cat ~/.azurite.pid` swagger: swagger-install # Generate swagger docs @@ -111,7 +111,7 @@ bench: serve: prepare swagger-install ## Run development server (auto recompiling) test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3 cp debian/aptly.conf ~/.aptly.conf - sed -i /enableSwaggerEndpoint/s/false/true/ ~/.aptly.conf + sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142 dpkg: prepare swagger ## Build debian packages @@ -210,7 +210,7 @@ man: ## Create man pages clean: ## remove local build and module cache # Clean all generated and build files - find .go/ -type d ! -perm -u=w -exec chmod u+w {} \; + test ! -e .go || find .go/ -type d ! -perm -u=w -exec chmod u+w {} \; rm -rf .go/ rm -rf build/ obj-*-linux-gnu* tmp/ rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf diff --git a/cmd/config_show.go b/cmd/config_show.go index 66adc8f0a..6568604f3 100644 --- a/cmd/config_show.go +++ b/cmd/config_show.go @@ -5,18 +5,29 @@ import ( "fmt" "github.com/smira/commander" + "gopkg.in/yaml.v3" ) func aptlyConfigShow(_ *commander.Command, _ []string) error { + showYaml := context.Flags().Lookup("yaml").Value.Get().(bool) config := context.Config() - prettyJSON, err := json.MarshalIndent(config, "", " ") - if err != nil { - return fmt.Errorf("unable to dump the config file: %s", err) - } + if showYaml { + yamlData, err := yaml.Marshal(&config) + if err != nil { + return fmt.Errorf("error marshaling to YAML: %s", err) + } + + fmt.Println(string(yamlData)) + } else { + prettyJSON, err := json.MarshalIndent(config, "", " ") + if err != nil { + return fmt.Errorf("unable to dump the config file: %s", err) + } - fmt.Println(string(prettyJSON)) + fmt.Println(string(prettyJSON)) + } return nil } @@ -35,5 +46,6 @@ Example: `, } + cmd.Flag.Bool("yaml", false, "show yaml config") return cmd } diff --git a/context/context_test.go b/context/context_test.go index f83d42f20..16ecbb2b4 100644 --- a/context/context_test.go +++ b/context/context_test.go @@ -84,6 +84,6 @@ func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) { // storage never exists. c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") }, FatalErrorPanicMatches, - &FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: EOF", + &FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: invalid yaml (EOF) or json (EOF)", os.Getenv("HOME"))}) } diff --git a/debian/aptly.conf b/debian/aptly.conf index 6eeed77ec..38a8a56a8 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -1,370 +1,343 @@ -// vim: : filetype=json -// json configuration file with comments -// validate with: sed '/\/\//d' aptly.conf | json_pp -{ +# Aptly Configuration File +########################### +# vim: : filetype=yaml -// Aptly Configuration File -//////////////////////////// +# aptly 1.6.0 supports yaml configuraiton files with inline documentation and examples. +# Legacy json config files are still supported, and may be converted to yaml with `aptly config show -yaml` - // Aptly storage directory for: - // - downloaded packages (`rootDir`/pool) - // - database (`rootDir`/db) - // - published repositories (`rootDir`/public) - "rootDir": "~/.aptly", +# Root directory for: +# - downloaded packages (`rootDir`/pool) +# - database (`rootDir`/db) +# - published repositories (`rootDir`/public) +root_dir: ~/.aptly - // Number of attempts to open database if it's locked by other instance - // * -1 (no retry) - "databaseOpenAttempts": -1, +# Log Level +# * debug +# * info +# * warning +# * error +log_level: info - // Log Level - // * debug - // * info - // * warning - // * error - "logLevel": "info", +# Log Format +# * default (text) +# * json +log_format: default - // Log Format - // * default (text) - // * json - "logFormat": "default", +# Number of attempts to open database if it's locked by other instance +# * -1 (no retry) +database_open_attempts: -1 - // Default Architectures - // empty array defaults to all available architectures - "architectures": [], +# Default Architectures +# empty list defaults to all available architectures +architectures: +# - amd64 - // Follow contents of `Suggests:` field when processing dependencies for the package - "dependencyFollowSuggests": false, +# OBSOLETE +# in aptly up to version 1.0.0, package files were stored in internal package pool +# with MD5-dervied path, since 1.1.0 package pool layout was changed; +# if option is enabled, aptly stops checking for legacy paths; +# by default option is enabled for new aptly installations and disabled when +# upgrading from older versions +skip_legacy_pool: true - // Follow contents of `Recommends:` field when processing dependencies for the package - "dependencyFollowRecommends": false, - // When dependency looks like `package-a | package-b`, follow both variants always - "dependencyFollowAllVariants": false, +# Dependency following +####################### - // Follow dependency from binary package to source package - "dependencyFollowSource": false, +# Follow contents of `Suggests:` field when processing dependencies for the package +dep_follow_suggests: false - // Log additional details while resolving dependencies (useful for debugging) - "dependencyVerboseResolve": false, +# Follow contents of `Recommends:` field when processing dependencies for the package +dep_follow_recommends: false - // Specifies paramaters for short PPA url expansion - // empty defaults to output of `lsb_release` command - "ppaDistributorID": "ubuntu", +# When dependency looks like `package-a | package-b`, follow both variants always +dep_follow_allvariants: false - // Codename for short PPA url expansion - "ppaCodename": "", +# Follow dependency from binary package to source package +dep_follow_source: false - // OBSOLETE - // in aptly up to version 1.0.0, package files were stored in internal package pool - // with MD5-dervied path, since 1.1.0 package pool layout was changed; - // if option is enabled, aptly stops checking for legacy paths; - // by default option is enabled for new aptly installations and disabled when - // upgrading from older versions - "skipLegacyPool": true, +# Log additional details while resolving dependencies (useful for debugging) +dep_verbose_resolve: false -// Aptly Server -//////////////// +# PPA +###### - // Serve published repos as well as API - "serveInAPIMode": false, +# Specify paramaters for short PPA url expansion +# empty defaults to output of `lsb_release` command +ppa_distributor_id: ubuntu - // Enable metrics for Prometheus client - "enableMetricsEndpoint": false, +# Codename for short PPA url expansion +ppa_codename: "" - // Enable API documentation on /docs - "enableSwaggerEndpoint": false, - // OBSOLETE: use via url param ?_async=true - "AsyncAPI": false, +# Aptly Server +############### +# Serve published repos as well as API +serve_in_api_mode: false -// Database -//////////// +# Enable metrics for Prometheus client +enable_metrics_endpoint: false + +# Enable API documentation on /docs +enable_swagger_endpoint: false + +# OBSOLETE: use via url param ?_async=true +async_api: false + + +# Database +########### + +# Database backend +# Type must be one of: +# * leveldb (default) +# * etcd +database_backend: + type: leveldb + # Path to leveldb files + # empty dbPath defaults to `rootDir`/db + db_path: "" + + # type: etcd + # # URL to db server + # url: "127.0.0.1:2379" + + +# Mirroring +############ + +# Downloader +# * "default" +# * "grab" (more robust) +downloader: default + +# Number of parallel download threads to use when downloading packages +download_concurrency: 4 + +# Limit in kbytes/sec on download speed while mirroring remote repositories +download_limit: 0 + +# Number of retries for download attempts +download_retries: 0 + +# Download source packages per default +download_sourcepackages: false + + +# Signing +########## + +# GPG Provider +# * "internal" (Go internal implementation) +# * "gpg" (External `gpg` utility) +gpg_provider: gpg + +# Disable signing of published repositories +gpg_disable_sign: false + +# Disable signature verification of remote repositories +gpg_disable_verify: false + + +# Publishing +############# + +# Do not publish Contents files +skip_contents_publishing: false + +# Do not create bz2 files +skip_bz2_publishing: false + + +# Storage +########## + +# Filesystem publishing endpoints +# +# aptly defaults to publish to a single publish directory under `rootDir`/public. For +# a more advanced publishing strategy, you can define one or more filesystem endpoints in the +# `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name +# and the following associated settings. +# +# In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` +# with `endpoint-name` as the name given in the aptly configuration file. For example: +# +# `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` +# +filesystem_publish_endpoints: + # # Endpoint Name + # test1: + # # Directory for publishing + # root_dir: /opt/srv/aptly_public + # # File Link Method for linking files from the internal pool to the published directory + # # * hardlink + # # * symlink + # # * copy + # link_method: hardlink + # # File Copare Method for comparing existing links from the internal pool to the published directory + # # Only used when "linkMethod" is set to "copy" + # # * md5 (default: compare md5 sum) + # # * size (compare file size) + # verify_method: md5 + +# S3 Endpoint Support +# +# cloud storage). First, publishing +# endpoints should be described in aptly configuration file. Each endpoint has name +# and associated settings. +# +# In order to publish to S3, specify endpoint as `s3:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot wheezy-main s3:test:` +# +s3_publish_endpoints: + # # Endpoint Name + # test: + # # Amazon region for S3 bucket + # region: us-east-1 + # # Bucket name + # bucket: test-bucket + # # Prefix (optional) + # # publishing under specified prefix in the bucket, defaults to + # # no prefix (bucket root) + # prefix: "" + # # Default ACLs (optional) + # # assign ACL to published files: + # # * private (default, for use with apt S3 transport) + # # * public-read (public repository) + # # * none (don't set ACL) + # acl: private + # # Credentials (optional) + # # Amazon credentials to access S3 bucket. If not supplied, environment variables + # # `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` are used + # access_key_id: "" + # secret_access_key: "" + # session_token: "" + # # Endpoint (optional) + # # When using S3-compatible cloud storage, specify hostname of service endpoint here, + # # region is ignored if endpoint is set (set region to some human-readable name) + # # (should be left blank for real Amazon S3) + # endpoint: "" + # # Storage Class (optional) + # # Amazon S3 storage class, defaults to `STANDARD`. Other values + # # available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + # storage_class: STANDARD + # # Encryption Method (optional) + # # Server-side encryption method, defaults to none. Currently + # # the only available encryption method is `AES256` + # encryption_method: none + # # Plus Workaround (optional) + # # Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + # # creating two copies of package files with `+` in filename: one original + # # and another one with spaces instead of plus signs + # # With `plusWorkaround` enabled, package files with plus sign + # # would be stored twice. aptly might not cleanup files with spaces when published + # # repository is dropped or updated (switched) to new version of repository (snapshot) + # plus_workaround: false + # # Disable MultiDel (optional) + # # For S3-compatible cloud storages which do not support `MultiDel` S3 API, + # # enable this setting (file deletion would be slower with this setting enabled) + # disable_multidel: false + # # Force Signature v2 (optional) + # # Disable Signature V4 support, useful with non-AWS S3-compatible object stores + # # which do not support SigV4, shouldn't be enabled for AWS + # force_sigv2: false + # # Force VirtualHosted Style (optional) + # # Disable path style visit, useful with non-AWS S3-compatible object stores + # # which only support virtual hosted style + # force_virtualhosted_style: false + # # Debug (optional) + # # Enables detailed request/response dump for each S3 operation + # debug: false + +# Swift Endpoint Support +# +# aptly can publish a repository directly to OpenStack Swift. +# Each endpoint has name and associated settings. +# +# In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot jessie-main swift:test:` +# +swift_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: taylor1 + # # Prefix (optional) + # # Publish under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials (optional) + # # OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + # username: "" + # password: "" + # # Domain (optional) + # # OpenStack domain + # domain: "" + # domain_id: "" + # # Tenant (optional) + # # OpenStack tenant (in order to use v2 authentication) + # tenant: "" + # tenant_id: "" + # tenant_domain: "" + # tenant_domain_id: "" + # # Auth URL (optional) + # # Full url of Keystone server (including port, and version). + # # Example `http://identity.example.com:5000/v2.0` + # auth_url: "" + +# Azure Endpoint Support +# +# aptly can be configured to publish repositories directly to Microsoft Azure Blob +# Storage. First, publishing endpoints should be described in the aptly +# configuration file. Each endpoint has its name and associated settings. +azure_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: container1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" + +# Package Pool +# +# Location for storing downloaded packages +# Type must be one of: +# * local +# * azure +packagepool_storage: + # Local Pool + type: local + # Local Pool Path + # empty path defaults to `rootDir`/pool + path: + + # # Azure Azure Blob Storage Pool + # type: azure + # # Container Name + # container: pool1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" - // Database backend - // Type must be one of: - // * leveldb (default) - // * etcd - "databaseBackend": { - // LevelDB - "type": "leveldb", - // Path to leveldb files - // empty dbPath defaults to `rootDir`/db - "dbPath": "" - - // // etcd - // "type": "etcd", - // // URL to db server - // "url": "127.0.0.1:2379" - }, - - -// Mirroring -///////////// - - // Downloader - // * "default" - // * "grab" (more robust) - "downloader": "default", - - // Number of parallel download threads to use when downloading packages - "downloadConcurrency": 4, - - // Limit in kbytes/sec on download speed while mirroring remote repositories - "downloadSpeedLimit": 0, - - // Number of retries for download attempts - "downloadRetries": 0, - - // Download source packages per default - "downloadSourcePackages": false, - - -// Signing -/////////// - - // GPG Provider - // * "internal" (Go internal implementation) - // * "gpg" (External `gpg` utility) - "gpgProvider": "gpg", - - // Disable signing of published repositories - "gpgDisableSign": false, - - // Disable signature verification of remote repositories - "gpgDisableVerify": false, - - -// Publishing -////////////// - - // Do not publish Contents files - "skipContentsPublishing": false, - - // Do not create bz2 files - "skipBz2Publishing": false, - - -// Storage -/////////// - - // Filesystem publishing endpoints - // - // aptly defaults to publish to a single publish directory under `rootDir`/public. For - // a more advanced publishing strategy, you can define one or more filesystem endpoints in the - // `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name - // and the following associated settings. - // - // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` - // with `endpoint-name` as the name given in the aptly configuration file. For example: - // - // `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` - // - "FileSystemPublishEndpoints": { - // // Endpoint Name - // "test1": { - // // Directory for publishing - // "rootDir": "/opt/srv/aptly_public", - - // // File Link Method for linking files from the internal pool to the published directory - // // * hardlink - // // * symlink - // // * copy - // "linkMethod": "hardlink", - - // // File Copare Method for comparing existing links from the internal pool to the published directory - // // Only used when "linkMethod" is set to "copy" - // // * md5 (default: compare md5 sum) - // // * size (compare file size) - // "verifyMethod": "md5" - // } - }, - - // S3 Endpoint Support - // - // cloud storage). First, publishing - // endpoints should be described in aptly configuration file. Each endpoint has name - // and associated settings. - // - // In order to publish to S3, specify endpoint as `s3:endpoint-name:` before - // publishing prefix on the command line, e.g.: - // - // `aptly publish snapshot wheezy-main s3:test:` - // - "S3PublishEndpoints": { - // // Endpoint Name - // "test": { - - // // Amazon region for S3 bucket - // "region": "us-east-1", - - // // Bucket name - // "bucket": "test-bucket", - - // // Endpoint (optional) - // // When using S3-compatible cloud storage, specify hostname of service endpoint here, - // // region is ignored if endpoint is set (set region to some human-readable name) - // // (should be left blank for real Amazon S3) - // "endpoint": "", - - // // Prefix (optional) - // // publishing under specified prefix in the bucket, defaults to - // // no prefix (bucket root) - // "prefix": "", - - // // Default ACLs (optional) - // // assign ACL to published files (one of the canned ACLs in Amazon - // // terminology). Useful values: `private` (default), `public-read` (public - // // repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using - // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), - // // for private repositories special apt S3 transport is required. - // "acl": "private", - - // // Credentials (optional) - // // Amazon credentials to access S3 bucket. If not supplied, - // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` - // // are used. - // "awsAccessKeyID": "", - // "awsSecretAccessKey": "", - - // // Storage Class (optional) - // // Amazon S3 storage class, defaults to `STANDARD`. Other values - // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) - // "storageClass": "STANDARD", - - // // Encryption Method (optional) - // // Server-side encryption method, defaults to none. Currently - // // the only available encryption method is `AES256` - // "encryptionMethod": "none", - - // // Plus Workaround (optional) - // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by - // // creating two copies of package files with `+` in filename: one original - // // and another one with spaces instead of plus signs - // // With `plusWorkaround` enabled, package files with plus sign - // // would be stored twice. aptly might not cleanup files with spaces when published - // // repository is dropped or updated (switched) to new version of repository (snapshot) - // "plusWorkaround": false, - - // // Disable MultiDel (optional) - // // For S3-compatible cloud storages which do not support `MultiDel` S3 API, - // // enable this setting (file deletion would be slower with this setting enabled) - // "disableMultiDel": false, - - // // ForceSig2 (optional) - // // Disable Signature V4 support, useful with non-AWS S3-compatible object stores - // // which do not support SigV4, shouldn't be enabled for AWS - // "forceSigV2": false, - - // // ForceVirtualHostedStyle (optional) - // // Disable path style visit, useful with non-AWS S3-compatible object stores - // // which only support virtual hosted style - // "forceVirtualHostedStyle": false, - - // // Debug (optional) - // // Enables detailed request/response dump for each S3 operation - // "debug": false - // } - }, - - // Swift Endpoint Support - // - // aptly could be configured to publish repository directly to OpenStack Swift. First, - // publishing endpoints should be described in aptly configuration file. Each endpoint - // has name and associated settings. - // - // In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before - // publishing prefix on the command line, e.g.: - // - // `aptly publish snapshot jessie-main swift:test:` - // - "SwiftPublishEndpoints": { - // Endpoint Name - // "test": { - - // // Container Name - // "container": "taylor1", - - // // Prefix (optional) - // // Publish under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials (optional) - // // OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used - // "osname": "", - // "password": "", - - // // Tenant (optional) - // // OpenStack tenant name and id (in order to use v2 authentication) - // "tenant": "", - // "tenantid": "", - - // // Auth URL (optional) - // // Full url of Keystone server (including port, and version). - // // Example `http://identity.example.com:5000/v2.0` - // "authurl": "" - // } - }, - - // Azure Endpoint Support - // - // aptly can be configured to publish repositories directly to Microsoft Azure Blob - // Storage. First, publishing endpoints should be described in the aptly - // configuration file. Each endpoint has its name and associated settings. - "AzurePublishEndpoints": { - // // Endpoint Name - // "test": { - - // // Container Name - // "container": "container1", - - // // Prefix (optional) - // // Publishing under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials - // // Azure storage account access key to access blob storage - // "accountName": "", - // "accountKey": "", - - // // Endpoint URL - // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://.blob.core.windows.net" - // "endpoint": "" - // } - }, - - // Package Pool - // Location for storing downloaded packages - // Type must be one of: - // * local - // * azure - "packagePoolStorage": { - // Local Pool - "type": "local", - // Local Pool Path - // empty path defaults to `rootDir`/pool - "path": "" - - // // Azure Azure Blob Storage Pool - // "type": "azure", - // "container": "pool1", - - // // Prefix (optional) - // // Publishing under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials - // // Azure storage account access key to access blob storage - // "accountName": "", - // "accountKey": "", - - // // Endpoint URL - // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://.blob.core.windows.net" - // "endpoint": "" - } - -// End of config -} diff --git a/debian/control b/debian/control index f7ed7f09a..101dbcf12 100644 --- a/debian/control +++ b/debian/control @@ -77,6 +77,7 @@ Build-Depends: bash-completion, golang-go.uber-multierr-dev, golang-go.uber-zap-dev, golang-etcd-server-dev (>= 3.5.15-7), + golang-gopkg-yaml.v3-dev, git Standards-Version: 4.7.0 Homepage: https://www.aptly.info diff --git a/go.mod b/go.mod index 91f7e5657..35786ebe5 100644 --- a/go.mod +++ b/go.mod @@ -113,7 +113,6 @@ require ( google.golang.org/grpc v1.64.1 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -129,4 +128,5 @@ require ( github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.3 go.etcd.io/etcd/client/v3 v3.5.15 + gopkg.in/yaml.v3 v3.0.1 ) diff --git a/man/aptly.1 b/man/aptly.1 index d49d2f6f0..523440f8f 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -1,7 +1,7 @@ .\" generated with Ronn/v0.7.3 .\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "APTLY" "1" "October 2024" "" "" +.TH "APTLY" "1" "December 2024" "" "" . .SH "NAME" \fBaptly\fR \- Debian repository management tool @@ -28,330 +28,390 @@ aptly\(cqs goal is to establish repeatability and controlled changes in a packag aptly looks for configuration file first in \fB~/\.aptly\.conf\fR then in \fB/usr/local/etc/aptly\.conf\fR and \fB/etc/aptly\.conf\fR\. If no config file found (or they are not readable), a new one is created in the home directory\. If \fB\-config=\fR flag is specified, aptly would use config file at specified location\. Also aptly needs root directory for database, package and published repository storage\. If not specified, directory defaults to \fB~/\.aptly/\fR, it will be created if missing\. . .P -Configuration file is stored in JSON format (default values shown below): +With aptly version 1\.6\.0, yaml configuration with inline documentation is supported and recommended (see \fBdebian/aptly\.conf\fR)\. +. +.P +The legacy json configuration is still supported: . .IP "" 4 . .nf +// vim: : filetype=json +// json configuration file with comments +// validate with: sed \(cq/\e/\e//d\(cq aptly\.conf | json_pp { - "rootDir": "$HOME/\.aptly", - "databaseBackend": { - "type": "", - "url": "" - }, - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "downloadRetries": 0, - "downloader": "default", - "databaseOpenAttempts": 10, + +// Aptly Configuration File +//////////////////////////// + + // Root directory for: + // \- downloaded packages (`rootDir`/pool) + // \- database (`rootDir`/db) + // \- published repositories (`rootDir`/public) + "rootDir": "~/\.aptly", + + // Number of attempts to open database if it\(cqs locked by other instance + // * \-1 (no retry) + "databaseOpenAttempts": \-1, + + // Log Level + // * debug + // * info + // * warning + // * error + "logLevel": "info", + + // Log Format + // * default (text) + // * json + "logFormat": "default", + + // Default Architectures + // empty array defaults to all available architectures "architectures": [], + + // Follow contents of `Suggests:` field when processing dependencies for the package "dependencyFollowSuggests": false, + + // Follow contents of `Recommends:` field when processing dependencies for the package "dependencyFollowRecommends": false, + + // When dependency looks like `package\-a | package\-b`, follow both variants always "dependencyFollowAllVariants": false, + + // Follow dependency from binary package to source package "dependencyFollowSource": false, + + // Log additional details while resolving dependencies (useful for debugging) "dependencyVerboseResolve": false, - "gpgDisableSign": false, - "gpgDisableVerify": false, - "gpgProvider": "gpg", - "downloadSourcePackages": false, - "packagePoolStorage": { - "path": "$ROOTDIR/pool", - "azure": { - "accountName": "", - "accountKey": "", - "container": "repo", - "prefix": "", - "endpoint": "" - } - }, - "skipLegacyPool": true, + + // Specifies paramaters for short PPA url expansion + // empty defaults to output of `lsb_release` command "ppaDistributorID": "ubuntu", + + // Codename for short PPA url expansion "ppaCodename": "", + + // OBSOLETE + // in aptly up to version 1\.0\.0, package files were stored in internal package pool + // with MD5\-dervied path, since 1\.1\.0 package pool layout was changed; + // if option is enabled, aptly stops checking for legacy paths; + // by default option is enabled for new aptly installations and disabled when + // upgrading from older versions + "skipLegacyPool": true, + + +// Aptly Server +//////////////// + + // Serve published repos as well as API + "serveInAPIMode": false, + + // Enable metrics for Prometheus client + "enableMetricsEndpoint": false, + + // Enable API documentation on /docs + "enableSwaggerEndpoint": false, + + // OBSOLETE: use via url param ?_async=true + "AsyncAPI": false, + + +// Database +//////////// + + // Database backend + // Type must be one of: + // * leveldb (default) + // * etcd + "databaseBackend": { + // LevelDB + "type": "leveldb", + // Path to leveldb files + // empty dbPath defaults to `rootDir`/db + "dbPath": "" + + // // etcd + // "type": "etcd", + // // URL to db server + // "url": "127\.0\.0\.1:2379" + }, + + +// Mirroring +///////////// + + // Downloader + // * "default" + // * "grab" (more robust) + "downloader": "default", + + // Number of parallel download threads to use when downloading packages + "downloadConcurrency": 4, + + // Limit in kbytes/sec on download speed while mirroring remote repositories + "downloadSpeedLimit": 0, + + // Number of retries for download attempts + "downloadRetries": 0, + + // Download source packages per default + "downloadSourcePackages": false, + + +// Signing +/////////// + + // GPG Provider + // * "internal" (Go internal implementation) + // * "gpg" (External `gpg` utility) + "gpgProvider": "gpg", + + // Disable signing of published repositories + "gpgDisableSign": false, + + // Disable signature verification of remote repositories + "gpgDisableVerify": false, + + +// Publishing +////////////// + + // Do not publish Contents files "skipContentsPublishing": false, + + // Do not create bz2 files + "skipBz2Publishing": false, + + +// Storage +/////////// + + // Filesystem publishing endpoints + // + // aptly defaults to publish to a single publish directory under `rootDir`/public\. For + // a more advanced publishing strategy, you can define one or more filesystem endpoints in the + // `FileSystemPublishEndpoints` list of the aptly configuration file\. Each endpoint has a name + // and the following associated settings\. + // + // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint\-name` + // with `endpoint\-name` as the name given in the aptly configuration file\. For example: + // + // `aptly publish snapshot wheezy\-main filesystem:test1:wheezy/daily` + // "FileSystemPublishEndpoints": { - "test1": { - "rootDir": "/opt/srv1/aptly_public", - "linkMethod": "symlink" - }, - "test2": { - "rootDir": "/opt/srv2/aptly_public", - "linkMethod": "copy", - "verifyMethod": "md5" - }, - "test3": { - "rootDir": "/opt/srv3/aptly_public", - "linkMethod": "hardlink" - } + // // Endpoint Name + // "test1": { + // // Directory for publishing + // "rootDir": "/opt/srv/aptly_public", + + // // File Link Method for linking files from the internal pool to the published directory + // // * hardlink + // // * symlink + // // * copy + // "linkMethod": "hardlink", + + // // File Copare Method for comparing existing links from the internal pool to the published directory + // // Only used when "linkMethod" is set to "copy" + // // * md5 (default: compare md5 sum) + // // * size (compare file size) + // "verifyMethod": "md5" + // } }, + + // S3 Endpoint Support + // + // cloud storage)\. First, publishing + // endpoints should be described in aptly configuration file\. Each endpoint has name + // and associated settings\. + // + // In order to publish to S3, specify endpoint as `s3:endpoint\-name:` before + // publishing prefix on the command line, e\.g\.: + // + // `aptly publish snapshot wheezy\-main s3:test:` + // "S3PublishEndpoints": { - "test": { - "region": "us\-east\-1", - "bucket": "repo", - "endpoint": "", - "awsAccessKeyID": "", - "awsSecretAccessKey": "", - "prefix": "", - "acl": "public\-read", - "storageClass": "", - "encryptionMethod": "", - "plusWorkaround": false, - "disableMultiDel": false, - "forceSigV2": false, - "forceVirtualHostedStyle": true, - "debug": false - } + // // Endpoint Name + // "test": { + + // // Amazon region for S3 bucket + // "region": "us\-east\-1", + + // // Bucket name + // "bucket": "test\-bucket", + + // // Endpoint (optional) + // // When using S3\-compatible cloud storage, specify hostname of service endpoint here, + // // region is ignored if endpoint is set (set region to some human\-readable name) + // // (should be left blank for real Amazon S3) + // "endpoint": "", + + // // Prefix (optional) + // // publishing under specified prefix in the bucket, defaults to + // // no prefix (bucket root) + // "prefix": "", + + // // Default ACLs (optional) + // // assign ACL to published files (one of the canned ACLs in Amazon + // // terminology)\. Useful values: `private` (default), `public\-read` (public + // // repository) or `none` (don\(cqt set ACL)\. Public repositories could be consumed by `apt` using + // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), + // // for private repositories special apt S3 transport is required\. + // "acl": "private", + + // // Credentials (optional) + // // Amazon credentials to access S3 bucket\. If not supplied, + // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + // // are used\. + // "awsAccessKeyID": "", + // "awsSecretAccessKey": "", + + // // Storage Class (optional) + // // Amazon S3 storage class, defaults to `STANDARD`\. Other values + // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + // "storageClass": "STANDARD", + + // // Encryption Method (optional) + // // Server\-side encryption method, defaults to none\. Currently + // // the only available encryption method is `AES256` + // "encryptionMethod": "none", + + // // Plus Workaround (optional) + // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + // // creating two copies of package files with `+` in filename: one original + // // and another one with spaces instead of plus signs + // // With `plusWorkaround` enabled, package files with plus sign + // // would be stored twice\. aptly might not cleanup files with spaces when published + // // repository is dropped or updated (switched) to new version of repository (snapshot) + // "plusWorkaround": false, + + // // Disable MultiDel (optional) + // // For S3\-compatible cloud storages which do not support `MultiDel` S3 API, + // // enable this setting (file deletion would be slower with this setting enabled) + // "disableMultiDel": false, + + // // ForceSig2 (optional) + // // Disable Signature V4 support, useful with non\-AWS S3\-compatible object stores + // // which do not support SigV4, shouldn\(cqt be enabled for AWS + // "forceSigV2": false, + + // // ForceVirtualHostedStyle (optional) + // // Disable path style visit, useful with non\-AWS S3\-compatible object stores + // // which only support virtual hosted style + // "forceVirtualHostedStyle": false, + + // // Debug (optional) + // // Enables detailed request/response dump for each S3 operation + // "debug": false + // } }, + + // Swift Endpoint Support + // + // aptly could be configured to publish repository directly to OpenStack Swift\. First, + // publishing endpoints should be described in aptly configuration file\. Each endpoint + // has name and associated settings\. + // + // In order to publish to Swift, specify endpoint as `swift:endpoint\-name:` before + // publishing prefix on the command line, e\.g\.: + // + // `aptly publish snapshot jessie\-main swift:test:` + // "SwiftPublishEndpoints": { - "test": { - "container": "repo", - "osname": "", - "password": "", - "prefix": "", - "authurl": "", - "tenant": "", - "tenantid": "" - } + // Endpoint Name + // "test": { + + // // Container Name + // "container": "taylor1", + + // // Prefix (optional) + // // Publish under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials (optional) + // // OpenStack credentials to access Keystone\. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + // "osname": "", + // "password": "", + + // // Tenant (optional) + // // OpenStack tenant name and id (in order to use v2 authentication) + // "tenant": "", + // "tenantid": "", + + // // Auth URL (optional) + // // Full url of Keystone server (including port, and version)\. + // // Example `http://identity\.example\.com:5000/v2\.0` + // "authurl": "" + // } }, + + // Azure Endpoint Support + // + // aptly can be configured to publish repositories directly to Microsoft Azure Blob + // Storage\. First, publishing endpoints should be described in the aptly + // configuration file\. Each endpoint has its name and associated settings\. "AzurePublishEndpoints": { - "test": { - "accountName": "", - "accountKey": "", - "container": "repo", - "prefix": "", - "endpoint": "blob\.core\.windows\.net" - } + // // Endpoint Name + // "test": { + + // // Container Name + // "container": "container1", + + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + + // // Endpoint URL + // // See: Azure documentation https://docs\.microsoft\.com/en\-us/azure/storage/common/storage\-configure\-connection\-string + // // defaults to "https://\.blob\.core\.windows\.net" + // "endpoint": "" + // } + }, + + // Package Pool + // Location for storing downloaded packages + // Type must be one of: + // * local + // * azure + "packagePoolStorage": { + // Local Pool + "type": "local", + // Local Pool Path + // empty path defaults to `rootDir`/pool + "path": "" + + // // Azure Azure Blob Storage Pool + // "type": "azure", + // "container": "pool1", + + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + + // // Endpoint URL + // // See: Azure documentation https://docs\.microsoft\.com/en\-us/azure/storage/common/storage\-configure\-connection\-string + // // defaults to "https://\.blob\.core\.windows\.net" + // "endpoint": "" } + +// End of config } . .fi . .IP "" 0 . -.P -Options: -. -.IP "\[ci]" 4 -\fBrootDir\fR: is root of directory storage to store database (\fBrootDir\fR/db), the default for downloaded packages (\fBrootDir\fR/pool) and the default for published repositories (\fBrootDir\fR/public) -. -.IP "\[ci]" 4 -\fBdatabaseBackend\fR: the database config; if this config is empty, use levledb backend by default -. -.IP "\[ci]" 4 -\fBdownloadConcurrency\fR: is a number of parallel download threads to use when downloading packages -. -.IP "\[ci]" 4 -\fBdownloadSpeedLimit\fR: limit in kbytes/sec on download speed while mirroring remote repositories -. -.IP "\[ci]" 4 -\fBdownloadRetries\fR: number of retries for download attempts -. -.IP "\[ci]" 4 -\fBdatabaseOpenAttempts\fR: number of attempts to open DB if it\(cqs locked by other instance; could be overridden with option \fB\-db\-open\-attempts\fR -. -.IP "\[ci]" 4 -\fBarchitectures\fR: is a list of architectures to process; if left empty defaults to all available architectures; could be overridden with option \fB\-architectures\fR -. -.IP "\[ci]" 4 -\fBdependencyFollowSuggests\fR: follow contents of \fBSuggests:\fR field when processing dependencies for the package -. -.IP "\[ci]" 4 -\fBdependencyFollowRecommends\fR: follow contents of \fBRecommends:\fR field when processing dependencies for the package -. -.IP "\[ci]" 4 -\fBdependencyFollowAllVariants\fR: when dependency looks like \fBpackage\-a | package\-b\fR, follow both variants always -. -.IP "\[ci]" 4 -\fBdependencyFollowSource\fR: follow dependency from binary package to source package -. -.IP "\[ci]" 4 -\fBdependencyVerboseResolve\fR: print additional details while resolving dependencies (useful for debugging) -. -.IP "\[ci]" 4 -\fBgpgDisableSign\fR: don\(cqt sign published repositories with gpg(1), also can be disabled on per\-repo basis using \fB\-skip\-signing\fR flag when publishing -. -.IP "\[ci]" 4 -\fBgpgDisableVerify\fR: don\(cqt verify remote mirrors with gpg(1), also can be disabled on per\-mirror basis using \fB\-ignore\-signatures\fR flag when creating and updating mirrors -. -.IP "\[ci]" 4 -\fBgpgProvider\fR: implementation of PGP signing/validation \- \fBgpg\fR for external \fBgpg\fR utility or \fBinternal\fR to use Go internal implementation; \fBgpg1\fR might be used to force use of GnuPG 1\.x, \fBgpg2\fR enables GnuPG 2\.x only; default is to use GnuPG 1\.x if available and GnuPG 2\.x otherwise -. -.IP "\[ci]" 4 -\fBdownloadSourcePackages\fR: if enabled, all mirrors created would have flag set to download source packages; this setting could be controlled on per\-mirror basis with \fB\-with\-sources\fR flag -. -.IP "\[ci]" 4 -\fBpackagePoolStorage\fR: configures the location to store downloaded packages (defaults to the path \fB$ROOTDIR/pool\fR), by setting the value of the \fBtype\fR: -. -.IP "\[ci]" 4 -\fBpath\fR: store the packages in the given path -. -.IP "\[ci]" 4 -\fBazure\fR: store the packages in the given Azure Blob Storage container (see the section on Azure publishing below for information on the configuration) -. -.IP "" 0 - -. -.IP "\[ci]" 4 -\fBskipLegacyPool\fR: in aptly up to version 1\.0\.0, package files were stored in internal package pool with MD5\-dervied path, since 1\.1\.0 package pool layout was changed; if option is enabled, aptly stops checking for legacy paths; by default option is enabled for new aptly installations and disabled when upgrading from older versions -. -.IP "\[ci]" 4 -\fBppaDistributorID\fR, \fBppaCodename\fR: specifies paramaters for short PPA url expansion, if left blank they default to output of \fBlsb_release\fR command -. -.IP "\[ci]" 4 -\fBFileSystemPublishEndpoints\fR: configuration of local filesystem publishing endpoints (see below) -. -.IP "\[ci]" 4 -\fBS3PublishEndpoints\fR: configuration of Amazon S3 publishing endpoints (see below) -. -.IP "\[ci]" 4 -\fBSwiftPublishEndpoints\fR: configuration of OpenStack Swift publishing endpoints (see below) -. -.IP "\[ci]" 4 -\fBAzurePublishEndpoints\fR: configuration of Azure publishing endpoints (see below) -. -.IP "" 0 -. -.SH "CUSTOM PACKAGE POOLS" -aptly defaults to storing downloaded packages at \fBrootDir/\fRpool\. In order to change this, you can set the \fBtype\fR key within \fBpackagePoolStorage\fR to one of two values: -. -.IP "\[ci]" 4 -\fBlocal\fR: Store the package pool locally (the default)\. In order to change the path, additionally set the \fBpath\fR key within \fBpackagePoolStorage\fR to the desired location\. -. -.IP "\[ci]" 4 -\fBazure\fR: Store the package pool in an Azure Blob Storage container\. Any keys in the below section on Azure publishing may be set on the \fBpackagePoolStorage\fR object in order to configure the Azure connection\. -. -.IP "" 0 -. -.SH "FILESYSTEM PUBLISHING ENDPOINTS" -aptly defaults to publish to a single publish directory under \fBrootDir\fR/public\. For a more advanced publishing strategy, you can define one or more filesystem endpoints in the \fBFileSystemPublishEndpoints\fR list of the aptly configuration file\. Each endpoint has a name and the following associated settings: -. -.TP -\fBrootDir\fR -The publish directory, e\.g\., \fB/opt/srv/aptly_public\fR\. -. -.TP -\fBlinkMethod\fR -This is one of \fBhardlink\fR, \fBsymlink\fR or \fBcopy\fR\. It specifies how aptly links the files from the internal pool to the published directory\. If not specified, empty or wrong, this defaults to \fBhardlink\fR\. -. -.TP -\fBverifyMethod\fR -This is used only when setting the \fBlinkMethod\fR to \fBcopy\fR\. Possible values are \fBmd5\fR and \fBsize\fR\. It specifies how aptly compares existing links from the internal pool to the published directory\. The \fBsize\fR method compares only the file sizes, whereas the \fBmd5\fR method calculates the md5 checksum of the found file and compares it to the desired one\. If not specified, empty or wrong, this defaults to \fBmd5\fR\. -. -.P -In order to publish to such an endpoint, specify the endpoint as \fBfilesystem:endpoint\-name\fR with \fBendpoint\-name\fR as the name given in the aptly configuration file\. For example: -. -.P -\fBaptly publish snapshot wheezy\-main filesystem:test1:wheezy/daily\fR -. -.SH "S3 PUBLISHING ENDPOINTS" -aptly could be configured to publish repository directly to Amazon S3 (or S3\-compatible cloud storage)\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings: -. -.TP -\fBregion\fR -Amazon region for S3 bucket (e\.g\. \fBus\-east\-1\fR) -. -.TP -\fBbucket\fR -bucket name -. -.TP -\fBendpoint\fR -(optional) when using S3\-compatible cloud storage, specify hostname of service endpoint here, region is ignored if endpoint is set (set region to some human\-readable name) (should be left blank for real Amazon S3) -. -.TP -\fBprefix\fR -(optional) do publishing under specified prefix in the bucket, defaults to no prefix (bucket root) -. -.TP -\fBacl\fR -(optional) assign ACL to published files (one of the canned ACLs in Amazon terminology)\. Useful values: \fBprivate\fR (default), \fBpublic\-read\fR (public repository) or \fBnone\fR (don\(cqt set ACL)\. Public repositories could be consumed by \fBapt\fR using HTTP endpoint (Amazon bucket should be configured for "website hosting"), for private repositories special apt S3 transport is required\. -. -.TP -\fBawsAccessKeyID\fR, \fBawsSecretAccessKey\fR -(optional) Amazon credentials to access S3 bucket\. If not supplied, environment variables \fBAWS_ACCESS_KEY_ID\fR and \fBAWS_SECRET_ACCESS_KEY\fR are used\. -. -.TP -\fBstorageClass\fR -(optional) Amazon S3 storage class, defaults to \fBSTANDARD\fR\. Other values available: \fBREDUCED_REDUNDANCY\fR (lower price, lower redundancy) -. -.TP -\fBencryptionMethod\fR -(optional) server\-side encryption method, defaults to none\. Currently the only available encryption method is \fBAES256\fR -. -.TP -\fBplusWorkaround\fR -(optional) workaround misbehavior in apt and Amazon S3 for files with \fB+\fR in filename by creating two copies of package files with \fB+\fR in filename: one original and another one with spaces instead of plus signs With \fBplusWorkaround\fR enabled, package files with plus sign would be stored twice\. aptly might not cleanup files with spaces when published repository is dropped or updated (switched) to new version of repository (snapshot) -. -.TP -\fBdisableMultiDel\fR -(optional) for S3\-compatible cloud storages which do not support \fBMultiDel\fR S3 API, enable this setting (file deletion would be slower with this setting enabled) -. -.TP -\fBforceSigV2\fR -(optional) disable Signature V4 support, useful with non\-AWS S3\-compatible object stores which do not support SigV4, shouldn\(cqt be enabled for AWS -. -.TP -\fBforceVirtualHostedStyle\fR -(optional) disable path style visit, useful with non\-AWS S3\-compatible object stores which only support virtual hosted style -. -.TP -\fBdebug\fR -(optional) enables detailed request/response dump for each S3 operation -. -.P -In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.: -. -.P -\fBaptly publish snapshot wheezy\-main s3:test:\fR -. -.SH "OPENSTACK SWIFT PUBLISHING ENDPOINTS" -aptly could be configured to publish repository directly to OpenStack Swift\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings: -. -.TP -\fBcontainer\fR -container name -. -.TP -\fBprefix\fR -(optional) do publishing under specified prefix in the container, defaults to no prefix (container root) -. -.TP -\fBosname\fR, \fBpassword\fR -(optional) OpenStack credentials to access Keystone\. If not supplied, environment variables \fBOS_USERNAME\fR and \fBOS_PASSWORD\fR are used\. -. -.TP -\fBtenant\fR, \fBtenantid\fR -(optional) OpenStack tenant name and id (in order to use v2 authentication)\. -. -.TP -\fBauthurl\fR -(optional) the full url of Keystone server (including port, and version)\. example \fBhttp://identity\.example\.com:5000/v2\.0\fR -. -.P -In order to publish to Swift, specify endpoint as \fBswift:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.: -. -.P -\fBaptly publish snapshot jessie\-main swift:test:\fR -. -.SH "AZURE PUBLISHING ENDPOINTS" -aptly can be configured to publish repositories directly to Microsoft Azure Blob Storage\. First, publishing endpoints should be described in the aptly configuration file\. Each endpoint has its name and associated settings: -. -.TP -\fBcontainer\fR -container name -. -.TP -\fBprefix\fR -(optional) do publishing under specified prefix in the container, defaults to no prefix (container root) -. -.TP -\fBaccountName\fR, \fBaccountKey\fR -Azure storage account access key to access blob storage -. -.TP -\fBendpoint\fR -endpoint URL to connect to, as described in the Azure documentation \fIhttps://docs\.microsoft\.com/en\-us/azure/storage/common/storage\-configure\-connection\-string\fR; defaults to \fBhttps://$accountName\.blob\.core\.windows\.net\fR -. .SH "PACKAGE QUERY" Some commands accept package queries to identify list of packages to process\. Package query syntax almost matches \fBreprepro\fR query language\. Query consists of the following simple terms: . @@ -771,7 +831,7 @@ custom format for result printing include dependencies into search results . .SH "ADD PACKAGES TO LOCAL REPOSITORY" -\fBaptly\fR \fBrepo\fR \fBadd\fR \fIname\fR +\fBaptly\fR \fBrepo\fR \fBadd\fR \fIname\fR \fB(|)\|\.\|\.\|\.\fR . .P Command adds packages to local repository from \.deb, \.udeb (binary packages) and \.dsc (source packages) files\. When importing from directory aptly would do recursive scan looking for all files matching \fI\.[u]deb or\fR\.dsc patterns\. Every file discovered would be analyzed to extract metadata, package would then be created and added to the database\. Files would be imported to internal package pool\. For source packages, all required files are added automatically as well\. Extra files for source package should be in the same directory as *\.dsc file\. @@ -1057,7 +1117,7 @@ custom format for result printing include dependencies into search results . .SH "ADD PACKAGES TO LOCAL REPOSITORIES BASED ON \.CHANGES FILES" -\fBaptly\fR \fBrepo\fR \fBinclude\fR +\fBaptly\fR \fBrepo\fR \fBinclude\fR \fB(|)\|\.\|\.\|\.\fR . .P Command include looks for \.changes files in list of arguments or specified directories\. Each \.changes file is verified, parsed, referenced files are put into separate temporary directory and added into local repository\. Successfully imported files are removed by default\. @@ -1103,7 +1163,7 @@ which repo should files go to, defaults to Distribution field of \.changes file path to uploaders\.json file . .SH "CREATES SNAPSHOT OF MIRROR (LOCAL REPOSITORY) CONTENTS" -\fBaptly\fR \fBsnapshot\fR \fBcreate\fR \fIname\fR \fBfrom\fR \fBmirror\fR \fImirror\-name\fR \fB|\fR \fBfrom\fR \fBrepo\fR \fIrepo\-name\fR \fB|\fR \fBempty\fR +\fBaptly\fR \fBsnapshot\fR \fBcreate\fR \fIname\fR \fB(from\fR \fBmirror\fR \fImirror\-name\fR \fB|\fR \fBfrom\fR \fBrepo\fR \fIrepo\-name\fR \fB|\fR \fBempty)\fR . .P Command create \fIname\fR from mirror makes persistent immutable snapshot of remote repository mirror\. Snapshot could be published or further modified using merge, pull and other aptly features\. @@ -1706,11 +1766,14 @@ don\(cqt sign Release files with GPG \-\fBsuite\fR= suite to publish (defaults to distribution) . -.SH "ADD SOURCE TO STAGED SOURCE LIST OF PUBLISHED REPOSITORY" +.SH "ADD SOURCE COMPONENTS TO A PUBLISHED REPO" \fBaptly\fR \fBpublish\fR \fBsource\fR \fBadd\fR \fIdistribution\fR \fIsource\fR . .P -The command adds sources to the staged source list of the published repository\. +The command adds components of a snapshot or local repository to be published\. +. +.P +This does not publish the changes directly, but rather schedules them for a subsequent \(cqaptly publish update\(cq\. . .P The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be modified\. The number of given components must be equal to the number of given sources, e\.g\.: @@ -1719,7 +1782,7 @@ The flag \-component is mandatory\. Use a comma\-separated list of components, i . .nf -aptly publish add \-component=main,contrib wheezy wheezy\-main wheezy\-contrib +aptly publish source add \-component=main,contrib wheezy wheezy\-main wheezy\-contrib . .fi . @@ -1732,7 +1795,7 @@ Example: . .nf -$ aptly publish add \-component=contrib wheezy ppa wheezy\-contrib +$ aptly publish source add \-component=contrib wheezy ppa wheezy\-contrib . .fi . @@ -1752,11 +1815,11 @@ component names to add (for multi\-component publishing, separate components wit \-\fBprefix\fR=\. publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR . -.SH "DROPS STAGED SOURCE CHANGES OF PUBLISHED REPOSITORY" +.SH "DROP PENDING SOURCE COMPONENT CHANGES OF A PUBLISHED REPOSITORY" \fBaptly\fR \fBpublish\fR \fBsource\fR \fBdrop\fR \fIdistribution\fR . .P -Command drops the staged source changes of the published repository\. +Remove all pending changes what would be applied with a subsequent \(cqaptly publish update\(cq\. . .P Example: @@ -1816,11 +1879,14 @@ display record in JSON format \-\fBprefix\fR=\. publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR . -.SH "REMOVE SOURCE FROM STAGED SOURCE LIST OF PUBLISHED REPOSITORY" +.SH "REMOVE SOURCE COMPONENTS FROM A PUBLISHED REPO" \fBaptly\fR \fBpublish\fR \fBsource\fR \fBremove\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] \fIsource\fR . .P -The command removes sources from the staged source list of the published repository\. +The command removes source components (snapshot / local repo) from a published repository\. +. +.P +This does not publish the changes directly, but rather schedules them for a subsequent \(cqaptly publish update\(cq\. . .P The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be removed, e\.g\.: @@ -1832,7 +1898,7 @@ Example: . .nf -$ aptly publish remove \-component=contrib,non\-free wheezy filesystem:symlink:debian +$ aptly publish source remove \-component=contrib,non\-free wheezy filesystem:symlink:debian . .fi . @@ -1849,11 +1915,60 @@ component names to remove (for multi\-component publishing, separate components \-\fBprefix\fR=\. publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR . -.SH "UPDATE SOURCE IN STAGED SOURCE LIST OF PUBLISHED REPOSITORY" +.SH "REPLACE THE SOURCE COMPONENTS OF A PUBLISHED REPOSITORY" +\fBaptly\fR \fBpublish\fR \fBsource\fR \fBreplace\fR \fIdistribution\fR \fIsource\fR +. +.P +The command replaces the source components of a snapshot or local repository to be published\. +. +.P +This does not publish the changes directly, but rather schedules them for a subsequent \(cqaptly publish update\(cq\. +. +.P +The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be modified\. The number of given components must be equal to the number of given sources, e\.g\.: +. +.IP "" 4 +. +.nf + +aptly publish source replace \-component=main,contrib wheezy wheezy\-main wheezy\-contrib +. +.fi +. +.IP "" 0 +. +.P +Example: +. +.IP "" 4 +. +.nf + +$ aptly publish source replace \-component=contrib wheezy ppa wheezy\-contrib +. +.fi +. +.IP "" 0 +. +.P +Options: +. +.TP +\-\fBcomponent\fR= +component names to add (for multi\-component publishing, separate components with commas) +. +.TP +\-\fBprefix\fR=\. +publishing prefix in the form of [\fIendpoint\fR:]\fIprefix\fR +. +.SH "UPDATE THE SOURCE COMPONENTS OF A PUBLISHED REPOSITORY" \fBaptly\fR \fBpublish\fR \fBsource\fR \fBupdate\fR \fIdistribution\fR \fIsource\fR . .P -The command updates sources in the staged source list of the published repository\. +The command updates the source components of a snapshot or local repository to be published\. +. +.P +This does not publish the changes directly, but rather schedules them for a subsequent \(cqaptly publish update\(cq\. . .P The flag \-component is mandatory\. Use a comma\-separated list of components, if multiple components should be modified\. The number of given components must be equal to the number of given sources, e\.g\.: @@ -1862,7 +1977,7 @@ The flag \-component is mandatory\. Use a comma\-separated list of components, i . .nf -aptly publish update \-component=main,contrib wheezy wheezy\-main wheezy\-contrib +aptly publish source update \-component=main,contrib wheezy wheezy\-main wheezy\-contrib . .fi . @@ -1875,7 +1990,7 @@ Example: . .nf -$ aptly publish update \-component=contrib wheezy ppa wheezy\-contrib +$ aptly publish source update \-component=contrib wheezy ppa wheezy\-contrib . .fi . @@ -1982,11 +2097,40 @@ don\(cqt generate Contents indexes \-\fBskip\-signing\fR don\(cqt sign Release files with GPG . -.SH "UPDATE PUBLISHED LOCAL REPOSITORY" +.SH "UPDATE PUBLISHED REPOSITORY" \fBaptly\fR \fBpublish\fR \fBupdate\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR] . .P -Command re\-publishes (updates) published local repository\. \fIdistribution\fR and \fIprefix\fR should be occupied with local repository published using command aptly publish repo\. Update happens in\-place with minimum possible downtime for published repository\. +The command updates updates a published repository after applying pending changes to the sources\. +. +.P +For published local repositories: +. +.IP "" 4 +. +.nf + +* update to match local repository contents +. +.fi +. +.IP "" 0 +. +.P +For published snapshots: +. +.IP "" 4 +. +.nf + +* switch components to new snapshot +. +.fi +. +.IP "" 0 +. +.P +The update happens in\-place with minimum possible downtime for published repository\. . .P For multiple component published repositories, all local repositories are updated\. @@ -2230,8 +2374,15 @@ Example: .P $ aptly config show . +.P +Options: +. +.TP +\-\fByaml\fR +show yaml config +. .SH "RUN APTLY TASKS" -\fBaptly\fR \fBtask\fR \fBrun\fR \-filename=\fIfilename\fR \fB|\fR \fIcommand1\fR, \fIcommand2\fR, \fB\|\.\|\.\|\.\fR +\fBaptly\fR \fBtask\fR \fBrun\fR (\-filename=\fIfilename\fR \fB|\fR \fIcommands\fR\|\.\|\.\|\.) . .P Command helps organise multiple aptly commands in one single aptly task, running as single thread\. @@ -2273,6 +2424,13 @@ Example: .P $ aptly config show . +.P +Options: +. +.TP +\-\fByaml\fR +show yaml config +. .SH "ENVIRONMENT" If environment variable \fBHTTP_PROXY\fR is set \fBaptly\fR would use its value to proxy all HTTP requests\. . @@ -2484,7 +2642,16 @@ Golf Hu (https://github\.com/hudeng\-go) Cookie Fei (https://github\.com/wuhuang26) . .IP "\[ci]" 4 +Andrey Loukhnov (https://github\.com/aol\-nnov) +. +.IP "\[ci]" 4 Christoph Fiehe (https://github\.com/cfiehe) . +.IP "\[ci]" 4 +Blake Kostner (https://github\.com/btkostner) +. +.IP "\[ci]" 4 +Leigh London (https://github\.com/leighlondon) +. .IP "" 0 diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index 0b319c749..b948aaf42 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -18,336 +18,384 @@ aptly has integrated help that matches contents of this manual page, to get help ## CONFIGURATION -aptly looks for configuration file first in `~/.aptly.conf` then -in `/usr/local/etc/aptly.conf` and `/etc/aptly.conf`. If no config file found (or they are not readable), a new one is created in the -home directory. If `-config=` flag is specified, aptly would use config file at specified -location. Also aptly needs root directory for database, package and published repository storage. -If not specified, directory defaults to `~/.aptly/`, it will be created if missing. +aptly looks for configuration file first in `~/.aptly.conf` then in `/usr/local/etc/aptly.conf` and `/etc/aptly.conf`. If no config file found (or they are not readable), a new one is created in the +home directory. If `-config=` flag is specified, aptly would use config file at specified location. Also aptly needs root directory for database, package and published repository storage. If not specified, directory defaults to `~/.aptly/`, it will be created if missing. -Configuration file is stored in JSON format (default values shown below): +With aptly version 1.6.0, yaml configuration with inline documentation is supported and recommended (see `debian/aptly.conf`). +The legacy json configuration is still supported: + + // vim: : filetype=json + // json configuration file with comments + // validate with: sed '/\/\//d' aptly.conf | json_pp { - "rootDir": "$HOME/.aptly", - "databaseBackend": { - "type": "", - "url": "" - }, - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "downloadRetries": 0, - "downloader": "default", - "databaseOpenAttempts": 10, + + // Aptly Configuration File + //////////////////////////// + + // Root directory for: + // - downloaded packages (`rootDir`/pool) + // - database (`rootDir`/db) + // - published repositories (`rootDir`/public) + "rootDir": "~/.aptly", + + // Number of attempts to open database if it's locked by other instance + // * -1 (no retry) + "databaseOpenAttempts": -1, + + // Log Level + // * debug + // * info + // * warning + // * error + "logLevel": "info", + + // Log Format + // * default (text) + // * json + "logFormat": "default", + + // Default Architectures + // empty array defaults to all available architectures "architectures": [], + + // Follow contents of `Suggests:` field when processing dependencies for the package "dependencyFollowSuggests": false, + + // Follow contents of `Recommends:` field when processing dependencies for the package "dependencyFollowRecommends": false, + + // When dependency looks like `package-a | package-b`, follow both variants always "dependencyFollowAllVariants": false, + + // Follow dependency from binary package to source package "dependencyFollowSource": false, + + // Log additional details while resolving dependencies (useful for debugging) "dependencyVerboseResolve": false, - "gpgDisableSign": false, - "gpgDisableVerify": false, - "gpgProvider": "gpg", - "downloadSourcePackages": false, - "packagePoolStorage": { - "path": "$ROOTDIR/pool", - "azure": { - "accountName": "", - "accountKey": "", - "container": "repo", - "prefix": "", - "endpoint": "" - } - }, - "skipLegacyPool": true, + + // Specifies paramaters for short PPA url expansion + // empty defaults to output of `lsb_release` command "ppaDistributorID": "ubuntu", + + // Codename for short PPA url expansion "ppaCodename": "", - "skipContentsPublishing": false, - "FileSystemPublishEndpoints": { - "test1": { - "rootDir": "/opt/srv1/aptly_public", - "linkMethod": "symlink" - }, - "test2": { - "rootDir": "/opt/srv2/aptly_public", - "linkMethod": "copy", - "verifyMethod": "md5" - }, - "test3": { - "rootDir": "/opt/srv3/aptly_public", - "linkMethod": "hardlink" - } - }, - "S3PublishEndpoints": { - "test": { - "region": "us-east-1", - "bucket": "repo", - "endpoint": "", - "awsAccessKeyID": "", - "awsSecretAccessKey": "", - "prefix": "", - "acl": "public-read", - "storageClass": "", - "encryptionMethod": "", - "plusWorkaround": false, - "disableMultiDel": false, - "forceSigV2": false, - "forceVirtualHostedStyle": true, - "debug": false - } - }, - "SwiftPublishEndpoints": { - "test": { - "container": "repo", - "osname": "", - "password": "", - "prefix": "", - "authurl": "", - "tenant": "", - "tenantid": "" - } + + // OBSOLETE + // in aptly up to version 1.0.0, package files were stored in internal package pool + // with MD5-dervied path, since 1.1.0 package pool layout was changed; + // if option is enabled, aptly stops checking for legacy paths; + // by default option is enabled for new aptly installations and disabled when + // upgrading from older versions + "skipLegacyPool": true, + + + // Aptly Server + //////////////// + + // Serve published repos as well as API + "serveInAPIMode": false, + + // Enable metrics for Prometheus client + "enableMetricsEndpoint": false, + + // Enable API documentation on /docs + "enableSwaggerEndpoint": false, + + // OBSOLETE: use via url param ?_async=true + "AsyncAPI": false, + + + // Database + //////////// + + // Database backend + // Type must be one of: + // * leveldb (default) + // * etcd + "databaseBackend": { + // LevelDB + "type": "leveldb", + // Path to leveldb files + // empty dbPath defaults to `rootDir`/db + "dbPath": "" + + // // etcd + // "type": "etcd", + // // URL to db server + // "url": "127.0.0.1:2379" }, - "AzurePublishEndpoints": { - "test": { - "accountName": "", - "accountKey": "", - "container": "repo", - "prefix": "", - "endpoint": "blob.core.windows.net" - } - } - } -Options: - * `rootDir`: - is root of directory storage to store database (`rootDir`/db), - the default for downloaded packages (`rootDir`/pool) and - the default for published repositories (`rootDir`/public) and - skeleton files (`rootDir`/skel) + // Mirroring + ///////////// + + // Downloader + // * "default" + // * "grab" (more robust) + "downloader": "default", + + // Number of parallel download threads to use when downloading packages + "downloadConcurrency": 4, + + // Limit in kbytes/sec on download speed while mirroring remote repositories + "downloadSpeedLimit": 0, + + // Number of retries for download attempts + "downloadRetries": 0, + + // Download source packages per default + "downloadSourcePackages": false, + + + // Signing + /////////// + + // GPG Provider + // * "internal" (Go internal implementation) + // * "gpg" (External `gpg` utility) + "gpgProvider": "gpg", - * `databaseBackend`: - the database config; if this config is empty, use levledb backend by default + // Disable signing of published repositories + "gpgDisableSign": false, - * `downloadConcurrency`: - is a number of parallel download threads to use when downloading packages + // Disable signature verification of remote repositories + "gpgDisableVerify": false, - * `downloadSpeedLimit`: - limit in kbytes/sec on download speed while mirroring remote repositories - * `downloadRetries`: - number of retries for download attempts + // Publishing + ////////////// + + // Do not publish Contents files + "skipContentsPublishing": false, - * `databaseOpenAttempts`: - number of attempts to open DB if it's locked by other instance; could be overridden with option - `-db-open-attempts` - - * `architectures`: - is a list of architectures to process; if left empty defaults to all available architectures; could be - overridden with option `-architectures` - - * `dependencyFollowSuggests`: - follow contents of `Suggests:` field when processing dependencies for the package + // Do not create bz2 files + "skipBz2Publishing": false, + + + // Storage + /////////// + + // Filesystem publishing endpoints + // + // aptly defaults to publish to a single publish directory under `rootDir`/public. For + // a more advanced publishing strategy, you can define one or more filesystem endpoints in the + // `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name + // and the following associated settings. + // + // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` + // with `endpoint-name` as the name given in the aptly configuration file. For example: + // + // `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` + // + "FileSystemPublishEndpoints": { + // // Endpoint Name + // "test1": { + // // Directory for publishing + // "rootDir": "/opt/srv/aptly_public", + + // // File Link Method for linking files from the internal pool to the published directory + // // * hardlink + // // * symlink + // // * copy + // "linkMethod": "hardlink", + + // // File Copare Method for comparing existing links from the internal pool to the published directory + // // Only used when "linkMethod" is set to "copy" + // // * md5 (default: compare md5 sum) + // // * size (compare file size) + // "verifyMethod": "md5" + // } + }, - * `dependencyFollowRecommends`: - follow contents of `Recommends:` field when processing dependencies for the package + // S3 Endpoint Support + // + // cloud storage). First, publishing + // endpoints should be described in aptly configuration file. Each endpoint has name + // and associated settings. + // + // In order to publish to S3, specify endpoint as `s3:endpoint-name:` before + // publishing prefix on the command line, e.g.: + // + // `aptly publish snapshot wheezy-main s3:test:` + // + "S3PublishEndpoints": { + // // Endpoint Name + // "test": { + + // // Amazon region for S3 bucket + // "region": "us-east-1", + + // // Bucket name + // "bucket": "test-bucket", + + // // Endpoint (optional) + // // When using S3-compatible cloud storage, specify hostname of service endpoint here, + // // region is ignored if endpoint is set (set region to some human-readable name) + // // (should be left blank for real Amazon S3) + // "endpoint": "", + + // // Prefix (optional) + // // publishing under specified prefix in the bucket, defaults to + // // no prefix (bucket root) + // "prefix": "", + + // // Default ACLs (optional) + // // assign ACL to published files (one of the canned ACLs in Amazon + // // terminology). Useful values: `private` (default), `public-read` (public + // // repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using + // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), + // // for private repositories special apt S3 transport is required. + // "acl": "private", + + // // Credentials (optional) + // // Amazon credentials to access S3 bucket. If not supplied, + // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` + // // are used. + // "awsAccessKeyID": "", + // "awsSecretAccessKey": "", + + // // Storage Class (optional) + // // Amazon S3 storage class, defaults to `STANDARD`. Other values + // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + // "storageClass": "STANDARD", + + // // Encryption Method (optional) + // // Server-side encryption method, defaults to none. Currently + // // the only available encryption method is `AES256` + // "encryptionMethod": "none", + + // // Plus Workaround (optional) + // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + // // creating two copies of package files with `+` in filename: one original + // // and another one with spaces instead of plus signs + // // With `plusWorkaround` enabled, package files with plus sign + // // would be stored twice. aptly might not cleanup files with spaces when published + // // repository is dropped or updated (switched) to new version of repository (snapshot) + // "plusWorkaround": false, + + // // Disable MultiDel (optional) + // // For S3-compatible cloud storages which do not support `MultiDel` S3 API, + // // enable this setting (file deletion would be slower with this setting enabled) + // "disableMultiDel": false, + + // // ForceSig2 (optional) + // // Disable Signature V4 support, useful with non-AWS S3-compatible object stores + // // which do not support SigV4, shouldn't be enabled for AWS + // "forceSigV2": false, + + // // ForceVirtualHostedStyle (optional) + // // Disable path style visit, useful with non-AWS S3-compatible object stores + // // which only support virtual hosted style + // "forceVirtualHostedStyle": false, + + // // Debug (optional) + // // Enables detailed request/response dump for each S3 operation + // "debug": false + // } + }, - * `dependencyFollowAllVariants`: - when dependency looks like `package-a | package-b`, follow both variants always - - * `dependencyFollowSource`: - follow dependency from binary package to source package + // Swift Endpoint Support + // + // aptly could be configured to publish repository directly to OpenStack Swift. First, + // publishing endpoints should be described in aptly configuration file. Each endpoint + // has name and associated settings. + // + // In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before + // publishing prefix on the command line, e.g.: + // + // `aptly publish snapshot jessie-main swift:test:` + // + "SwiftPublishEndpoints": { + // Endpoint Name + // "test": { + + // // Container Name + // "container": "taylor1", + + // // Prefix (optional) + // // Publish under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials (optional) + // // OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + // "osname": "", + // "password": "", + + // // Tenant (optional) + // // OpenStack tenant name and id (in order to use v2 authentication) + // "tenant": "", + // "tenantid": "", + + // // Auth URL (optional) + // // Full url of Keystone server (including port, and version). + // // Example `http://identity.example.com:5000/v2.0` + // "authurl": "" + // } + }, + + // Azure Endpoint Support + // + // aptly can be configured to publish repositories directly to Microsoft Azure Blob + // Storage. First, publishing endpoints should be described in the aptly + // configuration file. Each endpoint has its name and associated settings. + "AzurePublishEndpoints": { + // // Endpoint Name + // "test": { + + // // Container Name + // "container": "container1", + + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + + // // Endpoint URL + // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + // // defaults to "https://.blob.core.windows.net" + // "endpoint": "" + // } + }, + + // Package Pool + // Location for storing downloaded packages + // Type must be one of: + // * local + // * azure + "packagePoolStorage": { + // Local Pool + "type": "local", + // Local Pool Path + // empty path defaults to `rootDir`/pool + "path": "" + + // // Azure Azure Blob Storage Pool + // "type": "azure", + // "container": "pool1", + + // // Prefix (optional) + // // Publishing under specified prefix in the container, defaults to no prefix (container root) + // "prefix": "", + + // // Credentials + // // Azure storage account access key to access blob storage + // "accountName": "", + // "accountKey": "", + + // // Endpoint URL + // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + // // defaults to "https://.blob.core.windows.net" + // "endpoint": "" + } + + // End of config + } - * `dependencyVerboseResolve`: - print additional details while resolving dependencies (useful for debugging) - - * `gpgDisableSign`: - don't sign published repositories with gpg(1), also can be disabled on - per-repo basis using `-skip-signing` flag when publishing - - * `gpgDisableVerify`: - don't verify remote mirrors with gpg(1), also can be disabled on - per-mirror basis using `-ignore-signatures` flag when creating and updating mirrors - - * `gpgProvider`: - implementation of PGP signing/validation - `gpg` for external `gpg` utility or - `internal` to use Go internal implementation; `gpg1` might be used to force use - of GnuPG 1.x, `gpg2` enables GnuPG 2.x only; default is to use GnuPG 1.x if - available and GnuPG 2.x otherwise - - * `downloadSourcePackages`: - if enabled, all mirrors created would have flag set to download source packages; - this setting could be controlled on per-mirror basis with `-with-sources` flag - - * `packagePoolStorage`: - configures the location to store downloaded packages (defaults to the - path `$ROOTDIR/pool`), by setting the value of the `type`: - * `path`: store the packages in the given path - * `azure`: store the packages in the given Azure Blob Storage container - (see the section on Azure publishing below for information on the - configuration) - - * `skipLegacyPool`: - in aptly up to version 1.0.0, package files were stored in internal package pool - with MD5-dervied path, since 1.1.0 package pool layout was changed; - if option is enabled, aptly stops checking for legacy paths; - by default option is enabled for new aptly installations and disabled when - upgrading from older versions - - * `ppaDistributorID`, `ppaCodename`: - specifies paramaters for short PPA url expansion, if left blank they default - to output of `lsb_release` command - - * `FileSystemPublishEndpoints`: - configuration of local filesystem publishing endpoints (see below) - - * `S3PublishEndpoints`: - configuration of Amazon S3 publishing endpoints (see below) - - * `SwiftPublishEndpoints`: - configuration of OpenStack Swift publishing endpoints (see below) - - * `AzurePublishEndpoints`: - configuration of Azure publishing endpoints (see below) - -## CUSTOM PACKAGE POOLS - -aptly defaults to storing downloaded packages at `rootDir/`pool. In order to -change this, you can set the `type` key within `packagePoolStorage` to one of -two values: - - * `local`: Store the package pool locally (the default). In order to change - the path, additionally set the `path` key within `packagePoolStorage` to - the desired location. - * `azure`: Store the package pool in an Azure Blob Storage container. Any - keys in the below section on Azure publishing may be set on the - `packagePoolStorage` object in order to configure the Azure connection. - -## FILESYSTEM PUBLISHING ENDPOINTS - -aptly defaults to publish to a single publish directory under `rootDir`/public. For -a more advanced publishing strategy, you can define one or more filesystem endpoints in the -`FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name -and the following associated settings: - - * `rootDir`: - The publish directory, e.g., `/opt/srv/aptly_public`. - * `linkMethod`: - This is one of `hardlink`, `symlink` or `copy`. It specifies how aptly links the - files from the internal pool to the published directory. - If not specified, empty or wrong, this defaults to `hardlink`. - * `verifyMethod`: - This is used only when setting the `linkMethod` to `copy`. Possible values are - `md5` and `size`. It specifies how aptly compares existing links from the - internal pool to the published directory. The `size` method compares only the - file sizes, whereas the `md5` method calculates the md5 checksum of the found - file and compares it to the desired one. - If not specified, empty or wrong, this defaults to `md5`. - -In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` -with `endpoint-name` as the name given in the aptly configuration file. For example: - - `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` - -## S3 PUBLISHING ENDPOINTS - -aptly could be configured to publish repository directly to Amazon S3 (or S3-compatible -cloud storage). First, publishing -endpoints should be described in aptly configuration file. Each endpoint has name -and associated settings: - - * `region`: - Amazon region for S3 bucket (e.g. `us-east-1`) - * `bucket`: - bucket name - * `endpoint`: - (optional) when using S3-compatible cloud storage, specify hostname of service endpoint here, - region is ignored if endpoint is set (set region to some human-readable name) - (should be left blank for real Amazon S3) - * `prefix`: - (optional) do publishing under specified prefix in the bucket, defaults to - no prefix (bucket root) - * `acl`: - (optional) assign ACL to published files (one of the canned ACLs in Amazon - terminology). Useful values: `private` (default), `public-read` (public - repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using - HTTP endpoint (Amazon bucket should be configured for "website hosting"), - for private repositories special apt S3 transport is required. - * `awsAccessKeyID`, `awsSecretAccessKey`: - (optional) Amazon credentials to access S3 bucket. If not supplied, - environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` - are used. - * `storageClass`: - (optional) Amazon S3 storage class, defaults to `STANDARD`. Other values - available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) - * `encryptionMethod`: - (optional) server-side encryption method, defaults to none. Currently - the only available encryption method is `AES256` - * `plusWorkaround`: - (optional) workaround misbehavior in apt and Amazon S3 - for files with `+` in filename by - creating two copies of package files with `+` in filename: one original - and another one with spaces instead of plus signs - With `plusWorkaround` enabled, package files with plus sign - would be stored twice. aptly might not cleanup files with spaces when published - repository is dropped or updated (switched) to new version of repository (snapshot) - * `disableMultiDel`: - (optional) for S3-compatible cloud storages which do not support `MultiDel` S3 API, - enable this setting (file deletion would be slower with this setting enabled) - * `forceSigV2`: - (optional) disable Signature V4 support, useful with non-AWS S3-compatible object stores - which do not support SigV4, shouldn't be enabled for AWS - * `forceVirtualHostedStyle`: - (optional) disable path style visit, useful with non-AWS S3-compatible object stores - which only support virtual hosted style - * `debug`: - (optional) enables detailed request/response dump for each S3 operation - -In order to publish to S3, specify endpoint as `s3:endpoint-name:` before -publishing prefix on the command line, e.g.: - - `aptly publish snapshot wheezy-main s3:test:` - -## OPENSTACK SWIFT PUBLISHING ENDPOINTS - -aptly could be configured to publish repository directly to OpenStack Swift. First, -publishing endpoints should be described in aptly configuration file. Each endpoint -has name and associated settings: - - * `container`: - container name - * `prefix`: - (optional) do publishing under specified prefix in the container, defaults to - no prefix (container root) - * `osname`, `password`: - (optional) OpenStack credentials to access Keystone. If not supplied, - environment variables `OS_USERNAME` and `OS_PASSWORD` are used. - * `tenant`, `tenantid`: - (optional) OpenStack tenant name and id (in order to use v2 authentication). - * `authurl`: - (optional) the full url of Keystone server (including port, and version). - example `http://identity.example.com:5000/v2.0` - -In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before -publishing prefix on the command line, e.g.: - - `aptly publish snapshot jessie-main swift:test:` - -## AZURE PUBLISHING ENDPOINTS - -aptly can be configured to publish repositories directly to Microsoft Azure Blob -Storage. First, publishing endpoints should be described in the aptly -configuration file. Each endpoint has its name and associated settings: - - * `container`: - container name - * `prefix`: - (optional) do publishing under specified prefix in the container, defaults to - no prefix (container root) - * `accountName`, `accountKey`: - Azure storage account access key to access blob storage - * `endpoint`: - endpoint URL to connect to, as described in - [the Azure documentation](https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string); - defaults to `https://$accountName.blob.core.windows.net` ## PACKAGE QUERY diff --git a/system/t02_config/BadConfigTest_gold b/system/t02_config/BadConfigTest_gold index 10f5b8ce8..f6864d06f 100644 --- a/system/t02_config/BadConfigTest_gold +++ b/system/t02_config/BadConfigTest_gold @@ -1 +1 @@ -ERROR: error loading config file ${HOME}/.aptly.conf: invalid character 's' looking for beginning of object key string +ERROR: error loading config file ${HOME}/.aptly.conf: invalid yaml (yaml: line 1: did not find expected ',' or '}') or json (invalid character 's' looking for beginning of object key string) diff --git a/system/t02_config/ConfigShowTest_gold b/system/t02_config/ConfigShowTest_gold index 0529de71a..5a4a2273a 100644 --- a/system/t02_config/ConfigShowTest_gold +++ b/system/t02_config/ConfigShowTest_gold @@ -1,39 +1,39 @@ { "rootDir": "${HOME}/.aptly", - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "downloadRetries": 5, - "downloader": "default", + "logLevel": "debug", + "logFormat": "default", "databaseOpenAttempts": 10, "architectures": [], + "skipLegacyPool": false, "dependencyFollowSuggests": false, "dependencyFollowRecommends": false, "dependencyFollowAllVariants": false, "dependencyFollowSource": false, "dependencyVerboseResolve": false, - "gpgDisableSign": false, - "gpgDisableVerify": false, - "gpgProvider": "gpg", - "downloadSourcePackages": false, - "packagePoolStorage": {}, - "skipLegacyPool": false, "ppaDistributorID": "ubuntu", "ppaCodename": "", + "serveInAPIMode": true, + "enableMetricsEndpoint": true, + "enableSwaggerEndpoint": false, + "AsyncAPI": false, + "databaseBackend": { + "type": "", + "dbPath": "", + "url": "" + }, + "downloader": "default", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "downloadRetries": 5, + "downloadSourcePackages": false, + "gpgProvider": "gpg", + "gpgDisableSign": false, + "gpgDisableVerify": false, "skipContentsPublishing": false, "skipBz2Publishing": false, "FileSystemPublishEndpoints": {}, "S3PublishEndpoints": {}, "SwiftPublishEndpoints": {}, "AzurePublishEndpoints": {}, - "AsyncAPI": false, - "enableMetricsEndpoint": true, - "logLevel": "debug", - "logFormat": "default", - "serveInAPIMode": true, - "databaseBackend": { - "type": "", - "url": "", - "dbPath": "" - }, - "enableSwaggerEndpoint": false + "packagePoolStorage": {} } diff --git a/system/t02_config/ConfigShowYAMLTest_gold b/system/t02_config/ConfigShowYAMLTest_gold new file mode 100644 index 000000000..615982b0d --- /dev/null +++ b/system/t02_config/ConfigShowYAMLTest_gold @@ -0,0 +1,37 @@ +root_dir: ${HOME}/.aptly +log_level: debug +log_format: default +database_open_attempts: 10 +architectures: [] +skip_legacy_pool: false +dep_follow_suggests: false +dep_follow_recommends: false +dep_follow_all_variants: false +dep_follow_source: false +dep_verboseresolve: false +ppa_distributor_id: ubuntu +ppa_codename: "" +serve_in_api_mode: true +enable_metrics_endpoint: true +enable_swagger_endpoint: false +async_api: false +database_backend: + type: "" + db_path: "" + url: "" +downloader: default +download_concurrency: 4 +download_limit: 0 +download_retries: 5 +download_sourcepackages: false +gpg_provider: gpg +gpg_disable_sign: false +gpg_disable_verify: false +skip_contents_publishing: false +skip_bz2_publishing: false +filesystem_publish_endpoints: {} +s3_publish_endpoints: {} +swift_publish_endpoints: {} +azure_publish_endpoints: {} +packagepool_storage: {} + diff --git a/system/t02_config/CreateConfigTest_gold b/system/t02_config/CreateConfigTest_gold index 6eeed77ec..38a8a56a8 100644 --- a/system/t02_config/CreateConfigTest_gold +++ b/system/t02_config/CreateConfigTest_gold @@ -1,370 +1,343 @@ -// vim: : filetype=json -// json configuration file with comments -// validate with: sed '/\/\//d' aptly.conf | json_pp -{ +# Aptly Configuration File +########################### +# vim: : filetype=yaml -// Aptly Configuration File -//////////////////////////// +# aptly 1.6.0 supports yaml configuraiton files with inline documentation and examples. +# Legacy json config files are still supported, and may be converted to yaml with `aptly config show -yaml` - // Aptly storage directory for: - // - downloaded packages (`rootDir`/pool) - // - database (`rootDir`/db) - // - published repositories (`rootDir`/public) - "rootDir": "~/.aptly", +# Root directory for: +# - downloaded packages (`rootDir`/pool) +# - database (`rootDir`/db) +# - published repositories (`rootDir`/public) +root_dir: ~/.aptly - // Number of attempts to open database if it's locked by other instance - // * -1 (no retry) - "databaseOpenAttempts": -1, +# Log Level +# * debug +# * info +# * warning +# * error +log_level: info - // Log Level - // * debug - // * info - // * warning - // * error - "logLevel": "info", +# Log Format +# * default (text) +# * json +log_format: default - // Log Format - // * default (text) - // * json - "logFormat": "default", +# Number of attempts to open database if it's locked by other instance +# * -1 (no retry) +database_open_attempts: -1 - // Default Architectures - // empty array defaults to all available architectures - "architectures": [], +# Default Architectures +# empty list defaults to all available architectures +architectures: +# - amd64 - // Follow contents of `Suggests:` field when processing dependencies for the package - "dependencyFollowSuggests": false, +# OBSOLETE +# in aptly up to version 1.0.0, package files were stored in internal package pool +# with MD5-dervied path, since 1.1.0 package pool layout was changed; +# if option is enabled, aptly stops checking for legacy paths; +# by default option is enabled for new aptly installations and disabled when +# upgrading from older versions +skip_legacy_pool: true - // Follow contents of `Recommends:` field when processing dependencies for the package - "dependencyFollowRecommends": false, - // When dependency looks like `package-a | package-b`, follow both variants always - "dependencyFollowAllVariants": false, +# Dependency following +####################### - // Follow dependency from binary package to source package - "dependencyFollowSource": false, +# Follow contents of `Suggests:` field when processing dependencies for the package +dep_follow_suggests: false - // Log additional details while resolving dependencies (useful for debugging) - "dependencyVerboseResolve": false, +# Follow contents of `Recommends:` field when processing dependencies for the package +dep_follow_recommends: false - // Specifies paramaters for short PPA url expansion - // empty defaults to output of `lsb_release` command - "ppaDistributorID": "ubuntu", +# When dependency looks like `package-a | package-b`, follow both variants always +dep_follow_allvariants: false - // Codename for short PPA url expansion - "ppaCodename": "", +# Follow dependency from binary package to source package +dep_follow_source: false - // OBSOLETE - // in aptly up to version 1.0.0, package files were stored in internal package pool - // with MD5-dervied path, since 1.1.0 package pool layout was changed; - // if option is enabled, aptly stops checking for legacy paths; - // by default option is enabled for new aptly installations and disabled when - // upgrading from older versions - "skipLegacyPool": true, +# Log additional details while resolving dependencies (useful for debugging) +dep_verbose_resolve: false -// Aptly Server -//////////////// +# PPA +###### - // Serve published repos as well as API - "serveInAPIMode": false, +# Specify paramaters for short PPA url expansion +# empty defaults to output of `lsb_release` command +ppa_distributor_id: ubuntu - // Enable metrics for Prometheus client - "enableMetricsEndpoint": false, +# Codename for short PPA url expansion +ppa_codename: "" - // Enable API documentation on /docs - "enableSwaggerEndpoint": false, - // OBSOLETE: use via url param ?_async=true - "AsyncAPI": false, +# Aptly Server +############### +# Serve published repos as well as API +serve_in_api_mode: false -// Database -//////////// +# Enable metrics for Prometheus client +enable_metrics_endpoint: false + +# Enable API documentation on /docs +enable_swagger_endpoint: false + +# OBSOLETE: use via url param ?_async=true +async_api: false + + +# Database +########### + +# Database backend +# Type must be one of: +# * leveldb (default) +# * etcd +database_backend: + type: leveldb + # Path to leveldb files + # empty dbPath defaults to `rootDir`/db + db_path: "" + + # type: etcd + # # URL to db server + # url: "127.0.0.1:2379" + + +# Mirroring +############ + +# Downloader +# * "default" +# * "grab" (more robust) +downloader: default + +# Number of parallel download threads to use when downloading packages +download_concurrency: 4 + +# Limit in kbytes/sec on download speed while mirroring remote repositories +download_limit: 0 + +# Number of retries for download attempts +download_retries: 0 + +# Download source packages per default +download_sourcepackages: false + + +# Signing +########## + +# GPG Provider +# * "internal" (Go internal implementation) +# * "gpg" (External `gpg` utility) +gpg_provider: gpg + +# Disable signing of published repositories +gpg_disable_sign: false + +# Disable signature verification of remote repositories +gpg_disable_verify: false + + +# Publishing +############# + +# Do not publish Contents files +skip_contents_publishing: false + +# Do not create bz2 files +skip_bz2_publishing: false + + +# Storage +########## + +# Filesystem publishing endpoints +# +# aptly defaults to publish to a single publish directory under `rootDir`/public. For +# a more advanced publishing strategy, you can define one or more filesystem endpoints in the +# `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name +# and the following associated settings. +# +# In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` +# with `endpoint-name` as the name given in the aptly configuration file. For example: +# +# `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` +# +filesystem_publish_endpoints: + # # Endpoint Name + # test1: + # # Directory for publishing + # root_dir: /opt/srv/aptly_public + # # File Link Method for linking files from the internal pool to the published directory + # # * hardlink + # # * symlink + # # * copy + # link_method: hardlink + # # File Copare Method for comparing existing links from the internal pool to the published directory + # # Only used when "linkMethod" is set to "copy" + # # * md5 (default: compare md5 sum) + # # * size (compare file size) + # verify_method: md5 + +# S3 Endpoint Support +# +# cloud storage). First, publishing +# endpoints should be described in aptly configuration file. Each endpoint has name +# and associated settings. +# +# In order to publish to S3, specify endpoint as `s3:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot wheezy-main s3:test:` +# +s3_publish_endpoints: + # # Endpoint Name + # test: + # # Amazon region for S3 bucket + # region: us-east-1 + # # Bucket name + # bucket: test-bucket + # # Prefix (optional) + # # publishing under specified prefix in the bucket, defaults to + # # no prefix (bucket root) + # prefix: "" + # # Default ACLs (optional) + # # assign ACL to published files: + # # * private (default, for use with apt S3 transport) + # # * public-read (public repository) + # # * none (don't set ACL) + # acl: private + # # Credentials (optional) + # # Amazon credentials to access S3 bucket. If not supplied, environment variables + # # `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` are used + # access_key_id: "" + # secret_access_key: "" + # session_token: "" + # # Endpoint (optional) + # # When using S3-compatible cloud storage, specify hostname of service endpoint here, + # # region is ignored if endpoint is set (set region to some human-readable name) + # # (should be left blank for real Amazon S3) + # endpoint: "" + # # Storage Class (optional) + # # Amazon S3 storage class, defaults to `STANDARD`. Other values + # # available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) + # storage_class: STANDARD + # # Encryption Method (optional) + # # Server-side encryption method, defaults to none. Currently + # # the only available encryption method is `AES256` + # encryption_method: none + # # Plus Workaround (optional) + # # Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by + # # creating two copies of package files with `+` in filename: one original + # # and another one with spaces instead of plus signs + # # With `plusWorkaround` enabled, package files with plus sign + # # would be stored twice. aptly might not cleanup files with spaces when published + # # repository is dropped or updated (switched) to new version of repository (snapshot) + # plus_workaround: false + # # Disable MultiDel (optional) + # # For S3-compatible cloud storages which do not support `MultiDel` S3 API, + # # enable this setting (file deletion would be slower with this setting enabled) + # disable_multidel: false + # # Force Signature v2 (optional) + # # Disable Signature V4 support, useful with non-AWS S3-compatible object stores + # # which do not support SigV4, shouldn't be enabled for AWS + # force_sigv2: false + # # Force VirtualHosted Style (optional) + # # Disable path style visit, useful with non-AWS S3-compatible object stores + # # which only support virtual hosted style + # force_virtualhosted_style: false + # # Debug (optional) + # # Enables detailed request/response dump for each S3 operation + # debug: false + +# Swift Endpoint Support +# +# aptly can publish a repository directly to OpenStack Swift. +# Each endpoint has name and associated settings. +# +# In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before +# publishing prefix on the command line, e.g.: +# +# `aptly publish snapshot jessie-main swift:test:` +# +swift_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: taylor1 + # # Prefix (optional) + # # Publish under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials (optional) + # # OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used + # username: "" + # password: "" + # # Domain (optional) + # # OpenStack domain + # domain: "" + # domain_id: "" + # # Tenant (optional) + # # OpenStack tenant (in order to use v2 authentication) + # tenant: "" + # tenant_id: "" + # tenant_domain: "" + # tenant_domain_id: "" + # # Auth URL (optional) + # # Full url of Keystone server (including port, and version). + # # Example `http://identity.example.com:5000/v2.0` + # auth_url: "" + +# Azure Endpoint Support +# +# aptly can be configured to publish repositories directly to Microsoft Azure Blob +# Storage. First, publishing endpoints should be described in the aptly +# configuration file. Each endpoint has its name and associated settings. +azure_publish_endpoints: + # # Endpoint Name + # test: + # # Container Name + # container: container1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" + +# Package Pool +# +# Location for storing downloaded packages +# Type must be one of: +# * local +# * azure +packagepool_storage: + # Local Pool + type: local + # Local Pool Path + # empty path defaults to `rootDir`/pool + path: + + # # Azure Azure Blob Storage Pool + # type: azure + # # Container Name + # container: pool1 + # # Prefix (optional) + # # Publishing under specified prefix in the container, defaults to no prefix (container root) + # prefix: "" + # # Credentials + # # Azure storage account access key to access blob storage + # account_name: "" + # account_key: "" + # # Endpoint URL + # # See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string + # # defaults to "https://.blob.core.windows.net" + # endpoint: "" - // Database backend - // Type must be one of: - // * leveldb (default) - // * etcd - "databaseBackend": { - // LevelDB - "type": "leveldb", - // Path to leveldb files - // empty dbPath defaults to `rootDir`/db - "dbPath": "" - - // // etcd - // "type": "etcd", - // // URL to db server - // "url": "127.0.0.1:2379" - }, - - -// Mirroring -///////////// - - // Downloader - // * "default" - // * "grab" (more robust) - "downloader": "default", - - // Number of parallel download threads to use when downloading packages - "downloadConcurrency": 4, - - // Limit in kbytes/sec on download speed while mirroring remote repositories - "downloadSpeedLimit": 0, - - // Number of retries for download attempts - "downloadRetries": 0, - - // Download source packages per default - "downloadSourcePackages": false, - - -// Signing -/////////// - - // GPG Provider - // * "internal" (Go internal implementation) - // * "gpg" (External `gpg` utility) - "gpgProvider": "gpg", - - // Disable signing of published repositories - "gpgDisableSign": false, - - // Disable signature verification of remote repositories - "gpgDisableVerify": false, - - -// Publishing -////////////// - - // Do not publish Contents files - "skipContentsPublishing": false, - - // Do not create bz2 files - "skipBz2Publishing": false, - - -// Storage -/////////// - - // Filesystem publishing endpoints - // - // aptly defaults to publish to a single publish directory under `rootDir`/public. For - // a more advanced publishing strategy, you can define one or more filesystem endpoints in the - // `FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name - // and the following associated settings. - // - // In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name` - // with `endpoint-name` as the name given in the aptly configuration file. For example: - // - // `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily` - // - "FileSystemPublishEndpoints": { - // // Endpoint Name - // "test1": { - // // Directory for publishing - // "rootDir": "/opt/srv/aptly_public", - - // // File Link Method for linking files from the internal pool to the published directory - // // * hardlink - // // * symlink - // // * copy - // "linkMethod": "hardlink", - - // // File Copare Method for comparing existing links from the internal pool to the published directory - // // Only used when "linkMethod" is set to "copy" - // // * md5 (default: compare md5 sum) - // // * size (compare file size) - // "verifyMethod": "md5" - // } - }, - - // S3 Endpoint Support - // - // cloud storage). First, publishing - // endpoints should be described in aptly configuration file. Each endpoint has name - // and associated settings. - // - // In order to publish to S3, specify endpoint as `s3:endpoint-name:` before - // publishing prefix on the command line, e.g.: - // - // `aptly publish snapshot wheezy-main s3:test:` - // - "S3PublishEndpoints": { - // // Endpoint Name - // "test": { - - // // Amazon region for S3 bucket - // "region": "us-east-1", - - // // Bucket name - // "bucket": "test-bucket", - - // // Endpoint (optional) - // // When using S3-compatible cloud storage, specify hostname of service endpoint here, - // // region is ignored if endpoint is set (set region to some human-readable name) - // // (should be left blank for real Amazon S3) - // "endpoint": "", - - // // Prefix (optional) - // // publishing under specified prefix in the bucket, defaults to - // // no prefix (bucket root) - // "prefix": "", - - // // Default ACLs (optional) - // // assign ACL to published files (one of the canned ACLs in Amazon - // // terminology). Useful values: `private` (default), `public-read` (public - // // repository) or `none` (don't set ACL). Public repositories could be consumed by `apt` using - // // HTTP endpoint (Amazon bucket should be configured for "website hosting"), - // // for private repositories special apt S3 transport is required. - // "acl": "private", - - // // Credentials (optional) - // // Amazon credentials to access S3 bucket. If not supplied, - // // environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` - // // are used. - // "awsAccessKeyID": "", - // "awsSecretAccessKey": "", - - // // Storage Class (optional) - // // Amazon S3 storage class, defaults to `STANDARD`. Other values - // // available: `REDUCED_REDUNDANCY` (lower price, lower redundancy) - // "storageClass": "STANDARD", - - // // Encryption Method (optional) - // // Server-side encryption method, defaults to none. Currently - // // the only available encryption method is `AES256` - // "encryptionMethod": "none", - - // // Plus Workaround (optional) - // // Workaround misbehavior in apt and Amazon S3 for files with `+` in filename by - // // creating two copies of package files with `+` in filename: one original - // // and another one with spaces instead of plus signs - // // With `plusWorkaround` enabled, package files with plus sign - // // would be stored twice. aptly might not cleanup files with spaces when published - // // repository is dropped or updated (switched) to new version of repository (snapshot) - // "plusWorkaround": false, - - // // Disable MultiDel (optional) - // // For S3-compatible cloud storages which do not support `MultiDel` S3 API, - // // enable this setting (file deletion would be slower with this setting enabled) - // "disableMultiDel": false, - - // // ForceSig2 (optional) - // // Disable Signature V4 support, useful with non-AWS S3-compatible object stores - // // which do not support SigV4, shouldn't be enabled for AWS - // "forceSigV2": false, - - // // ForceVirtualHostedStyle (optional) - // // Disable path style visit, useful with non-AWS S3-compatible object stores - // // which only support virtual hosted style - // "forceVirtualHostedStyle": false, - - // // Debug (optional) - // // Enables detailed request/response dump for each S3 operation - // "debug": false - // } - }, - - // Swift Endpoint Support - // - // aptly could be configured to publish repository directly to OpenStack Swift. First, - // publishing endpoints should be described in aptly configuration file. Each endpoint - // has name and associated settings. - // - // In order to publish to Swift, specify endpoint as `swift:endpoint-name:` before - // publishing prefix on the command line, e.g.: - // - // `aptly publish snapshot jessie-main swift:test:` - // - "SwiftPublishEndpoints": { - // Endpoint Name - // "test": { - - // // Container Name - // "container": "taylor1", - - // // Prefix (optional) - // // Publish under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials (optional) - // // OpenStack credentials to access Keystone. If not supplied, environment variables `OS_USERNAME` and `OS_PASSWORD` are used - // "osname": "", - // "password": "", - - // // Tenant (optional) - // // OpenStack tenant name and id (in order to use v2 authentication) - // "tenant": "", - // "tenantid": "", - - // // Auth URL (optional) - // // Full url of Keystone server (including port, and version). - // // Example `http://identity.example.com:5000/v2.0` - // "authurl": "" - // } - }, - - // Azure Endpoint Support - // - // aptly can be configured to publish repositories directly to Microsoft Azure Blob - // Storage. First, publishing endpoints should be described in the aptly - // configuration file. Each endpoint has its name and associated settings. - "AzurePublishEndpoints": { - // // Endpoint Name - // "test": { - - // // Container Name - // "container": "container1", - - // // Prefix (optional) - // // Publishing under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials - // // Azure storage account access key to access blob storage - // "accountName": "", - // "accountKey": "", - - // // Endpoint URL - // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://.blob.core.windows.net" - // "endpoint": "" - // } - }, - - // Package Pool - // Location for storing downloaded packages - // Type must be one of: - // * local - // * azure - "packagePoolStorage": { - // Local Pool - "type": "local", - // Local Pool Path - // empty path defaults to `rootDir`/pool - "path": "" - - // // Azure Azure Blob Storage Pool - // "type": "azure", - // "container": "pool1", - - // // Prefix (optional) - // // Publishing under specified prefix in the container, defaults to no prefix (container root) - // "prefix": "", - - // // Credentials - // // Azure storage account access key to access blob storage - // "accountName": "", - // "accountKey": "", - - // // Endpoint URL - // // See: Azure documentation https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string - // // defaults to "https://.blob.core.windows.net" - // "endpoint": "" - } - -// End of config -} diff --git a/system/t02_config/config.py b/system/t02_config/config.py index 240e59446..4bee72d81 100644 --- a/system/t02_config/config.py +++ b/system/t02_config/config.py @@ -64,3 +64,11 @@ class ConfigShowTest(BaseTest): """ runCmd = ["aptly", "config", "show"] gold_processor = BaseTest.expand_environ + + +class ConfigShowYAMLTest(BaseTest): + """ + config showing + """ + runCmd = ["aptly", "config", "show", "-yaml"] + gold_processor = BaseTest.expand_environ diff --git a/utils/config.go b/utils/config.go index 4665c8307..da7dfcd6a 100644 --- a/utils/config.go +++ b/utils/config.go @@ -8,54 +8,72 @@ import ( "strings" "github.com/DisposaBoy/JsonConfigReader" + "gopkg.in/yaml.v3" ) // ConfigStructure is structure of main configuration type ConfigStructure struct { // nolint: maligned - RootDir string `json:"rootDir"` - DownloadConcurrency int `json:"downloadConcurrency"` - DownloadLimit int64 `json:"downloadSpeedLimit"` - DownloadRetries int `json:"downloadRetries"` - Downloader string `json:"downloader"` - DatabaseOpenAttempts int `json:"databaseOpenAttempts"` - Architectures []string `json:"architectures"` - DepFollowSuggests bool `json:"dependencyFollowSuggests"` - DepFollowRecommends bool `json:"dependencyFollowRecommends"` - DepFollowAllVariants bool `json:"dependencyFollowAllVariants"` - DepFollowSource bool `json:"dependencyFollowSource"` - DepVerboseResolve bool `json:"dependencyVerboseResolve"` - GpgDisableSign bool `json:"gpgDisableSign"` - GpgDisableVerify bool `json:"gpgDisableVerify"` - GpgProvider string `json:"gpgProvider"` - DownloadSourcePackages bool `json:"downloadSourcePackages"` - PackagePoolStorage PackagePoolStorage `json:"packagePoolStorage"` - SkipLegacyPool bool `json:"skipLegacyPool"` - PpaDistributorID string `json:"ppaDistributorID"` - PpaCodename string `json:"ppaCodename"` - SkipContentsPublishing bool `json:"skipContentsPublishing"` - SkipBz2Publishing bool `json:"skipBz2Publishing"` - FileSystemPublishRoots map[string]FileSystemPublishRoot `json:"FileSystemPublishEndpoints"` - S3PublishRoots map[string]S3PublishRoot `json:"S3PublishEndpoints"` - SwiftPublishRoots map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints"` - AzurePublishRoots map[string]AzureEndpoint `json:"AzurePublishEndpoints"` - AsyncAPI bool `json:"AsyncAPI"` - EnableMetricsEndpoint bool `json:"enableMetricsEndpoint"` - LogLevel string `json:"logLevel"` - LogFormat string `json:"logFormat"` - ServeInAPIMode bool `json:"serveInAPIMode"` - DatabaseBackend DBConfig `json:"databaseBackend"` - EnableSwaggerEndpoint bool `json:"enableSwaggerEndpoint"` + // General + RootDir string `json:"rootDir" yaml:"root_dir"` + LogLevel string `json:"logLevel" yaml:"log_level"` + LogFormat string `json:"logFormat" yaml:"log_format"` + DatabaseOpenAttempts int `json:"databaseOpenAttempts" yaml:"database_open_attempts"` + Architectures []string `json:"architectures" yaml:"architectures"` + SkipLegacyPool bool `json:"skipLegacyPool" yaml:"skip_legacy_pool"` // OBSOLETE + + // Dependency following + DepFollowSuggests bool `json:"dependencyFollowSuggests" yaml:"dep_follow_suggests"` + DepFollowRecommends bool `json:"dependencyFollowRecommends" yaml:"dep_follow_recommends"` + DepFollowAllVariants bool `json:"dependencyFollowAllVariants" yaml:"dep_follow_all_variants"` + DepFollowSource bool `json:"dependencyFollowSource" yaml:"dep_follow_source"` + DepVerboseResolve bool `json:"dependencyVerboseResolve" yaml:"dep_verboseresolve"` + + // PPA + PpaDistributorID string `json:"ppaDistributorID" yaml:"ppa_distributor_id"` + PpaCodename string `json:"ppaCodename" yaml:"ppa_codename"` + + // Server + ServeInAPIMode bool `json:"serveInAPIMode" yaml:"serve_in_api_mode"` + EnableMetricsEndpoint bool `json:"enableMetricsEndpoint" yaml:"enable_metrics_endpoint"` + EnableSwaggerEndpoint bool `json:"enableSwaggerEndpoint" yaml:"enable_swagger_endpoint"` + AsyncAPI bool `json:"AsyncAPI" yaml:"async_api"` // OBSOLETE + + // Database + DatabaseBackend DBConfig `json:"databaseBackend" yaml:"database_backend"` + + // Mirroring + Downloader string `json:"downloader" yaml:"downloader"` + DownloadConcurrency int `json:"downloadConcurrency" yaml:"download_concurrency"` + DownloadLimit int64 `json:"downloadSpeedLimit" yaml:"download_limit"` + DownloadRetries int `json:"downloadRetries" yaml:"download_retries"` + DownloadSourcePackages bool `json:"downloadSourcePackages" yaml:"download_sourcepackages"` + + // Signing + GpgProvider string `json:"gpgProvider" yaml:"gpg_provider"` + GpgDisableSign bool `json:"gpgDisableSign" yaml:"gpg_disable_sign"` + GpgDisableVerify bool `json:"gpgDisableVerify" yaml:"gpg_disable_verify"` + + // Publishing + SkipContentsPublishing bool `json:"skipContentsPublishing" yaml:"skip_contents_publishing"` + SkipBz2Publishing bool `json:"skipBz2Publishing" yaml:"skip_bz2_publishing"` + + // Storage + FileSystemPublishRoots map[string]FileSystemPublishRoot `json:"FileSystemPublishEndpoints" yaml:"filesystem_publish_endpoints"` + S3PublishRoots map[string]S3PublishRoot `json:"S3PublishEndpoints" yaml:"s3_publish_endpoints"` + SwiftPublishRoots map[string]SwiftPublishRoot `json:"SwiftPublishEndpoints" yaml:"swift_publish_endpoints"` + AzurePublishRoots map[string]AzureEndpoint `json:"AzurePublishEndpoints" yaml:"azure_publish_endpoints"` + PackagePoolStorage PackagePoolStorage `json:"packagePoolStorage" yaml:"packagepool_storage"` } // DBConfig type DBConfig struct { - Type string `json:"type"` - URL string `json:"url"` - DbPath string `json:"dbPath"` + Type string `json:"type" yaml:"type"` + DbPath string `json:"dbPath" yaml:"db_path"` + URL string `json:"url" yaml:"url"` } type LocalPoolStorage struct { - Path string `json:"path,omitempty"` + Path string `json:"path,omitempty" yaml:"path,omitempty"` } type PackagePoolStorage struct { @@ -63,6 +81,9 @@ type PackagePoolStorage struct { Azure *AzureEndpoint } +var AZURE = "azure" +var LOCAL = "local" + func (pool *PackagePoolStorage) UnmarshalJSON(data []byte) error { var discriminator struct { Type string `json:"type"` @@ -73,10 +94,10 @@ func (pool *PackagePoolStorage) UnmarshalJSON(data []byte) error { } switch discriminator.Type { - case "azure": + case AZURE: pool.Azure = &AzureEndpoint{} return json.Unmarshal(data, &pool.Azure) - case "local", "": + case LOCAL, "": pool.Local = &LocalPoolStorage{} return json.Unmarshal(data, &pool.Local) default: @@ -84,6 +105,26 @@ func (pool *PackagePoolStorage) UnmarshalJSON(data []byte) error { } } +func (pool *PackagePoolStorage) UnmarshalYAML(unmarshal func(interface{}) error) error { + var discriminator struct { + Type string `yaml:"type"` + } + if err := unmarshal(&discriminator); err != nil { + return err + } + + switch discriminator.Type { + case AZURE: + pool.Azure = &AzureEndpoint{} + return unmarshal(&pool.Azure) + case LOCAL, "": + pool.Local = &LocalPoolStorage{} + return unmarshal(&pool.Local) + default: + return fmt.Errorf("unknown pool storage type: %s", discriminator.Type) + } +} + func (pool *PackagePoolStorage) MarshalJSON() ([]byte, error) { var wrapper struct { Type string `json:"type,omitempty"` @@ -102,54 +143,72 @@ func (pool *PackagePoolStorage) MarshalJSON() ([]byte, error) { return json.Marshal(wrapper) } +func (pool PackagePoolStorage) MarshalYAML() (interface{}, error) { + var wrapper struct { + Type string `yaml:"type,omitempty"` + *LocalPoolStorage `yaml:",inline"` + *AzureEndpoint `yaml:",inline"` + } + + if pool.Azure != nil { + wrapper.Type = "azure" + wrapper.AzureEndpoint = pool.Azure + } else if pool.Local.Path != "" { + wrapper.Type = "local" + wrapper.LocalPoolStorage = pool.Local + } + + return wrapper, nil +} + // FileSystemPublishRoot describes single filesystem publishing entry point type FileSystemPublishRoot struct { - RootDir string `json:"rootDir"` - LinkMethod string `json:"linkMethod"` - VerifyMethod string `json:"verifyMethod"` + RootDir string `json:"rootDir" yaml:"root_dir"` + LinkMethod string `json:"linkMethod" yaml:"link_method"` + VerifyMethod string `json:"verifyMethod" yaml:"verify_method"` } // S3PublishRoot describes single S3 publishing entry point type S3PublishRoot struct { - Region string `json:"region"` - Bucket string `json:"bucket"` - Endpoint string `json:"endpoint"` - AccessKeyID string `json:"awsAccessKeyID"` - SecretAccessKey string `json:"awsSecretAccessKey"` - SessionToken string `json:"awsSessionToken"` - Prefix string `json:"prefix"` - ACL string `json:"acl"` - StorageClass string `json:"storageClass"` - EncryptionMethod string `json:"encryptionMethod"` - PlusWorkaround bool `json:"plusWorkaround"` - DisableMultiDel bool `json:"disableMultiDel"` - ForceSigV2 bool `json:"forceSigV2"` - ForceVirtualHostedStyle bool `json:"forceVirtualHostedStyle"` - Debug bool `json:"debug"` + Region string `json:"region" yaml:"region"` + Bucket string `json:"bucket" yaml:"bucket"` + Prefix string `json:"prefix" yaml:"prefix"` + ACL string `json:"acl" yaml:"acl"` + AccessKeyID string `json:"awsAccessKeyID" yaml:"access_key_id"` + SecretAccessKey string `json:"awsSecretAccessKey" yaml:"secret_access_key"` + SessionToken string `json:"awsSessionToken" yaml:"session_token"` + Endpoint string `json:"endpoint" yaml:"endpoint"` + StorageClass string `json:"storageClass" yaml:"storage_class"` + EncryptionMethod string `json:"encryptionMethod" yaml:"encryption_method"` + PlusWorkaround bool `json:"plusWorkaround" yaml:"plus_workaround"` + DisableMultiDel bool `json:"disableMultiDel" yaml:"disable_multidel"` + ForceSigV2 bool `json:"forceSigV2" yaml:"force_sigv2"` + ForceVirtualHostedStyle bool `json:"forceVirtualHostedStyle" yaml:"force_virtualhosted_style"` + Debug bool `json:"debug" yaml:"debug"` } // SwiftPublishRoot describes single OpenStack Swift publishing entry point type SwiftPublishRoot struct { - UserName string `json:"osname"` - Password string `json:"password"` - AuthURL string `json:"authurl"` - Tenant string `json:"tenant"` - TenantID string `json:"tenantid"` - Domain string `json:"domain"` - DomainID string `json:"domainid"` - TenantDomain string `json:"tenantdomain"` - TenantDomainID string `json:"tenantdomainid"` - Prefix string `json:"prefix"` - Container string `json:"container"` + Container string `json:"container" yaml:"container"` + Prefix string `json:"prefix" yaml:"prefix"` + UserName string `json:"osname" yaml:"username"` + Password string `json:"password" yaml:"password"` + Tenant string `json:"tenant" yaml:"tenant"` + TenantID string `json:"tenantid" yaml:"tenant_id"` + Domain string `json:"domain" yaml:"domain"` + DomainID string `json:"domainid" yaml:"domain_id"` + TenantDomain string `json:"tenantdomain" yaml:"tenant_domain"` + TenantDomainID string `json:"tenantdomainid" yaml:"tenant_domain_id"` + AuthURL string `json:"authurl" yaml:"auth_url"` } // AzureEndpoint describes single Azure publishing entry point type AzureEndpoint struct { - AccountName string `json:"accountName"` - AccountKey string `json:"accountKey"` - Container string `json:"container"` - Prefix string `json:"prefix"` - Endpoint string `json:"endpoint"` + Container string `json:"container" yaml:"container"` + Prefix string `json:"prefix" yaml:"prefix"` + AccountName string `json:"accountName" yaml:"account_name"` + AccountKey string `json:"accountKey" yaml:"account_key"` + Endpoint string `json:"endpoint" yaml:"endpoint"` } // Config is configuration for aptly, shared by all modules @@ -180,7 +239,7 @@ var Config = ConfigStructure{ AzurePublishRoots: map[string]AzureEndpoint{}, AsyncAPI: false, EnableMetricsEndpoint: false, - LogLevel: "debug", + LogLevel: "info", LogFormat: "default", ServeInAPIMode: false, EnableSwaggerEndpoint: false, @@ -194,8 +253,17 @@ func LoadConfig(filename string, config *ConfigStructure) error { } defer f.Close() - dec := json.NewDecoder(JsonConfigReader.New(f)) - return dec.Decode(&config) + decJSON := json.NewDecoder(JsonConfigReader.New(f)) + if err = decJSON.Decode(&config); err != nil { + f.Seek(0, 0) + decYAML := yaml.NewDecoder(f) + if err2 := decYAML.Decode(&config); err2 != nil { + err = fmt.Errorf("invalid yaml (%s) or json (%s)", err2, err) + } else { + err = nil + } + } + return err } // SaveConfig write configuration to json file @@ -227,6 +295,23 @@ func SaveConfigRaw(filename string, conf []byte) error { return err } +// SaveConfigYAML write configuration to yaml file +func SaveConfigYAML(filename string, config *ConfigStructure) error { + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + yamlData, err := yaml.Marshal(&config) + if err != nil { + return fmt.Errorf("error marshaling to YAML: %s", err) + } + + _, err = f.Write(yamlData) + return err +} + // GetRootDir returns the RootDir with expanded ~ as home directory func (conf *ConfigStructure) GetRootDir() string { return strings.Replace(conf.RootDir, "~", os.Getenv("HOME"), 1) diff --git a/utils/config_test.go b/utils/config_test.go index ddfa4ba91..66f1ad03e 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -14,11 +14,14 @@ type ConfigSuite struct { var _ = Suite(&ConfigSuite{}) func (s *ConfigSuite) TestLoadConfig(c *C) { - configname := filepath.Join(c.MkDir(), "aptly.json") + configname := filepath.Join(c.MkDir(), "aptly.json1") f, _ := os.Create(configname) f.WriteString(configFile) f.Close() + // start with empty config + s.config = ConfigStructure{} + err := LoadConfig(configname, &s.config) c.Assert(err, IsNil) c.Check(s.config.GetRootDir(), Equals, "/opt/aptly/") @@ -27,7 +30,10 @@ func (s *ConfigSuite) TestLoadConfig(c *C) { } func (s *ConfigSuite) TestSaveConfig(c *C) { - configname := filepath.Join(c.MkDir(), "aptly.json") + configname := filepath.Join(c.MkDir(), "aptly.json2") + + // start with empty config + s.config = ConfigStructure{} s.config.RootDir = "/tmp/aptly" s.config.DownloadConcurrency = 5 @@ -63,94 +69,305 @@ func (s *ConfigSuite) TestSaveConfig(c *C) { f.Read(buf) c.Check(string(buf), Equals, ""+ - "{\n"+ - " \"rootDir\": \"/tmp/aptly\",\n"+ - " \"downloadConcurrency\": 5,\n"+ - " \"downloadSpeedLimit\": 0,\n"+ - " \"downloadRetries\": 0,\n"+ - " \"downloader\": \"\",\n"+ - " \"databaseOpenAttempts\": 5,\n"+ - " \"architectures\": null,\n"+ - " \"dependencyFollowSuggests\": false,\n"+ - " \"dependencyFollowRecommends\": false,\n"+ - " \"dependencyFollowAllVariants\": false,\n"+ - " \"dependencyFollowSource\": false,\n"+ - " \"dependencyVerboseResolve\": false,\n"+ - " \"gpgDisableSign\": false,\n"+ - " \"gpgDisableVerify\": false,\n"+ - " \"gpgProvider\": \"gpg\",\n"+ - " \"downloadSourcePackages\": false,\n"+ - " \"packagePoolStorage\": {\n"+ - " \"type\": \"local\",\n"+ - " \"path\": \"/tmp/aptly-pool\"\n"+ - " },\n"+ - " \"skipLegacyPool\": false,\n"+ - " \"ppaDistributorID\": \"\",\n"+ - " \"ppaCodename\": \"\",\n"+ - " \"skipContentsPublishing\": false,\n"+ - " \"skipBz2Publishing\": false,\n"+ - " \"FileSystemPublishEndpoints\": {\n"+ - " \"test\": {\n"+ - " \"rootDir\": \"/opt/aptly-publish\",\n"+ - " \"linkMethod\": \"\",\n"+ - " \"verifyMethod\": \"\"\n"+ - " }\n"+ - " },\n"+ - " \"S3PublishEndpoints\": {\n"+ - " \"test\": {\n"+ - " \"region\": \"us-east-1\",\n"+ - " \"bucket\": \"repo\",\n"+ - " \"endpoint\": \"\",\n"+ - " \"awsAccessKeyID\": \"\",\n"+ - " \"awsSecretAccessKey\": \"\",\n"+ - " \"awsSessionToken\": \"\",\n"+ - " \"prefix\": \"\",\n"+ - " \"acl\": \"\",\n"+ - " \"storageClass\": \"\",\n"+ - " \"encryptionMethod\": \"\",\n"+ - " \"plusWorkaround\": false,\n"+ - " \"disableMultiDel\": false,\n"+ - " \"forceSigV2\": false,\n"+ - " \"forceVirtualHostedStyle\": false,\n"+ - " \"debug\": false\n"+ - " }\n"+ - " },\n"+ - " \"SwiftPublishEndpoints\": {\n"+ - " \"test\": {\n"+ - " \"osname\": \"\",\n"+ - " \"password\": \"\",\n"+ - " \"authurl\": \"\",\n"+ - " \"tenant\": \"\",\n"+ - " \"tenantid\": \"\",\n"+ - " \"domain\": \"\",\n"+ - " \"domainid\": \"\",\n"+ - " \"tenantdomain\": \"\",\n"+ - " \"tenantdomainid\": \"\",\n"+ - " \"prefix\": \"\",\n"+ - " \"container\": \"repo\"\n"+ - " }\n"+ - " },\n"+ - " \"AzurePublishEndpoints\": {\n"+ - " \"test\": {\n"+ - " \"accountName\": \"\",\n"+ - " \"accountKey\": \"\",\n"+ - " \"container\": \"repo\",\n"+ - " \"prefix\": \"\",\n"+ - " \"endpoint\": \"\"\n"+ - " }\n"+ - " },\n"+ - " \"AsyncAPI\": false,\n"+ - " \"enableMetricsEndpoint\": false,\n"+ - " \"logLevel\": \"info\",\n"+ - " \"logFormat\": \"json\",\n"+ - " \"serveInAPIMode\": false,\n"+ - " \"databaseBackend\": {\n"+ - " \"type\": \"\",\n"+ - " \"url\": \"\",\n"+ - " \"dbPath\": \"\"\n" + - " },\n"+ - " \"enableSwaggerEndpoint\": false\n" + + "{\n" + + " \"rootDir\": \"/tmp/aptly\",\n" + + " \"logLevel\": \"info\",\n" + + " \"logFormat\": \"json\",\n" + + " \"databaseOpenAttempts\": 5,\n" + + " \"architectures\": null,\n" + + " \"skipLegacyPool\": false,\n" + + " \"dependencyFollowSuggests\": false,\n" + + " \"dependencyFollowRecommends\": false,\n" + + " \"dependencyFollowAllVariants\": false,\n" + + " \"dependencyFollowSource\": false,\n" + + " \"dependencyVerboseResolve\": false,\n" + + " \"ppaDistributorID\": \"\",\n" + + " \"ppaCodename\": \"\",\n" + + " \"serveInAPIMode\": false,\n" + + " \"enableMetricsEndpoint\": false,\n" + + " \"enableSwaggerEndpoint\": false,\n" + + " \"AsyncAPI\": false,\n" + + " \"databaseBackend\": {\n" + + " \"type\": \"\",\n" + + " \"dbPath\": \"\",\n" + + " \"url\": \"\"\n" + + " },\n" + + " \"downloader\": \"\",\n" + + " \"downloadConcurrency\": 5,\n" + + " \"downloadSpeedLimit\": 0,\n" + + " \"downloadRetries\": 0,\n" + + " \"downloadSourcePackages\": false,\n" + + " \"gpgProvider\": \"gpg\",\n" + + " \"gpgDisableSign\": false,\n" + + " \"gpgDisableVerify\": false,\n" + + " \"skipContentsPublishing\": false,\n" + + " \"skipBz2Publishing\": false,\n" + + " \"FileSystemPublishEndpoints\": {\n" + + " \"test\": {\n" + + " \"rootDir\": \"/opt/aptly-publish\",\n" + + " \"linkMethod\": \"\",\n" + + " \"verifyMethod\": \"\"\n" + + " }\n" + + " },\n" + + " \"S3PublishEndpoints\": {\n" + + " \"test\": {\n" + + " \"region\": \"us-east-1\",\n" + + " \"bucket\": \"repo\",\n" + + " \"prefix\": \"\",\n" + + " \"acl\": \"\",\n" + + " \"awsAccessKeyID\": \"\",\n" + + " \"awsSecretAccessKey\": \"\",\n" + + " \"awsSessionToken\": \"\",\n" + + " \"endpoint\": \"\",\n" + + " \"storageClass\": \"\",\n" + + " \"encryptionMethod\": \"\",\n" + + " \"plusWorkaround\": false,\n" + + " \"disableMultiDel\": false,\n" + + " \"forceSigV2\": false,\n" + + " \"forceVirtualHostedStyle\": false,\n" + + " \"debug\": false\n" + + " }\n" + + " },\n" + + " \"SwiftPublishEndpoints\": {\n" + + " \"test\": {\n" + + " \"container\": \"repo\",\n" + + " \"prefix\": \"\",\n" + + " \"osname\": \"\",\n" + + " \"password\": \"\",\n" + + " \"tenant\": \"\",\n" + + " \"tenantid\": \"\",\n" + + " \"domain\": \"\",\n" + + " \"domainid\": \"\",\n" + + " \"tenantdomain\": \"\",\n" + + " \"tenantdomainid\": \"\",\n" + + " \"authurl\": \"\"\n" + + " }\n" + + " },\n" + + " \"AzurePublishEndpoints\": {\n" + + " \"test\": {\n" + + " \"container\": \"repo\",\n" + + " \"prefix\": \"\",\n" + + " \"accountName\": \"\",\n" + + " \"accountKey\": \"\",\n" + + " \"endpoint\": \"\"\n" + + " }\n" + + " },\n" + + " \"packagePoolStorage\": {\n" + + " \"type\": \"local\",\n" + + " \"path\": \"/tmp/aptly-pool\"\n" + + " }\n" + "}") } +func (s *ConfigSuite) TestLoadYAMLConfig(c *C) { + configname := filepath.Join(c.MkDir(), "aptly.yaml1") + f, _ := os.Create(configname) + f.WriteString(configFileYAML) + f.Close() + + // start with empty config + s.config = ConfigStructure{} + + err := LoadConfig(configname, &s.config) + c.Assert(err, IsNil) + c.Check(s.config.GetRootDir(), Equals, "/opt/aptly/") + c.Check(s.config.DownloadConcurrency, Equals, 40) + c.Check(s.config.DatabaseOpenAttempts, Equals, 10) +} + +func (s *ConfigSuite) TestLoadYAMLErrorConfig(c *C) { + configname := filepath.Join(c.MkDir(), "aptly.yaml2") + f, _ := os.Create(configname) + f.WriteString(configFileYAMLError) + f.Close() + + // start with empty config + s.config = ConfigStructure{} + + err := LoadConfig(configname, &s.config) + c.Assert(err.Error(), Equals, "invalid yaml (unknown pool storage type: invalid) or json (invalid character 'p' looking for beginning of value)") +} + +func (s *ConfigSuite) TestSaveYAMLConfig(c *C) { + configname := filepath.Join(c.MkDir(), "aptly.yaml3") + f, _ := os.Create(configname) + f.WriteString(configFileYAML) + f.Close() + + // start with empty config + s.config = ConfigStructure{} + + err := LoadConfig(configname, &s.config) + c.Assert(err, IsNil) + + err = SaveConfigYAML(configname, &s.config) + c.Assert(err, IsNil) + + f, _ = os.Open(configname) + defer f.Close() + + st, _ := f.Stat() + buf := make([]byte, st.Size()) + f.Read(buf) + + c.Check(string(buf), Equals, configFileYAML) +} + +func (s *ConfigSuite) TestSaveYAML2Config(c *C) { + // start with empty config + s.config = ConfigStructure{} + + s.config.PackagePoolStorage.Local = &LocalPoolStorage{"/tmp/aptly-pool"} + s.config.PackagePoolStorage.Azure = nil + + configname := filepath.Join(c.MkDir(), "aptly.yaml4") + err := SaveConfigYAML(configname, &s.config) + c.Assert(err, IsNil) + + f, _ := os.Open(configname) + defer f.Close() + + st, _ := f.Stat() + buf := make([]byte, st.Size()) + f.Read(buf) + + c.Check(string(buf), Equals, "" + + "root_dir: \"\"\n" + + "log_level: \"\"\n" + + "log_format: \"\"\n" + + "database_open_attempts: 0\n" + + "architectures: []\n" + + "skip_legacy_pool: false\n" + + "dep_follow_suggests: false\n" + + "dep_follow_recommends: false\n" + + "dep_follow_all_variants: false\n" + + "dep_follow_source: false\n" + + "dep_verboseresolve: false\n" + + "ppa_distributor_id: \"\"\n" + + "ppa_codename: \"\"\n" + + "serve_in_api_mode: false\n" + + "enable_metrics_endpoint: false\n" + + "enable_swagger_endpoint: false\n" + + "async_api: false\n" + + "database_backend:\n" + + " type: \"\"\n" + + " db_path: \"\"\n" + + " url: \"\"\n" + + "downloader: \"\"\n" + + "download_concurrency: 0\n" + + "download_limit: 0\n" + + "download_retries: 0\n" + + "download_sourcepackages: false\n" + + "gpg_provider: \"\"\n" + + "gpg_disable_sign: false\n" + + "gpg_disable_verify: false\n" + + "skip_contents_publishing: false\n" + + "skip_bz2_publishing: false\n" + + "filesystem_publish_endpoints: {}\n" + + "s3_publish_endpoints: {}\n" + + "swift_publish_endpoints: {}\n" + + "azure_publish_endpoints: {}\n" + + "packagepool_storage:\n" + + " type: local\n" + + " path: /tmp/aptly-pool\n") +} + +func (s *ConfigSuite) TestLoadEmptyConfig(c *C) { + configname := filepath.Join(c.MkDir(), "aptly.yaml5") + f, _ := os.Create(configname) + f.Close() + + // start with empty config + s.config = ConfigStructure{} + + err := LoadConfig(configname, &s.config) + c.Assert(err.Error(), Equals, "invalid yaml (EOF) or json (EOF)") +} + const configFile = `{"rootDir": "/opt/aptly/", "downloadConcurrency": 33, "databaseOpenAttempts": 33}` +const configFileYAML = `root_dir: /opt/aptly/ +log_level: error +log_format: json +database_open_attempts: 10 +architectures: + - amd64 + - arm64 +skip_legacy_pool: true +dep_follow_suggests: true +dep_follow_recommends: true +dep_follow_all_variants: true +dep_follow_source: true +dep_verboseresolve: true +ppa_distributor_id: Ubuntu +ppa_codename: code +serve_in_api_mode: true +enable_metrics_endpoint: true +enable_swagger_endpoint: true +async_api: true +database_backend: + type: etcd + db_path: "" + url: 127.0.0.1:2379 +downloader: grab +download_concurrency: 40 +download_limit: 100 +download_retries: 10 +download_sourcepackages: true +gpg_provider: gpg +gpg_disable_sign: true +gpg_disable_verify: true +skip_contents_publishing: true +skip_bz2_publishing: true +filesystem_publish_endpoints: + test1: + root_dir: /opt/srv/aptly_public + link_method: hardlink + verify_method: md5 +s3_publish_endpoints: + test: + region: us-east-1 + bucket: test-bucket + prefix: prfx + acl: public-read + access_key_id: "2" + secret_access_key: secret + session_token: none + endpoint: endpoint + storage_class: STANDARD + encryption_method: AES256 + plus_workaround: true + disable_multidel: true + force_sigv2: true + force_virtualhosted_style: true + debug: true +swift_publish_endpoints: + test: + container: c1 + prefix: pre + username: user + password: pass + tenant: t + tenant_id: "2" + domain: pop + domain_id: "1" + tenant_domain: td + tenant_domain_id: "3" + auth_url: http://auth.url +azure_publish_endpoints: + test: + container: container1 + prefix: pre2 + account_name: aname + account_key: akey + endpoint: https://end.point +packagepool_storage: + type: azure + container: test-pool1 + prefix: pre3 + account_name: a name + account_key: a key + endpoint: ep +` +const configFileYAMLError = `packagepool_storage: + type: invalid +`