diff --git a/docs/pages/deployment/clustering.rst b/docs/pages/deployment/clustering.rst index a6b05432e..e7520a28f 100644 --- a/docs/pages/deployment/clustering.rst +++ b/docs/pages/deployment/clustering.rst @@ -3,7 +3,6 @@ Clustering ########## -With the introduction of a SQL database and separate session storage, clustering with HA is now possible. Clustering is currently limited to nodes that have the ``did:nuts`` method disabled. To enable clustering, you must support the following: @@ -13,4 +12,10 @@ To enable clustering, you must support the following: - Read only mounts for configuration, policy, discovery and JSON-LD context files. It's recommended to use a level 4 load balancer to distribute the load across the nodes. -Each node should have a reverse proxy for TLS termination. \ No newline at end of file +Each node should have a reverse proxy for TLS termination. + +Clustering will not work if you use one of the following: + +- The did:nuts method +- SQLite +- Disk based private key storage diff --git a/docs/pages/deployment/storage.rst b/docs/pages/deployment/storage.rst index cce565bc1..ef83aa4b3 100644 --- a/docs/pages/deployment/storage.rst +++ b/docs/pages/deployment/storage.rst @@ -48,11 +48,11 @@ Session storage *************** Session storage is used for storing access tokens, nonces and other volatile data. -Data is stored in-memory only. There are 3 supported session storage types: +Session data is volatile by nature. There are 3 supported session storage types: -- local -- memcached -- redis (standalone, cluster, sentinel) +- In-memory +- Memcached +- Redis (standalone, cluster, sentinel) Local ===== @@ -73,6 +73,7 @@ Memcached can be enabled with the following config: You can add multiple memcached servers to the list. memcached is not capable of clustering. Each piece of data is stored on a single instance. If you want true HA, you'll need to use Redis. +For more information on Memcached connection strings, refer to the `Memcached documentation `_. Redis ===== diff --git a/e2e-tests/clustering/memcached/docker-compose.yml b/e2e-tests/clustering/memcached/docker-compose.yml index 3049b59af..bacdc0c1d 100644 --- a/e2e-tests/clustering/memcached/docker-compose.yml +++ b/e2e-tests/clustering/memcached/docker-compose.yml @@ -11,15 +11,18 @@ services: - "18081:8081" environment: NUTS_CONFIGFILE: /opt/nuts/nuts.yaml + NUTS_URL: "https://nodeA" + NUTS_DISCOVERY_SERVER_IDS: "e2e-test" + NUTS_STORAGE_SESSION_MEMCACHED_ADDRESS: "memcached:11211" volumes: - - "./node-A/nuts.yaml:/opt/nuts/nuts.yaml:ro" + - "../shared/nuts.yaml:/opt/nuts/nuts.yaml:ro" - "../../tls-certs/nodeA-backend-certificate.pem:/opt/nuts/certificate-and-key.pem:ro" - "../../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro" # did:web resolver uses the OS CA bundle, but e2e tests use a self-signed CA which can be found in truststore.pem # So we need to mount that file to the OS CA bundle location, otherwise did:web resolving will fail due to untrusted certs. - "../../tls-certs/truststore.pem:/etc/ssl/certs/Nuts_RootCA.pem:ro" - - "./node-A/presentationexchangemapping.json:/opt/nuts/policies/presentationexchangemapping.json:ro" - - "./shared/discovery:/nuts/discovery:ro" + - "../shared/presentationexchangemapping.json:/opt/nuts/policies/presentationexchangemapping.json:ro" + - "../shared/discovery:/nuts/discovery:ro" healthcheck: interval: 1s # Make test run quicker by checking health status more often nodeA: @@ -27,7 +30,7 @@ services: ports: - "10443:443" volumes: - - "./node-A/nginx.conf:/etc/nginx/nginx.conf:ro" + - "../shared/node-A/nginx.conf:/etc/nginx/nginx.conf:ro" - "../../tls-certs/nodeA-certificate.pem:/etc/nginx/ssl/server.pem:ro" - "../../tls-certs/nodeA-certificate.pem:/etc/nginx/ssl/key.pem:ro" - "../../tls-certs/truststore.pem:/etc/nginx/ssl/truststore.pem:ro" @@ -38,15 +41,17 @@ services: - "28081:8081" environment: NUTS_CONFIGFILE: /opt/nuts/nuts.yaml + NUTS_URL: "https://nodeB" volumes: - - "./node-B/nuts.yaml:/opt/nuts/nuts.yaml:ro" + - "../shared/nuts.yaml:/opt/nuts/nuts.yaml:ro" - "../../tls-certs/nodeB-certificate.pem:/opt/nuts/certificate-and-key.pem:ro" - "../../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro" - "../../tls-certs/truststore.pem:/etc/ssl/certs/truststore.pem:ro" # did:web resolver uses the OS CA bundle, but e2e tests use a self-signed CA which can be found in truststore.pem # So we need to mount that file to the OS CA bundle location, otherwise did:web resolving will fail due to untrusted certs. - "../../tls-certs/truststore.pem:/etc/ssl/certs/Nuts_RootCA.pem:ro" - - "./shared/discovery:/nuts/discovery:ro" + - "../shared/presentationexchangemapping.json:/opt/nuts/policies/presentationexchangemapping.json:ro" + - "../shared/discovery:/nuts/discovery:ro" healthcheck: interval: 1s # Make test run quicker by checking health status more often nodeB: diff --git a/e2e-tests/clustering/redis/docker-compose.yml b/e2e-tests/clustering/redis/docker-compose.yml index adb4c75c3..9eca6a6df 100644 --- a/e2e-tests/clustering/redis/docker-compose.yml +++ b/e2e-tests/clustering/redis/docker-compose.yml @@ -45,15 +45,19 @@ services: - "18081:8081" environment: NUTS_CONFIGFILE: /opt/nuts/nuts.yaml + NUTS_URL: "https://nodeA" + NUTS_DISCOVERY_SERVER_IDS: "e2e-test" + NUTS_STORAGE_SESSION_REDIS_SENTINEL_MASTER: "mymaster" + NUTS_STORAGE_SESSION_REDIS_SENTINEL_NODES: sentinelA:26379,sentinelB:26379,sentinelC:26379 volumes: - - "./node-A/nuts.yaml:/opt/nuts/nuts.yaml:ro" + - "../shared/nuts.yaml:/opt/nuts/nuts.yaml:ro" - "../../tls-certs/nodeA-backend-certificate.pem:/opt/nuts/certificate-and-key.pem:ro" - "../../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro" # did:web resolver uses the OS CA bundle, but e2e tests use a self-signed CA which can be found in truststore.pem # So we need to mount that file to the OS CA bundle location, otherwise did:web resolving will fail due to untrusted certs. - "../../tls-certs/truststore.pem:/etc/ssl/certs/Nuts_RootCA.pem:ro" - - "./node-A/presentationexchangemapping.json:/opt/nuts/policies/presentationexchangemapping.json:ro" - - "./shared/discovery:/nuts/discovery:ro" + - "../shared/presentationexchangemapping.json:/opt/nuts/policies/presentationexchangemapping.json:ro" + - "../shared/discovery:/nuts/discovery:ro" healthcheck: interval: 1s # Make test run quicker by checking health status more often nodeA: @@ -61,7 +65,7 @@ services: ports: - "10443:443" volumes: - - "./node-A/nginx.conf:/etc/nginx/nginx.conf:ro" + - "../shared/node-A/nginx.conf:/etc/nginx/nginx.conf:ro" - "../../tls-certs/nodeA-certificate.pem:/etc/nginx/ssl/server.pem:ro" - "../../tls-certs/nodeA-certificate.pem:/etc/nginx/ssl/key.pem:ro" - "../../tls-certs/truststore.pem:/etc/nginx/ssl/truststore.pem:ro" @@ -72,15 +76,17 @@ services: - "28081:8081" environment: NUTS_CONFIGFILE: /opt/nuts/nuts.yaml + NUTS_URL: "https://nodeB" volumes: - - "./node-B/nuts.yaml:/opt/nuts/nuts.yaml:ro" + - "../shared/nuts.yaml:/opt/nuts/nuts.yaml:ro" - "../../tls-certs/nodeB-certificate.pem:/opt/nuts/certificate-and-key.pem:ro" - "../../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro" - "../../tls-certs/truststore.pem:/etc/ssl/certs/truststore.pem:ro" # did:web resolver uses the OS CA bundle, but e2e tests use a self-signed CA which can be found in truststore.pem # So we need to mount that file to the OS CA bundle location, otherwise did:web resolving will fail due to untrusted certs. - "../../tls-certs/truststore.pem:/etc/ssl/certs/Nuts_RootCA.pem:ro" - - "./shared/discovery:/nuts/discovery:ro" + - "../shared/presentationexchangemapping.json:/opt/nuts/policies/presentationexchangemapping.json:ro" + - "../shared/discovery:/nuts/discovery:ro" healthcheck: interval: 1s # Make test run quicker by checking health status more often nodeB: diff --git a/e2e-tests/clustering/redis/node-A/nginx.conf b/e2e-tests/clustering/redis/node-A/nginx.conf deleted file mode 100644 index f3a537298..000000000 --- a/e2e-tests/clustering/redis/node-A/nginx.conf +++ /dev/null @@ -1,71 +0,0 @@ -load_module /usr/lib/nginx/modules/ngx_http_js_module.so; - -user nginx; -worker_processes 1; - -error_log /var/log/nginx/error.log debug; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - js_import oauth2.js; - include /etc/nginx/mime.types; - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - keepalive_timeout 65; - - include /etc/nginx/conf.d/*.conf; - - upstream nodeA-internal { - server nodeA-backend:8081; - } - upstream nodeA-external { - server nodeA-backend:8080; - } - - server { - server_name nodeA; - listen 443 ssl; - http2 on; - ssl_certificate /etc/nginx/ssl/server.pem; - ssl_certificate_key /etc/nginx/ssl/key.pem; - ssl_client_certificate /etc/nginx/ssl/truststore.pem; - ssl_verify_client optional; - ssl_verify_depth 1; - ssl_protocols TLSv1.3; - - location / { - proxy_set_header X-Ssl-Client-Cert $ssl_client_escaped_cert; - proxy_pass http://nodeA-external; - } - - # check access via token introspection as described by https://www.nginx.com/blog/validating-oauth-2-0-access-tokens-nginx/ - location /resource { - js_content oauth2.introspectAccessToken; - } - - # Location in javascript subrequest. - # this is needed to set headers and method - location /_oauth2_send_request { - internal; - proxy_method POST; - proxy_set_header Content-Type "application/x-www-form-urlencoded"; - proxy_pass http://nodeA-internal/internal/auth/v2/accesstoken/introspect; - } - location /_dpop_send_request { - internal; - proxy_method POST; - proxy_set_header Content-Type "application/json"; - proxy_pass http://nodeA-internal/internal/auth/v2/dpop/validate; - } - } -} diff --git a/e2e-tests/clustering/redis/node-A/presentationexchangemapping.json b/e2e-tests/clustering/redis/node-A/presentationexchangemapping.json deleted file mode 100644 index 992655c56..000000000 --- a/e2e-tests/clustering/redis/node-A/presentationexchangemapping.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "test": { - "organization": { - "format": { - "ldp_vp": { - "proof_type": [ - "JsonWebSignature2020" - ] - }, - "ldp_vc": { - "proof_type": [ - "JsonWebSignature2020" - ] - } - }, - "id": "pd_any_care_organization_with_employee", - "name": "Care organization with employee", - "purpose": "Finding a care organization with logged in user for authorizing access to medical metadata", - "input_descriptors": [ - { - "id": "id_nuts_care_organization_cred", - "constraints": { - "fields": [ - { - "path": [ - "$.type" - ], - "filter": { - "type": "string", - "const": "NutsOrganizationCredential" - } - }, - { - "path": [ - "$.credentialSubject.organization.name" - ], - "filter": { - "type": "string" - } - }, - { - "path": [ - "$.credentialSubject.organization.city" - ], - "filter": { - "type": "string" - } - } - ] - } - }, - { - "id": "id_employee_credential_cred", - "constraints": { - "fields": [ - { - "path": [ - "$.type" - ], - "filter": { - "type": "string", - "const": "EmployeeCredential" - } - }, - { - "id": "employee_identifier", - "path": [ - "$.credentialSubject.identifier", - "$.credentialSubject[0].identifier" - ], - "filter": { - "type": "string" - } - }, - { - "id": "employee_name", - "path": [ - "$.credentialSubject.name", - "$.credentialSubject[0].name" - ], - "filter": { - "type": "string" - } - }, - { - "id": "employee_role", - "path": [ - "$.credentialSubject.roleName", - "$.credentialSubject[0].roleName" - ], - "filter": { - "type": "string" - } - } - ] - } - } - ] - } - } -} diff --git a/e2e-tests/clustering/redis/sentinel/A/sentinel.conf b/e2e-tests/clustering/redis/sentinel/A/sentinel.conf index 52d2f15a5..82ba5197f 100644 --- a/e2e-tests/clustering/redis/sentinel/A/sentinel.conf +++ b/e2e-tests/clustering/redis/sentinel/A/sentinel.conf @@ -12,3 +12,16 @@ sentinel resolve-hostnames yes sentinel announce-hostnames yes # Generated by CONFIG REWRITE + +latency-tracking-info-percentiles 50 99 99.9 +user default on nopass sanitize-payload ~* &* +@all +sentinel myid d2edd2fd2762267af463b772e543253e17b00ec9 +sentinel config-epoch mymaster 0 +sentinel leader-epoch mymaster 0 +sentinel current-epoch 0 + +sentinel known-replica mymaster 172.16.4.7 6379 + +sentinel known-sentinel mymaster 172.16.4.9 26379 acdeeaa3cb7c53bf8965ba3046d1f212ef2bd342 + +sentinel known-sentinel mymaster 172.16.4.10 26379 bd75ae05071650f196c7ecdffb4abea1271c1e27 diff --git a/e2e-tests/clustering/redis/sentinel/B/sentinel.conf b/e2e-tests/clustering/redis/sentinel/B/sentinel.conf index 52d2f15a5..526321f5f 100644 --- a/e2e-tests/clustering/redis/sentinel/B/sentinel.conf +++ b/e2e-tests/clustering/redis/sentinel/B/sentinel.conf @@ -12,3 +12,16 @@ sentinel resolve-hostnames yes sentinel announce-hostnames yes # Generated by CONFIG REWRITE + +latency-tracking-info-percentiles 50 99 99.9 +user default on nopass sanitize-payload ~* &* +@all +sentinel myid acdeeaa3cb7c53bf8965ba3046d1f212ef2bd342 +sentinel config-epoch mymaster 0 +sentinel leader-epoch mymaster 0 +sentinel current-epoch 0 + +sentinel known-replica mymaster 172.16.4.7 6379 + +sentinel known-sentinel mymaster 172.16.4.10 26379 bd75ae05071650f196c7ecdffb4abea1271c1e27 + +sentinel known-sentinel mymaster 172.16.4.8 26379 d2edd2fd2762267af463b772e543253e17b00ec9 diff --git a/e2e-tests/clustering/redis/sentinel/C/sentinel.conf b/e2e-tests/clustering/redis/sentinel/C/sentinel.conf index 52d2f15a5..d153289db 100644 --- a/e2e-tests/clustering/redis/sentinel/C/sentinel.conf +++ b/e2e-tests/clustering/redis/sentinel/C/sentinel.conf @@ -12,3 +12,16 @@ sentinel resolve-hostnames yes sentinel announce-hostnames yes # Generated by CONFIG REWRITE + +latency-tracking-info-percentiles 50 99 99.9 +user default on nopass sanitize-payload ~* &* +@all +sentinel myid bd75ae05071650f196c7ecdffb4abea1271c1e27 +sentinel config-epoch mymaster 0 +sentinel leader-epoch mymaster 0 +sentinel current-epoch 0 + +sentinel known-replica mymaster 172.16.4.7 6379 + +sentinel known-sentinel mymaster 172.16.4.9 26379 acdeeaa3cb7c53bf8965ba3046d1f212ef2bd342 + +sentinel known-sentinel mymaster 172.16.4.8 26379 d2edd2fd2762267af463b772e543253e17b00ec9 diff --git a/e2e-tests/clustering/redis/shared/discovery/definition.json b/e2e-tests/clustering/redis/shared/discovery/definition.json deleted file mode 100644 index f43a88422..000000000 --- a/e2e-tests/clustering/redis/shared/discovery/definition.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "id": "e2e-test", - "endpoint": "http://nodeA-backend:8080/discovery/e2e-test", - "presentation_max_validity": 36000, - "presentation_definition": { - "id": "pd_eoverdracht_dev_care_organization", - "format": { - "ldp_vc": { - "proof_type": [ - "JsonWebSignature2020" - ] - } - }, - "input_descriptors": [ - { - "id": "id_nuts_care_organization_cred", - "constraints": { - "fields": [ - { - "path": [ - "$.type" - ], - "filter": { - "type": "string", - "const": "NutsOrganizationCredential" - } - }, - { - "path": [ - "$.credentialSubject.organization.name", - "$.credentialSubject[0].organization.name" - ], - "filter": { - "type": "string" - } - }, - { - "path": [ - "$.credentialSubject.organization.city", - "$.credentialSubject[0].organization.city" - ], - "filter": { - "type": "string" - } - } - ] - } - } - ] - } -} diff --git a/e2e-tests/clustering/memcached/shared/discovery/definition.json b/e2e-tests/clustering/shared/discovery/definition.json similarity index 100% rename from e2e-tests/clustering/memcached/shared/discovery/definition.json rename to e2e-tests/clustering/shared/discovery/definition.json diff --git a/e2e-tests/clustering/memcached/node-A/nginx.conf b/e2e-tests/clustering/shared/node-A/nginx.conf similarity index 100% rename from e2e-tests/clustering/memcached/node-A/nginx.conf rename to e2e-tests/clustering/shared/node-A/nginx.conf diff --git a/e2e-tests/clustering/memcached/node-A/presentationexchangemapping.json b/e2e-tests/clustering/shared/presentationexchangemapping.json similarity index 100% rename from e2e-tests/clustering/memcached/node-A/presentationexchangemapping.json rename to e2e-tests/clustering/shared/presentationexchangemapping.json