From 865d95e59a8dc948b24a9abb82779fc9b3fc0730 Mon Sep 17 00:00:00 2001 From: Gerard Snaauw <33763579+gerardsn@users.noreply.github.com> Date: Fri, 17 Nov 2023 09:18:21 +0100 Subject: [PATCH 01/23] Discovery: SQL based server implementation --- README.rst | 2 +- discoveryservice/api/v1/api.go | 29 ++ discoveryservice/client.go | 280 +++++++++++ discoveryservice/client_test.go | 457 ++++++++++++++++++ discoveryservice/config.go | 50 ++ discoveryservice/definition.go | 68 +++ discoveryservice/interface.go | 42 ++ discoveryservice/log/logger.go | 31 ++ discoveryservice/mock.go | 107 ++++ discoveryservice/module.go | 232 +++++++++ discoveryservice/module_test.go | 272 +++++++++++ .../service-definition-schema.json | 28 ++ discoveryservice/store.go | 230 +++++++++ discoveryservice/store_test.go | 78 +++ discoveryservice/test.go | 211 ++++++++ discoveryservice/test/duplicate_id/1.json | 49 ++ discoveryservice/test/duplicate_id/2.json | 49 ++ discoveryservice/test/duplicate_id/README.md | 1 + .../test/invalid_definition/1.json | 3 + .../test/invalid_definition/README.md | 1 + discoveryservice/test/invalid_json/1.json | 1 + discoveryservice/test/invalid_json/README.md | 1 + discoveryservice/test/valid/eoverdracht.json | 49 ++ discoveryservice/test/valid/subdir/README.md | 1 + discoveryservice/test/valid/subdir/empty.json | 2 + docs/pages/deployment/cli-reference.rst | 2 +- docs/pages/deployment/server_options.rst | 2 +- go.mod | 1 + makefile | 2 + storage/mock.go | 28 +- .../2_discoveryservice.down.sql | 4 + .../sql_migrations/2_discoveryservice.up.sql | 49 ++ storage/test.go | 1 + vcr/credential/resolver.go | 48 ++ vcr/pe/presentation_definition.go | 3 +- vcr/pe/schema/v2/schema.go | 12 +- vcr/pe/test/definition_mapping.json | 10 + 37 files changed, 2413 insertions(+), 23 deletions(-) create mode 100644 discoveryservice/api/v1/api.go create mode 100644 discoveryservice/client.go create mode 100644 discoveryservice/client_test.go create mode 100644 discoveryservice/config.go create mode 100644 discoveryservice/definition.go create mode 100644 discoveryservice/interface.go create mode 100644 discoveryservice/log/logger.go create mode 100644 discoveryservice/mock.go create mode 100644 discoveryservice/module.go create mode 100644 discoveryservice/module_test.go create mode 100644 discoveryservice/service-definition-schema.json create mode 100644 discoveryservice/store.go create mode 100644 discoveryservice/store_test.go create mode 100644 discoveryservice/test.go create mode 100644 discoveryservice/test/duplicate_id/1.json create mode 100644 discoveryservice/test/duplicate_id/2.json create mode 100644 discoveryservice/test/duplicate_id/README.md create mode 100644 discoveryservice/test/invalid_definition/1.json create mode 100644 discoveryservice/test/invalid_definition/README.md create mode 100644 discoveryservice/test/invalid_json/1.json create mode 100644 discoveryservice/test/invalid_json/README.md create mode 100644 discoveryservice/test/valid/eoverdracht.json create mode 100644 discoveryservice/test/valid/subdir/README.md create mode 100644 discoveryservice/test/valid/subdir/empty.json create mode 100644 storage/sql_migrations/2_discoveryservice.down.sql create mode 100644 storage/sql_migrations/2_discoveryservice.up.sql diff --git a/README.rst b/README.rst index f6316b814b..3f908d240f 100644 --- a/README.rst +++ b/README.rst @@ -224,7 +224,7 @@ The following options can be configured on the server: http.default.auth.type Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode. http.default.cors.origin [] When set, enables CORS from the specified origins on the default HTTP interface. **JSONLD** - jsonld.contexts.localmapping [https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson,https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. + jsonld.contexts.localmapping [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. jsonld.contexts.remoteallowlist [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json] In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. **Network** network.bootstrapnodes [] List of bootstrap nodes (':') which the node initially connect to. diff --git a/discoveryservice/api/v1/api.go b/discoveryservice/api/v1/api.go new file mode 100644 index 0000000000..2a90d4c29f --- /dev/null +++ b/discoveryservice/api/v1/api.go @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package v1 + +//// lists maps a list name (last path part of use case endpoint) to the list ID +//lists map[string]string +//// name is derived from endpoint: it's the last path part of the definition endpoint +//// It is used to route HTTP GET requests to the correct list. +//pathParts := strings.Split(definition.Endpoint, "/") +//name := pathParts[len(pathParts)-1] +//if name == "" { +//return nil, fmt.Errorf("can't derive list name from definition endpoint: %s", definition.Endpoint) +//} diff --git a/discoveryservice/client.go b/discoveryservice/client.go new file mode 100644 index 0000000000..568f6becef --- /dev/null +++ b/discoveryservice/client.go @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package discoveryservice + +// +//import ( +// "encoding/json" +// "errors" +// "fmt" +// "github.com/google/uuid" +// "github.com/nuts-foundation/go-did/vc" +// "github.com/nuts-foundation/nuts-node/discoveryservice/log" +// "gorm.io/gorm" +// "gorm.io/gorm/clause" +// "io" +// "net/http" +// "net/url" +// "strconv" +// "strings" +// "sync" +// "time" +//) +// +//func newClient(db *gorm.DB, definitions map[string]Definition) (*client, error) { +// result := &client{ +// db: db, +// definitions: definitions, +// } +// if err := initializeSQLStore(db, definitions); err != nil { +// return nil, err +// } +// return result, nil +//} +// +//type client struct { +// db *gorm.DB +// definitions map[string]Definition +//} +// +//func (c *client) Search(serviceID string, query map[string]string) ([]vc.VerifiablePresentation, error) { +// propertyColumns := map[string]string{ +// "id": "cred.credential_id", +// "issuer": "cred.credential_issuer", +// "type": "cred.credential_type", +// "credentialSubject.id": "cred.credential_subject_id", +// } +// +// stmt := c.db.Model(&entry{}). +// Where("usecase_id = ?", serviceID). +// Joins("inner join usecase_client_credential cred ON cred.entry_id = usecase_client_entries.id") +// numProps := 0 +// for jsonPath, value := range query { +// if value == "*" { +// continue +// } +// // sort out wildcard mode +// var eq = "=" +// if strings.HasPrefix(value, "*") { +// value = "%" + value[1:] +// eq = "LIKE" +// } +// if strings.HasSuffix(value, "*") { +// value = value[:len(value)-1] + "%" +// eq = "LIKE" +// } +// if column := propertyColumns[jsonPath]; column != "" { +// stmt = stmt.Where(column+" "+eq+" ?", value) +// } else { +// // This property is not present as column, but indexed as key-value property. +// // Multiple (inner) joins to filter on a dynamic number of properties to filter on is not pretty, but it works +// alias := "p" + strconv.Itoa(numProps) +// numProps++ +// stmt = stmt.Joins("inner join usecase_client_credential_props "+alias+" ON "+alias+".id = cred.id AND "+alias+".key = ? AND "+alias+".value "+eq+" ?", jsonPath, value) +// } +// } +// +// var matches []entry +// if err := stmt.Find(&matches).Error; err != nil { +// return nil, err +// } +// var results []vc.VerifiablePresentation +// for _, match := range matches { +// if match.PresentationExpiration <= time.Now().Unix() { +// continue +// } +// presentation, err := vc.ParseVerifiablePresentation(match.PresentationRaw) +// if err != nil { +// return nil, fmt.Errorf("failed to parse presentation '%s': %w", match.PresentationID, err) +// } +// results = append(results, *presentation) +// } +// return results, nil +//} +// +//func (c *client) refreshAll() { +// wg := &sync.WaitGroup{} +// for _, definition := range c.definitions { +// wg.Add(1) +// go func(definition Definition) { +// c.refreshList(definition) +// }(definition) +// } +// wg.Done() +//} +// +//func (c *client) refreshList(definition Definition) error { +// var currentService discoveryService +// if err := c.db.Find(¤tService, "usecase_id = ?", definition.ID).Error; errors.Is(err, gorm.ErrRecordNotFound) { +// // First refresh of the list +// if err := c.db.Create(&discoveryService{ID: definition.ID}).Error; err != nil { +// return err +// } +// } else if err != nil { +// // Other error +// return err +// } +// log.Logger().Debugf("Refreshing use case list %s", definition.ID) +// // replace with generated client later +// requestURL, _ := url.Parse(definition.Endpoint) +// requestURL.Query().Add("timestamp", fmt.Sprintf("%d", currentService.Timestamp)) +// httpResponse, err := http.Get(definition.Endpoint) +// if err != nil { +// return err +// } +// data, err := io.ReadAll(httpResponse.Body) +// if err != nil { +// return err +// } +// var response ListResponse +// if err = json.Unmarshal(data, &response); err != nil { +// return err +// } +// return c.applyDelta(currentService.UsecaseID, response.Entries, response.Tombstone, currentService.Timestamp, response.Timestamp) +//} +// +//// applyDelta applies the updateTimestamp, retrieved from the use case list server, to the local index of the use case lists. +//func (c *client) applyDelta(usecaseID string, presentations []vc.VerifiablePresentation, tombstoneSet []string, previousTimestamp uint64, timestamp uint64) error { +// // TODO: validate presentations +// if previousTimestamp == timestamp { +// // nothing to do +// return nil +// } +// // We use a transaction to make sure the complete updateTimestamp is applied, or nothing at all. +// // Use a lock on the list to make sure there are no concurrent updates being applied to the list, +// // which could lead to the client becoming out-of-sync with the server list. +// // This situation can only really occur in a distributed system (multiple nodes updating the same list at the same time, with a different timestamp), +// // or bug in the updateTimestamp scheduler. +// return c.db.Transaction(func(tx *gorm.DB) error { +// // Lock the list, check if we're applying the delta to the right starting point +// var currentList list +// if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). +// Where("usecase_id = ?", usecaseID). +// Find(¤tList). +// Error; err != nil { +// return err +// } +// // Make sure we don't apply stale data +// if currentList.Timestamp != previousTimestamp { +// log.Logger().Infof("Not applying delta to use case list '%s': timestamp mismatch (expected %d but was %d). "+ +// "Probably caused by multiple processes updating the list. This is not a problem/bug: stale data should be updated at next refresh.", usecaseID, previousTimestamp, currentList.Timestamp) +// return nil +// } +// // Now we can apply the delta: +// // - delete removed presentations +// // - add new presentations +// // - index the presentations' properties +// if len(tombstoneSet) > 0 { +// if err := tx.Delete(&entry{}, "usecase_id = ? AND presentation_id IN ?", usecaseID, tombstoneSet).Error; err != nil { +// return fmt.Errorf("failed to delete tombstone records: %w", err) +// } +// } +// for _, presentation := range presentations { +// err := c.writePresentation(tx, usecaseID, presentation) +// if err != nil { +// return err +// } +// } +// // Finally, updateTimestamp the list timestamp +// if err := tx.Model(&list{}).Where("usecase_id = ?", usecaseID).Update("timestamp", timestamp).Error; err != nil { +// return fmt.Errorf("failed to updateTimestamp timestamp: %w", err) +// } +// return nil +// }) +//} +// +//func (c *client) writePresentation(tx *gorm.DB, usecaseID string, presentation vc.VerifiablePresentation) error { +// entryID := uuid.NewString() +// // Store list entry / verifiable presentation +// newEntry := entry{ +// ID: entryID, +// UsecaseID: usecaseID, +// PresentationID: presentation.ID.String(), +// PresentationRaw: presentation.Raw(), +// PresentationExpiration: presentation.JWT().Expiration().Unix(), +// } +// // Store the credentials of the presentation +// for _, curr := range presentation.VerifiableCredential { +// var credentialType *string +// for _, currType := range curr.Type { +// if currType.String() != "VerifiableCredential" { +// credentialType = new(string) +// *credentialType = currType.String() +// break +// } +// } +// subjectDID, err := curr.SubjectDID() +// if err != nil { +// return fmt.Errorf("invalid credential subject ID for VP '%s': %w", presentation.ID, err) +// } +// credentialRecordID := uuid.NewString() +// cred := credential{ +// ID: credentialRecordID, +// EntryID: entryID, +// CredentialID: curr.ID.String(), +// CredentialIssuer: curr.Issuer.String(), +// CredentialSubjectID: subjectDID.String(), +// CredentialType: credentialType, +// } +// if len(curr.CredentialSubject) != 1 { +// return errors.New("credential must contain exactly one subject") +// } +// // Store credential properties +// keys, values := indexJSONObject(curr.CredentialSubject[0].(map[string]interface{}), nil, nil, "credentialSubject") +// for i, key := range keys { +// if key == "credentialSubject.id" { +// // present as column, don't index +// continue +// } +// cred.Properties = append(cred.Properties, credentialProperty{ +// ID: credentialRecordID, +// Key: key, +// Value: values[i], +// }) +// } +// newEntry.Credentials = append(newEntry.Credentials, cred) +// } +// if err := tx.Create(&newEntry).Error; err != nil { +// return fmt.Errorf("failed to create entry: %w", err) +// } +// return nil +//} +// +//// indexJSONObject indexes a JSON object, resulting in a slice of JSON paths and corresponding string values. +//// It only traverses JSON objects and only adds string values to the result. +//func indexJSONObject(target map[string]interface{}, jsonPaths []string, stringValues []string, currentPath string) ([]string, []string) { +// for key, value := range target { +// thisPath := currentPath +// if len(thisPath) > 0 { +// thisPath += "." +// } +// thisPath += key +// +// switch typedValue := value.(type) { +// case string: +// jsonPaths = append(jsonPaths, thisPath) +// stringValues = append(stringValues, typedValue) +// case map[string]interface{}: +// jsonPaths, stringValues = indexJSONObject(typedValue, jsonPaths, stringValues, thisPath) +// default: +// // other values (arrays, booleans, numbers, null) are not indexed +// } +// } +// return jsonPaths, stringValues +//} diff --git a/discoveryservice/client_test.go b/discoveryservice/client_test.go new file mode 100644 index 0000000000..160bd7b200 --- /dev/null +++ b/discoveryservice/client_test.go @@ -0,0 +1,457 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package discoveryservice + +// +//import ( +// "context" +// "crypto/ecdsa" +// "crypto/elliptic" +// "crypto/rand" +// "fmt" +// "github.com/google/uuid" +// "github.com/lestrrat-go/jwx/v2/jwa" +// "github.com/lestrrat-go/jwx/v2/jwk" +// "github.com/lestrrat-go/jwx/v2/jws" +// "github.com/lestrrat-go/jwx/v2/jwt" +// ssi "github.com/nuts-foundation/go-did" +// "github.com/nuts-foundation/go-did/did" +// "github.com/nuts-foundation/go-did/vc" +// "github.com/nuts-foundation/nuts-node/storage" +// "github.com/stretchr/testify/require" +// "gorm.io/gorm/schema" +// "testing" +// "time" +//) +// +//func Test_client_applyDelta(t *testing.T) { +// //storageEngine := storage.New() +// //storageEngine.(core.Injectable).Config().(*storage.Config).SQL = storage.SQLConfig{ConnectionString: "file:../../data/sqlite.db"} +// //require.NoError(t, storageEngine.Configure(core.TestServerConfig(core.ServerConfig{Datadir: "data"}))) +// //require.NoError(t, storageEngine.Start()) +// +// storageEngine := storage.NewTestStorageEngine(t) +// require.NoError(t, storageEngine.Start()) +// t.Cleanup(func() { +// _ = storageEngine.Shutdown() +// }) +// +// t.Run("fresh list, assert all persisted fields", func(t *testing.T) { +// c := setupClient(t, storageEngine) +// err := c.applyDelta(TestDefinition.ID, []vc.VerifiablePresentation{vpAlice, vpBob}, []string{"other", "and another"}, 0, 1000) +// require.NoError(t, err) +// +// var actualList list +// require.NoError(t, c.db.Find(&actualList, "usecase_id = ?", TestDefinition.ID).Error) +// require.Equal(t, TestDefinition.ID, actualList.UsecaseID) +// require.Equal(t, uint64(1000), actualList.Timestamp) +// +// var entries []entry +// require.NoError(t, c.db.Find(&entries, "usecase_id = ?", TestDefinition.ID).Error) +// require.Len(t, entries, 2) +// require.Equal(t, vpAlice.ID.String(), entries[0].PresentationID) +// require.Equal(t, vpBob.ID.String(), entries[1].PresentationID) +// }) +//} +// +//func Test_client_writePresentation(t *testing.T) { +// storageEngine := storage.NewTestStorageEngine(t) +// require.NoError(t, storageEngine.Start()) +// t.Cleanup(func() { +// _ = storageEngine.Shutdown() +// }) +// +// t.Run("1 credential", func(t *testing.T) { +// c := setupClient(t, storageEngine) +// err := c.writePresentation(c.db, TestDefinition.ID, vpAlice) +// require.NoError(t, err) +// +// var entries []entry +// require.NoError(t, c.db.Find(&entries, "usecase_id = ?", TestDefinition.ID).Error) +// require.Len(t, entries, 1) +// require.Equal(t, vpAlice.ID.String(), entries[0].PresentationID) +// require.Equal(t, vpAlice.Raw(), entries[0].PresentationRaw) +// require.Equal(t, vpAlice.JWT().Expiration().Unix(), entries[0].PresentationExpiration) +// +// var credentials []credential +// require.NoError(t, c.db.Find(&credentials, "entry_id = ?", entries[0].ID).Error) +// require.Len(t, credentials, 1) +// cred := credentials[0] +// require.Equal(t, vcAlice.ID.String(), cred.CredentialID) +// require.Equal(t, vcAlice.Issuer.String(), cred.CredentialIssuer) +// require.Equal(t, aliceDID.String(), cred.CredentialSubjectID) +// require.Equal(t, vcAlice.Type[1].String(), *cred.CredentialType) +// +// expectedProperties := map[string]map[string]string{ +// cred.ID: { +// "credentialSubject.person.givenName": "Alice", +// "credentialSubject.person.familyName": "Jones", +// "credentialSubject.person.city": "InfoSecLand", +// }, +// } +// for recordID, properties := range expectedProperties { +// for key, value := range properties { +// var prop credentialProperty +// require.NoError(t, c.db.Find(&prop, "id = ? AND key = ?", recordID, key).Error) +// require.Equal(t, value, prop.Value) +// } +// } +// }) +//} +// +//func Test_client_search(t *testing.T) { +// storageEngine := storage.NewTestStorageEngine(t) +// require.NoError(t, storageEngine.Start()) +// t.Cleanup(func() { +// _ = storageEngine.Shutdown() +// }) +// +// type testCase struct { +// name string +// inputVPs []vc.VerifiablePresentation +// query map[string]string +// expectedVPs []string +// } +// testCases := []testCase{ +// { +// name: "issuer", +// inputVPs: []vc.VerifiablePresentation{vpAlice}, +// query: map[string]string{ +// "issuer": authorityDID.String(), +// }, +// expectedVPs: []string{vpAlice.ID.String()}, +// }, +// { +// name: "id", +// inputVPs: []vc.VerifiablePresentation{vpAlice}, +// query: map[string]string{ +// "id": vcAlice.ID.String(), +// }, +// expectedVPs: []string{vpAlice.ID.String()}, +// }, +// { +// name: "type", +// inputVPs: []vc.VerifiablePresentation{vpAlice}, +// query: map[string]string{ +// "type": "TestCredential", +// }, +// expectedVPs: []string{vpAlice.ID.String()}, +// }, +// { +// name: "credentialSubject.id", +// inputVPs: []vc.VerifiablePresentation{vpAlice}, +// query: map[string]string{ +// "credentialSubject.id": aliceDID.String(), +// }, +// expectedVPs: []string{vpAlice.ID.String()}, +// }, +// { +// name: "1 property", +// inputVPs: []vc.VerifiablePresentation{vpAlice}, +// query: map[string]string{ +// "credentialSubject.person.givenName": "Alice", +// }, +// expectedVPs: []string{vpAlice.ID.String()}, +// }, +// { +// name: "2 properties", +// inputVPs: []vc.VerifiablePresentation{vpAlice}, +// query: map[string]string{ +// "credentialSubject.person.givenName": "Alice", +// "credentialSubject.person.familyName": "Jones", +// }, +// expectedVPs: []string{vpAlice.ID.String()}, +// }, +// { +// name: "properties and base properties", +// inputVPs: []vc.VerifiablePresentation{vpAlice}, +// query: map[string]string{ +// "issuer": authorityDID.String(), +// "credentialSubject.person.givenName": "Alice", +// }, +// expectedVPs: []string{vpAlice.ID.String()}, +// }, +// { +// name: "wildcard postfix", +// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, +// query: map[string]string{ +// "credentialSubject.person.familyName": "Jo*", +// }, +// expectedVPs: []string{vpAlice.ID.String(), vpBob.ID.String()}, +// }, +// { +// name: "wildcard prefix", +// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, +// query: map[string]string{ +// "credentialSubject.person.givenName": "*ce", +// }, +// expectedVPs: []string{vpAlice.ID.String()}, +// }, +// { +// name: "wildcard midway (no interpreted as wildcard)", +// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, +// query: map[string]string{ +// "credentialSubject.person.givenName": "A*ce", +// }, +// expectedVPs: []string{}, +// }, +// { +// name: "just wildcard", +// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, +// query: map[string]string{ +// "id": "*", +// }, +// expectedVPs: []string{vpAlice.ID.String(), vpBob.ID.String()}, +// }, +// { +// name: "2 VPs, 1 match", +// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, +// query: map[string]string{ +// "credentialSubject.person.givenName": "Alice", +// }, +// expectedVPs: []string{vpAlice.ID.String()}, +// }, +// { +// name: "multiple matches", +// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, +// query: map[string]string{ +// "issuer": authorityDID.String(), +// }, +// expectedVPs: []string{vpAlice.ID.String(), vpBob.ID.String()}, +// }, +// { +// name: "no match", +// inputVPs: []vc.VerifiablePresentation{vpAlice}, +// query: map[string]string{ +// "credentialSubject.person.givenName": "Bob", +// }, +// expectedVPs: []string{}, +// }, +// { +// name: "empty database", +// query: map[string]string{ +// "credentialSubject.person.givenName": "Bob", +// }, +// expectedVPs: []string{}, +// }, +// } +// +// for _, tc := range testCases { +// t.Run(tc.name, func(t *testing.T) { +// c := setupClient(t, storageEngine) +// for _, vp := range tc.inputVPs { +// err := c.writePresentation(c.db, TestDefinition.ID, vp) +// require.NoError(t, err) +// } +// actualVPs, err := c.Search(TestDefinition.ID, tc.query) +// require.NoError(t, err) +// require.Len(t, actualVPs, len(tc.expectedVPs)) +// for _, expectedVP := range tc.expectedVPs { +// found := false +// for _, actualVP := range actualVPs { +// if actualVP.ID.String() == expectedVP { +// found = true +// break +// } +// } +// require.True(t, found, "expected to find VP with ID %s", expectedVP) +// } +// }) +// } +//} +// +//func setupClient(t *testing.T, storageEngine storage.Engine) *client { +// t.Cleanup(func() { +// underlyingDB, err := storageEngine.GetSQLDatabase().DB() +// require.NoError(t, err) +// tables := []schema.Tabler{ +// &entry{}, +// &credential{}, +// &list{}, +// } +// for _, table := range tables { +// _, err = underlyingDB.Exec("DELETE FROM " + table.TableName()) +// require.NoError(t, err) +// } +// }) +// testDefinitions := map[string]Definition{ +// TestDefinition.ID: TestDefinition, +// } +// +// c, err := newClient(storageEngine.GetSQLDatabase(), testDefinitions) +// require.NoError(t, err) +// return c +//} +// +//var keyPairs map[string]*ecdsa.PrivateKey +//var authorityDID did.DID +//var aliceDID did.DID +//var vcAlice vc.VerifiableCredential +//var vpAlice vc.VerifiablePresentation +//var bobDID did.DID +//var vcBob vc.VerifiableCredential +// +//var vpBob vc.VerifiablePresentation +// +//func init() { +// keyPairs = make(map[string]*ecdsa.PrivateKey) +// authorityDID = did.MustParseDID("did:example:authority") +// keyPairs[authorityDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) +// aliceDID = did.MustParseDID("did:example:alice") +// keyPairs[aliceDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) +// bobDID = did.MustParseDID("did:example:bob") +// keyPairs[bobDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) +// +// vcAlice = createCredentialWithClaims(authorityDID, aliceDID, func() []interface{} { +// return []interface{}{ +// map[string]interface{}{ +// "id": aliceDID.String(), +// "person": map[string]interface{}{ +// "givenName": "Alice", +// "familyName": "Jones", +// "city": "InfoSecLand", +// }, +// }, +// } +// }, func(m map[string]interface{}) { +// // do nothing +// }) +// vpAlice = createPresentation(aliceDID, vcAlice) +// vcBob = createCredentialWithClaims(authorityDID, bobDID, func() []interface{} { +// return []interface{}{ +// map[string]interface{}{ +// "id": aliceDID.String(), +// "person": map[string]interface{}{ +// "givenName": "Bob", +// "familyName": "Johansson", +// "city": "InfoSecLand", +// }, +// }, +// } +// }, func(m map[string]interface{}) { +// // do nothing +// }) +// vpBob = createPresentation(bobDID, vcBob) +//} +// +//func createCredential(issuerDID did.DID, subjectDID did.DID) vc.VerifiableCredential { +// return createCredentialWithClaims(issuerDID, subjectDID, +// func() []interface{} { +// return []interface{}{ +// map[string]interface{}{ +// "id": subjectDID.String(), +// }, +// } +// }, +// func(claims map[string]interface{}) { +// // do nothing +// }) +//} +// +//func createCredentialWithClaims(issuerDID did.DID, subjectDID did.DID, credentialSubjectCreator func() []interface{}, claimVisitor func(map[string]interface{})) vc.VerifiableCredential { +// vcID := did.DIDURL{DID: issuerDID} +// vcID.Fragment = uuid.NewString() +// vcIDURI := vcID.URI() +// expirationDate := time.Now().Add(time.Hour * 24) +// +// result, err := vc.CreateJWTVerifiableCredential(context.Background(), vc.VerifiableCredential{ +// ID: &vcIDURI, +// Issuer: issuerDID.URI(), +// Type: []ssi.URI{ssi.MustParseURI("VerifiableCredential"), ssi.MustParseURI("TestCredential")}, +// IssuanceDate: time.Now(), +// ExpirationDate: &expirationDate, +// CredentialSubject: credentialSubjectCreator(), +// }, func(ctx context.Context, claims map[string]interface{}, headers map[string]interface{}) (string, error) { +// claimVisitor(claims) +// return signJWT(subjectDID, claims, headers) +// }) +// if err != nil { +// panic(err) +// } +// return *result +//} +// +//func createPresentation(subjectDID did.DID, credentials ...vc.VerifiableCredential) vc.VerifiablePresentation { +// return createPresentationCustom(subjectDID, func(claims map[string]interface{}) { +// // do nothing +// }, credentials...) +//} +// +//func createPresentationCustom(subjectDID did.DID, claimVisitor func(map[string]interface{}), credentials ...vc.VerifiableCredential) vc.VerifiablePresentation { +// headers := map[string]interface{}{ +// jws.TypeKey: "JWT", +// } +// claims := map[string]interface{}{ +// jwt.IssuerKey: subjectDID.String(), +// jwt.SubjectKey: subjectDID.String(), +// jwt.JwtIDKey: subjectDID.String() + "#" + uuid.NewString(), +// "vp": vc.VerifiablePresentation{ +// Type: append([]ssi.URI{ssi.MustParseURI("VerifiablePresentation")}), +// VerifiableCredential: credentials, +// }, +// jwt.NotBeforeKey: time.Now().Unix(), +// jwt.ExpirationKey: time.Now().Add(time.Hour * 8), +// } +// claimVisitor(claims) +// token, err := signJWT(subjectDID, claims, headers) +// if err != nil { +// panic(err) +// } +// presentation, err := vc.ParseVerifiablePresentation(token) +// if err != nil { +// panic(err) +// } +// return *presentation +//} +// +//func signJWT(subjectDID did.DID, claims map[string]interface{}, headers map[string]interface{}) (string, error) { +// // Build JWK +// signingKey := keyPairs[subjectDID.String()] +// if signingKey == nil { +// return "", fmt.Errorf("key not found for DID: %s", subjectDID) +// } +// subjectKeyJWK, err := jwk.FromRaw(signingKey) +// if err != nil { +// return "", nil +// } +// keyID := did.DIDURL{DID: subjectDID} +// keyID.Fragment = "0" +// if err := subjectKeyJWK.Set(jwk.AlgorithmKey, jwa.ES256); err != nil { +// return "", err +// } +// if err := subjectKeyJWK.Set(jwk.KeyIDKey, keyID.String()); err != nil { +// return "", err +// } +// +// // Build token +// token := jwt.New() +// for k, v := range claims { +// if err := token.Set(k, v); err != nil { +// return "", err +// } +// } +// hdr := jws.NewHeaders() +// for k, v := range headers { +// if err := hdr.Set(k, v); err != nil { +// return "", err +// } +// } +// bytes, err := jwt.Sign(token, jwt.WithKey(jwa.ES256, signingKey, jws.WithProtectedHeaders(hdr))) +// return string(bytes), err +//} diff --git a/discoveryservice/config.go b/discoveryservice/config.go new file mode 100644 index 0000000000..d2dd701a96 --- /dev/null +++ b/discoveryservice/config.go @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package discoveryservice + +// Config holds the config of the module +type Config struct { + Server ServerConfig `koanf:"server"` + Definitions DefinitionsConfig `koanf:"definitions"` +} + +// DefinitionsConfig holds the config for loading Service Definitions. +type DefinitionsConfig struct { + Directory string `koanf:"directory"` +} + +// ServerConfig holds the config for the server +type ServerConfig struct { + // DefinitionIDs specifies which use case lists the server serves. + DefinitionIDs []string `koanf:"definition_ids"` + // Directory is the directory where the server stores the lists. + Directory string `koanf:"directory"` +} + +// DefaultConfig returns the default configuration. +func DefaultConfig() Config { + return Config{ + Server: ServerConfig{}, + } +} + +// IsServer returns true if the node act as Discovery Server. +func (c Config) IsServer() bool { + return len(c.Server.DefinitionIDs) > 0 +} diff --git a/discoveryservice/definition.go b/discoveryservice/definition.go new file mode 100644 index 0000000000..b9af33283b --- /dev/null +++ b/discoveryservice/definition.go @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package discoveryservice + +import ( + "bytes" + "embed" + "encoding/json" + "github.com/nuts-foundation/nuts-node/vcr/pe" + v2 "github.com/nuts-foundation/nuts-node/vcr/pe/schema/v2" + "github.com/santhosh-tekuri/jsonschema" +) + +//go:embed *.json +var jsonSchemaFiles embed.FS +var definitionJsonSchema *jsonschema.Schema + +func init() { + serviceDefinitionSchemaData, err := jsonSchemaFiles.ReadFile("service-definition-schema.json") + if err != nil { + panic(err) + } + const schemaURL = "http://nuts.nl/schemas/discovery-service-v0.json" + if err := v2.Compiler.AddResource(schemaURL, bytes.NewReader(serviceDefinitionSchemaData)); err != nil { + panic(err) + } + definitionJsonSchema = v2.Compiler.MustCompile(schemaURL) +} + +// Definition holds the definition of a service. +type Definition struct { + // ID is the unique identifier of the use case. + ID string `json:"id"` + // Endpoint is the endpoint where the use case list is served. + Endpoint string `json:"endpoint"` + // PresentationDefinition specifies the Presentation Definition submissions to the list must conform to, + // according to the Presentation Exchange specification. + PresentationDefinition pe.PresentationDefinition `json:"presentation_definition"` + // PresentationMaxValidity specifies how long submitted presentations are allowed to be valid (in seconds). + PresentationMaxValidity int `json:"presentation_max_validity"` +} + +func ParseDefinition(data []byte) (*Definition, error) { + if err := definitionJsonSchema.Validate(bytes.NewReader(data)); err != nil { + return nil, err + } + var definition Definition + if err := json.Unmarshal(data, &definition); err != nil { + return nil, err + } + return &definition, nil +} diff --git a/discoveryservice/interface.go b/discoveryservice/interface.go new file mode 100644 index 0000000000..4a32bcf053 --- /dev/null +++ b/discoveryservice/interface.go @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package discoveryservice + +import ( + "github.com/nuts-foundation/go-did/vc" +) + +// Timestamp is value that references a point in the list. +// It is used by clients to request new entries since their last query. +// It's implemented as lamport timestamp (https://en.wikipedia.org/wiki/Lamport_timestamp); +// it is incremented when a new entry is added to the list. +// Pass 0 to start at the beginning of the list. +type Timestamp uint64 + +type Server interface { + // Add registers a presentation of the given Discovery Service. + // If the presentation is not valid or it does not conform to the Service Definition, it returns an error. + Add(serviceID string, presentation vc.VerifiablePresentation) error + // Get retrieves the presentations for the given service, starting at the given timestamp. + Get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresentation, *Timestamp, error) +} + +type Client interface { + Search(serviceID string, query map[string]string) ([]vc.VerifiablePresentation, error) +} diff --git a/discoveryservice/log/logger.go b/discoveryservice/log/logger.go new file mode 100644 index 0000000000..13914edbaf --- /dev/null +++ b/discoveryservice/log/logger.go @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package log + +import ( + "github.com/nuts-foundation/nuts-node/core" + "github.com/sirupsen/logrus" +) + +var _logger = logrus.StandardLogger().WithField(core.LogFieldModule, "DiscoveryService") + +// Logger returns a logger with the module field set +func Logger() *logrus.Entry { + return _logger +} diff --git a/discoveryservice/mock.go b/discoveryservice/mock.go new file mode 100644 index 0000000000..c0b8b10c9c --- /dev/null +++ b/discoveryservice/mock.go @@ -0,0 +1,107 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: discoveryservice/interface.go +// +// Generated by this command: +// +// mockgen -destination=discoveryservice/mock.go -package=discoveryservice -source=discoveryservice/interface.go +// +// Package discoveryservice is a generated GoMock package. +package discoveryservice + +import ( + reflect "reflect" + + vc "github.com/nuts-foundation/go-did/vc" + gomock "go.uber.org/mock/gomock" +) + +// MockServer is a mock of Server interface. +type MockServer struct { + ctrl *gomock.Controller + recorder *MockServerMockRecorder +} + +// MockServerMockRecorder is the mock recorder for MockServer. +type MockServerMockRecorder struct { + mock *MockServer +} + +// NewMockServer creates a new mock instance. +func NewMockServer(ctrl *gomock.Controller) *MockServer { + mock := &MockServer{ctrl: ctrl} + mock.recorder = &MockServerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockServer) EXPECT() *MockServerMockRecorder { + return m.recorder +} + +// Add mocks base method. +func (m *MockServer) Add(serviceID string, presentation vc.VerifiablePresentation) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Add", serviceID, presentation) + ret0, _ := ret[0].(error) + return ret0 +} + +// Add indicates an expected call of Add. +func (mr *MockServerMockRecorder) Add(serviceID, presentation any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Add", reflect.TypeOf((*MockServer)(nil).Add), serviceID, presentation) +} + +// Get mocks base method. +func (m *MockServer) Get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresentation, *Timestamp, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", serviceID, startAt) + ret0, _ := ret[0].([]vc.VerifiablePresentation) + ret1, _ := ret[1].(*Timestamp) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Get indicates an expected call of Get. +func (mr *MockServerMockRecorder) Get(serviceID, startAt any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockServer)(nil).Get), serviceID, startAt) +} + +// MockClient is a mock of Client interface. +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder +} + +// MockClientMockRecorder is the mock recorder for MockClient. +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance. +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// Search mocks base method. +func (m *MockClient) Search(serviceID string, query map[string]string) ([]vc.VerifiablePresentation, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Search", serviceID, query) + ret0, _ := ret[0].([]vc.VerifiablePresentation) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Search indicates an expected call of Search. +func (mr *MockClientMockRecorder) Search(serviceID, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Search", reflect.TypeOf((*MockClient)(nil).Search), serviceID, query) +} diff --git a/discoveryservice/module.go b/discoveryservice/module.go new file mode 100644 index 0000000000..1a4852cb98 --- /dev/null +++ b/discoveryservice/module.go @@ -0,0 +1,232 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package discoveryservice + +import ( + "errors" + "fmt" + ssi "github.com/nuts-foundation/go-did" + "github.com/nuts-foundation/go-did/did" + "github.com/nuts-foundation/go-did/vc" + "github.com/nuts-foundation/nuts-node/core" + "github.com/nuts-foundation/nuts-node/storage" + "os" + "path" + "strings" + "time" +) + +const ModuleName = "DiscoveryService" + +var ErrServerModeDisabled = errors.New("node is not a discovery server for this service") + +var _ core.Injectable = &Module{} +var _ core.Runnable = &Module{} +var _ core.Configurable = &Module{} +var _ Server = &Module{} + +// var _ Client = &Module{} +var retractionPresentationType = ssi.MustParseURI("RetractedVerifiablePresentation") + +func New(storageInstance storage.Engine) *Module { + return &Module{ + storageInstance: storageInstance, + } +} + +type Module struct { + config Config + storageInstance storage.Engine + store *sqlStore + serverDefinitions map[string]Definition + services map[string]Definition +} + +func (m *Module) Configure(_ core.ServerConfig) error { + if m.config.Definitions.Directory == "" { + return nil + } + var err error + m.services, err = loadDefinitions(m.config.Definitions.Directory) + if err != nil { + return err + } + return nil +} + +func (m *Module) Start() error { + var err error + m.store, err = newSQLStore(m.storageInstance.GetSQLDatabase(), m.services) + if err != nil { + return err + } + if len(m.config.Server.DefinitionIDs) > 0 { + // Get the definitions that are enabled for this server + serverDefinitions := make(map[string]Definition) + for _, definitionID := range m.config.Server.DefinitionIDs { + if definition, exists := m.services[definitionID]; !exists { + return fmt.Errorf("definition '%s' not found", definitionID) + } else { + serverDefinitions[definitionID] = definition + } + } + m.serverDefinitions = serverDefinitions + } + return nil +} + +func (m *Module) Shutdown() error { + return nil +} + +func (m *Module) Name() string { + return ModuleName +} + +func (m *Module) Config() interface{} { + return &m.config +} + +func (m *Module) Add(serviceID string, presentation vc.VerifiablePresentation) error { + definition, exists := m.services[serviceID] + if !exists { + return ErrServiceNotFound + } + if _, isMaintainer := m.serverDefinitions[serviceID]; !isMaintainer { + return ErrServerModeDisabled + } + if presentation.Format() != vc.JWTPresentationProofFormat { + return errors.New("only JWT presentations are supported") + } + // TODO: validate signature + if presentation.ID == nil { + return errors.New("presentation does not have an ID") + } + expiration := presentation.JWT().Expiration() + // VPs should not be valid for too short; unnecessary overhead at the server and clients. + // Also protects against expiration not being set at all. The factor is somewhat arbitrary. + minValidity := float64(definition.PresentationMaxValidity / 4) + if expiration.Sub(time.Now()).Seconds() < minValidity { + return fmt.Errorf("presentation is not valid for long enough (min %s)", time.Duration(minValidity)*time.Second) + } + // VPs should not be valid for too long, as that would prevent the server from pruning them. + if int(expiration.Sub(time.Now()).Seconds()) > definition.PresentationMaxValidity { + return fmt.Errorf("presentation is valid for too long (max %s)", time.Duration(definition.PresentationMaxValidity)*time.Second) + } + + if presentation.IsType(retractionPresentationType) { + return m.addRetraction(definition.ID, presentation) + } else { + return m.addPresentation(definition, presentation) + } +} + +func (m *Module) addPresentation(definition Definition, presentation vc.VerifiablePresentation) error { + // Must contain credentials + if len(presentation.VerifiableCredential) == 0 { + return errors.New("presentation must contain at least one credential") + } + // VP can't be valid longer than the credential it contains + expiration := presentation.JWT().Expiration() + for _, cred := range presentation.VerifiableCredential { + exp := cred.JWT().Expiration() + if !exp.IsZero() && expiration.After(exp) { + return fmt.Errorf("presentation is valid longer than the credential(s) it contains") + } + } + // VP must fulfill the PEX Presentation Definition + creds, _, err := definition.PresentationDefinition.Match(presentation.VerifiableCredential) + if err != nil || len(creds) != len(presentation.VerifiableCredential) { + return fmt.Errorf("presentation does not fulfill Presentation Definition: %w", err) + } + return m.store.add(definition.ID, presentation, nil) +} + +func (m *Module) addRetraction(serviceID string, presentation vc.VerifiablePresentation) error { + // Presentation might be a retraction (deletion of an earlier credential) must contain no credentials, and refer to the VP being retracted by ID. + // If those conditions aren't met, we don't need to register the retraction. + if len(presentation.VerifiableCredential) > 0 { + return errors.New("retraction presentation must not contain credentials") + } + // Check that the retraction refers to a presentation that: + // - is owned by the signer (same DID) + // - exists (if not, it might've already been removed due to expiry, or superseeded by a newer presentation) + var retractJTIString string + if retractJTIRaw, ok := presentation.JWT().Get("retract_jti"); !ok { + return errors.New("retraction presentation does not contain 'retract_jti' claim") + } else { + if retractJTIString, ok = retractJTIRaw.(string); !ok { + return errors.New("retraction presentation 'retract_jti' claim is not a string") + } + } + signer := presentation.JWT().Issuer() + signerDID, err := did.ParseDID(signer) + if err != nil { + return fmt.Errorf("retraction presentation issuer is not a valid DID: %w", err) + } + retractJTI, err := did.ParseDIDURL(retractJTIString) + if err != nil { + return fmt.Errorf("retraction presentation 'retract_jti' claim is not a valid DID URL: %w", err) + } + if !signerDID.Equals(retractJTI.DID) { + return errors.New("retraction presentation 'retract_jti' claim does not match JWT issuer") + } + exists, err := m.store.exists(serviceID, signer, retractJTIString) + if err != nil { + return err + } + if !exists { + return errors.New("retraction presentation refers to a non-existing presentation") + } + return m.store.add(serviceID, presentation, nil) +} + +func (m *Module) Get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresentation, *Timestamp, error) { + if _, exists := m.services[serviceID]; !exists { + return nil, nil, ErrServiceNotFound + } + return m.store.get(serviceID, startAt) +} + +func loadDefinitions(directory string) (map[string]Definition, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, fmt.Errorf("unable to read definitions directory '%s': %w", directory, err) + } + result := make(map[string]Definition) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + filePath := path.Join(directory, entry.Name()) + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("unable to read definition file '%s': %w", filePath, err) + } + definition, err := ParseDefinition(data) + if err != nil { + return nil, fmt.Errorf("unable to parse definition file '%s': %w", filePath, err) + } + if _, exists := result[definition.ID]; exists { + return nil, fmt.Errorf("duplicate definition ID '%s' in file '%s'", definition.ID, filePath) + } + result[definition.ID] = *definition + } + return result, nil +} diff --git a/discoveryservice/module_test.go b/discoveryservice/module_test.go new file mode 100644 index 0000000000..a978da620a --- /dev/null +++ b/discoveryservice/module_test.go @@ -0,0 +1,272 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package discoveryservice + +import ( + "github.com/nuts-foundation/go-did/vc" + "github.com/nuts-foundation/nuts-node/core" + "github.com/nuts-foundation/nuts-node/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +const serviceID = "urn:nuts.nl:usecase:eOverdrachtDev2023" + +func TestModule_Name(t *testing.T) { + assert.Equal(t, "DiscoveryService", (&Module{}).Name()) +} + +func TestModule_Shutdown(t *testing.T) { + assert.NoError(t, (&Module{}).Shutdown()) +} + +func Test_Module_Add(t *testing.T) { + storageEngine := storage.NewTestStorageEngine(t) + require.NoError(t, storageEngine.Start()) + t.Run("ok", func(t *testing.T) { + m := setupModule(t, storageEngine) + + err := m.Add(testServiceID, vpAlice) + assert.NoError(t, err) + + _, timestamp, err := m.Get(testServiceID, 0) + require.NoError(t, err) + assert.Equal(t, Timestamp(1), *timestamp) + }) + t.Run("replace presentation of same credential subject", func(t *testing.T) { + m := setupModule(t, storageEngine) + + vpAlice2 := createPresentation(aliceDID, vcAlice) + assert.NoError(t, m.Add(testServiceID, vpAlice)) + assert.NoError(t, m.Add(testServiceID, vpBob)) + assert.NoError(t, m.Add(testServiceID, vpAlice2)) + + presentations, timestamp, err := m.Get(testServiceID, 0) + require.NoError(t, err) + assert.Equal(t, []vc.VerifiablePresentation{vpBob, vpAlice2}, presentations) + assert.Equal(t, Timestamp(3), *timestamp) + }) + t.Run("already exists", func(t *testing.T) { + m := setupModule(t, storageEngine) + + err := m.Add(testServiceID, vpAlice) + assert.NoError(t, err) + err = m.Add(testServiceID, vpAlice) + assert.EqualError(t, err, "presentation already exists") + }) + t.Run("valid for too long", func(t *testing.T) { + m := setupModule(t, storageEngine) + def := m.services[testServiceID] + def.PresentationMaxValidity = 1 + m.services[testServiceID] = def + + err := m.Add(testServiceID, vpAlice) + assert.EqualError(t, err, "presentation is valid for too long (max 1s)") + }) + t.Run("valid longer than its credentials", func(t *testing.T) { + m := setupModule(t, storageEngine) + + vcAlice := createCredentialWithClaims(authorityDID, aliceDID, func(claims map[string]interface{}) { + claims["exp"] = time.Now().Add(time.Hour) + }) + vpAlice := createPresentation(aliceDID, vcAlice) + err := m.Add(testServiceID, vpAlice) + assert.EqualError(t, err, "presentation is valid longer than the credential(s) it contains") + }) + t.Run("not valid long enough", func(t *testing.T) { + m := setupModule(t, storageEngine) + def := m.services[testServiceID] + def.PresentationMaxValidity = int((24 * time.Hour).Seconds() * 365) + m.services[testServiceID] = def + + err := m.Add(testServiceID, vpAlice) + assert.EqualError(t, err, "presentation is not valid for long enough (min 2190h0m0s)") + }) + t.Run("presentation does not contain an ID", func(t *testing.T) { + m := setupModule(t, storageEngine) + + vpWithoutID := createPresentationCustom(aliceDID, func(claims map[string]interface{}, _ *vc.VerifiablePresentation) { + delete(claims, "jti") + }, vcAlice) + err := m.Add(testServiceID, vpWithoutID) + assert.EqualError(t, err, "presentation does not have an ID") + }) + t.Run("not a JWT", func(t *testing.T) { + m := setupModule(t, storageEngine) + err := m.Add(testServiceID, vc.VerifiablePresentation{}) + assert.EqualError(t, err, "only JWT presentations are supported") + }) + t.Run("service unknown", func(t *testing.T) { + m := setupModule(t, storageEngine) + err := m.Add("unknown", vpAlice) + assert.ErrorIs(t, err, ErrServiceNotFound) + }) + + t.Run("retraction", func(t *testing.T) { + vpAliceRetract := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { + vp.Type = append(vp.Type, retractionPresentationType) + claims["retract_jti"] = vpAlice.ID.String() + }) + t.Run("ok", func(t *testing.T) { + m := setupModule(t, storageEngine) + err := m.Add(testServiceID, vpAlice) + require.NoError(t, err) + err = m.Add(testServiceID, vpAliceRetract) + assert.NoError(t, err) + }) + t.Run("non-existent presentation", func(t *testing.T) { + m := setupModule(t, storageEngine) + err := m.Add(testServiceID, vpAliceRetract) + assert.EqualError(t, err, "retraction presentation refers to a non-existing presentation") + }) + t.Run("must not contain credentials", func(t *testing.T) { + m := setupModule(t, storageEngine) + vp := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { + vp.Type = append(vp.Type, retractionPresentationType) + }, vcAlice) + err := m.Add(testServiceID, vp) + assert.EqualError(t, err, "retraction presentation must not contain credentials") + }) + t.Run("missing 'retract_jti' claim", func(t *testing.T) { + m := setupModule(t, storageEngine) + vp := createPresentationCustom(aliceDID, func(_ map[string]interface{}, vp *vc.VerifiablePresentation) { + vp.Type = append(vp.Type, retractionPresentationType) + }) + err := m.Add(testServiceID, vp) + assert.EqualError(t, err, "retraction presentation does not contain 'retract_jti' claim") + }) + t.Run("'retract_jti' claim in not a string", func(t *testing.T) { + m := setupModule(t, storageEngine) + vp := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { + vp.Type = append(vp.Type, retractionPresentationType) + claims["retract_jti"] = 10 + }) + err := m.Add(testServiceID, vp) + assert.EqualError(t, err, "retraction presentation 'retract_jti' claim is not a string") + }) + t.Run("'retract_jti' claim in not a valid DID", func(t *testing.T) { + m := setupModule(t, storageEngine) + vp := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { + vp.Type = append(vp.Type, retractionPresentationType) + claims["retract_jti"] = "not a DID" + }) + err := m.Add(testServiceID, vp) + assert.EqualError(t, err, "retraction presentation 'retract_jti' claim is not a valid DID URL: invalid DID") + }) + t.Run("'retract_jti' claim does not reference a presentation of the signer", func(t *testing.T) { + m := setupModule(t, storageEngine) + vp := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { + vp.Type = append(vp.Type, retractionPresentationType) + claims["retract_jti"] = bobDID.String() + }) + err := m.Add(testServiceID, vp) + assert.EqualError(t, err, "retraction presentation 'retract_jti' claim does not match JWT issuer") + }) + }) +} + +func Test_Module_Get(t *testing.T) { + storageEngine := storage.NewTestStorageEngine(t) + require.NoError(t, storageEngine.Start()) + + t.Run("empty list, empty timestamp", func(t *testing.T) { + m := setupModule(t, storageEngine) + presentations, timestamp, err := m.Get(testServiceID, 0) + assert.NoError(t, err) + assert.Empty(t, presentations) + assert.Empty(t, timestamp) + }) + t.Run("1 entry, empty timestamp", func(t *testing.T) { + m := setupModule(t, storageEngine) + require.NoError(t, m.Add(testServiceID, vpAlice)) + presentations, timestamp, err := m.Get(testServiceID, 0) + assert.NoError(t, err) + assert.Equal(t, []vc.VerifiablePresentation{vpAlice}, presentations) + assert.Equal(t, Timestamp(1), *timestamp) + }) + t.Run("2 entries, empty timestamp", func(t *testing.T) { + m := setupModule(t, storageEngine) + require.NoError(t, m.Add(testServiceID, vpAlice)) + require.NoError(t, m.Add(testServiceID, vpBob)) + presentations, timestamp, err := m.Get(testServiceID, 0) + assert.NoError(t, err) + assert.Equal(t, []vc.VerifiablePresentation{vpAlice, vpBob}, presentations) + assert.Equal(t, Timestamp(2), *timestamp) + }) + t.Run("2 entries, start after first", func(t *testing.T) { + m := setupModule(t, storageEngine) + require.NoError(t, m.Add(testServiceID, vpAlice)) + require.NoError(t, m.Add(testServiceID, vpBob)) + presentations, timestamp, err := m.Get(testServiceID, 1) + assert.NoError(t, err) + assert.Equal(t, []vc.VerifiablePresentation{vpBob}, presentations) + assert.Equal(t, Timestamp(2), *timestamp) + }) + t.Run("2 entries, start after end", func(t *testing.T) { + m := setupModule(t, storageEngine) + require.NoError(t, m.Add(testServiceID, vpAlice)) + require.NoError(t, m.Add(testServiceID, vpBob)) + presentations, timestamp, err := m.Get(testServiceID, 2) + assert.NoError(t, err) + assert.Equal(t, []vc.VerifiablePresentation{}, presentations) + assert.Equal(t, Timestamp(2), *timestamp) + }) + t.Run("service unknown", func(t *testing.T) { + m := setupModule(t, storageEngine) + _, _, err := m.Get("unknown", 0) + assert.ErrorIs(t, err, ErrServiceNotFound) + }) +} + +func setupModule(t *testing.T, storageInstance storage.Engine) *Module { + resetStoreAfterTest(t, storageInstance.GetSQLDatabase()) + m := New(storageInstance) + require.NoError(t, m.Configure(core.ServerConfig{})) + m.services = testDefinitions() + m.serverDefinitions = map[string]Definition{ + testServiceID: m.services[testServiceID], + } + require.NoError(t, m.Start()) + return m +} + +func Test_loadDefinitions(t *testing.T) { + t.Run("duplicate ID", func(t *testing.T) { + definitions, err := loadDefinitions("test/duplicate_id") + assert.EqualError(t, err, "duplicate definition ID 'urn:nuts.nl:usecase:eOverdrachtDev2023' in file 'test/duplicate_id/2.json'") + assert.Nil(t, definitions) + }) + t.Run("invalid JSON", func(t *testing.T) { + definitions, err := loadDefinitions("test/invalid_json") + assert.ErrorContains(t, err, "unable to parse definition file 'test/invalid_json/1.json'") + assert.Nil(t, definitions) + }) + t.Run("invalid definition", func(t *testing.T) { + definitions, err := loadDefinitions("test/invalid_definition") + assert.ErrorContains(t, err, "unable to parse definition file 'test/invalid_definition/1.json'") + assert.Nil(t, definitions) + }) + t.Run("non-existent directory", func(t *testing.T) { + definitions, err := loadDefinitions("test/non_existent") + assert.ErrorContains(t, err, "unable to read definitions directory 'test/non_existent'") + assert.Nil(t, definitions) + }) +} diff --git a/discoveryservice/service-definition-schema.json b/discoveryservice/service-definition-schema.json new file mode 100644 index 0000000000..b1b3bc177e --- /dev/null +++ b/discoveryservice/service-definition-schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Service Definition", + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "endpoint": { + "type": "string", + "minLength": 1 + }, + "presentation_max_validity": { + "type": "integer", + "minimum": 1 + }, + "presentation_definition": { + "$ref": "http://identity.foundation/presentation-exchange/schemas/presentation-definition.json" + } + }, + "required": [ + "id", + "endpoint", + "presentation_max_validity", + "presentation_definition" + ] +} \ No newline at end of file diff --git a/discoveryservice/store.go b/discoveryservice/store.go new file mode 100644 index 0000000000..dd53898e9a --- /dev/null +++ b/discoveryservice/store.go @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package discoveryservice + +import ( + "errors" + "fmt" + "github.com/google/uuid" + "github.com/nuts-foundation/go-did/vc" + "github.com/nuts-foundation/nuts-node/discoveryservice/log" + credential2 "github.com/nuts-foundation/nuts-node/vcr/credential" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "gorm.io/gorm/schema" + "time" +) + +var ErrServiceNotFound = errors.New("discovery service not found") +var ErrPresentationAlreadyExists = errors.New("presentation already exists") + +type discoveryService struct { + ID string `gorm:"primaryKey"` + Timestamp uint64 +} + +func (s discoveryService) TableName() string { + return "discoveryservices" +} + +var _ schema.Tabler = (*servicePresentation)(nil) + +type servicePresentation struct { + ID string `gorm:"primaryKey"` + ServiceID string + Timestamp uint64 + CredentialSubjectID string + PresentationID string + PresentationRaw string + PresentationExpiration int64 + Credentials []credential `gorm:"foreignKey:PresentationID;references:ID"` +} + +func (s servicePresentation) TableName() string { + return "discoveryservice_presentations" +} + +// credential is a Verifiable Credential, part of a presentation (entry) on a use case list. +type credential struct { + // ID is the unique identifier of the entry. + ID string `gorm:"primaryKey"` + // PresentationID corresponds to the discoveryservice_presentations record ID (not VerifiablePresentation.ID) this credential belongs to. + PresentationID string + // CredentialID contains the 'id' property of the Verifiable Credential. + CredentialID string + // CredentialIssuer contains the 'issuer' property of the Verifiable Credential. + CredentialIssuer string + // CredentialSubjectID contains the 'credentialSubject.id' property of the Verifiable Credential. + CredentialSubjectID string + // CredentialType contains the 'type' property of the Verifiable Credential (not being 'VerifiableCredential'). + CredentialType *string + Properties []credentialProperty `gorm:"foreignKey:ID;references:ID"` +} + +// TableName returns the table name for this DTO. +func (p credential) TableName() string { + return "discoveryservice_credentials" +} + +// credentialProperty is a property of a Verifiable Credential in a Verifiable Presentation in a discovery service. +type credentialProperty struct { + // ID refers to the entry record in discoveryservice_credentials + ID string `gorm:"primaryKey"` + // Key is JSON path of the property. + Key string `gorm:"primaryKey"` + // Value is the value of the property. + Value string +} + +// TableName returns the table name for this DTO. +func (l credentialProperty) TableName() string { + return "discoveryservice_credential_props" +} + +type sqlStore struct { + db *gorm.DB +} + +func newSQLStore(db *gorm.DB, definitions map[string]Definition) (*sqlStore, error) { + // Creates entries in the discovery service table with initial timestamp, if they don't exist yet + for _, definition := range definitions { + currentList := discoveryService{ + ID: definition.ID, + } + if err := db.FirstOrCreate(¤tList, "id = ?", definition.ID).Error; err != nil { + return nil, err + } + } + return &sqlStore{ + db: db, + }, nil +} + +// Add adds a presentation to the list of presentations. +// Timestamp should be passed if the presentation was received from a remote Discovery Server, then it is stored alongside the presentation. +// If the local node is the Discovery Server and thus is responsible for the timestamping, +// nil should be passed to let the store determine the right value. +func (s *sqlStore) add(serviceID string, presentation vc.VerifiablePresentation, timestamp *Timestamp) error { + credentialSubjectID, err := credential2.PresentationSigner(presentation) + if err != nil { + return err + } + if exists, err := s.exists(serviceID, credentialSubjectID.String(), presentation.ID.String()); err != nil { + return err + } else if exists { + return ErrPresentationAlreadyExists + } + if err := s.prune(); err != nil { + return err + } + + return s.db.Transaction(func(tx *gorm.DB) error { + timestamp, err := s.updateTimestamp(tx, serviceID, timestamp) + if err != nil { + return err + } + // Delete any previous presentations of the subject + if err := tx.Delete(&servicePresentation{}, "service_id = ? AND credential_subject_id = ?", serviceID, credentialSubjectID.String()). + Error; err != nil { + return err + } + // Now store the presentation itself + return tx.Create(&servicePresentation{ + ID: uuid.NewString(), + ServiceID: serviceID, + Timestamp: uint64(timestamp), + CredentialSubjectID: credentialSubjectID.String(), + PresentationID: presentation.ID.String(), + PresentationRaw: presentation.Raw(), + PresentationExpiration: presentation.JWT().Expiration().Unix(), + }).Error + }) +} + +func (s *sqlStore) get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresentation, *Timestamp, error) { + var rows []servicePresentation + err := s.db.Order("timestamp ASC").Find(&rows, "service_id = ? AND timestamp > ?", serviceID, int(startAt)).Error + if err != nil { + return nil, nil, fmt.Errorf("query service '%s': %w", serviceID, err) + } + timestamp := startAt + presentations := make([]vc.VerifiablePresentation, 0, len(rows)) + for _, row := range rows { + presentation, err := vc.ParseVerifiablePresentation(row.PresentationRaw) + if err != nil { + return nil, nil, fmt.Errorf("parse presentation '%s' of service '%s': %w", row.PresentationID, serviceID, err) + } + presentations = append(presentations, *presentation) + timestamp = Timestamp(row.Timestamp) + } + return presentations, ×tamp, nil +} + +func (s *sqlStore) updateTimestamp(tx *gorm.DB, serviceID string, newTimestamp *Timestamp) (Timestamp, error) { + var result discoveryService + // Lock (SELECT FOR UPDATE) discoveryservices row to prevent concurrent updates to the same list, which could mess up the lamport timestamp. + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). + Where(discoveryService{ID: serviceID}). + Find(&result). + Error; err != nil { + return 0, err + } + result.ID = serviceID + if newTimestamp == nil { + // Increment timestamp + result.Timestamp++ + } else { + result.Timestamp = uint64(*newTimestamp) + } + if err := tx.Save(&result).Error; err != nil { + return 0, err + } + return Timestamp(result.Timestamp), nil +} + +func (s *sqlStore) exists(serviceID string, credentialSubjectID string, presentationID string) (bool, error) { + var count int64 + if err := s.db.Model(servicePresentation{}).Where(servicePresentation{ + ServiceID: serviceID, + CredentialSubjectID: credentialSubjectID, + PresentationID: presentationID, + }).Count(&count).Error; err != nil { + return false, fmt.Errorf("check presentation existence: %w", err) + } + return count > 0, nil +} + +func (s *sqlStore) prune() error { + num, err := s.removeExpired() + if err != nil { + return err + } + if num > 0 { + log.Logger().Debugf("Pruned %d expired presentations", num) + } + return nil +} + +func (s *sqlStore) removeExpired() (int, error) { + result := s.db.Where("presentation_expiration < ?", time.Now().Unix()).Delete(servicePresentation{}) + if result.Error != nil { + return 0, fmt.Errorf("prune presentations: %w", result.Error) + } + return int(result.RowsAffected), nil +} diff --git a/discoveryservice/store_test.go b/discoveryservice/store_test.go new file mode 100644 index 0000000000..1627a6d446 --- /dev/null +++ b/discoveryservice/store_test.go @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package discoveryservice + +import ( + "github.com/nuts-foundation/nuts-node/storage" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "testing" +) + +func Test_sqlStore_exists(t *testing.T) { + storageEngine := storage.NewTestStorageEngine(t) + require.NoError(t, storageEngine.Start()) + + t.Run("empty list", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + exists, err := m.exists(testServiceID, aliceDID.String(), vpAlice.ID.String()) + assert.NoError(t, err) + assert.False(t, exists) + }) + t.Run("non-empty list, no match (other subject and ID)", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + require.NoError(t, m.add(testServiceID, vpBob, nil)) + exists, err := m.exists(testServiceID, aliceDID.String(), vpAlice.ID.String()) + assert.NoError(t, err) + assert.False(t, exists) + }) + t.Run("non-empty list, no match (other list)", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + require.NoError(t, m.add(testServiceID, vpAlice, nil)) + exists, err := m.exists("other", aliceDID.String(), vpAlice.ID.String()) + assert.NoError(t, err) + assert.False(t, exists) + }) + t.Run("non-empty list, match", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + require.NoError(t, m.add(testServiceID, vpAlice, nil)) + exists, err := m.exists(testServiceID, aliceDID.String(), vpAlice.ID.String()) + assert.NoError(t, err) + assert.True(t, exists) + }) +} + +func setupStore(t *testing.T, db *gorm.DB) *sqlStore { + resetStoreAfterTest(t, db) + store, err := newSQLStore(db, testDefinitions()) + require.NoError(t, err) + return store +} + +func resetStoreAfterTest(t *testing.T, db *gorm.DB) { + t.Cleanup(func() { + underlyingDB, err := db.DB() + require.NoError(t, err) + _, err = underlyingDB.Exec("DELETE FROM discoveryservice_presentations") + require.NoError(t, err) + _, err = underlyingDB.Exec("DELETE FROM discoveryservices") + require.NoError(t, err) + }) +} diff --git a/discoveryservice/test.go b/discoveryservice/test.go new file mode 100644 index 0000000000..43f74082f8 --- /dev/null +++ b/discoveryservice/test.go @@ -0,0 +1,211 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + +package discoveryservice + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "fmt" + "github.com/google/uuid" + "github.com/lestrrat-go/jwx/v2/jwa" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/lestrrat-go/jwx/v2/jws" + "github.com/lestrrat-go/jwx/v2/jwt" + ssi "github.com/nuts-foundation/go-did" + "github.com/nuts-foundation/go-did/did" + "github.com/nuts-foundation/go-did/vc" + "github.com/nuts-foundation/nuts-node/vcr/pe" + "time" +) + +var keyPairs map[string]*ecdsa.PrivateKey +var authorityDID did.DID +var aliceDID did.DID +var vcAlice vc.VerifiableCredential +var vpAlice vc.VerifiablePresentation +var bobDID did.DID +var vcBob vc.VerifiableCredential +var vpBob vc.VerifiablePresentation + +var testServiceID = "usecase_v1" + +func testDefinitions() map[string]Definition { + return map[string]Definition{ + testServiceID: { + ID: testServiceID, + Endpoint: "http://example.com/usecase", + PresentationDefinition: pe.PresentationDefinition{ + InputDescriptors: []*pe.InputDescriptor{ + { + Constraints: &pe.Constraints{ + Fields: []pe.Field{ + { + Path: []string{"$.issuer"}, + Filter: &pe.Filter{ + Type: "string", + }, + }, + }, + }, + }, + }, + }, + PresentationMaxValidity: int((24 * time.Hour).Seconds()), + }, + "other": { + ID: "other", + Endpoint: "http://example.com/other", + PresentationDefinition: pe.PresentationDefinition{ + InputDescriptors: []*pe.InputDescriptor{ + { + Constraints: &pe.Constraints{ + Fields: []pe.Field{ + { + Path: []string{"$.issuer"}, + Filter: &pe.Filter{ + Type: "string", + }, + }, + }, + }, + }, + }, + }, + PresentationMaxValidity: int((24 * time.Hour).Seconds()), + }, + } +} + +func init() { + keyPairs = make(map[string]*ecdsa.PrivateKey) + authorityDID = did.MustParseDID("did:example:authority") + keyPairs[authorityDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + aliceDID = did.MustParseDID("did:example:alice") + keyPairs[aliceDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + bobDID = did.MustParseDID("did:example:bob") + keyPairs[bobDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + + vcAlice = createCredential(authorityDID, aliceDID) + vpAlice = createPresentation(aliceDID, vcAlice) + vcBob = createCredential(authorityDID, bobDID) + vpBob = createPresentation(bobDID, vcBob) +} + +func createCredential(issuerDID did.DID, subjectDID did.DID) vc.VerifiableCredential { + return createCredentialWithClaims(issuerDID, subjectDID, func(claims map[string]interface{}) { + // do nothing + }) +} + +func createCredentialWithClaims(issuerDID did.DID, subjectDID did.DID, claimVisitor func(map[string]interface{})) vc.VerifiableCredential { + vcID := did.DIDURL{DID: issuerDID} + vcID.Fragment = uuid.NewString() + vcIDURI := vcID.URI() + expirationDate := time.Now().Add(time.Hour * 24) + result, err := vc.CreateJWTVerifiableCredential(context.Background(), vc.VerifiableCredential{ + ID: &vcIDURI, + Issuer: issuerDID.URI(), + IssuanceDate: time.Now(), + ExpirationDate: &expirationDate, + CredentialSubject: []interface{}{ + map[string]interface{}{ + "id": subjectDID.String(), + }, + }, + }, func(ctx context.Context, claims map[string]interface{}, headers map[string]interface{}) (string, error) { + claimVisitor(claims) + return signJWT(subjectDID, claims, headers) + }) + if err != nil { + panic(err) + } + return *result +} + +func createPresentation(subjectDID did.DID, credentials ...vc.VerifiableCredential) vc.VerifiablePresentation { + return createPresentationCustom(subjectDID, func(_ map[string]interface{}, _ *vc.VerifiablePresentation) { + // do nothing + }, credentials...) +} + +func createPresentationCustom(subjectDID did.DID, visitor func(claims map[string]interface{}, vp *vc.VerifiablePresentation), credentials ...vc.VerifiableCredential) vc.VerifiablePresentation { + headers := map[string]interface{}{ + jws.TypeKey: "JWT", + } + innerVP := &vc.VerifiablePresentation{ + Type: append([]ssi.URI{ssi.MustParseURI("VerifiablePresentation")}), + VerifiableCredential: credentials, + } + claims := map[string]interface{}{ + jwt.IssuerKey: subjectDID.String(), + jwt.SubjectKey: subjectDID.String(), + jwt.JwtIDKey: subjectDID.String() + "#" + uuid.NewString(), + jwt.NotBeforeKey: time.Now().Unix(), + jwt.ExpirationKey: time.Now().Add(time.Hour * 8), + } + visitor(claims, innerVP) + claims["vp"] = *innerVP + token, err := signJWT(subjectDID, claims, headers) + if err != nil { + panic(err) + } + presentation, err := vc.ParseVerifiablePresentation(token) + if err != nil { + panic(err) + } + return *presentation +} + +func signJWT(subjectDID did.DID, claims map[string]interface{}, headers map[string]interface{}) (string, error) { + // Build JWK + signingKey := keyPairs[subjectDID.String()] + if signingKey == nil { + return "", fmt.Errorf("key not found for DID: %s", subjectDID) + } + subjectKeyJWK, err := jwk.FromRaw(signingKey) + if err != nil { + return "", nil + } + keyID := did.DIDURL{DID: subjectDID} + keyID.Fragment = "0" + if err := subjectKeyJWK.Set(jwk.AlgorithmKey, jwa.ES256); err != nil { + return "", err + } + if err := subjectKeyJWK.Set(jwk.KeyIDKey, keyID.String()); err != nil { + return "", err + } + + // Build token + token := jwt.New() + for k, v := range claims { + if err := token.Set(k, v); err != nil { + return "", err + } + } + hdr := jws.NewHeaders() + for k, v := range headers { + if err := hdr.Set(k, v); err != nil { + return "", err + } + } + bytes, err := jwt.Sign(token, jwt.WithKey(jwa.ES256, signingKey, jws.WithProtectedHeaders(hdr))) + return string(bytes), err +} diff --git a/discoveryservice/test/duplicate_id/1.json b/discoveryservice/test/duplicate_id/1.json new file mode 100644 index 0000000000..533dc08feb --- /dev/null +++ b/discoveryservice/test/duplicate_id/1.json @@ -0,0 +1,49 @@ +{ + "id": "urn:nuts.nl:usecase:eOverdrachtDev2023", + "endpoint": "https://example.com/usecase/eoverdracht_dev", + "presentation_max_validity": 36000, + "presentation_definition": { + "id": "pd_eoverdracht_dev_care_organization", + "format": { + "ldp_vc": { + "proof_type": [ + "JsonWebSignature2020" + ] + } + }, + "input_descriptors": [ + { + "id": "id_nuts_care_organization_cred", + "constraints": { + "fields": [ + { + "path": [ + "$.type" + ], + "filter": { + "type": "string", + "const": "NutsOrganizationCredential" + } + }, + { + "path": [ + "$.credentialSubject.organization.name" + ], + "filter": { + "type": "string" + } + }, + { + "path": [ + "$.credentialSubject.organization.city" + ], + "filter": { + "type": "string" + } + } + ] + } + } + ] + } +} diff --git a/discoveryservice/test/duplicate_id/2.json b/discoveryservice/test/duplicate_id/2.json new file mode 100644 index 0000000000..533dc08feb --- /dev/null +++ b/discoveryservice/test/duplicate_id/2.json @@ -0,0 +1,49 @@ +{ + "id": "urn:nuts.nl:usecase:eOverdrachtDev2023", + "endpoint": "https://example.com/usecase/eoverdracht_dev", + "presentation_max_validity": 36000, + "presentation_definition": { + "id": "pd_eoverdracht_dev_care_organization", + "format": { + "ldp_vc": { + "proof_type": [ + "JsonWebSignature2020" + ] + } + }, + "input_descriptors": [ + { + "id": "id_nuts_care_organization_cred", + "constraints": { + "fields": [ + { + "path": [ + "$.type" + ], + "filter": { + "type": "string", + "const": "NutsOrganizationCredential" + } + }, + { + "path": [ + "$.credentialSubject.organization.name" + ], + "filter": { + "type": "string" + } + }, + { + "path": [ + "$.credentialSubject.organization.city" + ], + "filter": { + "type": "string" + } + } + ] + } + } + ] + } +} diff --git a/discoveryservice/test/duplicate_id/README.md b/discoveryservice/test/duplicate_id/README.md new file mode 100644 index 0000000000..f0fc1802ed --- /dev/null +++ b/discoveryservice/test/duplicate_id/README.md @@ -0,0 +1 @@ +This directory contains an invalid use case definition: 2 definitions have the same ID. \ No newline at end of file diff --git a/discoveryservice/test/invalid_definition/1.json b/discoveryservice/test/invalid_definition/1.json new file mode 100644 index 0000000000..0db3279e44 --- /dev/null +++ b/discoveryservice/test/invalid_definition/1.json @@ -0,0 +1,3 @@ +{ + +} diff --git a/discoveryservice/test/invalid_definition/README.md b/discoveryservice/test/invalid_definition/README.md new file mode 100644 index 0000000000..6053c6729b --- /dev/null +++ b/discoveryservice/test/invalid_definition/README.md @@ -0,0 +1 @@ +This directory contains an invalid use case definition: it does not fields that are required according to the JSON schema. \ No newline at end of file diff --git a/discoveryservice/test/invalid_json/1.json b/discoveryservice/test/invalid_json/1.json new file mode 100644 index 0000000000..7e31dc3cad --- /dev/null +++ b/discoveryservice/test/invalid_json/1.json @@ -0,0 +1 @@ +this is not JSON \ No newline at end of file diff --git a/discoveryservice/test/invalid_json/README.md b/discoveryservice/test/invalid_json/README.md new file mode 100644 index 0000000000..30610e3784 --- /dev/null +++ b/discoveryservice/test/invalid_json/README.md @@ -0,0 +1 @@ +This directory contains an invalid use case definition: it is not valid JSON. \ No newline at end of file diff --git a/discoveryservice/test/valid/eoverdracht.json b/discoveryservice/test/valid/eoverdracht.json new file mode 100644 index 0000000000..533dc08feb --- /dev/null +++ b/discoveryservice/test/valid/eoverdracht.json @@ -0,0 +1,49 @@ +{ + "id": "urn:nuts.nl:usecase:eOverdrachtDev2023", + "endpoint": "https://example.com/usecase/eoverdracht_dev", + "presentation_max_validity": 36000, + "presentation_definition": { + "id": "pd_eoverdracht_dev_care_organization", + "format": { + "ldp_vc": { + "proof_type": [ + "JsonWebSignature2020" + ] + } + }, + "input_descriptors": [ + { + "id": "id_nuts_care_organization_cred", + "constraints": { + "fields": [ + { + "path": [ + "$.type" + ], + "filter": { + "type": "string", + "const": "NutsOrganizationCredential" + } + }, + { + "path": [ + "$.credentialSubject.organization.name" + ], + "filter": { + "type": "string" + } + }, + { + "path": [ + "$.credentialSubject.organization.city" + ], + "filter": { + "type": "string" + } + } + ] + } + } + ] + } +} diff --git a/discoveryservice/test/valid/subdir/README.md b/discoveryservice/test/valid/subdir/README.md new file mode 100644 index 0000000000..b1778a548c --- /dev/null +++ b/discoveryservice/test/valid/subdir/README.md @@ -0,0 +1 @@ +This directory (with an invalid definition) is there to assert subdirectories are not traversed. \ No newline at end of file diff --git a/discoveryservice/test/valid/subdir/empty.json b/discoveryservice/test/valid/subdir/empty.json new file mode 100644 index 0000000000..2c63c08510 --- /dev/null +++ b/discoveryservice/test/valid/subdir/empty.json @@ -0,0 +1,2 @@ +{ +} diff --git a/docs/pages/deployment/cli-reference.rst b/docs/pages/deployment/cli-reference.rst index 0bf3c272ea..ca69be9aee 100755 --- a/docs/pages/deployment/cli-reference.rst +++ b/docs/pages/deployment/cli-reference.rst @@ -44,7 +44,7 @@ The following options apply to the server commands below: --http.default.log string What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). (default "metadata") --http.default.tls string Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', --internalratelimiter When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. (default true) - --jsonld.contexts.localmapping stringToString This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. (default [https://schema.org=assets/contexts/schema-org-v13.ldjson,https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson]) + --jsonld.contexts.localmapping stringToString This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. (default [https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson,https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson]) --jsonld.contexts.remoteallowlist strings In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. (default [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json]) --loggerformat string Log format (text, json) (default "text") --network.bootstrapnodes strings List of bootstrap nodes (':') which the node initially connect to. diff --git a/docs/pages/deployment/server_options.rst b/docs/pages/deployment/server_options.rst index 03b274bb46..fbe48394e1 100755 --- a/docs/pages/deployment/server_options.rst +++ b/docs/pages/deployment/server_options.rst @@ -50,7 +50,7 @@ http.default.auth.type Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode. http.default.cors.origin [] When set, enables CORS from the specified origins on the default HTTP interface. **JSONLD** - jsonld.contexts.localmapping [https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson,https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. + jsonld.contexts.localmapping [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. jsonld.contexts.remoteallowlist [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json] In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. **Network** network.bootstrapnodes [] List of bootstrap nodes (':') which the node initially connect to. diff --git a/go.mod b/go.mod index 896b9cf3f9..594e72dbe2 100644 --- a/go.mod +++ b/go.mod @@ -106,6 +106,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/gorm v1.9.16 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.2 // indirect github.com/kr/text v0.2.0 // indirect diff --git a/makefile b/makefile index 6f6efae5be..4c92cfa65c 100644 --- a/makefile +++ b/makefile @@ -19,6 +19,7 @@ gen-mocks: mockgen -destination=crypto/mock.go -package=crypto -source=crypto/interface.go mockgen -destination=crypto/storage/spi/mock.go -package spi -source=crypto/storage/spi/interface.go mockgen -destination=didman/mock.go -package=didman -source=didman/types.go + mockgen -destination=discoveryservice/mock.go -package=discoveryservice -source=discoveryservice/interface.go mockgen -destination=events/events_mock.go -package=events -source=events/interface.go Event mockgen -destination=events/mock.go -package=events -source=events/conn.go Conn ConnectionPool mockgen -destination=http/echo_mock.go -package=http -source=http/echo.go -imports echo=github.com/labstack/echo/v4 @@ -55,6 +56,7 @@ gen-mocks: mockgen -destination=vdr/management/management_mock.go -package=management -source=vdr/management/management.go mockgen -destination=vdr/management/finder_mock.go -package=management -source=vdr/management/finder.go + gen-api: oapi-codegen --config codegen/configs/common_ssi_types.yaml docs/_static/common/ssi_types.yaml | gofmt > api/ssi_types.go oapi-codegen --config codegen/configs/crypto_v1.yaml -package v1 docs/_static/crypto/v1.yaml | gofmt > crypto/api/v1/generated.go diff --git a/storage/mock.go b/storage/mock.go index a4aa1c0bfa..ecbfce1b17 100644 --- a/storage/mock.go +++ b/storage/mock.go @@ -69,32 +69,32 @@ func (mr *MockEngineMockRecorder) GetProvider(moduleName any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProvider", reflect.TypeOf((*MockEngine)(nil).GetProvider), moduleName) } -// GetSessionDatabase mocks base method. -func (m *MockEngine) GetSessionDatabase() SessionDatabase { +// GetSQLDatabase mocks base method. +func (m *MockEngine) GetSQLDatabase() *gorm.DB { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSessionDatabase") - ret0, _ := ret[0].(SessionDatabase) + ret := m.ctrl.Call(m, "GetSQLDatabase") + ret0, _ := ret[0].(*gorm.DB) return ret0 } -// GetSessionDatabase indicates an expected call of GetSessionDatabase. -func (mr *MockEngineMockRecorder) GetSessionDatabase() *gomock.Call { +// GetSQLDatabase indicates an expected call of GetSQLDatabase. +func (mr *MockEngineMockRecorder) GetSQLDatabase() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionDatabase", reflect.TypeOf((*MockEngine)(nil).GetSessionDatabase)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSQLDatabase", reflect.TypeOf((*MockEngine)(nil).GetSQLDatabase)) } -// SQLDatabase mocks base method. -func (m *MockEngine) GetSQLDatabase() *gorm.DB { +// GetSessionDatabase mocks base method. +func (m *MockEngine) GetSessionDatabase() SessionDatabase { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetSQLDatabase") - ret0, _ := ret[0].(*gorm.DB) + ret := m.ctrl.Call(m, "GetSessionDatabase") + ret0, _ := ret[0].(SessionDatabase) return ret0 } -// SQLDatabase indicates an expected call of SQLDatabase. -func (mr *MockEngineMockRecorder) SQLDatabase() *gomock.Call { +// GetSessionDatabase indicates an expected call of GetSessionDatabase. +func (mr *MockEngineMockRecorder) GetSessionDatabase() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSQLDatabase", reflect.TypeOf((*MockEngine)(nil).GetSQLDatabase)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSessionDatabase", reflect.TypeOf((*MockEngine)(nil).GetSessionDatabase)) } // Shutdown mocks base method. diff --git a/storage/sql_migrations/2_discoveryservice.down.sql b/storage/sql_migrations/2_discoveryservice.down.sql new file mode 100644 index 0000000000..3922d7112b --- /dev/null +++ b/storage/sql_migrations/2_discoveryservice.down.sql @@ -0,0 +1,4 @@ +drop table discoveryservices; +drop table discoveryservice_presentations; +drop table discoveryservice_credentials; +drop table discoveryservice_credential_props; \ No newline at end of file diff --git a/storage/sql_migrations/2_discoveryservice.up.sql b/storage/sql_migrations/2_discoveryservice.up.sql new file mode 100644 index 0000000000..bfd78e9937 --- /dev/null +++ b/storage/sql_migrations/2_discoveryservice.up.sql @@ -0,0 +1,49 @@ +-- discoveryservices contains the known discovery services and the highest timestamp +create table discoveryservices +( + id text not null primary key, + timestamp integer not null +); + +-- discoveryservice_presentations contains the presentations of the discovery services +create table discoveryservice_presentations +( + id text not null primary key, + service_id text not null, + timestamp integer not null, + credential_subject_id text not null, + presentation_id text not null, + presentation_raw text not null, + presentation_expiration integer not null, + unique (service_id, credential_subject_id), + constraint fk_discovery_presentation_service_id foreign key (service_id) references discoveryservices (id) on delete cascade +); + +-- discoveryservice_credentials is a credential in a presentation of the discovery service. +-- We could do without the table, but having it allows to have a normalized index for credential properties that appear on every credential. +-- Then we don't need rows in the properties table for them (having a column for those is faster than having a row in the properties table which needs to be joined). +create table discoveryservice_credentials +( + id text not null primary key, + presentation_id text not null, + credential_id text not null, + credential_issuer text not null, + credential_subject_id text not null, + -- for now, credentials with at most 2 types are supported. + -- The type stored in the type column will be the 'other' type, not being 'VerifiableCredential'. + -- When credentials with 3 or more types appear, we could have to use a separate table for the types. + credential_type text, + constraint fk_discoveryservice_credential_presentation foreign key (presentation_id) references discoveryservice_presentations (id) on delete cascade +); + +-- discoveryservice_credential_props contains the credentialSubject properties of a credential in a presentation of the discovery service. +-- It is used by clients to search for presentations. +create table discoveryservice_credential_props +( + id text not null, + key text not null, + value text, + PRIMARY KEY (id, key), + -- cascading delete: if the presentation gets deleted, the properties get deleted as well + constraint fk_discoveryservice_credential_id foreign key (id) references discoveryservice_credentials (id) on delete cascade +); \ No newline at end of file diff --git a/storage/test.go b/storage/test.go index 34cc9a0ef7..299fc89faa 100644 --- a/storage/test.go +++ b/storage/test.go @@ -34,6 +34,7 @@ const SQLiteInMemoryConnectionString = "file::memory:?cache=shared" func NewTestStorageEngineInDir(dir string) Engine { result := New().(*engine) result.config.SQL = SQLConfig{ConnectionString: SQLiteInMemoryConnectionString} + //result.config.SQL = SQLConfig{ConnectionString: "file:../../data/sqlite.db"} _ = result.Configure(core.TestServerConfig(core.ServerConfig{Datadir: dir + "/data"})) return result } diff --git a/vcr/credential/resolver.go b/vcr/credential/resolver.go index fd4ec48ac8..5267a3b4f9 100644 --- a/vcr/credential/resolver.go +++ b/vcr/credential/resolver.go @@ -20,7 +20,10 @@ package credential import ( + "errors" + "github.com/nuts-foundation/go-did/did" "github.com/nuts-foundation/go-did/vc" + "strings" ) // FindValidator finds the Validator the provided credential based on its Type @@ -52,3 +55,48 @@ func ExtractTypes(credential vc.VerifiableCredential) []string { return vcTypes } + +func PresentationSigner(presentation vc.VerifiablePresentation) (*did.DID, error) { + switch presentation.Format() { + case vc.JWTPresentationProofFormat: + token := presentation.JWT() + issuer := token.Issuer() + if issuer == "" { + return nil, errors.New("JWT presentation does not have 'iss' claim") + } + return did.ParseDID(issuer) + default: + return nil, errors.New("unsupported presentation format") + } +} + +func PresentationSigningKeyID(presentation vc.VerifiablePresentation) (*did.DIDURL, error) { + switch presentation.Format() { + case vc.JWTPresentationProofFormat: + token := presentation.JWT() + keyID, exists := token.Get("kid") + if !exists { + return nil, errors.New("JWT presentation does not have 'kid' claim") + } + keyIDString, isString := keyID.(string) + if !isString { + return nil, errors.New("JWT presentation 'kid' claim is not a string") + } + issuer, err := PresentationSigner(presentation) + if err != nil { + return nil, err + } + if strings.HasPrefix(keyIDString, "#") { + // Key ID is a fragment, so it's a relative URL to the JWT issuer + keyIDString = issuer.String() + keyIDString + } else { + // Key ID is fully qualified, must be prefixed with JWT issuer + if !strings.HasPrefix(keyIDString, issuer.String()+"#") { + return nil, errors.New("JWT presentation 'kid' claim must be scoped to 'iss' claim if absolute") + } + } + return did.ParseDIDURL(keyIDString) + default: + return nil, errors.New("unsupported presentation format") + } +} diff --git a/vcr/pe/presentation_definition.go b/vcr/pe/presentation_definition.go index 027f957fbf..71b3d6b8dc 100644 --- a/vcr/pe/presentation_definition.go +++ b/vcr/pe/presentation_definition.go @@ -292,8 +292,9 @@ func matchConstraint(constraint *Constraints, credential vc.VerifiableCredential // matchField matches the field against the VC. // All fields need to match unless optional is set to true and no values are found for all the paths. func matchField(field Field, credential vc.VerifiableCredential) (bool, error) { + type Alias vc.VerifiableCredential // jsonpath works on interfaces, so convert the VC to an interface - asJSON, _ := json.Marshal(credential) + asJSON, _ := json.Marshal(Alias(credential)) var asInterface interface{} _ = json.Unmarshal(asJSON, &asInterface) diff --git a/vcr/pe/schema/v2/schema.go b/vcr/pe/schema/v2/schema.go index 44de55fdd8..8c3da1a2bc 100644 --- a/vcr/pe/schema/v2/schema.go +++ b/vcr/pe/schema/v2/schema.go @@ -51,19 +51,21 @@ var PresentationDefinition *jsonschema.Schema // PresentationSubmission is the JSON schema for a presentation submission. var PresentationSubmission *jsonschema.Schema +// Compiler is the JSON schema compiler. +var Compiler = jsonschema.NewCompiler() + func init() { // By default, it loads from filesystem, but that sounds unsafe. // Since register our schemas, we don't need to allow loading resources. loader.Load = func(url string) (io.ReadCloser, error) { return nil, fmt.Errorf("refusing to load unknown schema: %s", url) } - compiler := jsonschema.NewCompiler() - compiler.Draft = jsonschema.Draft7 - if err := loadSchemas(schemaFiles, compiler); err != nil { + Compiler.Draft = jsonschema.Draft7 + if err := loadSchemas(schemaFiles, Compiler); err != nil { panic(err) } - PresentationDefinition = compiler.MustCompile(presentationDefinition) - PresentationSubmission = compiler.MustCompile(presentationSubmission) + PresentationDefinition = Compiler.MustCompile(presentationDefinition) + PresentationSubmission = Compiler.MustCompile(presentationSubmission) } func loadSchemas(reader fs.ReadFileFS, compiler *jsonschema.Compiler) error { diff --git a/vcr/pe/test/definition_mapping.json b/vcr/pe/test/definition_mapping.json index b543faa577..5459a3ea65 100644 --- a/vcr/pe/test/definition_mapping.json +++ b/vcr/pe/test/definition_mapping.json @@ -20,6 +20,16 @@ "const": "NutsOrganizationCredential" } }, + { + "path": ["$.issuer"], + "filter": { + "type": "string", + "filter": { + "type": "string", + "pattern": "^did:example:123456789abcdefghi$" + } + } + }, { "path": ["$.credentialSubject.organization.name"], "filter": { From 9dacfc466c9b2015c9a62d4be3063086e18da5e9 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Sun, 19 Nov 2023 12:24:30 +0100 Subject: [PATCH 02/23] credential properties index --- README.rst | 8 +- discoveryservice/client.go | 280 -------------- discoveryservice/client_test.go | 457 ----------------------- discoveryservice/module.go | 26 +- discoveryservice/module_test.go | 78 ++-- discoveryservice/store.go | 205 ++++++++-- discoveryservice/store_test.go | 283 +++++++++++++- discoveryservice/test.go | 45 ++- docs/pages/deployment/cli-reference.rst | 4 +- docs/pages/deployment/server_options.rst | 164 ++++---- storage/cmd/cmd.go | 4 +- storage/engine.go | 2 +- storage/test.go | 3 +- 13 files changed, 606 insertions(+), 953 deletions(-) delete mode 100644 discoveryservice/client.go delete mode 100644 discoveryservice/client_test.go diff --git a/README.rst b/README.rst index 3f908d240f..d5e2702b58 100644 --- a/README.rst +++ b/README.rst @@ -176,9 +176,9 @@ The following options can be configured on the server: :widths: 20 30 50 :class: options-table - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================== + ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= Key Default Description - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================== + ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= configfile nuts.yaml Nuts config file cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. datadir ./data Directory where the node stores its files. @@ -252,12 +252,12 @@ The following options can be configured on the server: storage.redis.sentinel.password Password for authenticating to Redis Sentinels. storage.redis.sentinel.username Username for authenticating to Redis Sentinels. storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). - storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory + storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' option. **VCR** vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================== + ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= This table is automatically generated using the configuration flags in the core and engines. When they're changed the options table must be regenerated using the Makefile: diff --git a/discoveryservice/client.go b/discoveryservice/client.go deleted file mode 100644 index 568f6becef..0000000000 --- a/discoveryservice/client.go +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright (C) 2023 Nuts community - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - */ - -package discoveryservice - -// -//import ( -// "encoding/json" -// "errors" -// "fmt" -// "github.com/google/uuid" -// "github.com/nuts-foundation/go-did/vc" -// "github.com/nuts-foundation/nuts-node/discoveryservice/log" -// "gorm.io/gorm" -// "gorm.io/gorm/clause" -// "io" -// "net/http" -// "net/url" -// "strconv" -// "strings" -// "sync" -// "time" -//) -// -//func newClient(db *gorm.DB, definitions map[string]Definition) (*client, error) { -// result := &client{ -// db: db, -// definitions: definitions, -// } -// if err := initializeSQLStore(db, definitions); err != nil { -// return nil, err -// } -// return result, nil -//} -// -//type client struct { -// db *gorm.DB -// definitions map[string]Definition -//} -// -//func (c *client) Search(serviceID string, query map[string]string) ([]vc.VerifiablePresentation, error) { -// propertyColumns := map[string]string{ -// "id": "cred.credential_id", -// "issuer": "cred.credential_issuer", -// "type": "cred.credential_type", -// "credentialSubject.id": "cred.credential_subject_id", -// } -// -// stmt := c.db.Model(&entry{}). -// Where("usecase_id = ?", serviceID). -// Joins("inner join usecase_client_credential cred ON cred.entry_id = usecase_client_entries.id") -// numProps := 0 -// for jsonPath, value := range query { -// if value == "*" { -// continue -// } -// // sort out wildcard mode -// var eq = "=" -// if strings.HasPrefix(value, "*") { -// value = "%" + value[1:] -// eq = "LIKE" -// } -// if strings.HasSuffix(value, "*") { -// value = value[:len(value)-1] + "%" -// eq = "LIKE" -// } -// if column := propertyColumns[jsonPath]; column != "" { -// stmt = stmt.Where(column+" "+eq+" ?", value) -// } else { -// // This property is not present as column, but indexed as key-value property. -// // Multiple (inner) joins to filter on a dynamic number of properties to filter on is not pretty, but it works -// alias := "p" + strconv.Itoa(numProps) -// numProps++ -// stmt = stmt.Joins("inner join usecase_client_credential_props "+alias+" ON "+alias+".id = cred.id AND "+alias+".key = ? AND "+alias+".value "+eq+" ?", jsonPath, value) -// } -// } -// -// var matches []entry -// if err := stmt.Find(&matches).Error; err != nil { -// return nil, err -// } -// var results []vc.VerifiablePresentation -// for _, match := range matches { -// if match.PresentationExpiration <= time.Now().Unix() { -// continue -// } -// presentation, err := vc.ParseVerifiablePresentation(match.PresentationRaw) -// if err != nil { -// return nil, fmt.Errorf("failed to parse presentation '%s': %w", match.PresentationID, err) -// } -// results = append(results, *presentation) -// } -// return results, nil -//} -// -//func (c *client) refreshAll() { -// wg := &sync.WaitGroup{} -// for _, definition := range c.definitions { -// wg.Add(1) -// go func(definition Definition) { -// c.refreshList(definition) -// }(definition) -// } -// wg.Done() -//} -// -//func (c *client) refreshList(definition Definition) error { -// var currentService discoveryService -// if err := c.db.Find(¤tService, "usecase_id = ?", definition.ID).Error; errors.Is(err, gorm.ErrRecordNotFound) { -// // First refresh of the list -// if err := c.db.Create(&discoveryService{ID: definition.ID}).Error; err != nil { -// return err -// } -// } else if err != nil { -// // Other error -// return err -// } -// log.Logger().Debugf("Refreshing use case list %s", definition.ID) -// // replace with generated client later -// requestURL, _ := url.Parse(definition.Endpoint) -// requestURL.Query().Add("timestamp", fmt.Sprintf("%d", currentService.Timestamp)) -// httpResponse, err := http.Get(definition.Endpoint) -// if err != nil { -// return err -// } -// data, err := io.ReadAll(httpResponse.Body) -// if err != nil { -// return err -// } -// var response ListResponse -// if err = json.Unmarshal(data, &response); err != nil { -// return err -// } -// return c.applyDelta(currentService.UsecaseID, response.Entries, response.Tombstone, currentService.Timestamp, response.Timestamp) -//} -// -//// applyDelta applies the updateTimestamp, retrieved from the use case list server, to the local index of the use case lists. -//func (c *client) applyDelta(usecaseID string, presentations []vc.VerifiablePresentation, tombstoneSet []string, previousTimestamp uint64, timestamp uint64) error { -// // TODO: validate presentations -// if previousTimestamp == timestamp { -// // nothing to do -// return nil -// } -// // We use a transaction to make sure the complete updateTimestamp is applied, or nothing at all. -// // Use a lock on the list to make sure there are no concurrent updates being applied to the list, -// // which could lead to the client becoming out-of-sync with the server list. -// // This situation can only really occur in a distributed system (multiple nodes updating the same list at the same time, with a different timestamp), -// // or bug in the updateTimestamp scheduler. -// return c.db.Transaction(func(tx *gorm.DB) error { -// // Lock the list, check if we're applying the delta to the right starting point -// var currentList list -// if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). -// Where("usecase_id = ?", usecaseID). -// Find(¤tList). -// Error; err != nil { -// return err -// } -// // Make sure we don't apply stale data -// if currentList.Timestamp != previousTimestamp { -// log.Logger().Infof("Not applying delta to use case list '%s': timestamp mismatch (expected %d but was %d). "+ -// "Probably caused by multiple processes updating the list. This is not a problem/bug: stale data should be updated at next refresh.", usecaseID, previousTimestamp, currentList.Timestamp) -// return nil -// } -// // Now we can apply the delta: -// // - delete removed presentations -// // - add new presentations -// // - index the presentations' properties -// if len(tombstoneSet) > 0 { -// if err := tx.Delete(&entry{}, "usecase_id = ? AND presentation_id IN ?", usecaseID, tombstoneSet).Error; err != nil { -// return fmt.Errorf("failed to delete tombstone records: %w", err) -// } -// } -// for _, presentation := range presentations { -// err := c.writePresentation(tx, usecaseID, presentation) -// if err != nil { -// return err -// } -// } -// // Finally, updateTimestamp the list timestamp -// if err := tx.Model(&list{}).Where("usecase_id = ?", usecaseID).Update("timestamp", timestamp).Error; err != nil { -// return fmt.Errorf("failed to updateTimestamp timestamp: %w", err) -// } -// return nil -// }) -//} -// -//func (c *client) writePresentation(tx *gorm.DB, usecaseID string, presentation vc.VerifiablePresentation) error { -// entryID := uuid.NewString() -// // Store list entry / verifiable presentation -// newEntry := entry{ -// ID: entryID, -// UsecaseID: usecaseID, -// PresentationID: presentation.ID.String(), -// PresentationRaw: presentation.Raw(), -// PresentationExpiration: presentation.JWT().Expiration().Unix(), -// } -// // Store the credentials of the presentation -// for _, curr := range presentation.VerifiableCredential { -// var credentialType *string -// for _, currType := range curr.Type { -// if currType.String() != "VerifiableCredential" { -// credentialType = new(string) -// *credentialType = currType.String() -// break -// } -// } -// subjectDID, err := curr.SubjectDID() -// if err != nil { -// return fmt.Errorf("invalid credential subject ID for VP '%s': %w", presentation.ID, err) -// } -// credentialRecordID := uuid.NewString() -// cred := credential{ -// ID: credentialRecordID, -// EntryID: entryID, -// CredentialID: curr.ID.String(), -// CredentialIssuer: curr.Issuer.String(), -// CredentialSubjectID: subjectDID.String(), -// CredentialType: credentialType, -// } -// if len(curr.CredentialSubject) != 1 { -// return errors.New("credential must contain exactly one subject") -// } -// // Store credential properties -// keys, values := indexJSONObject(curr.CredentialSubject[0].(map[string]interface{}), nil, nil, "credentialSubject") -// for i, key := range keys { -// if key == "credentialSubject.id" { -// // present as column, don't index -// continue -// } -// cred.Properties = append(cred.Properties, credentialProperty{ -// ID: credentialRecordID, -// Key: key, -// Value: values[i], -// }) -// } -// newEntry.Credentials = append(newEntry.Credentials, cred) -// } -// if err := tx.Create(&newEntry).Error; err != nil { -// return fmt.Errorf("failed to create entry: %w", err) -// } -// return nil -//} -// -//// indexJSONObject indexes a JSON object, resulting in a slice of JSON paths and corresponding string values. -//// It only traverses JSON objects and only adds string values to the result. -//func indexJSONObject(target map[string]interface{}, jsonPaths []string, stringValues []string, currentPath string) ([]string, []string) { -// for key, value := range target { -// thisPath := currentPath -// if len(thisPath) > 0 { -// thisPath += "." -// } -// thisPath += key -// -// switch typedValue := value.(type) { -// case string: -// jsonPaths = append(jsonPaths, thisPath) -// stringValues = append(stringValues, typedValue) -// case map[string]interface{}: -// jsonPaths, stringValues = indexJSONObject(typedValue, jsonPaths, stringValues, thisPath) -// default: -// // other values (arrays, booleans, numbers, null) are not indexed -// } -// } -// return jsonPaths, stringValues -//} diff --git a/discoveryservice/client_test.go b/discoveryservice/client_test.go deleted file mode 100644 index 160bd7b200..0000000000 --- a/discoveryservice/client_test.go +++ /dev/null @@ -1,457 +0,0 @@ -/* - * Copyright (C) 2023 Nuts community - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - */ - -package discoveryservice - -// -//import ( -// "context" -// "crypto/ecdsa" -// "crypto/elliptic" -// "crypto/rand" -// "fmt" -// "github.com/google/uuid" -// "github.com/lestrrat-go/jwx/v2/jwa" -// "github.com/lestrrat-go/jwx/v2/jwk" -// "github.com/lestrrat-go/jwx/v2/jws" -// "github.com/lestrrat-go/jwx/v2/jwt" -// ssi "github.com/nuts-foundation/go-did" -// "github.com/nuts-foundation/go-did/did" -// "github.com/nuts-foundation/go-did/vc" -// "github.com/nuts-foundation/nuts-node/storage" -// "github.com/stretchr/testify/require" -// "gorm.io/gorm/schema" -// "testing" -// "time" -//) -// -//func Test_client_applyDelta(t *testing.T) { -// //storageEngine := storage.New() -// //storageEngine.(core.Injectable).Config().(*storage.Config).SQL = storage.SQLConfig{ConnectionString: "file:../../data/sqlite.db"} -// //require.NoError(t, storageEngine.Configure(core.TestServerConfig(core.ServerConfig{Datadir: "data"}))) -// //require.NoError(t, storageEngine.Start()) -// -// storageEngine := storage.NewTestStorageEngine(t) -// require.NoError(t, storageEngine.Start()) -// t.Cleanup(func() { -// _ = storageEngine.Shutdown() -// }) -// -// t.Run("fresh list, assert all persisted fields", func(t *testing.T) { -// c := setupClient(t, storageEngine) -// err := c.applyDelta(TestDefinition.ID, []vc.VerifiablePresentation{vpAlice, vpBob}, []string{"other", "and another"}, 0, 1000) -// require.NoError(t, err) -// -// var actualList list -// require.NoError(t, c.db.Find(&actualList, "usecase_id = ?", TestDefinition.ID).Error) -// require.Equal(t, TestDefinition.ID, actualList.UsecaseID) -// require.Equal(t, uint64(1000), actualList.Timestamp) -// -// var entries []entry -// require.NoError(t, c.db.Find(&entries, "usecase_id = ?", TestDefinition.ID).Error) -// require.Len(t, entries, 2) -// require.Equal(t, vpAlice.ID.String(), entries[0].PresentationID) -// require.Equal(t, vpBob.ID.String(), entries[1].PresentationID) -// }) -//} -// -//func Test_client_writePresentation(t *testing.T) { -// storageEngine := storage.NewTestStorageEngine(t) -// require.NoError(t, storageEngine.Start()) -// t.Cleanup(func() { -// _ = storageEngine.Shutdown() -// }) -// -// t.Run("1 credential", func(t *testing.T) { -// c := setupClient(t, storageEngine) -// err := c.writePresentation(c.db, TestDefinition.ID, vpAlice) -// require.NoError(t, err) -// -// var entries []entry -// require.NoError(t, c.db.Find(&entries, "usecase_id = ?", TestDefinition.ID).Error) -// require.Len(t, entries, 1) -// require.Equal(t, vpAlice.ID.String(), entries[0].PresentationID) -// require.Equal(t, vpAlice.Raw(), entries[0].PresentationRaw) -// require.Equal(t, vpAlice.JWT().Expiration().Unix(), entries[0].PresentationExpiration) -// -// var credentials []credential -// require.NoError(t, c.db.Find(&credentials, "entry_id = ?", entries[0].ID).Error) -// require.Len(t, credentials, 1) -// cred := credentials[0] -// require.Equal(t, vcAlice.ID.String(), cred.CredentialID) -// require.Equal(t, vcAlice.Issuer.String(), cred.CredentialIssuer) -// require.Equal(t, aliceDID.String(), cred.CredentialSubjectID) -// require.Equal(t, vcAlice.Type[1].String(), *cred.CredentialType) -// -// expectedProperties := map[string]map[string]string{ -// cred.ID: { -// "credentialSubject.person.givenName": "Alice", -// "credentialSubject.person.familyName": "Jones", -// "credentialSubject.person.city": "InfoSecLand", -// }, -// } -// for recordID, properties := range expectedProperties { -// for key, value := range properties { -// var prop credentialProperty -// require.NoError(t, c.db.Find(&prop, "id = ? AND key = ?", recordID, key).Error) -// require.Equal(t, value, prop.Value) -// } -// } -// }) -//} -// -//func Test_client_search(t *testing.T) { -// storageEngine := storage.NewTestStorageEngine(t) -// require.NoError(t, storageEngine.Start()) -// t.Cleanup(func() { -// _ = storageEngine.Shutdown() -// }) -// -// type testCase struct { -// name string -// inputVPs []vc.VerifiablePresentation -// query map[string]string -// expectedVPs []string -// } -// testCases := []testCase{ -// { -// name: "issuer", -// inputVPs: []vc.VerifiablePresentation{vpAlice}, -// query: map[string]string{ -// "issuer": authorityDID.String(), -// }, -// expectedVPs: []string{vpAlice.ID.String()}, -// }, -// { -// name: "id", -// inputVPs: []vc.VerifiablePresentation{vpAlice}, -// query: map[string]string{ -// "id": vcAlice.ID.String(), -// }, -// expectedVPs: []string{vpAlice.ID.String()}, -// }, -// { -// name: "type", -// inputVPs: []vc.VerifiablePresentation{vpAlice}, -// query: map[string]string{ -// "type": "TestCredential", -// }, -// expectedVPs: []string{vpAlice.ID.String()}, -// }, -// { -// name: "credentialSubject.id", -// inputVPs: []vc.VerifiablePresentation{vpAlice}, -// query: map[string]string{ -// "credentialSubject.id": aliceDID.String(), -// }, -// expectedVPs: []string{vpAlice.ID.String()}, -// }, -// { -// name: "1 property", -// inputVPs: []vc.VerifiablePresentation{vpAlice}, -// query: map[string]string{ -// "credentialSubject.person.givenName": "Alice", -// }, -// expectedVPs: []string{vpAlice.ID.String()}, -// }, -// { -// name: "2 properties", -// inputVPs: []vc.VerifiablePresentation{vpAlice}, -// query: map[string]string{ -// "credentialSubject.person.givenName": "Alice", -// "credentialSubject.person.familyName": "Jones", -// }, -// expectedVPs: []string{vpAlice.ID.String()}, -// }, -// { -// name: "properties and base properties", -// inputVPs: []vc.VerifiablePresentation{vpAlice}, -// query: map[string]string{ -// "issuer": authorityDID.String(), -// "credentialSubject.person.givenName": "Alice", -// }, -// expectedVPs: []string{vpAlice.ID.String()}, -// }, -// { -// name: "wildcard postfix", -// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, -// query: map[string]string{ -// "credentialSubject.person.familyName": "Jo*", -// }, -// expectedVPs: []string{vpAlice.ID.String(), vpBob.ID.String()}, -// }, -// { -// name: "wildcard prefix", -// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, -// query: map[string]string{ -// "credentialSubject.person.givenName": "*ce", -// }, -// expectedVPs: []string{vpAlice.ID.String()}, -// }, -// { -// name: "wildcard midway (no interpreted as wildcard)", -// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, -// query: map[string]string{ -// "credentialSubject.person.givenName": "A*ce", -// }, -// expectedVPs: []string{}, -// }, -// { -// name: "just wildcard", -// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, -// query: map[string]string{ -// "id": "*", -// }, -// expectedVPs: []string{vpAlice.ID.String(), vpBob.ID.String()}, -// }, -// { -// name: "2 VPs, 1 match", -// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, -// query: map[string]string{ -// "credentialSubject.person.givenName": "Alice", -// }, -// expectedVPs: []string{vpAlice.ID.String()}, -// }, -// { -// name: "multiple matches", -// inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, -// query: map[string]string{ -// "issuer": authorityDID.String(), -// }, -// expectedVPs: []string{vpAlice.ID.String(), vpBob.ID.String()}, -// }, -// { -// name: "no match", -// inputVPs: []vc.VerifiablePresentation{vpAlice}, -// query: map[string]string{ -// "credentialSubject.person.givenName": "Bob", -// }, -// expectedVPs: []string{}, -// }, -// { -// name: "empty database", -// query: map[string]string{ -// "credentialSubject.person.givenName": "Bob", -// }, -// expectedVPs: []string{}, -// }, -// } -// -// for _, tc := range testCases { -// t.Run(tc.name, func(t *testing.T) { -// c := setupClient(t, storageEngine) -// for _, vp := range tc.inputVPs { -// err := c.writePresentation(c.db, TestDefinition.ID, vp) -// require.NoError(t, err) -// } -// actualVPs, err := c.Search(TestDefinition.ID, tc.query) -// require.NoError(t, err) -// require.Len(t, actualVPs, len(tc.expectedVPs)) -// for _, expectedVP := range tc.expectedVPs { -// found := false -// for _, actualVP := range actualVPs { -// if actualVP.ID.String() == expectedVP { -// found = true -// break -// } -// } -// require.True(t, found, "expected to find VP with ID %s", expectedVP) -// } -// }) -// } -//} -// -//func setupClient(t *testing.T, storageEngine storage.Engine) *client { -// t.Cleanup(func() { -// underlyingDB, err := storageEngine.GetSQLDatabase().DB() -// require.NoError(t, err) -// tables := []schema.Tabler{ -// &entry{}, -// &credential{}, -// &list{}, -// } -// for _, table := range tables { -// _, err = underlyingDB.Exec("DELETE FROM " + table.TableName()) -// require.NoError(t, err) -// } -// }) -// testDefinitions := map[string]Definition{ -// TestDefinition.ID: TestDefinition, -// } -// -// c, err := newClient(storageEngine.GetSQLDatabase(), testDefinitions) -// require.NoError(t, err) -// return c -//} -// -//var keyPairs map[string]*ecdsa.PrivateKey -//var authorityDID did.DID -//var aliceDID did.DID -//var vcAlice vc.VerifiableCredential -//var vpAlice vc.VerifiablePresentation -//var bobDID did.DID -//var vcBob vc.VerifiableCredential -// -//var vpBob vc.VerifiablePresentation -// -//func init() { -// keyPairs = make(map[string]*ecdsa.PrivateKey) -// authorityDID = did.MustParseDID("did:example:authority") -// keyPairs[authorityDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) -// aliceDID = did.MustParseDID("did:example:alice") -// keyPairs[aliceDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) -// bobDID = did.MustParseDID("did:example:bob") -// keyPairs[bobDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) -// -// vcAlice = createCredentialWithClaims(authorityDID, aliceDID, func() []interface{} { -// return []interface{}{ -// map[string]interface{}{ -// "id": aliceDID.String(), -// "person": map[string]interface{}{ -// "givenName": "Alice", -// "familyName": "Jones", -// "city": "InfoSecLand", -// }, -// }, -// } -// }, func(m map[string]interface{}) { -// // do nothing -// }) -// vpAlice = createPresentation(aliceDID, vcAlice) -// vcBob = createCredentialWithClaims(authorityDID, bobDID, func() []interface{} { -// return []interface{}{ -// map[string]interface{}{ -// "id": aliceDID.String(), -// "person": map[string]interface{}{ -// "givenName": "Bob", -// "familyName": "Johansson", -// "city": "InfoSecLand", -// }, -// }, -// } -// }, func(m map[string]interface{}) { -// // do nothing -// }) -// vpBob = createPresentation(bobDID, vcBob) -//} -// -//func createCredential(issuerDID did.DID, subjectDID did.DID) vc.VerifiableCredential { -// return createCredentialWithClaims(issuerDID, subjectDID, -// func() []interface{} { -// return []interface{}{ -// map[string]interface{}{ -// "id": subjectDID.String(), -// }, -// } -// }, -// func(claims map[string]interface{}) { -// // do nothing -// }) -//} -// -//func createCredentialWithClaims(issuerDID did.DID, subjectDID did.DID, credentialSubjectCreator func() []interface{}, claimVisitor func(map[string]interface{})) vc.VerifiableCredential { -// vcID := did.DIDURL{DID: issuerDID} -// vcID.Fragment = uuid.NewString() -// vcIDURI := vcID.URI() -// expirationDate := time.Now().Add(time.Hour * 24) -// -// result, err := vc.CreateJWTVerifiableCredential(context.Background(), vc.VerifiableCredential{ -// ID: &vcIDURI, -// Issuer: issuerDID.URI(), -// Type: []ssi.URI{ssi.MustParseURI("VerifiableCredential"), ssi.MustParseURI("TestCredential")}, -// IssuanceDate: time.Now(), -// ExpirationDate: &expirationDate, -// CredentialSubject: credentialSubjectCreator(), -// }, func(ctx context.Context, claims map[string]interface{}, headers map[string]interface{}) (string, error) { -// claimVisitor(claims) -// return signJWT(subjectDID, claims, headers) -// }) -// if err != nil { -// panic(err) -// } -// return *result -//} -// -//func createPresentation(subjectDID did.DID, credentials ...vc.VerifiableCredential) vc.VerifiablePresentation { -// return createPresentationCustom(subjectDID, func(claims map[string]interface{}) { -// // do nothing -// }, credentials...) -//} -// -//func createPresentationCustom(subjectDID did.DID, claimVisitor func(map[string]interface{}), credentials ...vc.VerifiableCredential) vc.VerifiablePresentation { -// headers := map[string]interface{}{ -// jws.TypeKey: "JWT", -// } -// claims := map[string]interface{}{ -// jwt.IssuerKey: subjectDID.String(), -// jwt.SubjectKey: subjectDID.String(), -// jwt.JwtIDKey: subjectDID.String() + "#" + uuid.NewString(), -// "vp": vc.VerifiablePresentation{ -// Type: append([]ssi.URI{ssi.MustParseURI("VerifiablePresentation")}), -// VerifiableCredential: credentials, -// }, -// jwt.NotBeforeKey: time.Now().Unix(), -// jwt.ExpirationKey: time.Now().Add(time.Hour * 8), -// } -// claimVisitor(claims) -// token, err := signJWT(subjectDID, claims, headers) -// if err != nil { -// panic(err) -// } -// presentation, err := vc.ParseVerifiablePresentation(token) -// if err != nil { -// panic(err) -// } -// return *presentation -//} -// -//func signJWT(subjectDID did.DID, claims map[string]interface{}, headers map[string]interface{}) (string, error) { -// // Build JWK -// signingKey := keyPairs[subjectDID.String()] -// if signingKey == nil { -// return "", fmt.Errorf("key not found for DID: %s", subjectDID) -// } -// subjectKeyJWK, err := jwk.FromRaw(signingKey) -// if err != nil { -// return "", nil -// } -// keyID := did.DIDURL{DID: subjectDID} -// keyID.Fragment = "0" -// if err := subjectKeyJWK.Set(jwk.AlgorithmKey, jwa.ES256); err != nil { -// return "", err -// } -// if err := subjectKeyJWK.Set(jwk.KeyIDKey, keyID.String()); err != nil { -// return "", err -// } -// -// // Build token -// token := jwt.New() -// for k, v := range claims { -// if err := token.Set(k, v); err != nil { -// return "", err -// } -// } -// hdr := jws.NewHeaders() -// for k, v := range headers { -// if err := hdr.Set(k, v); err != nil { -// return "", err -// } -// } -// bytes, err := jwt.Sign(token, jwt.WithKey(jwa.ES256, signingKey, jws.WithProtectedHeaders(hdr))) -// return string(bytes), err -//} diff --git a/discoveryservice/module.go b/discoveryservice/module.go index 1a4852cb98..93be32ec21 100644 --- a/discoveryservice/module.go +++ b/discoveryservice/module.go @@ -67,15 +67,6 @@ func (m *Module) Configure(_ core.ServerConfig) error { if err != nil { return err } - return nil -} - -func (m *Module) Start() error { - var err error - m.store, err = newSQLStore(m.storageInstance.GetSQLDatabase(), m.services) - if err != nil { - return err - } if len(m.config.Server.DefinitionIDs) > 0 { // Get the definitions that are enabled for this server serverDefinitions := make(map[string]Definition) @@ -91,6 +82,15 @@ func (m *Module) Start() error { return nil } +func (m *Module) Start() error { + var err error + m.store, err = newSQLStore(m.storageInstance.GetSQLDatabase(), m.services) + if err != nil { + return err + } + return nil +} + func (m *Module) Shutdown() error { return nil } @@ -140,14 +140,14 @@ func (m *Module) Add(serviceID string, presentation vc.VerifiablePresentation) e func (m *Module) addPresentation(definition Definition, presentation vc.VerifiablePresentation) error { // Must contain credentials if len(presentation.VerifiableCredential) == 0 { - return errors.New("presentation must contain at least one credential") + return errors.New("presentation must contain at least one credentialRecord") } - // VP can't be valid longer than the credential it contains + // VP can't be valid longer than the credentialRecord it contains expiration := presentation.JWT().Expiration() for _, cred := range presentation.VerifiableCredential { exp := cred.JWT().Expiration() if !exp.IsZero() && expiration.After(exp) { - return fmt.Errorf("presentation is valid longer than the credential(s) it contains") + return fmt.Errorf("presentation is valid longer than the credentialRecord(s) it contains") } } // VP must fulfill the PEX Presentation Definition @@ -159,7 +159,7 @@ func (m *Module) addPresentation(definition Definition, presentation vc.Verifiab } func (m *Module) addRetraction(serviceID string, presentation vc.VerifiablePresentation) error { - // Presentation might be a retraction (deletion of an earlier credential) must contain no credentials, and refer to the VP being retracted by ID. + // Presentation might be a retraction (deletion of an earlier credentialRecord) must contain no credentials, and refer to the VP being retracted by ID. // If those conditions aren't met, we don't need to register the retraction. if len(presentation.VerifiableCredential) > 0 { return errors.New("retraction presentation must not contain credentials") diff --git a/discoveryservice/module_test.go b/discoveryservice/module_test.go index a978da620a..416c85258a 100644 --- a/discoveryservice/module_test.go +++ b/discoveryservice/module_test.go @@ -51,7 +51,7 @@ func Test_Module_Add(t *testing.T) { require.NoError(t, err) assert.Equal(t, Timestamp(1), *timestamp) }) - t.Run("replace presentation of same credential subject", func(t *testing.T) { + t.Run("replace presentation of same credentialRecord subject", func(t *testing.T) { m := setupModule(t, storageEngine) vpAlice2 := createPresentation(aliceDID, vcAlice) @@ -84,12 +84,12 @@ func Test_Module_Add(t *testing.T) { t.Run("valid longer than its credentials", func(t *testing.T) { m := setupModule(t, storageEngine) - vcAlice := createCredentialWithClaims(authorityDID, aliceDID, func(claims map[string]interface{}) { + vcAlice := createCredential(authorityDID, aliceDID, nil, func(claims map[string]interface{}) { claims["exp"] = time.Now().Add(time.Hour) }) vpAlice := createPresentation(aliceDID, vcAlice) err := m.Add(testServiceID, vpAlice) - assert.EqualError(t, err, "presentation is valid longer than the credential(s) it contains") + assert.EqualError(t, err, "presentation is valid longer than the credentialRecord(s) it contains") }) t.Run("not valid long enough", func(t *testing.T) { m := setupModule(t, storageEngine) @@ -186,14 +186,6 @@ func Test_Module_Add(t *testing.T) { func Test_Module_Get(t *testing.T) { storageEngine := storage.NewTestStorageEngine(t) require.NoError(t, storageEngine.Start()) - - t.Run("empty list, empty timestamp", func(t *testing.T) { - m := setupModule(t, storageEngine) - presentations, timestamp, err := m.Get(testServiceID, 0) - assert.NoError(t, err) - assert.Empty(t, presentations) - assert.Empty(t, timestamp) - }) t.Run("1 entry, empty timestamp", func(t *testing.T) { m := setupModule(t, storageEngine) require.NoError(t, m.Add(testServiceID, vpAlice)) @@ -202,33 +194,6 @@ func Test_Module_Get(t *testing.T) { assert.Equal(t, []vc.VerifiablePresentation{vpAlice}, presentations) assert.Equal(t, Timestamp(1), *timestamp) }) - t.Run("2 entries, empty timestamp", func(t *testing.T) { - m := setupModule(t, storageEngine) - require.NoError(t, m.Add(testServiceID, vpAlice)) - require.NoError(t, m.Add(testServiceID, vpBob)) - presentations, timestamp, err := m.Get(testServiceID, 0) - assert.NoError(t, err) - assert.Equal(t, []vc.VerifiablePresentation{vpAlice, vpBob}, presentations) - assert.Equal(t, Timestamp(2), *timestamp) - }) - t.Run("2 entries, start after first", func(t *testing.T) { - m := setupModule(t, storageEngine) - require.NoError(t, m.Add(testServiceID, vpAlice)) - require.NoError(t, m.Add(testServiceID, vpBob)) - presentations, timestamp, err := m.Get(testServiceID, 1) - assert.NoError(t, err) - assert.Equal(t, []vc.VerifiablePresentation{vpBob}, presentations) - assert.Equal(t, Timestamp(2), *timestamp) - }) - t.Run("2 entries, start after end", func(t *testing.T) { - m := setupModule(t, storageEngine) - require.NoError(t, m.Add(testServiceID, vpAlice)) - require.NoError(t, m.Add(testServiceID, vpBob)) - presentations, timestamp, err := m.Get(testServiceID, 2) - assert.NoError(t, err) - assert.Equal(t, []vc.VerifiablePresentation{}, presentations) - assert.Equal(t, Timestamp(2), *timestamp) - }) t.Run("service unknown", func(t *testing.T) { m := setupModule(t, storageEngine) _, _, err := m.Get("unknown", 0) @@ -237,7 +202,7 @@ func Test_Module_Get(t *testing.T) { } func setupModule(t *testing.T, storageInstance storage.Engine) *Module { - resetStoreAfterTest(t, storageInstance.GetSQLDatabase()) + resetStore(t, storageInstance.GetSQLDatabase()) m := New(storageInstance) require.NoError(t, m.Configure(core.ServerConfig{})) m.services = testDefinitions() @@ -248,25 +213,42 @@ func setupModule(t *testing.T, storageInstance storage.Engine) *Module { return m } -func Test_loadDefinitions(t *testing.T) { +func TestModule_Configure(t *testing.T) { + serverConfig := core.ServerConfig{} t.Run("duplicate ID", func(t *testing.T) { - definitions, err := loadDefinitions("test/duplicate_id") + config := Config{ + Definitions: DefinitionsConfig{ + Directory: "test/duplicate_id", + }, + } + err := (&Module{config: config}).Configure(serverConfig) assert.EqualError(t, err, "duplicate definition ID 'urn:nuts.nl:usecase:eOverdrachtDev2023' in file 'test/duplicate_id/2.json'") - assert.Nil(t, definitions) }) t.Run("invalid JSON", func(t *testing.T) { - definitions, err := loadDefinitions("test/invalid_json") + config := Config{ + Definitions: DefinitionsConfig{ + Directory: "test/invalid_json", + }, + } + err := (&Module{config: config}).Configure(serverConfig) assert.ErrorContains(t, err, "unable to parse definition file 'test/invalid_json/1.json'") - assert.Nil(t, definitions) }) t.Run("invalid definition", func(t *testing.T) { - definitions, err := loadDefinitions("test/invalid_definition") + config := Config{ + Definitions: DefinitionsConfig{ + Directory: "test/invalid_definition", + }, + } + err := (&Module{config: config}).Configure(serverConfig) assert.ErrorContains(t, err, "unable to parse definition file 'test/invalid_definition/1.json'") - assert.Nil(t, definitions) }) t.Run("non-existent directory", func(t *testing.T) { - definitions, err := loadDefinitions("test/non_existent") + config := Config{ + Definitions: DefinitionsConfig{ + Directory: "test/non_existent", + }, + } + err := (&Module{config: config}).Configure(serverConfig) assert.ErrorContains(t, err, "unable to read definitions directory 'test/non_existent'") - assert.Nil(t, definitions) }) } diff --git a/discoveryservice/store.go b/discoveryservice/store.go index dd53898e9a..d44ea31cbf 100644 --- a/discoveryservice/store.go +++ b/discoveryservice/store.go @@ -24,28 +24,30 @@ import ( "github.com/google/uuid" "github.com/nuts-foundation/go-did/vc" "github.com/nuts-foundation/nuts-node/discoveryservice/log" - credential2 "github.com/nuts-foundation/nuts-node/vcr/credential" + credential "github.com/nuts-foundation/nuts-node/vcr/credential" "gorm.io/gorm" "gorm.io/gorm/clause" "gorm.io/gorm/schema" + "strconv" + "strings" "time" ) var ErrServiceNotFound = errors.New("discovery service not found") var ErrPresentationAlreadyExists = errors.New("presentation already exists") -type discoveryService struct { +type serviceRecord struct { ID string `gorm:"primaryKey"` Timestamp uint64 } -func (s discoveryService) TableName() string { +func (s serviceRecord) TableName() string { return "discoveryservices" } -var _ schema.Tabler = (*servicePresentation)(nil) +var _ schema.Tabler = (*presentationRecord)(nil) -type servicePresentation struct { +type presentationRecord struct { ID string `gorm:"primaryKey"` ServiceID string Timestamp uint64 @@ -53,18 +55,18 @@ type servicePresentation struct { PresentationID string PresentationRaw string PresentationExpiration int64 - Credentials []credential `gorm:"foreignKey:PresentationID;references:ID"` + Credentials []credentialRecord `gorm:"foreignKey:PresentationID;references:ID"` } -func (s servicePresentation) TableName() string { +func (s presentationRecord) TableName() string { return "discoveryservice_presentations" } -// credential is a Verifiable Credential, part of a presentation (entry) on a use case list. -type credential struct { +// credentialRecord is a Verifiable Credential, part of a presentation (entry) on a use case list. +type credentialRecord struct { // ID is the unique identifier of the entry. ID string `gorm:"primaryKey"` - // PresentationID corresponds to the discoveryservice_presentations record ID (not VerifiablePresentation.ID) this credential belongs to. + // PresentationID corresponds to the discoveryservice_presentations record ID (not VerifiablePresentation.ID) this credentialRecord belongs to. PresentationID string // CredentialID contains the 'id' property of the Verifiable Credential. CredentialID string @@ -74,16 +76,16 @@ type credential struct { CredentialSubjectID string // CredentialType contains the 'type' property of the Verifiable Credential (not being 'VerifiableCredential'). CredentialType *string - Properties []credentialProperty `gorm:"foreignKey:ID;references:ID"` + Properties []credentialPropertyRecord `gorm:"foreignKey:ID;references:ID"` } // TableName returns the table name for this DTO. -func (p credential) TableName() string { +func (p credentialRecord) TableName() string { return "discoveryservice_credentials" } -// credentialProperty is a property of a Verifiable Credential in a Verifiable Presentation in a discovery service. -type credentialProperty struct { +// credentialPropertyRecord is a property of a Verifiable Credential in a Verifiable Presentation in a discovery service. +type credentialPropertyRecord struct { // ID refers to the entry record in discoveryservice_credentials ID string `gorm:"primaryKey"` // Key is JSON path of the property. @@ -93,7 +95,7 @@ type credentialProperty struct { } // TableName returns the table name for this DTO. -func (l credentialProperty) TableName() string { +func (l credentialPropertyRecord) TableName() string { return "discoveryservice_credential_props" } @@ -104,7 +106,7 @@ type sqlStore struct { func newSQLStore(db *gorm.DB, definitions map[string]Definition) (*sqlStore, error) { // Creates entries in the discovery service table with initial timestamp, if they don't exist yet for _, definition := range definitions { - currentList := discoveryService{ + currentList := serviceRecord{ ID: definition.ID, } if err := db.FirstOrCreate(¤tList, "id = ?", definition.ID).Error; err != nil { @@ -121,7 +123,7 @@ func newSQLStore(db *gorm.DB, definitions map[string]Definition) (*sqlStore, err // If the local node is the Discovery Server and thus is responsible for the timestamping, // nil should be passed to let the store determine the right value. func (s *sqlStore) add(serviceID string, presentation vc.VerifiablePresentation, timestamp *Timestamp) error { - credentialSubjectID, err := credential2.PresentationSigner(presentation) + credentialSubjectID, err := credential.PresentationSigner(presentation) if err != nil { return err } @@ -135,30 +137,87 @@ func (s *sqlStore) add(serviceID string, presentation vc.VerifiablePresentation, } return s.db.Transaction(func(tx *gorm.DB) error { - timestamp, err := s.updateTimestamp(tx, serviceID, timestamp) + newTimestamp, err := s.updateTimestamp(tx, serviceID, timestamp) if err != nil { return err } // Delete any previous presentations of the subject - if err := tx.Delete(&servicePresentation{}, "service_id = ? AND credential_subject_id = ?", serviceID, credentialSubjectID.String()). + if err := tx.Delete(&presentationRecord{}, "service_id = ? AND credential_subject_id = ?", serviceID, credentialSubjectID.String()). Error; err != nil { return err } - // Now store the presentation itself - return tx.Create(&servicePresentation{ - ID: uuid.NewString(), - ServiceID: serviceID, - Timestamp: uint64(timestamp), - CredentialSubjectID: credentialSubjectID.String(), - PresentationID: presentation.ID.String(), - PresentationRaw: presentation.Raw(), - PresentationExpiration: presentation.JWT().Expiration().Unix(), - }).Error + + newPresentation, err := createPresentationRecord(serviceID, newTimestamp, presentation) + if err != nil { + return err + } + + return tx.Create(&newPresentation).Error }) } +// createPresentationRecord creates a presentationRecord from a VerifiablePresentation. +// It creates the following types: +// - presentationRecord +// - presentationRecord.Credentials with credentialRecords of the credentials in the presentation +// - presentationRecord.Credentials.Properties of the credentialSubject properties of the credential (for s +func createPresentationRecord(serviceID string, timestamp Timestamp, presentation vc.VerifiablePresentation) (*presentationRecord, error) { + credentialSubjectID, err := credential.PresentationSigner(presentation) + if err != nil { + return nil, err + } + + newPresentation := presentationRecord{ + ID: uuid.NewString(), + ServiceID: serviceID, + Timestamp: uint64(timestamp), + CredentialSubjectID: credentialSubjectID.String(), + PresentationID: presentation.ID.String(), + PresentationRaw: presentation.Raw(), + PresentationExpiration: presentation.JWT().Expiration().Unix(), + } + + for _, currCred := range presentation.VerifiableCredential { + var credentialType *string + for _, currType := range currCred.Type { + if currType.String() != "VerifiableCredential" { + credentialType = new(string) + *credentialType = currType.String() + break + } + } + if len(currCred.CredentialSubject) != 1 { + return nil, errors.New("credential must contain exactly one subject") + } + + newCredential := credentialRecord{ + ID: uuid.NewString(), + PresentationID: newPresentation.ID, + CredentialID: currCred.ID.String(), + CredentialIssuer: currCred.Issuer.String(), + CredentialSubjectID: credentialSubjectID.String(), + CredentialType: credentialType, + } + // Store credential's properties + keys, values := indexJSONObject(currCred.CredentialSubject[0].(map[string]interface{}), nil, nil, "credentialSubject") + for i, key := range keys { + if key == "credentialSubject.id" { + // present as column, don't index + continue + } + newCredential.Properties = append(newCredential.Properties, credentialPropertyRecord{ + ID: newCredential.ID, + Key: key, + Value: values[i], + }) + } + newPresentation.Credentials = append(newPresentation.Credentials, newCredential) + } + return &newPresentation, nil +} + func (s *sqlStore) get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresentation, *Timestamp, error) { - var rows []servicePresentation + var rows []presentationRecord err := s.db.Order("timestamp ASC").Find(&rows, "service_id = ? AND timestamp > ?", serviceID, int(startAt)).Error if err != nil { return nil, nil, fmt.Errorf("query service '%s': %w", serviceID, err) @@ -176,11 +235,66 @@ func (s *sqlStore) get(serviceID string, startAt Timestamp) ([]vc.VerifiablePres return presentations, ×tamp, nil } +func (s *sqlStore) search(serviceID string, query map[string]string) ([]vc.VerifiablePresentation, error) { + propertyColumns := map[string]string{ + "id": "cred.credential_id", + "issuer": "cred.credential_issuer", + "type": "cred.credential_type", + "credentialSubject.id": "cred.credential_subject_id", + } + + stmt := s.db.Model(&presentationRecord{}). + Where("service_id = ?", serviceID). + Joins("inner join discoveryservice_credentials cred ON cred.presentation_id = discoveryservice_presentations.id") + numProps := 0 + for jsonPath, value := range query { + if value == "*" { + continue + } + // sort out wildcard mode + var eq = "=" + if strings.HasPrefix(value, "*") { + value = "%" + value[1:] + eq = "LIKE" + } + if strings.HasSuffix(value, "*") { + value = value[:len(value)-1] + "%" + eq = "LIKE" + } + if column := propertyColumns[jsonPath]; column != "" { + stmt = stmt.Where(column+" "+eq+" ?", value) + } else { + // This property is not present as column, but indexed as key-value property. + // Multiple (inner) joins to filter on a dynamic number of properties to filter on is not pretty, but it works + alias := "p" + strconv.Itoa(numProps) + numProps++ + stmt = stmt.Joins("inner join discoveryservice_credential_props "+alias+" ON "+alias+".id = cred.id AND "+alias+".key = ? AND "+alias+".value "+eq+" ?", jsonPath, value) + } + } + + var matches []presentationRecord + if err := stmt.Find(&matches).Error; err != nil { + return nil, err + } + var results []vc.VerifiablePresentation + for _, match := range matches { + if match.PresentationExpiration <= time.Now().Unix() { + continue + } + presentation, err := vc.ParseVerifiablePresentation(match.PresentationRaw) + if err != nil { + return nil, fmt.Errorf("failed to parse presentation '%s': %w", match.PresentationID, err) + } + results = append(results, *presentation) + } + return results, nil +} + func (s *sqlStore) updateTimestamp(tx *gorm.DB, serviceID string, newTimestamp *Timestamp) (Timestamp, error) { - var result discoveryService + var result serviceRecord // Lock (SELECT FOR UPDATE) discoveryservices row to prevent concurrent updates to the same list, which could mess up the lamport timestamp. if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). - Where(discoveryService{ID: serviceID}). + Where(serviceRecord{ID: serviceID}). Find(&result). Error; err != nil { return 0, err @@ -200,7 +314,7 @@ func (s *sqlStore) updateTimestamp(tx *gorm.DB, serviceID string, newTimestamp * func (s *sqlStore) exists(serviceID string, credentialSubjectID string, presentationID string) (bool, error) { var count int64 - if err := s.db.Model(servicePresentation{}).Where(servicePresentation{ + if err := s.db.Model(presentationRecord{}).Where(presentationRecord{ ServiceID: serviceID, CredentialSubjectID: credentialSubjectID, PresentationID: presentationID, @@ -222,9 +336,32 @@ func (s *sqlStore) prune() error { } func (s *sqlStore) removeExpired() (int, error) { - result := s.db.Where("presentation_expiration < ?", time.Now().Unix()).Delete(servicePresentation{}) + result := s.db.Where("presentation_expiration < ?", time.Now().Unix()).Delete(presentationRecord{}) if result.Error != nil { return 0, fmt.Errorf("prune presentations: %w", result.Error) } return int(result.RowsAffected), nil } + +// indexJSONObject indexes a JSON object, resulting in a slice of JSON paths and corresponding string values. +// It only traverses JSON objects and only adds string values to the result. +func indexJSONObject(target map[string]interface{}, jsonPaths []string, stringValues []string, currentPath string) ([]string, []string) { + for key, value := range target { + thisPath := currentPath + if len(thisPath) > 0 { + thisPath += "." + } + thisPath += key + + switch typedValue := value.(type) { + case string: + jsonPaths = append(jsonPaths, thisPath) + stringValues = append(stringValues, typedValue) + case map[string]interface{}: + jsonPaths, stringValues = indexJSONObject(typedValue, jsonPaths, stringValues, thisPath) + default: + // other values (arrays, booleans, numbers, null) are not indexed + } + } + return jsonPaths, stringValues +} diff --git a/discoveryservice/store_test.go b/discoveryservice/store_test.go index 1627a6d446..5828d8d5f7 100644 --- a/discoveryservice/store_test.go +++ b/discoveryservice/store_test.go @@ -19,6 +19,7 @@ package discoveryservice import ( + "github.com/nuts-foundation/go-did/vc" "github.com/nuts-foundation/nuts-node/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -59,20 +60,282 @@ func Test_sqlStore_exists(t *testing.T) { }) } +func Test_sqlStore_add(t *testing.T) { + storageEngine := storage.NewTestStorageEngine(t) + require.NoError(t, storageEngine.Start()) + + t.Run("no credentials in presentation", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + err := m.add(testServiceID, createPresentation(aliceDID), nil) + assert.NoError(t, err) + }) + t.Run("with indexable properties in credential", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + err := m.add(testServiceID, createPresentation(aliceDID, createCredential(authorityDID, aliceDID, map[string]interface{}{ + "name": "Alice", + "placeOfBirth": "Bristol", + }, nil)), nil) + assert.NoError(t, err) + + var actual []credentialPropertyRecord + assert.NoError(t, m.db.Find(&actual).Error) + require.Len(t, actual, 2) + assert.Equal(t, "Alice", sliceToMap(actual)["credentialSubject.name"]) + assert.Equal(t, "Bristol", sliceToMap(actual)["credentialSubject.placeOfBirth"]) + }) + t.Run("with non-indexable properties in credential", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + err := m.add(testServiceID, createPresentation(aliceDID, createCredential(authorityDID, aliceDID, map[string]interface{}{ + "name": "Alice", + "age": 35, + }, nil)), nil) + assert.NoError(t, err) + + var actual []credentialPropertyRecord + assert.NoError(t, m.db.Find(&actual).Error) + require.Len(t, actual, 1) + assert.Equal(t, "Alice", sliceToMap(actual)["credentialSubject.name"]) + }) + t.Run("without indexable properties in credential", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + presentation := createCredential(authorityDID, aliceDID, map[string]interface{}{}, nil) + err := m.add(testServiceID, createPresentation(aliceDID, presentation), nil) + assert.NoError(t, err) + + var actual []credentialPropertyRecord + assert.NoError(t, m.db.Find(&actual).Error) + assert.Empty(t, actual) + }) +} + +func Test_sqlStore_get(t *testing.T) { + storageEngine := storage.NewTestStorageEngine(t) + require.NoError(t, storageEngine.Start()) + + t.Run("empty list, empty timestamp", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + presentations, timestamp, err := m.get(testServiceID, 0) + assert.NoError(t, err) + assert.Empty(t, presentations) + assert.Empty(t, timestamp) + }) + t.Run("1 entry, empty timestamp", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + require.NoError(t, m.add(testServiceID, vpAlice, nil)) + presentations, timestamp, err := m.get(testServiceID, 0) + assert.NoError(t, err) + assert.Equal(t, []vc.VerifiablePresentation{vpAlice}, presentations) + assert.Equal(t, Timestamp(1), *timestamp) + }) + t.Run("2 entries, empty timestamp", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + require.NoError(t, m.add(testServiceID, vpAlice, nil)) + require.NoError(t, m.add(testServiceID, vpBob, nil)) + presentations, timestamp, err := m.get(testServiceID, 0) + assert.NoError(t, err) + assert.Equal(t, []vc.VerifiablePresentation{vpAlice, vpBob}, presentations) + assert.Equal(t, Timestamp(2), *timestamp) + }) + t.Run("2 entries, start after first", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + require.NoError(t, m.add(testServiceID, vpAlice, nil)) + require.NoError(t, m.add(testServiceID, vpBob, nil)) + presentations, timestamp, err := m.get(testServiceID, 1) + assert.NoError(t, err) + assert.Equal(t, []vc.VerifiablePresentation{vpBob}, presentations) + assert.Equal(t, Timestamp(2), *timestamp) + }) + t.Run("2 entries, start after end", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + require.NoError(t, m.add(testServiceID, vpAlice, nil)) + require.NoError(t, m.add(testServiceID, vpBob, nil)) + presentations, timestamp, err := m.get(testServiceID, 2) + assert.NoError(t, err) + assert.Equal(t, []vc.VerifiablePresentation{}, presentations) + assert.Equal(t, Timestamp(2), *timestamp) + }) +} + +func Test_sqlStore_search(t *testing.T) { + storageEngine := storage.NewTestStorageEngine(t) + require.NoError(t, storageEngine.Start()) + t.Cleanup(func() { + _ = storageEngine.Shutdown() + }) + + type testCase struct { + name string + inputVPs []vc.VerifiablePresentation + query map[string]string + expectedVPs []string + } + testCases := []testCase{ + { + name: "issuer", + inputVPs: []vc.VerifiablePresentation{vpAlice}, + query: map[string]string{ + "issuer": authorityDID.String(), + }, + expectedVPs: []string{vpAlice.ID.String()}, + }, + { + name: "id", + inputVPs: []vc.VerifiablePresentation{vpAlice}, + query: map[string]string{ + "id": vcAlice.ID.String(), + }, + expectedVPs: []string{vpAlice.ID.String()}, + }, + { + name: "type", + inputVPs: []vc.VerifiablePresentation{vpAlice}, + query: map[string]string{ + "type": "TestCredential", + }, + expectedVPs: []string{vpAlice.ID.String()}, + }, + { + name: "credentialSubject.id", + inputVPs: []vc.VerifiablePresentation{vpAlice}, + query: map[string]string{ + "credentialSubject.id": aliceDID.String(), + }, + expectedVPs: []string{vpAlice.ID.String()}, + }, + { + name: "1 property", + inputVPs: []vc.VerifiablePresentation{vpAlice}, + query: map[string]string{ + "credentialSubject.person.givenName": "Alice", + }, + expectedVPs: []string{vpAlice.ID.String()}, + }, + { + name: "2 properties", + inputVPs: []vc.VerifiablePresentation{vpAlice}, + query: map[string]string{ + "credentialSubject.person.givenName": "Alice", + "credentialSubject.person.familyName": "Jones", + }, + expectedVPs: []string{vpAlice.ID.String()}, + }, + { + name: "properties and base properties", + inputVPs: []vc.VerifiablePresentation{vpAlice}, + query: map[string]string{ + "issuer": authorityDID.String(), + "credentialSubject.person.givenName": "Alice", + }, + expectedVPs: []string{vpAlice.ID.String()}, + }, + { + name: "wildcard postfix", + inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, + query: map[string]string{ + "credentialSubject.person.familyName": "Jo*", + }, + expectedVPs: []string{vpAlice.ID.String(), vpBob.ID.String()}, + }, + { + name: "wildcard prefix", + inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, + query: map[string]string{ + "credentialSubject.person.givenName": "*ce", + }, + expectedVPs: []string{vpAlice.ID.String()}, + }, + { + name: "wildcard midway (no interpreted as wildcard)", + inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, + query: map[string]string{ + "credentialSubject.person.givenName": "A*ce", + }, + expectedVPs: []string{}, + }, + { + name: "just wildcard", + inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, + query: map[string]string{ + "id": "*", + }, + expectedVPs: []string{vpAlice.ID.String(), vpBob.ID.String()}, + }, + { + name: "2 VPs, 1 match", + inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, + query: map[string]string{ + "credentialSubject.person.givenName": "Alice", + }, + expectedVPs: []string{vpAlice.ID.String()}, + }, + { + name: "multiple matches", + inputVPs: []vc.VerifiablePresentation{vpAlice, vpBob}, + query: map[string]string{ + "issuer": authorityDID.String(), + }, + expectedVPs: []string{vpAlice.ID.String(), vpBob.ID.String()}, + }, + { + name: "no match", + inputVPs: []vc.VerifiablePresentation{vpAlice}, + query: map[string]string{ + "credentialSubject.person.givenName": "Bob", + }, + expectedVPs: []string{}, + }, + { + name: "empty database", + query: map[string]string{ + "credentialSubject.person.givenName": "Bob", + }, + expectedVPs: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + c := setupStore(t, storageEngine.GetSQLDatabase()) + for _, vp := range tc.inputVPs { + err := c.add(testServiceID, vp, nil) + require.NoError(t, err) + } + actualVPs, err := c.search(testServiceID, tc.query) + require.NoError(t, err) + require.Len(t, actualVPs, len(tc.expectedVPs)) + for _, expectedVP := range tc.expectedVPs { + found := false + for _, actualVP := range actualVPs { + if actualVP.ID.String() == expectedVP { + found = true + break + } + } + require.True(t, found, "expected to find VP with ID %s", expectedVP) + } + }) + } +} + func setupStore(t *testing.T, db *gorm.DB) *sqlStore { - resetStoreAfterTest(t, db) + resetStore(t, db) store, err := newSQLStore(db, testDefinitions()) require.NoError(t, err) return store } -func resetStoreAfterTest(t *testing.T, db *gorm.DB) { - t.Cleanup(func() { - underlyingDB, err := db.DB() - require.NoError(t, err) - _, err = underlyingDB.Exec("DELETE FROM discoveryservice_presentations") - require.NoError(t, err) - _, err = underlyingDB.Exec("DELETE FROM discoveryservices") - require.NoError(t, err) - }) +func resetStore(t *testing.T, db *gorm.DB) { + underlyingDB, err := db.DB() + require.NoError(t, err) + // related tables are emptied due to on-delete-cascade clause + _, err = underlyingDB.Exec("DELETE FROM discoveryservices") + require.NoError(t, err) +} + +func sliceToMap(slice []credentialPropertyRecord) map[string]string { + var result = make(map[string]string) + for _, curr := range slice { + result[curr.Key] = curr.Value + } + return result } diff --git a/discoveryservice/test.go b/discoveryservice/test.go index 43f74082f8..38764f7da5 100644 --- a/discoveryservice/test.go +++ b/discoveryservice/test.go @@ -103,35 +103,42 @@ func init() { bobDID = did.MustParseDID("did:example:bob") keyPairs[bobDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - vcAlice = createCredential(authorityDID, aliceDID) + vcAlice = createCredential(authorityDID, aliceDID, map[string]interface{}{ + "person": map[string]interface{}{ + "givenName": "Alice", + "familyName": "Jones", + }, + }, nil) vpAlice = createPresentation(aliceDID, vcAlice) - vcBob = createCredential(authorityDID, bobDID) + vcBob = createCredential(authorityDID, bobDID, map[string]interface{}{ + "person": map[string]interface{}{ + "givenName": "Bob", + "familyName": "Jomper", + }, + }, nil) vpBob = createPresentation(bobDID, vcBob) } -func createCredential(issuerDID did.DID, subjectDID did.DID) vc.VerifiableCredential { - return createCredentialWithClaims(issuerDID, subjectDID, func(claims map[string]interface{}) { - // do nothing - }) -} - -func createCredentialWithClaims(issuerDID did.DID, subjectDID did.DID, claimVisitor func(map[string]interface{})) vc.VerifiableCredential { +func createCredential(issuerDID did.DID, subjectDID did.DID, credentialSubject map[string]interface{}, claimVisitor func(map[string]interface{})) vc.VerifiableCredential { vcID := did.DIDURL{DID: issuerDID} vcID.Fragment = uuid.NewString() vcIDURI := vcID.URI() expirationDate := time.Now().Add(time.Hour * 24) + if credentialSubject == nil { + credentialSubject = make(map[string]interface{}) + } + credentialSubject["id"] = subjectDID.String() result, err := vc.CreateJWTVerifiableCredential(context.Background(), vc.VerifiableCredential{ - ID: &vcIDURI, - Issuer: issuerDID.URI(), - IssuanceDate: time.Now(), - ExpirationDate: &expirationDate, - CredentialSubject: []interface{}{ - map[string]interface{}{ - "id": subjectDID.String(), - }, - }, + ID: &vcIDURI, + Type: []ssi.URI{ssi.MustParseURI("VerifiableCredential"), ssi.MustParseURI("TestCredential")}, + Issuer: issuerDID.URI(), + IssuanceDate: time.Now(), + ExpirationDate: &expirationDate, + CredentialSubject: []interface{}{credentialSubject}, }, func(ctx context.Context, claims map[string]interface{}, headers map[string]interface{}) (string, error) { - claimVisitor(claims) + if claimVisitor != nil { + claimVisitor(claims) + } return signJWT(subjectDID, claims, headers) }) if err != nil { diff --git a/docs/pages/deployment/cli-reference.rst b/docs/pages/deployment/cli-reference.rst index ca69be9aee..abd69714d5 100755 --- a/docs/pages/deployment/cli-reference.rst +++ b/docs/pages/deployment/cli-reference.rst @@ -44,7 +44,7 @@ The following options apply to the server commands below: --http.default.log string What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). (default "metadata") --http.default.tls string Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', --internalratelimiter When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. (default true) - --jsonld.contexts.localmapping stringToString This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. (default [https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson,https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson]) + --jsonld.contexts.localmapping stringToString This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. (default [https://schema.org=assets/contexts/schema-org-v13.ldjson,https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson]) --jsonld.contexts.remoteallowlist strings In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. (default [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json]) --loggerformat string Log format (text, json) (default "text") --network.bootstrapnodes strings List of bootstrap nodes (':') which the node initially connect to. @@ -70,7 +70,7 @@ The following options apply to the server commands below: --storage.redis.sentinel.username string Username for authenticating to Redis Sentinels. --storage.redis.tls.truststorefile string PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). --storage.redis.username string Redis database username. If set, it overrides the username in the connection URL. - --storage.sql.connection string Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory + --storage.sql.connection string Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' option. --strictmode When set, insecure settings are forbidden. (default true) --tls.certfile string PEM file containing the certificate for the server (also used as client certificate). --tls.certheader string Name of the HTTP header that will contain the client certificate when TLS is offloaded. diff --git a/docs/pages/deployment/server_options.rst b/docs/pages/deployment/server_options.rst index fbe48394e1..c20776dd0b 100755 --- a/docs/pages/deployment/server_options.rst +++ b/docs/pages/deployment/server_options.rst @@ -2,85 +2,85 @@ :widths: 20 30 50 :class: options-table - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================== - Key Default Description - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================== - configfile nuts.yaml Nuts config file - cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. - datadir ./data Directory where the node stores its files. - internalratelimiter true When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. - loggerformat text Log format (text, json) - strictmode true When set, insecure settings are forbidden. - verbosity info Log level (trace, debug, info, warn, error) - tls.certfile PEM file containing the certificate for the server (also used as client certificate). - tls.certheader Name of the HTTP header that will contain the client certificate when TLS is offloaded. - tls.certkeyfile PEM file containing the private key of the server certificate. - tls.offload Whether to enable TLS offloading for incoming connections. Enable by setting it to 'incoming'. If enabled 'tls.certheader' must be configured as well. - tls.truststorefile truststore.pem PEM file containing the trusted CA certificates for authenticating remote servers. - **Auth** - auth.accesstokenlifespan 60 defines how long (in seconds) an access token is valid. Uses default in strict mode. - auth.clockskew 5000 allowed JWT Clock skew in milliseconds - auth.contractvalidators [irma,uzi,dummy,employeeid] sets the different contract validators to use - auth.publicurl public URL which can be reached by a users IRMA client, this should include the scheme and domain: https://example.com. Additional paths should only be added if some sort of url-rewriting is done in a reverse-proxy. - auth.http.timeout 30 HTTP timeout (in seconds) used by the Auth API HTTP client - auth.irma.autoupdateschemas true set if you want automatically update the IRMA schemas every 60 minutes. - auth.irma.schememanager pbdf IRMA schemeManager to use for attributes. Can be either 'pbdf' or 'irma-demo'. - **Crypto** - crypto.storage fs Storage to use, 'external' for an external backend (experimental), 'fs' for file system (for development purposes), 'vaultkv' for Vault KV store (recommended, will be replaced by external backend in future). - crypto.external.address Address of the external storage service. - crypto.external.timeout 100ms Time-out when invoking the external storage backend, in Golang time.Duration string format (e.g. 1s). - crypto.vault.address The Vault address. If set it overwrites the VAULT_ADDR env var. - crypto.vault.pathprefix kv The Vault path prefix. - crypto.vault.timeout 5s Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). - crypto.vault.token The Vault token. If set it overwrites the VAULT_TOKEN env var. - **Events** - events.nats.hostname 0.0.0.0 Hostname for the NATS server - events.nats.port 4222 Port where the NATS server listens on - events.nats.storagedir Directory where file-backed streams are stored in the NATS server - events.nats.timeout 30 Timeout for NATS server operations - **GoldenHammer** - goldenhammer.enabled true Whether to enable automatically fixing DID documents with the required endpoints. - goldenhammer.interval 10m0s The interval in which to check for DID documents to fix. - **HTTP** - http.default.address \:1323 Address and port the server will be listening to - http.default.log metadata What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). - http.default.tls Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', - http.default.auth.audience Expected audience for JWT tokens (default: hostname) - http.default.auth.authorizedkeyspath Path to an authorized_keys file for trusted JWT signers - http.default.auth.type Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode. - http.default.cors.origin [] When set, enables CORS from the specified origins on the default HTTP interface. - **JSONLD** - jsonld.contexts.localmapping [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. - jsonld.contexts.remoteallowlist [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json] In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. - **Network** - network.bootstrapnodes [] List of bootstrap nodes (':') which the node initially connect to. - network.connectiontimeout 5000 Timeout before an outbound connection attempt times out (in milliseconds). - network.enablediscovery true Whether to enable automatic connecting to other nodes. - network.enabletls true Whether to enable TLS for gRPC connections, which can be disabled for demo/development purposes. It is NOT meant for TLS offloading (see 'tls.offload'). Disabling TLS is not allowed in strict-mode. - network.grpcaddr \:5555 Local address for gRPC to listen on. If empty the gRPC server won't be started and other nodes will not be able to connect to this node (outbound connections can still be made). - network.maxbackoff 24h0m0s Maximum between outbound connections attempts to unresponsive nodes (in Golang duration format, e.g. '1h', '30m'). - network.nodedid Specifies the DID of the organization that operates this node, typically a vendor for EPD software. It is used to identify the node on the network. If the DID document does not exist of is deactivated, the node will not start. - network.protocols [] Specifies the list of network protocols to enable on the server. They are specified by version (1, 2). If not set, all protocols are enabled. - network.v2.diagnosticsinterval 5000 Interval (in milliseconds) that specifies how often the node should broadcast its diagnostic information to other nodes (specify 0 to disable). - network.v2.gossipinterval 5000 Interval (in milliseconds) that specifies how often the node should gossip its new hashes to other nodes. - **PKI** - pki.maxupdatefailhours 4 Maximum number of hours that a denylist update can fail - pki.softfail true Do not reject certificates if their revocation status cannot be established when softfail is true - **Storage** - storage.bbolt.backup.directory Target directory for BBolt database backups. - storage.bbolt.backup.interval 0s Interval, formatted as Golang duration (e.g. 10m, 1h) at which BBolt database backups will be performed. - storage.redis.address Redis database server address. This can be a simple 'host:port' or a Redis connection URL with scheme, auth and other options. - storage.redis.database Redis database name, which is used as prefix every key. Can be used to have multiple instances use the same Redis instance. - storage.redis.password Redis database password. If set, it overrides the username in the connection URL. - storage.redis.username Redis database username. If set, it overrides the username in the connection URL. - storage.redis.sentinel.master Name of the Redis Sentinel master. Setting this property enables Redis Sentinel. - storage.redis.sentinel.nodes [] Addresses of the Redis Sentinels to connect to initially. Setting this property enables Redis Sentinel. - storage.redis.sentinel.password Password for authenticating to Redis Sentinels. - storage.redis.sentinel.username Username for authenticating to Redis Sentinels. - storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). - storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory - **VCR** - vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). - vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. - vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================== + ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= + Key Default Description + ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= + configfile nuts.yaml Nuts config file + cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. + datadir ./data Directory where the node stores its files. + internalratelimiter true When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. + loggerformat text Log format (text, json) + strictmode true When set, insecure settings are forbidden. + verbosity info Log level (trace, debug, info, warn, error) + tls.certfile PEM file containing the certificate for the server (also used as client certificate). + tls.certheader Name of the HTTP header that will contain the client certificate when TLS is offloaded. + tls.certkeyfile PEM file containing the private key of the server certificate. + tls.offload Whether to enable TLS offloading for incoming connections. Enable by setting it to 'incoming'. If enabled 'tls.certheader' must be configured as well. + tls.truststorefile truststore.pem PEM file containing the trusted CA certificates for authenticating remote servers. + **Auth** + auth.accesstokenlifespan 60 defines how long (in seconds) an access token is valid. Uses default in strict mode. + auth.clockskew 5000 allowed JWT Clock skew in milliseconds + auth.contractvalidators [irma,uzi,dummy,employeeid] sets the different contract validators to use + auth.publicurl public URL which can be reached by a users IRMA client, this should include the scheme and domain: https://example.com. Additional paths should only be added if some sort of url-rewriting is done in a reverse-proxy. + auth.http.timeout 30 HTTP timeout (in seconds) used by the Auth API HTTP client + auth.irma.autoupdateschemas true set if you want automatically update the IRMA schemas every 60 minutes. + auth.irma.schememanager pbdf IRMA schemeManager to use for attributes. Can be either 'pbdf' or 'irma-demo'. + **Crypto** + crypto.storage fs Storage to use, 'external' for an external backend (experimental), 'fs' for file system (for development purposes), 'vaultkv' for Vault KV store (recommended, will be replaced by external backend in future). + crypto.external.address Address of the external storage service. + crypto.external.timeout 100ms Time-out when invoking the external storage backend, in Golang time.Duration string format (e.g. 1s). + crypto.vault.address The Vault address. If set it overwrites the VAULT_ADDR env var. + crypto.vault.pathprefix kv The Vault path prefix. + crypto.vault.timeout 5s Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). + crypto.vault.token The Vault token. If set it overwrites the VAULT_TOKEN env var. + **Events** + events.nats.hostname 0.0.0.0 Hostname for the NATS server + events.nats.port 4222 Port where the NATS server listens on + events.nats.storagedir Directory where file-backed streams are stored in the NATS server + events.nats.timeout 30 Timeout for NATS server operations + **GoldenHammer** + goldenhammer.enabled true Whether to enable automatically fixing DID documents with the required endpoints. + goldenhammer.interval 10m0s The interval in which to check for DID documents to fix. + **HTTP** + http.default.address \:1323 Address and port the server will be listening to + http.default.log metadata What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). + http.default.tls Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', + http.default.auth.audience Expected audience for JWT tokens (default: hostname) + http.default.auth.authorizedkeyspath Path to an authorized_keys file for trusted JWT signers + http.default.auth.type Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode. + http.default.cors.origin [] When set, enables CORS from the specified origins on the default HTTP interface. + **JSONLD** + jsonld.contexts.localmapping [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. + jsonld.contexts.remoteallowlist [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json] In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. + **Network** + network.bootstrapnodes [] List of bootstrap nodes (':') which the node initially connect to. + network.connectiontimeout 5000 Timeout before an outbound connection attempt times out (in milliseconds). + network.enablediscovery true Whether to enable automatic connecting to other nodes. + network.enabletls true Whether to enable TLS for gRPC connections, which can be disabled for demo/development purposes. It is NOT meant for TLS offloading (see 'tls.offload'). Disabling TLS is not allowed in strict-mode. + network.grpcaddr \:5555 Local address for gRPC to listen on. If empty the gRPC server won't be started and other nodes will not be able to connect to this node (outbound connections can still be made). + network.maxbackoff 24h0m0s Maximum between outbound connections attempts to unresponsive nodes (in Golang duration format, e.g. '1h', '30m'). + network.nodedid Specifies the DID of the organization that operates this node, typically a vendor for EPD software. It is used to identify the node on the network. If the DID document does not exist of is deactivated, the node will not start. + network.protocols [] Specifies the list of network protocols to enable on the server. They are specified by version (1, 2). If not set, all protocols are enabled. + network.v2.diagnosticsinterval 5000 Interval (in milliseconds) that specifies how often the node should broadcast its diagnostic information to other nodes (specify 0 to disable). + network.v2.gossipinterval 5000 Interval (in milliseconds) that specifies how often the node should gossip its new hashes to other nodes. + **PKI** + pki.maxupdatefailhours 4 Maximum number of hours that a denylist update can fail + pki.softfail true Do not reject certificates if their revocation status cannot be established when softfail is true + **Storage** + storage.bbolt.backup.directory Target directory for BBolt database backups. + storage.bbolt.backup.interval 0s Interval, formatted as Golang duration (e.g. 10m, 1h) at which BBolt database backups will be performed. + storage.redis.address Redis database server address. This can be a simple 'host:port' or a Redis connection URL with scheme, auth and other options. + storage.redis.database Redis database name, which is used as prefix every key. Can be used to have multiple instances use the same Redis instance. + storage.redis.password Redis database password. If set, it overrides the username in the connection URL. + storage.redis.username Redis database username. If set, it overrides the username in the connection URL. + storage.redis.sentinel.master Name of the Redis Sentinel master. Setting this property enables Redis Sentinel. + storage.redis.sentinel.nodes [] Addresses of the Redis Sentinels to connect to initially. Setting this property enables Redis Sentinel. + storage.redis.sentinel.password Password for authenticating to Redis Sentinels. + storage.redis.sentinel.username Username for authenticating to Redis Sentinels. + storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). + storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' option. + **VCR** + vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). + vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. + vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. + ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= diff --git a/storage/cmd/cmd.go b/storage/cmd/cmd.go index 9cf8d59626..5fe495bf2b 100644 --- a/storage/cmd/cmd.go +++ b/storage/cmd/cmd.go @@ -38,6 +38,8 @@ func FlagSet() *pflag.FlagSet { flagSet.StringSlice("storage.redis.sentinel.nodes", defs.Redis.Sentinel.Nodes, "Addresses of the Redis Sentinels to connect to initially. Setting this property enables Redis Sentinel.") flagSet.String("storage.redis.sentinel.username", defs.Redis.Sentinel.Username, "Username for authenticating to Redis Sentinels.") flagSet.String("storage.redis.sentinel.password", defs.Redis.Sentinel.Password, "Password for authenticating to Redis Sentinels.") - flagSet.String("storage.sql.connection", defs.SQL.ConnectionString, "Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory") + flagSet.String("storage.sql.connection", defs.SQL.ConnectionString, "Connection string for the SQL database. "+ + "If not set, it defaults to a SQLite database stored inside the configured data directory. "+ + "If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' query parameter.") return flagSet } diff --git a/storage/engine.go b/storage/engine.go index fecd2db7d6..10f7309960 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -162,7 +162,7 @@ func (e *engine) GetSQLDatabase() *gorm.DB { func (e *engine) initSQLDatabase() error { connectionString := e.config.SQL.ConnectionString if len(connectionString) == 0 { - connectionString = "file:" + path.Join(e.datadir, "sqlite.db") + connectionString = "file:" + path.Join(e.datadir, "sqlite.db?_foreign_keys=on") } var err error e.sqlDB, err = gorm.Open(sqlite.Open(connectionString), &gorm.Config{}) diff --git a/storage/test.go b/storage/test.go index 299fc89faa..9b5820b42a 100644 --- a/storage/test.go +++ b/storage/test.go @@ -29,12 +29,11 @@ import ( ) // SQLiteInMemoryConnectionString is a connection string for an in-memory SQLite database -const SQLiteInMemoryConnectionString = "file::memory:?cache=shared" +const SQLiteInMemoryConnectionString = "file::memory:?cache=shared&_foreign_keys=on" func NewTestStorageEngineInDir(dir string) Engine { result := New().(*engine) result.config.SQL = SQLConfig{ConnectionString: SQLiteInMemoryConnectionString} - //result.config.SQL = SQLConfig{ConnectionString: "file:../../data/sqlite.db"} _ = result.Configure(core.TestServerConfig(core.ServerConfig{Datadir: dir + "/data"})) return result } From ff9b719707d02d2eb0daebe2d30be06bea9968e4 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Mon, 20 Nov 2023 07:05:36 +0100 Subject: [PATCH 03/23] regen --- README.rst | 8 +- docs/pages/deployment/cli-reference.rst | 2 +- docs/pages/deployment/server_options.rst | 164 +++++++++++------------ 3 files changed, 87 insertions(+), 87 deletions(-) diff --git a/README.rst b/README.rst index d5e2702b58..32f22cdc14 100644 --- a/README.rst +++ b/README.rst @@ -176,9 +176,9 @@ The following options can be configured on the server: :widths: 20 30 50 :class: options-table - ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ Key Default Description - ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ configfile nuts.yaml Nuts config file cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. datadir ./data Directory where the node stores its files. @@ -252,12 +252,12 @@ The following options can be configured on the server: storage.redis.sentinel.password Password for authenticating to Redis Sentinels. storage.redis.sentinel.username Username for authenticating to Redis Sentinels. storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). - storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' option. + storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' query parameter. **VCR** vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. - ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ This table is automatically generated using the configuration flags in the core and engines. When they're changed the options table must be regenerated using the Makefile: diff --git a/docs/pages/deployment/cli-reference.rst b/docs/pages/deployment/cli-reference.rst index abd69714d5..2cda060033 100755 --- a/docs/pages/deployment/cli-reference.rst +++ b/docs/pages/deployment/cli-reference.rst @@ -70,7 +70,7 @@ The following options apply to the server commands below: --storage.redis.sentinel.username string Username for authenticating to Redis Sentinels. --storage.redis.tls.truststorefile string PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). --storage.redis.username string Redis database username. If set, it overrides the username in the connection URL. - --storage.sql.connection string Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' option. + --storage.sql.connection string Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' query parameter. --strictmode When set, insecure settings are forbidden. (default true) --tls.certfile string PEM file containing the certificate for the server (also used as client certificate). --tls.certheader string Name of the HTTP header that will contain the client certificate when TLS is offloaded. diff --git a/docs/pages/deployment/server_options.rst b/docs/pages/deployment/server_options.rst index c20776dd0b..e80a7c02d3 100755 --- a/docs/pages/deployment/server_options.rst +++ b/docs/pages/deployment/server_options.rst @@ -2,85 +2,85 @@ :widths: 20 30 50 :class: options-table - ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= - Key Default Description - ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= - configfile nuts.yaml Nuts config file - cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. - datadir ./data Directory where the node stores its files. - internalratelimiter true When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. - loggerformat text Log format (text, json) - strictmode true When set, insecure settings are forbidden. - verbosity info Log level (trace, debug, info, warn, error) - tls.certfile PEM file containing the certificate for the server (also used as client certificate). - tls.certheader Name of the HTTP header that will contain the client certificate when TLS is offloaded. - tls.certkeyfile PEM file containing the private key of the server certificate. - tls.offload Whether to enable TLS offloading for incoming connections. Enable by setting it to 'incoming'. If enabled 'tls.certheader' must be configured as well. - tls.truststorefile truststore.pem PEM file containing the trusted CA certificates for authenticating remote servers. - **Auth** - auth.accesstokenlifespan 60 defines how long (in seconds) an access token is valid. Uses default in strict mode. - auth.clockskew 5000 allowed JWT Clock skew in milliseconds - auth.contractvalidators [irma,uzi,dummy,employeeid] sets the different contract validators to use - auth.publicurl public URL which can be reached by a users IRMA client, this should include the scheme and domain: https://example.com. Additional paths should only be added if some sort of url-rewriting is done in a reverse-proxy. - auth.http.timeout 30 HTTP timeout (in seconds) used by the Auth API HTTP client - auth.irma.autoupdateschemas true set if you want automatically update the IRMA schemas every 60 minutes. - auth.irma.schememanager pbdf IRMA schemeManager to use for attributes. Can be either 'pbdf' or 'irma-demo'. - **Crypto** - crypto.storage fs Storage to use, 'external' for an external backend (experimental), 'fs' for file system (for development purposes), 'vaultkv' for Vault KV store (recommended, will be replaced by external backend in future). - crypto.external.address Address of the external storage service. - crypto.external.timeout 100ms Time-out when invoking the external storage backend, in Golang time.Duration string format (e.g. 1s). - crypto.vault.address The Vault address. If set it overwrites the VAULT_ADDR env var. - crypto.vault.pathprefix kv The Vault path prefix. - crypto.vault.timeout 5s Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). - crypto.vault.token The Vault token. If set it overwrites the VAULT_TOKEN env var. - **Events** - events.nats.hostname 0.0.0.0 Hostname for the NATS server - events.nats.port 4222 Port where the NATS server listens on - events.nats.storagedir Directory where file-backed streams are stored in the NATS server - events.nats.timeout 30 Timeout for NATS server operations - **GoldenHammer** - goldenhammer.enabled true Whether to enable automatically fixing DID documents with the required endpoints. - goldenhammer.interval 10m0s The interval in which to check for DID documents to fix. - **HTTP** - http.default.address \:1323 Address and port the server will be listening to - http.default.log metadata What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). - http.default.tls Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', - http.default.auth.audience Expected audience for JWT tokens (default: hostname) - http.default.auth.authorizedkeyspath Path to an authorized_keys file for trusted JWT signers - http.default.auth.type Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode. - http.default.cors.origin [] When set, enables CORS from the specified origins on the default HTTP interface. - **JSONLD** - jsonld.contexts.localmapping [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. - jsonld.contexts.remoteallowlist [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json] In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. - **Network** - network.bootstrapnodes [] List of bootstrap nodes (':') which the node initially connect to. - network.connectiontimeout 5000 Timeout before an outbound connection attempt times out (in milliseconds). - network.enablediscovery true Whether to enable automatic connecting to other nodes. - network.enabletls true Whether to enable TLS for gRPC connections, which can be disabled for demo/development purposes. It is NOT meant for TLS offloading (see 'tls.offload'). Disabling TLS is not allowed in strict-mode. - network.grpcaddr \:5555 Local address for gRPC to listen on. If empty the gRPC server won't be started and other nodes will not be able to connect to this node (outbound connections can still be made). - network.maxbackoff 24h0m0s Maximum between outbound connections attempts to unresponsive nodes (in Golang duration format, e.g. '1h', '30m'). - network.nodedid Specifies the DID of the organization that operates this node, typically a vendor for EPD software. It is used to identify the node on the network. If the DID document does not exist of is deactivated, the node will not start. - network.protocols [] Specifies the list of network protocols to enable on the server. They are specified by version (1, 2). If not set, all protocols are enabled. - network.v2.diagnosticsinterval 5000 Interval (in milliseconds) that specifies how often the node should broadcast its diagnostic information to other nodes (specify 0 to disable). - network.v2.gossipinterval 5000 Interval (in milliseconds) that specifies how often the node should gossip its new hashes to other nodes. - **PKI** - pki.maxupdatefailhours 4 Maximum number of hours that a denylist update can fail - pki.softfail true Do not reject certificates if their revocation status cannot be established when softfail is true - **Storage** - storage.bbolt.backup.directory Target directory for BBolt database backups. - storage.bbolt.backup.interval 0s Interval, formatted as Golang duration (e.g. 10m, 1h) at which BBolt database backups will be performed. - storage.redis.address Redis database server address. This can be a simple 'host:port' or a Redis connection URL with scheme, auth and other options. - storage.redis.database Redis database name, which is used as prefix every key. Can be used to have multiple instances use the same Redis instance. - storage.redis.password Redis database password. If set, it overrides the username in the connection URL. - storage.redis.username Redis database username. If set, it overrides the username in the connection URL. - storage.redis.sentinel.master Name of the Redis Sentinel master. Setting this property enables Redis Sentinel. - storage.redis.sentinel.nodes [] Addresses of the Redis Sentinels to connect to initially. Setting this property enables Redis Sentinel. - storage.redis.sentinel.password Password for authenticating to Redis Sentinels. - storage.redis.sentinel.username Username for authenticating to Redis Sentinels. - storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). - storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' option. - **VCR** - vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). - vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. - vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. - ==================================== =============================================================================================================================================================================================================================================================================================================== ======================================================================================================================================================================================================================================= + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ + Key Default Description + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ + configfile nuts.yaml Nuts config file + cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. + datadir ./data Directory where the node stores its files. + internalratelimiter true When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. + loggerformat text Log format (text, json) + strictmode true When set, insecure settings are forbidden. + verbosity info Log level (trace, debug, info, warn, error) + tls.certfile PEM file containing the certificate for the server (also used as client certificate). + tls.certheader Name of the HTTP header that will contain the client certificate when TLS is offloaded. + tls.certkeyfile PEM file containing the private key of the server certificate. + tls.offload Whether to enable TLS offloading for incoming connections. Enable by setting it to 'incoming'. If enabled 'tls.certheader' must be configured as well. + tls.truststorefile truststore.pem PEM file containing the trusted CA certificates for authenticating remote servers. + **Auth** + auth.accesstokenlifespan 60 defines how long (in seconds) an access token is valid. Uses default in strict mode. + auth.clockskew 5000 allowed JWT Clock skew in milliseconds + auth.contractvalidators [irma,uzi,dummy,employeeid] sets the different contract validators to use + auth.publicurl public URL which can be reached by a users IRMA client, this should include the scheme and domain: https://example.com. Additional paths should only be added if some sort of url-rewriting is done in a reverse-proxy. + auth.http.timeout 30 HTTP timeout (in seconds) used by the Auth API HTTP client + auth.irma.autoupdateschemas true set if you want automatically update the IRMA schemas every 60 minutes. + auth.irma.schememanager pbdf IRMA schemeManager to use for attributes. Can be either 'pbdf' or 'irma-demo'. + **Crypto** + crypto.storage fs Storage to use, 'external' for an external backend (experimental), 'fs' for file system (for development purposes), 'vaultkv' for Vault KV store (recommended, will be replaced by external backend in future). + crypto.external.address Address of the external storage service. + crypto.external.timeout 100ms Time-out when invoking the external storage backend, in Golang time.Duration string format (e.g. 1s). + crypto.vault.address The Vault address. If set it overwrites the VAULT_ADDR env var. + crypto.vault.pathprefix kv The Vault path prefix. + crypto.vault.timeout 5s Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). + crypto.vault.token The Vault token. If set it overwrites the VAULT_TOKEN env var. + **Events** + events.nats.hostname 0.0.0.0 Hostname for the NATS server + events.nats.port 4222 Port where the NATS server listens on + events.nats.storagedir Directory where file-backed streams are stored in the NATS server + events.nats.timeout 30 Timeout for NATS server operations + **GoldenHammer** + goldenhammer.enabled true Whether to enable automatically fixing DID documents with the required endpoints. + goldenhammer.interval 10m0s The interval in which to check for DID documents to fix. + **HTTP** + http.default.address \:1323 Address and port the server will be listening to + http.default.log metadata What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). + http.default.tls Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', + http.default.auth.audience Expected audience for JWT tokens (default: hostname) + http.default.auth.authorizedkeyspath Path to an authorized_keys file for trusted JWT signers + http.default.auth.type Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode. + http.default.cors.origin [] When set, enables CORS from the specified origins on the default HTTP interface. + **JSONLD** + jsonld.contexts.localmapping [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. + jsonld.contexts.remoteallowlist [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json] In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. + **Network** + network.bootstrapnodes [] List of bootstrap nodes (':') which the node initially connect to. + network.connectiontimeout 5000 Timeout before an outbound connection attempt times out (in milliseconds). + network.enablediscovery true Whether to enable automatic connecting to other nodes. + network.enabletls true Whether to enable TLS for gRPC connections, which can be disabled for demo/development purposes. It is NOT meant for TLS offloading (see 'tls.offload'). Disabling TLS is not allowed in strict-mode. + network.grpcaddr \:5555 Local address for gRPC to listen on. If empty the gRPC server won't be started and other nodes will not be able to connect to this node (outbound connections can still be made). + network.maxbackoff 24h0m0s Maximum between outbound connections attempts to unresponsive nodes (in Golang duration format, e.g. '1h', '30m'). + network.nodedid Specifies the DID of the organization that operates this node, typically a vendor for EPD software. It is used to identify the node on the network. If the DID document does not exist of is deactivated, the node will not start. + network.protocols [] Specifies the list of network protocols to enable on the server. They are specified by version (1, 2). If not set, all protocols are enabled. + network.v2.diagnosticsinterval 5000 Interval (in milliseconds) that specifies how often the node should broadcast its diagnostic information to other nodes (specify 0 to disable). + network.v2.gossipinterval 5000 Interval (in milliseconds) that specifies how often the node should gossip its new hashes to other nodes. + **PKI** + pki.maxupdatefailhours 4 Maximum number of hours that a denylist update can fail + pki.softfail true Do not reject certificates if their revocation status cannot be established when softfail is true + **Storage** + storage.bbolt.backup.directory Target directory for BBolt database backups. + storage.bbolt.backup.interval 0s Interval, formatted as Golang duration (e.g. 10m, 1h) at which BBolt database backups will be performed. + storage.redis.address Redis database server address. This can be a simple 'host:port' or a Redis connection URL with scheme, auth and other options. + storage.redis.database Redis database name, which is used as prefix every key. Can be used to have multiple instances use the same Redis instance. + storage.redis.password Redis database password. If set, it overrides the username in the connection URL. + storage.redis.username Redis database username. If set, it overrides the username in the connection URL. + storage.redis.sentinel.master Name of the Redis Sentinel master. Setting this property enables Redis Sentinel. + storage.redis.sentinel.nodes [] Addresses of the Redis Sentinels to connect to initially. Setting this property enables Redis Sentinel. + storage.redis.sentinel.password Password for authenticating to Redis Sentinels. + storage.redis.sentinel.username Username for authenticating to Redis Sentinels. + storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). + storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' query parameter. + **VCR** + vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). + vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. + vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ From 984f8aa6d4a255b15d66bd2b4092e33c07542721 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Mon, 20 Nov 2023 14:01:36 +0100 Subject: [PATCH 04/23] sqlite concurrency --- README.rst | 8 +- discoveryservice/store.go | 21 ++- discoveryservice/store_test.go | 15 +++ docs/pages/deployment/cli-reference.rst | 4 +- docs/pages/deployment/server_options.rst | 164 +++++++++++------------ storage/cmd/cmd.go | 2 +- storage/engine.go | 7 +- storage/test.go | 6 +- 8 files changed, 130 insertions(+), 97 deletions(-) diff --git a/README.rst b/README.rst index 32f22cdc14..c3fb894efe 100644 --- a/README.rst +++ b/README.rst @@ -176,9 +176,9 @@ The following options can be configured on the server: :widths: 20 30 50 :class: options-table - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== Key Default Description - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== configfile nuts.yaml Nuts config file cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. datadir ./data Directory where the node stores its files. @@ -252,12 +252,12 @@ The following options can be configured on the server: storage.redis.sentinel.password Password for authenticating to Redis Sentinels. storage.redis.sentinel.username Username for authenticating to Redis Sentinels. storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). - storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' query parameter. + storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If using a SQLite database, make sure to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL'). **VCR** vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== This table is automatically generated using the configuration flags in the core and engines. When they're changed the options table must be regenerated using the Makefile: diff --git a/discoveryservice/store.go b/discoveryservice/store.go index d44ea31cbf..98bf3b8c49 100644 --- a/discoveryservice/store.go +++ b/discoveryservice/store.go @@ -25,11 +25,13 @@ import ( "github.com/nuts-foundation/go-did/vc" "github.com/nuts-foundation/nuts-node/discoveryservice/log" credential "github.com/nuts-foundation/nuts-node/vcr/credential" + "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/clause" "gorm.io/gorm/schema" "strconv" "strings" + "sync" "time" ) @@ -100,7 +102,8 @@ func (l credentialPropertyRecord) TableName() string { } type sqlStore struct { - db *gorm.DB + db *gorm.DB + writeLock sync.Mutex } func newSQLStore(db *gorm.DB, definitions map[string]Definition) (*sqlStore, error) { @@ -114,7 +117,8 @@ func newSQLStore(db *gorm.DB, definitions map[string]Definition) (*sqlStore, err } } return &sqlStore{ - db: db, + db: db, + writeLock: sync.Mutex{}, }, nil } @@ -132,10 +136,19 @@ func (s *sqlStore) add(serviceID string, presentation vc.VerifiablePresentation, } else if exists { return ErrPresentationAlreadyExists } + if _, isSQLite := s.db.Config.Dialector.(*sqlite.Dialector); isSQLite { + // SQLite does not support SELECT FOR UPDATE and allows only 1 active write transaction at any time, + // and any other attempt to acquire a write transaction will directly return an error. + // This is in contrast to most other SQL-databases, which let the 2nd thread wait for some time to acquire the lock. + // The general advice for SQLite is to retry the operation, which is just poor-man's scheduling. + // So to keep behavior consistent across databases, we'll just lock the entire store for the duration of the transaction. + // See https://github.com/nuts-foundation/nuts-node/pull/2589#discussion_r1399130608 + s.writeLock.Lock() + defer s.writeLock.Unlock() + } if err := s.prune(); err != nil { return err } - return s.db.Transaction(func(tx *gorm.DB) error { newTimestamp, err := s.updateTimestamp(tx, serviceID, timestamp) if err != nil { @@ -293,6 +306,8 @@ func (s *sqlStore) search(serviceID string, query map[string]string) ([]vc.Verif func (s *sqlStore) updateTimestamp(tx *gorm.DB, serviceID string, newTimestamp *Timestamp) (Timestamp, error) { var result serviceRecord // Lock (SELECT FOR UPDATE) discoveryservices row to prevent concurrent updates to the same list, which could mess up the lamport timestamp. + // But, it is not supported by SQLite. SQLite defaults to table-level write-locks upon the first write action in the transaction. + // if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). Where(serviceRecord{ID: serviceID}). Find(&result). diff --git a/discoveryservice/store_test.go b/discoveryservice/store_test.go index 5828d8d5f7..afea3c33b7 100644 --- a/discoveryservice/store_test.go +++ b/discoveryservice/store_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gorm.io/gorm" + "sync" "testing" ) @@ -315,6 +316,20 @@ func Test_sqlStore_search(t *testing.T) { } }) } + + t.Run("concurrency", func(t *testing.T) { + c := setupStore(t, storageEngine.GetSQLDatabase()) + wg := &sync.WaitGroup{} + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + err := c.add(testServiceID, createPresentation(aliceDID, vcAlice), nil) + require.NoError(t, err) + }() + } + wg.Wait() + }) } func setupStore(t *testing.T, db *gorm.DB) *sqlStore { diff --git a/docs/pages/deployment/cli-reference.rst b/docs/pages/deployment/cli-reference.rst index 2cda060033..6974629a28 100755 --- a/docs/pages/deployment/cli-reference.rst +++ b/docs/pages/deployment/cli-reference.rst @@ -44,7 +44,7 @@ The following options apply to the server commands below: --http.default.log string What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). (default "metadata") --http.default.tls string Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', --internalratelimiter When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. (default true) - --jsonld.contexts.localmapping stringToString This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. (default [https://schema.org=assets/contexts/schema-org-v13.ldjson,https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson]) + --jsonld.contexts.localmapping stringToString This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. (default [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson]) --jsonld.contexts.remoteallowlist strings In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. (default [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json]) --loggerformat string Log format (text, json) (default "text") --network.bootstrapnodes strings List of bootstrap nodes (':') which the node initially connect to. @@ -70,7 +70,7 @@ The following options apply to the server commands below: --storage.redis.sentinel.username string Username for authenticating to Redis Sentinels. --storage.redis.tls.truststorefile string PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). --storage.redis.username string Redis database username. If set, it overrides the username in the connection URL. - --storage.sql.connection string Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' query parameter. + --storage.sql.connection string Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If using a SQLite database, make sure to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL'). --strictmode When set, insecure settings are forbidden. (default true) --tls.certfile string PEM file containing the certificate for the server (also used as client certificate). --tls.certheader string Name of the HTTP header that will contain the client certificate when TLS is offloaded. diff --git a/docs/pages/deployment/server_options.rst b/docs/pages/deployment/server_options.rst index e80a7c02d3..f45c8bc310 100755 --- a/docs/pages/deployment/server_options.rst +++ b/docs/pages/deployment/server_options.rst @@ -2,85 +2,85 @@ :widths: 20 30 50 :class: options-table - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ - Key Default Description - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ - configfile nuts.yaml Nuts config file - cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. - datadir ./data Directory where the node stores its files. - internalratelimiter true When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. - loggerformat text Log format (text, json) - strictmode true When set, insecure settings are forbidden. - verbosity info Log level (trace, debug, info, warn, error) - tls.certfile PEM file containing the certificate for the server (also used as client certificate). - tls.certheader Name of the HTTP header that will contain the client certificate when TLS is offloaded. - tls.certkeyfile PEM file containing the private key of the server certificate. - tls.offload Whether to enable TLS offloading for incoming connections. Enable by setting it to 'incoming'. If enabled 'tls.certheader' must be configured as well. - tls.truststorefile truststore.pem PEM file containing the trusted CA certificates for authenticating remote servers. - **Auth** - auth.accesstokenlifespan 60 defines how long (in seconds) an access token is valid. Uses default in strict mode. - auth.clockskew 5000 allowed JWT Clock skew in milliseconds - auth.contractvalidators [irma,uzi,dummy,employeeid] sets the different contract validators to use - auth.publicurl public URL which can be reached by a users IRMA client, this should include the scheme and domain: https://example.com. Additional paths should only be added if some sort of url-rewriting is done in a reverse-proxy. - auth.http.timeout 30 HTTP timeout (in seconds) used by the Auth API HTTP client - auth.irma.autoupdateschemas true set if you want automatically update the IRMA schemas every 60 minutes. - auth.irma.schememanager pbdf IRMA schemeManager to use for attributes. Can be either 'pbdf' or 'irma-demo'. - **Crypto** - crypto.storage fs Storage to use, 'external' for an external backend (experimental), 'fs' for file system (for development purposes), 'vaultkv' for Vault KV store (recommended, will be replaced by external backend in future). - crypto.external.address Address of the external storage service. - crypto.external.timeout 100ms Time-out when invoking the external storage backend, in Golang time.Duration string format (e.g. 1s). - crypto.vault.address The Vault address. If set it overwrites the VAULT_ADDR env var. - crypto.vault.pathprefix kv The Vault path prefix. - crypto.vault.timeout 5s Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). - crypto.vault.token The Vault token. If set it overwrites the VAULT_TOKEN env var. - **Events** - events.nats.hostname 0.0.0.0 Hostname for the NATS server - events.nats.port 4222 Port where the NATS server listens on - events.nats.storagedir Directory where file-backed streams are stored in the NATS server - events.nats.timeout 30 Timeout for NATS server operations - **GoldenHammer** - goldenhammer.enabled true Whether to enable automatically fixing DID documents with the required endpoints. - goldenhammer.interval 10m0s The interval in which to check for DID documents to fix. - **HTTP** - http.default.address \:1323 Address and port the server will be listening to - http.default.log metadata What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). - http.default.tls Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', - http.default.auth.audience Expected audience for JWT tokens (default: hostname) - http.default.auth.authorizedkeyspath Path to an authorized_keys file for trusted JWT signers - http.default.auth.type Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode. - http.default.cors.origin [] When set, enables CORS from the specified origins on the default HTTP interface. - **JSONLD** - jsonld.contexts.localmapping [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. - jsonld.contexts.remoteallowlist [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json] In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. - **Network** - network.bootstrapnodes [] List of bootstrap nodes (':') which the node initially connect to. - network.connectiontimeout 5000 Timeout before an outbound connection attempt times out (in milliseconds). - network.enablediscovery true Whether to enable automatic connecting to other nodes. - network.enabletls true Whether to enable TLS for gRPC connections, which can be disabled for demo/development purposes. It is NOT meant for TLS offloading (see 'tls.offload'). Disabling TLS is not allowed in strict-mode. - network.grpcaddr \:5555 Local address for gRPC to listen on. If empty the gRPC server won't be started and other nodes will not be able to connect to this node (outbound connections can still be made). - network.maxbackoff 24h0m0s Maximum between outbound connections attempts to unresponsive nodes (in Golang duration format, e.g. '1h', '30m'). - network.nodedid Specifies the DID of the organization that operates this node, typically a vendor for EPD software. It is used to identify the node on the network. If the DID document does not exist of is deactivated, the node will not start. - network.protocols [] Specifies the list of network protocols to enable on the server. They are specified by version (1, 2). If not set, all protocols are enabled. - network.v2.diagnosticsinterval 5000 Interval (in milliseconds) that specifies how often the node should broadcast its diagnostic information to other nodes (specify 0 to disable). - network.v2.gossipinterval 5000 Interval (in milliseconds) that specifies how often the node should gossip its new hashes to other nodes. - **PKI** - pki.maxupdatefailhours 4 Maximum number of hours that a denylist update can fail - pki.softfail true Do not reject certificates if their revocation status cannot be established when softfail is true - **Storage** - storage.bbolt.backup.directory Target directory for BBolt database backups. - storage.bbolt.backup.interval 0s Interval, formatted as Golang duration (e.g. 10m, 1h) at which BBolt database backups will be performed. - storage.redis.address Redis database server address. This can be a simple 'host:port' or a Redis connection URL with scheme, auth and other options. - storage.redis.database Redis database name, which is used as prefix every key. Can be used to have multiple instances use the same Redis instance. - storage.redis.password Redis database password. If set, it overrides the username in the connection URL. - storage.redis.username Redis database username. If set, it overrides the username in the connection URL. - storage.redis.sentinel.master Name of the Redis Sentinel master. Setting this property enables Redis Sentinel. - storage.redis.sentinel.nodes [] Addresses of the Redis Sentinels to connect to initially. Setting this property enables Redis Sentinel. - storage.redis.sentinel.password Password for authenticating to Redis Sentinels. - storage.redis.sentinel.username Username for authenticating to Redis Sentinels. - storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). - storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' query parameter. - **VCR** - vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). - vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. - vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================ + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== + Key Default Description + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== + configfile nuts.yaml Nuts config file + cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. + datadir ./data Directory where the node stores its files. + internalratelimiter true When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. + loggerformat text Log format (text, json) + strictmode true When set, insecure settings are forbidden. + verbosity info Log level (trace, debug, info, warn, error) + tls.certfile PEM file containing the certificate for the server (also used as client certificate). + tls.certheader Name of the HTTP header that will contain the client certificate when TLS is offloaded. + tls.certkeyfile PEM file containing the private key of the server certificate. + tls.offload Whether to enable TLS offloading for incoming connections. Enable by setting it to 'incoming'. If enabled 'tls.certheader' must be configured as well. + tls.truststorefile truststore.pem PEM file containing the trusted CA certificates for authenticating remote servers. + **Auth** + auth.accesstokenlifespan 60 defines how long (in seconds) an access token is valid. Uses default in strict mode. + auth.clockskew 5000 allowed JWT Clock skew in milliseconds + auth.contractvalidators [irma,uzi,dummy,employeeid] sets the different contract validators to use + auth.publicurl public URL which can be reached by a users IRMA client, this should include the scheme and domain: https://example.com. Additional paths should only be added if some sort of url-rewriting is done in a reverse-proxy. + auth.http.timeout 30 HTTP timeout (in seconds) used by the Auth API HTTP client + auth.irma.autoupdateschemas true set if you want automatically update the IRMA schemas every 60 minutes. + auth.irma.schememanager pbdf IRMA schemeManager to use for attributes. Can be either 'pbdf' or 'irma-demo'. + **Crypto** + crypto.storage fs Storage to use, 'external' for an external backend (experimental), 'fs' for file system (for development purposes), 'vaultkv' for Vault KV store (recommended, will be replaced by external backend in future). + crypto.external.address Address of the external storage service. + crypto.external.timeout 100ms Time-out when invoking the external storage backend, in Golang time.Duration string format (e.g. 1s). + crypto.vault.address The Vault address. If set it overwrites the VAULT_ADDR env var. + crypto.vault.pathprefix kv The Vault path prefix. + crypto.vault.timeout 5s Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). + crypto.vault.token The Vault token. If set it overwrites the VAULT_TOKEN env var. + **Events** + events.nats.hostname 0.0.0.0 Hostname for the NATS server + events.nats.port 4222 Port where the NATS server listens on + events.nats.storagedir Directory where file-backed streams are stored in the NATS server + events.nats.timeout 30 Timeout for NATS server operations + **GoldenHammer** + goldenhammer.enabled true Whether to enable automatically fixing DID documents with the required endpoints. + goldenhammer.interval 10m0s The interval in which to check for DID documents to fix. + **HTTP** + http.default.address \:1323 Address and port the server will be listening to + http.default.log metadata What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). + http.default.tls Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', + http.default.auth.audience Expected audience for JWT tokens (default: hostname) + http.default.auth.authorizedkeyspath Path to an authorized_keys file for trusted JWT signers + http.default.auth.type Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode. + http.default.cors.origin [] When set, enables CORS from the specified origins on the default HTTP interface. + **JSONLD** + jsonld.contexts.localmapping [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. + jsonld.contexts.remoteallowlist [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json] In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. + **Network** + network.bootstrapnodes [] List of bootstrap nodes (':') which the node initially connect to. + network.connectiontimeout 5000 Timeout before an outbound connection attempt times out (in milliseconds). + network.enablediscovery true Whether to enable automatic connecting to other nodes. + network.enabletls true Whether to enable TLS for gRPC connections, which can be disabled for demo/development purposes. It is NOT meant for TLS offloading (see 'tls.offload'). Disabling TLS is not allowed in strict-mode. + network.grpcaddr \:5555 Local address for gRPC to listen on. If empty the gRPC server won't be started and other nodes will not be able to connect to this node (outbound connections can still be made). + network.maxbackoff 24h0m0s Maximum between outbound connections attempts to unresponsive nodes (in Golang duration format, e.g. '1h', '30m'). + network.nodedid Specifies the DID of the organization that operates this node, typically a vendor for EPD software. It is used to identify the node on the network. If the DID document does not exist of is deactivated, the node will not start. + network.protocols [] Specifies the list of network protocols to enable on the server. They are specified by version (1, 2). If not set, all protocols are enabled. + network.v2.diagnosticsinterval 5000 Interval (in milliseconds) that specifies how often the node should broadcast its diagnostic information to other nodes (specify 0 to disable). + network.v2.gossipinterval 5000 Interval (in milliseconds) that specifies how often the node should gossip its new hashes to other nodes. + **PKI** + pki.maxupdatefailhours 4 Maximum number of hours that a denylist update can fail + pki.softfail true Do not reject certificates if their revocation status cannot be established when softfail is true + **Storage** + storage.bbolt.backup.directory Target directory for BBolt database backups. + storage.bbolt.backup.interval 0s Interval, formatted as Golang duration (e.g. 10m, 1h) at which BBolt database backups will be performed. + storage.redis.address Redis database server address. This can be a simple 'host:port' or a Redis connection URL with scheme, auth and other options. + storage.redis.database Redis database name, which is used as prefix every key. Can be used to have multiple instances use the same Redis instance. + storage.redis.password Redis database password. If set, it overrides the username in the connection URL. + storage.redis.username Redis database username. If set, it overrides the username in the connection URL. + storage.redis.sentinel.master Name of the Redis Sentinel master. Setting this property enables Redis Sentinel. + storage.redis.sentinel.nodes [] Addresses of the Redis Sentinels to connect to initially. Setting this property enables Redis Sentinel. + storage.redis.sentinel.password Password for authenticating to Redis Sentinels. + storage.redis.sentinel.username Username for authenticating to Redis Sentinels. + storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). + storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If using a SQLite database, make sure to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL'). + **VCR** + vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). + vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. + vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== diff --git a/storage/cmd/cmd.go b/storage/cmd/cmd.go index 5fe495bf2b..5747b9ab72 100644 --- a/storage/cmd/cmd.go +++ b/storage/cmd/cmd.go @@ -40,6 +40,6 @@ func FlagSet() *pflag.FlagSet { flagSet.String("storage.redis.sentinel.password", defs.Redis.Sentinel.Password, "Password for authenticating to Redis Sentinels.") flagSet.String("storage.sql.connection", defs.SQL.ConnectionString, "Connection string for the SQL database. "+ "If not set, it defaults to a SQLite database stored inside the configured data directory. "+ - "If specifying a SQLite database, make sure to enable foreign keys with the '_foreign_keys=on' query parameter.") + "If using a SQLite database, make sure to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL').") return flagSet } diff --git a/storage/engine.go b/storage/engine.go index 10f7309960..17c1197c3a 100644 --- a/storage/engine.go +++ b/storage/engine.go @@ -162,8 +162,9 @@ func (e *engine) GetSQLDatabase() *gorm.DB { func (e *engine) initSQLDatabase() error { connectionString := e.config.SQL.ConnectionString if len(connectionString) == 0 { - connectionString = "file:" + path.Join(e.datadir, "sqlite.db?_foreign_keys=on") + connectionString = sqliteConnectionString(e.datadir) } + var err error e.sqlDB, err = gorm.Open(sqlite.Open(connectionString), &gorm.Config{}) if err != nil { @@ -195,6 +196,10 @@ func (e *engine) initSQLDatabase() error { return err } +func sqliteConnectionString(datadir string) string { + return "file:" + path.Join(datadir, "sqlite.db?_journal_mode=WAL&_foreign_keys=on") +} + type provider struct { moduleName string engine *engine diff --git a/storage/test.go b/storage/test.go index 9b5820b42a..0366208ea0 100644 --- a/storage/test.go +++ b/storage/test.go @@ -28,12 +28,10 @@ import ( "testing" ) -// SQLiteInMemoryConnectionString is a connection string for an in-memory SQLite database -const SQLiteInMemoryConnectionString = "file::memory:?cache=shared&_foreign_keys=on" - func NewTestStorageEngineInDir(dir string) Engine { result := New().(*engine) - result.config.SQL = SQLConfig{ConnectionString: SQLiteInMemoryConnectionString} + + result.config.SQL = SQLConfig{ConnectionString: sqliteConnectionString(dir)} _ = result.Configure(core.TestServerConfig(core.ServerConfig{Datadir: dir + "/data"})) return result } From fe929e3ec97f56b353ccc82683c328bb3d04c272 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Mon, 20 Nov 2023 15:40:24 +0100 Subject: [PATCH 05/23] SQL schema feedback fixes --- discoveryservice/api/v1/api.go | 29 --------- discoveryservice/config.go | 2 - discoveryservice/definition.go | 5 +- discoveryservice/log/logger.go | 2 +- discoveryservice/module.go | 2 +- discoveryservice/module_test.go | 2 +- discoveryservice/store.go | 46 +++++++------- discoveryservice/store_test.go | 2 +- .../sql_migrations/2_discoveryservice.up.sql | 60 +++++++++---------- vcr/pe/schema/v2/schema.go | 22 ++++--- 10 files changed, 74 insertions(+), 98 deletions(-) delete mode 100644 discoveryservice/api/v1/api.go diff --git a/discoveryservice/api/v1/api.go b/discoveryservice/api/v1/api.go deleted file mode 100644 index 2a90d4c29f..0000000000 --- a/discoveryservice/api/v1/api.go +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (C) 2023 Nuts community - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - * - */ - -package v1 - -//// lists maps a list name (last path part of use case endpoint) to the list ID -//lists map[string]string -//// name is derived from endpoint: it's the last path part of the definition endpoint -//// It is used to route HTTP GET requests to the correct list. -//pathParts := strings.Split(definition.Endpoint, "/") -//name := pathParts[len(pathParts)-1] -//if name == "" { -//return nil, fmt.Errorf("can't derive list name from definition endpoint: %s", definition.Endpoint) -//} diff --git a/discoveryservice/config.go b/discoveryservice/config.go index d2dd701a96..f6d8647a8a 100644 --- a/discoveryservice/config.go +++ b/discoveryservice/config.go @@ -33,8 +33,6 @@ type DefinitionsConfig struct { type ServerConfig struct { // DefinitionIDs specifies which use case lists the server serves. DefinitionIDs []string `koanf:"definition_ids"` - // Directory is the directory where the server stores the lists. - Directory string `koanf:"directory"` } // DefaultConfig returns the default configuration. diff --git a/discoveryservice/definition.go b/discoveryservice/definition.go index b9af33283b..2919c93e71 100644 --- a/discoveryservice/definition.go +++ b/discoveryservice/definition.go @@ -37,10 +37,11 @@ func init() { panic(err) } const schemaURL = "http://nuts.nl/schemas/discovery-service-v0.json" - if err := v2.Compiler.AddResource(schemaURL, bytes.NewReader(serviceDefinitionSchemaData)); err != nil { + compiler := v2.Compiler() + if err := compiler.AddResource(schemaURL, bytes.NewReader(serviceDefinitionSchemaData)); err != nil { panic(err) } - definitionJsonSchema = v2.Compiler.MustCompile(schemaURL) + definitionJsonSchema = compiler.MustCompile(schemaURL) } // Definition holds the definition of a service. diff --git a/discoveryservice/log/logger.go b/discoveryservice/log/logger.go index 13914edbaf..3a4fcf4b4a 100644 --- a/discoveryservice/log/logger.go +++ b/discoveryservice/log/logger.go @@ -23,7 +23,7 @@ import ( "github.com/sirupsen/logrus" ) -var _logger = logrus.StandardLogger().WithField(core.LogFieldModule, "DiscoveryService") +var _logger = logrus.StandardLogger().WithField(core.LogFieldModule, "Discovery") // Logger returns a logger with the module field set func Logger() *logrus.Entry { diff --git a/discoveryservice/module.go b/discoveryservice/module.go index 93be32ec21..fa12d1ef9d 100644 --- a/discoveryservice/module.go +++ b/discoveryservice/module.go @@ -32,7 +32,7 @@ import ( "time" ) -const ModuleName = "DiscoveryService" +const ModuleName = "Discovery" var ErrServerModeDisabled = errors.New("node is not a discovery server for this service") diff --git a/discoveryservice/module_test.go b/discoveryservice/module_test.go index 416c85258a..0a4af7aae6 100644 --- a/discoveryservice/module_test.go +++ b/discoveryservice/module_test.go @@ -31,7 +31,7 @@ import ( const serviceID = "urn:nuts.nl:usecase:eOverdrachtDev2023" func TestModule_Name(t *testing.T) { - assert.Equal(t, "DiscoveryService", (&Module{}).Name()) + assert.Equal(t, "Discovery", (&Module{}).Name()) } func TestModule_Shutdown(t *testing.T) { diff --git a/discoveryservice/store.go b/discoveryservice/store.go index 98bf3b8c49..2b805c5f20 100644 --- a/discoveryservice/store.go +++ b/discoveryservice/store.go @@ -39,12 +39,12 @@ var ErrServiceNotFound = errors.New("discovery service not found") var ErrPresentationAlreadyExists = errors.New("presentation already exists") type serviceRecord struct { - ID string `gorm:"primaryKey"` - Timestamp uint64 + ID string `gorm:"primaryKey"` + LamportTimestamp uint64 } func (s serviceRecord) TableName() string { - return "discoveryservices" + return "discovery_service" } var _ schema.Tabler = (*presentationRecord)(nil) @@ -52,7 +52,7 @@ var _ schema.Tabler = (*presentationRecord)(nil) type presentationRecord struct { ID string `gorm:"primaryKey"` ServiceID string - Timestamp uint64 + LamportTimestamp uint64 CredentialSubjectID string PresentationID string PresentationRaw string @@ -61,14 +61,14 @@ type presentationRecord struct { } func (s presentationRecord) TableName() string { - return "discoveryservice_presentations" + return "discovery_presentation" } // credentialRecord is a Verifiable Credential, part of a presentation (entry) on a use case list. type credentialRecord struct { // ID is the unique identifier of the entry. ID string `gorm:"primaryKey"` - // PresentationID corresponds to the discoveryservice_presentations record ID (not VerifiablePresentation.ID) this credentialRecord belongs to. + // PresentationID corresponds to the discovery_presentation record ID (not VerifiablePresentation.ID) this credentialRecord belongs to. PresentationID string // CredentialID contains the 'id' property of the Verifiable Credential. CredentialID string @@ -78,18 +78,18 @@ type credentialRecord struct { CredentialSubjectID string // CredentialType contains the 'type' property of the Verifiable Credential (not being 'VerifiableCredential'). CredentialType *string - Properties []credentialPropertyRecord `gorm:"foreignKey:ID;references:ID"` + Properties []credentialPropertyRecord `gorm:"foreignKey:CredentialID;references:ID"` } // TableName returns the table name for this DTO. func (p credentialRecord) TableName() string { - return "discoveryservice_credentials" + return "discovery_credential" } // credentialPropertyRecord is a property of a Verifiable Credential in a Verifiable Presentation in a discovery service. type credentialPropertyRecord struct { - // ID refers to the entry record in discoveryservice_credentials - ID string `gorm:"primaryKey"` + // CredentialID refers to the entry record in discovery_credential + CredentialID string `gorm:"primaryKey"` // Key is JSON path of the property. Key string `gorm:"primaryKey"` // Value is the value of the property. @@ -98,7 +98,7 @@ type credentialPropertyRecord struct { // TableName returns the table name for this DTO. func (l credentialPropertyRecord) TableName() string { - return "discoveryservice_credential_props" + return "discovery_credential_prop" } type sqlStore struct { @@ -183,7 +183,7 @@ func createPresentationRecord(serviceID string, timestamp Timestamp, presentatio newPresentation := presentationRecord{ ID: uuid.NewString(), ServiceID: serviceID, - Timestamp: uint64(timestamp), + LamportTimestamp: uint64(timestamp), CredentialSubjectID: credentialSubjectID.String(), PresentationID: presentation.ID.String(), PresentationRaw: presentation.Raw(), @@ -219,9 +219,9 @@ func createPresentationRecord(serviceID string, timestamp Timestamp, presentatio continue } newCredential.Properties = append(newCredential.Properties, credentialPropertyRecord{ - ID: newCredential.ID, - Key: key, - Value: values[i], + CredentialID: newCredential.ID, + Key: key, + Value: values[i], }) } newPresentation.Credentials = append(newPresentation.Credentials, newCredential) @@ -231,7 +231,7 @@ func createPresentationRecord(serviceID string, timestamp Timestamp, presentatio func (s *sqlStore) get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresentation, *Timestamp, error) { var rows []presentationRecord - err := s.db.Order("timestamp ASC").Find(&rows, "service_id = ? AND timestamp > ?", serviceID, int(startAt)).Error + err := s.db.Order("lamport_timestamp ASC").Find(&rows, "service_id = ? AND lamport_timestamp > ?", serviceID, int(startAt)).Error if err != nil { return nil, nil, fmt.Errorf("query service '%s': %w", serviceID, err) } @@ -243,7 +243,7 @@ func (s *sqlStore) get(serviceID string, startAt Timestamp) ([]vc.VerifiablePres return nil, nil, fmt.Errorf("parse presentation '%s' of service '%s': %w", row.PresentationID, serviceID, err) } presentations = append(presentations, *presentation) - timestamp = Timestamp(row.Timestamp) + timestamp = Timestamp(row.LamportTimestamp) } return presentations, ×tamp, nil } @@ -258,7 +258,7 @@ func (s *sqlStore) search(serviceID string, query map[string]string) ([]vc.Verif stmt := s.db.Model(&presentationRecord{}). Where("service_id = ?", serviceID). - Joins("inner join discoveryservice_credentials cred ON cred.presentation_id = discoveryservice_presentations.id") + Joins("inner join discovery_credential cred ON cred.presentation_id = discovery_presentation.id") numProps := 0 for jsonPath, value := range query { if value == "*" { @@ -281,7 +281,7 @@ func (s *sqlStore) search(serviceID string, query map[string]string) ([]vc.Verif // Multiple (inner) joins to filter on a dynamic number of properties to filter on is not pretty, but it works alias := "p" + strconv.Itoa(numProps) numProps++ - stmt = stmt.Joins("inner join discoveryservice_credential_props "+alias+" ON "+alias+".id = cred.id AND "+alias+".key = ? AND "+alias+".value "+eq+" ?", jsonPath, value) + stmt = stmt.Joins("inner join discovery_credential_prop "+alias+" ON "+alias+".credential_id = cred.id AND "+alias+".key = ? AND "+alias+".value "+eq+" ?", jsonPath, value) } } @@ -305,7 +305,7 @@ func (s *sqlStore) search(serviceID string, query map[string]string) ([]vc.Verif func (s *sqlStore) updateTimestamp(tx *gorm.DB, serviceID string, newTimestamp *Timestamp) (Timestamp, error) { var result serviceRecord - // Lock (SELECT FOR UPDATE) discoveryservices row to prevent concurrent updates to the same list, which could mess up the lamport timestamp. + // Lock (SELECT FOR UPDATE) discoveryservice row to prevent concurrent updates to the same list, which could mess up the lamport timestamp. // But, it is not supported by SQLite. SQLite defaults to table-level write-locks upon the first write action in the transaction. // if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). @@ -317,14 +317,14 @@ func (s *sqlStore) updateTimestamp(tx *gorm.DB, serviceID string, newTimestamp * result.ID = serviceID if newTimestamp == nil { // Increment timestamp - result.Timestamp++ + result.LamportTimestamp++ } else { - result.Timestamp = uint64(*newTimestamp) + result.LamportTimestamp = uint64(*newTimestamp) } if err := tx.Save(&result).Error; err != nil { return 0, err } - return Timestamp(result.Timestamp), nil + return Timestamp(result.LamportTimestamp), nil } func (s *sqlStore) exists(serviceID string, credentialSubjectID string, presentationID string) (bool, error) { diff --git a/discoveryservice/store_test.go b/discoveryservice/store_test.go index afea3c33b7..463f78a646 100644 --- a/discoveryservice/store_test.go +++ b/discoveryservice/store_test.go @@ -343,7 +343,7 @@ func resetStore(t *testing.T, db *gorm.DB) { underlyingDB, err := db.DB() require.NoError(t, err) // related tables are emptied due to on-delete-cascade clause - _, err = underlyingDB.Exec("DELETE FROM discoveryservices") + _, err = underlyingDB.Exec("DELETE FROM discovery_service") require.NoError(t, err) } diff --git a/storage/sql_migrations/2_discoveryservice.up.sql b/storage/sql_migrations/2_discoveryservice.up.sql index bfd78e9937..4f7dc05414 100644 --- a/storage/sql_migrations/2_discoveryservice.up.sql +++ b/storage/sql_migrations/2_discoveryservice.up.sql @@ -1,49 +1,49 @@ --- discoveryservices contains the known discovery services and the highest timestamp -create table discoveryservices +-- discovery contains the known discovery services and the highest timestamp +create table discovery_service ( - id text not null primary key, - timestamp integer not null + id varchar(36) not null primary key, + lamport_timestamp integer not null ); --- discoveryservice_presentations contains the presentations of the discovery services -create table discoveryservice_presentations +-- discovery_presentation contains the presentations of the discovery services +create table discovery_presentation ( - id text not null primary key, - service_id text not null, - timestamp integer not null, - credential_subject_id text not null, - presentation_id text not null, - presentation_raw text not null, - presentation_expiration integer not null, + id varchar(36) not null primary key, + service_id varchar(36) not null, + lamport_timestamp integer not null, + credential_subject_id varchar not null, + presentation_id varchar not null, + presentation_raw varchar not null, + presentation_expiration integer not null, unique (service_id, credential_subject_id), - constraint fk_discovery_presentation_service_id foreign key (service_id) references discoveryservices (id) on delete cascade + constraint fk_discovery_presentation_service_id foreign key (service_id) references discovery_service (id) on delete cascade ); --- discoveryservice_credentials is a credential in a presentation of the discovery service. +-- discovery_credential is a credential in a presentation of the discovery service. -- We could do without the table, but having it allows to have a normalized index for credential properties that appear on every credential. -- Then we don't need rows in the properties table for them (having a column for those is faster than having a row in the properties table which needs to be joined). -create table discoveryservice_credentials +create table discovery_credential ( - id text not null primary key, - presentation_id text not null, - credential_id text not null, - credential_issuer text not null, - credential_subject_id text not null, + id varchar(36) not null primary key, + presentation_id varchar(36) not null, + credential_id varchar not null, + credential_issuer varchar not null, + credential_subject_id varchar not null, -- for now, credentials with at most 2 types are supported. -- The type stored in the type column will be the 'other' type, not being 'VerifiableCredential'. -- When credentials with 3 or more types appear, we could have to use a separate table for the types. - credential_type text, - constraint fk_discoveryservice_credential_presentation foreign key (presentation_id) references discoveryservice_presentations (id) on delete cascade + credential_type varchar, + constraint fk_discoveryservice_credential_presentation foreign key (presentation_id) references discovery_presentation (id) on delete cascade ); --- discoveryservice_credential_props contains the credentialSubject properties of a credential in a presentation of the discovery service. +-- discovery_credential_prop contains the credentialSubject properties of a credential in a presentation of the discovery service. -- It is used by clients to search for presentations. -create table discoveryservice_credential_props +create table discovery_credential_prop ( - id text not null, - key text not null, - value text, - PRIMARY KEY (id, key), + credential_id varchar(36) not null, + key varchar not null, + value varchar, + PRIMARY KEY (credential_id, key), -- cascading delete: if the presentation gets deleted, the properties get deleted as well - constraint fk_discoveryservice_credential_id foreign key (id) references discoveryservice_credentials (id) on delete cascade + constraint fk_discoveryservice_credential_id foreign key (credential_id) references discovery_credential (id) on delete cascade ); \ No newline at end of file diff --git a/vcr/pe/schema/v2/schema.go b/vcr/pe/schema/v2/schema.go index 8c3da1a2bc..dd4c1ce59a 100644 --- a/vcr/pe/schema/v2/schema.go +++ b/vcr/pe/schema/v2/schema.go @@ -51,8 +51,17 @@ var PresentationDefinition *jsonschema.Schema // PresentationSubmission is the JSON schema for a presentation submission. var PresentationSubmission *jsonschema.Schema -// Compiler is the JSON schema compiler. -var Compiler = jsonschema.NewCompiler() +// Compiler returns a JSON schema compiler with the Presentation Exchange schemas loaded. +func Compiler() *jsonschema.Compiler { + compiler := jsonschema.NewCompiler() + compiler.Draft = jsonschema.Draft7 + if err := loadSchemas(schemaFiles, compiler); err != nil { + panic(err) + } + PresentationDefinition = compiler.MustCompile(presentationDefinition) + PresentationSubmission = compiler.MustCompile(presentationSubmission) + return compiler +} func init() { // By default, it loads from filesystem, but that sounds unsafe. @@ -60,12 +69,9 @@ func init() { loader.Load = func(url string) (io.ReadCloser, error) { return nil, fmt.Errorf("refusing to load unknown schema: %s", url) } - Compiler.Draft = jsonschema.Draft7 - if err := loadSchemas(schemaFiles, Compiler); err != nil { - panic(err) - } - PresentationDefinition = Compiler.MustCompile(presentationDefinition) - PresentationSubmission = Compiler.MustCompile(presentationSubmission) + compiler := Compiler() + PresentationDefinition = compiler.MustCompile(presentationDefinition) + PresentationSubmission = compiler.MustCompile(presentationSubmission) } func loadSchemas(reader fs.ReadFileFS, compiler *jsonschema.Compiler) error { From 8b7bce74c27fed1c6b027330fb514dfcf6d9091c Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Mon, 20 Nov 2023 15:44:48 +0100 Subject: [PATCH 06/23] feedback --- {discoveryservice => discovery}/config.go | 2 +- {discoveryservice => discovery}/definition.go | 2 +- {discoveryservice => discovery}/interface.go | 6 ++++-- {discoveryservice => discovery}/log/logger.go | 0 {discoveryservice => discovery}/mock.go | 8 ++++---- {discoveryservice => discovery}/module.go | 5 +++-- {discoveryservice => discovery}/module_test.go | 2 +- .../service-definition-schema.json | 0 {discoveryservice => discovery}/store.go | 6 +++--- {discoveryservice => discovery}/store_test.go | 2 +- {discoveryservice => discovery}/test.go | 2 +- {discoveryservice => discovery}/test/duplicate_id/1.json | 0 {discoveryservice => discovery}/test/duplicate_id/2.json | 0 .../test/duplicate_id/README.md | 0 .../test/invalid_definition/1.json | 0 .../test/invalid_definition/README.md | 0 {discoveryservice => discovery}/test/invalid_json/1.json | 0 .../test/invalid_json/README.md | 0 .../test/valid/eoverdracht.json | 0 .../test/valid/subdir/README.md | 0 .../test/valid/subdir/empty.json | 0 makefile | 2 +- storage/sql_migrations/2_discoveryservice.down.sql | 8 ++++---- storage/sql_migrations/2_discoveryservice.up.sql | 4 ++-- 24 files changed, 26 insertions(+), 23 deletions(-) rename {discoveryservice => discovery}/config.go (98%) rename {discoveryservice => discovery}/definition.go (98%) rename {discoveryservice => discovery}/interface.go (89%) rename {discoveryservice => discovery}/log/logger.go (100%) rename {discoveryservice => discovery}/mock.go (92%) rename {discoveryservice => discovery}/module.go (98%) rename {discoveryservice => discovery}/module_test.go (99%) rename {discoveryservice => discovery}/service-definition-schema.json (100%) rename {discoveryservice => discovery}/store.go (98%) rename {discoveryservice => discovery}/store_test.go (99%) rename {discoveryservice => discovery}/test.go (99%) rename {discoveryservice => discovery}/test/duplicate_id/1.json (100%) rename {discoveryservice => discovery}/test/duplicate_id/2.json (100%) rename {discoveryservice => discovery}/test/duplicate_id/README.md (100%) rename {discoveryservice => discovery}/test/invalid_definition/1.json (100%) rename {discoveryservice => discovery}/test/invalid_definition/README.md (100%) rename {discoveryservice => discovery}/test/invalid_json/1.json (100%) rename {discoveryservice => discovery}/test/invalid_json/README.md (100%) rename {discoveryservice => discovery}/test/valid/eoverdracht.json (100%) rename {discoveryservice => discovery}/test/valid/subdir/README.md (100%) rename {discoveryservice => discovery}/test/valid/subdir/empty.json (100%) diff --git a/discoveryservice/config.go b/discovery/config.go similarity index 98% rename from discoveryservice/config.go rename to discovery/config.go index f6d8647a8a..c4a325e9e9 100644 --- a/discoveryservice/config.go +++ b/discovery/config.go @@ -16,7 +16,7 @@ * */ -package discoveryservice +package discovery // Config holds the config of the module type Config struct { diff --git a/discoveryservice/definition.go b/discovery/definition.go similarity index 98% rename from discoveryservice/definition.go rename to discovery/definition.go index 2919c93e71..be0168dee0 100644 --- a/discoveryservice/definition.go +++ b/discovery/definition.go @@ -16,7 +16,7 @@ * */ -package discoveryservice +package discovery import ( "bytes" diff --git a/discoveryservice/interface.go b/discovery/interface.go similarity index 89% rename from discoveryservice/interface.go rename to discovery/interface.go index 4a32bcf053..40351f3ef9 100644 --- a/discoveryservice/interface.go +++ b/discovery/interface.go @@ -16,7 +16,7 @@ * */ -package discoveryservice +package discovery import ( "github.com/nuts-foundation/go-did/vc" @@ -29,14 +29,16 @@ import ( // Pass 0 to start at the beginning of the list. type Timestamp uint64 +// Server defines the API for Discovery Servers. type Server interface { - // Add registers a presentation of the given Discovery Service. + // Add registers a presentation on the given Discovery Service. // If the presentation is not valid or it does not conform to the Service Definition, it returns an error. Add(serviceID string, presentation vc.VerifiablePresentation) error // Get retrieves the presentations for the given service, starting at the given timestamp. Get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresentation, *Timestamp, error) } +// Client defines the API for Discovery Clients. type Client interface { Search(serviceID string, query map[string]string) ([]vc.VerifiablePresentation, error) } diff --git a/discoveryservice/log/logger.go b/discovery/log/logger.go similarity index 100% rename from discoveryservice/log/logger.go rename to discovery/log/logger.go diff --git a/discoveryservice/mock.go b/discovery/mock.go similarity index 92% rename from discoveryservice/mock.go rename to discovery/mock.go index c0b8b10c9c..335c3bcd0d 100644 --- a/discoveryservice/mock.go +++ b/discovery/mock.go @@ -1,12 +1,12 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: discoveryservice/interface.go +// Source: discovery/interface.go // // Generated by this command: // -// mockgen -destination=discoveryservice/mock.go -package=discoveryservice -source=discoveryservice/interface.go +// mockgen -destination=discovery/mock.go -package=discovery -source=discovery/interface.go // -// Package discoveryservice is a generated GoMock package. -package discoveryservice +// Package discovery is a generated GoMock package. +package discovery import ( reflect "reflect" diff --git a/discoveryservice/module.go b/discovery/module.go similarity index 98% rename from discoveryservice/module.go rename to discovery/module.go index fa12d1ef9d..1001695e16 100644 --- a/discoveryservice/module.go +++ b/discovery/module.go @@ -16,7 +16,7 @@ * */ -package discoveryservice +package discovery import ( "errors" @@ -41,15 +41,16 @@ var _ core.Runnable = &Module{} var _ core.Configurable = &Module{} var _ Server = &Module{} -// var _ Client = &Module{} var retractionPresentationType = ssi.MustParseURI("RetractedVerifiablePresentation") +// New creates a new Module. func New(storageInstance storage.Engine) *Module { return &Module{ storageInstance: storageInstance, } } +// Module is the main entry point for discovery services. type Module struct { config Config storageInstance storage.Engine diff --git a/discoveryservice/module_test.go b/discovery/module_test.go similarity index 99% rename from discoveryservice/module_test.go rename to discovery/module_test.go index 0a4af7aae6..b9fbace3df 100644 --- a/discoveryservice/module_test.go +++ b/discovery/module_test.go @@ -16,7 +16,7 @@ * */ -package discoveryservice +package discovery import ( "github.com/nuts-foundation/go-did/vc" diff --git a/discoveryservice/service-definition-schema.json b/discovery/service-definition-schema.json similarity index 100% rename from discoveryservice/service-definition-schema.json rename to discovery/service-definition-schema.json diff --git a/discoveryservice/store.go b/discovery/store.go similarity index 98% rename from discoveryservice/store.go rename to discovery/store.go index 2b805c5f20..61fc502d83 100644 --- a/discoveryservice/store.go +++ b/discovery/store.go @@ -16,14 +16,14 @@ * */ -package discoveryservice +package discovery import ( "errors" "fmt" "github.com/google/uuid" "github.com/nuts-foundation/go-did/vc" - "github.com/nuts-foundation/nuts-node/discoveryservice/log" + "github.com/nuts-foundation/nuts-node/discovery/log" credential "github.com/nuts-foundation/nuts-node/vcr/credential" "gorm.io/driver/sqlite" "gorm.io/gorm" @@ -305,7 +305,7 @@ func (s *sqlStore) search(serviceID string, query map[string]string) ([]vc.Verif func (s *sqlStore) updateTimestamp(tx *gorm.DB, serviceID string, newTimestamp *Timestamp) (Timestamp, error) { var result serviceRecord - // Lock (SELECT FOR UPDATE) discoveryservice row to prevent concurrent updates to the same list, which could mess up the lamport timestamp. + // Lock (SELECT FOR UPDATE) discovery_service row to prevent concurrent updates to the same list, which could mess up the lamport timestamp. // But, it is not supported by SQLite. SQLite defaults to table-level write-locks upon the first write action in the transaction. // if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). diff --git a/discoveryservice/store_test.go b/discovery/store_test.go similarity index 99% rename from discoveryservice/store_test.go rename to discovery/store_test.go index 463f78a646..d44c3d2c16 100644 --- a/discoveryservice/store_test.go +++ b/discovery/store_test.go @@ -16,7 +16,7 @@ * */ -package discoveryservice +package discovery import ( "github.com/nuts-foundation/go-did/vc" diff --git a/discoveryservice/test.go b/discovery/test.go similarity index 99% rename from discoveryservice/test.go rename to discovery/test.go index 38764f7da5..3b836924b4 100644 --- a/discoveryservice/test.go +++ b/discovery/test.go @@ -16,7 +16,7 @@ * */ -package discoveryservice +package discovery import ( "context" diff --git a/discoveryservice/test/duplicate_id/1.json b/discovery/test/duplicate_id/1.json similarity index 100% rename from discoveryservice/test/duplicate_id/1.json rename to discovery/test/duplicate_id/1.json diff --git a/discoveryservice/test/duplicate_id/2.json b/discovery/test/duplicate_id/2.json similarity index 100% rename from discoveryservice/test/duplicate_id/2.json rename to discovery/test/duplicate_id/2.json diff --git a/discoveryservice/test/duplicate_id/README.md b/discovery/test/duplicate_id/README.md similarity index 100% rename from discoveryservice/test/duplicate_id/README.md rename to discovery/test/duplicate_id/README.md diff --git a/discoveryservice/test/invalid_definition/1.json b/discovery/test/invalid_definition/1.json similarity index 100% rename from discoveryservice/test/invalid_definition/1.json rename to discovery/test/invalid_definition/1.json diff --git a/discoveryservice/test/invalid_definition/README.md b/discovery/test/invalid_definition/README.md similarity index 100% rename from discoveryservice/test/invalid_definition/README.md rename to discovery/test/invalid_definition/README.md diff --git a/discoveryservice/test/invalid_json/1.json b/discovery/test/invalid_json/1.json similarity index 100% rename from discoveryservice/test/invalid_json/1.json rename to discovery/test/invalid_json/1.json diff --git a/discoveryservice/test/invalid_json/README.md b/discovery/test/invalid_json/README.md similarity index 100% rename from discoveryservice/test/invalid_json/README.md rename to discovery/test/invalid_json/README.md diff --git a/discoveryservice/test/valid/eoverdracht.json b/discovery/test/valid/eoverdracht.json similarity index 100% rename from discoveryservice/test/valid/eoverdracht.json rename to discovery/test/valid/eoverdracht.json diff --git a/discoveryservice/test/valid/subdir/README.md b/discovery/test/valid/subdir/README.md similarity index 100% rename from discoveryservice/test/valid/subdir/README.md rename to discovery/test/valid/subdir/README.md diff --git a/discoveryservice/test/valid/subdir/empty.json b/discovery/test/valid/subdir/empty.json similarity index 100% rename from discoveryservice/test/valid/subdir/empty.json rename to discovery/test/valid/subdir/empty.json diff --git a/makefile b/makefile index 4c92cfa65c..7c3d233b67 100644 --- a/makefile +++ b/makefile @@ -19,7 +19,7 @@ gen-mocks: mockgen -destination=crypto/mock.go -package=crypto -source=crypto/interface.go mockgen -destination=crypto/storage/spi/mock.go -package spi -source=crypto/storage/spi/interface.go mockgen -destination=didman/mock.go -package=didman -source=didman/types.go - mockgen -destination=discoveryservice/mock.go -package=discoveryservice -source=discoveryservice/interface.go + mockgen -destination=discovery/mock.go -package=discovery -source=discovery/interface.go mockgen -destination=events/events_mock.go -package=events -source=events/interface.go Event mockgen -destination=events/mock.go -package=events -source=events/conn.go Conn ConnectionPool mockgen -destination=http/echo_mock.go -package=http -source=http/echo.go -imports echo=github.com/labstack/echo/v4 diff --git a/storage/sql_migrations/2_discoveryservice.down.sql b/storage/sql_migrations/2_discoveryservice.down.sql index 3922d7112b..02fe661c06 100644 --- a/storage/sql_migrations/2_discoveryservice.down.sql +++ b/storage/sql_migrations/2_discoveryservice.down.sql @@ -1,4 +1,4 @@ -drop table discoveryservices; -drop table discoveryservice_presentations; -drop table discoveryservice_credentials; -drop table discoveryservice_credential_props; \ No newline at end of file +drop table discovery_service; +drop table discovery_presentation; +drop table discovery_credential; +drop table discovery_credential_prop; \ No newline at end of file diff --git a/storage/sql_migrations/2_discoveryservice.up.sql b/storage/sql_migrations/2_discoveryservice.up.sql index 4f7dc05414..824d848bcb 100644 --- a/storage/sql_migrations/2_discoveryservice.up.sql +++ b/storage/sql_migrations/2_discoveryservice.up.sql @@ -33,7 +33,7 @@ create table discovery_credential -- The type stored in the type column will be the 'other' type, not being 'VerifiableCredential'. -- When credentials with 3 or more types appear, we could have to use a separate table for the types. credential_type varchar, - constraint fk_discoveryservice_credential_presentation foreign key (presentation_id) references discovery_presentation (id) on delete cascade + constraint fk_discovery_credential_presentation foreign key (presentation_id) references discovery_presentation (id) on delete cascade ); -- discovery_credential_prop contains the credentialSubject properties of a credential in a presentation of the discovery service. @@ -45,5 +45,5 @@ create table discovery_credential_prop value varchar, PRIMARY KEY (credential_id, key), -- cascading delete: if the presentation gets deleted, the properties get deleted as well - constraint fk_discoveryservice_credential_id foreign key (credential_id) references discovery_credential (id) on delete cascade + constraint fk_discovery_credential_id foreign key (credential_id) references discovery_credential (id) on delete cascade ); \ No newline at end of file From 6d8a88db305a113baf155c65a894e9b75b19a5c0 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Mon, 20 Nov 2023 17:05:33 +0100 Subject: [PATCH 07/23] unusedfunc --- vcr/credential/resolver.go | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/vcr/credential/resolver.go b/vcr/credential/resolver.go index 5267a3b4f9..7b0eb8092b 100644 --- a/vcr/credential/resolver.go +++ b/vcr/credential/resolver.go @@ -23,7 +23,6 @@ import ( "errors" "github.com/nuts-foundation/go-did/did" "github.com/nuts-foundation/go-did/vc" - "strings" ) // FindValidator finds the Validator the provided credential based on its Type @@ -69,34 +68,3 @@ func PresentationSigner(presentation vc.VerifiablePresentation) (*did.DID, error return nil, errors.New("unsupported presentation format") } } - -func PresentationSigningKeyID(presentation vc.VerifiablePresentation) (*did.DIDURL, error) { - switch presentation.Format() { - case vc.JWTPresentationProofFormat: - token := presentation.JWT() - keyID, exists := token.Get("kid") - if !exists { - return nil, errors.New("JWT presentation does not have 'kid' claim") - } - keyIDString, isString := keyID.(string) - if !isString { - return nil, errors.New("JWT presentation 'kid' claim is not a string") - } - issuer, err := PresentationSigner(presentation) - if err != nil { - return nil, err - } - if strings.HasPrefix(keyIDString, "#") { - // Key ID is a fragment, so it's a relative URL to the JWT issuer - keyIDString = issuer.String() + keyIDString - } else { - // Key ID is fully qualified, must be prefixed with JWT issuer - if !strings.HasPrefix(keyIDString, issuer.String()+"#") { - return nil, errors.New("JWT presentation 'kid' claim must be scoped to 'iss' claim if absolute") - } - } - return did.ParseDIDURL(keyIDString) - default: - return nil, errors.New("unsupported presentation format") - } -} From 55dcedec1517d02199d0248d0d8f69ac48234def Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Mon, 20 Nov 2023 17:08:23 +0100 Subject: [PATCH 08/23] godoc, moved errors to interface.go --- discovery/interface.go | 8 ++++++++ discovery/module.go | 6 +----- discovery/store.go | 3 --- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/discovery/interface.go b/discovery/interface.go index 40351f3ef9..dc66642a37 100644 --- a/discovery/interface.go +++ b/discovery/interface.go @@ -19,6 +19,7 @@ package discovery import ( + "errors" "github.com/nuts-foundation/go-did/vc" ) @@ -29,6 +30,13 @@ import ( // Pass 0 to start at the beginning of the list. type Timestamp uint64 +// ErrServiceNotFound is returned when a service (ID) is not found in the discovery service. +var ErrServiceNotFound = errors.New("discovery service not found") + +// ErrPresentationAlreadyExists is returned when a presentation is added to the discovery service, +// but a presentation with this ID already exists. +var ErrPresentationAlreadyExists = errors.New("presentation already exists") + // Server defines the API for Discovery Servers. type Server interface { // Add registers a presentation on the given Discovery Service. diff --git a/discovery/module.go b/discovery/module.go index 1001695e16..5ff3fa10fb 100644 --- a/discovery/module.go +++ b/discovery/module.go @@ -139,16 +139,12 @@ func (m *Module) Add(serviceID string, presentation vc.VerifiablePresentation) e } func (m *Module) addPresentation(definition Definition, presentation vc.VerifiablePresentation) error { - // Must contain credentials - if len(presentation.VerifiableCredential) == 0 { - return errors.New("presentation must contain at least one credentialRecord") - } // VP can't be valid longer than the credentialRecord it contains expiration := presentation.JWT().Expiration() for _, cred := range presentation.VerifiableCredential { exp := cred.JWT().Expiration() if !exp.IsZero() && expiration.After(exp) { - return fmt.Errorf("presentation is valid longer than the credentialRecord(s) it contains") + return fmt.Errorf("presentation is valid longer than the credential(s) it contains") } } // VP must fulfill the PEX Presentation Definition diff --git a/discovery/store.go b/discovery/store.go index 61fc502d83..e506cbf5e0 100644 --- a/discovery/store.go +++ b/discovery/store.go @@ -35,9 +35,6 @@ import ( "time" ) -var ErrServiceNotFound = errors.New("discovery service not found") -var ErrPresentationAlreadyExists = errors.New("presentation already exists") - type serviceRecord struct { ID string `gorm:"primaryKey"` LamportTimestamp uint64 From e44fa5f127e96ee33a0cc0392ba8530da87a9ff0 Mon Sep 17 00:00:00 2001 From: reinkrul Date: Mon, 20 Nov 2023 11:34:22 +0100 Subject: [PATCH 09/23] PEX: Changed path_nested to be object instead of slice (#2609) --- vcr/pe/presentation_submission.go | 60 +++++++----- vcr/pe/presentation_submission_test.go | 124 ++++++++++++++++++++++--- vcr/pe/test/test_definitions.go | 4 +- vcr/pe/types.go | 8 +- 4 files changed, 153 insertions(+), 43 deletions(-) diff --git a/vcr/pe/presentation_submission.go b/vcr/pe/presentation_submission.go index fa858d199a..66fecd8f75 100644 --- a/vcr/pe/presentation_submission.go +++ b/vcr/pe/presentation_submission.go @@ -71,8 +71,9 @@ type SignInstruction struct { // Holder contains the DID of the holder that should sign the VP. Holder did.DID // VerifiableCredentials contains the VCs that should be included in the VP. - VerifiableCredentials []vc.VerifiableCredential - inputDescriptorMappingObjects []InputDescriptorMappingObject + VerifiableCredentials []vc.VerifiableCredential + // Mappings contains the Input Descriptor that are mapped by this SignInstruction. + Mappings []InputDescriptorMappingObject } // Empty returns true if there are no VCs in the SignInstruction. @@ -100,7 +101,6 @@ func (b *PresentationSubmissionBuilder) Build(format string) (PresentationSubmis Id: uuid.New().String(), DefinitionId: b.presentationDefinition.Id, } - signInstructions := make([]SignInstruction, len(b.wallets)) // first we need to select the VCs from all wallets that match the presentation definition allVCs := make([]vc.VerifiableCredential, 0) @@ -115,43 +115,57 @@ func (b *PresentationSubmissionBuilder) Build(format string) (PresentationSubmis // next we need to map the selected VCs to the correct wallet // loop over all selected VCs and find the wallet that contains the VC + signInstructions := make([]SignInstruction, len(b.wallets)) + walletCredentialIndex := map[did.DID]int{} for j := range selectedVCs { for i, walletVCs := range b.wallets { - var index int for _, walletVC := range walletVCs { // do a JSON equality check - if vcEqual(selectedVCs[j], walletVC) { + if selectedVCs[j].Raw() == walletVC.Raw() { signInstructions[i].Holder = b.holders[i] signInstructions[i].VerifiableCredentials = append(signInstructions[i].VerifiableCredentials, selectedVCs[j]) // remap the path to the correct wallet index - inputDescriptorMappingObjectForVC := inputDescriptorMappingObjects[j] - inputDescriptorMappingObjectForVC.Path = fmt.Sprintf("$.verifiableCredential[%d]", index) - signInstructions[i].inputDescriptorMappingObjects = append(signInstructions[i].inputDescriptorMappingObjects, inputDescriptorMappingObjectForVC) - index++ + mapping := inputDescriptorMappingObjects[j] + mapping.Format = selectedVCs[j].Format() + mapping.Path = fmt.Sprintf("$.verifiableCredential[%d]", walletCredentialIndex[b.holders[i]]) + signInstructions[i].Mappings = append(signInstructions[i].Mappings, mapping) + walletCredentialIndex[b.holders[i]]++ } } } } + // filter out empty sign instructions + nonEmptySignInstructions := make([]SignInstruction, 0) + for _, signInstruction := range signInstructions { + if !signInstruction.Empty() { + nonEmptySignInstructions = append(nonEmptySignInstructions, signInstruction) + } + } + index := 0 // last we create the descriptor map for the presentation submission - // If there's only one sign instruction the Path will be $. If there are multiple sign instructions the Path will be $[0], $[1], etc. - for _, signInstruction := range signInstructions { - if len(signInstruction.VerifiableCredentials) > 0 { - // wrap each InputDescriptorMappingObject for the outer VP - nestedDescriptorMap := InputDescriptorMappingObject{ - Id: "", // todo what to add here? - Format: format, - Path: "$.", - PathNested: signInstruction.inputDescriptorMappingObjects, - } - if len(signInstructions) > 1 { - nestedDescriptorMap.Path = fmt.Sprintf("$[%d]", index) + // If there's only one sign instruction the Path will be $. + // If there are multiple sign instructions (each yielding a VP) the Path will be $[0], $[1], etc. + for _, signInstruction := range nonEmptySignInstructions { + if len(signInstruction.Mappings) > 0 { + for _, inputDescriptorMapping := range signInstruction.Mappings { + // If we have multiple VPs in the resulting submission, wrap each in a nested descriptor map (see path_nested in PEX specification). + if len(nonEmptySignInstructions) > 1 { + presentationSubmission.DescriptorMap = append(presentationSubmission.DescriptorMap, InputDescriptorMappingObject{ + Id: inputDescriptorMapping.Id, + Format: format, + Path: fmt.Sprintf("$[%d]", index), + PathNested: &inputDescriptorMapping, + }) + } else { + // Just 1 VP, no nesting needed + presentationSubmission.DescriptorMap = append(presentationSubmission.DescriptorMap, inputDescriptorMapping) + } } - presentationSubmission.DescriptorMap = append(presentationSubmission.DescriptorMap, nestedDescriptorMap) index++ } } - return presentationSubmission, signInstructions, nil + return presentationSubmission, nonEmptySignInstructions, nil } diff --git a/vcr/pe/presentation_submission_test.go b/vcr/pe/presentation_submission_test.go index 6c9f31c5e2..aa6b837599 100644 --- a/vcr/pe/presentation_submission_test.go +++ b/vcr/pe/presentation_submission_test.go @@ -46,10 +46,29 @@ func TestPresentationSubmissionBuilder_Build(t *testing.T) { holder2 := did.MustParseDID("did:example:2") id1 := ssi.MustParseURI("1") id2 := ssi.MustParseURI("2") - vc1 := vc.VerifiableCredential{ID: &id1} - vc2 := vc.VerifiableCredential{ID: &id2} + id3 := ssi.MustParseURI("3") + vc1 := credentialToJSONLD(vc.VerifiableCredential{ID: &id1}) + vc2 := credentialToJSONLD(vc.VerifiableCredential{ID: &id2}) + vc3 := credentialToJSONLD(vc.VerifiableCredential{ID: &id3}) - t.Run("ok - single wallet", func(t *testing.T) { + t.Run("1 presentation", func(t *testing.T) { + expectedJSON := ` + { + "id": "for-test", + "definition_id": "", + "descriptor_map": [ + { + "format": "ldp_vc", + "id": "Match ID=1", + "path": "$.verifiableCredential[0]" + }, + { + "format": "ldp_vc", + "id": "Match ID=2", + "path": "$.verifiableCredential[1]" + } + ] + }` presentationDefinition := PresentationDefinition{} _ = json.Unmarshal([]byte(test.All), &presentationDefinition) builder := presentationDefinition.PresentationSubmissionBuilder() @@ -60,12 +79,42 @@ func TestPresentationSubmissionBuilder_Build(t *testing.T) { require.NoError(t, err) require.NotNil(t, signInstructions) assert.Len(t, signInstructions, 1) - assert.Len(t, submission.DescriptorMap, 1) - assert.Equal(t, "$.", submission.DescriptorMap[0].Path) - require.Len(t, submission.DescriptorMap[0].PathNested, 2) - assert.Equal(t, "$.verifiableCredential[0]", submission.DescriptorMap[0].PathNested[0].Path) + require.Len(t, submission.DescriptorMap, 2) + + submission.Id = "for-test" // easier assertion + actualJSON, _ := json.MarshalIndent(submission, "", " ") + println(string(actualJSON)) + assert.JSONEq(t, expectedJSON, string(actualJSON)) }) - t.Run("ok - two wallets", func(t *testing.T) { + t.Run("2 presentations", func(t *testing.T) { + expectedJSON := ` +{ + "id": "for-test", + "definition_id": "", + "descriptor_map": [ + { + "format": "ldp_vp", + "id": "Match ID=1", + "path": "$[0]", + "path_nested": { + "format": "ldp_vc", + "id": "Match ID=1", + "path": "$.verifiableCredential[0]" + } + }, + { + "format": "ldp_vp", + "id": "Match ID=2", + "path": "$[1]", + "path_nested": { + "format": "ldp_vc", + "id": "Match ID=2", + "path": "$.verifiableCredential[0]" + } + } + ] +} +` presentationDefinition := PresentationDefinition{} _ = json.Unmarshal([]byte(test.All), &presentationDefinition) builder := presentationDefinition.PresentationSubmissionBuilder() @@ -78,11 +127,58 @@ func TestPresentationSubmissionBuilder_Build(t *testing.T) { require.NotNil(t, signInstructions) assert.Len(t, signInstructions, 2) assert.Len(t, submission.DescriptorMap, 2) - assert.Equal(t, "$[0]", submission.DescriptorMap[0].Path) - require.Len(t, submission.DescriptorMap[0].PathNested, 1) - assert.Equal(t, "$.verifiableCredential[0]", submission.DescriptorMap[0].PathNested[0].Path) - assert.Equal(t, "$[1]", submission.DescriptorMap[1].Path) - require.Len(t, submission.DescriptorMap[1].PathNested, 1) - assert.Equal(t, "$.verifiableCredential[0]", submission.DescriptorMap[1].PathNested[0].Path) + + submission.Id = "for-test" // easier assertion + actualJSON, _ := json.MarshalIndent(submission, "", " ") + assert.JSONEq(t, expectedJSON, string(actualJSON)) + }) + t.Run("2 wallets, but 1 VP", func(t *testing.T) { + expectedJSON := ` + { + "id": "for-test", + "definition_id": "", + "descriptor_map": [ + { + "format": "ldp_vc", + "id": "Match ID=1", + "path": "$.verifiableCredential[0]" + }, + { + "format": "ldp_vc", + "id": "Match ID=2", + "path": "$.verifiableCredential[1]" + } + ] + }` + presentationDefinition := PresentationDefinition{} + _ = json.Unmarshal([]byte(test.All), &presentationDefinition) + builder := presentationDefinition.PresentationSubmissionBuilder() + builder.AddWallet(holder1, []vc.VerifiableCredential{vc1, vc2}) + builder.AddWallet(holder2, []vc.VerifiableCredential{vc3}) + + submission, signInstructions, err := builder.Build("ldp_vp") + + require.NoError(t, err) + require.NotNil(t, signInstructions) + assert.Len(t, signInstructions, 1) + assert.Len(t, submission.DescriptorMap, 2) + + submission.Id = "for-test" // easier assertion + actualJSON, _ := json.MarshalIndent(submission, "", " ") + println(string(actualJSON)) + assert.JSONEq(t, expectedJSON, string(actualJSON)) }) } + +func credentialToJSONLD(credential vc.VerifiableCredential) vc.VerifiableCredential { + bytes, err := credential.MarshalJSON() + if err != nil { + panic(err) + } + var result vc.VerifiableCredential + err = json.Unmarshal(bytes, &result) + if err != nil { + panic(err) + } + return result +} diff --git a/vcr/pe/test/test_definitions.go b/vcr/pe/test/test_definitions.go index 4ce7954022..3562eebb96 100644 --- a/vcr/pe/test/test_definitions.go +++ b/vcr/pe/test/test_definitions.go @@ -183,7 +183,7 @@ const All = ` ], "input_descriptors": [ { - "name": "Match ID=1", + "id": "Match ID=1", "group": ["A"], "constraints": { "fields": [ @@ -200,7 +200,7 @@ const All = ` } }, { - "name": "Match ID=2", + "id": "Match ID=2", "group": ["A"], "constraints": { "fields": [ diff --git a/vcr/pe/types.go b/vcr/pe/types.go index aa57bdd4e4..c0cadfbdef 100644 --- a/vcr/pe/types.go +++ b/vcr/pe/types.go @@ -34,10 +34,10 @@ type PresentationSubmission struct { // InputDescriptorMappingObject type InputDescriptorMappingObject struct { - Format string `json:"format"` - Id string `json:"id"` - Path string `json:"path"` - PathNested []InputDescriptorMappingObject `json:"path_nested,omitempty"` + Format string `json:"format"` + Id string `json:"id"` + Path string `json:"path"` + PathNested *InputDescriptorMappingObject `json:"path_nested,omitempty"` } // Constraints From 4d815cb83d7ebcc9f9aded3b07b6abb7a0879702 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Tue, 21 Nov 2023 13:42:49 +0100 Subject: [PATCH 10/23] wip --- discovery/module.go | 7 ++----- discovery/module_test.go | 14 ++++++-------- storage/engine_test.go | 1 - vcr/pe/presentation_definition.go | 3 +-- 4 files changed, 9 insertions(+), 16 deletions(-) diff --git a/discovery/module.go b/discovery/module.go index 5ff3fa10fb..fe10d80129 100644 --- a/discovery/module.go +++ b/discovery/module.go @@ -120,11 +120,8 @@ func (m *Module) Add(serviceID string, presentation vc.VerifiablePresentation) e return errors.New("presentation does not have an ID") } expiration := presentation.JWT().Expiration() - // VPs should not be valid for too short; unnecessary overhead at the server and clients. - // Also protects against expiration not being set at all. The factor is somewhat arbitrary. - minValidity := float64(definition.PresentationMaxValidity / 4) - if expiration.Sub(time.Now()).Seconds() < minValidity { - return fmt.Errorf("presentation is not valid for long enough (min %s)", time.Duration(minValidity)*time.Second) + if expiration.IsZero() { + return errors.New("presentation does not have an expiration") } // VPs should not be valid for too long, as that would prevent the server from pruning them. if int(expiration.Sub(time.Now()).Seconds()) > definition.PresentationMaxValidity { diff --git a/discovery/module_test.go b/discovery/module_test.go index b9fbace3df..82e019f442 100644 --- a/discovery/module_test.go +++ b/discovery/module_test.go @@ -89,16 +89,14 @@ func Test_Module_Add(t *testing.T) { }) vpAlice := createPresentation(aliceDID, vcAlice) err := m.Add(testServiceID, vpAlice) - assert.EqualError(t, err, "presentation is valid longer than the credentialRecord(s) it contains") + assert.EqualError(t, err, "presentation is valid longer than the credential(s) it contains") }) - t.Run("not valid long enough", func(t *testing.T) { + t.Run("no expiration", func(t *testing.T) { m := setupModule(t, storageEngine) - def := m.services[testServiceID] - def.PresentationMaxValidity = int((24 * time.Hour).Seconds() * 365) - m.services[testServiceID] = def - - err := m.Add(testServiceID, vpAlice) - assert.EqualError(t, err, "presentation is not valid for long enough (min 2190h0m0s)") + err := m.Add(testServiceID, createPresentationCustom(aliceDID, func(claims map[string]interface{}, _ *vc.VerifiablePresentation) { + delete(claims, "exp") + })) + assert.EqualError(t, err, "presentation does not have an expiration") }) t.Run("presentation does not contain an ID", func(t *testing.T) { m := setupModule(t, storageEngine) diff --git a/storage/engine_test.go b/storage/engine_test.go index 82f1af7524..3594169207 100644 --- a/storage/engine_test.go +++ b/storage/engine_test.go @@ -132,7 +132,6 @@ func Test_engine_sqlDatabase(t *testing.T) { }) t.Run("runs migrations", func(t *testing.T) { e := New().(*engine) - e.config.SQL.ConnectionString = SQLiteInMemoryConnectionString require.NoError(t, e.Configure(*core.NewServerConfig())) require.NoError(t, e.Start()) t.Cleanup(func() { diff --git a/vcr/pe/presentation_definition.go b/vcr/pe/presentation_definition.go index 71b3d6b8dc..027f957fbf 100644 --- a/vcr/pe/presentation_definition.go +++ b/vcr/pe/presentation_definition.go @@ -292,9 +292,8 @@ func matchConstraint(constraint *Constraints, credential vc.VerifiableCredential // matchField matches the field against the VC. // All fields need to match unless optional is set to true and no values are found for all the paths. func matchField(field Field, credential vc.VerifiableCredential) (bool, error) { - type Alias vc.VerifiableCredential // jsonpath works on interfaces, so convert the VC to an interface - asJSON, _ := json.Marshal(Alias(credential)) + asJSON, _ := json.Marshal(credential) var asInterface interface{} _ = json.Unmarshal(asJSON, &asInterface) From 28f27956162a897f290023a5149b5beb83cf2077 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Fri, 24 Nov 2023 12:57:37 +0100 Subject: [PATCH 11/23] PR feedback --- README.rst | 8 +- discovery/config.go | 8 +- discovery/definition.go | 18 ++- discovery/interface.go | 2 +- discovery/module.go | 26 ++-- discovery/module_test.go | 20 +-- discovery/store.go | 2 +- discovery/test.go | 4 +- discovery/test/invalid_definition/README.md | 2 +- docs/pages/deployment/cli-reference.rst | 2 +- docs/pages/deployment/server_options.rst | 164 ++++++++++---------- storage/cmd/cmd.go | 5 +- 12 files changed, 132 insertions(+), 129 deletions(-) diff --git a/README.rst b/README.rst index c3fb894efe..4ef69925ff 100644 --- a/README.rst +++ b/README.rst @@ -176,9 +176,9 @@ The following options can be configured on the server: :widths: 20 30 50 :class: options-table - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================================================================================ Key Default Description - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================================================================================ configfile nuts.yaml Nuts config file cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. datadir ./data Directory where the node stores its files. @@ -252,12 +252,12 @@ The following options can be configured on the server: storage.redis.sentinel.password Password for authenticating to Redis Sentinels. storage.redis.sentinel.username Username for authenticating to Redis Sentinels. storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). - storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If using a SQLite database, make sure to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL'). + storage.sql.connection Connection string for the SQL database. If not set it, defaults to a SQLite database stored inside the configured data directory. Note: using SQLite is not recommended in production environments. If using SQLite anyways, remember to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL'). **VCR** vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================================================================================ This table is automatically generated using the configuration flags in the core and engines. When they're changed the options table must be regenerated using the Makefile: diff --git a/discovery/config.go b/discovery/config.go index c4a325e9e9..1a432cbbf6 100644 --- a/discovery/config.go +++ b/discovery/config.go @@ -20,12 +20,12 @@ package discovery // Config holds the config of the module type Config struct { - Server ServerConfig `koanf:"server"` - Definitions DefinitionsConfig `koanf:"definitions"` + Server ServerConfig `koanf:"server"` + Definitions ServiceDefinitionsConfig `koanf:"definitions"` } -// DefinitionsConfig holds the config for loading Service Definitions. -type DefinitionsConfig struct { +// ServiceDefinitionsConfig holds the config for loading Service Definitions. +type ServiceDefinitionsConfig struct { Directory string `koanf:"directory"` } diff --git a/discovery/definition.go b/discovery/definition.go index be0168dee0..6315424b65 100644 --- a/discovery/definition.go +++ b/discovery/definition.go @@ -29,7 +29,7 @@ import ( //go:embed *.json var jsonSchemaFiles embed.FS -var definitionJsonSchema *jsonschema.Schema +var serviceDefinitionJsonSchema *jsonschema.Schema func init() { serviceDefinitionSchemaData, err := jsonSchemaFiles.ReadFile("service-definition-schema.json") @@ -41,27 +41,29 @@ func init() { if err := compiler.AddResource(schemaURL, bytes.NewReader(serviceDefinitionSchemaData)); err != nil { panic(err) } - definitionJsonSchema = compiler.MustCompile(schemaURL) + serviceDefinitionJsonSchema = compiler.MustCompile(schemaURL) } -// Definition holds the definition of a service. -type Definition struct { +// ServiceDefinition holds the definition of a service. +type ServiceDefinition struct { // ID is the unique identifier of the use case. ID string `json:"id"` // Endpoint is the endpoint where the use case list is served. Endpoint string `json:"endpoint"` - // PresentationDefinition specifies the Presentation Definition submissions to the list must conform to, + // PresentationDefinition specifies the Presentation ServiceDefinition submissions to the list must conform to, // according to the Presentation Exchange specification. PresentationDefinition pe.PresentationDefinition `json:"presentation_definition"` // PresentationMaxValidity specifies how long submitted presentations are allowed to be valid (in seconds). PresentationMaxValidity int `json:"presentation_max_validity"` } -func ParseDefinition(data []byte) (*Definition, error) { - if err := definitionJsonSchema.Validate(bytes.NewReader(data)); err != nil { +// ParseServiceDefinition validates the input against the JSON schema for service definitions. +// If the input is valid, it is parsed and returned as a ServiceDefinition. +func ParseServiceDefinition(data []byte) (*ServiceDefinition, error) { + if err := serviceDefinitionJsonSchema.Validate(bytes.NewReader(data)); err != nil { return nil, err } - var definition Definition + var definition ServiceDefinition if err := json.Unmarshal(data, &definition); err != nil { return nil, err } diff --git a/discovery/interface.go b/discovery/interface.go index dc66642a37..81ffc78d19 100644 --- a/discovery/interface.go +++ b/discovery/interface.go @@ -40,7 +40,7 @@ var ErrPresentationAlreadyExists = errors.New("presentation already exists") // Server defines the API for Discovery Servers. type Server interface { // Add registers a presentation on the given Discovery Service. - // If the presentation is not valid or it does not conform to the Service Definition, it returns an error. + // If the presentation is not valid or it does not conform to the Service ServiceDefinition, it returns an error. Add(serviceID string, presentation vc.VerifiablePresentation) error // Get retrieves the presentations for the given service, starting at the given timestamp. Get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresentation, *Timestamp, error) diff --git a/discovery/module.go b/discovery/module.go index fe10d80129..64c8c5447e 100644 --- a/discovery/module.go +++ b/discovery/module.go @@ -55,8 +55,8 @@ type Module struct { config Config storageInstance storage.Engine store *sqlStore - serverDefinitions map[string]Definition - services map[string]Definition + serverDefinitions map[string]ServiceDefinition + services map[string]ServiceDefinition } func (m *Module) Configure(_ core.ServerConfig) error { @@ -70,10 +70,10 @@ func (m *Module) Configure(_ core.ServerConfig) error { } if len(m.config.Server.DefinitionIDs) > 0 { // Get the definitions that are enabled for this server - serverDefinitions := make(map[string]Definition) + serverDefinitions := make(map[string]ServiceDefinition) for _, definitionID := range m.config.Server.DefinitionIDs { if definition, exists := m.services[definitionID]; !exists { - return fmt.Errorf("definition '%s' not found", definitionID) + return fmt.Errorf("service definition '%s' not found", definitionID) } else { serverDefinitions[definitionID] = definition } @@ -135,7 +135,7 @@ func (m *Module) Add(serviceID string, presentation vc.VerifiablePresentation) e } } -func (m *Module) addPresentation(definition Definition, presentation vc.VerifiablePresentation) error { +func (m *Module) addPresentation(definition ServiceDefinition, presentation vc.VerifiablePresentation) error { // VP can't be valid longer than the credentialRecord it contains expiration := presentation.JWT().Expiration() for _, cred := range presentation.VerifiableCredential { @@ -144,10 +144,10 @@ func (m *Module) addPresentation(definition Definition, presentation vc.Verifiab return fmt.Errorf("presentation is valid longer than the credential(s) it contains") } } - // VP must fulfill the PEX Presentation Definition + // VP must fulfill the PEX Presentation ServiceDefinition creds, _, err := definition.PresentationDefinition.Match(presentation.VerifiableCredential) if err != nil || len(creds) != len(presentation.VerifiableCredential) { - return fmt.Errorf("presentation does not fulfill Presentation Definition: %w", err) + return fmt.Errorf("presentation does not fulfill Presentation ServiceDefinition: %w", err) } return m.store.add(definition.ID, presentation, nil) } @@ -198,12 +198,12 @@ func (m *Module) Get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresen return m.store.get(serviceID, startAt) } -func loadDefinitions(directory string) (map[string]Definition, error) { +func loadDefinitions(directory string) (map[string]ServiceDefinition, error) { entries, err := os.ReadDir(directory) if err != nil { return nil, fmt.Errorf("unable to read definitions directory '%s': %w", directory, err) } - result := make(map[string]Definition) + result := make(map[string]ServiceDefinition) for _, entry := range entries { if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { continue @@ -211,14 +211,14 @@ func loadDefinitions(directory string) (map[string]Definition, error) { filePath := path.Join(directory, entry.Name()) data, err := os.ReadFile(filePath) if err != nil { - return nil, fmt.Errorf("unable to read definition file '%s': %w", filePath, err) + return nil, fmt.Errorf("unable to read service definition file '%s': %w", filePath, err) } - definition, err := ParseDefinition(data) + definition, err := ParseServiceDefinition(data) if err != nil { - return nil, fmt.Errorf("unable to parse definition file '%s': %w", filePath, err) + return nil, fmt.Errorf("unable to parse service definition file '%s': %w", filePath, err) } if _, exists := result[definition.ID]; exists { - return nil, fmt.Errorf("duplicate definition ID '%s' in file '%s'", definition.ID, filePath) + return nil, fmt.Errorf("duplicate service definition ID '%s' in file '%s'", definition.ID, filePath) } result[definition.ID] = *definition } diff --git a/discovery/module_test.go b/discovery/module_test.go index 82e019f442..61d0eff0f7 100644 --- a/discovery/module_test.go +++ b/discovery/module_test.go @@ -51,7 +51,7 @@ func Test_Module_Add(t *testing.T) { require.NoError(t, err) assert.Equal(t, Timestamp(1), *timestamp) }) - t.Run("replace presentation of same credentialRecord subject", func(t *testing.T) { + t.Run("replace presentation of same credential subject", func(t *testing.T) { m := setupModule(t, storageEngine) vpAlice2 := createPresentation(aliceDID, vcAlice) @@ -204,7 +204,7 @@ func setupModule(t *testing.T, storageInstance storage.Engine) *Module { m := New(storageInstance) require.NoError(t, m.Configure(core.ServerConfig{})) m.services = testDefinitions() - m.serverDefinitions = map[string]Definition{ + m.serverDefinitions = map[string]ServiceDefinition{ testServiceID: m.services[testServiceID], } require.NoError(t, m.Start()) @@ -215,34 +215,34 @@ func TestModule_Configure(t *testing.T) { serverConfig := core.ServerConfig{} t.Run("duplicate ID", func(t *testing.T) { config := Config{ - Definitions: DefinitionsConfig{ + Definitions: ServiceDefinitionsConfig{ Directory: "test/duplicate_id", }, } err := (&Module{config: config}).Configure(serverConfig) - assert.EqualError(t, err, "duplicate definition ID 'urn:nuts.nl:usecase:eOverdrachtDev2023' in file 'test/duplicate_id/2.json'") + assert.EqualError(t, err, "duplicate service definition ID 'urn:nuts.nl:usecase:eOverdrachtDev2023' in file 'test/duplicate_id/2.json'") }) t.Run("invalid JSON", func(t *testing.T) { config := Config{ - Definitions: DefinitionsConfig{ + Definitions: ServiceDefinitionsConfig{ Directory: "test/invalid_json", }, } err := (&Module{config: config}).Configure(serverConfig) - assert.ErrorContains(t, err, "unable to parse definition file 'test/invalid_json/1.json'") + assert.ErrorContains(t, err, "unable to parse service definition file 'test/invalid_json/1.json'") }) - t.Run("invalid definition", func(t *testing.T) { + t.Run("invalid service definition", func(t *testing.T) { config := Config{ - Definitions: DefinitionsConfig{ + Definitions: ServiceDefinitionsConfig{ Directory: "test/invalid_definition", }, } err := (&Module{config: config}).Configure(serverConfig) - assert.ErrorContains(t, err, "unable to parse definition file 'test/invalid_definition/1.json'") + assert.ErrorContains(t, err, "unable to parse service definition file 'test/invalid_definition/1.json'") }) t.Run("non-existent directory", func(t *testing.T) { config := Config{ - Definitions: DefinitionsConfig{ + Definitions: ServiceDefinitionsConfig{ Directory: "test/non_existent", }, } diff --git a/discovery/store.go b/discovery/store.go index e506cbf5e0..06833331db 100644 --- a/discovery/store.go +++ b/discovery/store.go @@ -103,7 +103,7 @@ type sqlStore struct { writeLock sync.Mutex } -func newSQLStore(db *gorm.DB, definitions map[string]Definition) (*sqlStore, error) { +func newSQLStore(db *gorm.DB, definitions map[string]ServiceDefinition) (*sqlStore, error) { // Creates entries in the discovery service table with initial timestamp, if they don't exist yet for _, definition := range definitions { currentList := serviceRecord{ diff --git a/discovery/test.go b/discovery/test.go index 3b836924b4..987713ff5a 100644 --- a/discovery/test.go +++ b/discovery/test.go @@ -47,8 +47,8 @@ var vpBob vc.VerifiablePresentation var testServiceID = "usecase_v1" -func testDefinitions() map[string]Definition { - return map[string]Definition{ +func testDefinitions() map[string]ServiceDefinition { + return map[string]ServiceDefinition{ testServiceID: { ID: testServiceID, Endpoint: "http://example.com/usecase", diff --git a/discovery/test/invalid_definition/README.md b/discovery/test/invalid_definition/README.md index 6053c6729b..0d166a7da4 100644 --- a/discovery/test/invalid_definition/README.md +++ b/discovery/test/invalid_definition/README.md @@ -1 +1 @@ -This directory contains an invalid use case definition: it does not fields that are required according to the JSON schema. \ No newline at end of file +This directory contains an invalid use case definition: it does not contain the fields that are required according to the JSON schema. \ No newline at end of file diff --git a/docs/pages/deployment/cli-reference.rst b/docs/pages/deployment/cli-reference.rst index 6974629a28..9f902661cd 100755 --- a/docs/pages/deployment/cli-reference.rst +++ b/docs/pages/deployment/cli-reference.rst @@ -70,7 +70,7 @@ The following options apply to the server commands below: --storage.redis.sentinel.username string Username for authenticating to Redis Sentinels. --storage.redis.tls.truststorefile string PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). --storage.redis.username string Redis database username. If set, it overrides the username in the connection URL. - --storage.sql.connection string Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If using a SQLite database, make sure to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL'). + --storage.sql.connection string Connection string for the SQL database. If not set it, defaults to a SQLite database stored inside the configured data directory. Note: using SQLite is not recommended in production environments. If using SQLite anyways, remember to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL'). --strictmode When set, insecure settings are forbidden. (default true) --tls.certfile string PEM file containing the certificate for the server (also used as client certificate). --tls.certheader string Name of the HTTP header that will contain the client certificate when TLS is offloaded. diff --git a/docs/pages/deployment/server_options.rst b/docs/pages/deployment/server_options.rst index f45c8bc310..a4f9973030 100755 --- a/docs/pages/deployment/server_options.rst +++ b/docs/pages/deployment/server_options.rst @@ -2,85 +2,85 @@ :widths: 20 30 50 :class: options-table - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== - Key Default Description - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== - configfile nuts.yaml Nuts config file - cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. - datadir ./data Directory where the node stores its files. - internalratelimiter true When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. - loggerformat text Log format (text, json) - strictmode true When set, insecure settings are forbidden. - verbosity info Log level (trace, debug, info, warn, error) - tls.certfile PEM file containing the certificate for the server (also used as client certificate). - tls.certheader Name of the HTTP header that will contain the client certificate when TLS is offloaded. - tls.certkeyfile PEM file containing the private key of the server certificate. - tls.offload Whether to enable TLS offloading for incoming connections. Enable by setting it to 'incoming'. If enabled 'tls.certheader' must be configured as well. - tls.truststorefile truststore.pem PEM file containing the trusted CA certificates for authenticating remote servers. - **Auth** - auth.accesstokenlifespan 60 defines how long (in seconds) an access token is valid. Uses default in strict mode. - auth.clockskew 5000 allowed JWT Clock skew in milliseconds - auth.contractvalidators [irma,uzi,dummy,employeeid] sets the different contract validators to use - auth.publicurl public URL which can be reached by a users IRMA client, this should include the scheme and domain: https://example.com. Additional paths should only be added if some sort of url-rewriting is done in a reverse-proxy. - auth.http.timeout 30 HTTP timeout (in seconds) used by the Auth API HTTP client - auth.irma.autoupdateschemas true set if you want automatically update the IRMA schemas every 60 minutes. - auth.irma.schememanager pbdf IRMA schemeManager to use for attributes. Can be either 'pbdf' or 'irma-demo'. - **Crypto** - crypto.storage fs Storage to use, 'external' for an external backend (experimental), 'fs' for file system (for development purposes), 'vaultkv' for Vault KV store (recommended, will be replaced by external backend in future). - crypto.external.address Address of the external storage service. - crypto.external.timeout 100ms Time-out when invoking the external storage backend, in Golang time.Duration string format (e.g. 1s). - crypto.vault.address The Vault address. If set it overwrites the VAULT_ADDR env var. - crypto.vault.pathprefix kv The Vault path prefix. - crypto.vault.timeout 5s Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). - crypto.vault.token The Vault token. If set it overwrites the VAULT_TOKEN env var. - **Events** - events.nats.hostname 0.0.0.0 Hostname for the NATS server - events.nats.port 4222 Port where the NATS server listens on - events.nats.storagedir Directory where file-backed streams are stored in the NATS server - events.nats.timeout 30 Timeout for NATS server operations - **GoldenHammer** - goldenhammer.enabled true Whether to enable automatically fixing DID documents with the required endpoints. - goldenhammer.interval 10m0s The interval in which to check for DID documents to fix. - **HTTP** - http.default.address \:1323 Address and port the server will be listening to - http.default.log metadata What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). - http.default.tls Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', - http.default.auth.audience Expected audience for JWT tokens (default: hostname) - http.default.auth.authorizedkeyspath Path to an authorized_keys file for trusted JWT signers - http.default.auth.type Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode. - http.default.cors.origin [] When set, enables CORS from the specified origins on the default HTTP interface. - **JSONLD** - jsonld.contexts.localmapping [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. - jsonld.contexts.remoteallowlist [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json] In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. - **Network** - network.bootstrapnodes [] List of bootstrap nodes (':') which the node initially connect to. - network.connectiontimeout 5000 Timeout before an outbound connection attempt times out (in milliseconds). - network.enablediscovery true Whether to enable automatic connecting to other nodes. - network.enabletls true Whether to enable TLS for gRPC connections, which can be disabled for demo/development purposes. It is NOT meant for TLS offloading (see 'tls.offload'). Disabling TLS is not allowed in strict-mode. - network.grpcaddr \:5555 Local address for gRPC to listen on. If empty the gRPC server won't be started and other nodes will not be able to connect to this node (outbound connections can still be made). - network.maxbackoff 24h0m0s Maximum between outbound connections attempts to unresponsive nodes (in Golang duration format, e.g. '1h', '30m'). - network.nodedid Specifies the DID of the organization that operates this node, typically a vendor for EPD software. It is used to identify the node on the network. If the DID document does not exist of is deactivated, the node will not start. - network.protocols [] Specifies the list of network protocols to enable on the server. They are specified by version (1, 2). If not set, all protocols are enabled. - network.v2.diagnosticsinterval 5000 Interval (in milliseconds) that specifies how often the node should broadcast its diagnostic information to other nodes (specify 0 to disable). - network.v2.gossipinterval 5000 Interval (in milliseconds) that specifies how often the node should gossip its new hashes to other nodes. - **PKI** - pki.maxupdatefailhours 4 Maximum number of hours that a denylist update can fail - pki.softfail true Do not reject certificates if their revocation status cannot be established when softfail is true - **Storage** - storage.bbolt.backup.directory Target directory for BBolt database backups. - storage.bbolt.backup.interval 0s Interval, formatted as Golang duration (e.g. 10m, 1h) at which BBolt database backups will be performed. - storage.redis.address Redis database server address. This can be a simple 'host:port' or a Redis connection URL with scheme, auth and other options. - storage.redis.database Redis database name, which is used as prefix every key. Can be used to have multiple instances use the same Redis instance. - storage.redis.password Redis database password. If set, it overrides the username in the connection URL. - storage.redis.username Redis database username. If set, it overrides the username in the connection URL. - storage.redis.sentinel.master Name of the Redis Sentinel master. Setting this property enables Redis Sentinel. - storage.redis.sentinel.nodes [] Addresses of the Redis Sentinels to connect to initially. Setting this property enables Redis Sentinel. - storage.redis.sentinel.password Password for authenticating to Redis Sentinels. - storage.redis.sentinel.username Username for authenticating to Redis Sentinels. - storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). - storage.sql.connection Connection string for the SQL database. If not set, it defaults to a SQLite database stored inside the configured data directory. If using a SQLite database, make sure to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL'). - **VCR** - vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). - vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. - vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. - ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================== + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================================================================================ + Key Default Description + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================================================================================ + configfile nuts.yaml Nuts config file + cpuprofile When set, a CPU profile is written to the given path. Ignored when strictmode is set. + datadir ./data Directory where the node stores its files. + internalratelimiter true When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. + loggerformat text Log format (text, json) + strictmode true When set, insecure settings are forbidden. + verbosity info Log level (trace, debug, info, warn, error) + tls.certfile PEM file containing the certificate for the server (also used as client certificate). + tls.certheader Name of the HTTP header that will contain the client certificate when TLS is offloaded. + tls.certkeyfile PEM file containing the private key of the server certificate. + tls.offload Whether to enable TLS offloading for incoming connections. Enable by setting it to 'incoming'. If enabled 'tls.certheader' must be configured as well. + tls.truststorefile truststore.pem PEM file containing the trusted CA certificates for authenticating remote servers. + **Auth** + auth.accesstokenlifespan 60 defines how long (in seconds) an access token is valid. Uses default in strict mode. + auth.clockskew 5000 allowed JWT Clock skew in milliseconds + auth.contractvalidators [irma,uzi,dummy,employeeid] sets the different contract validators to use + auth.publicurl public URL which can be reached by a users IRMA client, this should include the scheme and domain: https://example.com. Additional paths should only be added if some sort of url-rewriting is done in a reverse-proxy. + auth.http.timeout 30 HTTP timeout (in seconds) used by the Auth API HTTP client + auth.irma.autoupdateschemas true set if you want automatically update the IRMA schemas every 60 minutes. + auth.irma.schememanager pbdf IRMA schemeManager to use for attributes. Can be either 'pbdf' or 'irma-demo'. + **Crypto** + crypto.storage fs Storage to use, 'external' for an external backend (experimental), 'fs' for file system (for development purposes), 'vaultkv' for Vault KV store (recommended, will be replaced by external backend in future). + crypto.external.address Address of the external storage service. + crypto.external.timeout 100ms Time-out when invoking the external storage backend, in Golang time.Duration string format (e.g. 1s). + crypto.vault.address The Vault address. If set it overwrites the VAULT_ADDR env var. + crypto.vault.pathprefix kv The Vault path prefix. + crypto.vault.timeout 5s Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). + crypto.vault.token The Vault token. If set it overwrites the VAULT_TOKEN env var. + **Events** + events.nats.hostname 0.0.0.0 Hostname for the NATS server + events.nats.port 4222 Port where the NATS server listens on + events.nats.storagedir Directory where file-backed streams are stored in the NATS server + events.nats.timeout 30 Timeout for NATS server operations + **GoldenHammer** + goldenhammer.enabled true Whether to enable automatically fixing DID documents with the required endpoints. + goldenhammer.interval 10m0s The interval in which to check for DID documents to fix. + **HTTP** + http.default.address \:1323 Address and port the server will be listening to + http.default.log metadata What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). + http.default.tls Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', + http.default.auth.audience Expected audience for JWT tokens (default: hostname) + http.default.auth.authorizedkeyspath Path to an authorized_keys file for trusted JWT signers + http.default.auth.type Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode. + http.default.cors.origin [] When set, enables CORS from the specified origins on the default HTTP interface. + **JSONLD** + jsonld.contexts.localmapping [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson] This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. + jsonld.contexts.remoteallowlist [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json] In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. + **Network** + network.bootstrapnodes [] List of bootstrap nodes (':') which the node initially connect to. + network.connectiontimeout 5000 Timeout before an outbound connection attempt times out (in milliseconds). + network.enablediscovery true Whether to enable automatic connecting to other nodes. + network.enabletls true Whether to enable TLS for gRPC connections, which can be disabled for demo/development purposes. It is NOT meant for TLS offloading (see 'tls.offload'). Disabling TLS is not allowed in strict-mode. + network.grpcaddr \:5555 Local address for gRPC to listen on. If empty the gRPC server won't be started and other nodes will not be able to connect to this node (outbound connections can still be made). + network.maxbackoff 24h0m0s Maximum between outbound connections attempts to unresponsive nodes (in Golang duration format, e.g. '1h', '30m'). + network.nodedid Specifies the DID of the organization that operates this node, typically a vendor for EPD software. It is used to identify the node on the network. If the DID document does not exist of is deactivated, the node will not start. + network.protocols [] Specifies the list of network protocols to enable on the server. They are specified by version (1, 2). If not set, all protocols are enabled. + network.v2.diagnosticsinterval 5000 Interval (in milliseconds) that specifies how often the node should broadcast its diagnostic information to other nodes (specify 0 to disable). + network.v2.gossipinterval 5000 Interval (in milliseconds) that specifies how often the node should gossip its new hashes to other nodes. + **PKI** + pki.maxupdatefailhours 4 Maximum number of hours that a denylist update can fail + pki.softfail true Do not reject certificates if their revocation status cannot be established when softfail is true + **Storage** + storage.bbolt.backup.directory Target directory for BBolt database backups. + storage.bbolt.backup.interval 0s Interval, formatted as Golang duration (e.g. 10m, 1h) at which BBolt database backups will be performed. + storage.redis.address Redis database server address. This can be a simple 'host:port' or a Redis connection URL with scheme, auth and other options. + storage.redis.database Redis database name, which is used as prefix every key. Can be used to have multiple instances use the same Redis instance. + storage.redis.password Redis database password. If set, it overrides the username in the connection URL. + storage.redis.username Redis database username. If set, it overrides the username in the connection URL. + storage.redis.sentinel.master Name of the Redis Sentinel master. Setting this property enables Redis Sentinel. + storage.redis.sentinel.nodes [] Addresses of the Redis Sentinels to connect to initially. Setting this property enables Redis Sentinel. + storage.redis.sentinel.password Password for authenticating to Redis Sentinels. + storage.redis.sentinel.username Username for authenticating to Redis Sentinels. + storage.redis.tls.truststorefile PEM file containing the trusted CA certificate(s) for authenticating remote Redis servers. Can only be used when connecting over TLS (use 'rediss://' as scheme in address). + storage.sql.connection Connection string for the SQL database. If not set it, defaults to a SQLite database stored inside the configured data directory. Note: using SQLite is not recommended in production environments. If using SQLite anyways, remember to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL'). + **VCR** + vcr.openid4vci.definitionsdir Directory with the additional credential definitions the node could issue (experimental, may change without notice). + vcr.openid4vci.enabled true Enable issuing and receiving credentials over OpenID4VCI. + vcr.openid4vci.timeout 30s Time-out for OpenID4VCI HTTP client operations. + ==================================== =============================================================================================================================================================================================================================================================================================================== ================================================================================================================================================================================================================================================================================================================================ diff --git a/storage/cmd/cmd.go b/storage/cmd/cmd.go index 5747b9ab72..535046274b 100644 --- a/storage/cmd/cmd.go +++ b/storage/cmd/cmd.go @@ -39,7 +39,8 @@ func FlagSet() *pflag.FlagSet { flagSet.String("storage.redis.sentinel.username", defs.Redis.Sentinel.Username, "Username for authenticating to Redis Sentinels.") flagSet.String("storage.redis.sentinel.password", defs.Redis.Sentinel.Password, "Password for authenticating to Redis Sentinels.") flagSet.String("storage.sql.connection", defs.SQL.ConnectionString, "Connection string for the SQL database. "+ - "If not set, it defaults to a SQLite database stored inside the configured data directory. "+ - "If using a SQLite database, make sure to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL').") + "If not set it, defaults to a SQLite database stored inside the configured data directory. "+ + "Note: using SQLite is not recommended in production environments. "+ + "If using SQLite anyways, remember to enable foreign keys ('_foreign_keys=on') and the write-ahead-log ('_journal_mode=WAL').") return flagSet } From f6f4caa1999f4dabcda96e052156da694465c396 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Fri, 24 Nov 2023 13:15:00 +0100 Subject: [PATCH 12/23] fixed tests --- discovery/module_test.go | 130 +++++++++++++----------------- discovery/store.go | 2 - discovery/store_test.go | 17 ++++ discovery/test.go | 1 + vcr/pe/presentation_definition.go | 9 ++- 5 files changed, 84 insertions(+), 75 deletions(-) diff --git a/discovery/module_test.go b/discovery/module_test.go index 61d0eff0f7..c535d4ee3b 100644 --- a/discovery/module_test.go +++ b/discovery/module_test.go @@ -28,8 +28,6 @@ import ( "time" ) -const serviceID = "urn:nuts.nl:usecase:eOverdrachtDev2023" - func TestModule_Name(t *testing.T) { assert.Equal(t, "Discovery", (&Module{}).Name()) } @@ -41,83 +39,71 @@ func TestModule_Shutdown(t *testing.T) { func Test_Module_Add(t *testing.T) { storageEngine := storage.NewTestStorageEngine(t) require.NoError(t, storageEngine.Start()) - t.Run("ok", func(t *testing.T) { - m := setupModule(t, storageEngine) - - err := m.Add(testServiceID, vpAlice) - assert.NoError(t, err) - - _, timestamp, err := m.Get(testServiceID, 0) - require.NoError(t, err) - assert.Equal(t, Timestamp(1), *timestamp) - }) - t.Run("replace presentation of same credential subject", func(t *testing.T) { - m := setupModule(t, storageEngine) + t.Run("registration", func(t *testing.T) { + t.Run("ok", func(t *testing.T) { + m := setupModule(t, storageEngine) - vpAlice2 := createPresentation(aliceDID, vcAlice) - assert.NoError(t, m.Add(testServiceID, vpAlice)) - assert.NoError(t, m.Add(testServiceID, vpBob)) - assert.NoError(t, m.Add(testServiceID, vpAlice2)) + err := m.Add(testServiceID, vpAlice) + require.NoError(t, err) - presentations, timestamp, err := m.Get(testServiceID, 0) - require.NoError(t, err) - assert.Equal(t, []vc.VerifiablePresentation{vpBob, vpAlice2}, presentations) - assert.Equal(t, Timestamp(3), *timestamp) - }) - t.Run("already exists", func(t *testing.T) { - m := setupModule(t, storageEngine) + _, timestamp, err := m.Get(testServiceID, 0) + require.NoError(t, err) + assert.Equal(t, Timestamp(1), *timestamp) + }) + t.Run("already exists", func(t *testing.T) { + m := setupModule(t, storageEngine) - err := m.Add(testServiceID, vpAlice) - assert.NoError(t, err) - err = m.Add(testServiceID, vpAlice) - assert.EqualError(t, err, "presentation already exists") - }) - t.Run("valid for too long", func(t *testing.T) { - m := setupModule(t, storageEngine) - def := m.services[testServiceID] - def.PresentationMaxValidity = 1 - m.services[testServiceID] = def + err := m.Add(testServiceID, vpAlice) + assert.NoError(t, err) + err = m.Add(testServiceID, vpAlice) + assert.EqualError(t, err, "presentation already exists") + }) + t.Run("valid for too long", func(t *testing.T) { + m := setupModule(t, storageEngine) + def := m.services[testServiceID] + def.PresentationMaxValidity = 1 + m.services[testServiceID] = def - err := m.Add(testServiceID, vpAlice) - assert.EqualError(t, err, "presentation is valid for too long (max 1s)") - }) - t.Run("valid longer than its credentials", func(t *testing.T) { - m := setupModule(t, storageEngine) + err := m.Add(testServiceID, vpAlice) + assert.EqualError(t, err, "presentation is valid for too long (max 1s)") + }) + t.Run("valid longer than its credentials", func(t *testing.T) { + m := setupModule(t, storageEngine) - vcAlice := createCredential(authorityDID, aliceDID, nil, func(claims map[string]interface{}) { - claims["exp"] = time.Now().Add(time.Hour) + vcAlice := createCredential(authorityDID, aliceDID, nil, func(claims map[string]interface{}) { + claims["exp"] = time.Now().Add(time.Hour) + }) + vpAlice := createPresentation(aliceDID, vcAlice) + err := m.Add(testServiceID, vpAlice) + assert.EqualError(t, err, "presentation is valid longer than the credential(s) it contains") }) - vpAlice := createPresentation(aliceDID, vcAlice) - err := m.Add(testServiceID, vpAlice) - assert.EqualError(t, err, "presentation is valid longer than the credential(s) it contains") - }) - t.Run("no expiration", func(t *testing.T) { - m := setupModule(t, storageEngine) - err := m.Add(testServiceID, createPresentationCustom(aliceDID, func(claims map[string]interface{}, _ *vc.VerifiablePresentation) { - delete(claims, "exp") - })) - assert.EqualError(t, err, "presentation does not have an expiration") - }) - t.Run("presentation does not contain an ID", func(t *testing.T) { - m := setupModule(t, storageEngine) + t.Run("no expiration", func(t *testing.T) { + m := setupModule(t, storageEngine) + err := m.Add(testServiceID, createPresentationCustom(aliceDID, func(claims map[string]interface{}, _ *vc.VerifiablePresentation) { + delete(claims, "exp") + })) + assert.EqualError(t, err, "presentation does not have an expiration") + }) + t.Run("presentation does not contain an ID", func(t *testing.T) { + m := setupModule(t, storageEngine) - vpWithoutID := createPresentationCustom(aliceDID, func(claims map[string]interface{}, _ *vc.VerifiablePresentation) { - delete(claims, "jti") - }, vcAlice) - err := m.Add(testServiceID, vpWithoutID) - assert.EqualError(t, err, "presentation does not have an ID") - }) - t.Run("not a JWT", func(t *testing.T) { - m := setupModule(t, storageEngine) - err := m.Add(testServiceID, vc.VerifiablePresentation{}) - assert.EqualError(t, err, "only JWT presentations are supported") - }) - t.Run("service unknown", func(t *testing.T) { - m := setupModule(t, storageEngine) - err := m.Add("unknown", vpAlice) - assert.ErrorIs(t, err, ErrServiceNotFound) + vpWithoutID := createPresentationCustom(aliceDID, func(claims map[string]interface{}, _ *vc.VerifiablePresentation) { + delete(claims, "jti") + }, vcAlice) + err := m.Add(testServiceID, vpWithoutID) + assert.EqualError(t, err, "presentation does not have an ID") + }) + t.Run("not a JWT", func(t *testing.T) { + m := setupModule(t, storageEngine) + err := m.Add(testServiceID, vc.VerifiablePresentation{}) + assert.EqualError(t, err, "only JWT presentations are supported") + }) + t.Run("service unknown", func(t *testing.T) { + m := setupModule(t, storageEngine) + err := m.Add("unknown", vpAlice) + assert.ErrorIs(t, err, ErrServiceNotFound) + }) }) - t.Run("retraction", func(t *testing.T) { vpAliceRetract := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { vp.Type = append(vp.Type, retractionPresentationType) @@ -184,7 +170,7 @@ func Test_Module_Add(t *testing.T) { func Test_Module_Get(t *testing.T) { storageEngine := storage.NewTestStorageEngine(t) require.NoError(t, storageEngine.Start()) - t.Run("1 entry, empty timestamp", func(t *testing.T) { + t.Run("ok", func(t *testing.T) { m := setupModule(t, storageEngine) require.NoError(t, m.Add(testServiceID, vpAlice)) presentations, timestamp, err := m.Get(testServiceID, 0) diff --git a/discovery/store.go b/discovery/store.go index 06833331db..c38df5ad91 100644 --- a/discovery/store.go +++ b/discovery/store.go @@ -303,8 +303,6 @@ func (s *sqlStore) search(serviceID string, query map[string]string) ([]vc.Verif func (s *sqlStore) updateTimestamp(tx *gorm.DB, serviceID string, newTimestamp *Timestamp) (Timestamp, error) { var result serviceRecord // Lock (SELECT FOR UPDATE) discovery_service row to prevent concurrent updates to the same list, which could mess up the lamport timestamp. - // But, it is not supported by SQLite. SQLite defaults to table-level write-locks upon the first write action in the transaction. - // if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). Where(serviceRecord{ID: serviceID}). Find(&result). diff --git a/discovery/store_test.go b/discovery/store_test.go index d44c3d2c16..11b1af52c1 100644 --- a/discovery/store_test.go +++ b/discovery/store_test.go @@ -107,6 +107,23 @@ func Test_sqlStore_add(t *testing.T) { assert.NoError(t, m.db.Find(&actual).Error) assert.Empty(t, actual) }) + t.Run("replaces previous presentation of same subject", func(t *testing.T) { + m := setupStore(t, storageEngine.GetSQLDatabase()) + + secondVP := createPresentation(aliceDID, vcAlice) + require.NoError(t, m.add(testServiceID, vpAlice, nil)) + require.NoError(t, m.add(testServiceID, secondVP, nil)) + + // First VP should not exist + exists, err := m.exists(testServiceID, aliceDID.String(), vpAlice.ID.String()) + require.NoError(t, err) + assert.False(t, exists) + + // Only second VP should exist + exists, err = m.exists(testServiceID, aliceDID.String(), secondVP.ID.String()) + require.NoError(t, err) + assert.True(t, exists) + }) } func Test_sqlStore_get(t *testing.T) { diff --git a/discovery/test.go b/discovery/test.go index 987713ff5a..13372d396c 100644 --- a/discovery/test.go +++ b/discovery/test.go @@ -55,6 +55,7 @@ func testDefinitions() map[string]ServiceDefinition { PresentationDefinition: pe.PresentationDefinition{ InputDescriptors: []*pe.InputDescriptor{ { + Id: "1", Constraints: &pe.Constraints{ Fields: []pe.Field{ { diff --git a/vcr/pe/presentation_definition.go b/vcr/pe/presentation_definition.go index 027f957fbf..0640337839 100644 --- a/vcr/pe/presentation_definition.go +++ b/vcr/pe/presentation_definition.go @@ -293,7 +293,14 @@ func matchConstraint(constraint *Constraints, credential vc.VerifiableCredential // All fields need to match unless optional is set to true and no values are found for all the paths. func matchField(field Field, credential vc.VerifiableCredential) (bool, error) { // jsonpath works on interfaces, so convert the VC to an interface - asJSON, _ := json.Marshal(credential) + var asJSON []byte + if credential.Format() == vc.JWTCredentialProofFormat { + // json.Marshal on a JWT VC leads to the JWT string, not a map with the VC properties + type Alias vc.VerifiableCredential + asJSON, _ = json.Marshal(Alias(credential)) + } else { + asJSON, _ = json.Marshal(credential) + } var asInterface interface{} _ = json.Unmarshal(asJSON, &asInterface) From ffd37cb3e09b1d3a7184f614883ae22701be0b01 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Fri, 24 Nov 2023 13:17:52 +0100 Subject: [PATCH 13/23] Storage: do not write sqlite.db to source dir --- storage/engine_test.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/storage/engine_test.go b/storage/engine_test.go index 3594169207..befd3bae6f 100644 --- a/storage/engine_test.go +++ b/storage/engine_test.go @@ -52,7 +52,7 @@ func Test_engine_lifecycle(t *testing.T) { func Test_engine_GetProvider(t *testing.T) { sut := New() - _ = sut.Configure(*core.NewServerConfig()) + _ = sut.Configure(core.ServerConfig{Datadir: io.TestDirectory(t)}) t.Run("moduleName is empty", func(t *testing.T) { store, err := sut.GetProvider("").GetKVStore("store", VolatileStorageClass) assert.Nil(t, store) @@ -62,7 +62,7 @@ func Test_engine_GetProvider(t *testing.T) { func Test_engine_GetKVStore(t *testing.T) { sut := New() - _ = sut.Configure(*core.NewServerConfig()) + _ = sut.Configure(core.ServerConfig{Datadir: io.TestDirectory(t)}) t.Run("store is empty", func(t *testing.T) { store, err := sut.GetProvider("engine").GetKVStore("", VolatileStorageClass) assert.Nil(t, store) @@ -132,7 +132,7 @@ func Test_engine_sqlDatabase(t *testing.T) { }) t.Run("runs migrations", func(t *testing.T) { e := New().(*engine) - require.NoError(t, e.Configure(*core.NewServerConfig())) + require.NoError(t, e.Configure(core.ServerConfig{Datadir: io.TestDirectory(t)})) require.NoError(t, e.Start()) t.Cleanup(func() { _ = e.Shutdown() @@ -146,5 +146,4 @@ func Test_engine_sqlDatabase(t *testing.T) { assert.NoError(t, row.Scan(&count)) assert.Equal(t, 1, count) }) - } From b6957d3803b355df80e1d5068c89b98f24976887 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Fri, 24 Nov 2023 16:03:19 +0100 Subject: [PATCH 14/23] added VP verify, more tests, organized tests --- discovery/module.go | 55 +++++++++----- discovery/module_test.go | 154 +++++++++++++++++++++++++-------------- discovery/store.go | 21 ++++-- discovery/test.go | 7 +- 4 files changed, 156 insertions(+), 81 deletions(-) diff --git a/discovery/module.go b/discovery/module.go index 64c8c5447e..108117c65d 100644 --- a/discovery/module.go +++ b/discovery/module.go @@ -26,6 +26,8 @@ import ( "github.com/nuts-foundation/go-did/vc" "github.com/nuts-foundation/nuts-node/core" "github.com/nuts-foundation/nuts-node/storage" + "github.com/nuts-foundation/nuts-node/vcr" + "github.com/nuts-foundation/nuts-node/vcr/credential" "os" "path" "strings" @@ -44,9 +46,10 @@ var _ Server = &Module{} var retractionPresentationType = ssi.MustParseURI("RetractedVerifiablePresentation") // New creates a new Module. -func New(storageInstance storage.Engine) *Module { +func New(storageInstance storage.Engine, vcrInstance vcr.VCR) *Module { return &Module{ storageInstance: storageInstance, + vcrInstance: vcrInstance, } } @@ -57,6 +60,7 @@ type Module struct { store *sqlStore serverDefinitions map[string]ServiceDefinition services map[string]ServiceDefinition + vcrInstance vcr.VCR } func (m *Module) Configure(_ core.ServerConfig) error { @@ -105,8 +109,9 @@ func (m *Module) Config() interface{} { } func (m *Module) Add(serviceID string, presentation vc.VerifiablePresentation) error { - definition, exists := m.services[serviceID] - if !exists { + // First, simple sanity checks + definition, serviceExists := m.services[serviceID] + if !serviceExists { return ErrServiceNotFound } if _, isMaintainer := m.serverDefinitions[serviceID]; !isMaintainer { @@ -115,7 +120,6 @@ func (m *Module) Add(serviceID string, presentation vc.VerifiablePresentation) e if presentation.Format() != vc.JWTPresentationProofFormat { return errors.New("only JWT presentations are supported") } - // TODO: validate signature if presentation.ID == nil { return errors.New("presentation does not have an ID") } @@ -127,15 +131,36 @@ func (m *Module) Add(serviceID string, presentation vc.VerifiablePresentation) e if int(expiration.Sub(time.Now()).Seconds()) > definition.PresentationMaxValidity { return fmt.Errorf("presentation is valid for too long (max %s)", time.Duration(definition.PresentationMaxValidity)*time.Second) } - + // Check if the presentation already exists + credentialSubjectID, err := credential.PresentationSigner(presentation) + if err != nil { + return err + } + exists, err := m.store.exists(serviceID, credentialSubjectID.String(), presentation.ID.String()) + if err != nil { + return err + } + if exists { + return ErrPresentationAlreadyExists + } + // Depending on the presentation type, we need to validate different properties before storing it. if presentation.IsType(retractionPresentationType) { - return m.addRetraction(definition.ID, presentation) + err = m.validateRetraction(definition.ID, presentation) } else { - return m.addPresentation(definition, presentation) + err = m.validateRegistration(definition, presentation) + } + if err != nil { + return err } + // Check signature of presentation and contained credential(s) + _, err = m.vcrInstance.Verifier().VerifyVP(presentation, true, true, nil) + if err != nil { + return fmt.Errorf("presentation verification failed: %w", err) + } + return m.store.add(definition.ID, presentation, nil) } -func (m *Module) addPresentation(definition ServiceDefinition, presentation vc.VerifiablePresentation) error { +func (m *Module) validateRegistration(definition ServiceDefinition, presentation vc.VerifiablePresentation) error { // VP can't be valid longer than the credentialRecord it contains expiration := presentation.JWT().Expiration() for _, cred := range presentation.VerifiableCredential { @@ -149,10 +174,10 @@ func (m *Module) addPresentation(definition ServiceDefinition, presentation vc.V if err != nil || len(creds) != len(presentation.VerifiableCredential) { return fmt.Errorf("presentation does not fulfill Presentation ServiceDefinition: %w", err) } - return m.store.add(definition.ID, presentation, nil) + return nil } -func (m *Module) addRetraction(serviceID string, presentation vc.VerifiablePresentation) error { +func (m *Module) validateRetraction(serviceID string, presentation vc.VerifiablePresentation) error { // Presentation might be a retraction (deletion of an earlier credentialRecord) must contain no credentials, and refer to the VP being retracted by ID. // If those conditions aren't met, we don't need to register the retraction. if len(presentation.VerifiableCredential) > 0 { @@ -169,11 +194,7 @@ func (m *Module) addRetraction(serviceID string, presentation vc.VerifiablePrese return errors.New("retraction presentation 'retract_jti' claim is not a string") } } - signer := presentation.JWT().Issuer() - signerDID, err := did.ParseDID(signer) - if err != nil { - return fmt.Errorf("retraction presentation issuer is not a valid DID: %w", err) - } + signerDID, _ := credential.PresentationSigner(presentation) // checked before retractJTI, err := did.ParseDIDURL(retractJTIString) if err != nil { return fmt.Errorf("retraction presentation 'retract_jti' claim is not a valid DID URL: %w", err) @@ -181,14 +202,14 @@ func (m *Module) addRetraction(serviceID string, presentation vc.VerifiablePrese if !signerDID.Equals(retractJTI.DID) { return errors.New("retraction presentation 'retract_jti' claim does not match JWT issuer") } - exists, err := m.store.exists(serviceID, signer, retractJTIString) + exists, err := m.store.exists(serviceID, signerDID.String(), retractJTIString) if err != nil { return err } if !exists { return errors.New("retraction presentation refers to a non-existing presentation") } - return m.store.add(serviceID, presentation, nil) + return nil } func (m *Module) Get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresentation, *Timestamp, error) { diff --git a/discovery/module_test.go b/discovery/module_test.go index c535d4ee3b..5b0b0c68b7 100644 --- a/discovery/module_test.go +++ b/discovery/module_test.go @@ -19,11 +19,15 @@ package discovery import ( + "errors" "github.com/nuts-foundation/go-did/vc" "github.com/nuts-foundation/nuts-node/core" "github.com/nuts-foundation/nuts-node/storage" + "github.com/nuts-foundation/nuts-node/vcr" + "github.com/nuts-foundation/nuts-node/vcr/verifier" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" "testing" "time" ) @@ -39,9 +43,73 @@ func TestModule_Shutdown(t *testing.T) { func Test_Module_Add(t *testing.T) { storageEngine := storage.NewTestStorageEngine(t) require.NoError(t, storageEngine.Start()) + + t.Run("not a maintainer", func(t *testing.T) { + m, _ := setupModule(t, storageEngine) + + err := m.Add("other", vpAlice) + require.EqualError(t, err, "node is not a discovery server for this service") + }) + t.Run("VP verification fails (e.g. invalid signature)", func(t *testing.T) { + m, presentationVerifier := setupModule(t, storageEngine) + presentationVerifier.EXPECT().VerifyVP(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New("failed")) + + err := m.Add(testServiceID, vpAlice) + require.EqualError(t, err, "presentation verification failed: failed") + + _, timestamp, err := m.Get(testServiceID, 0) + require.NoError(t, err) + assert.Equal(t, Timestamp(0), *timestamp) + }) + t.Run("already exists", func(t *testing.T) { + m, presentationVerifier := setupModule(t, storageEngine) + presentationVerifier.EXPECT().VerifyVP(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()) + + err := m.Add(testServiceID, vpAlice) + assert.NoError(t, err) + err = m.Add(testServiceID, vpAlice) + assert.EqualError(t, err, "presentation already exists") + }) + t.Run("valid for too long", func(t *testing.T) { + m, _ := setupModule(t, storageEngine) + def := m.services[testServiceID] + def.PresentationMaxValidity = 1 + m.services[testServiceID] = def + + err := m.Add(testServiceID, vpAlice) + assert.EqualError(t, err, "presentation is valid for too long (max 1s)") + }) + t.Run("no expiration", func(t *testing.T) { + m, _ := setupModule(t, storageEngine) + err := m.Add(testServiceID, createPresentationCustom(aliceDID, func(claims map[string]interface{}, _ *vc.VerifiablePresentation) { + delete(claims, "exp") + })) + assert.EqualError(t, err, "presentation does not have an expiration") + }) + t.Run("presentation does not contain an ID", func(t *testing.T) { + m, _ := setupModule(t, storageEngine) + + vpWithoutID := createPresentationCustom(aliceDID, func(claims map[string]interface{}, _ *vc.VerifiablePresentation) { + delete(claims, "jti") + }, vcAlice) + err := m.Add(testServiceID, vpWithoutID) + assert.EqualError(t, err, "presentation does not have an ID") + }) + t.Run("not a JWT", func(t *testing.T) { + m, _ := setupModule(t, storageEngine) + err := m.Add(testServiceID, vc.VerifiablePresentation{}) + assert.EqualError(t, err, "only JWT presentations are supported") + }) + t.Run("service unknown", func(t *testing.T) { + m, _ := setupModule(t, storageEngine) + err := m.Add("unknown", vpAlice) + assert.ErrorIs(t, err, ErrServiceNotFound) + }) + t.Run("registration", func(t *testing.T) { t.Run("ok", func(t *testing.T) { - m := setupModule(t, storageEngine) + m, presentationVerifier := setupModule(t, storageEngine) + presentationVerifier.EXPECT().VerifyVP(gomock.Any(), true, true, nil) err := m.Add(testServiceID, vpAlice) require.NoError(t, err) @@ -50,25 +118,8 @@ func Test_Module_Add(t *testing.T) { require.NoError(t, err) assert.Equal(t, Timestamp(1), *timestamp) }) - t.Run("already exists", func(t *testing.T) { - m := setupModule(t, storageEngine) - - err := m.Add(testServiceID, vpAlice) - assert.NoError(t, err) - err = m.Add(testServiceID, vpAlice) - assert.EqualError(t, err, "presentation already exists") - }) - t.Run("valid for too long", func(t *testing.T) { - m := setupModule(t, storageEngine) - def := m.services[testServiceID] - def.PresentationMaxValidity = 1 - m.services[testServiceID] = def - - err := m.Add(testServiceID, vpAlice) - assert.EqualError(t, err, "presentation is valid for too long (max 1s)") - }) t.Run("valid longer than its credentials", func(t *testing.T) { - m := setupModule(t, storageEngine) + m, _ := setupModule(t, storageEngine) vcAlice := createCredential(authorityDID, aliceDID, nil, func(claims map[string]interface{}) { claims["exp"] = time.Now().Add(time.Hour) @@ -77,31 +128,16 @@ func Test_Module_Add(t *testing.T) { err := m.Add(testServiceID, vpAlice) assert.EqualError(t, err, "presentation is valid longer than the credential(s) it contains") }) - t.Run("no expiration", func(t *testing.T) { - m := setupModule(t, storageEngine) - err := m.Add(testServiceID, createPresentationCustom(aliceDID, func(claims map[string]interface{}, _ *vc.VerifiablePresentation) { - delete(claims, "exp") - })) - assert.EqualError(t, err, "presentation does not have an expiration") - }) - t.Run("presentation does not contain an ID", func(t *testing.T) { - m := setupModule(t, storageEngine) + t.Run("not conform to Presentation Definition", func(t *testing.T) { + m, _ := setupModule(t, storageEngine) - vpWithoutID := createPresentationCustom(aliceDID, func(claims map[string]interface{}, _ *vc.VerifiablePresentation) { - delete(claims, "jti") - }, vcAlice) - err := m.Add(testServiceID, vpWithoutID) - assert.EqualError(t, err, "presentation does not have an ID") - }) - t.Run("not a JWT", func(t *testing.T) { - m := setupModule(t, storageEngine) - err := m.Add(testServiceID, vc.VerifiablePresentation{}) - assert.EqualError(t, err, "only JWT presentations are supported") - }) - t.Run("service unknown", func(t *testing.T) { - m := setupModule(t, storageEngine) - err := m.Add("unknown", vpAlice) - assert.ErrorIs(t, err, ErrServiceNotFound) + // Presentation Definition only allows did:example DIDs + otherVP := createPresentation(unsupportedDID, createCredential(unsupportedDID, unsupportedDID, nil, nil)) + err := m.Add(testServiceID, otherVP) + require.ErrorContains(t, err, "presentation does not fulfill Presentation ServiceDefinition") + + _, timestamp, _ := m.Get(testServiceID, 0) + assert.Equal(t, Timestamp(0), *timestamp) }) }) t.Run("retraction", func(t *testing.T) { @@ -110,19 +146,21 @@ func Test_Module_Add(t *testing.T) { claims["retract_jti"] = vpAlice.ID.String() }) t.Run("ok", func(t *testing.T) { - m := setupModule(t, storageEngine) + m, presentationVerifier := setupModule(t, storageEngine) + presentationVerifier.EXPECT().VerifyVP(gomock.Any(), true, true, nil).Times(2) + err := m.Add(testServiceID, vpAlice) require.NoError(t, err) err = m.Add(testServiceID, vpAliceRetract) assert.NoError(t, err) }) t.Run("non-existent presentation", func(t *testing.T) { - m := setupModule(t, storageEngine) + m, _ := setupModule(t, storageEngine) err := m.Add(testServiceID, vpAliceRetract) assert.EqualError(t, err, "retraction presentation refers to a non-existing presentation") }) t.Run("must not contain credentials", func(t *testing.T) { - m := setupModule(t, storageEngine) + m, _ := setupModule(t, storageEngine) vp := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { vp.Type = append(vp.Type, retractionPresentationType) }, vcAlice) @@ -130,7 +168,7 @@ func Test_Module_Add(t *testing.T) { assert.EqualError(t, err, "retraction presentation must not contain credentials") }) t.Run("missing 'retract_jti' claim", func(t *testing.T) { - m := setupModule(t, storageEngine) + m, _ := setupModule(t, storageEngine) vp := createPresentationCustom(aliceDID, func(_ map[string]interface{}, vp *vc.VerifiablePresentation) { vp.Type = append(vp.Type, retractionPresentationType) }) @@ -138,7 +176,7 @@ func Test_Module_Add(t *testing.T) { assert.EqualError(t, err, "retraction presentation does not contain 'retract_jti' claim") }) t.Run("'retract_jti' claim in not a string", func(t *testing.T) { - m := setupModule(t, storageEngine) + m, _ := setupModule(t, storageEngine) vp := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { vp.Type = append(vp.Type, retractionPresentationType) claims["retract_jti"] = 10 @@ -147,7 +185,7 @@ func Test_Module_Add(t *testing.T) { assert.EqualError(t, err, "retraction presentation 'retract_jti' claim is not a string") }) t.Run("'retract_jti' claim in not a valid DID", func(t *testing.T) { - m := setupModule(t, storageEngine) + m, _ := setupModule(t, storageEngine) vp := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { vp.Type = append(vp.Type, retractionPresentationType) claims["retract_jti"] = "not a DID" @@ -156,7 +194,7 @@ func Test_Module_Add(t *testing.T) { assert.EqualError(t, err, "retraction presentation 'retract_jti' claim is not a valid DID URL: invalid DID") }) t.Run("'retract_jti' claim does not reference a presentation of the signer", func(t *testing.T) { - m := setupModule(t, storageEngine) + m, _ := setupModule(t, storageEngine) vp := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { vp.Type = append(vp.Type, retractionPresentationType) claims["retract_jti"] = bobDID.String() @@ -171,30 +209,34 @@ func Test_Module_Get(t *testing.T) { storageEngine := storage.NewTestStorageEngine(t) require.NoError(t, storageEngine.Start()) t.Run("ok", func(t *testing.T) { - m := setupModule(t, storageEngine) - require.NoError(t, m.Add(testServiceID, vpAlice)) + m, _ := setupModule(t, storageEngine) + require.NoError(t, m.store.add(testServiceID, vpAlice, nil)) presentations, timestamp, err := m.Get(testServiceID, 0) assert.NoError(t, err) assert.Equal(t, []vc.VerifiablePresentation{vpAlice}, presentations) assert.Equal(t, Timestamp(1), *timestamp) }) t.Run("service unknown", func(t *testing.T) { - m := setupModule(t, storageEngine) + m, _ := setupModule(t, storageEngine) _, _, err := m.Get("unknown", 0) assert.ErrorIs(t, err, ErrServiceNotFound) }) } -func setupModule(t *testing.T, storageInstance storage.Engine) *Module { +func setupModule(t *testing.T, storageInstance storage.Engine) (*Module, *verifier.MockVerifier) { resetStore(t, storageInstance.GetSQLDatabase()) - m := New(storageInstance) + ctrl := gomock.NewController(t) + mockVerifier := verifier.NewMockVerifier(ctrl) + mockVCR := vcr.NewMockVCR(ctrl) + mockVCR.EXPECT().Verifier().Return(mockVerifier).AnyTimes() + m := New(storageInstance, mockVCR) require.NoError(t, m.Configure(core.ServerConfig{})) m.services = testDefinitions() m.serverDefinitions = map[string]ServiceDefinition{ testServiceID: m.services[testServiceID], } require.NoError(t, m.Start()) - return m + return m, mockVerifier } func TestModule_Configure(t *testing.T) { diff --git a/discovery/store.go b/discovery/store.go index c38df5ad91..c86da5f898 100644 --- a/discovery/store.go +++ b/discovery/store.go @@ -128,11 +128,6 @@ func (s *sqlStore) add(serviceID string, presentation vc.VerifiablePresentation, if err != nil { return err } - if exists, err := s.exists(serviceID, credentialSubjectID.String(), presentation.ID.String()); err != nil { - return err - } else if exists { - return ErrPresentationAlreadyExists - } if _, isSQLite := s.db.Config.Dialector.(*sqlite.Dialector); isSQLite { // SQLite does not support SELECT FOR UPDATE and allows only 1 active write transaction at any time, // and any other attempt to acquire a write transaction will directly return an error. @@ -208,7 +203,7 @@ func createPresentationRecord(serviceID string, timestamp Timestamp, presentatio CredentialSubjectID: credentialSubjectID.String(), CredentialType: credentialType, } - // Store credential's properties + // Create key-value properties of the credential subject, which is then stored in the property table for searching. keys, values := indexJSONObject(currCred.CredentialSubject[0].(map[string]interface{}), nil, nil, "credentialSubject") for i, key := range keys { if key == "credentialSubject.id" { @@ -226,6 +221,9 @@ func createPresentationRecord(serviceID string, timestamp Timestamp, presentatio return &newPresentation, nil } +// get returns all presentations, registered on the given service, starting after the given timestamp. +// It also returns the latest timestamp of the returned presentations. +// This timestamp can then be used next time to only retrieve presentations that were added after that timestamp. func (s *sqlStore) get(serviceID string, startAt Timestamp) ([]vc.VerifiablePresentation, *Timestamp, error) { var rows []presentationRecord err := s.db.Order("lamport_timestamp ASC").Find(&rows, "service_id = ? AND lamport_timestamp > ?", serviceID, int(startAt)).Error @@ -245,6 +243,10 @@ func (s *sqlStore) get(serviceID string, startAt Timestamp) ([]vc.VerifiablePres return presentations, ×tamp, nil } +// search searches for presentations, registered on the given service, matching the given query. +// The query is a map of JSON paths and expected string values, matched against the presentation's credentials. +// Wildcard matching is supported by prefixing or suffixing the value with an asterisk (*). +// It returns the presentations which contain credentials that match the given query. func (s *sqlStore) search(serviceID string, query map[string]string) ([]vc.VerifiablePresentation, error) { propertyColumns := map[string]string{ "id": "cred.credential_id", @@ -261,7 +263,8 @@ func (s *sqlStore) search(serviceID string, query map[string]string) ([]vc.Verif if value == "*" { continue } - // sort out wildcard mode + // sort out wildcard mode: prefix and postfix asterisks (*) are replaced with %, which then is used in a LIKE query. + // Otherwise, exact match (=) is used. var eq = "=" if strings.HasPrefix(value, "*") { value = "%" + value[1:] @@ -300,6 +303,9 @@ func (s *sqlStore) search(serviceID string, query map[string]string) ([]vc.Verif return results, nil } +// updateTimestamp updates the timestamp of the given service. +// Clients should pass the timestamp they received from the server (which simply sets it). +// Servers should pass nil (since they "own" the timestamp), which causes it to be incremented. func (s *sqlStore) updateTimestamp(tx *gorm.DB, serviceID string, newTimestamp *Timestamp) (Timestamp, error) { var result serviceRecord // Lock (SELECT FOR UPDATE) discovery_service row to prevent concurrent updates to the same list, which could mess up the lamport timestamp. @@ -322,6 +328,7 @@ func (s *sqlStore) updateTimestamp(tx *gorm.DB, serviceID string, newTimestamp * return Timestamp(result.LamportTimestamp), nil } +// exists checks whether a presentation of the given subject is registered on a service. func (s *sqlStore) exists(serviceID string, credentialSubjectID string, presentationID string) (bool, error) { var count int64 if err := s.db.Model(presentationRecord{}).Where(presentationRecord{ diff --git a/discovery/test.go b/discovery/test.go index 13372d396c..c8a2444202 100644 --- a/discovery/test.go +++ b/discovery/test.go @@ -44,10 +44,12 @@ var vpAlice vc.VerifiablePresentation var bobDID did.DID var vcBob vc.VerifiableCredential var vpBob vc.VerifiablePresentation +var unsupportedDID did.DID var testServiceID = "usecase_v1" func testDefinitions() map[string]ServiceDefinition { + issuerPattern := "did:example:*" return map[string]ServiceDefinition{ testServiceID: { ID: testServiceID, @@ -61,7 +63,8 @@ func testDefinitions() map[string]ServiceDefinition { { Path: []string{"$.issuer"}, Filter: &pe.Filter{ - Type: "string", + Type: "string", + Pattern: &issuerPattern, }, }, }, @@ -103,6 +106,8 @@ func init() { keyPairs[aliceDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) bobDID = did.MustParseDID("did:example:bob") keyPairs[bobDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + unsupportedDID = did.MustParseDID("did:web:example.com") + keyPairs[unsupportedDID.String()], _ = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) vcAlice = createCredential(authorityDID, aliceDID, map[string]interface{}{ "person": map[string]interface{}{ From 442cab13125fe042666a9c86887e6a3338d000e0 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Fri, 24 Nov 2023 16:17:18 +0100 Subject: [PATCH 15/23] feedback --- discovery/module.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/discovery/module.go b/discovery/module.go index 108117c65d..5e007778ba 100644 --- a/discovery/module.go +++ b/discovery/module.go @@ -164,12 +164,12 @@ func (m *Module) validateRegistration(definition ServiceDefinition, presentation // VP can't be valid longer than the credentialRecord it contains expiration := presentation.JWT().Expiration() for _, cred := range presentation.VerifiableCredential { - exp := cred.JWT().Expiration() - if !exp.IsZero() && expiration.After(exp) { + if cred.ExpirationDate != nil && expiration.After(*cred.ExpirationDate) { return fmt.Errorf("presentation is valid longer than the credential(s) it contains") } } // VP must fulfill the PEX Presentation ServiceDefinition + // TODO: Use Validate() once PR is merged creds, _, err := definition.PresentationDefinition.Match(presentation.VerifiableCredential) if err != nil || len(creds) != len(presentation.VerifiableCredential) { return fmt.Errorf("presentation does not fulfill Presentation ServiceDefinition: %w", err) From 4eaadb51c958c497abc712400361f57f7490ccf4 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Wed, 29 Nov 2023 06:47:34 +0100 Subject: [PATCH 16/23] feedback --- vcr/pe/presentation_definition.go | 4 ++-- vcr/pe/schema/v2/schema.go | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/vcr/pe/presentation_definition.go b/vcr/pe/presentation_definition.go index 0640337839..b4d9ea67ce 100644 --- a/vcr/pe/presentation_definition.go +++ b/vcr/pe/presentation_definition.go @@ -296,8 +296,8 @@ func matchField(field Field, credential vc.VerifiableCredential) (bool, error) { var asJSON []byte if credential.Format() == vc.JWTCredentialProofFormat { // json.Marshal on a JWT VC leads to the JWT string, not a map with the VC properties - type Alias vc.VerifiableCredential - asJSON, _ = json.Marshal(Alias(credential)) + type altType vc.VerifiableCredential + asJSON, _ = json.Marshal(altType(credential)) } else { asJSON, _ = json.Marshal(credential) } diff --git a/vcr/pe/schema/v2/schema.go b/vcr/pe/schema/v2/schema.go index dd4c1ce59a..c1db1c59a5 100644 --- a/vcr/pe/schema/v2/schema.go +++ b/vcr/pe/schema/v2/schema.go @@ -58,8 +58,6 @@ func Compiler() *jsonschema.Compiler { if err := loadSchemas(schemaFiles, compiler); err != nil { panic(err) } - PresentationDefinition = compiler.MustCompile(presentationDefinition) - PresentationSubmission = compiler.MustCompile(presentationSubmission) return compiler } From 91b0bef07c7ba37497aac9fbd7a288b93fc73915 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Wed, 29 Nov 2023 06:57:55 +0100 Subject: [PATCH 17/23] config flags --- README.rst | 3 +++ cmd/root.go | 5 +++++ discovery/cmd/cmd.go | 19 +++++++++++++++++++ discovery/cmd/cmd_test.go | 11 +++++++++++ docs/pages/deployment/cli-reference.rst | 4 +++- docs/pages/deployment/server_options.rst | 3 +++ 6 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 discovery/cmd/cmd.go create mode 100644 discovery/cmd/cmd_test.go diff --git a/README.rst b/README.rst index 4ef69925ff..350f0ccb54 100644 --- a/README.rst +++ b/README.rst @@ -207,6 +207,9 @@ The following options can be configured on the server: crypto.vault.pathprefix kv The Vault path prefix. crypto.vault.timeout 5s Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). crypto.vault.token The Vault token. If set it overwrites the VAULT_TOKEN env var. + **Discovery** + discovery.definitions.directory Directory to load Discovery Service Definitions from. If not set, the discovery service will be disabled. If the directory contains JSON files that can't be parsed as service definition, the node will fail to start. + discovery.server.definition_ids [] IDs of the Discovery Service Definitions for which to act as server. If an ID does not map to a loaded service definition, the node will fail to start. **Events** events.nats.hostname 0.0.0.0 Hostname for the NATS server events.nats.port 4222 Port where the NATS server listens on diff --git a/cmd/root.go b/cmd/root.go index 4c3c6b302e..a531de27d0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -23,6 +23,7 @@ import ( "context" "errors" "fmt" + "github.com/nuts-foundation/nuts-node/discovery" "github.com/nuts-foundation/nuts-node/vdr/resolver" "github.com/nuts-foundation/nuts-node/golden_hammer" @@ -46,6 +47,7 @@ import ( "github.com/nuts-foundation/nuts-node/didman" didmanAPI "github.com/nuts-foundation/nuts-node/didman/api/v1" didmanCmd "github.com/nuts-foundation/nuts-node/didman/cmd" + discoveryCmd "github.com/nuts-foundation/nuts-node/discovery/cmd" "github.com/nuts-foundation/nuts-node/events" eventsCmd "github.com/nuts-foundation/nuts-node/events/cmd" httpEngine "github.com/nuts-foundation/nuts-node/http" @@ -192,6 +194,7 @@ func CreateSystem(shutdownCallback context.CancelFunc) *core.System { vdrInstance := vdr.NewVDR(cryptoInstance, networkInstance, didStore, eventManager) credentialInstance := vcr.NewVCRInstance(cryptoInstance, vdrInstance, networkInstance, jsonld, eventManager, storageInstance, pkiInstance) didmanInstance := didman.NewDidmanInstance(vdrInstance, credentialInstance, jsonld) + discoveryInstance := discovery.New(storageInstance, credentialInstance) authInstance := auth.NewAuthInstance(auth.DefaultConfig(), vdrInstance, credentialInstance, cryptoInstance, didmanInstance, jsonld, pkiInstance) statusEngine := status.NewStatusEngine(system) metricsEngine := core.NewMetricsEngine() @@ -233,6 +236,7 @@ func CreateSystem(shutdownCallback context.CancelFunc) *core.System { system.RegisterEngine(credentialInstance) system.RegisterEngine(networkInstance) system.RegisterEngine(authInstance) + system.RegisterEngine(discoveryInstance) system.RegisterEngine(didmanInstance) system.RegisterEngine(goldenHammer) // HTTP engine MUST be registered last, because when started it dispatches HTTP calls to the registered routes. @@ -333,6 +337,7 @@ func serverConfigFlags() *pflag.FlagSet { set.AddFlagSet(eventsCmd.FlagSet()) set.AddFlagSet(pki.FlagSet()) set.AddFlagSet(goldenHammerCmd.FlagSet()) + set.AddFlagSet(discoveryCmd.FlagSet()) return set } diff --git a/discovery/cmd/cmd.go b/discovery/cmd/cmd.go new file mode 100644 index 0000000000..821e2b5ef5 --- /dev/null +++ b/discovery/cmd/cmd.go @@ -0,0 +1,19 @@ +package cmd + +import ( + "github.com/nuts-foundation/nuts-node/discovery" + "github.com/spf13/pflag" +) + +// FlagSet contains flags relevant for the module. +func FlagSet() *pflag.FlagSet { + defs := discovery.DefaultConfig() + flagSet := pflag.NewFlagSet("discovery", pflag.ContinueOnError) + flagSet.String("discovery.definitions.directory", defs.Definitions.Directory, + "Directory to load Discovery Service Definitions from. If not set, the discovery service will be disabled. "+ + "If the directory contains JSON files that can't be parsed as service definition, the node will fail to start.") + flagSet.StringSlice("discovery.server.definition_ids", defs.Server.DefinitionIDs, + "IDs of the Discovery Service Definitions for which to act as server. "+ + "If an ID does not map to a loaded service definition, the node will fail to start.") + return flagSet +} diff --git a/discovery/cmd/cmd_test.go b/discovery/cmd/cmd_test.go new file mode 100644 index 0000000000..2a5f341b09 --- /dev/null +++ b/discovery/cmd/cmd_test.go @@ -0,0 +1,11 @@ +package cmd + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestFlagSet(t *testing.T) { + flagset := FlagSet() + assert.NotNil(t, flagset) +} diff --git a/docs/pages/deployment/cli-reference.rst b/docs/pages/deployment/cli-reference.rst index 9f902661cd..fea8bf69eb 100755 --- a/docs/pages/deployment/cli-reference.rst +++ b/docs/pages/deployment/cli-reference.rst @@ -30,6 +30,8 @@ The following options apply to the server commands below: --crypto.vault.timeout duration Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). (default 5s) --crypto.vault.token string The Vault token. If set it overwrites the VAULT_TOKEN env var. --datadir string Directory where the node stores its files. (default "./data") + --discovery.definitions.directory string Directory to load Discovery Service Definitions from. If not set, the discovery service will be disabled. If the directory contains JSON files that can't be parsed as service definition, the node will fail to start. + --discovery.server.definition_ids strings IDs of the Discovery Service Definitions for which to act as server. If an ID does not map to a loaded service definition, the node will fail to start. --events.nats.hostname string Hostname for the NATS server (default "0.0.0.0") --events.nats.port int Port where the NATS server listens on (default 4222) --events.nats.storagedir string Directory where file-backed streams are stored in the NATS server @@ -44,7 +46,7 @@ The following options apply to the server commands below: --http.default.log string What to log about HTTP requests. Options are 'nothing', 'metadata' (log request method, URI, IP and response code), and 'metadata-and-body' (log the request and response body, in addition to the metadata). (default "metadata") --http.default.tls string Whether to enable TLS for the default interface, options are 'disabled', 'server', 'server-client'. Leaving it empty is synonymous to 'disabled', --internalratelimiter When set, expensive internal calls are rate-limited to protect the network. Always enabled in strict mode. (default true) - --jsonld.contexts.localmapping stringToString This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. (default [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson]) + --jsonld.contexts.localmapping stringToString This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist. (default [https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson,https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson]) --jsonld.contexts.remoteallowlist strings In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here. (default [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json]) --loggerformat string Log format (text, json) (default "text") --network.bootstrapnodes strings List of bootstrap nodes (':') which the node initially connect to. diff --git a/docs/pages/deployment/server_options.rst b/docs/pages/deployment/server_options.rst index a4f9973030..cb1e7afef7 100755 --- a/docs/pages/deployment/server_options.rst +++ b/docs/pages/deployment/server_options.rst @@ -33,6 +33,9 @@ crypto.vault.pathprefix kv The Vault path prefix. crypto.vault.timeout 5s Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s). crypto.vault.token The Vault token. If set it overwrites the VAULT_TOKEN env var. + **Discovery** + discovery.definitions.directory Directory to load Discovery Service Definitions from. If not set, the discovery service will be disabled. If the directory contains JSON files that can't be parsed as service definition, the node will fail to start. + discovery.server.definition_ids [] IDs of the Discovery Service Definitions for which to act as server. If an ID does not map to a loaded service definition, the node will fail to start. **Events** events.nats.hostname 0.0.0.0 Hostname for the NATS server events.nats.port 4222 Port where the NATS server listens on From e751c88c1ee0b9e4843c1ff8e28eae3d9b9dc513 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Wed, 29 Nov 2023 06:58:15 +0100 Subject: [PATCH 18/23] copyriht --- discovery/cmd/cmd.go | 18 ++++++++++++++++++ discovery/cmd/cmd_test.go | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/discovery/cmd/cmd.go b/discovery/cmd/cmd.go index 821e2b5ef5..302f812471 100644 --- a/discovery/cmd/cmd.go +++ b/discovery/cmd/cmd.go @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + package cmd import ( diff --git a/discovery/cmd/cmd_test.go b/discovery/cmd/cmd_test.go index 2a5f341b09..6f4a8c74ef 100644 --- a/discovery/cmd/cmd_test.go +++ b/discovery/cmd/cmd_test.go @@ -1,3 +1,21 @@ +/* + * Copyright (C) 2023 Nuts community + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + */ + package cmd import ( From 96d172d473b9a051635bb4e30f59301023ef1c34 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Wed, 29 Nov 2023 07:18:34 +0100 Subject: [PATCH 19/23] documentation --- docs/index.rst | 1 + docs/pages/deployment/discovery.rst | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 docs/pages/deployment/discovery.rst diff --git a/docs/index.rst b/docs/index.rst index 131c752e84..733a67b69f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -37,6 +37,7 @@ Nuts documentation pages/deployment/monitoring.rst pages/deployment/administering-your-node.rst pages/deployment/cli-reference.rst + pages/deployment/discovery.rst pages/deployment/backup-restore.rst pages/deployment/key-rotation.rst pages/deployment/storage-configuration.rst diff --git a/docs/pages/deployment/discovery.rst b/docs/pages/deployment/discovery.rst new file mode 100644 index 0000000000..e00527fde3 --- /dev/null +++ b/docs/pages/deployment/discovery.rst @@ -0,0 +1,29 @@ +.. _discovery: + +Discovery +######### + +.. warning:: + This feature is under development and subject to change. + +Discovery allows parties to publish information about themselves as a Verifiable Presentation, +so that other parties can discover them for further (data) exchange. + +In this Discovery Service protocol there are clients and servers: clients register their Verifiable Presentations on a server, +which can be queried by other clients. +Where to find the server and what is allowed in the Verifiable Presentations is defined in a Discovery Service Definition. +These are JSON documents that are loaded by both client and server. + +The Nuts node always acts as client for every loaded service definition, meaning it can register itself on the server and query it. +It only acts as server for a specific server if configured to do so. + +Configuration +************* + +Service definitions are JSON files loaded from the ``discovery.definitions.directory`` directory. +It loads all files wih the ``.json`` extension in this directory. It does not load subdirectories. +If the directory contains JSON files that are not (valid) service definitions, the node will fail to start. + +To act as server for a specific discovery service definition, +the service ID from the definition needs to be specified in ``discovery.server.defition_ids``. +The IDs in this list must correspond to the ``id`` fields of the loaded service definition, otherwise the node will fail to start. \ No newline at end of file From 3a759fcd115e4781c0e3e703877009c40c745477 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Wed, 29 Nov 2023 10:05:28 +0100 Subject: [PATCH 20/23] tests, todo --- cmd/root_test.go | 2 +- discovery/module.go | 9 ++++++--- vcr/credential/resolver.go | 2 -- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/root_test.go b/cmd/root_test.go index 3db937fa69..9f12c828d5 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -163,7 +163,7 @@ func Test_CreateSystem(t *testing.T) { system.VisitEngines(func(engine core.Engine) { numEngines++ }) - assert.Equal(t, 15, numEngines) + assert.Equal(t, 16, numEngines) } func Test_ClientCommand_ErrorHandlers(t *testing.T) { diff --git a/discovery/module.go b/discovery/module.go index 5e007778ba..bf7aa40ee7 100644 --- a/discovery/module.go +++ b/discovery/module.go @@ -169,10 +169,13 @@ func (m *Module) validateRegistration(definition ServiceDefinition, presentation } } // VP must fulfill the PEX Presentation ServiceDefinition - // TODO: Use Validate() once PR is merged + // We don't have a PresentationSubmission, so we can't use Validate(). creds, _, err := definition.PresentationDefinition.Match(presentation.VerifiableCredential) - if err != nil || len(creds) != len(presentation.VerifiableCredential) { - return fmt.Errorf("presentation does not fulfill Presentation ServiceDefinition: %w", err) + if err != nil { + return err + } + if len(creds) != len(presentation.VerifiableCredential) { + return errors.New("presentation does not fulfill Presentation ServiceDefinition") } return nil } diff --git a/vcr/credential/resolver.go b/vcr/credential/resolver.go index 21cf94a235..eda669ff84 100644 --- a/vcr/credential/resolver.go +++ b/vcr/credential/resolver.go @@ -23,8 +23,6 @@ import ( "errors" "fmt" "github.com/nuts-foundation/go-did/did" - "errors" - "github.com/nuts-foundation/go-did/did" "github.com/nuts-foundation/go-did/vc" "github.com/nuts-foundation/nuts-node/vcr/signature/proof" ) From 2467cc728498448f426434f5cdbdbaa2e371276b Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Wed, 29 Nov 2023 15:12:27 +0100 Subject: [PATCH 21/23] feedback --- storage/sql_migrations/2_discoveryservice.up.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/storage/sql_migrations/2_discoveryservice.up.sql b/storage/sql_migrations/2_discoveryservice.up.sql index 824d848bcb..d2e87102fa 100644 --- a/storage/sql_migrations/2_discoveryservice.up.sql +++ b/storage/sql_migrations/2_discoveryservice.up.sql @@ -1,8 +1,9 @@ -- discovery contains the known discovery services and the highest timestamp create table discovery_service ( - id varchar(36) not null primary key, - lamport_timestamp integer not null + -- id is the unique identifier for the service. It comes from the service definition. + id varchar(200) not null primary key, + lamport_timestamp integer not null ); -- discovery_presentation contains the presentations of the discovery services @@ -18,6 +19,8 @@ create table discovery_presentation unique (service_id, credential_subject_id), constraint fk_discovery_presentation_service_id foreign key (service_id) references discovery_service (id) on delete cascade ); +-- index for the presentation_expiration column, used by prune() +create index idx_discovery_presentation_expiration on discovery_presentation (presentation_expiration); -- discovery_credential is a credential in a presentation of the discovery service. -- We could do without the table, but having it allows to have a normalized index for credential properties that appear on every credential. @@ -25,6 +28,7 @@ create table discovery_presentation create table discovery_credential ( id varchar(36) not null primary key, + -- presentation_id is NOT the ID of the presentation (VerifiablePresentation.ID), but refers to the presentation record in the discovery_presentation table. presentation_id varchar(36) not null, credential_id varchar not null, credential_issuer varchar not null, From 7c25c863e96729ca54c0f930a81af27ed02c1a74 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Thu, 30 Nov 2023 14:39:13 +0100 Subject: [PATCH 22/23] tweak error msg --- discovery/module.go | 2 +- discovery/module_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/discovery/module.go b/discovery/module.go index bf7aa40ee7..5e359e2ce2 100644 --- a/discovery/module.go +++ b/discovery/module.go @@ -203,7 +203,7 @@ func (m *Module) validateRetraction(serviceID string, presentation vc.Verifiable return fmt.Errorf("retraction presentation 'retract_jti' claim is not a valid DID URL: %w", err) } if !signerDID.Equals(retractJTI.DID) { - return errors.New("retraction presentation 'retract_jti' claim does not match JWT issuer") + return errors.New("retraction presentation 'retract_jti' claim DID does not match JWT issuer") } exists, err := m.store.exists(serviceID, signerDID.String(), retractJTIString) if err != nil { diff --git a/discovery/module_test.go b/discovery/module_test.go index 5b0b0c68b7..4c93de5cc0 100644 --- a/discovery/module_test.go +++ b/discovery/module_test.go @@ -200,7 +200,7 @@ func Test_Module_Add(t *testing.T) { claims["retract_jti"] = bobDID.String() }) err := m.Add(testServiceID, vp) - assert.EqualError(t, err, "retraction presentation 'retract_jti' claim does not match JWT issuer") + assert.EqualError(t, err, "retraction presentation 'retract_jti' claim DID does not match JWT issuer") }) }) } From 0306b39d8a1d08c83a0129cd88113163bb893797 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Thu, 30 Nov 2023 14:54:33 +0100 Subject: [PATCH 23/23] remove requirement retraction ID is scoped to issuer --- discovery/module.go | 13 ++----------- discovery/module_test.go | 18 ------------------ 2 files changed, 2 insertions(+), 29 deletions(-) diff --git a/discovery/module.go b/discovery/module.go index 5e359e2ce2..ce7fd89642 100644 --- a/discovery/module.go +++ b/discovery/module.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" ssi "github.com/nuts-foundation/go-did" - "github.com/nuts-foundation/go-did/did" "github.com/nuts-foundation/go-did/vc" "github.com/nuts-foundation/nuts-node/core" "github.com/nuts-foundation/nuts-node/storage" @@ -186,9 +185,8 @@ func (m *Module) validateRetraction(serviceID string, presentation vc.Verifiable if len(presentation.VerifiableCredential) > 0 { return errors.New("retraction presentation must not contain credentials") } - // Check that the retraction refers to a presentation that: - // - is owned by the signer (same DID) - // - exists (if not, it might've already been removed due to expiry, or superseeded by a newer presentation) + // Check that the retraction refers to an existing presentation. + // If not, it might've already been removed due to expiry or superseded by a newer presentation. var retractJTIString string if retractJTIRaw, ok := presentation.JWT().Get("retract_jti"); !ok { return errors.New("retraction presentation does not contain 'retract_jti' claim") @@ -198,13 +196,6 @@ func (m *Module) validateRetraction(serviceID string, presentation vc.Verifiable } } signerDID, _ := credential.PresentationSigner(presentation) // checked before - retractJTI, err := did.ParseDIDURL(retractJTIString) - if err != nil { - return fmt.Errorf("retraction presentation 'retract_jti' claim is not a valid DID URL: %w", err) - } - if !signerDID.Equals(retractJTI.DID) { - return errors.New("retraction presentation 'retract_jti' claim DID does not match JWT issuer") - } exists, err := m.store.exists(serviceID, signerDID.String(), retractJTIString) if err != nil { return err diff --git a/discovery/module_test.go b/discovery/module_test.go index 4c93de5cc0..4c6d96b570 100644 --- a/discovery/module_test.go +++ b/discovery/module_test.go @@ -184,24 +184,6 @@ func Test_Module_Add(t *testing.T) { err := m.Add(testServiceID, vp) assert.EqualError(t, err, "retraction presentation 'retract_jti' claim is not a string") }) - t.Run("'retract_jti' claim in not a valid DID", func(t *testing.T) { - m, _ := setupModule(t, storageEngine) - vp := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { - vp.Type = append(vp.Type, retractionPresentationType) - claims["retract_jti"] = "not a DID" - }) - err := m.Add(testServiceID, vp) - assert.EqualError(t, err, "retraction presentation 'retract_jti' claim is not a valid DID URL: invalid DID") - }) - t.Run("'retract_jti' claim does not reference a presentation of the signer", func(t *testing.T) { - m, _ := setupModule(t, storageEngine) - vp := createPresentationCustom(aliceDID, func(claims map[string]interface{}, vp *vc.VerifiablePresentation) { - vp.Type = append(vp.Type, retractionPresentationType) - claims["retract_jti"] = bobDID.String() - }) - err := m.Add(testServiceID, vp) - assert.EqualError(t, err, "retraction presentation 'retract_jti' claim DID does not match JWT issuer") - }) }) }