Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Encapsulate templating data source and optimize locking #1149

Merged
merged 1 commit into from
Aug 28, 2024
Merged

Encapsulate templating data source and optimize locking #1149

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 19 additions & 14 deletions core/hoverfly_service_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1523,25 +1523,30 @@ func TestHoverfly_SetMultipleTemplateDataSource(t *testing.T) {
Expect(err1).To(BeNil())
Expect(err2).To(BeNil())

Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources).ToNot(BeNil())
Expect(unit.templator.TemplateHelper.TemplateDataSource.GetAllDataSources()).ToNot(BeNil())

Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv1"]).NotTo(BeNil())
Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv2"]).NotTo(BeNil())
csv1, exists1:= unit.templator.TemplateHelper.TemplateDataSource.GetDataSource("test-csv1")
Expect(csv1).NotTo(BeNil())
Expect(exists1).To(BeTrue())

Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv1"].Name).To(Equal("test-csv1"))
Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv2"].Name).To(Equal("test-csv2"))
csv2, exists2:= unit.templator.TemplateHelper.TemplateDataSource.GetDataSource("test-csv2")
Expect(csv2).NotTo(BeNil())
Expect(exists2).To(BeTrue())

Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv1"].SourceType).To(Equal("csv"))
Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv2"].SourceType).To(Equal("csv"))
Expect(csv1.Name).To(Equal("test-csv1"))
Expect(csv2.Name).To(Equal("test-csv2"))

Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv1"].Data).To(HaveLen(3))
Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv2"].Data).To(HaveLen(4))
Expect(csv1.SourceType).To(Equal("csv"))
Expect(csv2.SourceType).To(Equal("csv"))

Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv1"].Data[1][2]).To(Equal("55"))
Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv2"].Data[2][2]).To(Equal("New York"))
Expect(csv1.Data).To(HaveLen(3))
Expect(csv2.Data).To(HaveLen(4))

Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv1"].Data[2][1]).To(Equal("Test2"))
Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources["test-csv2"].Data[3][0]).To(Equal("31"))
Expect(csv1.Data[1][2]).To(Equal("55"))
Expect(csv2.Data[2][2]).To(Equal("New York"))

Expect(csv1.Data[2][1]).To(Equal("Test2"))
Expect(csv2.Data[3][0]).To(Equal("31"))
}

func TestHoverfly_DeleteTemplateDataSource(t *testing.T) {
Expand All @@ -1557,7 +1562,7 @@ func TestHoverfly_DeleteTemplateDataSource(t *testing.T) {
unit.DeleteDataSource("test-csv1")

Expect(err).To(BeNil())
Expect(unit.templator.TemplateHelper.TemplateDataSource.DataSources).To(HaveLen(0))
Expect(unit.templator.TemplateHelper.TemplateDataSource.GetAllDataSources()).To(HaveLen(0))
}

func TestHoverfly_GetTemplateDataSources(t *testing.T) {
Expand Down
13 changes: 8 additions & 5 deletions core/templating/datasource_sql_over_csv.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,16 @@ func parseSqlCommand(query string, datasource *TemplateDataSource) (SQLStatement
}
columnsPart = matches[1]
dataSourceName = matches[2]
if !datasource.DataSourceExists(dataSourceName) {
source, exits := datasource.GetDataSource(dataSourceName)
if !exits {
return SQLStatement{}, errors.New("data source does not exist")
}

if len(matches) == 4 {
wherePart = matches[3]
}
headers := datasource.DataSources[dataSourceName].Data[0]

headers := source.Data[0]
columns, err := parseColumns(columnsPart, headers)
if err != nil {
return SQLStatement{}, err
Expand All @@ -86,14 +88,15 @@ func parseSqlCommand(query string, datasource *TemplateDataSource) (SQLStatement
return SQLStatement{}, errors.New("invalid UPDATE query format")
}
dataSourceName = matches[1]
if !datasource.DataSourceExists(dataSourceName) {
source, exits := datasource.GetDataSource(dataSourceName)
if !exits {
return SQLStatement{}, errors.New("data source does not exist")
}
setPart := matches[2]
if len(matches) == 4 {
wherePart = matches[3]
}
headers := datasource.DataSources[dataSourceName].Data[0]
headers := source.Data[0]
setClauses, err := parseSetClauses(setPart, headers)
if err != nil {
return SQLStatement{}, err
Expand All @@ -116,7 +119,7 @@ func parseSqlCommand(query string, datasource *TemplateDataSource) (SQLStatement
return SQLStatement{}, errors.New("invalid DELETE query format")
}
dataSourceName = matches[1]
if !datasource.DataSourceExists(dataSourceName) {
if _, exits := datasource.GetDataSource(dataSourceName); !exits {
return SQLStatement{}, errors.New("data source does not exist")
}
if len(matches) == 3 {
Expand Down
2 changes: 1 addition & 1 deletion core/templating/datasource_sql_over_csv_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func TestParseCommand(t *testing.T) {
},
}
templateDataSource := NewTemplateDataSource()
templateDataSource.DataSources = dataSources
templateDataSource.dataSources = dataSources

tests := []struct {
query string
Expand Down
46 changes: 22 additions & 24 deletions core/templating/template_datasource.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,47 +5,45 @@ import (
)

type TemplateDataSource struct {
DataSources map[string]*DataSource
RWMutex sync.RWMutex
dataSources map[string]*DataSource
rwMutex sync.RWMutex
}

func NewTemplateDataSource() *TemplateDataSource {

return &TemplateDataSource{
DataSources: make(map[string]*DataSource),
dataSources: make(map[string]*DataSource),
}
}

func (templateDataSource *TemplateDataSource) SetDataSource(dataSourceName string, dataSource *DataSource) {
func (t *TemplateDataSource) SetDataSource(dataSourceName string, dataSource *DataSource) {

templateDataSource.RWMutex.Lock()
templateDataSource.DataSources[dataSourceName] = dataSource
templateDataSource.RWMutex.Unlock()
t.rwMutex.Lock()
defer t.rwMutex.Unlock()

t.dataSources[dataSourceName] = dataSource
}

func (templateDataSource *TemplateDataSource) DeleteDataSource(dataSourceName string) {
func (t *TemplateDataSource) DeleteDataSource(dataSourceName string) {

templateDataSource.RWMutex.Lock()
t.rwMutex.Lock()
defer t.rwMutex.Unlock()

if _, ok := templateDataSource.DataSources[dataSourceName]; ok {
delete(templateDataSource.DataSources, dataSourceName)
}
templateDataSource.RWMutex.Unlock()
delete(t.dataSources, dataSourceName)
}

func (templateDataSource *TemplateDataSource) GetAllDataSources() map[string]*DataSource {
func (t *TemplateDataSource) GetAllDataSources() map[string]*DataSource {

return templateDataSource.DataSources
t.rwMutex.RLock()
defer t.rwMutex.RUnlock()

return t.dataSources
}

func (templateDataSource *TemplateDataSource) DataSourceExists(name string) bool {
templateDataSource.RWMutex.Lock()
defer templateDataSource.RWMutex.Unlock()
func (t *TemplateDataSource) GetDataSource(name string) (*DataSource, bool) {
t.rwMutex.RLock()
defer t.rwMutex.RUnlock()

for _, dataSource := range templateDataSource.DataSources {
if dataSource.Name == name {
return true
}
}
return false
source, exits := t.dataSources[name]
return source, exits
Comment on lines +43 to +48
Copy link
Member Author

@tommysitu tommysitu Aug 28, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ensure safe concurrent access using read lock

}
23 changes: 8 additions & 15 deletions core/templating/template_helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,8 +239,7 @@ func (t templateHelpers) faker(fakerType string) []reflect.Value {
}

func (t templateHelpers) fetchSingleFieldCsv(dataSourceName, searchFieldName, searchFieldValue, returnFieldName string, options *raymond.Options) string {
templateDataSources := t.TemplateDataSource.DataSources
source, exists := templateDataSources[dataSourceName]
source, exists := t.TemplateDataSource.GetDataSource(dataSourceName)
if !exists {
log.Error("could not find datasource " + dataSourceName)
return getEvaluationString("csv", options)
Expand Down Expand Up @@ -274,8 +273,7 @@ func (t templateHelpers) fetchSingleFieldCsv(dataSourceName, searchFieldName, se
}

func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchFieldName string, searchFieldValue string) []RowMap {
templateDataSources := t.TemplateDataSource.DataSources
source, exists := templateDataSources[dataSourceName]
source, exists := t.TemplateDataSource.GetDataSource(dataSourceName)
if !exists {
log.Error("could not find datasource " + dataSourceName)
return []RowMap{}
Expand Down Expand Up @@ -316,8 +314,7 @@ func (t templateHelpers) fetchMatchingRowsCsv(dataSourceName string, searchField
}

func (t templateHelpers) csvAsArray(dataSourceName string) [][]string {
templateDataSources := t.TemplateDataSource.DataSources
source, exists := templateDataSources[dataSourceName]
source, exists := t.TemplateDataSource.GetDataSource(dataSourceName)
if exists {
source.mu.Lock()
defer source.mu.Unlock()
Expand All @@ -330,8 +327,7 @@ func (t templateHelpers) csvAsArray(dataSourceName string) [][]string {

func (t templateHelpers) csvAsMap(dataSourceName string) []RowMap {

templateDataSources := t.TemplateDataSource.DataSources
source, exists := templateDataSources[dataSourceName]
source, exists := t.TemplateDataSource.GetDataSource(dataSourceName)
if !exists {
log.Error("could not find datasource " + dataSourceName)
return []RowMap{}
Expand All @@ -357,8 +353,7 @@ func (t templateHelpers) csvAsMap(dataSourceName string) []RowMap {
}

func (t templateHelpers) csvAddRow(dataSourceName string, newRow []string) string {
templateDataSources := t.TemplateDataSource.DataSources
source, exists := templateDataSources[dataSourceName]
source, exists := t.TemplateDataSource.GetDataSource(dataSourceName)
if exists {
source.mu.Lock()
defer source.mu.Unlock()
Expand All @@ -370,8 +365,7 @@ func (t templateHelpers) csvAddRow(dataSourceName string, newRow []string) strin
}

func (t templateHelpers) csvDeleteRows(dataSourceName, searchFieldName, searchFieldValue string, output bool) string {
templateDataSources := t.TemplateDataSource.DataSources
source, exists := templateDataSources[dataSourceName]
source, exists := t.TemplateDataSource.GetDataSource(dataSourceName)
if !exists {
log.Error("could not find datasource " + dataSourceName)
return ""
Expand Down Expand Up @@ -411,8 +405,7 @@ func (t templateHelpers) csvDeleteRows(dataSourceName, searchFieldName, searchFi
}

func (t templateHelpers) csvCountRows(dataSourceName string) string {
templateDataSources := t.TemplateDataSource.DataSources
source, exists := templateDataSources[dataSourceName]
source, exists := t.TemplateDataSource.GetDataSource(dataSourceName)
if !exists {
log.Error("could not find datasource " + dataSourceName)
return ""
Expand All @@ -436,7 +429,7 @@ func (t templateHelpers) csvSqlCommand(commandString string) []RowMap {
}

// Find the data source by name
source, exists := t.TemplateDataSource.DataSources[command.DataSourceName]
source, exists := t.TemplateDataSource.GetDataSource(command.DataSourceName)
if !exists {
log.Error("Could not find datasource " + command.DataSourceName)
return []RowMap{}
Expand Down
Loading