From 914cb00176a38d3f8cb887bd8136eba22d0ae664 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:39:00 +0200 Subject: [PATCH 01/21] fmt: make the linter happy and add document starts to yamls --- .github/dependabot.yml | 1 + .goreleaser.yaml | 1 + docker-compose.yml | 1 + testdata/testconfig.yaml | 1 + 4 files changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5e562fb..9ebc4be 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,3 +1,4 @@ +--- # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 10ce928..aa23872 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,3 +1,4 @@ +--- before: hooks: - go mod tidy diff --git a/docker-compose.yml b/docker-compose.yml index 0f16bf6..abf716d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,3 +1,4 @@ +--- version: '3.4' services: diff --git a/testdata/testconfig.yaml b/testdata/testconfig.yaml index 14f7a6b..ea9a279 100644 --- a/testdata/testconfig.yaml +++ b/testdata/testconfig.yaml @@ -1,3 +1,4 @@ +--- sync: start: identifier: MonthStart # 1st of the current month From b266ae9dc537e10d0ab2b6c1206293aefa778351 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:39:33 +0200 Subject: [PATCH 02/21] feat: add Accepted field to models.Event --- internal/models/event.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/models/event.go b/internal/models/event.go index db0e3d2..7713c2f 100644 --- a/internal/models/event.go +++ b/internal/models/event.go @@ -27,6 +27,7 @@ type Event struct { Attendees Attendees Reminders Reminders MeetingLink string + Accepted bool } type Reminders []Reminder From ee2b4be3621da00e600838086954158e106fd954 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:40:11 +0200 Subject: [PATCH 03/21] feat(outlook): add ResponseStatus field and map to models.Event.Accepted --- internal/adapter/outlook_http/client.go | 5 ++++ internal/adapter/outlook_http/models.go | 37 +++++++++++++++---------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/internal/adapter/outlook_http/client.go b/internal/adapter/outlook_http/client.go index 223f5c9..ddccc9e 100644 --- a/internal/adapter/outlook_http/client.go +++ b/internal/adapter/outlook_http/client.go @@ -267,6 +267,10 @@ func (o OutlookClient) outlookEventToEvent(oe Event, adapterSourceID string) (e }, }) } + var hasEventAccepted bool = true + if oe.ResponseStatus.Response == "declined" { + hasEventAccepted = false + } bufEvent = models.Event{ ICalUID: oe.UID, @@ -280,6 +284,7 @@ func (o OutlookClient) outlookEventToEvent(oe Event, adapterSourceID string) (e Attendees: attendees, Reminders: reminders, MeetingLink: oe.OnlineMeetingUrl, + Accepted: hasEventAccepted, } if oe.IsAllDay { diff --git a/internal/adapter/outlook_http/models.go b/internal/adapter/outlook_http/models.go index e59ef3e..4ba9fa6 100644 --- a/internal/adapter/outlook_http/models.go +++ b/internal/adapter/outlook_http/models.go @@ -12,21 +12,22 @@ type EventList struct { } type Event struct { - ID string `json:"id"` - UID string `json:"iCalUId"` - ChangeKey string `json:"changeKey"` - HtmlLink string `json:"webLink"` - Subject string `json:"subject"` - Start Time `json:"start"` - End Time `json:"end"` - Body Body `json:"body,omitempty"` - Attendees []Attendee `json:"attendees,omitempty"` - Location Location `json:"location"` - IsReminderOn bool `json:"isReminderOn"` - ReminderMinutesBeforeStart int `json:"reminderMinutesBeforeStart"` - Extensions []Extensions `json:"extensions"` - IsAllDay bool `json:"isAllDay"` - OnlineMeetingUrl string `json:"onlineMeetingUrl"` + ID string `json:"id"` + UID string `json:"iCalUId"` + ChangeKey string `json:"changeKey"` + HtmlLink string `json:"webLink"` + Subject string `json:"subject"` + Start Time `json:"start"` + End Time `json:"end"` + Body Body `json:"body,omitempty"` + Attendees []Attendee `json:"attendees,omitempty"` + Location Location `json:"location"` + IsReminderOn bool `json:"isReminderOn"` + ReminderMinutesBeforeStart int `json:"reminderMinutesBeforeStart"` + Extensions []Extensions `json:"extensions"` + IsAllDay bool `json:"isAllDay"` + OnlineMeetingUrl string `json:"onlineMeetingUrl"` + ResponseStatus ResponseStatus `json:"responseStatus,omitempty"` } type Extensions struct { @@ -36,6 +37,12 @@ type Extensions struct { models.Metadata } +type ResponseStatus struct { + Response string `json:"response,omitempty"` + // there's an additional field called `time` which returns date and time when the response was returned + // but we don't need that +} + type Body struct { ContentType string `json:"contentType,omitempty"` Content string `json:"content,omitempty"` From 7a547e453ef289686971ea4108af95aa516c2fc7 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:40:42 +0200 Subject: [PATCH 04/21] feat(google): check for Event Response and map to models.Event.Accepted --- internal/adapter/google/event.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/adapter/google/event.go b/internal/adapter/google/event.go index 9f5385b..f3ffca5 100644 --- a/internal/adapter/google/event.go +++ b/internal/adapter/google/event.go @@ -15,7 +15,11 @@ func calendarEventToEvent(e *calendar.Event, adapterSourceID string) models.Even metadata := ensureMetadata(e, adapterSourceID) var attendees []models.Attendee + var hasEventAccepted bool = true for _, eventAttendee := range e.Attendees { + if eventAttendee.Self && eventAttendee.ResponseStatus == "declined" { + hasEventAccepted = false + } attendees = append(attendees, models.Attendee{ Email: eventAttendee.Email, DisplayName: eventAttendee.DisplayName, @@ -47,6 +51,7 @@ func calendarEventToEvent(e *calendar.Event, adapterSourceID string) models.Even Attendees: attendees, Reminders: reminders, MeetingLink: e.HangoutLink, + Accepted: hasEventAccepted, } } From cf05cca25cec86121e3a6c5978c862579b06f8d1 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:41:03 +0200 Subject: [PATCH 05/21] feat: add sync_declined_events field to config --- example.sync.yaml | 1 + internal/config/config.go | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/example.sync.yaml b/example.sync.yaml index b0d0b0f..54c6867 100644 --- a/example.sync.yaml +++ b/example.sync.yaml @@ -6,6 +6,7 @@ sync: end: identifier: MonthEnd # last day of the current month offset: +1 # MonthEnd +1 month (end of next month) + sync_declined_events: false auth: storage_mode: yaml diff --git a/internal/config/config.go b/internal/config/config.go index 2efccb4..902e250 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -79,8 +79,9 @@ type Transformer struct { // Sync configuration type Sync struct { - StartTime SyncTime `yaml:"start"` - EndTime SyncTime `yaml:"end"` + StartTime SyncTime `yaml:"start"` + EndTime SyncTime `yaml:"end"` + SyncDeclinedEvents bool `yaml:"sync_declined_events"` } type SyncTime struct { From 01355ec571713d83bcc5e1a9d8516bfc6ce878e3 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:41:39 +0200 Subject: [PATCH 06/21] feat: add SyncDeclinedEvents field to controller struct --- internal/sync/controller.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/internal/sync/controller.go b/internal/sync/controller.go index f9a87bc..c636e37 100644 --- a/internal/sync/controller.go +++ b/internal/sync/controller.go @@ -20,20 +20,22 @@ var ( type Controller struct { source Source // transformers are applied in order - transformers []Transformer - sink Sink - concurrency int - logger *log.Logger + transformers []Transformer + sink Sink + concurrency int + logger *log.Logger + SyncDeclinedEvents bool } // NewController constructs a new Controller. -func NewController(logger *log.Logger, source Source, sink Sink, transformer ...Transformer) Controller { +func NewController(logger *log.Logger, source Source, sink Sink, SyncDeclinedEvents bool, transformer ...Transformer) Controller { return Controller{ - concurrency: 1, - source: source, - transformers: transformer, - sink: sink, - logger: logger, + concurrency: 1, + source: source, + transformers: transformer, + sink: sink, + logger: logger, + SyncDeclinedEvents: SyncDeclinedEvents, } } From 7c23a11d01f27f0935ac32c8184346321907d2a0 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:42:08 +0200 Subject: [PATCH 07/21] test: modify tests for the newly introduced accepted field and controller property --- cmd/calendarsync/main.go | 2 +- internal/sync/controller_test.go | 56 +++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/cmd/calendarsync/main.go b/cmd/calendarsync/main.go index 403ecfc..8583418 100644 --- a/cmd/calendarsync/main.go +++ b/cmd/calendarsync/main.go @@ -164,7 +164,7 @@ func Run(c *cli.Context) error { } } - controller := sync.NewController(log.Default(), sourceAdapter, sinkAdapter, sync.TransformerFactory(cfg.Transformations)...) + controller := sync.NewController(log.Default(), sourceAdapter, sinkAdapter, cfg.Sync.SyncDeclinedEvents, sync.TransformerFactory(cfg.Transformations)...) if cfg.UpdateConcurrency != 0 { controller.SetConcurrency(cfg.UpdateConcurrency) } diff --git a/internal/sync/controller_test.go b/internal/sync/controller_test.go index cade14c..eaea199 100644 --- a/internal/sync/controller_test.go +++ b/internal/sync/controller_test.go @@ -34,7 +34,7 @@ func (suite *ControllerTestSuite) SetupTest() { {Name: "KeepTitle"}, {Name: "KeepReminders"}, }) - suite.controller = NewController(log.Default(), suite.source, suite.sink, transformers...) + suite.controller = NewController(log.Default(), suite.source, suite.sink, false, transformers...) } // TestDryRun tests that no acutal adapter func is called @@ -54,6 +54,7 @@ func (suite *ControllerTestSuite) TestDryRun() { AllDay: false, Metadata: models.NewEventMetadata("seed1", "uri", "sourceID"), Reminders: []models.Reminder{{Actions: models.ReminderActionDisplay, Trigger: models.ReminderTrigger{PointInTime: startTime.Add(-10 * time.Minute)}}}, + Accepted: true, }, { ICalUID: "testID3", @@ -65,6 +66,7 @@ func (suite *ControllerTestSuite) TestDryRun() { AllDay: false, Metadata: models.NewEventMetadata("seed3", "uri", "sourceID"), Reminders: []models.Reminder{{Actions: models.ReminderActionDisplay, Trigger: models.ReminderTrigger{PointInTime: startTime.Add(-10 * time.Minute)}}}, + Accepted: true, }, } @@ -197,6 +199,7 @@ func (suite *ControllerTestSuite) TestCreateEventsEmptySink() { AllDay: false, Metadata: models.NewEventMetadata("seed1", "uri", "sourceID"), Reminders: []models.Reminder{{Actions: models.ReminderActionDisplay, Trigger: models.ReminderTrigger{PointInTime: time.Now().Add(-10 * time.Minute)}}}, + Accepted: true, }, { ICalUID: "testID2", @@ -207,6 +210,7 @@ func (suite *ControllerTestSuite) TestCreateEventsEmptySink() { EndTime: time.Now().Add(time.Hour), AllDay: false, Metadata: models.NewEventMetadata("seed2", "uri", "sourceID"), + Accepted: true, }, } @@ -241,6 +245,7 @@ func (suite *ControllerTestSuite) TestDeleteEventsNotInSink() { AllDay: false, Metadata: models.NewEventMetadata("seed1", "uri", "sourceID"), Reminders: []models.Reminder{{Actions: models.ReminderActionDisplay, Trigger: models.ReminderTrigger{PointInTime: startTime.Add(-10 * time.Minute)}}}, + Accepted: true, }, } sinkEvents := []models.Event{ @@ -325,6 +330,7 @@ func (suite *ControllerTestSuite) TestUpdateEventsPrefilledSink() { AllDay: false, Metadata: models.NewEventMetadata("seed1", "uri", "sourceID"), Reminders: []models.Reminder{{Actions: models.ReminderActionDisplay, Trigger: models.ReminderTrigger{PointInTime: time.Now().Add(-10 * time.Minute)}}}, + Accepted: true, }, { ICalUID: "testID2", @@ -335,6 +341,7 @@ func (suite *ControllerTestSuite) TestUpdateEventsPrefilledSink() { EndTime: endTime, AllDay: false, Metadata: models.NewEventMetadata("seed2", "uri", "sourceID"), + Accepted: true, }, { ICalUID: "testID3", @@ -345,6 +352,7 @@ func (suite *ControllerTestSuite) TestUpdateEventsPrefilledSink() { EndTime: endTime, AllDay: false, Metadata: models.NewEventMetadata("seed3", "uri", "sourceID"), + Accepted: true, }, { ICalUID: "testID4", @@ -356,6 +364,7 @@ func (suite *ControllerTestSuite) TestUpdateEventsPrefilledSink() { AllDay: false, Metadata: models.NewEventMetadata("seed4", "uri", "sourceID"), Reminders: []models.Reminder{{Actions: models.ReminderActionDisplay, Trigger: models.ReminderTrigger{PointInTime: time.Now().Add(-30 * time.Minute)}}}, + Accepted: true, }, } @@ -437,6 +446,51 @@ func (suite *ControllerTestSuite) TestUpdateEventsPrefilledSink() { suite.sink.AssertNotCalled(suite.T(), "DeleteEvent", ctx, mock.AnythingOfType("models.Event")) } +// TestCreateEventsDeclined asserts that, only the accepted event gets synced +func (suite *ControllerTestSuite) TestCreateEventsDeclined() { + ctx := context.Background() + startTime := time.Now() + endTime := startTime.Add(2 * time.Hour) + eventsToCreate := []models.Event{ + { + ICalUID: "testID", + ID: "testUID", + Title: "Title", + Description: "Description", + StartTime: time.Now(), + EndTime: time.Now().Add(time.Hour), + AllDay: false, + Metadata: models.NewEventMetadata("seed1", "uri", "sourceID"), + Reminders: []models.Reminder{{Actions: models.ReminderActionDisplay, Trigger: models.ReminderTrigger{PointInTime: time.Now().Add(-10 * time.Minute)}}}, + Accepted: true, + }, + { + ICalUID: "testID2", + ID: "testUID2", + Title: "Title", + Description: "Description", + StartTime: time.Now(), + EndTime: time.Now().Add(time.Hour), + AllDay: false, + Metadata: models.NewEventMetadata("seed2", "uri", "sourceID"), + Accepted: false, + }, + } + + suite.source.On("EventsInTimeframe", ctx, startTime, endTime).Return(eventsToCreate, nil) + suite.sink.On("EventsInTimeframe", ctx, startTime, endTime).Return(nil, nil) + suite.sink.On("CreateEvent", ctx, mock.AnythingOfType("models.Event")).Return(nil) + + err := suite.controller.SynchroniseTimeframe(ctx, startTime, endTime, false) + assert.NoError(suite.T(), err) + + suite.source.AssertCalled(suite.T(), "EventsInTimeframe", ctx, startTime, endTime) + suite.sink.AssertCalled(suite.T(), "EventsInTimeframe", ctx, startTime, endTime) + suite.sink.AssertNumberOfCalls(suite.T(), "CreateEvent", len(eventsToCreate)-1) + suite.sink.AssertNotCalled(suite.T(), "UpdateEvent", ctx, mock.AnythingOfType("models.Event")) + suite.sink.AssertNotCalled(suite.T(), "DeleteEvent", ctx, mock.AnythingOfType("models.Event")) +} + func TestControllerTestSuite(t *testing.T) { suite.Run(t, new(ControllerTestSuite)) } From 1ad213e0e8ce2f1c51395fa457561b1312bc3dd0 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:42:31 +0200 Subject: [PATCH 08/21] feat: add function to remove declined events if config parameter is set --- internal/sync/controller.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/internal/sync/controller.go b/internal/sync/controller.go index c636e37..7c2141a 100644 --- a/internal/sync/controller.go +++ b/internal/sync/controller.go @@ -3,6 +3,7 @@ package sync import ( "context" "fmt" + "slices" "time" "github.com/inovex/CalendarSync/internal/models" @@ -77,6 +78,12 @@ func (p Controller) SynchroniseTimeframe(ctx context.Context, start time.Time, e return err } + // remove declined events + if !p.SyncDeclinedEvents { + log.Debug("We're not syncing declined events, to enable set sync.sync_declined_events to true in your config.yaml") + eventsInSource = removeDeclinedEvents(eventsInSource) + } + // Transform source events before comparing them to the sink events transformedEventsInSource := []models.Event{} @@ -228,3 +235,18 @@ func maps(events []models.Event) map[string]models.Event { } return result } + +func removeDeclinedEvents(events []models.Event) []models.Event { + var toDeleteIndex []int + for i, event := range events { + log.Warn("event accept check", "accepted", event.Accepted) + if !event.Accepted { + toDeleteIndex = append(toDeleteIndex, i) + } + } + for _, k := range toDeleteIndex { + log.Warn("Deleting event id", "id", k) + events = slices.Delete(events, k, k+1) + } + return events +} From fc70f5f695f09084e76da6c7a8dc3e2d9ae35401 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:45:00 +0200 Subject: [PATCH 09/21] docs: add sync_declined_fields parameter to readme.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index fb60a39..50ad4e4 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,7 @@ sync: end: identifier: MonthEnd # last day of the current month offset: +1 # MonthEnd +1 month (end of next month) + sync_declined_events: false # If you want to sync events which you declined in the source calendar, set to true ``` ## Source From c0f7c2ba5fe564db26f7cd9f7a03430a2c2e8dbc Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:53:41 +0200 Subject: [PATCH 10/21] feat: bump go version to 1.21 --- .github/workflows/build.yaml | 4 ++-- .github/workflows/release.yaml | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4045380..b2d96a9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -3,8 +3,8 @@ name: Build on: [ push ] env: - GO_VERSION: '1.20.4' - GO_VERSION_SHORT: '1.20' + GO_VERSION: '1.21.0' + GO_VERSION_SHORT: '1.21' GOLANGCI_VERSION: 'v1.52.2' STATICCHECK_VERSION: '2023.1.3' diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0bab143..31dab9c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -5,6 +5,9 @@ on: tags: - 'v*' +env: + GO_VERSION: '1.21.0' + permissions: contents: write packages: write @@ -19,6 +22,8 @@ jobs: fetch-depth: 0 - name: Set up Go uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} - name: Log in to GitHub container registry uses: docker/login-action@v2 From 0f56847704df8a0065526de5d7d26892021b8b5d Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 12 Aug 2023 12:58:01 +0200 Subject: [PATCH 11/21] feat: bump golangci-lint version to 1.54.0 --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b2d96a9..6c0fb46 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -5,7 +5,7 @@ on: [ push ] env: GO_VERSION: '1.21.0' GO_VERSION_SHORT: '1.21' - GOLANGCI_VERSION: 'v1.52.2' + GOLANGCI_VERSION: 'v1.54.0' STATICCHECK_VERSION: '2023.1.3' permissions: From 42a71d57091d679c7afa39e60524df31804953c6 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Wed, 20 Sep 2023 18:36:43 +0200 Subject: [PATCH 12/21] fix: remove SyncAllDayEvents field and add filters to the config file --- internal/config/config.go | 22 ++++++++++++++-------- internal/sync/controller.go | 14 +++++++------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 902e250..8c46529 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -8,11 +8,11 @@ import ( ) type File struct { - Path string - Auth AuthStorage - Source Source `yaml:"source"` - Sink Sink `yaml:"sink"` - // TODO: filters for source events + Path string + Auth AuthStorage + Source Source `yaml:"source"` + Sink Sink `yaml:"sink"` + Filters []Filter `yaml:"filters,omitempty"` Transformations []Transformer `yaml:"transformations,omitempty"` Sync Sync `yaml:"sync"` UpdateConcurrency int `yaml:"updateConcurrency,omitempty"` @@ -77,11 +77,17 @@ type Transformer struct { Config CustomMap `yaml:"config"` } +type Filter struct { + // Name of the filter + Name string `yaml:"name"` + // Any kind of parameter which can be passed to a filter. + Config CustomMap `yaml:"config"` +} + // Sync configuration type Sync struct { - StartTime SyncTime `yaml:"start"` - EndTime SyncTime `yaml:"end"` - SyncDeclinedEvents bool `yaml:"sync_declined_events"` + StartTime SyncTime `yaml:"start"` + EndTime SyncTime `yaml:"end"` } type SyncTime struct { diff --git a/internal/sync/controller.go b/internal/sync/controller.go index 7c2141a..14f0fc3 100644 --- a/internal/sync/controller.go +++ b/internal/sync/controller.go @@ -29,14 +29,14 @@ type Controller struct { } // NewController constructs a new Controller. -func NewController(logger *log.Logger, source Source, sink Sink, SyncDeclinedEvents bool, transformer ...Transformer) Controller { +func NewController(logger *log.Logger, source Source, sink Sink, transformer []Transformer, filters []Filter) Controller { return Controller{ - concurrency: 1, - source: source, - transformers: transformer, - sink: sink, - logger: logger, - SyncDeclinedEvents: SyncDeclinedEvents, + concurrency: 1, + source: source, + transformers: transformer, + filters: filters, + sink: sink, + logger: logger, } } From ecd9f3504ea0871edb543a9efb9c897ec50868a6 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Wed, 20 Sep 2023 18:37:47 +0200 Subject: [PATCH 13/21] feat: add Filter Interface, autoconfigure, etc mostly copy & paste from transformers --- internal/sync/filter.go | 101 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 internal/sync/filter.go diff --git a/internal/sync/filter.go b/internal/sync/filter.go new file mode 100644 index 0000000..1dfc1b0 --- /dev/null +++ b/internal/sync/filter.go @@ -0,0 +1,101 @@ +package sync + +import ( + "fmt" + "reflect" + "strings" + + "github.com/inovex/CalendarSync/internal/config" + "github.com/inovex/CalendarSync/internal/filter" + "github.com/inovex/CalendarSync/internal/models" +) + +type Filter interface { + NamedComponent + Filter(events []models.Event) []models.Event +} + +func FilterEvents(events []models.Event, filters ...Filter) (filteredEvents []models.Event) { + for _, filter := range filters { + events = filter.Filter(events) + } + return events +} + +var ( + filterConfigMapping = map[string]Filter{ + "DeclinedEvents": &filter.DeclinedEvents{}, + "AllDayEvents": &filter.AllDayEvents{}, + } + + filterOrder = []string{ + "DeclinedEvents", + "AllDayEvents", + } +) + +func FilterFactory(configuredFilters []config.Filter) (loadedFilters []Filter) { + for _, configuredFilter := range configuredFilters { + if _, nameExists := filterConfigMapping[configuredFilter.Name]; !nameExists { + // todo: handle properly + panic(fmt.Sprintf("unknown filter: %s", configuredFilter.Name)) + } + // load the default Transformer for the configured name and initialize it based on the config + filterDefault := filterConfigMapping[configuredFilter.Name] + loadedFilters = append(loadedFilters, filterFromConfig(filterDefault, configuredFilter.Config)) + } + + var sortedAndLoadedFilter []Filter + for _, name := range filterOrder { + for _, v := range loadedFilters { + if strings.EqualFold(name, v.Name()) { + sortedAndLoadedFilter = append(sortedAndLoadedFilter, v) + } + } + } + + return sortedAndLoadedFilter +} + +func autoConfigureFilter(filter Filter, config config.CustomMap) { + ps := reflect.ValueOf(filter) + s := ps.Elem() + if s.Kind() == reflect.Struct { + for key, value := range config { + field := s.FieldByName(key) + if field.IsValid() && field.CanSet() { + switch field.Kind() { + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64: + field.SetInt(value.(int64)) + case reflect.Bool: + field.SetBool(value.(bool)) + case reflect.String: + field.SetString(value.(string)) + default: + panic(fmt.Sprintf("autoConfigure(): unknown kind '%s' for field '%s'", key, field.Kind().String())) + } + } + } + } +} + +func filterFromConfig(filter Filter, config config.CustomMap) Filter { + autoConfigureFilter(filter, config) + return filter +} + +func removeDuplicateInt(intSlice []int) []int { + allKeys := make(map[int]bool) + list := []int{} + for _, item := range intSlice { + if _, value := allKeys[item]; !value { + allKeys[item] = true + list = append(list, item) + } + } + return list +} From ab8b72c6e97d5b0fae988058c4886dd8e6d1dc9e Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Wed, 20 Sep 2023 18:38:03 +0200 Subject: [PATCH 14/21] feat: use filters in the new controller --- cmd/calendarsync/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/calendarsync/main.go b/cmd/calendarsync/main.go index 8583418..9373f4e 100644 --- a/cmd/calendarsync/main.go +++ b/cmd/calendarsync/main.go @@ -164,7 +164,7 @@ func Run(c *cli.Context) error { } } - controller := sync.NewController(log.Default(), sourceAdapter, sinkAdapter, cfg.Sync.SyncDeclinedEvents, sync.TransformerFactory(cfg.Transformations)...) + controller := sync.NewController(log.Default(), sourceAdapter, sinkAdapter, sync.TransformerFactory(cfg.Transformations), sync.FilterFactory(cfg.Filters)) if cfg.UpdateConcurrency != 0 { controller.SetConcurrency(cfg.UpdateConcurrency) } From ca5c52153012f35df99c3fa300d4dbfbcdabcc65 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Wed, 20 Sep 2023 18:38:19 +0200 Subject: [PATCH 15/21] feat: add filters "AllDayEvents" and "DeclinedEvents" --- internal/filter/allDayEvents.go | 25 +++++++++++++++++++++++++ internal/filter/declinedEvents.go | 20 ++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 internal/filter/allDayEvents.go create mode 100644 internal/filter/declinedEvents.go diff --git a/internal/filter/allDayEvents.go b/internal/filter/allDayEvents.go new file mode 100644 index 0000000..4b49c10 --- /dev/null +++ b/internal/filter/allDayEvents.go @@ -0,0 +1,25 @@ +package filter + +import ( + "fmt" + + "github.com/inovex/CalendarSync/internal/models" +) + +type AllDayEvents struct { +} + +func (a AllDayEvents) Name() string { + return "AllDayEvents" +} + +func (a AllDayEvents) Filter(events []models.Event) (filteredEvents []models.Event) { + for _, event := range events { + if !event.AllDay { + filteredEvents = append(filteredEvents, event) + } else { + fmt.Println("Filtered!") + } + } + return filteredEvents +} diff --git a/internal/filter/declinedEvents.go b/internal/filter/declinedEvents.go new file mode 100644 index 0000000..e1fe04f --- /dev/null +++ b/internal/filter/declinedEvents.go @@ -0,0 +1,20 @@ +package filter + +import ( + "github.com/inovex/CalendarSync/internal/models" +) + +type DeclinedEvents struct{} + +func (d DeclinedEvents) Name() string { + return "DeclinedEvents" +} + +func (d DeclinedEvents) Filter(events []models.Event) (filteredEvents []models.Event) { + for _, event := range events { + if event.Accepted { + filteredEvents = append(filteredEvents, event) + } + } + return filteredEvents +} From bcf96b413dc2a75084ec55fa582f8f4a0f9daf42 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Wed, 20 Sep 2023 18:38:45 +0200 Subject: [PATCH 16/21] feat: add filter logic to SynchroniseTimeframe --- internal/sync/controller.go | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/internal/sync/controller.go b/internal/sync/controller.go index 14f0fc3..9ad880b 100644 --- a/internal/sync/controller.go +++ b/internal/sync/controller.go @@ -21,11 +21,11 @@ var ( type Controller struct { source Source // transformers are applied in order - transformers []Transformer - sink Sink - concurrency int - logger *log.Logger - SyncDeclinedEvents bool + transformers []Transformer + filters []Filter + sink Sink + concurrency int + logger *log.Logger } // NewController constructs a new Controller. @@ -78,12 +78,14 @@ func (p Controller) SynchroniseTimeframe(ctx context.Context, start time.Time, e return err } - // remove declined events - if !p.SyncDeclinedEvents { - log.Debug("We're not syncing declined events, to enable set sync.sync_declined_events to true in your config.yaml") - eventsInSource = removeDeclinedEvents(eventsInSource) + filteredEventsInSource := []models.Event{} + + for _, filter := range p.filters { + p.logger.Info("loaded filter", "name", filter.Name()) } + filteredEventsInSource = FilterEvents(eventsInSource, p.filters...) + // Transform source events before comparing them to the sink events transformedEventsInSource := []models.Event{} @@ -92,7 +94,7 @@ func (p Controller) SynchroniseTimeframe(ctx context.Context, start time.Time, e p.logger.Info("loaded transformer", "name", trans.Name()) } - for _, event := range eventsInSource { + for _, event := range filteredEventsInSource { transformedEventsInSource = append(transformedEventsInSource, TransformEvent(event, p.transformers...)) } @@ -235,18 +237,3 @@ func maps(events []models.Event) map[string]models.Event { } return result } - -func removeDeclinedEvents(events []models.Event) []models.Event { - var toDeleteIndex []int - for i, event := range events { - log.Warn("event accept check", "accepted", event.Accepted) - if !event.Accepted { - toDeleteIndex = append(toDeleteIndex, i) - } - } - for _, k := range toDeleteIndex { - log.Warn("Deleting event id", "id", k) - events = slices.Delete(events, k, k+1) - } - return events -} From 9d55da945c1ffec93a8829a0af776bf27851a724 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Wed, 20 Sep 2023 18:38:55 +0200 Subject: [PATCH 17/21] docs: add filters to example config --- example.sync.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/example.sync.yaml b/example.sync.yaml index 54c6867..96fc865 100644 --- a/example.sync.yaml +++ b/example.sync.yaml @@ -55,6 +55,13 @@ transformations: config: UseEmailAsDisplayName: true +# Filters remove events from being synced due to different criteria +filters: + # Events where you declined the invitation aren't synced + - name: DeclinedEvents + # Events which cover the full day aren't synced + - name: AllDayEvents + # Perform multiple calendar updates concurrently # Defaults to 1 if not set updateConcurrency: 3 From e957eabac4a3a48d5dd9a2965681368985f56652 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Sat, 23 Sep 2023 22:08:34 +0200 Subject: [PATCH 18/21] fix: move autoConfigure to its own file and allow any object to be passed --- internal/sync/filter.go | 41 +----------------------------------- internal/sync/transformer.go | 29 ------------------------- internal/sync/util.go | 34 ++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 69 deletions(-) create mode 100644 internal/sync/util.go diff --git a/internal/sync/filter.go b/internal/sync/filter.go index 1dfc1b0..d1bc4c8 100644 --- a/internal/sync/filter.go +++ b/internal/sync/filter.go @@ -2,7 +2,6 @@ package sync import ( "fmt" - "reflect" "strings" "github.com/inovex/CalendarSync/internal/config" @@ -57,45 +56,7 @@ func FilterFactory(configuredFilters []config.Filter) (loadedFilters []Filter) { return sortedAndLoadedFilter } -func autoConfigureFilter(filter Filter, config config.CustomMap) { - ps := reflect.ValueOf(filter) - s := ps.Elem() - if s.Kind() == reflect.Struct { - for key, value := range config { - field := s.FieldByName(key) - if field.IsValid() && field.CanSet() { - switch field.Kind() { - case reflect.Int, - reflect.Int8, - reflect.Int16, - reflect.Int32, - reflect.Int64: - field.SetInt(value.(int64)) - case reflect.Bool: - field.SetBool(value.(bool)) - case reflect.String: - field.SetString(value.(string)) - default: - panic(fmt.Sprintf("autoConfigure(): unknown kind '%s' for field '%s'", key, field.Kind().String())) - } - } - } - } -} - func filterFromConfig(filter Filter, config config.CustomMap) Filter { - autoConfigureFilter(filter, config) + autoConfigure(filter, config) return filter } - -func removeDuplicateInt(intSlice []int) []int { - allKeys := make(map[int]bool) - list := []int{} - for _, item := range intSlice { - if _, value := allKeys[item]; !value { - allKeys[item] = true - list = append(list, item) - } - } - return list -} diff --git a/internal/sync/transformer.go b/internal/sync/transformer.go index e9c0c27..34eb567 100644 --- a/internal/sync/transformer.go +++ b/internal/sync/transformer.go @@ -2,7 +2,6 @@ package sync import ( "fmt" - "reflect" "strings" "github.com/inovex/CalendarSync/internal/config" @@ -79,34 +78,6 @@ func TransformerFactory(configuredTransformers []config.Transformer) (loadedTran return sortedAndLoadedTransformer } -// autoConfigure can automatically map keys of the config.CustomMap to fields of a given Transformer implementation. -// TODO: I kinda wrote this in a hurry. Re-visit this. -func autoConfigure(transformer Transformer, config config.CustomMap) { - ps := reflect.ValueOf(transformer) - s := ps.Elem() - if s.Kind() == reflect.Struct { - for key, value := range config { - field := s.FieldByName(key) - if field.IsValid() && field.CanSet() { - switch field.Kind() { - case reflect.Int, - reflect.Int8, - reflect.Int16, - reflect.Int32, - reflect.Int64: - field.SetInt(value.(int64)) - case reflect.Bool: - field.SetBool(value.(bool)) - case reflect.String: - field.SetString(value.(string)) - default: - panic(fmt.Sprintf("autoConfigure(): unknown kind '%s' for field '%s'", key, field.Kind().String())) - } - } - } - } -} - func TransformerFromConfig(transformer Transformer, config config.CustomMap) Transformer { autoConfigure(transformer, config) return transformer diff --git a/internal/sync/util.go b/internal/sync/util.go new file mode 100644 index 0000000..8512c55 --- /dev/null +++ b/internal/sync/util.go @@ -0,0 +1,34 @@ +package sync + +import ( + "fmt" + "reflect" + + "github.com/inovex/CalendarSync/internal/config" +) + +func autoConfigure(object any, config config.CustomMap) { + ps := reflect.ValueOf(object) + s := ps.Elem() + if s.Kind() == reflect.Struct { + for key, value := range config { + field := s.FieldByName(key) + if field.IsValid() && field.CanSet() { + switch field.Kind() { + case reflect.Int, + reflect.Int8, + reflect.Int16, + reflect.Int32, + reflect.Int64: + field.SetInt(value.(int64)) + case reflect.Bool: + field.SetBool(value.(bool)) + case reflect.String: + field.SetString(value.(string)) + default: + panic(fmt.Sprintf("autoConfigure(): unknown kind '%s' for field '%s'", key, field.Kind().String())) + } + } + } + } +} From 17dfea0672e1f0ce7599e2b45865eb63d538b524 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Tue, 26 Sep 2023 16:43:59 +0200 Subject: [PATCH 19/21] refactor: filter events one by one we have aligned the processing of the filters to work like the processing of the transformers --- internal/filter/allDayEvents.go | 13 ++----------- internal/filter/declinedEvents.go | 9 ++------- internal/sync/controller.go | 9 +++++++-- internal/sync/filter.go | 14 ++++++++++---- 4 files changed, 21 insertions(+), 24 deletions(-) diff --git a/internal/filter/allDayEvents.go b/internal/filter/allDayEvents.go index 4b49c10..119bb68 100644 --- a/internal/filter/allDayEvents.go +++ b/internal/filter/allDayEvents.go @@ -1,8 +1,6 @@ package filter import ( - "fmt" - "github.com/inovex/CalendarSync/internal/models" ) @@ -13,13 +11,6 @@ func (a AllDayEvents) Name() string { return "AllDayEvents" } -func (a AllDayEvents) Filter(events []models.Event) (filteredEvents []models.Event) { - for _, event := range events { - if !event.AllDay { - filteredEvents = append(filteredEvents, event) - } else { - fmt.Println("Filtered!") - } - } - return filteredEvents +func (a AllDayEvents) Filter(event models.Event) bool { + return !event.AllDay } diff --git a/internal/filter/declinedEvents.go b/internal/filter/declinedEvents.go index e1fe04f..8bff962 100644 --- a/internal/filter/declinedEvents.go +++ b/internal/filter/declinedEvents.go @@ -10,11 +10,6 @@ func (d DeclinedEvents) Name() string { return "DeclinedEvents" } -func (d DeclinedEvents) Filter(events []models.Event) (filteredEvents []models.Event) { - for _, event := range events { - if event.Accepted { - filteredEvents = append(filteredEvents, event) - } - } - return filteredEvents +func (d DeclinedEvents) Filter(event models.Event) bool { + return event.Accepted } diff --git a/internal/sync/controller.go b/internal/sync/controller.go index 9ad880b..e95f45e 100644 --- a/internal/sync/controller.go +++ b/internal/sync/controller.go @@ -3,7 +3,6 @@ package sync import ( "context" "fmt" - "slices" "time" "github.com/inovex/CalendarSync/internal/models" @@ -84,7 +83,13 @@ func (p Controller) SynchroniseTimeframe(ctx context.Context, start time.Time, e p.logger.Info("loaded filter", "name", filter.Name()) } - filteredEventsInSource = FilterEvents(eventsInSource, p.filters...) + for _, event := range eventsInSource { + if FilterEvent(event, p.filters...) { + filteredEventsInSource = append(filteredEventsInSource, event) + } else { + p.logger.Debug("filter rejects event", logFields(event)...) + } + } // Transform source events before comparing them to the sink events transformedEventsInSource := []models.Event{} diff --git a/internal/sync/filter.go b/internal/sync/filter.go index d1bc4c8..7004e9a 100644 --- a/internal/sync/filter.go +++ b/internal/sync/filter.go @@ -11,14 +11,20 @@ import ( type Filter interface { NamedComponent - Filter(events []models.Event) []models.Event + // Filter returns true to keep the event + Filter(event models.Event) bool } -func FilterEvents(events []models.Event, filters ...Filter) (filteredEvents []models.Event) { +// FilterEvent returns false if one of the filters rejects the event +func FilterEvent(event models.Event, filters ...Filter) (result bool) { for _, filter := range filters { - events = filter.Filter(events) + // If the filter returns false (or: filters the event), then return false + if !filter.Filter(event) { + return false + } } - return events + // otherwise keep the event + return true } var ( From 6460e556910c1a7fd9101daec12685bd565d7467 Mon Sep 17 00:00:00 2001 From: Alexander Huck Date: Tue, 26 Sep 2023 16:51:04 +0200 Subject: [PATCH 20/21] test: modify filter test to the new format without the global boolean, using filters in the controller struct --- internal/sync/controller_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/sync/controller_test.go b/internal/sync/controller_test.go index eaea199..f1a8f27 100644 --- a/internal/sync/controller_test.go +++ b/internal/sync/controller_test.go @@ -34,7 +34,10 @@ func (suite *ControllerTestSuite) SetupTest() { {Name: "KeepTitle"}, {Name: "KeepReminders"}, }) - suite.controller = NewController(log.Default(), suite.source, suite.sink, false, transformers...) + filters := FilterFactory([]config.Filter{ + {Name: "DeclinedEvents"}, + }) + suite.controller = NewController(log.Default(), suite.source, suite.sink, transformers, filters) } // TestDryRun tests that no acutal adapter func is called From 45de524e0fbadeb32aa48eec029dc6025f248899 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 26 Sep 2023 19:47:57 +0200 Subject: [PATCH 21/21] docs: update readme.md with filter mechanism --- README.md | 16 +++++++++++++++- example.sync.yaml | 1 - 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 50ad4e4..2fcd3a0 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,6 @@ sync: end: identifier: MonthEnd # last day of the current month offset: +1 # MonthEnd +1 month (end of next month) - sync_declined_events: false # If you want to sync events which you declined in the source calendar, set to true ``` ## Source @@ -136,6 +135,21 @@ transformations: The transformers are applied in a specific order. The order is defined here: [`internal/sync/transformer.go`](./internal/sync/transformer.go) +## Filters + +In some cases events should not be synced. For example, declined events might +create too much noise in the target calendar. These can be filtered by enabling +the corresponding filter. + +```yaml +# Filters remove events from being synced due to different criteria +filters: + # Events where you declined the invitation aren't synced + - name: DeclinedEvents + # Events which cover the full day aren't synced + - name: AllDayEvents +``` + # Cleaning Up You just synced a lot of events in your calendar and decide you want to use a diff --git a/example.sync.yaml b/example.sync.yaml index 96fc865..1dfaadc 100644 --- a/example.sync.yaml +++ b/example.sync.yaml @@ -6,7 +6,6 @@ sync: end: identifier: MonthEnd # last day of the current month offset: +1 # MonthEnd +1 month (end of next month) - sync_declined_events: false auth: storage_mode: yaml