From 3b785e41657335caae66faaec5f362cec168ec62 Mon Sep 17 00:00:00 2001 From: Gordian Schoenherr Date: Tue, 26 Nov 2024 16:19:23 +0900 Subject: [PATCH 1/4] Refactor Filter options into a struct It was already a lot of options for one method and I am going to add another one in the next commit. --- api/api.go | 14 ++++-- api/mirror.go | 8 +++- api/repos.go | 9 +++- api/snapshot.go | 9 +++- cmd/repo_move.go | 9 +++- cmd/repo_remove.go | 2 +- cmd/snapshot_filter.go | 9 +++- cmd/snapshot_pull.go | 9 +++- cmd/snapshot_search.go | 9 +++- deb/list.go | 49 +++++++++++--------- deb/list_test.go | 102 +++++++++++++++++++++++++++-------------- deb/remote.go | 9 +++- 12 files changed, 167 insertions(+), 71 deletions(-) diff --git a/api/api.go b/api/api.go index a52406d80..6c9ed7e6c 100644 --- a/api/api.go +++ b/api/api.go @@ -227,8 +227,13 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory list.PrepareIndex() - list, err = list.Filter([]deb.PackageQuery{q}, withDeps, - nil, context.DependencyOptions(), architecturesList) + list, err = list.Filter(deb.FilterOptions{ + Queries: []deb.PackageQuery{q}, + WithDependencies: withDeps, + Source: nil, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + }) if err != nil { AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err)) return @@ -244,8 +249,9 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory fmt.Println("filter packages by version, query string parse err: ", err) c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err)) } else { - tmpList, err := list.Filter([]deb.PackageQuery{versionQ}, false, - nil, 0, []string{}) + tmpList, err := list.Filter(deb.FilterOptions{ + Queries: []deb.PackageQuery{versionQ}, + }) if err == nil { if tmpList.Len() > 0 { diff --git a/api/mirror.go b/api/mirror.go index 0052f2a59..803551dd4 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -306,8 +306,12 @@ func apiMirrorsPackages(c *gin.Context) { list.PrepareIndex() - list, err = list.Filter([]deb.PackageQuery{q}, withDeps, - nil, context.DependencyOptions(), architecturesList) + list, err = list.Filter(deb.FilterOptions{ + Queries: []deb.PackageQuery{q}, + WithDependencies: withDeps, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + }) if err != nil { AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err)) } diff --git a/api/repos.go b/api/repos.go index 005c2aaa1..3e3ddd3b6 100644 --- a/api/repos.go +++ b/api/repos.go @@ -586,7 +586,14 @@ func apiReposCopyPackage(c *gin.Context) { return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to parse query '%s': %s", fileName, err) } - toProcess, err := srcList.FilterWithProgress(queries, jsonBody.WithDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress()) + toProcess, err := srcList.Filter(deb.FilterOptions{ + Queries: queries, + WithDependencies: jsonBody.WithDeps, + Source: dstList, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("filter error: %s", err) } diff --git a/api/snapshot.go b/api/snapshot.go index 1a9c22786..df51adcd8 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -636,7 +636,14 @@ func apiSnapshotsPull(c *gin.Context) { } // Filter with dependencies as requested - destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, context.DependencyOptions(), architecturesList, context.Progress()) + destinationPackageList, err := sourcePackageList.Filter(deb.FilterOptions{ + Queries: queries, + WithDependencies: !noDeps, + Source: toPackageList, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } diff --git a/cmd/repo_move.go b/cmd/repo_move.go index 8be6698b6..e53c3a7c1 100644 --- a/cmd/repo_move.go +++ b/cmd/repo_move.go @@ -116,7 +116,14 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error { } } - toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress()) + toProcess, err := srcList.Filter(deb.FilterOptions{ + Queries: queries, + WithDependencies: withDeps, + Source: dstList, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return fmt.Errorf("unable to %s: %s", command, err) } diff --git a/cmd/repo_remove.go b/cmd/repo_remove.go index 93e8535c7..287a42d5e 100644 --- a/cmd/repo_remove.go +++ b/cmd/repo_remove.go @@ -45,7 +45,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error { } list.PrepareIndex() - toRemove, err := list.Filter(queries, false, nil, 0, nil) + toRemove, err := list.Filter(deb.FilterOptions{Queries: queries}) if err != nil { return fmt.Errorf("unable to remove: %s", err) } diff --git a/cmd/snapshot_filter.go b/cmd/snapshot_filter.go index b81a9cfc2..c712ae67c 100644 --- a/cmd/snapshot_filter.go +++ b/cmd/snapshot_filter.go @@ -67,7 +67,14 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error { } // Filter with dependencies as requested - result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress()) + result, err := packageList.Filter(deb.FilterOptions{ + Queries: queries, + WithDependencies: withDeps, + Source: nil, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return fmt.Errorf("unable to filter: %s", err) } diff --git a/cmd/snapshot_pull.go b/cmd/snapshot_pull.go index 9c2559f55..27593b777 100644 --- a/cmd/snapshot_pull.go +++ b/cmd/snapshot_pull.go @@ -97,7 +97,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error { } // Filter with dependencies as requested - result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress()) + result, err := sourcePackageList.Filter(deb.FilterOptions{ + Queries: queries, + WithDependencies: !noDeps, + Source: packageList, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return fmt.Errorf("unable to pull: %s", err) } diff --git a/cmd/snapshot_search.go b/cmd/snapshot_search.go index d771af7ce..7af78e1e4 100644 --- a/cmd/snapshot_search.go +++ b/cmd/snapshot_search.go @@ -103,8 +103,13 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error } } - result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps, - nil, context.DependencyOptions(), architecturesList, context.Progress()) + result, err := list.Filter(deb.FilterOptions{ + Queries: []deb.PackageQuery{q}, + WithDependencies: withDeps, + DependencyOptions: context.DependencyOptions(), + Architectures: architecturesList, + Progress: context.Progress(), + }) if err != nil { return fmt.Errorf("unable to search: %s", err) } diff --git a/deb/list.go b/deb/list.go index 4f863ccf1..9f3f67a9b 100644 --- a/deb/list.go +++ b/deb/list.go @@ -503,32 +503,37 @@ func (l *PackageList) Search(dep Dependency, allMatches bool, searchProvided boo return } -// Filter filters package index by specified queries (ORed together), possibly pulling dependencies -func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) { - return l.FilterWithProgress(queries, withDependencies, source, dependencyOptions, architecturesList, nil) +// FilterOptions specifies options for Filter() +type FilterOptions struct { + Queries []PackageQuery + WithDependencies bool + Source *PackageList + DependencyOptions int + Architectures []string + Progress aptly.Progress // set to non-nil to report progress } -// FilterWithProgress filters package index by specified queries (ORed together), possibly pulling dependencies and displays progress -func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) { +// Filter filters package index by specified queries (ORed together), possibly pulling dependencies +func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) { if !l.indexed { panic("list not indexed, can't filter") } result := NewPackageList() - for _, query := range queries { - result.Append(query.Query(l)) + for _, query := range options.Queries { + _ = result.Append(query.Query(l)) } - if withDependencies { + if options.WithDependencies { added := result.Len() result.PrepareIndex() dependencySource := NewPackageList() - if source != nil { - dependencySource.Append(source) + if options.Source != nil { + _ = dependencySource.Append(options.Source) } - dependencySource.Append(result) + _ = dependencySource.Append(result) dependencySource.PrepareIndex() // while some new dependencies were discovered @@ -536,22 +541,22 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie added = 0 // find missing dependencies - missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress) + missing, err := result.VerifyDependencies(options.DependencyOptions, options.Architectures, dependencySource, options.Progress) if err != nil { return nil, err } // try to satisfy dependencies for _, dep := range missing { - if dependencyOptions&DepFollowAllVariants == 0 { + if options.DependencyOptions&DepFollowAllVariants == 0 { // dependency might have already been satisfied // with packages already been added // // when follow-all-variants is enabled, we need to try to expand anyway, // as even if dependency is satisfied now, there might be other ways to satisfy dependency if result.Search(dep, false, true) != nil { - if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil { - progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true)) + if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil { + options.Progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true)) } continue } @@ -564,19 +569,19 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie continue } - if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil { - progress.ColoredPrintf("@{g}Injecting package@|: %s", p) + if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil { + options.Progress.ColoredPrintf("@{g}Injecting package@|: %s", p) } - result.Add(p) - dependencySource.Add(p) + _ = result.Add(p) + _ = dependencySource.Add(p) added++ - if dependencyOptions&DepFollowAllVariants == 0 { + if options.DependencyOptions&DepFollowAllVariants == 0 { break } } } else { - if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil { - progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String()) + if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil { + options.Progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String()) } } diff --git a/deb/list_test.go b/deb/list_test.go index 4209f125e..e13cd4f24 100644 --- a/deb/list_test.go +++ b/deb/list_test.go @@ -131,7 +131,7 @@ func (s *PackageListSuite) TestAddLen(c *C) { c.Check(s.list.Len(), Equals, 1) c.Check(s.list.Add(s.p3), IsNil) c.Check(s.list.Len(), Equals, 2) - c.Check(s.list.Add(s.p4), ErrorMatches, "package already exists and is different: .*") + c.Check(s.list.Add(s.p4), ErrorMatches, "package already exists and is different: .*") } func (s *PackageListSuite) TestRemove(c *C) { @@ -311,7 +311,11 @@ func (s *PackageListSuite) TestSearch(c *C) { } func (s *PackageListSuite) TestFilter(c *C) { - c.Check(func() { s.list.Filter([]PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter") + c.Check(func() { + s.list.Filter(FilterOptions{ + Queries: []PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}}, + }) + }, Panics, "list not indexed, can't filter") plString := func(l *PackageList) string { list := make([]string, 0, l.Len()) @@ -324,81 +328,111 @@ func (s *PackageListSuite) TestFilter(c *C) { return strings.Join(list, " ") } - result, err := s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}}, false, nil, 0, nil) + result, err := s.il.Filter(FilterOptions{Queries: []PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}}}) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386") - result, err = s.il.Filter([]PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &PkgQuery{"dpkg", "1.7", "source"}, - &PkgQuery{"dpkg", "1.8", "amd64"}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{Queries: []PackageQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &PkgQuery{"dpkg", "1.7", "source"}, + &PkgQuery{"dpkg", "1.8", "amd64"}}}) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_source") - result, err = s.il.Filter([]PackageQuery{ + result, err = s.il.Filter(FilterOptions{Queries: []PackageQuery{ &DependencyQuery{Dep: Dependency{Pkg: "app"}}, &DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}, &DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.0"}}, &DependencyQuery{Dep: Dependency{Pkg: "xyz"}}, - &DependencyQuery{Dep: Dependency{Pkg: "aa", Relation: VersionGreater, Version: "3.0"}}}, false, nil, 0, nil) + &DependencyQuery{Dep: Dependency{Pkg: "aa", Relation: VersionGreater, Version: "3.0"}}}}) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source") - result, err = s.il.Filter([]PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "app", Architecture: "i386"}}}, true, NewPackageList(), 0, []string{"i386"}) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "app", Architecture: "i386"}}}, + WithDependencies: true, + Source: NewPackageList(), + Architectures: []string{"i386"}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386") - result, err = s.il.Filter([]PackageQuery{ - &DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "0.9"}}, - &DependencyQuery{Dep: Dependency{Pkg: "lib"}}, - &DependencyQuery{Dep: Dependency{Pkg: "data"}}}, true, NewPackageList(), 0, []string{"i386", "amd64"}) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{ + &DependencyQuery{Dep: Dependency{Pkg: "app", Relation: VersionGreaterOrEqual, Version: "0.9"}}, + &DependencyQuery{Dep: Dependency{Pkg: "lib"}}, + &DependencyQuery{Dep: Dependency{Pkg: "data"}}}, + WithDependencies: true, + Source: NewPackageList(), + Architectures: []string{"i386", "amd64"}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386") - result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, - &DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, + &DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source") - result, err = s.il.Filter([]PackageQuery{&AndQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, - &DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, + &DependencyQuery{Dep: Dependency{Pkg: "dpkg", Relation: VersionGreater, Version: "1.6.1-3"}}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "") - result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, - &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil) + //result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, + // &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, + &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_i386 data_1.1~bp1_all") - result, err = s.il.Filter([]PackageQuery{&AndQuery{&FieldQuery{Field: "Version", Relation: VersionGreaterOrEqual, Value: "1.0"}, - &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{&FieldQuery{Field: "Version", Relation: VersionGreaterOrEqual, Value: "1.0"}, + &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.0_s390 data_1.1~bp1_all") - result, err = s.il.Filter([]PackageQuery{&AndQuery{ - &FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{ + &FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386") - result, err = s.il.Filter([]PackageQuery{&NotQuery{ - &FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&NotQuery{ + &FieldQuery{Field: "$Architecture", Relation: VersionPatternMatch, Value: "i*6"}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.6.1-3_arm dpkg_1.6.1-3_source dpkg_1.7_source libx_1.5_arm") - result, err = s.il.Filter([]PackageQuery{&AndQuery{ - &FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{ + &FieldQuery{Field: "$Architecture", Relation: VersionRegexp, Value: "i.*6", Regexp: regexp.MustCompile("i.*6")}, &PkgQuery{"app", "1.1~bp1", "i386"}}}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "app_1.1~bp1_i386") - result, err = s.il.Filter([]PackageQuery{&AndQuery{ - &FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")}, - &NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}}, - }}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{ + &FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")}, + &NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}}, + }}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386") - result, err = s.il.Filter([]PackageQuery{&AndQuery{ - &NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}}, - &FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")}, - }}, false, nil, 0, nil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&AndQuery{ + &NotQuery{Q: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: "data"}}, + &FieldQuery{Field: "Name", Relation: VersionRegexp, Value: "a", Regexp: regexp.MustCompile("a")}, + }}, + }) c.Check(err, IsNil) c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386") } diff --git a/deb/remote.go b/deb/remote.go index b12f0dd3a..6845e0d8e 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -596,7 +596,14 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu emptyList.PrepareIndex() oldLen = repo.packageList.Len() - repo.packageList, err = repo.packageList.FilterWithProgress([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures, progress) + repo.packageList, err = repo.packageList.Filter(FilterOptions{ + Queries: []PackageQuery{filterQuery}, + WithDependencies: repo.FilterWithDeps, + Source: emptyList, + DependencyOptions: dependencyOptions, + Architectures: repo.Architectures, + Progress: progress, + }) if repo.packageList != nil { newLen = repo.packageList.Len() } From 0c76677b16aabdbdc3b109c19bd2bc0e5fb4aa26 Mon Sep 17 00:00:00 2001 From: Gordian Schoenherr Date: Tue, 26 Nov 2024 16:19:46 +0900 Subject: [PATCH 2/4] Fix -with-sources not downloading differently named sources Such as e.g. downloading 'glibc' when the sources for 'libc6' are requested. --- deb/list.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ deb/package.go | 2 +- deb/remote.go | 1 + 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/deb/list.go b/deb/list.go index 9f3f67a9b..daca4372d 100644 --- a/deb/list.go +++ b/deb/list.go @@ -2,6 +2,7 @@ package deb import ( "fmt" + "regexp" "sort" "strings" @@ -507,12 +508,24 @@ func (l *PackageList) Search(dep Dependency, allMatches bool, searchProvided boo type FilterOptions struct { Queries []PackageQuery WithDependencies bool + WithSources bool // Source packages correspond to binary packages are included Source *PackageList DependencyOptions int Architectures []string Progress aptly.Progress // set to non-nil to report progress } +// SourceRegex is a regular expression to match source package names. +// > In a binary package control file [...], the source package name may be followed by a version number in +// > parentheses. This version number may be omitted [...] if it has the same value as the Version field of +// > the binary package in question. +// > [...] +// > Package names (both source and binary, see Package) must consist only of lower case letters (a-z), +// > digits (0-9), plus (+) and minus (-) signs, and periods (.). +// > They must be at least two characters long and must start with an alphanumeric character. +// -- https://www.debian.org/doc/debian-policy/ch-controlfields.html#s-f-source +var SourceRegex = regexp.MustCompile(`^([a-z0-9][-+.a-z0-9]+)(?:\s+\(([^)]+)\))?$`) + // Filter filters package index by specified queries (ORed together), possibly pulling dependencies func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) { if !l.indexed { @@ -524,6 +537,37 @@ func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) { for _, query := range options.Queries { _ = result.Append(query.Query(l)) } + // The above loop already finds source packages that are named equal to their binary package, but we still need + // to account for those that are named differently. + if options.WithSources { + sourceQueries := make([]PackageQuery, 0) + for _, pkg := range result.packages { + if pkg.Source == "" { + continue + } + matches := SourceRegex.FindStringSubmatch(pkg.Source) + if matches == nil { + return nil, fmt.Errorf("invalid Source field: %s", pkg.Source) + } + sourceName := matches[1] + if sourceName == pkg.Name { + continue + } + sourceVersion := pkg.Version + if matches[2] != "" { + sourceVersion = matches[2] + } + sourceQueries = append(sourceQueries, &DependencyQuery{Dependency{ + Pkg: sourceName, + Version: sourceVersion, + Relation: VersionEqual, + Architecture: ArchitectureSource, + }}) + } + for _, query := range sourceQueries { + _ = result.Append(query.Query(l)) + } + } if options.WithDependencies { added := result.Len() diff --git a/deb/package.go b/deb/package.go index cf6a98ca0..fc4c9b68c 100644 --- a/deb/package.go +++ b/deb/package.go @@ -450,7 +450,7 @@ func (p *Package) GetArchitecture() string { return p.Architecture } -// GetDependencies compiles list of dependncies by flags from options +// GetDependencies compiles list of dependencies by flags from options func (p *Package) GetDependencies(options int) (dependencies []string) { deps := p.Deps() diff --git a/deb/remote.go b/deb/remote.go index 6845e0d8e..9f89f905b 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -600,6 +600,7 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu Queries: []PackageQuery{filterQuery}, WithDependencies: repo.FilterWithDeps, Source: emptyList, + WithSources: repo.DownloadSources, DependencyOptions: dependencyOptions, Architectures: repo.Architectures, Progress: progress, From ef6815222cfd05498d65622fa980c64c4cb6641b Mon Sep 17 00:00:00 2001 From: Gordian Schoenherr Date: Tue, 26 Nov 2024 16:43:51 +0900 Subject: [PATCH 3/4] Add unit tests for filtering with source packages --- deb/list_test.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/deb/list_test.go b/deb/list_test.go index e13cd4f24..efff77a36 100644 --- a/deb/list_test.go +++ b/deb/list_test.go @@ -381,8 +381,6 @@ func (s *PackageListSuite) TestFilter(c *C) { c.Check(err, IsNil) c.Check(plString(result), Equals, "") - //result, err = s.il.Filter([]PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, - // &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, false, nil, 0, nil) result, err = s.il.Filter(FilterOptions{ Queries: []PackageQuery{&OrQuery{&PkgQuery{"app", "1.1~bp1", "i386"}, &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: "s390"}}}, @@ -435,6 +433,31 @@ func (s *PackageListSuite) TestFilter(c *C) { }) c.Check(err, IsNil) c.Check(plString(result), Equals, "aa_2.0-1_i386 app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 mailer_3.5.8_i386") + + // Different version for the source package + for _, p := range s.sourcePackages { + c.Check(s.il.Add(p), IsNil) + } + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "lib"}}}, + Architectures: []string{"i386", "amd64"}, + WithSources: true, + }) + c.Check(err, IsNil) + c.Check(plString(result), Equals, "lib_0.9_source lib_1.0_i386") + + // Different name for the source package + err = s.il.Add(&Package{Name: "glibc", Version: "1.0", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}}) + c.Check(err, IsNil) + err = s.il.Add(&Package{Name: "libc1", Version: "1.0", Architecture: "i386", Source: "glibc", deps: &PackageDependencies{}}) + c.Check(err, IsNil) + result, err = s.il.Filter(FilterOptions{ + Queries: []PackageQuery{&DependencyQuery{Dep: Dependency{Pkg: "libc1"}}}, + Architectures: []string{"i386"}, + WithSources: true, + }) + c.Check(err, IsNil) + c.Check(plString(result), Equals, "glibc_1.0_source libc1_1.0_i386") } func (s *PackageListSuite) TestVerifyDependencies(c *C) { From 32e458c01ad69018240231336bdccc99f2a63ae9 Mon Sep 17 00:00:00 2001 From: Gordian Schoenherr Date: Mon, 9 Dec 2024 14:16:42 +0900 Subject: [PATCH 4/4] Fix failing system test The fix of the -with-filter flag causes the following previously missing source files to be downloaded, so I updated the test file. ``` rkward_0.7.5-1~bullseyecran.0.debian.tar.xz rkward_0.7.5-1~bullseyecran.0.dsc rkward_0.7.5.orig.tar.gz rpy2_3.5.12-1~bullseyecran.0.debian.tar.xz rpy2_3.5.12-1~bullseyecran.0.dsc rpy2_3.5.12.orig.tar.gz ``` --- system/t04_mirror/UpdateMirror10Test_gold | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/system/t04_mirror/UpdateMirror10Test_gold b/system/t04_mirror/UpdateMirror10Test_gold index 54251a64e..3b4ab6f29 100644 --- a/system/t04_mirror/UpdateMirror10Test_gold +++ b/system/t04_mirror/UpdateMirror10Test_gold @@ -2,7 +2,7 @@ Applying filter... Building download queue... -Download queue: 11 items (39.66 MiB) +Download queue: 17 items (47.22 MiB) Downloading & parsing package files... Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/InRelease Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/Packages.bz2 @@ -16,10 +16,16 @@ Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/d Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward-data_0.7.5-1~bullseyecran.0_all.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward-dbgsym_0.7.5-1~bullseyecran.0_amd64.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward-dbgsym_0.7.5-1~bullseyecran.0_i386.deb +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0.debian.tar.xz +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0.dsc Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0_amd64.deb Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5-1~bullseyecran.0_i386.deb +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rkward_0.7.5.orig.tar.gz +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rpy2_3.5.12-1~bullseyecran.0.debian.tar.xz +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rpy2_3.5.12-1~bullseyecran.0.dsc +Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/rpy2_3.5.12.orig.tar.gz Mirror `flat-src` has been updated successfully. -Packages filtered: 110 -> 11. +Packages filtered: 110 -> 13. gpgv: using RSA key 7BA040A510E4E66ED3743EC1B8F25A8A73EACF41 gpgv: Good signature from "Johannes Ranke " gpgv: Signature made Thu Nov 2 07:43:52 2023 UTC \ No newline at end of file