dnf: mark packages

Package marking is enabled per image type and propagated down to the OS
pipeline. The package marking information is output by `dnf-json` based
on what `dnf` gave as reasons during the depsolve transactions. This is
stored on the `rpmmd.PackageSpec`.

Note that we currently output 'user' for any 'group'-installed package.

There's also a bit of complication here; as `dnf-json` does chain
depsolving it marks all packages from the previous transaction for
`install`. This leads to all packages from the previous transaction
being marked as `user`-installed. To circumvent this we keep track of
the reasons a package was selected and the oldest one wins.

dnf-json

@@ -1,5 +1,6 @@
 #!/usr/bin/python3
 # pylint: disable=invalid-name
+# pylint: disable=fixme
 
 """
 A JSON-based interface for depsolving using DNF.
@@ -14,9 +15,14 @@ import tempfile
 from datetime import datetime
 
 import dnf
+import libdnf
 import hawkey
 
 
+def package_nevra(pkg):
+    return f"{pkg.name}-{pkg.evr}.{pkg.arch}"
+
+
 class Solver():
 
     # pylint: disable=too-many-arguments
@@ -199,6 +205,20 @@ class Solver():
     def depsolve(self, transactions):
         last_transaction = []
 
+        # Here's a tricky bit; because we do a chain of depsolves and select
+        # all packages from the previous depsolve with `package_install`
+        # `dnf` thinks they are all user-requested (as if `dnf install
+        # $package`)
+
+        # The reasons describe *why* the package was added to the transaction.
+        # Either 'user' (user-selected), 'group' (group-selected), or 'dependency'
+        # when neither of the above ('weak-dependency' also exists).
+
+        # So we have a separate dict keyed on NEVRA containing (reason,
+        # reason_group). If the NEVRA already exists in there we keep it,
+        # restoring the original reason the package was selected.
+        historical_reasons = {}
+
         for idx, transaction in enumerate(transactions):
             self.base.reset(goal=True)
             self.base.sack.reset_excludes()
@@ -226,10 +246,30 @@ class Solver():
                 # a stable order
                 if tsi.action not in dnf.transaction.FORWARD_ACTIONS:
                     continue
+
                 last_transaction.append(tsi.pkg)
 
+                # store the reason the package was added to the historical reasons,
+                # this lets us uncover the original reason
+                if package_nevra(tsi.pkg) not in historical_reasons:
+                    # XXX: Note that `get_group()` is currently empty:
+                    # XXX: https://github.com/rpm-software-management/libdnf/issues/1608
+                    historical_reasons[package_nevra(tsi.pkg)] = (
+                        libdnf.transaction.TransactionItemReasonToString(tsi.reason),
+                        tsi.get_group()
+                    )
+
         dependencies = []
         for package in last_transaction:
+            # get the initial reason for the package
+            reason, reason_group = historical_reasons[package_nevra(package)]
+
+            # XXX since the group reason is always empty we will mark all
+            # XXX 'group' packages as 'user' for now we're doing that here
+            # XXX in `dnf-json` so other code won't have to change later
+            # XXX when the group becomes available
+            reason = "user" if reason == "group" else reason
+
             dependencies.append({
                 "name": package.name,
                 "epoch": package.epoch,
@@ -242,7 +282,9 @@ class Solver():
                 "checksum": (
                     f"{hawkey.chksum_name(package.chksum[0])}:"
                     f"{package.chksum[1].hex()}"
-                )
+                ),
+                "reason": reason,
+                "reason_group": reason_group,
             })
 
         return dependencies

internal/dnfjson/dnfjson.go

@@ -458,6 +458,10 @@ func (pkgs packageSpecs) toRPMMD(repos map[string]rpmmd.RepoConfig) []rpmmd.Pack
 		rpmDependencies[i].Arch = dep.Arch
 		rpmDependencies[i].RemoteLocation = dep.RemoteLocation
 		rpmDependencies[i].Checksum = dep.Checksum
+
+		rpmDependencies[i].Reason = dep.Reason
+		rpmDependencies[i].ReasonGroup = dep.ReasonGroup
+
 		if repo.CheckGPG != nil {
 			rpmDependencies[i].CheckGPG = *repo.CheckGPG
 		}
@@ -554,6 +558,8 @@ type PackageSpec struct {
 	RemoteLocation string `json:"remote_location,omitempty"`
 	Checksum       string `json:"checksum,omitempty"`
 	Secrets        string `json:"secrets,omitempty"`
+	Reason         string `json:"reason,omitempty"`
+	ReasonGroup    string `json:"reason_group,omitempty"`
 }
 
 // dnf-json error structure

pkg/distro/fedora/images.go

@@ -223,6 +223,13 @@ func liveImage(workload workload.Workload,
 
 	img.Filename = t.Filename()
 
+	distro := t.Arch().Distro()
+
+	// We start marking packages in the dnf state database in Fedora 39 and up
+	if !common.VersionLessThan(distro.Releasever(), "39") {
+		img.MarkPackages = true
+	}
+
 	return img, nil
 }
 
@@ -266,6 +273,11 @@ func liveInstallerImage(workload workload.Workload,
 		img.AdditionalKernelOpts = []string{"inst.webui"}
 	}
 
+	// We start marking packages in the dnf state database in Fedora 39 and up
+	if !common.VersionLessThan(distro.Releasever(), "39") {
+		img.MarkPackages = true
+	}
+
 	img.Platform = t.platform
 	img.Workload = workload
 	img.ExtraBasePackages = packageSets[installerPkgsKey]

pkg/image/anaconda_live_installer.go

@@ -30,6 +30,8 @@ type AnacondaLiveInstaller struct {
 	OSVersion     string
 	Release       string
 
+	MarkPackages bool
+
 	Filename string
 
 	AdditionalKernelOpts []string

pkg/image/live.go

@@ -25,6 +25,7 @@ type LiveImage struct {
 	Workload         workload.Workload
 	Filename         string
 	Compression      string
+	MarkPackages     bool
 	ForceSize        *bool
 	PartTool         osbuild.PartTool
 
@@ -57,6 +58,7 @@ func (img *LiveImage) InstantiateManifest(m *manifest.Manifest,
 	osPipeline.OSProduct = img.OSProduct
 	osPipeline.OSVersion = img.OSVersion
 	osPipeline.OSNick = img.OSNick
+	osPipeline.MarkPackages = img.MarkPackages
 
 	imagePipeline := manifest.NewRawImage(m, buildPipeline, osPipeline)
 	imagePipeline.PartTool = img.PartTool

pkg/manifest/os.go

@@ -155,6 +155,8 @@ type OS struct {
 	containerSpecs   []container.Spec
 	ostreeParentSpec *ostree.CommitSpec
 
+	MarkPackages bool
+
 	platform  platform.Platform
 	kernelVer string
 
@@ -317,7 +319,7 @@ func (p *OS) getContainerSpecs() []container.Spec {
 }
 
 func (p *OS) serializeStart(packages []rpmmd.PackageSpec, containers []container.Spec, commits []ostree.CommitSpec) {
-	if len(p.packageSpecs) > 0 {
+	if len(p.getPackageSpecs()) > 0 {
 		panic("double call to serializeStart()")
 	}
 
@@ -331,12 +333,12 @@ func (p *OS) serializeStart(packages []rpmmd.PackageSpec, containers []container
 	}
 
 	if p.KernelName != "" {
-		p.kernelVer = rpmmd.GetVerStrFromPackageSpecListPanic(p.packageSpecs, p.KernelName)
+		p.kernelVer = rpmmd.GetVerStrFromPackageSpecListPanic(p.getPackageSpecs(), p.KernelName)
 	}
 }
 
 func (p *OS) serializeEnd() {
-	if len(p.packageSpecs) == 0 {
+	if len(p.getPackageSpecs()) == 0 {
 		panic("serializeEnd() call when serialization not in progress")
 	}
 	p.kernelVer = ""
@@ -346,7 +348,7 @@ func (p *OS) serializeEnd() {
 }
 
 func (p *OS) serialize() osbuild.Pipeline {
-	if len(p.packageSpecs) == 0 {
+	if len(p.getPackageSpecs()) == 0 {
 		panic("serialization not started")
 	}
 
@@ -373,7 +375,12 @@ func (p *OS) serialize() osbuild.Pipeline {
 		rpmOptions.OSTreeBooted = common.ToPtr(true)
 		rpmOptions.DBPath = "/usr/share/rpm"
 	}
-	pipeline.AddStage(osbuild.NewRPMStage(rpmOptions, osbuild.NewRpmStageSourceFilesInputs(p.packageSpecs)))
+
+	pipeline.AddStage(osbuild.NewRPMStage(rpmOptions, osbuild.NewRpmStageSourceFilesInputs(p.getPackageSpecs())))
+
+	if p.MarkPackages {
+		pipeline.AddStage(osbuild.NewDNFMarkStageFromPackageSpecs(p.getPackageSpecs()))
+	}
 
 	if !p.NoBLS {
 		// If the /boot is on a separate partition, the prefix for the BLS stage must be ""
@@ -608,7 +615,7 @@ func (p *OS) serialize() osbuild.Pipeline {
 					Nick:    p.OSNick,
 				}
 
-				_, err := rpmmd.GetVerStrFromPackageSpecList(p.packageSpecs, "dracut-config-rescue")
+				_, err := rpmmd.GetVerStrFromPackageSpecList(p.getPackageSpecs(), "dracut-config-rescue")
 				hasRescue := err == nil
 				bootloader = osbuild.NewGrub2LegacyStage(
 					osbuild.NewGrub2LegacyStageOptions(

pkg/osbuild/dnf_mark_stage.go

@@ -0,0 +1,56 @@
+package osbuild
+
+import (
+	"github.com/osbuild/images/pkg/rpmmd"
+)
+
+type DNFMarkStagePackageOptions struct {
+	Name  string `json:"name"`
+	Mark  string `json:"mark"`
+	Group string `json:"group,omitempty"`
+}
+
+type DNFMarkStageOptions struct {
+	Packages []DNFMarkStagePackageOptions `json:"packages"`
+}
+
+func (o DNFMarkStageOptions) isStageOptions() {}
+
+func (o DNFMarkStageOptions) validate() error {
+	return nil
+}
+
+func NewDNFMarkStageOptions(packages []DNFMarkStagePackageOptions) *DNFMarkStageOptions {
+	return &DNFMarkStageOptions{
+		Packages: packages,
+	}
+}
+
+type DNFMarkStage struct {
+}
+
+func NewDNFMarkStage(options *DNFMarkStageOptions) *Stage {
+	if err := options.validate(); err != nil {
+		panic(err)
+	}
+
+	return &Stage{
+		Type:    "org.osbuild.dnf.mark",
+		Options: options,
+	}
+}
+
+func NewDNFMarkStageFromPackageSpecs(packageSpecs []rpmmd.PackageSpec) *Stage {
+	var packages []DNFMarkStagePackageOptions
+
+	for _, ps := range packageSpecs {
+		packages = append(packages, DNFMarkStagePackageOptions{
+			Name: ps.Name,
+			Mark: ps.Reason,
+		})
+	}
+
+	options := NewDNFMarkStageOptions(packages)
+
+	return NewDNFMarkStage(options)
+}

pkg/rpmmd/repository.go

@@ -167,6 +167,8 @@ type PackageSpec struct {
 	Secrets        string `json:"secrets,omitempty"`
 	CheckGPG       bool   `json:"check_gpg,omitempty"`
 	IgnoreSSL      bool   `json:"ignore_ssl,omitempty"`
+	Reason         string `json:"reason,omitempty"`
+	ReasonGroup    string `json:"reason_group,omitempty"`
 }
 
 type PackageSource struct {