Merge pull request #24305 from Luap99/quadlet-pasta

quadlet: make user units wait for network

.gitignore

@@ -4,7 +4,7 @@
 /build/
 /conmon/
 contrib/spec/podman.spec
-contrib/systemd/*/*.service
+contrib/systemd/system/*.service
 *.coverprofile
 coverprofile
 /.coverage

Makefile

@@ -982,7 +982,7 @@ install.docker-full: install.docker install.docker-docs
 
 .PHONY: install.systemd
 ifneq (,$(findstring systemd,$(BUILDTAGS)))
-PODMAN_UNIT_FILES = contrib/systemd/auto-update/podman-auto-update.service \
+PODMAN_GENERATED_UNIT_FILES = contrib/systemd/system/podman-auto-update.service \
 		    contrib/systemd/system/podman.service \
 		    contrib/systemd/system/podman-restart.service \
 		    contrib/systemd/system/[email protected] \
@@ -992,25 +992,18 @@ PODMAN_UNIT_FILES = contrib/systemd/auto-update/podman-auto-update.service \
 	sed -e 's;@@PODMAN@@;$(BINDIR)/podman;g' $< >$@.tmp.$$ \
 		&& mv -f $@.tmp.$$ $@
 
-install.systemd: $(PODMAN_UNIT_FILES)
+install.systemd: $(PODMAN_GENERATED_UNIT_FILES)
 	install ${SELINUXOPT} -m 755 -d $(DESTDIR)${SYSTEMDDIR}  $(DESTDIR)${USERSYSTEMDDIR}
-	# User services
-	install ${SELINUXOPT} -m 644 contrib/systemd/auto-update/podman-auto-update.service $(DESTDIR)${USERSYSTEMDDIR}/podman-auto-update.service
-	install ${SELINUXOPT} -m 644 contrib/systemd/auto-update/podman-auto-update.timer $(DESTDIR)${USERSYSTEMDDIR}/podman-auto-update.timer
-	install ${SELINUXOPT} -m 644 contrib/systemd/system/podman.socket $(DESTDIR)${USERSYSTEMDDIR}/podman.socket
-	install ${SELINUXOPT} -m 644 contrib/systemd/system/podman.service $(DESTDIR)${USERSYSTEMDDIR}/podman.service
-	install ${SELINUXOPT} -m 644 contrib/systemd/system/podman-restart.service $(DESTDIR)${USERSYSTEMDDIR}/podman-restart.service
-	install ${SELINUXOPT} -m 644 contrib/systemd/system/[email protected] $(DESTDIR)${USERSYSTEMDDIR}/[email protected]
-	install ${SELINUXOPT} -m 644 contrib/systemd/system/podman-clean-transient.service $(DESTDIR)${USERSYSTEMDDIR}/podman-clean-transient.service
-	# System services
-	install ${SELINUXOPT} -m 644 contrib/systemd/auto-update/podman-auto-update.service $(DESTDIR)${SYSTEMDDIR}/podman-auto-update.service
-	install ${SELINUXOPT} -m 644 contrib/systemd/auto-update/podman-auto-update.timer $(DESTDIR)${SYSTEMDDIR}/podman-auto-update.timer
-	install ${SELINUXOPT} -m 644 contrib/systemd/system/podman.socket $(DESTDIR)${SYSTEMDDIR}/podman.socket
-	install ${SELINUXOPT} -m 644 contrib/systemd/system/podman.service $(DESTDIR)${SYSTEMDDIR}/podman.service
-	install ${SELINUXOPT} -m 644 contrib/systemd/system/podman-restart.service $(DESTDIR)${SYSTEMDDIR}/podman-restart.service
-	install ${SELINUXOPT} -m 644 contrib/systemd/system/[email protected] $(DESTDIR)${SYSTEMDDIR}/[email protected]
-	install ${SELINUXOPT} -m 644 contrib/systemd/system/podman-clean-transient.service $(DESTDIR)${SYSTEMDDIR}/podman-clean-transient.service
-	rm -f $(PODMAN_UNIT_FILES)
+	for unit in $^ \
+				contrib/systemd/system/podman-auto-update.timer \
+				contrib/systemd/system/podman.socket; do \
+		install ${SELINUXOPT} -m 644 $$unit $(DESTDIR)${USERSYSTEMDDIR}/$$(basename $$unit); \
+		install ${SELINUXOPT} -m 644 $$unit $(DESTDIR)${SYSTEMDDIR}/$$(basename $$unit); \
+	done
+	# Important this unit should only be installed for the user session and is thus not added to the loop above.
+	install ${SELINUXOPT} -m 644 contrib/systemd/user/podman-user-wait-network-online.service \
+		$(DESTDIR)${USERSYSTEMDDIR}/podman-user-wait-network-online.service
+	rm -f $^
 else
 install.systemd:
 endif

cmd/quadlet/main.go

@@ -732,16 +732,16 @@ func process() error {
 			service, err = quadlet.ConvertContainer(unit, isUserFlag, unitsInfoMap)
 		case strings.HasSuffix(unit.Filename, ".volume"):
 			warnIfAmbiguousName(unit, quadlet.VolumeGroup)
-			service, err = quadlet.ConvertVolume(unit, unit.Filename, unitsInfoMap)
+			service, err = quadlet.ConvertVolume(unit, unit.Filename, unitsInfoMap, isUserFlag)
 		case strings.HasSuffix(unit.Filename, ".kube"):
 			service, err = quadlet.ConvertKube(unit, unitsInfoMap, isUserFlag)
 		case strings.HasSuffix(unit.Filename, ".network"):
-			service, err = quadlet.ConvertNetwork(unit, unit.Filename, unitsInfoMap)
+			service, err = quadlet.ConvertNetwork(unit, unit.Filename, unitsInfoMap, isUserFlag)
 		case strings.HasSuffix(unit.Filename, ".image"):
 			warnIfAmbiguousName(unit, quadlet.ImageGroup)
-			service, err = quadlet.ConvertImage(unit, unitsInfoMap)
+			service, err = quadlet.ConvertImage(unit, unitsInfoMap, isUserFlag)
 		case strings.HasSuffix(unit.Filename, ".build"):
-			service, err = quadlet.ConvertBuild(unit, unitsInfoMap)
+			service, err = quadlet.ConvertBuild(unit, unitsInfoMap, isUserFlag)
 		case strings.HasSuffix(unit.Filename, ".pod"):
 			service, err = quadlet.ConvertPod(unit, unit.Filename, unitsInfoMap, isUserFlag)
 		default:

contrib/systemd/user/podman-auto-update.service.in

@@ -0,0 +1 @@
+../system/podman-auto-update.service.in

contrib/systemd/user/podman-auto-update.timer

@@ -0,0 +1 @@
+../system/podman-auto-update.timer

contrib/systemd/user/podman-clean-transient.service.in

@@ -0,0 +1 @@
+../system/podman-clean-transient.service.in

contrib/systemd/user/podman-kube@.service.in

@@ -0,0 +1 @@
+../system/[email protected]

contrib/systemd/user/podman-restart.service.in

@@ -0,0 +1 @@
+../system/podman-restart.service.in

contrib/systemd/user/podman-user-wait-network-online.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Wait for system level network-online.target as user.
+Documentation=https://github.com/containers/podman/issues/22197
+Documentation=man:podman-systemd.unit(5)
+
+[Service]
+Type=oneshot
+# Set a timeout as by default oneshot does not have one and in case network-online.target
+# never comes online we do not want to block forever, 90s is the default systemd unit timeout.
+TimeoutStartSec=90s
+ExecStart=sh -c 'until systemctl is-active network-online.target; do sleep 0.5; done'
+RemainAfterExit=yes

contrib/systemd/user/podman.service.in

@@ -0,0 +1 @@
+../system/podman.service.in

contrib/systemd/user/podman.socket

@@ -0,0 +1 @@
+../system/podman.socket

docs/source/markdown/podman-systemd.unit.5.md

@@ -238,9 +238,14 @@ that limit the output to only the units you are debugging.
 
 ### Implicit network dependencies
 
-In the case of Container, Image and Build units, Quadlet will add dependencies on the `network-online.target`
-by adding `After=` and `Wants=` properties to the unit. This is to ensure that the network is reachable if
-an image needs to be pulled.
+Quadlet will add dependencies on the `network-online.target` (as root) or `podman-user-wait-network-online.service`
+(as user) by adding `After=` and `Wants=` properties to the unit. This is to ensure that the network is reachable
+if an image needs to be pulled and by the time the container is started.
+
+The special case `podman-user-wait-network-online.service` unit is needed as user because user units are unable to wait
+for system (root) units so `network-online.target` doesn't do anything there and is instead ignored. As this caused
+a significant amount of issues we decided to work around this with our own special purpose unit that simply checks if
+the `network-online.target` unit is active with `systemctl is-active network-online.target`.
 
 This behavior can be disabled by adding `DefaultDependencies=false` in the `Quadlet` section.
 
@@ -1791,11 +1796,6 @@ exists on the host, pulling it if needed.
 Using image units allows containers and volumes to depend on images being automatically pulled. This is
 particularly interesting when using special options to control image pulls.
 
-Note: The generated service have a dependency on `network-online.target` assuring the network is reachable if
-an image needs to be pulled.
-If the image service needs to run without available network (e.g. early in boot), the requirement can be
-overridden simply by adding an empty `After=` in the unit file. This will unset all previously set After's.
-
 Valid options for `[Image]` are listed below:
 
 | **[Image] options**                    | **podman image pull equivalent**                 |
@@ -1936,7 +1936,8 @@ Valid options for `[Quadlet]` are listed below:
 
 Add Quadlet's default network dependencies to the unit (default is `true`).
 
-When set to false, Quadlet will **not** add a dependency (After=, Wants=) to `network-online.target` to the generated unit.
+When set to false, Quadlet will **not** add a dependency (After=, Wants=) to
+`network-online.target`/`podman-user-wait-network-online.service` to the generated unit.
 
 ## EXAMPLES
 

pkg/systemd/quadlet/quadlet.go

@@ -524,13 +524,7 @@ func ConvertContainer(container *parser.UnitFile, isUser bool, unitsInfoMap map[
 	service := container.Dup()
 	service.Filename = unitInfo.ServiceFileName()
 
-	// Add a dependency on network-online.target so the image pull does not happen
-	// before network is ready
-	// https://github.com/containers/podman/issues/21873
-	if service.LookupBooleanWithDefault(QuadletGroup, KeyDefaultDependencies, true) {
-		service.PrependUnitLine(UnitGroup, "After", "network-online.target")
-		service.PrependUnitLine(UnitGroup, "Wants", "network-online.target")
-	}
+	addDefaultDependencies(service, isUser)
 
 	if container.Path != "" {
 		service.Add(UnitGroup, "SourcePath", container.Path)
@@ -888,7 +882,7 @@ func ConvertContainer(container *parser.UnitFile, isUser bool, unitsInfoMap map[
 // The original Network group is kept around as X-Network.
 // Also returns the canonical network name, either auto-generated or user-defined via the
 // NetworkName key-value.
-func ConvertNetwork(network *parser.UnitFile, name string, unitsInfoMap map[string]*UnitInfo) (*parser.UnitFile, error) {
+func ConvertNetwork(network *parser.UnitFile, name string, unitsInfoMap map[string]*UnitInfo, isUser bool) (*parser.UnitFile, error) {
 	unitInfo, ok := unitsInfoMap[network.Filename]
 	if !ok {
 		return nil, fmt.Errorf("internal error while processing network %s", network.Filename)
@@ -897,6 +891,8 @@ func ConvertNetwork(network *parser.UnitFile, name string, unitsInfoMap map[stri
 	service := network.Dup()
 	service.Filename = unitInfo.ServiceFileName()
 
+	addDefaultDependencies(service, isUser)
+
 	if network.Path != "" {
 		service.Add(UnitGroup, "SourcePath", network.Path)
 	}
@@ -998,7 +994,7 @@ func ConvertNetwork(network *parser.UnitFile, name string, unitsInfoMap map[stri
 // The original Volume group is kept around as X-Volume.
 // Also returns the canonical volume name, either auto-generated or user-defined via the VolumeName
 // key-value.
-func ConvertVolume(volume *parser.UnitFile, name string, unitsInfoMap map[string]*UnitInfo) (*parser.UnitFile, error) {
+func ConvertVolume(volume *parser.UnitFile, name string, unitsInfoMap map[string]*UnitInfo, isUser bool) (*parser.UnitFile, error) {
 	unitInfo, ok := unitsInfoMap[volume.Filename]
 	if !ok {
 		return nil, fmt.Errorf("internal error while processing network %s", volume.Filename)
@@ -1007,6 +1003,8 @@ func ConvertVolume(volume *parser.UnitFile, name string, unitsInfoMap map[string
 	service := volume.Dup()
 	service.Filename = unitInfo.ServiceFileName()
 
+	addDefaultDependencies(service, isUser)
+
 	if volume.Path != "" {
 		service.Add(UnitGroup, "SourcePath", volume.Path)
 	}
@@ -1148,6 +1146,8 @@ func ConvertKube(kube *parser.UnitFile, unitsInfoMap map[string]*UnitInfo, isUse
 	service := kube.Dup()
 	service.Filename = unitInfo.ServiceFileName()
 
+	addDefaultDependencies(service, isUser)
+
 	if kube.Path != "" {
 		service.Add(UnitGroup, "SourcePath", kube.Path)
 	}
@@ -1282,7 +1282,7 @@ func ConvertKube(kube *parser.UnitFile, unitsInfoMap map[string]*UnitInfo, isUse
 	return service, nil
 }
 
-func ConvertImage(image *parser.UnitFile, unitsInfoMap map[string]*UnitInfo) (*parser.UnitFile, error) {
+func ConvertImage(image *parser.UnitFile, unitsInfoMap map[string]*UnitInfo, isUser bool) (*parser.UnitFile, error) {
 	unitInfo, ok := unitsInfoMap[image.Filename]
 	if !ok {
 		return nil, fmt.Errorf("internal error while processing network %s", image.Filename)
@@ -1291,13 +1291,7 @@ func ConvertImage(image *parser.UnitFile, unitsInfoMap map[string]*UnitInfo) (*p
 	service := image.Dup()
 	service.Filename = unitInfo.ServiceFileName()
 
-	// Add a dependency on network-online.target so the image pull does not happen
-	// before network is ready
-	// https://github.com/containers/podman/issues/21873
-	if service.LookupBooleanWithDefault(QuadletGroup, KeyDefaultDependencies, true) {
-		service.PrependUnitLine(UnitGroup, "After", "network-online.target")
-		service.PrependUnitLine(UnitGroup, "Wants", "network-online.target")
-	}
+	addDefaultDependencies(service, isUser)
 
 	if image.Path != "" {
 		service.Add(UnitGroup, "SourcePath", image.Path)
@@ -1365,7 +1359,7 @@ func ConvertImage(image *parser.UnitFile, unitsInfoMap map[string]*UnitInfo) (*p
 	return service, nil
 }
 
-func ConvertBuild(build *parser.UnitFile, unitsInfoMap map[string]*UnitInfo) (*parser.UnitFile, error) {
+func ConvertBuild(build *parser.UnitFile, unitsInfoMap map[string]*UnitInfo, isUser bool) (*parser.UnitFile, error) {
 	unitInfo, ok := unitsInfoMap[build.Filename]
 	if !ok {
 		return nil, fmt.Errorf("internal error while processing network %s", build.Filename)
@@ -1379,13 +1373,7 @@ func ConvertBuild(build *parser.UnitFile, unitsInfoMap map[string]*UnitInfo) (*p
 	service := build.Dup()
 	service.Filename = unitInfo.ServiceFileName()
 
-	// Add a dependency on network-online.target so the image pull does not happen
-	// before network is ready
-	// https://github.com/containers/podman/issues/21873
-	if service.LookupBooleanWithDefault(QuadletGroup, KeyDefaultDependencies, true) {
-		service.PrependUnitLine(UnitGroup, "After", "network-online.target")
-		service.PrependUnitLine(UnitGroup, "Wants", "network-online.target")
-	}
+	addDefaultDependencies(service, isUser)
 
 	/* Rename old Build group to X-Build so that systemd ignores it */
 	service.RenameGroup(BuildGroup, XBuildGroup)
@@ -1550,6 +1538,8 @@ func ConvertPod(podUnit *parser.UnitFile, name string, unitsInfoMap map[string]*
 	service := podUnit.Dup()
 	service.Filename = unitInfo.ServiceFileName()
 
+	addDefaultDependencies(service, isUser)
+
 	if podUnit.Path != "" {
 		service.Add(UnitGroup, "SourcePath", podUnit.Path)
 	}
@@ -2184,3 +2174,22 @@ func addVolumes(quadletUnitFile, serviceUnitFile *parser.UnitFile, groupName str
 
 	return nil
 }
+
+func addDefaultDependencies(service *parser.UnitFile, isUser bool) {
+	// Add a dependency on network-online.target so the image pull container startup
+	// does not happen before network is ready.
+	// https://github.com/containers/podman/issues/21873
+	if service.LookupBooleanWithDefault(QuadletGroup, KeyDefaultDependencies, true) {
+		networkUnit := "network-online.target"
+		// network-online.target only exists as root and user session cannot wait for it
+		// https://github.com/systemd/systemd/issues/3312
+		// Given this is a bad problem with pasta which can fail to start or use the
+		// wrong interface if the network is not fully set up we need to work around
+		// that: https://github.com/containers/podman/issues/22197.
+		if isUser {
+			networkUnit = "podman-user-wait-network-online.service"
+		}
+		service.PrependUnitLine(UnitGroup, "After", networkUnit)
+		service.PrependUnitLine(UnitGroup, "Wants", networkUnit)
+	}
+}

test/e2e/quadlet/basic.build

@@ -1,7 +1,7 @@
 ## assert-podman-final-args-regex /.*/podman-e2e-.*/subtest-.*/quadlet
 ## assert-podman-args "--tag" "localhost/imagename"
-## assert-key-is "Unit" "After" "network-online.target"
-## assert-key-is "Unit" "Wants" "network-online.target"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service"
+## assert-key-is-regex "Unit" "Wants" "network-online.target|podman-user-wait-network-online.service"
 ## assert-key-is "Unit" "RequiresMountsFor" "%t/containers"
 ## assert-key-is-regex "Service" "WorkingDirectory" "/.*/podman-e2e-.*/subtest-.*/quadlet"
 ## assert-key-is "Service" "Type" "oneshot"

test/e2e/quadlet/basic.container

@@ -15,8 +15,8 @@
 ## assert-key-is-regex "Service" "ExecStopPost" "-[/S].*/podman rm -v -f -i --cidfile=%t/%N.cid"
 ## assert-key-is-regex "Service" "ExecStop" ".*/podman rm -v -f -i --cidfile=%t/%N.cid"
 ## assert-key-is "Service" "Environment" "PODMAN_SYSTEMD_UNIT=%n"
-## assert-key-is "Unit" "After" "network-online.target"
-## assert-key-is "Unit" "Wants" "network-online.target"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service"
+## assert-key-is-regex "Unit" "Wants" "network-online.target|podman-user-wait-network-online.service"
 
 [Container]
 Image=localhost/imagename

test/e2e/quadlet/basic.image

@@ -1,6 +1,6 @@
 ## assert-podman-final-args localhost/imagename
-## assert-key-is "Unit" "After" "network-online.target"
-## assert-key-is "Unit" "Wants" "network-online.target"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service"
+## assert-key-is-regex "Unit" "Wants" "network-online.target|podman-user-wait-network-online.service"
 ## assert-key-is "Unit" "RequiresMountsFor" "%t/containers"
 ## assert-key-is "Service" "Type" "oneshot"
 ## assert-key-is "Service" "RemainAfterExit" "yes"

test/e2e/quadlet/build.quadlet.servicename.volume

@@ -1,7 +1,7 @@
 ## assert-podman-args --driver image
 ## assert-podman-args --opt image=localhost/imagename
 ## assert-key-is "Unit" "Requires" "basic.service"
-## assert-key-is "Unit" "After" "basic.service"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service" "basic.service"
 
 [Volume]
 Driver=image

test/e2e/quadlet/build.quadlet.volume

@@ -1,7 +1,7 @@
 ## assert-podman-args --driver image
 ## assert-podman-args --opt image=localhost/imagename
 ## assert-key-is "Unit" "Requires" "basic-build.service"
-## assert-key-is "Unit" "After" "basic-build.service"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service" "basic-build.service"
 
 [Volume]
 Driver=image

test/e2e/quadlet/image.quadlet.servicename.volume

@@ -1,7 +1,7 @@
 ## assert-podman-args --driver image
 ## assert-podman-args --opt image=localhost/imagename
 ## assert-key-is "Unit" "Requires" "basic.service"
-## assert-key-is "Unit" "After" "basic.service"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service" "basic.service"
 
 [Volume]
 Driver=image

test/e2e/quadlet/image.quadlet.volume

@@ -1,7 +1,7 @@
 ## assert-podman-args --driver image
 ## assert-podman-args --opt image=localhost/imagename
 ## assert-key-is "Unit" "Requires" "basic-image.service"
-## assert-key-is "Unit" "After" "basic-image.service"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service" "basic-image.service"
 
 [Volume]
 Driver=image

test/e2e/quadlet/mount.container

@@ -10,7 +10,7 @@ Mount=type=bind,src=/path/on/host,dst=/path/in/container,relabel=shared,U=true
 Mount=type=volume,source=vol1,destination=/path/in/container,ro=true
 ## assert-podman-args-key-val "--mount" "," "type=volume,source=systemd-basic,destination=/path/in/container,ro=true"
 ## assert-key-is "Unit" "Requires" "basic-volume.service"
-## assert-key-is "Unit" "After" "network-online.target" "basic-volume.service"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service" "basic-volume.service"
 Mount=type=volume,source=basic.volume,destination=/path/in/container,ro=true
 ## assert-podman-args-key-val "--mount" "," "type=tmpfs,tmpfs-size=512M,destination=/path/in/container"
 Mount=type=tmpfs,tmpfs-size=512M,destination=/path/in/container

test/e2e/quadlet/mount.servicename.container

@@ -2,5 +2,5 @@
 Image=localhost/imagename
 ## assert-podman-args-key-val "--mount" "," "type=volume,source=test-volume,destination=/path/in/container,ro=true"
 ## assert-key-is "Unit" "Requires" "basic.service"
-## assert-key-is "Unit" "After" "network-online.target" "basic.service"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service" "basic.service"
 Mount=type=volume,source=service-name.volume,destination=/path/in/container,ro=true

test/e2e/quadlet/network.quadlet.build

@@ -1,6 +1,6 @@
 ## assert-podman-args "--network" "systemd-basic"
 ## assert-key-is "Unit" "Requires" "basic-network.service"
-## assert-key-is "Unit" "After" "network-online.target" "basic-network.service"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service" "basic-network.service"
 
 [Build]
 ImageTag=localhost/imagename

test/e2e/quadlet/network.quadlet.container

@@ -1,6 +1,6 @@
 ## assert-podman-args "--network" "systemd-basic"
 ## assert-key-is "Unit" "Requires" "basic-network.service"
-## assert-key-is "Unit" "After" "network-online.target" "basic-network.service"
+## assert-key-is-regex "Unit" "After" "network-online.target|podman-user-wait-network-online.service" "basic-network.service"
 
 [Container]
 Image=localhost/imagename