diff --git a/cmd/podman/common/build.go b/cmd/podman/common/build.go
index 25be5154e8..c34055615c 100644
--- a/cmd/podman/common/build.go
+++ b/cmd/podman/common/build.go
@@ -113,6 +113,9 @@ func DefineBuildFlags(cmd *cobra.Command, buildOpts *BuildFlagsWrapper, isFarmBu
 	completion.CompleteCommandFlags(cmd, fromAndBudFlagsCompletions)
 	flags.SetNormalizeFunc(buildahCLI.AliasFlags)
 	if registry.IsRemote() {
+		// Unset the isolation default as we never want to send this over the API
+		// as it can be wrong (root vs rootless).
+		_ = flags.Lookup("isolation").Value.Set("")
 		_ = flags.MarkHidden("disable-content-trust")
 		_ = flags.MarkHidden("sign-by")
 		_ = flags.MarkHidden("signature-policy")