From f017fcbec4673aa18a6e6ea7c787e2fb8058a782 Mon Sep 17 00:00:00 2001 From: Daniel Golle Date: Fri, 31 May 2024 02:32:07 +0100 Subject: [PATCH] phase1: remove kmods in target packages if archive is enabled OPKG gets confused if kmod packages are present in both, target packages as well as kernel version specific folder. Remove them from target packages to make opkg pick the kmods from kmod archive folder only. We reorder the step by: 1) Move the kmods to dedicated directory 2) Index packages 3) Index kmods 4) Make checksums 5) Extract kmods sums from remote sums 6) Merge the local sums with the remote kmods-only sums This permits to produce a better sha256sums that also reflect the hash of the entire directory. Co-developed-by: Christian Marangi Signed-off-by: Daniel Golle [ rework implementation, sha256sumsclean + --remove-source-files ] Signed-off-by: Christian Marangi --- phase1/master.cfg | 168 +++++++++++++++++++++++++++++----------------- 1 file changed, 107 insertions(+), 61 deletions(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index f869dfc..9709576 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -1218,23 +1218,6 @@ def prepareFactory(target): ) ) - factory.addStep( - ShellCommand( - name="pkgindex", - description="Indexing packages", - descriptionDone="Packages indexed", - command=[ - "make", - Interpolate("-j%(prop:nproc:-1)s"), - "package/index", - "V=s", - "CONFIG_SIGNED_PACKAGES=", - ], - env=MakeEnv(), - haltOnFailure=True, - ) - ) - factory.addStep( ShellCommand( name="images", @@ -1271,17 +1254,6 @@ def prepareFactory(target): ) ) - factory.addStep( - ShellCommand( - name="checksums", - description="Calculating checksums", - descriptionDone="Checksums calculated", - command=["make", "-j1", "checksum", "V=s"], - env=MakeEnv(), - haltOnFailure=True, - ) - ) - factory.addStep( ShellCommand( name="kmoddir", @@ -1303,10 +1275,11 @@ def prepareFactory(target): factory.addStep( ShellCommand( name="kmodprepare", - description="Preparing kmod archive", - descriptionDone="Kmod archive prepared", + description="Moving kmod to archive", + descriptionDone="Kmod moved", command=[ "rsync", + "--remove-source-files", "--include=/kmod-*.ipk", "--include=/kmod-*.apk", "--exclude=*", @@ -1327,6 +1300,23 @@ def prepareFactory(target): ) ) + factory.addStep( + ShellCommand( + name="pkgindex", + description="Indexing packages", + descriptionDone="Packages indexed", + command=[ + "make", + Interpolate("-j%(prop:nproc:-1)s"), + "package/index", + "V=s", + "CONFIG_SIGNED_PACKAGES=", + ], + env=MakeEnv(), + haltOnFailure=True, + ) + ) + factory.addStep( ShellCommand( name="kmodindex", @@ -1350,6 +1340,93 @@ def prepareFactory(target): ) ) + factory.addStep( + ShellCommand( + name="checksums", + description="Calculating checksums", + descriptionDone="Checksums calculated", + command=["make", "-j1", "checksum", "V=s"], + env=MakeEnv(), + haltOnFailure=True, + ) + ) + + # download remote sha256sums to 'target-sha256sums' + factory.addStep( + ShellCommand( + name="target-sha256sums", + description="Fetching remote sha256sums for target", + descriptionDone="Remote sha256sums for target fetched", + command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")] + + rsync_defopts + + [ + Interpolate( + "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", + url=GetRsyncParams.withArgs("bin", "url"), + target=target, + subtarget=subtarget, + prefix=GetVersionPrefix, + ), + "target-sha256sums", + ], + env={ + "RSYNC_PASSWORD": Interpolate( + "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key") + ) + }, + logEnviron=False, + haltOnFailure=False, + flunkOnFailure=False, + warnOnFailure=False, + doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")), + ) + ) + + factory.addStep( + ShellCommand( + name="target-sha256sums_kmodsparse", + description="Extract kmods from remote sha256sums", + descriptionDone="Kmods extracted", + command=[ + "sed", + "/ \\*kmods\\//! d", + "target-sha256sums", + "| tee", + "target-sha256sums-kmods" + ], + haltOnFailure=False, + doStepIf=IsKmodArchiveEnabled, + ) + ) + + factory.addStep( + ShellCommand( + name="mergesha256sum", + description="Merge sha256sums kmods with sha256sums", + descriptionDone="Sha256sums merged", + command=[ + "sort", + "-t \"\"", + "-k 2", + "-u", + Interpolate( + "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", + target=target, + subtarget=subtarget, + ), + "target-sha256sums-kmods", + "-o", + Interpolate( + "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", + target=target, + subtarget=subtarget, + ), + ], + haltOnFailure=False, + doStepIf=IsKmodArchiveEnabled, + ) + ) + # sign factory.addStep( MasterShellCommand( @@ -1508,37 +1585,6 @@ def prepareFactory(target): ) ) - # download remote sha256sums to 'target-sha256sums' - factory.addStep( - ShellCommand( - name="target-sha256sums", - description="Fetching remote sha256sums for target", - descriptionDone="Remote sha256sums for target fetched", - command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")] - + rsync_defopts - + [ - Interpolate( - "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", - url=GetRsyncParams.withArgs("bin", "url"), - target=target, - subtarget=subtarget, - prefix=GetVersionPrefix, - ), - "target-sha256sums", - ], - env={ - "RSYNC_PASSWORD": Interpolate( - "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key") - ) - }, - logEnviron=False, - haltOnFailure=False, - flunkOnFailure=False, - warnOnFailure=False, - doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")), - ) - ) - # build list of files to upload factory.addStep( FileDownload(