From 8473ea70b2bcd7816fd53ff768cd9cdf20119042 Mon Sep 17 00:00:00 2001 From: Jon Carstens Date: Sun, 26 May 2024 18:40:53 -0600 Subject: [PATCH] Build from from rootfs.tar system files Starts support to allow building firmware from a `rootfs.tar` tarball file more efficiently in Elixir, rather than relying on bash scripts to unsquash, combine, and resquash an FS. It also adjusts the build process to allow using EROFS instead of squashfs when making the firmware --- .circleci/config.yml | 2 +- lib/mix/nerves/preflight.ex | 16 +- lib/mix/tasks/firmware.ex | 302 ++++++++++++++++++++++-------------- mix.exs | 5 +- mix.lock | 1 + 5 files changed, 207 insertions(+), 119 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6f8a29b0..dfc27658 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: steps: - run: name: Install system dependencies - command: apk add --no-cache build-base procps + command: apk add --no-cache build-base procps git - checkout - run: name: Install hex, rebar, and nerves_bootstrap diff --git a/lib/mix/nerves/preflight.ex b/lib/mix/nerves/preflight.ex index 48ac1696..aea53cad 100644 --- a/lib/mix/nerves/preflight.ex +++ b/lib/mix/nerves/preflight.ex @@ -22,7 +22,7 @@ defmodule Mix.Nerves.Preflight do # OSX defp check_platform!(:darwin) do ensure_fwup_version!() - ensure_available!("mksquashfs", package: "squashfs") + ensure_fs_tools!() ensure_available!("gstat", package: "gstat (coreutils)") end @@ -35,13 +35,23 @@ defmodule Mix.Nerves.Preflight do defp check_platform!(:wsl) do ensure_fwup_version!() ensure_fwup_version!("fwup.exe") - ensure_available!("mksquashfs", package: "squashfs") + ensure_fs_tools!() end # Non-WSL Linux defp check_platform!(_) do ensure_fwup_version!() - ensure_available!("mksquashfs", package: "squashfs") + ensure_fs_tools!() + end + + defp ensure_fs_tools!() do + fs_type = Application.get_env(:nerves, :firmware, [])[:fs_type] + + if fs_type == :erofs do + ensure_available!("mkfs.erofs", package: "erofs-utils") + else + ensure_available!("mksquashfs", package: "squashfs") + end end @doc """ diff --git a/lib/mix/tasks/firmware.ex b/lib/mix/tasks/firmware.ex index 108f27c8..74c1ca7f 100644 --- a/lib/mix/tasks/firmware.ex +++ b/lib/mix/tasks/firmware.ex @@ -22,23 +22,31 @@ defmodule Mix.Tasks.Firmware do Nerves toolchain (C/C++ crosscompiler) that is used """ use Mix.Task - import Mix.Nerves.Utils - alias Mix.Nerves.Preflight - @default_mksquashfs_flags ["-no-xattrs", "-quiet"] + import Mix.Nerves.Utils, + only: [ + check_nerves_system_is_set!: 0, + check_nerves_toolchain_is_set!: 0, + parse_otp_version: 1, + set_provisioning: 1, + shell: 3 + ] + + import Mix.Nerves.IO, only: [debug_info: 1] + alias Mix.Nerves.Preflight @switches [verbose: :boolean, output: :string] @impl Mix.Task def run(args) do - Preflight.check!() - debug_info("Nerves Firmware Assembler") - {opts, _, _} = OptionParser.parse(args, switches: @switches) + if opts[:verbose], do: System.put_env("NERVES_DEBUG", "1") + debug_info("firmware build start") - system_path = check_nerves_system_is_set!() + Preflight.check!() - _ = check_nerves_toolchain_is_set!() + config = build_config!(opts) + compiler_check!() # By this point, paths have already been loaded. # We just want to ensure any custom systems are compiled @@ -46,71 +54,47 @@ defmodule Mix.Tasks.Firmware do Mix.Task.run("nerves.precompile", ["--no-loadpaths"]) Mix.Task.run("compile", []) - Mix.Nerves.IO.shell_info("Building OTP Release...") + {time, _result} = :timer.tc(fn -> Mix.Task.run("release", []) end) + debug_info("OTP release : #{time / 1.0e6}s") - build_release() + write_erlinit_config!(config) + prevent_overlay_overwrites!(config) - config = Mix.Project.config() - fw_out = opts[:output] || Nerves.Env.firmware_path(config) - build_firmware(config, system_path, fw_out) - end + build_from_tar? = + config.fs_type == :erofs or Path.extname(config.system_rootfs_path) == ".tar" - @doc false - @spec result({Collectable.t(), exit_status :: non_neg_integer()}) :: :ok - def result({_, 0}) do - Mix.shell().info(""" - Firmware built successfully! 🎉 - - Now you may install it to a MicroSD card using `mix burn` or upload it - to a device with `mix upload` or `mix firmware.gen.script`+`./upload.sh`. - """) - end - - def result({%IO.Stream{}, err}) do - # Any output was already sent through the stream, - # so just halt at this point - System.halt(err) - end + # build_result = + {time, build_result} = + :timer.tc(fn -> + if build_from_tar?, + do: build_firmware(config), + else: build_firmware_legacy(config) + end) - def result({result, _}) do - Mix.raise(""" - Nerves encountered an error. #{inspect(result)} - """) - end + debug_info("mkfs : #{time / 1.0e6}s") - defp build_release() do - Mix.Task.run("release", []) + result(build_result, config) + debug_info("firmware build end") end - defp build_firmware(config, system_path, fw_out) do - otp_app = config[:app] - compiler_check() - firmware_config = Application.get_env(:nerves, :firmware) - - mksquashfs_flags = firmware_config[:mksquashfs_flags] || @default_mksquashfs_flags - set_mksquashfs_flags(mksquashfs_flags) - - rootfs_priorities = - Nerves.Env.package(:nerves_system_br) - |> rootfs_priorities() + defp build_config!(opts) do + firmware_config = Application.get_env(:nerves, :firmware, []) + mix_config = Mix.Project.config() - rel2fw_path = Path.join(system_path, "scripts/rel2fw.sh") - cmd = "bash" - args = [rel2fw_path] - - if firmware_config[:rootfs_additions] do - Mix.shell().error( - "The :rootfs_additions configuration option has been deprecated. Please use :rootfs_overlay instead." - ) - end + # Enforce required pieces + system_path = check_nerves_system_is_set!() + toolchain_path = check_nerves_toolchain_is_set!() + set_provisioning(firmware_config[:provisioning]) + # Build configuration build_rootfs_overlay = Path.join([Mix.Project.build_path(), "nerves", "rootfs_overlay"]) File.mkdir_p!(build_rootfs_overlay) - write_erlinit_config(build_rootfs_overlay) + tmp_dir = Path.join(Mix.Project.build_path(), "_nerves-tmp") + File.mkdir_p!(tmp_dir) - project_rootfs_overlay = - case firmware_config[:rootfs_overlay] || firmware_config[:rootfs_additions] do + project_rootfs_overlays = + case firmware_config[:rootfs_overlay] do nil -> [] @@ -121,69 +105,165 @@ defmodule Mix.Tasks.Firmware do [Path.expand(overlay)] end - prevent_overlay_overwrites!(project_rootfs_overlay) - - rootfs_overlays = - [build_rootfs_overlay | project_rootfs_overlay] - |> Enum.map(&["-a", &1]) - |> List.flatten() + system_images = Path.join(system_path, "images") fwup_conf = - case firmware_config[:fwup_conf] do - nil -> [] - fwup_conf -> ["-c", Path.join(File.cwd!(), fwup_conf)] + if conf_path = firmware_config[:fwup_conf] do + Path.join(File.cwd!(), conf_path) + else + Path.join(system_images, "fwup.conf") end - fw = ["-f", fw_out] - release_path = Path.join(Mix.Project.build_path(), "rel/#{otp_app}") - output = [release_path] - args = args ++ fwup_conf ++ rootfs_overlays ++ fw ++ rootfs_priorities ++ output - env = [{"MIX_BUILD_PATH", Mix.Project.build_path()} | standard_fwup_variables(config)] - - set_provisioning(firmware_config[:provisioning]) - - config - |> Nerves.Env.images_path() - |> File.mkdir_p!() + system_rootfs = + with path <- Path.join(system_images, "rootfs.tar"), + true <- File.exists?(path) do + path + else + _ -> Path.join(system_images, "rootfs.squashfs") + end - shell(cmd, args, env: env) - |> result() + output = opts[:output] || Nerves.Env.firmware_path(mix_config) + # Make sure the fw dir path exists for fwup to write to + File.mkdir_p!(Path.dirname(output)) + + %{ + build_rootfs_overlay: build_rootfs_overlay, + env: build_env(mix_config), + erofs_options: firmware_config[:erofs_options] || [], + fs_type: firmware_config[:fs_type] || :squashfs, + fwup_conf: fwup_conf, + mksquashfs_flags: firmware_config[:mksquashfs_flags] || [], + output: output, + project_rootfs_overlays: project_rootfs_overlays, + release_path: Path.join(Mix.Project.build_path(), "rel/#{mix_config[:app]}"), + system_path: system_path, + system_rootfs_path: system_rootfs, + tmp_dir: tmp_dir, + toolchain_path: toolchain_path, + verbose: opts[:verbose] + } end - defp standard_fwup_variables(config) do + defp build_env(mix_config) do # Assuming the fwup.conf file respects these variable like the official # systems do, this will set the .fw metadata to what's in the mix.exs. [ - {"NERVES_FW_VERSION", config[:version]}, - {"NERVES_FW_PRODUCT", config[:name] || to_string(config[:app])}, - {"NERVES_FW_DESCRIPTION", config[:description]}, - {"NERVES_FW_AUTHOR", config[:author]} + {"MIX_BUILD_PATH", Mix.Project.build_path()}, + {"NERVES_FW_VERSION", mix_config[:version]}, + {"NERVES_FW_PRODUCT", mix_config[:name] || to_string(mix_config[:app])}, + {"NERVES_FW_DESCRIPTION", mix_config[:description]}, + {"NERVES_FW_AUTHOR", mix_config[:author]} ] end - # Need to check min version for nerves_system_br to check if passing the - # rootfs priorities option is supported. This was added in the 1.7.1 release - # https://github.com/nerves-project/nerves_system_br/releases/tag/v1.7.1 - defp rootfs_priorities(%Nerves.Package{app: :nerves_system_br, version: vsn}) do - case Version.compare(vsn, "1.7.1") do - r when r in [:gt, :eq] -> - rootfs_priorities_file = - Path.join([Mix.Project.build_path(), "nerves", "rootfs.priorities"]) - - if File.exists?(rootfs_priorities_file) do - ["-p", rootfs_priorities_file] - else - [] - end + defp result({_, 0}, config) do + args = ["-m", "--metadata-key", "meta-uuid", "-i", config.output] + {uuid, _} = shell("fwup", args, stream: "") + formatted = IO.ANSI.format([:green, String.trim(uuid)]) + + Mix.shell().info(""" + Firmware built successfully! 🎉 [#{formatted}] + + Now you may install it to a MicroSD card using `mix burn` or upload it + to a device with `mix upload` or `mix firmware.gen.script`+`./upload.sh`. + """) + end + + defp result(:error, _config), do: System.halt(1) + + defp result({%IO.Stream{}, err}, _config) do + # Any output was already sent through the stream, + # so just halt at this point + System.halt(err) + end + + defp result({result, _}, _config) do + Mix.raise(""" + Nerves encountered an error. #{inspect(result)} + """) + end - _ -> - [] + defp build_firmware(config) do + # Order matters. First == highest priority + entries = + [ + Enum.map(config.project_rootfs_overlays, &TarMerger.scan_directory/1), + TarMerger.scan_directory(config.build_rootfs_overlay), + TarMerger.scan_directory(config.release_path, "/srv/erlang"), + TarMerger.read_tar(config.system_rootfs_path) + ] + |> TarMerger.merge() + |> TarMerger.sort() + + rootfs = Path.join(config.tmp_dir, "rootfs.#{config.fs_type}") + + with :ok <- mkfs(rootfs, entries, config) do + args = ["-c", "-f", config.fwup_conf, "-o", config.output] + env = [{"ROOTFS", rootfs} | config.env] + shell("fwup", args, env: env) end end - defp rootfs_priorities(_), do: [] + defp mkfs(rootfs, entries, %{fs_type: :erofs} = config) do + mkfs_tmp = Path.join(config.tmp_dir, "mkfs.erofs-tmp") + File.mkdir_p!(mkfs_tmp) + TarMerger.mkfs_erofs(rootfs, entries, erofs_options: config.erofs_options, tmp_dir: mkfs_tmp) + end + + defp mkfs(rootfs, entries, config) do + mkfs_tmp = Path.join(config.tmp_dir, "mkfs.squashfs-tmp") + File.mkdir_p!(mkfs_tmp) + + TarMerger.mkfs_squashfs(rootfs, entries, + mksquashfs_options: config.mksquashfs_flags, + tmp_dir: mkfs_tmp + ) + end + + defp build_firmware_legacy(config) do + # Need to check min version for nerves_system_br to check if passing the + # rootfs priorities option is supported. This was added in the 1.7.1 release + # https://github.com/nerves-project/nerves_system_br/releases/tag/v1.7.1 + rootfs_priorities_arg = + with %Nerves.Package{app: :nerves_system_br, version: vsn} <- + Nerves.Env.package(:nerves_system_br), + r when r in [:gt, :eq] <- Version.compare(vsn, "1.7.1"), + rootfs_priorities_file = + Path.join([Mix.Project.build_path(), "nerves", "rootfs.priorities"]), + true <- File.exists?(rootfs_priorities_file) do + ["-p", rootfs_priorities_file] + else + _ -> [] + end - defp compiler_check() do + rootfs_overlay_args = + [config.build_rootfs_overlay | config.project_rootfs_overlays] + |> Enum.map(&["-a", &1]) + + args = + [ + Path.join(config.system_path, "scripts/rel2fw.sh"), + "-c", + config.fwup_conf, + "-f", + config.output, + rootfs_overlay_args, + rootfs_priorities_arg, + config.release_path + ] + |> List.flatten() + + flags = + if Enum.empty?(config.mksquashfs_flags), + do: ["-no-xattrs", "-quiet"], + else: config.mksquashfs_flags + + env = [{"NERVES_MKSQUASHFS_FLAGS", Enum.join(flags, " ")} | config.env] + + shell("bash", args, env: env) + end + + defp compiler_check!() do {:ok, otpc} = erlang_compiler_version() {:ok, elixirc} = elixir_compiler_version() @@ -227,14 +307,14 @@ defmodule Mix.Tasks.Firmware do |> parse_otp_version() end - defp write_erlinit_config(build_overlay) do + defp write_erlinit_config!(config) do with user_opts <- Application.get_env(:nerves, :erlinit, []), {:ok, system_config_file} <- Nerves.Erlinit.system_config_file(Nerves.Env.system()), {:ok, system_config_file} <- File.read(system_config_file), system_opts <- Nerves.Erlinit.decode_config(system_config_file), erlinit_opts <- Nerves.Erlinit.merge_opts(system_opts, user_opts), erlinit_config <- Nerves.Erlinit.encode_config(erlinit_opts) do - erlinit_config_file = Path.join(build_overlay, "etc/erlinit.config") + erlinit_config_file = Path.join(config.build_rootfs_overlay, "etc/erlinit.config") Path.dirname(erlinit_config_file) |> File.mkdir_p!() @@ -269,14 +349,10 @@ defmodule Mix.Tasks.Firmware do end end - defp set_mksquashfs_flags(flags) when is_list(flags) do - System.put_env("NERVES_MKSQUASHFS_FLAGS", Enum.join(flags, " ")) - end - @restricted_fs ["data", "root", "tmp", "dev", "sys", "proc"] - defp prevent_overlay_overwrites!(overlay_dirs) do + defp prevent_overlay_overwrites!(config) do shadow_mounts = - for dir <- overlay_dirs, + for dir <- config.project_rootfs_overlays, p <- Path.wildcard([dir, "/*"]), fs_dir = Path.relative_to(p, dir), fs_dir in @restricted_fs, diff --git a/mix.exs b/mix.exs index 0584075f..6f4e3d9c 100644 --- a/mix.exs +++ b/mix.exs @@ -30,7 +30,7 @@ defmodule Nerves.MixProject do end def application do - [extra_applications: [:ssl, :inets, :eex, :nerves_bootstrap]] + [extra_applications: [:ssl, :inets, :eex, :nerves_bootstrap, :tar_merger]] end defp elixirc_paths(:test), do: ["lib", "test/support"] @@ -46,7 +46,8 @@ defmodule Nerves.MixProject do {:dialyxir, "~> 1.0", only: :dev, runtime: false}, {:plug, "~> 1.10", only: :test}, {:mime, "~> 2.0", only: :test}, - {:plug_cowboy, "~> 1.0 or ~> 2.0", only: :test} + {:plug_cowboy, "~> 1.0 or ~> 2.0", only: :test}, + {:tar_merger, github: "jjcarstens/tar_merger", branch: "scan-dir-root", runtime: false} ] end diff --git a/mix.lock b/mix.lock index a82dfaaf..33cbd54f 100644 --- a/mix.lock +++ b/mix.lock @@ -21,5 +21,6 @@ "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "tar_merger": {:git, "https://github.com/jjcarstens/tar_merger.git", "4ba437d595870f6efe2418d8db8603eff3d593c6", [branch: "scan-dir-root"]}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, }