diff --git a/.circleci/config.yml b/.circleci/config.yml index 6f8a29b0..dfc27658 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,7 +26,7 @@ jobs: steps: - run: name: Install system dependencies - command: apk add --no-cache build-base procps + command: apk add --no-cache build-base procps git - checkout - run: name: Install hex, rebar, and nerves_bootstrap diff --git a/lib/mix/nerves/preflight.ex b/lib/mix/nerves/preflight.ex index 48ac1696..aea53cad 100644 --- a/lib/mix/nerves/preflight.ex +++ b/lib/mix/nerves/preflight.ex @@ -22,7 +22,7 @@ defmodule Mix.Nerves.Preflight do # OSX defp check_platform!(:darwin) do ensure_fwup_version!() - ensure_available!("mksquashfs", package: "squashfs") + ensure_fs_tools!() ensure_available!("gstat", package: "gstat (coreutils)") end @@ -35,13 +35,23 @@ defmodule Mix.Nerves.Preflight do defp check_platform!(:wsl) do ensure_fwup_version!() ensure_fwup_version!("fwup.exe") - ensure_available!("mksquashfs", package: "squashfs") + ensure_fs_tools!() end # Non-WSL Linux defp check_platform!(_) do ensure_fwup_version!() - ensure_available!("mksquashfs", package: "squashfs") + ensure_fs_tools!() + end + + defp ensure_fs_tools!() do + fs_type = Application.get_env(:nerves, :firmware, [])[:fs_type] + + if fs_type == :erofs do + ensure_available!("mkfs.erofs", package: "erofs-utils") + else + ensure_available!("mksquashfs", package: "squashfs") + end end @doc """ diff --git a/lib/mix/tasks/firmware.ex b/lib/mix/tasks/firmware.ex index 108f27c8..74c1ca7f 100644 --- a/lib/mix/tasks/firmware.ex +++ b/lib/mix/tasks/firmware.ex @@ -22,23 +22,31 @@ defmodule Mix.Tasks.Firmware do Nerves toolchain (C/C++ crosscompiler) that is used """ use Mix.Task - import Mix.Nerves.Utils - alias Mix.Nerves.Preflight - @default_mksquashfs_flags ["-no-xattrs", "-quiet"] + import Mix.Nerves.Utils, + only: [ + check_nerves_system_is_set!: 0, + check_nerves_toolchain_is_set!: 0, + parse_otp_version: 1, + set_provisioning: 1, + shell: 3 + ] + + import Mix.Nerves.IO, only: [debug_info: 1] + alias Mix.Nerves.Preflight @switches [verbose: :boolean, output: :string] @impl Mix.Task def run(args) do - Preflight.check!() - debug_info("Nerves Firmware Assembler") - {opts, _, _} = OptionParser.parse(args, switches: @switches) + if opts[:verbose], do: System.put_env("NERVES_DEBUG", "1") + debug_info("firmware build start") - system_path = check_nerves_system_is_set!() + Preflight.check!() - _ = check_nerves_toolchain_is_set!() + config = build_config!(opts) + compiler_check!() # By this point, paths have already been loaded. # We just want to ensure any custom systems are compiled @@ -46,71 +54,47 @@ defmodule Mix.Tasks.Firmware do Mix.Task.run("nerves.precompile", ["--no-loadpaths"]) Mix.Task.run("compile", []) - Mix.Nerves.IO.shell_info("Building OTP Release...") + {time, _result} = :timer.tc(fn -> Mix.Task.run("release", []) end) + debug_info("OTP release : #{time / 1.0e6}s") - build_release() + write_erlinit_config!(config) + prevent_overlay_overwrites!(config) - config = Mix.Project.config() - fw_out = opts[:output] || Nerves.Env.firmware_path(config) - build_firmware(config, system_path, fw_out) - end + build_from_tar? = + config.fs_type == :erofs or Path.extname(config.system_rootfs_path) == ".tar" - @doc false - @spec result({Collectable.t(), exit_status :: non_neg_integer()}) :: :ok - def result({_, 0}) do - Mix.shell().info(""" - Firmware built successfully! 🎉 - - Now you may install it to a MicroSD card using `mix burn` or upload it - to a device with `mix upload` or `mix firmware.gen.script`+`./upload.sh`. - """) - end - - def result({%IO.Stream{}, err}) do - # Any output was already sent through the stream, - # so just halt at this point - System.halt(err) - end + # build_result = + {time, build_result} = + :timer.tc(fn -> + if build_from_tar?, + do: build_firmware(config), + else: build_firmware_legacy(config) + end) - def result({result, _}) do - Mix.raise(""" - Nerves encountered an error. #{inspect(result)} - """) - end + debug_info("mkfs : #{time / 1.0e6}s") - defp build_release() do - Mix.Task.run("release", []) + result(build_result, config) + debug_info("firmware build end") end - defp build_firmware(config, system_path, fw_out) do - otp_app = config[:app] - compiler_check() - firmware_config = Application.get_env(:nerves, :firmware) - - mksquashfs_flags = firmware_config[:mksquashfs_flags] || @default_mksquashfs_flags - set_mksquashfs_flags(mksquashfs_flags) - - rootfs_priorities = - Nerves.Env.package(:nerves_system_br) - |> rootfs_priorities() + defp build_config!(opts) do + firmware_config = Application.get_env(:nerves, :firmware, []) + mix_config = Mix.Project.config() - rel2fw_path = Path.join(system_path, "scripts/rel2fw.sh") - cmd = "bash" - args = [rel2fw_path] - - if firmware_config[:rootfs_additions] do - Mix.shell().error( - "The :rootfs_additions configuration option has been deprecated. Please use :rootfs_overlay instead." - ) - end + # Enforce required pieces + system_path = check_nerves_system_is_set!() + toolchain_path = check_nerves_toolchain_is_set!() + set_provisioning(firmware_config[:provisioning]) + # Build configuration build_rootfs_overlay = Path.join([Mix.Project.build_path(), "nerves", "rootfs_overlay"]) File.mkdir_p!(build_rootfs_overlay) - write_erlinit_config(build_rootfs_overlay) + tmp_dir = Path.join(Mix.Project.build_path(), "_nerves-tmp") + File.mkdir_p!(tmp_dir) - project_rootfs_overlay = - case firmware_config[:rootfs_overlay] || firmware_config[:rootfs_additions] do + project_rootfs_overlays = + case firmware_config[:rootfs_overlay] do nil -> [] @@ -121,69 +105,165 @@ defmodule Mix.Tasks.Firmware do [Path.expand(overlay)] end - prevent_overlay_overwrites!(project_rootfs_overlay) - - rootfs_overlays = - [build_rootfs_overlay | project_rootfs_overlay] - |> Enum.map(&["-a", &1]) - |> List.flatten() + system_images = Path.join(system_path, "images") fwup_conf = - case firmware_config[:fwup_conf] do - nil -> [] - fwup_conf -> ["-c", Path.join(File.cwd!(), fwup_conf)] + if conf_path = firmware_config[:fwup_conf] do + Path.join(File.cwd!(), conf_path) + else + Path.join(system_images, "fwup.conf") end - fw = ["-f", fw_out] - release_path = Path.join(Mix.Project.build_path(), "rel/#{otp_app}") - output = [release_path] - args = args ++ fwup_conf ++ rootfs_overlays ++ fw ++ rootfs_priorities ++ output - env = [{"MIX_BUILD_PATH", Mix.Project.build_path()} | standard_fwup_variables(config)] - - set_provisioning(firmware_config[:provisioning]) - - config - |> Nerves.Env.images_path() - |> File.mkdir_p!() + system_rootfs = + with path <- Path.join(system_images, "rootfs.tar"), + true <- File.exists?(path) do + path + else + _ -> Path.join(system_images, "rootfs.squashfs") + end - shell(cmd, args, env: env) - |> result() + output = opts[:output] || Nerves.Env.firmware_path(mix_config) + # Make sure the fw dir path exists for fwup to write to + File.mkdir_p!(Path.dirname(output)) + + %{ + build_rootfs_overlay: build_rootfs_overlay, + env: build_env(mix_config), + erofs_options: firmware_config[:erofs_options] || [], + fs_type: firmware_config[:fs_type] || :squashfs, + fwup_conf: fwup_conf, + mksquashfs_flags: firmware_config[:mksquashfs_flags] || [], + output: output, + project_rootfs_overlays: project_rootfs_overlays, + release_path: Path.join(Mix.Project.build_path(), "rel/#{mix_config[:app]}"), + system_path: system_path, + system_rootfs_path: system_rootfs, + tmp_dir: tmp_dir, + toolchain_path: toolchain_path, + verbose: opts[:verbose] + } end - defp standard_fwup_variables(config) do + defp build_env(mix_config) do # Assuming the fwup.conf file respects these variable like the official # systems do, this will set the .fw metadata to what's in the mix.exs. [ - {"NERVES_FW_VERSION", config[:version]}, - {"NERVES_FW_PRODUCT", config[:name] || to_string(config[:app])}, - {"NERVES_FW_DESCRIPTION", config[:description]}, - {"NERVES_FW_AUTHOR", config[:author]} + {"MIX_BUILD_PATH", Mix.Project.build_path()}, + {"NERVES_FW_VERSION", mix_config[:version]}, + {"NERVES_FW_PRODUCT", mix_config[:name] || to_string(mix_config[:app])}, + {"NERVES_FW_DESCRIPTION", mix_config[:description]}, + {"NERVES_FW_AUTHOR", mix_config[:author]} ] end - # Need to check min version for nerves_system_br to check if passing the - # rootfs priorities option is supported. This was added in the 1.7.1 release - # https://github.com/nerves-project/nerves_system_br/releases/tag/v1.7.1 - defp rootfs_priorities(%Nerves.Package{app: :nerves_system_br, version: vsn}) do - case Version.compare(vsn, "1.7.1") do - r when r in [:gt, :eq] -> - rootfs_priorities_file = - Path.join([Mix.Project.build_path(), "nerves", "rootfs.priorities"]) - - if File.exists?(rootfs_priorities_file) do - ["-p", rootfs_priorities_file] - else - [] - end + defp result({_, 0}, config) do + args = ["-m", "--metadata-key", "meta-uuid", "-i", config.output] + {uuid, _} = shell("fwup", args, stream: "") + formatted = IO.ANSI.format([:green, String.trim(uuid)]) + + Mix.shell().info(""" + Firmware built successfully! 🎉 [#{formatted}] + + Now you may install it to a MicroSD card using `mix burn` or upload it + to a device with `mix upload` or `mix firmware.gen.script`+`./upload.sh`. + """) + end + + defp result(:error, _config), do: System.halt(1) + + defp result({%IO.Stream{}, err}, _config) do + # Any output was already sent through the stream, + # so just halt at this point + System.halt(err) + end + + defp result({result, _}, _config) do + Mix.raise(""" + Nerves encountered an error. #{inspect(result)} + """) + end - _ -> - [] + defp build_firmware(config) do + # Order matters. First == highest priority + entries = + [ + Enum.map(config.project_rootfs_overlays, &TarMerger.scan_directory/1), + TarMerger.scan_directory(config.build_rootfs_overlay), + TarMerger.scan_directory(config.release_path, "/srv/erlang"), + TarMerger.read_tar(config.system_rootfs_path) + ] + |> TarMerger.merge() + |> TarMerger.sort() + + rootfs = Path.join(config.tmp_dir, "rootfs.#{config.fs_type}") + + with :ok <- mkfs(rootfs, entries, config) do + args = ["-c", "-f", config.fwup_conf, "-o", config.output] + env = [{"ROOTFS", rootfs} | config.env] + shell("fwup", args, env: env) end end - defp rootfs_priorities(_), do: [] + defp mkfs(rootfs, entries, %{fs_type: :erofs} = config) do + mkfs_tmp = Path.join(config.tmp_dir, "mkfs.erofs-tmp") + File.mkdir_p!(mkfs_tmp) + TarMerger.mkfs_erofs(rootfs, entries, erofs_options: config.erofs_options, tmp_dir: mkfs_tmp) + end + + defp mkfs(rootfs, entries, config) do + mkfs_tmp = Path.join(config.tmp_dir, "mkfs.squashfs-tmp") + File.mkdir_p!(mkfs_tmp) + + TarMerger.mkfs_squashfs(rootfs, entries, + mksquashfs_options: config.mksquashfs_flags, + tmp_dir: mkfs_tmp + ) + end + + defp build_firmware_legacy(config) do + # Need to check min version for nerves_system_br to check if passing the + # rootfs priorities option is supported. This was added in the 1.7.1 release + # https://github.com/nerves-project/nerves_system_br/releases/tag/v1.7.1 + rootfs_priorities_arg = + with %Nerves.Package{app: :nerves_system_br, version: vsn} <- + Nerves.Env.package(:nerves_system_br), + r when r in [:gt, :eq] <- Version.compare(vsn, "1.7.1"), + rootfs_priorities_file = + Path.join([Mix.Project.build_path(), "nerves", "rootfs.priorities"]), + true <- File.exists?(rootfs_priorities_file) do + ["-p", rootfs_priorities_file] + else + _ -> [] + end - defp compiler_check() do + rootfs_overlay_args = + [config.build_rootfs_overlay | config.project_rootfs_overlays] + |> Enum.map(&["-a", &1]) + + args = + [ + Path.join(config.system_path, "scripts/rel2fw.sh"), + "-c", + config.fwup_conf, + "-f", + config.output, + rootfs_overlay_args, + rootfs_priorities_arg, + config.release_path + ] + |> List.flatten() + + flags = + if Enum.empty?(config.mksquashfs_flags), + do: ["-no-xattrs", "-quiet"], + else: config.mksquashfs_flags + + env = [{"NERVES_MKSQUASHFS_FLAGS", Enum.join(flags, " ")} | config.env] + + shell("bash", args, env: env) + end + + defp compiler_check!() do {:ok, otpc} = erlang_compiler_version() {:ok, elixirc} = elixir_compiler_version() @@ -227,14 +307,14 @@ defmodule Mix.Tasks.Firmware do |> parse_otp_version() end - defp write_erlinit_config(build_overlay) do + defp write_erlinit_config!(config) do with user_opts <- Application.get_env(:nerves, :erlinit, []), {:ok, system_config_file} <- Nerves.Erlinit.system_config_file(Nerves.Env.system()), {:ok, system_config_file} <- File.read(system_config_file), system_opts <- Nerves.Erlinit.decode_config(system_config_file), erlinit_opts <- Nerves.Erlinit.merge_opts(system_opts, user_opts), erlinit_config <- Nerves.Erlinit.encode_config(erlinit_opts) do - erlinit_config_file = Path.join(build_overlay, "etc/erlinit.config") + erlinit_config_file = Path.join(config.build_rootfs_overlay, "etc/erlinit.config") Path.dirname(erlinit_config_file) |> File.mkdir_p!() @@ -269,14 +349,10 @@ defmodule Mix.Tasks.Firmware do end end - defp set_mksquashfs_flags(flags) when is_list(flags) do - System.put_env("NERVES_MKSQUASHFS_FLAGS", Enum.join(flags, " ")) - end - @restricted_fs ["data", "root", "tmp", "dev", "sys", "proc"] - defp prevent_overlay_overwrites!(overlay_dirs) do + defp prevent_overlay_overwrites!(config) do shadow_mounts = - for dir <- overlay_dirs, + for dir <- config.project_rootfs_overlays, p <- Path.wildcard([dir, "/*"]), fs_dir = Path.relative_to(p, dir), fs_dir in @restricted_fs, diff --git a/mix.exs b/mix.exs index 0584075f..6f4e3d9c 100644 --- a/mix.exs +++ b/mix.exs @@ -30,7 +30,7 @@ defmodule Nerves.MixProject do end def application do - [extra_applications: [:ssl, :inets, :eex, :nerves_bootstrap]] + [extra_applications: [:ssl, :inets, :eex, :nerves_bootstrap, :tar_merger]] end defp elixirc_paths(:test), do: ["lib", "test/support"] @@ -46,7 +46,8 @@ defmodule Nerves.MixProject do {:dialyxir, "~> 1.0", only: :dev, runtime: false}, {:plug, "~> 1.10", only: :test}, {:mime, "~> 2.0", only: :test}, - {:plug_cowboy, "~> 1.0 or ~> 2.0", only: :test} + {:plug_cowboy, "~> 1.0 or ~> 2.0", only: :test}, + {:tar_merger, github: "jjcarstens/tar_merger", branch: "scan-dir-root", runtime: false} ] end diff --git a/mix.lock b/mix.lock index a82dfaaf..33cbd54f 100644 --- a/mix.lock +++ b/mix.lock @@ -21,5 +21,6 @@ "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, + "tar_merger": {:git, "https://github.com/jjcarstens/tar_merger.git", "4ba437d595870f6efe2418d8db8603eff3d593c6", [branch: "scan-dir-root"]}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, }