fileio: add example system for testing filesystems

This adds a new filesystem testing system to examples/. The system
runs MicroPython and includes two scripts, fs_test.py and bench.py,
which can be used to test a filesystem component's functionality and
benchmark its sequential read speed, respectively.

Signed-off-by: James Archer <[email protected]>
Co-authored-by: Cheng Li <[email protected]>

ci/examples.sh

@@ -20,3 +20,4 @@ MICROKIT_SDK=$2
 $LIONSOS/ci/kitty.sh $LIONSOS $MICROKIT_SDK
 $LIONSOS/ci/webserver.sh $LIONSOS $MICROKIT_SDK
 $LIONSOS/ci/vmm.sh $LIONSOS $MICROKIT_SDK
+$LIONSOS/ci/fileio.sh $LIONSOS $MICROKIT_SDK

ci/fileio.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2024, UNSW
+# SPDX-License-Identifier: BSD-2-Clause
+
+#
+# This script aims to build an already checked out version of LionsOS.
+#
+
+set -e
+
+if [ "$#" -ne 2 ]; then
+    echo "usage: fileio.sh /path/to/lionsos /path/to/microkit/sdk"
+    exit 1
+fi
+
+LIONSOS=$1
+MICROKIT_SDK=$2
+
+build() {
+    MICROKIT_BOARD=$1
+    MICROKIT_CONFIG=debug
+
+    echo "CI|INFO: building fileio for board: ${MICROKIT_BOARD}"
+
+    BUILD_DIR=$LIONSOS/ci_build/fileio/${MICROKIT_BOARD}/${MICROKIT_CONFIG}
+    rm -rf $BUILD_DIR
+
+    export BUILD_DIR=$BUILD_DIR
+    export MICROKIT_SDK=$MICROKIT_SDK
+    export MICROKIT_CONFIG=$MICROKIT_CONFIG
+    export MICROKIT_BOARD=$MICROKIT_BOARD
+    export LIONSOS=$LIONSOS
+
+    cd $LIONSOS/examples/fileio
+    make
+}
+
+BOARDS=("maaxboard" "qemu_virt_aarch64")
+for BOARD in "${BOARDS[@]}"
+do
+    build ${BOARD}
+done
+
+echo ""
+echo "CI|INFO: Passed all fileio build tests"

components/micropython/mpconfigport.h

@@ -91,6 +91,12 @@ typedef long mp_off_t;
 #if defined(CONFIG_PLAT_ODROIDC4)
 #define MICROPY_HW_BOARD_NAME "Odroid-C4"
 #define MICROPY_HW_MCU_NAME   "Cortex A55"
+#elif defined(CONFIG_PLAT_IMX8MM_EVK)
+#define MICROPY_HW_BOARD_NAME "i.MX 8M Mini"
+#define MICROPY_HW_MCU_NAME   "Cortex A53"
+#elif defined(CONFIG_PLAT_MAAXBOARD)
+#define MICROPY_HW_BOARD_NAME "MaaXBoard"
+#define MICROPY_HW_MCU_NAME   "Cortex A53"
 #elif defined(CONFIG_PLAT_QEMU_ARM_VIRT)
 #define MICROPY_HW_BOARD_NAME "QEMU virt AArch64"
 #define MICROPY_HW_MCU_NAME   "Cortex A53"

examples/fileio/Makefile

@@ -0,0 +1,46 @@
+#
+# Copyright 2024, UNSW
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+ifeq ($(strip $(MICROKIT_SDK)),)
+$(error MICROKIT_SDK must be specified)
+endif
+override MICROKIT_SDK:=$(abspath ${MICROKIT_SDK})
+ifeq ($(strip $(LIBGCC)),)
+export LIBGCC:=$(dir $(realpath $(shell aarch64-none-elf-gcc --print-file-name libgcc.a)))
+endif
+ifeq ($(strip $(LIBMATH)),)
+export LIBMATH:=$(dir $(realpath $(shell aarch64-none-elf-gcc --print-file-name libm.a)))
+endif
+
+export LIONSOS ?= $(abspath ../..)
+export FILEIO_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
+export CONFIG_INCLUDE ?= $(abspath src/config)
+export MICROKIT_CONFIG ?= debug
+export BUILD_DIR ?= $(abspath build)
+export MICROKIT_BOARD ?= qemu_virt_aarch64
+
+IMAGE_FILE := $(BUILD_DIR)/fileio.img
+REPORT_FILE := $(BUILD_DIR)/report.txt
+
+all: ${IMAGE_FILE}
+
+qemu ${IMAGE_FILE} ${REPORT_FILE} clean clobber: ${BUILD_DIR}/Makefile FORCE
+	${MAKE} -C ${BUILD_DIR} MICROKIT_SDK=${MICROKIT_SDK} $(notdir $@)
+
+${BUILD_DIR}/Makefile: fileio.mk
+	mkdir -p ${BUILD_DIR}
+	cp fileio.mk $@
+
+submodules:
+	git submodule update --init $(LIONSOS)/dep/libvmm
+	git submodule update --init $(LIONSOS)/dep/libnfs
+	git submodule update --init $(LIONSOS)/dep/micropython
+	git submodule update --init $(LIONSOS)/dep/musllibc
+	git submodule update --init $(LIONSOS)/dep/sddf
+	git submodule update --init $(LIONSOS)/dep/libmicrokitco
+	cd ${LIONSOS}/dep/micropython && git submodule update --init lib/micropython-lib
+
+FORCE:

examples/fileio/bench.py

@@ -0,0 +1,31 @@
+# Copyright 2024, UNSW
+# SPDX-License-Identifier: BSD-2-Clause
+
+import time
+
+path = 'bench.dat'
+
+def bench():
+    with open(path, 'w') as f:
+        for i in range(8):
+            f.write('#' * 1024**2)
+
+    bytes_read = 0
+    with open(path) as f:
+        buffer_size = 0x8000
+        _buffer = bytearray(buffer_size)
+        _bmview = memoryview(_buffer)
+        t_pre = time.time()
+        for i in range(16):
+            while True:
+                n = f.readinto(_buffer)
+                if n == 0:
+                    break
+                bytes_read += n
+            f.seek(0)
+        t = time.time() - t_pre
+
+    mbytes_read = bytes_read / 1024**2
+    seconds = t / 1000
+    mbytes_per_s = mbytes_read / seconds
+    print(f'Read {mbytes_read} MiB at a rate of {mbytes_per_s} MiB/s')

examples/fileio/fileio.mk

@@ -0,0 +1,185 @@
+#
+# Copyright 2024, UNSW
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+IMAGES := \
+	timer_driver.elf \
+	eth_driver.elf \
+	micropython.elf \
+	fat.elf \
+	copy.elf \
+	network_virt_rx.elf \
+	network_virt_tx.elf \
+	uart_driver.elf \
+	serial_virt_rx.elf \
+	serial_virt_tx.elf \
+	blk_virt.elf
+
+ifeq ($(strip $(MICROKIT_BOARD)), maaxboard)
+	NET_DRIV_DIR := imx
+	BLK_DRIV_DIR := mmc/imx
+	UART_DRIV_DIR := imx
+	TIMER_DRIV_DIR := imx
+	IMAGES += mmc_driver.elf
+	BLK_MK := mmc_driver.mk
+	CPU := cortex-a53
+else ifeq ($(strip $(MICROKIT_BOARD)), qemu_virt_aarch64)
+	NET_DRIV_DIR := virtio
+	BLK_DRIV_DIR := virtio
+	UART_DRIV_DIR := arm
+	TIMER_DRIV_DIR := arm
+	IMAGES += blk_driver.elf
+	BLK_MK := blk_driver.mk
+	CPU := cortex-a53
+	QEMU := qemu-system-aarch64
+else
+	$(error Unsupported MICROKIT_BOARD given)
+endif
+
+TOOLCHAIN := clang
+CC := clang
+LD := ld.lld
+RANLIB := llvm-ranlib
+AR := llvm-ar
+TARGET := aarch64-none-elf
+MICROKIT_TOOL ?= $(MICROKIT_SDK)/bin/microkit
+DTC := dtc
+
+BOARD_DIR := $(MICROKIT_SDK)/board/$(MICROKIT_BOARD)/$(MICROKIT_CONFIG)
+PLATFORM := meson
+SDDF := $(LIONSOS)/dep/sddf
+LIBVMM_DIR := $(LIONSOS)/dep/libvmm
+
+VMM_IMAGE_DIR := ${FILEIO_DIR}/src/vmm/images
+LINUX := 90c4247bcd24cbca1a3db4b7489a835ce87a486e-linux
+INITRD := 08c10529dc2806559d5c4b7175686a8206e10494-rootfs.cpio.gz
+DTS := $(VMM_IMAGE_DIR)/linux.dts
+DTB := linux.dtb
+
+LWIP := $(SDDF)/network/ipstacks/lwip/src
+FAT := $(LIONSOS)/components/fs/fat
+MUSL := $(LIONSOS)/dep/musllibc
+
+CFLAGS := \
+	-mtune=$(CPU) \
+	-mstrict-align \
+	-ffreestanding \
+	-g \
+	-O1 \
+	-Wall \
+	-Wno-unused-function \
+	-I$(BOARD_DIR)/include \
+	-target $(TARGET) \
+	-DBOARD_$(MICROKIT_BOARD) \
+	-I$(SDDF)/include \
+	-I${CONFIG_INCLUDE} \
+	-DVIRTIO_MMIO_NET_OFFSET=0xc00
+
+
+LDFLAGS := -L$(BOARD_DIR)/lib
+LIBS := -lmicrokit -Tmicrokit.ld libsddf_util_debug.a
+
+IMAGE_FILE := fileio.img
+REPORT_FILE := report.txt
+
+all: cache.o
+CHECK_FLAGS_BOARD_MD5:=.board_cflags-$(shell echo -- ${CFLAGS} ${BOARD} ${MICROKIT_CONFIG} | shasum | sed 's/ *-//')
+
+${CHECK_FLAGS_BOARD_MD5}:
+	-rm -f .board_cflags-*
+	touch $@
+
+
+%.elf: %.o
+	${LD} ${LDFLAGS} -o $@ $< ${LIBS}
+
+BLK_DRIVER := $(SDDF)/drivers/blk/${BLK_DRIV_DIR}
+BLK_COMPONENTS := $(SDDF)/blk/components
+
+include ${SDDF}/util/util.mk
+include ${LIBVMM_DIR}/vmm.mk
+include ${SDDF}/drivers/timer/${TIMER_DRIV_DIR}/timer_driver.mk
+include ${SDDF}/drivers/network/${NET_DRIV_DIR}/eth_driver.mk
+include ${SDDF}/drivers/serial/${UART_DRIV_DIR}/uart_driver.mk
+include ${SDDF}/network/components/network_components.mk
+include ${SDDF}/serial/components/serial_components.mk
+include ${SDDF}/libco/libco.mk
+include ${BLK_DRIVER}/${BLK_MK}
+include ${BLK_COMPONENTS}/blk_components.mk
+
+micropython.elf: mpy-cross libsddf_util_debug.a libco.a
+	cp $(LIONSOS)/examples/fileio/fs_test.py .
+	cp $(LIONSOS)/examples/fileio/manifest.py .
+	cp $(LIONSOS)/examples/fileio/bench.py .
+	make  -C $(LIONSOS)/components/micropython -j$(nproc) \
+		MICROKIT_SDK=$(MICROKIT_SDK) \
+		MICROKIT_BOARD=$(MICROKIT_BOARD) \
+		MICROKIT_CONFIG=$(MICROKIT_CONFIG) \
+		MICROPY_MPYCROSS=$(abspath mpy_cross/mpy-cross) \
+		MICROPY_MPYCROSS_DEPENDENCY=$(abspath mpy_cross/mpy-cross) \
+		BUILD=$(abspath .) \
+		LIBMATH=${LIBMATH} \
+		CONFIG_INCLUDE=$(abspath $(CONFIG_INCLUDE)) \
+		FROZEN_MANIFEST=$(abspath ./manifest.py) \
+		V=1
+
+fat.elf: musllibc/lib/libc.a
+	make -C $(LIONSOS)/components/fs/fat \
+		CC=$(CC) \
+		LD=$(LD) \
+		CPU=$(CPU) \
+		BUILD_DIR=$(abspath .) \
+		CONFIG=$(MICROKIT_CONFIG) \
+		MICROKIT_SDK=$(MICROKIT_SDK) \
+		MICROKIT_BOARD=$(MICROKIT_BOARD) \
+		LIBC_DIR=$(abspath $(BUILD_DIR)/musllibc) \
+		BUILD_DIR=$(abspath .) \
+		CONFIG_INCLUDE=$(abspath $(CONFIG_INCLUDE)) \
+		TARGET=$(TARGET)
+
+musllibc/lib/libc.a:
+	make -C $(MUSL) \
+		C_COMPILER=aarch64-none-elf-gcc \
+		TOOLPREFIX=aarch64-none-elf- \
+		CONFIG_ARCH_AARCH64=y \
+		STAGE_DIR=$(abspath $(BUILD_DIR)/musllibc) \
+		SOURCE_DIR=.
+
+${IMAGES}: libsddf_util_debug.a
+
+%.o: %.c
+	${CC} ${CFLAGS} -c -o $@ $<
+
+$(IMAGE_FILE) $(REPORT_FILE): $(IMAGES) ${FILEIO_DIR}/board/$(MICROKIT_BOARD)/fileio.system
+	$(MICROKIT_TOOL) ${FILEIO_DIR}/board/$(MICROKIT_BOARD)/fileio.system \
+		--search-path $(BUILD_DIR) \
+		--board $(MICROKIT_BOARD) \
+		--config $(MICROKIT_CONFIG) \
+		-o $(IMAGE_FILE) \
+		-r $(REPORT_FILE)
+
+FORCE:
+
+%.elf: %.o
+	${LD} ${LDFLAGS} -o $@ $< ${LIBS}
+
+mpy-cross: FORCE
+	${MAKE} -C ${LIONSOS}/dep/micropython/mpy-cross BUILD=$(abspath ./mpy_cross)
+
+qemu_disk:
+	$(LIONSOS)/dep/sddf/examples/blk/mkvirtdisk $@ 1 512 16777216
+
+qemu: ${IMAGE_FILE} qemu_disk
+	$(QEMU) -machine virt,virtualization=on \
+		-cpu cortex-a53 \
+		-serial mon:stdio \
+		-device loader,file=$(IMAGE_FILE),addr=0x70000000,cpu-num=0 \
+		-m size=2G \
+		-nographic \
+		-global virtio-mmio.force-legacy=false \
+		-d guest_errors \
+		-drive file=qemu_disk,if=none,format=raw,id=hd \
+		-device virtio-blk-device,drive=hd \
+		-device virtio-net-device,netdev=netdev0 \
+		-netdev user,id=netdev0