From 8c96044d76d7fe1bb7745f0b79cb69c7c85698ac Mon Sep 17 00:00:00 2001 From: Gabe Parmer Date: Mon, 13 Jun 2022 12:09:45 -0400 Subject: [PATCH 1/5] Updated Makefiles to pass along the make include path. --- .gitignore | 1 + src/Makefile | 1 + src/components/Makefile | 6 +++++- src/components/implementation/Makefile | 6 +++++- src/components/implementation/Makefile.subdir | 4 ++-- src/composer/src/passes.rs | 2 +- 6 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 07c88537b4..b439265404 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,4 @@ constructor kernel.iso grub.cfg gdbinit +src/external_packages/* diff --git a/src/Makefile b/src/Makefile index 1190131c98..a1cf5c4e49 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,6 +1,7 @@ #MAKEFLAGS=--no-print-directory --section-alignment 0x1000 -I$(PWD) # Note: can't use $(PWD) here if make -Ced from elsewhere. MAKEOPTIONS=--no-print-directory -I$(shell pwd) +export MAKEOPTIONS PLAT_FILE=.PLATFORM_ID .PHONY: default all composer component comps platclean plat cpplat clean distclean init update config diff --git a/src/components/Makefile b/src/components/Makefile index 9aa6d489f8..c169efee7b 100644 --- a/src/components/Makefile +++ b/src/components/Makefile @@ -1,4 +1,5 @@ -MAKEOPTIONS=-I$(shell pwd) +MAKEOPTIONS+= -I$(shell pwd) +export MAKEOPTIONS # Order of which subdirectories are built first matters here. # Interface files rely on library files, and component implementations @@ -24,3 +25,6 @@ init: component: $(MAKE) $(MAKEOPTIONS) -C implementation component + +component_external: + $(MAKE) $(MAKEOPTIONS) -C implementation component_external diff --git a/src/components/implementation/Makefile b/src/components/implementation/Makefile index 69ad4c1aa1..80442cba2b 100644 --- a/src/components/implementation/Makefile +++ b/src/components/implementation/Makefile @@ -1,7 +1,8 @@ include Makefile.src Makefile.comp SUBDIRS=$(filter-out skel/, $(wildcard */)) -MAKEOPTIONS=-I$(shell pwd) +MAKEOPTIONS+= -I$(shell pwd) +export MAKEOPTIONS .PHONY: all all: @@ -25,3 +26,6 @@ distclean: component: $(MAKE) $(MAKEOPTIONS) -C $(COMP_INTERFACE) component + +component_external: + $(MAKE) $(MAKEOPTIONS) -C $(COMPONENT_EXTERNAL_PATH) component diff --git a/src/components/implementation/Makefile.subdir b/src/components/implementation/Makefile.subdir index 8bd73fc235..7c0a9fd392 100644 --- a/src/components/implementation/Makefile.subdir +++ b/src/components/implementation/Makefile.subdir @@ -21,7 +21,7 @@ print: .PHONY: subdirs subdirs: @for dir in $(SUBDIRS) ; do \ - $(MAKE) -C $$dir ; \ + $(MAKE) $(MAKEOPTIONS) -C $$dir ; \ done $(CLIB):$(OBJS) @@ -61,6 +61,6 @@ clean: .PHONY: component component: - $(MAKE) -C $(COMP_NAME) component + $(MAKE) $(MAKEOPTIONS) -C $(COMP_NAME) component -include $(SOURCE_DEPENDENCIES) diff --git a/src/composer/src/passes.rs b/src/composer/src/passes.rs index 352c96a7cf..64c7a77a49 100644 --- a/src/composer/src/passes.rs +++ b/src/composer/src/passes.rs @@ -326,7 +326,7 @@ pub trait AddressAssignmentPass { // Compute the resource table, and resource allocations for each // component. pub trait ResPass { - fn args(&self, &ComponentId) -> &Vec; + fn args(&self, _: &ComponentId) -> &Vec; } // The initparam, objects, and synchronous invocation passes are all From fdc89783deaa8a43b3df98bf111894425f1cfa68 Mon Sep 17 00:00:00 2001 From: Gabe Parmer Date: Mon, 13 Jun 2022 14:28:07 -0400 Subject: [PATCH 2/5] CPU burn component, and simple composition script on top of the composer --- composition_scripts/burn.toml | 15 +++++++++++++++ .../implementation/no_interface/burn/Makefile | 18 ++++++++++++++++++ .../implementation/no_interface/burn/burn.c | 11 +++++++++++ .../implementation/no_interface/burn/doc.md | 12 ++++++++++++ 4 files changed, 56 insertions(+) create mode 100644 composition_scripts/burn.toml create mode 100644 src/components/implementation/no_interface/burn/Makefile create mode 100644 src/components/implementation/no_interface/burn/burn.c create mode 100644 src/components/implementation/no_interface/burn/doc.md diff --git a/composition_scripts/burn.toml b/composition_scripts/burn.toml new file mode 100644 index 0000000000..5adeaa8af8 --- /dev/null +++ b/composition_scripts/burn.toml @@ -0,0 +1,15 @@ +[system] +description = "Simple system running only a CPU-burn component." + +[[components]] +name = "booter" +img = "no_interface.llbooter" +implements = [{interface = "init"}, {interface = "addr"}] +deps = [{srv = "kernel", interface = "init", variant = "kernel"}] +constructor = "kernel" + +[[components]] +name = "burn" +img = "no_interface.burn" +deps = [{srv = "booter", interface = "init"}, {srv = "booter", interface = "addr"}] +constructor = "booter" diff --git a/src/components/implementation/no_interface/burn/Makefile b/src/components/implementation/no_interface/burn/Makefile new file mode 100644 index 0000000000..babc8cf858 --- /dev/null +++ b/src/components/implementation/no_interface/burn/Makefile @@ -0,0 +1,18 @@ +# Required variables used to drive the compilation process. It is OK +# for many of these to be empty. +# +# The set of interfaces that this component exports for use by other +# components. This is a list of the interface names. +INTERFACE_EXPORTS = +# The interfaces this component is dependent on for compilation (this +# is a list of directory names in interface/) +INTERFACE_DEPENDENCIES = +# The library dependencies this component is reliant on for +# compilation/linking (this is a list of directory names in lib/) +LIBRARY_DEPENDENCIES = component +# Note: Both the interface and library dependencies should be +# *minimal*. That is to say that removing a dependency should cause +# the build to fail. The build system does not validate this +# minimality; that's on you! + +include Makefile.subsubdir diff --git a/src/components/implementation/no_interface/burn/burn.c b/src/components/implementation/no_interface/burn/burn.c new file mode 100644 index 0000000000..55c22c06ff --- /dev/null +++ b/src/components/implementation/no_interface/burn/burn.c @@ -0,0 +1,11 @@ +#include + +void +cos_init(void) +{ + /* Burn all of the cycles! CPUs are glorified space heaters, right? */ + printc("CPU cycle burn: component %ld, thread id %ld, core %d.\n", + cos_compid(), cos_thdid(), cos_coreid()); + + while (1) ; +} diff --git a/src/components/implementation/no_interface/burn/doc.md b/src/components/implementation/no_interface/burn/doc.md new file mode 100644 index 0000000000..266be2a6ad --- /dev/null +++ b/src/components/implementation/no_interface/burn/doc.md @@ -0,0 +1,12 @@ +## CPU Burn Component + +A simple component that burns CPU cycles, nothing else. +Can be used in test scenarios as background, CPU-intensive computations. + +### Description + +Simple infinite loop, nothing more. + +### Usage and Assumptions + +Only depends on the `init` interface, so it can be used anywhere from on top of the constructor, all the way to as a component under the scheduler. From f041af40d883252ac30565e5deceb1871f27998c Mon Sep 17 00:00:00 2001 From: Gabe Parmer Date: Fri, 17 Jun 2022 07:18:33 -0400 Subject: [PATCH 3/5] TRANSIENT COMMIT --- composition_scripts/external_hello_world.toml | 9 +++++++++ src/composer/src/build.rs | 14 +++++++++++--- src/composer/src/cossystem.rs | 2 ++ src/composer/src/main.rs | 1 + src/composer/src/passes.rs | 9 ++++++++- 5 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 composition_scripts/external_hello_world.toml diff --git a/composition_scripts/external_hello_world.toml b/composition_scripts/external_hello_world.toml new file mode 100644 index 0000000000..1f01d72a6c --- /dev/null +++ b/composition_scripts/external_hello_world.toml @@ -0,0 +1,9 @@ +[system] +description = "Test for the hello world external component" + +[[components]] +name = "hw" +img = "hw" +location = "https://www.github.com/gparmer/hello_world" +deps = [{srv = "kernel", interface = "init", variant = "kernel"}] +constructor = "kernel" diff --git a/src/composer/src/build.rs b/src/composer/src/build.rs index 9d3833da07..33e2fd1c6f 100644 --- a/src/composer/src/build.rs +++ b/src/composer/src/build.rs @@ -41,15 +41,23 @@ use tar::Builder; // - COMP_INITARGS_FILE - the path to the generated initial arguments .c file // - COMP_TAR_FILE - the path to an initargs tarball to compile into the component // +// Building external repositories (not in the main composite repo) +// doesn't use COMP_INTERFACE and instead uses the following: +// +// - COMP_EXTERNAL_REPO - The URL to the to the external repo that +// holds the component. +// // In the end, this should result in a command line for each component // along these (artificial) lines: // -// `make COMP_INTERFACES="pong/log" COMP_IFDEPS="capmgr/stubs sched/lock" COMP_LIBS="ps heap" COMP_INTERFACE=pong COMP_NAME=pingpong COMP_VARNAME=pongcomp component` +// `make COMP_INTERFACES="pong/log" COMP_IFDEPS="capmgr/stubs+sched/lock" COMP_LIBS="ps heap" COMP_INTERFACE=pong COMP_NAME=pingpong COMP_VARNAME=pongcomp component` // // ...which should output the executable pong.pingpong.pongcomp in the // build directory which is the "sealed" version of the component that // is ready for loading. +// + // The key within the initargs for the tarball, the path of the // tarball, and the set of paths to the files to include in the // tarball and name of them within the tarball. @@ -340,7 +348,7 @@ impl BuildState for DefaultBuilder { fn comp_dir_path(&self, c: &ComponentId, state: &SystemState) -> Result { let name = state.get_named().ids().get(c).unwrap(); - Ok(self.file_path(&format!("{}.{}", name.scope_name, name.var_name))?) + Ok(self.file_path(&format!("{}", name))?) } fn comp_file_path( @@ -357,7 +365,7 @@ impl BuildState for DefaultBuilder { fn comp_obj_file(&self, c: &ComponentId, s: &SystemState) -> String { let comp = component(&s, &c); - format!("{}.{}", &comp.source, &comp.name) + format!("{}", &comp) } fn comp_obj_path(&self, c: &ComponentId, s: &SystemState) -> Result { diff --git a/src/composer/src/cossystem.rs b/src/composer/src/cossystem.rs index 2153754ac2..b8991c7118 100644 --- a/src/composer/src/cossystem.rs +++ b/src/composer/src/cossystem.rs @@ -32,6 +32,7 @@ pub struct InterfaceVariant { pub struct TomlComponent { name: String, img: String, + location: Option, baseaddr: Option, deps: Option>, params: Option>, @@ -509,6 +510,7 @@ impl Transition for SystemSpec { constructor: ComponentName::new(&c.constructor, &String::from("global")), scheduler: sched_name, source: c.img.clone(), + location: c.location.clone(), base_vaddr: c .baseaddr .as_ref() diff --git a/src/composer/src/main.rs b/src/composer/src/main.rs index bfb27751d3..ac7fb57b48 100644 --- a/src/composer/src/main.rs +++ b/src/composer/src/main.rs @@ -54,6 +54,7 @@ pub fn exec() -> Result<(), String> { sys.add_address_assign(AddressAssignmentx86_64::transition(&sys, &mut build)?); sys.add_properties(CompProperties::transition(&sys, &mut build)?); sys.add_restbls(ResAssignPass::transition(&sys, &mut build)?); + sys.add_repos(RepoPass::transition(&sys, &mut build)?); // process these in reverse order of dependencies (e.g. booter last) let reverse_ids: Vec = sys diff --git a/src/composer/src/passes.rs b/src/composer/src/passes.rs index 64c7a77a49..051cc47456 100644 --- a/src/composer/src/passes.rs +++ b/src/composer/src/passes.rs @@ -209,12 +209,19 @@ pub struct Component { pub constructor: ComponentName, // the constructor that loads this component pub scheduler: ComponentName, // our scheduler (that creates or initial thread) - pub source: String, // Where is the component source located? + pub source: String, // Where is the component source located in repo? + pub location: Option, // ...and should we use an external repo? pub base_vaddr: String, // The lowest virtual address for the component -- could be hex, so not a VAddr pub params: Vec, // initialization parameters pub fsimg: Option, } +impl fmt::Display for Component { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}.{}", self.source, self.name) + } +} + // Input/frontend pass taking the specification, and outputing the // first intermediate representation. Expected to populate the // Component structure. From 1645b075787e99fe1df1151c4da1c76312074ca9 Mon Sep 17 00:00:00 2001 From: Gabe Parmer Date: Sat, 25 Jun 2022 11:41:52 -0400 Subject: [PATCH 4/5] Added logic to drive component's compilation path and use github repos. DONE: 1. Add support for specifying the repo in the composition script, and supporting build-system updates, and validate the previous scripts still work. TODO: 2. Validate that the external repo download and usage works. 3. Update `img` to `src` and enable it to not just have the form of two nested directories (i.e. "x.y"), but also a single subdir (i.e. "x"), or no directory (in the case that a repo is the single component). FUTURE FEATURES: I won't be able to get to these soon, but they are desirable. - Add external library and interface repo support. This is more challenging as it requires more build system logic as dependencies on libraries/interfaces are specified in the build system, not in the composition scripts. --- composition_scripts/external_hello_world.toml | 4 +- src/Makefile | 4 + src/Makefile.src | 6 ++ src/components/Makefile | 10 ++- src/components/implementation/Makefile | 12 ++- .../implementation/Makefile.subsubdir | 10 ++- src/components/repos/.gitignore | 3 + src/components/repos/README.md | 24 ++++++ src/composer/src/build.rs | 77 ++++++++++++++++-- src/composer/src/cossystem.rs | 4 +- src/composer/src/dir_location.rs | 78 +++++++++++++++++++ src/composer/src/main.rs | 6 +- src/composer/src/passes.rs | 21 ++++- src/composer/src/syshelpers.rs | 6 +- 14 files changed, 238 insertions(+), 27 deletions(-) create mode 100644 src/components/repos/.gitignore create mode 100644 src/components/repos/README.md create mode 100644 src/composer/src/dir_location.rs diff --git a/composition_scripts/external_hello_world.toml b/composition_scripts/external_hello_world.toml index 1f01d72a6c..5743810521 100644 --- a/composition_scripts/external_hello_world.toml +++ b/composition_scripts/external_hello_world.toml @@ -3,7 +3,7 @@ description = "Test for the hello world external component" [[components]] name = "hw" -img = "hw" -location = "https://www.github.com/gparmer/hello_world" +img = "" +location = "github.com:gparmer/hello_world" deps = [{srv = "kernel", interface = "init", variant = "kernel"}] constructor = "kernel" diff --git a/src/Makefile b/src/Makefile index a1cf5c4e49..48e5295882 100644 --- a/src/Makefile +++ b/src/Makefile @@ -21,6 +21,10 @@ composer: component: $(MAKE) $(MAKEOPTIONS) -C components component +# Download an external repository +repo_external: + $(MAKE) $(MAKEOPTIONS) -C components repo_external + comps: $(info ) $(info ***********************************************) diff --git a/src/Makefile.src b/src/Makefile.src index 56badc076b..4441e9cab5 100644 --- a/src/Makefile.src +++ b/src/Makefile.src @@ -11,6 +11,7 @@ KINC=$(TOP_DIR)/kernel/include/ SHAREDINC=$(TOP_DIR)/kernel/include/shared/ CHALSHAREDINC=$(TOP_DIR)/kernel/include/chal/shared/ CDIR=$(TOP_DIR)/components/ +REPO_DIR=$(CDIR)/repos # tools MAKE=make @@ -25,3 +26,8 @@ CP=cp RM=rm PLAT_FILE=$(TOP_DIR)/.PLATFORM_ID PLATFORM=$(shell cat $(PLAT_FILE)) + +# disable the terminal prompts, and fail out instead; typically fails +# if ssh isn't set up (known hosts), or if the specified repo doesn't +# exist. +GITCLONE=GIT_TERMINAL_PROMPT=0 git clone diff --git a/src/components/Makefile b/src/components/Makefile index c169efee7b..809d9b952c 100644 --- a/src/components/Makefile +++ b/src/components/Makefile @@ -26,5 +26,11 @@ init: component: $(MAKE) $(MAKEOPTIONS) -C implementation component -component_external: - $(MAKE) $(MAKEOPTIONS) -C implementation component_external +# Download a remote repository (e.g. from github). We're setting up a +# specific rule for the repo's path so that it won't redundantly clone +# the repo. Once it has been cloned, never clone it again. +REPO_ABSPATH = $(REPO_DIR)/$(REPO_PATH) +$(REPO_ABSPATH): + $(GITCLONE) $(REPO_URL) $@ + +repo_external: $(REPO_ABSPATH) diff --git a/src/components/implementation/Makefile b/src/components/implementation/Makefile index 80442cba2b..75a36c5638 100644 --- a/src/components/implementation/Makefile +++ b/src/components/implementation/Makefile @@ -24,8 +24,12 @@ clean: init: distclean: +# NOTE: we're skipping the compilation of the libraries in the +# interfaces directories (the immediate subdirectories here) on +# composer-directed compilation. Soon, we'll remove the need for +# interface directories, and will move all of their shared code into +# the `lib/` code. +# +# NOTE: COMP_PATH is relative to the `src/component` (CDIR) directory. component: - $(MAKE) $(MAKEOPTIONS) -C $(COMP_INTERFACE) component - -component_external: - $(MAKE) $(MAKEOPTIONS) -C $(COMPONENT_EXTERNAL_PATH) component + $(MAKE) $(MAKEOPTIONS) -C $(CDIR)/$(COMP_PATH) component diff --git a/src/components/implementation/Makefile.subsubdir b/src/components/implementation/Makefile.subsubdir index 76ac119096..19dddb1e7f 100644 --- a/src/components/implementation/Makefile.subsubdir +++ b/src/components/implementation/Makefile.subsubdir @@ -40,7 +40,7 @@ TAR_SYMBOL_NAME=crt_init.tar ifeq ($(PLATFORM), i386) LINKFLAG=-Xlinker -melf_i386 else ifeq ($(PLATFORM), x86_64) -LINKFLAG=-Xlinker -melf_x86_64 +LINKFLAG=-Xlinker -melf_x86_64 else ifeq ($(PLATFORM), armv7a) LINKFLAG=-Xlinker -marmelf endif @@ -80,13 +80,15 @@ COMP_DEP_OBJS=$(foreach D,$(COMP_IFDEPS_CLEAN),$(INTERDIR)/$(D)/cosrt_c_stub.o) # NOTE: we're currently ignoring the *variants* library requirements, # which will break if an interface's code requires a library + comp_header: - $(info | Composing $(COMP_INTERFACE).$(COMP_NAME) for variable $(COMP_VARNAME) by linking with:) + $(info | Composing $(if $(COMP_INTERFACE),$(COMP_INTERFACE).)$(COMP_NAME) for variable $(COMP_VARNAME) by linking with:) $(info | Exported interfaces: $(COMP_INTERFACES_CLEAN)) $(info | Interface dependencies: $(COMP_IFDEPS_CLEAN)) $(info | Libraries: $(DEPENDENCY_LIBS) $(DEPENDENCY_LIBOBJS)) + $(info | Component location: $(COMP_REPO)) -.PHONY: component +.PHONY: component comp_header component: clean comp_header $(COMPOBJ) $(if $(COMP_INITARGS_FILE), $(CC) $(INCLUDE) $(CFLAGS) -c -o $(COMP_INITARGS_FILE:%.c=%.o) $(COMP_INITARGS_FILE)) $(if $(COMP_TAR_FILE), cp $(COMP_TAR_FILE) $(TAR_SYMBOL_NAME)) @@ -95,6 +97,8 @@ component: clean comp_header $(COMPOBJ) $(MUSLCC) $(COMPNAME).linked_libs_ifs.o $(MUSLCFLAGS) $(LINKFLAG) -o $(COMPNAME).linked_musl.o $(LD) $(LDFLAGS) -Ttext=$(COMP_BASEADDR) -T $(COMP_LD_SCRIPT) -o $(COMP_OUTPUT) $(COMPNAME).linked_musl.o +component_external: + %.o:%.c $(info | [CC] $<: Compiling) @$(CC) $(INCLUDE) $(CFLAGS) -o $@ -c $< diff --git a/src/components/repos/.gitignore b/src/components/repos/.gitignore new file mode 100644 index 0000000000..7c9d611b59 --- /dev/null +++ b/src/components/repos/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!README.md diff --git a/src/components/repos/README.md b/src/components/repos/README.md new file mode 100644 index 0000000000..f4262cbffe --- /dev/null +++ b/src/components/repos/README.md @@ -0,0 +1,24 @@ +# External Repositories + +Composite current enables components to be defined externally from the main repo, thus providing increased autonomy in development. +Each repository in this directory is pulled from github, and can contain a set of components and composition scripts. +These repositories can be thought of as part of the Composite ecosystem, but not as part of the "standard library" of necessary functionalities. + +A number of awkward realities with the current system: + +- Composite does *not* maintain the repos downloaded here in any way. + If the upstream repo is changed, you must manually do a `git pull` in the repo (or remove it so it is re-cloned). + Comparably, if you update the external repo, you should manually commit/push, etc... +- The Composite repo ignores all repos in here. + Never add them. + The proper course of action would be to do a PR to the main Composite repo adding your component instead. + +## TODO: Libraries and Interfaces + +It should be possible to also define libraries and interfaces in external repositories. +This is more complicated as the dependencies for these are + +1. not properly maintained by the composer (libraries aren't properly rebuilt when one of their dependencies changed), and +2. these dependencies are tracked by the build system, so updating them to enable dependencies are repositories requires most structural updates to the build system. + +There are no hard blockers to doing this; we just need more time and will! diff --git a/src/composer/src/build.rs b/src/composer/src/build.rs index 33e2fd1c6f..b4a4e4de54 100644 --- a/src/composer/src/build.rs +++ b/src/composer/src/build.rs @@ -40,23 +40,28 @@ use tar::Builder; // - COMP_BASEADDR - the base address of .text for the component // - COMP_INITARGS_FILE - the path to the generated initial arguments .c file // - COMP_TAR_FILE - the path to an initargs tarball to compile into the component +// - COMP_PATH - path to the component directory relative to `src/components` // // Building external repositories (not in the main composite repo) // doesn't use COMP_INTERFACE and instead uses the following: // -// - COMP_EXTERNAL_REPO - The URL to the to the external repo that -// holds the component. +// - REPO_URL - The URL to the to the external repo that holds the +// component. +// - REPO_PATH - The path within the repo directory into which the +// external repo should be cloned. // // In the end, this should result in a command line for each component // along these (artificial) lines: // -// `make COMP_INTERFACES="pong/log" COMP_IFDEPS="capmgr/stubs+sched/lock" COMP_LIBS="ps heap" COMP_INTERFACE=pong COMP_NAME=pingpong COMP_VARNAME=pongcomp component` +// `make COMP_INTERFACES="pong/log" COMP_IFDEPS="capmgr/stubs+sched/lock" COMP_LIBS="ps heap" COMP_INTERFACE=pong COMP_NAME=pingpong COMP_PATH="implementation/pong/pingpong COMP_VARNAME=pongcomp component` // // ...which should output the executable pong.pingpong.pongcomp in the // build directory which is the "sealed" version of the component that // is ready for loading. -// +// A key design goal of the composer and Composite build system is +// that the absolute paths are all taken care of by the build system, +// thus this code focuses on simply properly guiding the build system. // The key within the initargs for the tarball, the path of the // tarball, and the set of paths to the files to include in the @@ -283,6 +288,13 @@ fn comp_gen_make_cmd( ifpath.push_str(&d.variant.clone()); (false, ifpath) }); + // TODO: replace the above with this if it is the same + let deps_test = ds + .iter() + .map(|d| format!("{}/{}", d.interface, d.variant)) + .collect::>() + .join("+"); + assert_eq!(if_deps, deps_test); let mut optional_cmds = String::from(""); optional_cmds.push_str(&format!("COMP_INITARGS_FILE={} ", args_file)); @@ -294,10 +306,11 @@ fn comp_gen_make_cmd( // unwrap as we've already validated the name. let compid = s.get_named().rmap().get(&c.name).unwrap(); let baseaddr = s.get_address_assignments().component_baseaddr(compid); + let comp_location = s.get_dir_location().dir_location(&c.name); let cmd = format!( - r#"make -C src COMP_INTERFACES="{}" COMP_IFDEPS="{}" COMP_LIBDEPS="" COMP_INTERFACE={} COMP_NAME={} COMP_VARNAME={} COMP_OUTPUT={} COMP_BASEADDR={:#X} {} component"#, - if_exp, if_deps, &decomp[0], &decomp[1], &c.name, output_name, baseaddr, &optional_cmds + r#"make -C src COMP_INTERFACES="{}" COMP_IFDEPS="{}" COMP_LIBDEPS="" COMP_INTERFACE={} COMP_NAME={} COMP_VARNAME={} COMP_OUTPUT={} COMP_PATH={} COMP_BASEADDR={:#X} {} component"#, + if_exp, if_deps, &decomp[0], &decomp[1], &c.name, output_name, comp_location, baseaddr, &optional_cmds ); cmd @@ -310,6 +323,13 @@ fn kern_gen_make_cmd(input_constructor: &String, kern_output: &String, _s: &Syst ) } +fn repo_download_gen_make_cmd(repo_url: &String, repo_path: &String) -> String { + format!( + r#"make -C src REPO_URL="{}" REPO_PATH="{}" repo_external"#, + repo_url, repo_path + ) +} + pub struct DefaultBuilder { builddir: String, } @@ -469,4 +489,49 @@ impl BuildState for DefaultBuilder { Ok(()) } + + // Take the repo specification from the specification, and the + // path in `src/components/` for the repo. + fn repo_download(&self, repo_url: &String, repo_path: &String, _s: &SystemState) -> Result<(), String> { + let cmd = repo_download_gen_make_cmd(&repo_url, &repo_path); + let (_out, err) = exec_pipeline(vec![cmd.clone()]); + + if err.len() > 0 { + let lines = err.lines(); + let correct_str = "Cloning into "; + let correct_str_len = correct_str.len(); + + let errs: String = lines + .filter_map(|l| { + if l.chars().take(correct_str_len).collect::() != correct_str { + None + } else { + Some(l) + } + }) + .collect(); + + return Err( + format!("Error creating external repository \"{}\":\n", repo_url) + + &errs + + "\nCommon causes of this error include:\n" + + "1. The specified repository doesn't exist (i.e. incorrect URL).\n" + + "2. `git` isn't installed.\n"MUSTMUST + + "3. `ssh` isn't installed (we access the repo using git's s + + "3. `ssh` isn't installed (we access the repo using gitshMUST support).\n" + + "4. `ssh` asked for user input because the known hosts aren't set up.\n", + ); + } + + Ok(()) + } +} + + + "3. `ssh` isn't installed (we access the repo using gitshMUST support).\n" + + "4. `ssh` asked for user input because the known hosts aren't set up.\n", + ); + } + + Ok(()) + } } diff --git a/src/composer/src/cossystem.rs b/src/composer/src/cossystem.rs index b8991c7118..82e2cfe83f 100644 --- a/src/composer/src/cossystem.rs +++ b/src/composer/src/cossystem.rs @@ -32,7 +32,7 @@ pub struct InterfaceVariant { pub struct TomlComponent { name: String, img: String, - location: Option, + repo: Option, baseaddr: Option, deps: Option>, params: Option>, @@ -510,7 +510,7 @@ impl Transition for SystemSpec { constructor: ComponentName::new(&c.constructor, &String::from("global")), scheduler: sched_name, source: c.img.clone(), - location: c.location.clone(), + repo: c.repo.clone(), base_vaddr: c .baseaddr .as_ref() diff --git a/src/composer/src/dir_location.rs b/src/composer/src/dir_location.rs new file mode 100644 index 0000000000..758a7060f7 --- /dev/null +++ b/src/composer/src/dir_location.rs @@ -0,0 +1,78 @@ +use passes::{BuildState, ComponentName, DirLocationPass, SystemState, Transition}; +use std::collections::HashMap; + +pub struct Repos { + dirs: HashMap, +} + +impl DirLocationPass for Repos { + fn dir_location(&self, c: &ComponentName) -> &String { + // unwrap valid as a location is added for each component + self.dirs.get(&c).unwrap() + } +} + +impl Transition for Repos { + fn transition(s: &SystemState, b: &mut dyn BuildState) -> Result, String> { + let spec = s.get_spec(); + let mut repos: Vec<&String> = spec + .names() + .iter() + .filter_map(|cn| spec.component_named(cn).repo.as_ref()) + .collect(); + repos.sort(); + repos.dedup(); + let comp_locations = spec.names().iter().map(|c| { + ( + c.clone(), + &spec.component_named(&c).source, + &spec.component_named(&c).repo, + ) + }); + + // Download any repos that aren't already present. + for r in repos { + // Lets parse the repo into the URL to pass to `git`, and into + // the path in the source repo. + let repo_split: Vec<&str> = r.split(':').collect(); + if repo_split.len() != 2 { + return Err(format!( + r#"Repo specification "{}" must have the form : (e.g. "github.com:gparmer/hello_world").\n"#, + r + )); + } + if repo_split[0] != "github" { + return Err(format!( + r#"Repo specification provider "{}" is not supported (supported: "github").\n"#, + repo_split[0] + )); + } + + let repo_url = format!("git@github.com:{}", repo_split[0]); + let repo_path = repo_split[1].replace("/", "-"); + + b.repo_download(&repo_url, &repo_path, &s)?; + } + + // Add the paths of the components (in external repos, and in + // the standard source) to the DirLocationpass-queryable + // hashmap. + let mut dirs = HashMap::new(); + for (name, source, location) in comp_locations { + let loc = source + .as_str() + .split(&['.', '/'][..]) + .collect::>() + .join("/"); + dirs.insert( + name, + match location { + Some(r) => r.clone() + &loc, + None => String::from("implementation/") + &loc, + }, + ); + } + + Ok(Box::new(Repos { dirs })) + } +} diff --git a/src/composer/src/main.rs b/src/composer/src/main.rs index ac7fb57b48..25b5b8a7f3 100644 --- a/src/composer/src/main.rs +++ b/src/composer/src/main.rs @@ -18,6 +18,7 @@ mod syshelpers; mod tot_order; mod properties; mod address_assignment; +mod dir_location; use build::DefaultBuilder; use compobject::{Constructor, ElfObject}; @@ -27,9 +28,10 @@ use invocations::Invocations; use passes::{BuildState, ComponentId, SystemState, Transition, TransitionIter}; use resources::ResAssignPass; use properties::CompProperties; -use std::env; use tot_order::CompTotOrd; use address_assignment::AddressAssignmentx86_64; +use dir_location::Repos; +use std::env; pub fn exec() -> Result<(), String> { let mut args = env::args(); @@ -51,10 +53,10 @@ pub fn exec() -> Result<(), String> { sys.add_parsed(SystemSpec::transition(&sys, &mut build)?); sys.add_named(CompTotOrd::transition(&sys, &mut build)?); + sys.add_dir_location(Repos::transition(&sys, &mut build)?); sys.add_address_assign(AddressAssignmentx86_64::transition(&sys, &mut build)?); sys.add_properties(CompProperties::transition(&sys, &mut build)?); sys.add_restbls(ResAssignPass::transition(&sys, &mut build)?); - sys.add_repos(RepoPass::transition(&sys, &mut build)?); // process these in reverse order of dependencies (e.g. booter last) let reverse_ids: Vec = sys diff --git a/src/composer/src/passes.rs b/src/composer/src/passes.rs index 051cc47456..cf64b30927 100644 --- a/src/composer/src/passes.rs +++ b/src/composer/src/passes.rs @@ -21,6 +21,7 @@ pub struct SystemState { parse: Option>, named: Option>, + dir_location: Option>, address_assignment: Option>, properties: Option>, restbls: Option>, @@ -36,6 +37,7 @@ impl SystemState { spec, parse: None, named: None, + dir_location: None, address_assignment: None, properties: None, restbls: None, @@ -54,6 +56,10 @@ impl SystemState { self.named = Some(n); } + pub fn add_dir_location(&mut self, l: Box) { + self.dir_location = Some(l); + } + pub fn add_address_assign(&mut self, a: Box) { self.address_assignment = Some(a); } @@ -94,6 +100,10 @@ impl SystemState { &**(self.named.as_ref().unwrap()) } + pub fn get_dir_location(&self) -> &dyn DirLocationPass { + &**(self.dir_location.as_ref().unwrap()) + } + pub fn get_address_assignments(&self) -> &dyn AddressAssignmentPass { &**(self.address_assignment.as_ref().unwrap()) } @@ -143,6 +153,7 @@ pub trait BuildState { fn comp_build(&self, c: &ComponentId, state: &SystemState) -> Result; // build the component, and return the path to the resulting object fn constructor_build(&self, c: &ComponentId, state: &SystemState) -> Result; // build a constructor, including all components it is responsible for booting fn kernel_build(&self, kern_output: &String, constructor_input: &String, s: &SystemState) -> Result<(), String>; // build the final kernel image + fn repo_download(&self, github_repo: &String, dir_location: &String, s: &SystemState) -> Result<(), String>; } // The following describes the means of transitioning the system @@ -210,7 +221,7 @@ pub struct Component { pub scheduler: ComponentName, // our scheduler (that creates or initial thread) pub source: String, // Where is the component source located in repo? - pub location: Option, // ...and should we use an external repo? + pub repo: Option, // ...and should we use an external repo? pub base_vaddr: String, // The lowest virtual address for the component -- could be hex, so not a VAddr pub params: Vec, // initialization parameters pub fsimg: Option, @@ -258,6 +269,14 @@ pub trait SpecificationPass { fn address_spaces(&self) -> &AddrSpaces; } +// Find and initialize the location in the FS of a component. This is +// in the main Composite repo by default (in +// `components/implementation/*`), and in `components/repos/` otherwise. +pub trait DirLocationPass { + // return the path to the directory to start building the component. + fn dir_location(&self, &ComponentName) -> &String; +} + // Integer namespacing pass. Convert the component variable names to // component ids, and create a total order for the components based on // their stated dependencies (lower ids are more trusted (more diff --git a/src/composer/src/syshelpers.rs b/src/composer/src/syshelpers.rs index c660dcfcca..f8ebae7260 100644 --- a/src/composer/src/syshelpers.rs +++ b/src/composer/src/syshelpers.rs @@ -62,9 +62,5 @@ pub fn reset_dir(dirname: &String) -> Result<(), String> { } pub fn dir_exists(dirname: &String) -> bool { - if let Err(_) = fs::read_dir(&dirname) { - false - } else { - true - } + fs::read_dir(&dirname).is_ok() } From 115b8e46b2e269ef4499abf9d7a4b4acee9c161d Mon Sep 17 00:00:00 2001 From: Gabe Parmer Date: Sun, 26 Jun 2022 17:47:55 -0400 Subject: [PATCH 5/5] External components can be pulled into the system. Demonstrated with `external_hello_world.toml`, and the `github.com/gparmer/hello_world/` component. DONE: 1. Add support for specifying the repo in the composition script, and supporting build-system updates, and validate the previous scripts still work. 2. Validate that the external repo download and usage works. TODO: 3. Update `img` to `src` and enable it to not just have the form of two nested directories (i.e. "x.y"), but also a single subdir (i.e. "x"), or no directory (in the case that a repo ish. FUTURE FEATURES: I won't be able to get to these soon, but they are desirable. - Enable the composition scripts to be referenced in github repos, rather than only in local files. - Add external library and interface repo support. This is more challenging as it requires more build system logic as dependencies on libraries/interfaces are specified in the build system, not in the composition scripts. --- composition_scripts/external_hello_world.toml | 4 +- src/components/Makefile | 1 + .../implementation/Makefile.subsubdir | 3 +- src/composer/src/build.rs | 62 ++++++------------- src/composer/src/dir_location.rs | 15 +++-- 5 files changed, 31 insertions(+), 54 deletions(-) diff --git a/composition_scripts/external_hello_world.toml b/composition_scripts/external_hello_world.toml index 5743810521..a5c81a96a3 100644 --- a/composition_scripts/external_hello_world.toml +++ b/composition_scripts/external_hello_world.toml @@ -2,8 +2,8 @@ description = "Test for the hello world external component" [[components]] -name = "hw" +name = "hw1" img = "" -location = "github.com:gparmer/hello_world" +repo = "github:gparmer/hello_world" deps = [{srv = "kernel", interface = "init", variant = "kernel"}] constructor = "kernel" diff --git a/src/components/Makefile b/src/components/Makefile index 809d9b952c..014545ec26 100644 --- a/src/components/Makefile +++ b/src/components/Makefile @@ -1,3 +1,4 @@ +include Makefile.src MAKEOPTIONS+= -I$(shell pwd) export MAKEOPTIONS diff --git a/src/components/implementation/Makefile.subsubdir b/src/components/implementation/Makefile.subsubdir index 19dddb1e7f..a784e7b25f 100644 --- a/src/components/implementation/Makefile.subsubdir +++ b/src/components/implementation/Makefile.subsubdir @@ -82,11 +82,10 @@ COMP_DEP_OBJS=$(foreach D,$(COMP_IFDEPS_CLEAN),$(INTERDIR)/$(D)/cosrt_c_stub.o) comp_header: - $(info | Composing $(if $(COMP_INTERFACE),$(COMP_INTERFACE).)$(COMP_NAME) for variable $(COMP_VARNAME) by linking with:) + $(info | Composing component in src/components/$(COMP_PATH) for variable $(COMP_VARNAME) by linking with:) $(info | Exported interfaces: $(COMP_INTERFACES_CLEAN)) $(info | Interface dependencies: $(COMP_IFDEPS_CLEAN)) $(info | Libraries: $(DEPENDENCY_LIBS) $(DEPENDENCY_LIBOBJS)) - $(info | Component location: $(COMP_REPO)) .PHONY: component comp_header component: clean comp_header $(COMPOBJ) diff --git a/src/composer/src/build.rs b/src/composer/src/build.rs index b4a4e4de54..2804606536 100644 --- a/src/composer/src/build.rs +++ b/src/composer/src/build.rs @@ -264,53 +264,31 @@ fn comp_gen_make_cmd( let ds = deps(&s, id); let exports = exports(&s, id); - let (_, if_exp) = exports + let if_exp = exports .iter() - .fold((true, String::from("")), |(first, accum), e| { - let mut ifpath = accum.clone(); - if !first { - ifpath.push_str("+"); - } - ifpath.push_str(&e.interface.clone()); - ifpath.push_str("/"); - ifpath.push_str(&e.variant.clone()); - (false, ifpath) - }); - let (_, if_deps) = ds - .iter() - .fold((true, String::from("")), |(first, accum), d| { - let mut ifpath = accum.clone(); - if !first { - ifpath.push_str("+"); - } - ifpath.push_str(&d.interface.clone()); - ifpath.push_str("/"); - ifpath.push_str(&d.variant.clone()); - (false, ifpath) - }); - // TODO: replace the above with this if it is the same - let deps_test = ds + .map(|e| format!("{}/{}", e.interface, e.variant)) + .collect::>() + .join("+"); + let if_deps = ds .iter() .map(|d| format!("{}/{}", d.interface, d.variant)) .collect::>() .join("+"); - assert_eq!(if_deps, deps_test); let mut optional_cmds = String::from(""); optional_cmds.push_str(&format!("COMP_INITARGS_FILE={} ", args_file)); if let Some(s) = tar_file { optional_cmds.push_str(&format!("COMP_TAR_FILE={} ", s)); } - let decomp: Vec<&str> = c.source.split(".").collect(); - assert!(decomp.len() == 2); + // unwrap as we've already validated the name. let compid = s.get_named().rmap().get(&c.name).unwrap(); let baseaddr = s.get_address_assignments().component_baseaddr(compid); let comp_location = s.get_dir_location().dir_location(&c.name); let cmd = format!( - r#"make -C src COMP_INTERFACES="{}" COMP_IFDEPS="{}" COMP_LIBDEPS="" COMP_INTERFACE={} COMP_NAME={} COMP_VARNAME={} COMP_OUTPUT={} COMP_PATH={} COMP_BASEADDR={:#X} {} component"#, - if_exp, if_deps, &decomp[0], &decomp[1], &c.name, output_name, comp_location, baseaddr, &optional_cmds + r#"make -C src COMP_INTERFACES="{}" COMP_IFDEPS="{}" COMP_LIBDEPS="" COMP_VARNAME={} COMP_OUTPUT={} COMP_PATH={} COMP_BASEADDR={:#X} {} component"#, + if_exp, if_deps, &c.name, output_name, comp_location, baseaddr, &optional_cmds ); cmd @@ -325,7 +303,7 @@ fn kern_gen_make_cmd(input_constructor: &String, kern_output: &String, _s: &Syst fn repo_download_gen_make_cmd(repo_url: &String, repo_path: &String) -> String { format!( - r#"make -C src REPO_URL="{}" REPO_PATH="{}" repo_external"#, + r#"make -C src REPO_URL={} REPO_PATH={} repo_external"#, repo_url, repo_path ) } @@ -492,8 +470,14 @@ impl BuildState for DefaultBuilder { // Take the repo specification from the specification, and the // path in `src/components/` for the repo. - fn repo_download(&self, repo_url: &String, repo_path: &String, _s: &SystemState) -> Result<(), String> { + fn repo_download( + &self, + repo_url: &String, + repo_path: &String, + _s: &SystemState, + ) -> Result<(), String> { let cmd = repo_download_gen_make_cmd(&repo_url, &repo_path); + println!("Executing `git clone {}`, which may ask for your password...", repo_url); let (_out, err) = exec_pipeline(vec![cmd.clone()]); if err.len() > 0 { @@ -516,18 +500,8 @@ impl BuildState for DefaultBuilder { + &errs + "\nCommon causes of this error include:\n" + "1. The specified repository doesn't exist (i.e. incorrect URL).\n" - + "2. `git` isn't installed.\n"MUSTMUST - + "3. `ssh` isn't installed (we access the repo using git's s - + "3. `ssh` isn't installed (we access the repo using gitshMUST support).\n" - + "4. `ssh` asked for user input because the known hosts aren't set up.\n", - ); - } - - Ok(()) - } -} - - + "3. `ssh` isn't installed (we access the repo using gitshMUST support).\n" + + "2. `git` isn't installed.\n" + + "3. `ssh` isn't installed (we access the repo using git's ssh support).\n" + "4. `ssh` asked for user input because the known hosts aren't set up.\n", ); } diff --git a/src/composer/src/dir_location.rs b/src/composer/src/dir_location.rs index 758a7060f7..924c9bb61c 100644 --- a/src/composer/src/dir_location.rs +++ b/src/composer/src/dir_location.rs @@ -30,6 +30,10 @@ impl Transition for Repos { ) }); + fn repo_dir_create(r: &String) -> String { + r.replace("/", "-").replace(":", "-") + } + // Download any repos that aren't already present. for r in repos { // Lets parse the repo into the URL to pass to `git`, and into @@ -48,17 +52,16 @@ impl Transition for Repos { )); } - let repo_url = format!("git@github.com:{}", repo_split[0]); - let repo_path = repo_split[1].replace("/", "-"); + let repo_url = format!("git@github.com:{}", repo_split[1]); - b.repo_download(&repo_url, &repo_path, &s)?; + b.repo_download(&repo_url, &repo_dir_create(&r), &s)?; } // Add the paths of the components (in external repos, and in // the standard source) to the DirLocationpass-queryable // hashmap. let mut dirs = HashMap::new(); - for (name, source, location) in comp_locations { + for (name, source, repo) in comp_locations { let loc = source .as_str() .split(&['.', '/'][..]) @@ -66,8 +69,8 @@ impl Transition for Repos { .join("/"); dirs.insert( name, - match location { - Some(r) => r.clone() + &loc, + match repo { + Some(r) => format!("repos/{}/{}", repo_dir_create(&r.clone()), &loc), None => String::from("implementation/") + &loc, }, );