20230206 refresh compile file

resources/tests/hello.bin

@@ -0,0 +1 @@
+hello

resources/tests/secret_number.cl

@@ -0,0 +1,7 @@
+(mod (A)
+  (include *standard-cl-21*)
+  (include test_sub_include.cl)
+  (let ((X 19))
+    (+ A X)
+    )
+  )

resources/tests/secret_number2.cl

@@ -0,0 +1,7 @@
+(mod (A)
+  (include *standard-cl-21*)
+  (include test_sub_include.cl)
+  (let ((X 13))
+    (+ A X)
+    )
+  )

resources/tests/test_sub_include.cl

@@ -0,0 +1,2 @@
+(
+)

resources/tests/test_treehash_constant.cl

@@ -0,0 +1,8 @@
+(mod ()
+  (include sha256tree.clib)
+  (compile-file secret-number secret_number.cl)
+  (defconst A (sha256 1 H))
+  (defconst H (sha256tree secret-number))
+  (defconst I (sha256 1 H))
+  H
+  )

resources/tests/test_treehash_constant_2.cl

@@ -0,0 +1,8 @@
+(mod ()
+  (include sha256tree.clib)
+  (compile-file secret-number secret_number2.cl)
+  (defconst A (sha256 1 H))
+  (defconst H (sha256tree secret-number))
+  (defconst I (sha256 1 H))
+  H
+  )

resources/tests/test_treehash_constant_21.cl

@@ -0,0 +1,9 @@
+(mod ()
+  (include *standard-cl-21*)
+  (include sha256tree.clib)
+  (compile-file secret-number secret_number.cl)
+  (defconst A (sha256 1 H))
+  (defconst H (sha256tree secret-number))
+  (defconst I (sha256 1 H))
+  H
+  )

resources/tests/test_treehash_constant_21_2.cl

@@ -0,0 +1,9 @@
+(mod ()
+  (include *standard-cl-21*)
+  (include sha256tree.clib)
+  (compile-file secret-number secret_number2.cl)
+  (defconst A (sha256 1 H))
+  (defconst H (sha256tree secret-number))
+  (defconst I (sha256 1 H))
+  H
+  )

src/classic/clvm_tools/clvmc.rs

@@ -38,8 +38,9 @@ pub fn write_sym_output(
         .map(|_| ())
 }
 
-pub fn compile_clvm_text(
+pub fn compile_clvm_text_maybe_opt(
     allocator: &mut Allocator,
+    do_optimize: bool,
     opts: Rc<dyn CompilerOpts>,
     symbol_table: &mut HashMap<String, String>,
     text: &str,
@@ -57,10 +58,16 @@ pub fn compile_clvm_text(
     // to get more members that are somewhat independent.
     if let Some(stepping) = dialect.stepping {
         let runner = Rc::new(DefaultProgramRunner::new());
-        let opts = opts.set_optimize(true).set_frontend_opt(stepping > 21);
+        let opts = opts
+            .set_optimize(do_optimize)
+            .set_frontend_opt(stepping > 21);
 
         let unopt_res = compile_file(allocator, runner.clone(), opts, text, symbol_table);
-        let res = unopt_res.and_then(|x| run_optimizer(allocator, runner, Rc::new(x)));
+        let res = if do_optimize {
+            unopt_res.and_then(|x| run_optimizer(allocator, runner, Rc::new(x)))
+        } else {
+            unopt_res.map(Rc::new)
+        };
 
         res.and_then(|x| {
             convert_to_clvm_rs(allocator, x).map_err(|r| match r {
@@ -82,6 +89,25 @@ pub fn compile_clvm_text(
     }
 }
 
+pub fn compile_clvm_text(
+    allocator: &mut Allocator,
+    opts: Rc<dyn CompilerOpts>,
+    symbol_table: &mut HashMap<String, String>,
+    text: &str,
+    input_path: &str,
+    classic_with_opts: bool,
+) -> Result<NodePtr, EvalErr> {
+    compile_clvm_text_maybe_opt(
+        allocator,
+        true,
+        opts,
+        symbol_table,
+        text,
+        input_path,
+        classic_with_opts,
+    )
+}
+
 pub fn compile_clvm_inner(
     allocator: &mut Allocator,
     opts: Rc<dyn CompilerOpts>,

src/classic/clvm_tools/stages/stage_2/compile.rs

@@ -10,11 +10,17 @@ use crate::classic::clvm::OPERATORS_LATEST_VERSION;
 use crate::classic::clvm::{keyword_from_atom, keyword_to_atom};
 
 use crate::classic::clvm_tools::binutils::{assemble, disassemble};
+use crate::classic::clvm_tools::clvmc::{compile_clvm_text, write_sym_output};
 use crate::classic::clvm_tools::node_path::NodePath;
 use crate::classic::clvm_tools::stages::stage_0::TRunProgram;
 use crate::classic::clvm_tools::stages::stage_2::defaults::default_macro_lookup;
 use crate::classic::clvm_tools::stages::stage_2::helpers::{brun, evaluate, quote};
 use crate::classic::clvm_tools::stages::stage_2::module::compile_mod;
+use crate::classic::clvm_tools::stages::stage_2::reader::read_file;
+
+use crate::compiler::compiler::DefaultCompilerOpts;
+use crate::compiler::comptypes::CompilerOpts;
+use crate::compiler::sexp::decode_string;
 
 const DIAG_OUTPUT: bool = false;
 
@@ -844,3 +850,63 @@ pub fn make_symbols_name(current_filename: &str, name: &str) -> String {
 
     format!("{take_start}_{take_end}.sym")
 }
+
+pub fn compile_file(
+    runner: Rc<dyn TRunProgram>,
+    allocator: &mut Allocator,
+    parent_sexp: NodePtr,
+    name: &str,
+    filename: &str,
+) -> Response {
+    let search_paths = get_search_paths(runner.clone(), allocator)?;
+    let opts = Rc::new(DefaultCompilerOpts::new(filename)).set_search_paths(&search_paths);
+    let mut symtab = HashMap::new();
+    let current_filename = get_compile_filename(runner.clone(), allocator)?;
+    let file = read_file(runner, allocator, parent_sexp, filename)?;
+    let compiled = compile_clvm_text(
+        allocator,
+        opts,
+        &mut symtab,
+        &decode_string(&file.data),
+        &file.full_path,
+        true,
+    )?;
+
+    // Write symbols for the compiled inner module.
+    if let Some(filename) = current_filename {
+        let target_symbols_name = make_symbols_name(&filename, name);
+
+        // Not a hard error if we can't write the symbols,
+        // given the way most write chialisp.
+        write_sym_output(&symtab, &target_symbols_name).ok();
+    }
+
+    Ok(Reduction(1, compiled))
+}
+
+pub fn process_compile_file(
+    allocator: &mut Allocator,
+    runner: Rc<dyn TRunProgram>,
+    declaration_sexp: NodePtr,
+    name: Vec<u8>,
+) -> Result<(Vec<u8>, NodePtr), EvalErr> {
+    // A fancy include for the compilation result of a program.
+    let r_of_declaration = rest(allocator, declaration_sexp)?;
+    let rr_of_declaration = rest(allocator, r_of_declaration)?;
+    let frr_of_declaration = first(allocator, rr_of_declaration)?;
+    if let SExp::Atom = allocator.sexp(frr_of_declaration) {
+        // Referenced above.
+        let b_name = allocator.atom(frr_of_declaration).to_vec();
+        let compiled_output = compile_file(
+            runner,
+            allocator,
+            declaration_sexp,
+            &Bytes::new(Some(BytesFromType::Raw(name.clone()))).decode(),
+            &Bytes::new(Some(BytesFromType::Raw(b_name))).decode(),
+        )?;
+
+        Ok((name, quote(allocator, compiled_output.1)?))
+    } else {
+        Err(EvalErr(declaration_sexp, "expected filename".to_string()))
+    }
+}

src/classic/clvm_tools/stages/stage_2/module.rs

@@ -15,6 +15,7 @@ use crate::classic::clvm_tools::debug::{build_symbol_dump, FunctionExtraInfo};
 use crate::classic::clvm_tools::node_path::NodePath;
 use crate::classic::clvm_tools::stages::assemble;
 use crate::classic::clvm_tools::stages::stage_0::TRunProgram;
+use crate::classic::clvm_tools::stages::stage_2::compile::process_compile_file;
 use crate::classic::clvm_tools::stages::stage_2::helpers::{evaluate, quote};
 use crate::classic::clvm_tools::stages::stage_2::inline::{
     formulate_path_selections_for_destructuring, is_at_capture, is_inline_destructure,
@@ -351,6 +352,11 @@ fn parse_mod_sexp(
             process_embed_file(allocator, run_program.clone(), declaration_sexp)?;
         constants.insert(name, constant);
         Ok(())
+    } else if op == "compile-file".as_bytes() {
+        let (name, constant) =
+            process_compile_file(allocator, run_program.clone(), declaration_sexp, name)?;
+        constants.insert(name, constant);
+        Ok(())
     } else if namespace.contains(&name) {
         Err(EvalErr(
             declaration_sexp,

src/compiler/compiler.rs

@@ -267,6 +267,9 @@ impl CompilerOpts for DefaultCompilerOpts {
         Rc::new(copy)
     }
     fn set_search_paths(&self, dirs: &[String]) -> Rc<dyn CompilerOpts> {
+        if dirs.len() == 1 && dirs[0].len() == 1 {
+            todo!();
+        }
         let mut copy = self.clone();
         copy.include_dirs = dirs.to_owned();
         Rc::new(copy)

src/compiler/comptypes.rs

@@ -292,6 +292,9 @@ pub enum IncludeProcessType {
     Hex,
     /// Read clvm in s-expression form as a clvm value.
     SExpression,
+    /// The file is a program that should be compiled as clvm.
+    /// The output is s-expression data.
+    Compiled,
 }
 
 /// A description of an include form.  Here, records the locations of the various

src/compiler/preprocessor.rs

@@ -5,8 +5,10 @@ use std::rc::Rc;
 use clvmr::allocator::Allocator;
 
 use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType};
+use crate::classic::clvm_tools::clvmc::compile_clvm_text_maybe_opt;
 
 use crate::compiler::cldb::hex_to_modern_sexp;
+use crate::compiler::clvm::convert_from_clvm_rs;
 use crate::compiler::compiler::KNOWN_DIALECTS;
 use crate::compiler::comptypes::{CompileErr, CompilerOpts, IncludeDesc, IncludeProcessType};
 use crate::compiler::runtypes::RunFailure;
@@ -108,6 +110,23 @@ fn process_embed(
 
             parsed[0].clone()
         }
+        IncludeProcessType::Compiled => {
+            let decoded_content = decode_string(&content);
+            let mut symtab = HashMap::new();
+            let newly_compiled = compile_clvm_text_maybe_opt(
+                &mut allocator,
+                opts.optimize(),
+                opts,
+                &mut symtab,
+                &decoded_content,
+                &full_name,
+                true,
+            )
+            .map_err(|e| CompileErr(loc.clone(), format!("Subcompile failed: {}", e.1)))?;
+
+            convert_from_clvm_rs(&mut allocator, loc.clone(), newly_compiled)
+                .map_err(run_to_compile_err)?
+        }
     };
 
     Ok(vec![compose_defconst(loc, constant_name, content)])
@@ -122,6 +141,7 @@ fn process_pp_form(
     // Support using the preprocessor to collect dependencies recursively.
     let recurse_dependencies = |opts: Rc<dyn CompilerOpts>,
                                 includes: &mut Vec<IncludeDesc>,
+                                kind: IncludeProcessType,
                                 desc: IncludeDesc|
      -> Result<(), CompileErr> {
         let name_string = decode_string(&desc.name);
@@ -135,6 +155,10 @@ fn process_pp_form(
             ..desc
         });
 
+        if !matches!(kind, IncludeProcessType::Compiled) {
+            return Ok(());
+        }
+
         let parsed = parse_sexp(Srcloc::start(&full_name), content.iter().copied())
             .map_err(|e| CompileErr(e.0, e.1))?;
         if parsed.is_empty() {
@@ -167,6 +191,22 @@ fn process_pp_form(
                     }
                 }
 
+                [SExp::Atom(kl, compile_file), SExp::Atom(_, name), SExp::Atom(nl, fname)] => {
+                    if compile_file == b"compile-file" {
+                        return Ok(Some(IncludeType::Processed(
+                            IncludeDesc {
+                                kw: kl.clone(),
+                                nl: nl.clone(),
+                                kind: Some(IncludeProcessType::Compiled),
+
+                                name: fname.clone(),
+                            },
+                            IncludeProcessType::Compiled,
+                            name.clone()
+                        )));
+                    }
+                }
+
                 // Accepted forms:
                 // (embed-file varname bin file.dat)
                 // (embed-file varname sexp file.clvm)
@@ -238,16 +278,24 @@ fn process_pp_form(
 
     match include_type {
         Some(IncludeType::Basic(f)) => {
-            recurse_dependencies(opts.clone(), includes, f.clone())?;
+            recurse_dependencies(
+                opts.clone(),
+                includes,
+                IncludeProcessType::Compiled,
+                f.clone(),
+            )?;
             process_include(opts, f)
         }
-        Some(IncludeType::Processed(f, kind, name)) => process_embed(
-            body.loc(),
-            opts,
-            &Bytes::new(Some(BytesFromType::Raw(f.name.to_vec()))).decode(),
-            kind,
-            &name,
-        ),
+        Some(IncludeType::Processed(f, kind, name)) => {
+            recurse_dependencies(opts.clone(), includes, kind.clone(), f.clone())?;
+            process_embed(
+                body.loc(),
+                opts,
+                &Bytes::new(Some(BytesFromType::Raw(f.name.to_vec()))).decode(),
+                kind,
+                &name,
+            )
+        }
         _ => Ok(vec![body]),
     }
 }