From 936f279dff02f0dafa12306c96198817b69d0199 Mon Sep 17 00:00:00 2001 From: ZhaoXi Date: Thu, 21 Dec 2023 17:10:37 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9E=20fix(penglai-enclave-driver):=20f?= =?UTF-8?q?ree=20metadata=20&=20pmp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the problem that metadata memory is not fully freed when clearing pmp issues#100 --- .../penglai-enclave-driver.c | 2 +- .../penglai-enclave-elfloader.c | 36 ++++++++++- .../penglai-enclave-ioctl.c | 63 +++++++++++++++---- .../penglai-enclave-ioctl.h | 11 ++++ penglai-enclave-driver/penglai-enclave.h | 4 +- 5 files changed, 101 insertions(+), 15 deletions(-) diff --git a/penglai-enclave-driver/penglai-enclave-driver.c b/penglai-enclave-driver/penglai-enclave-driver.c index 968532922..0f3131685 100644 --- a/penglai-enclave-driver/penglai-enclave-driver.c +++ b/penglai-enclave-driver/penglai-enclave-driver.c @@ -114,7 +114,7 @@ void enclave_ioctl_exit(void) { free_pages((long unsigned int)__va(addr), (order - RISCV_PGSHIFT)); } - + *size = 0; sbiret = SBI_CALL_2(SBI_SM_FREE_ENCLAVE_MEM, __pa(size), FREE_MAX_MEMORY); addr = (unsigned long)(sbiret.value); diff --git a/penglai-enclave-driver/penglai-enclave-elfloader.c b/penglai-enclave-driver/penglai-enclave-elfloader.c index 41e9fcaeb..da1925cd5 100644 --- a/penglai-enclave-driver/penglai-enclave-elfloader.c +++ b/penglai-enclave-driver/penglai-enclave-elfloader.c @@ -1,4 +1,26 @@ #include "penglai-enclave-elfloader.h" +#define ROUND_TO(x, align) (((x) + ((align)-1)) & ~((align)-1)) + +// Function to print hex data +static void print_hex(const void *data, size_t size) { + const unsigned char *byte_data = data; + size_t i; + + for (i = 0; i < size; ) { + printk("%02x%02x%02x%02x %02x%02x%02x%02x %02x%02x%02x%02x %02x%02x%02x%02x\n%02x%02x%02x%02x %02x%02x%02x%02x %02x%02x%02x%02x %02x%02x%02x%02x", + byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++], + byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++], + byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++], + byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++],byte_data[i++]); + // // 每16个字节输出一个换行符 + // if ((i + 1) % 32 == 0) + // printk("\n"); + } + + // 如果最后一行不足16个字节，输出换行符 + if (i % 16 != 0) + printk("\n"); +} int penglai_enclave_load_NOBITS_section(enclave_mem_t* enclave_mem, void * elf_sect_addr, int elf_sect_size) { @@ -25,8 +47,9 @@ int penglai_enclave_load_program(enclave_mem_t* enclave_mem, vaddr_t elf_prog_in { vaddr_t addr; vaddr_t enclave_new_page; + vaddr_t begin_page = 0; int size; - int r; + unsigned long r = 0; for(addr = (vaddr_t)elf_prog_addr; addr < (vaddr_t)elf_prog_addr + elf_prog_size; addr += RISCV_PGSIZE) { @@ -35,8 +58,17 @@ int penglai_enclave_load_program(enclave_mem_t* enclave_mem, vaddr_t elf_prog_in size = elf_prog_size % RISCV_PGSIZE; else size = RISCV_PGSIZE; - r = copy_from_user((void* )enclave_new_page, (void *)(elf_prog_infile_addr + addr - (vaddr_t)elf_prog_addr), size); + r += copy_from_user((void* )enclave_new_page, (void *)(elf_prog_infile_addr + addr - (vaddr_t)elf_prog_addr), size); + if (r) + { + printk("KERNEL MODULE: load_program copy_from_user failed r=0x%lx\n",r); + } + + // dprint("[Penglai Driver@%s]copy segment 0x%lx to va:0x%lx -pa:0x%lx \n",__func__ ,(unsigned long)((void *)(elf_prog_addr + addr - (vaddr_t)elf_prog_addr)) ,(vaddr_t)enclave_new_page ,__pa((vaddr_t)enclave_new_page)); + if(!begin_page)begin_page = enclave_new_page; + // print_hex(enclave_new_page, size); } + dprint("[Penglai Driver@%s] SUCCESS! from 0x%lx to 0x%lx r=0x%x\n",__func__,(unsigned long)elf_prog_addr,(unsigned long)(elf_prog_addr + elf_prog_size) ,r); return 0; } diff --git a/penglai-enclave-driver/penglai-enclave-ioctl.c b/penglai-enclave-driver/penglai-enclave-ioctl.c index 4e2e9ac29..5000b309b 100644 --- a/penglai-enclave-driver/penglai-enclave-ioctl.c +++ b/penglai-enclave-driver/penglai-enclave-ioctl.c @@ -28,7 +28,7 @@ unsigned int total_enclave_page(int elf_size, int stack_size) int create_sbi_param(enclave_t* enclave, struct penglai_enclave_sbi_param * enclave_sbi_param, unsigned long paddr, unsigned long size, unsigned long entry_point, - unsigned long untrusted_ptr, unsigned long untrusted_size, unsigned long free_mem) + unsigned long untrusted_ptr, unsigned long untrusted_ptr_paddr, unsigned long untrusted_size, unsigned long free_mem, unsigned long kbuffer_ptr, unsigned long kbuffer_paddr) { enclave_sbi_param -> eid_ptr = (unsigned int* )__pa(&enclave -> eid); enclave_sbi_param -> ecall_arg0 = (unsigned long* )__pa(&enclave -> ocall_func_id); @@ -39,10 +39,13 @@ int create_sbi_param(enclave_t* enclave, struct penglai_enclave_sbi_param * encl enclave_sbi_param -> size = size; enclave_sbi_param -> entry_point = entry_point; enclave_sbi_param -> untrusted_ptr = untrusted_ptr; - enclave_sbi_param -> untrusted_size = untrusted_size; + enclave_sbi_param->untrusted_paddr = untrusted_ptr_paddr; + enclave_sbi_param->untrusted_size = untrusted_size; enclave_sbi_param -> free_mem = free_mem; //enclave share mem with kernel - enclave_sbi_param->kbuffer = ENCLAVE_DEFAULT_KBUFFER; + // enclave_sbi_param->kbuffer = ENCLAVE_DEFAULT_KBUFFER; + enclave_sbi_param->kbuffer = kbuffer_ptr; + enclave_sbi_param->kbuffer_paddr = kbuffer_paddr; enclave_sbi_param->kbuffer_size = enclave->kbuffer_size; return 0; } @@ -155,8 +158,8 @@ int penglai_enclave_create(struct file * filep, unsigned long args) } enclave->untrusted_mem->addr = (vaddr_t)untrusted_mem_ptr; enclave->untrusted_mem->size = untrusted_mem_size; - dprint("[Penglai Driver@%s] untrusted_mem->addr:0x%lx untrusted_mem->size:0x%lx\n", - __func__, (vaddr_t)untrusted_mem_ptr, untrusted_mem_size); + dprint("[Penglai Driver@%s] untrusted_mem->addr:0x%lx ,paddr:%lx, untrusted_mem->size:0x%lx\n", + __func__, (vaddr_t)untrusted_mem_ptr, __pa(untrusted_mem_ptr), untrusted_mem_size); alloc_kbuffer(ENCLAVE_DEFAULT_KBUFFER_SIZE, &kbuffer_ptr, enclave); //May sleep enclave->kbuffer = (vaddr_t)kbuffer_ptr; @@ -164,9 +167,9 @@ int penglai_enclave_create(struct file * filep, unsigned long args) free_mem = get_free_mem(&(enclave->enclave_mem->free_mem)); create_sbi_param(enclave, enclave_sbi_param, - (unsigned long)(enclave->enclave_mem->paddr), - enclave->enclave_mem->size, elf_entry, DEFAULT_UNTRUSTED_PTR, - untrusted_mem_size, __pa(free_mem)); + (unsigned long)(enclave->enclave_mem->paddr), + enclave->enclave_mem->size, elf_entry, DEFAULT_UNTRUSTED_PTR, __pa(untrusted_mem_ptr), + untrusted_mem_size, __pa(free_mem), ENCLAVE_DEFAULT_KBUFFER, __pa(kbuffer_ptr)); dprint("[Penglai Driver@%s] enclave_mem->paddr:0x%lx, size:0x%lx\n", __func__, (unsigned long)(enclave->enclave_mem->paddr), @@ -181,6 +184,7 @@ int penglai_enclave_create(struct file * filep, unsigned long args) if(ret.value == ENCLAVE_NO_MEMORY){ //TODO: allocate certain memory region like sm_init unsigned long addr; + int retry = 5; addr = __get_free_pages(GFP_ATOMIC, DEFAULT_SECURE_PAGES_ORDER); if(!addr) { @@ -189,8 +193,15 @@ int penglai_enclave_create(struct file * filep, unsigned long args) } dprint("[Penglai Driver@%s] new alloc paddr:0x%lx\n",__func__, addr); ret = SBI_CALL_2(SBI_SM_MEMORY_EXTEND, __pa(addr), (1 << (DEFAULT_SECURE_PAGES_ORDER + RISCV_PGSHIFT)) ); - if(ret.error) + + while (ret.value == RETRY_SPIN_LOCK && retry) { + retry--; + ret = SBI_CALL_2(SBI_SM_MEMORY_EXTEND, __pa(addr), (1 << (DEFAULT_SECURE_PAGES_ORDER + RISCV_PGSHIFT))); + } + + if(ret.error) + { printk("KERNEL MODULE: sbi call extend memory is failed\n"); goto destroy_enclave; } @@ -350,8 +361,11 @@ int penglai_enclave_run(struct file *filep, unsigned long args) return -EINVAL; } memset((void*)enclave->untrusted_mem->addr, 0, enclave->untrusted_mem->size); - if(copy_from_user((void*)enclave->untrusted_mem->addr, (void*)untrusted_mem_ptr, untrusted_mem_size)) + if (copy_from_user((void *)enclave->untrusted_mem->addr, (void *)untrusted_mem_ptr, untrusted_mem_size)) + { + printk("KERNEL MODULE: copy_from_user failed \n"); return -EFAULT; + } } dprint("[Penglai Driver@%s] goto infinite run loop\n", __func__); @@ -583,7 +597,7 @@ int penglai_enclave_resume(struct file * filep, unsigned long args) if (copy_to_user((void*)untrusted_mem_ptr, (void*)enclave->untrusted_mem->addr, ocall_buf_size)) return -EFAULT; } - printk("[Penglai Driver@%s] return user for ocall \n", __func__); + dprint("[Penglai Driver@%s] return user for ocall,enclave->untrusted_mem:%lx\n\t kbuffer:%lx kbuffer_val:%s \n", __func__, (void *)(enclave->untrusted_mem->addr), enclave->kbuffer, (void *)(enclave->kbuffer)); return RETURN_USER_FOR_OCALL; } default: @@ -631,6 +645,30 @@ int penglai_enclave_resume(struct file * filep, unsigned long args) return retval; } +int penglai_enclave_memory_reclaim(struct file * filep, unsigned long args){ + int retval; + struct sbiret ret = {0}; + struct mm_reclaim_arg_t* mm_reclaim; + + struct penglai_enclave_user_param * enclave_param = (struct penglai_enclave_user_param*) args; + unsigned long eid = enclave_param ->eid; + enclave_t * enclave; + + enclave = get_enclave_by_id(eid); + + mm_reclaim = kmalloc(sizeof(struct mm_reclaim_arg_t), GFP_KERNEL); + + acquire_big_lock(__func__); + ret = SBI_CALL_2(SBI_SM_MEMORY_RECLAIM, __pa(mm_reclaim), enclave->eid); + retval = ret.value; + + release_big_lock(__func__); + + dprint("[Penglai Driver@%s]A total of %lx enclave memory was reclaimed\n",__func__, mm_reclaim->resp_size); + kfree(mm_reclaim); + return retval; +} + long penglai_enclave_ioctl(struct file* filep, unsigned int cmd, unsigned long args) { char ioctl_data[1024]; @@ -670,6 +708,9 @@ long penglai_enclave_ioctl(struct file* filep, unsigned int cmd, unsigned long a case PENGLAI_ENCLAVE_IOC_DESTROY_ENCLAVE: ret = penglai_enclave_destroy(filep, (unsigned long)ioctl_data); break; + case PENGLAI_ENCLAVE_MEMORY_RECLAIM: + ret = penglai_enclave_memory_reclaim(filep, (unsigned long)ioctl_data); + break; case PENGLAI_ENCLAVE_IOC_DEBUG_PRINT: sbiret = SBI_CALL_1(SBI_SM_DEBUG_PRINT, 0); ret = sbiret.value; diff --git a/penglai-enclave-driver/penglai-enclave-ioctl.h b/penglai-enclave-driver/penglai-enclave-ioctl.h index 7d6e043d2..1d97eda57 100644 --- a/penglai-enclave-driver/penglai-enclave-ioctl.h +++ b/penglai-enclave-driver/penglai-enclave-ioctl.h @@ -20,6 +20,8 @@ _IOW(PENGLAI_ENCLAVE_IOC_MAGIC, 0x05, struct penglai_enclave_user_param) #define PENGLAI_ENCLAVE_IOC_DEBUG_PRINT \ _IOW(PENGLAI_ENCLAVE_IOC_MAGIC, 0x06, struct penglai_enclave_user_param) +#define PENGLAI_ENCLAVE_MEMORY_RECLAIM \ + _IOW(PENGLAI_ENCLAVE_IOC_MAGIC, 0x07, struct penglai_enclave_user_param) #define DEFAULT_SECURE_PAGES_ORDER 10 #define DEFAULT_CLOCK_DELAY 100000 @@ -48,10 +50,12 @@ struct penglai_enclave_sbi_param unsigned long size; unsigned long entry_point; unsigned long untrusted_ptr; + unsigned long untrusted_paddr; unsigned long untrusted_size; unsigned long free_mem; //enclave shared mem with kernel unsigned long kbuffer; + unsigned long kbuffer_paddr; unsigned long kbuffer_size; unsigned long *ecall_arg0; unsigned long *ecall_arg1; @@ -108,6 +112,13 @@ struct penglai_enclave_ioctl_attest_enclave struct report_t report; }; +struct mm_reclaim_arg_t +{ + unsigned long req_size; + unsigned long req_addr; + unsigned long resp_size; +}; + long penglai_enclave_ioctl(struct file* filep, unsigned int cmd, unsigned long args); #endif diff --git a/penglai-enclave-driver/penglai-enclave.h b/penglai-enclave-driver/penglai-enclave.h index b8dfd338c..2415638e3 100644 --- a/penglai-enclave-driver/penglai-enclave.h +++ b/penglai-enclave-driver/penglai-enclave.h @@ -28,10 +28,12 @@ #define SBI_SM_DESTROY_ENCLAVE 94 #define SBI_SM_ALLOC_ENCLAVE_MEM 93 #define SBI_SM_MEMORY_EXTEND 92 -#define SBI_SM_FREE_ENCLAVE_MEM 91 +#define SBI_SM_MEMORY_RECLAIM 91 +#define SBI_SM_FREE_ENCLAVE_MEM 90 #define SBI_SM_DEBUG_PRINT 88 //Error codes of SBI_SM_ALLOC_ENCLAVE_MEM +#define RETRY_SPIN_LOCK -3 #define ENCLAVE_NO_MEMORY -2 #define ENCLAVE_UNKNOWN_ERROR -1 #define ENCLAVE_SUCCESS 0