diff --git a/main.py b/main.py index d7d6f91..b71584e 100644 --- a/main.py +++ b/main.py @@ -16,6 +16,7 @@ template_vars = { 'navbar': [ ('/', 'Home'), + ('blog/', 'Blog'), ('about/', 'About'), ('quick_start/', 'Quick start guide'), ('docs/', 'Documentation'), diff --git a/mdrend.py b/mdrend.py index f21dc64..448b411 100644 --- a/mdrend.py +++ b/mdrend.py @@ -58,16 +58,18 @@ def ascii_render(self, code): scene = '{' + d[1] color = int(props['color']) if 'color' in props else 0 - w = int(props['w']) if 'w' in props else 64 + w = int(props['w']) if 'w' in props else 72 h = int(props['h']) if 'h' in props else 32 aspect = float(w / (2 * h)) + + # These parameters are not actually used in latest asciirend ortho = bool(props['ortho']) if 'ortho' in props else True fov = float(props['fov']) if 'fov' in props else 1.0 znear = float(props['znear']) if 'znear' in props else 0.1 zfar = float(props['zfar']) if 'zfar' in props else 100.0 self.scene_props[div_id] = { - 'scene': scene, + 'scene': scene.replace('\n', ''), 'w': w, 'h': h, 'aspect': aspect, @@ -77,9 +79,11 @@ def ascii_render(self, code): 'zfar': zfar, 'dynamic_w': bool(props['dynamic_w']) if 'dynamic_w' in props else False, 'dynamic_h': bool(props['dynamic_h']) if 'dynamic_h' in props else False, + 'show_usage': bool(props['show_usage']) if 'show_usage' in props else True, + 'disable_zoom': bool(props['disable_zoom']) if 'disable_zoom' in props else False, } - rendered = ar.ascii_render(scene, color, w, h, aspect, ortho, fov, znear, zfar) + rendered = ar.ascii_render(scene, color, w, h, aspect, ortho, fov, znear, zfar, 0.0) return f'
{rendered}
'; def gh_users_render(self, code): @@ -102,8 +106,6 @@ def gh_users_render(self, code):
""" - print(line) - return output def block_code(self, code, lang=None): @@ -119,8 +121,6 @@ def block_code(self, code, lang=None): except ClassNotFound: lexer = get_lexer_by_name("html", stripall=True) - print(str(lexer)) - formatter = HtmlFormatter() return highlight(code, lexer, formatter) @@ -159,19 +159,20 @@ def shortdown(value): md_rend = md.create_markdown(renderer=renderer, plugins=['strikethrough']) return md_rend(trimmed) -def markdown(value): +def markdown(value, backlink): + print(backlink) renderer = CustomizedRenderer() md_rend = md.create_markdown(renderer=renderer, plugins=['task_lists', 'table', 'footnotes', 'strikethrough'], escape=False) rendered = md_rend(value) if renderer.scene_cnt > 0: - javascript = """ + javascript = f""" """ diff --git a/src/blog/index.md b/src/blog/index.md new file mode 100644 index 0000000..094a19b --- /dev/null +++ b/src/blog/index.md @@ -0,0 +1,3 @@ +# Blog + +## [Announcing memflow 0.2.0](memflow-0.2.0/) diff --git a/src/blog/memflow-0.2.0.md b/src/blog/memflow-0.2.0.md new file mode 100644 index 0000000..4ea0876 --- /dev/null +++ b/src/blog/memflow-0.2.0.md @@ -0,0 +1,280 @@ +# Announcing memflow 0.2.0 + +Today, we are proud to release the first stable version of memflow 0.2! 3 years in the making, this +is certainly a monumental release. In this post, we will go through the key changes to the fastest +and most flexible physical memory introspection and forensics framework to date. + +## Key changes + +### 0. [memflowup](https://github.com/memflow/memflowup) + +Not a library change, but the ecosystem change! We now have a rust-written memflowup utility that +makes it much easier to manage your memflow installation. Key features: + +- Download binary builds (optional). +- Split between stable and dev versions. +- Custom install scripts, for more complicated plugins + - Used by [`memflow-kvm`](https://github.com/memflow/memflow-kvm) for DKMS install. + - Entry point for these is `install.rhai` script at the root of the package's repo. + +You can get started with memflowup by running the following: + +``` +> curl --proto '=https' --tlsv1.2 -sSf https://sh.memflow.io | sh +``` + +### 1. OS layers and modularity + +With the advent of 0.2 series, we now abstracted most of `memflow-win32` functionality behind +shared set of traits. These traits allow the user to interact with the operating system in unified +manner. In addition, we now made OS a plugin, just as Connectors were in 0.1! And finally, we do +indeed have multiple OS backends available, right now: + +- [`memflow-win32`](https://github.com/memflow/memflow-win32), for Windows analysis, given physical + memory access. +- [`memflow-native`](https://github.com/memflow/memflow-native), for syscall based interaction with + the running operating system. +- WIP: `memflow-linux` + - Don't expect much anytime soon, because the challenge of cross-version, zero-knowledge linux + support is a tricky one. + +With this, OS-independent code that works with `memflow-win32`, should also work on local OS. Here's +an example of such code: + +```rust +use memflow::prelude::v1::*; + +// We don't care what type of process we get, so long as it's a process +fn module_address(process: &mut impl Process, module: &str) -> Result
{ + let module = process.module_by_name(module)?; + Ok(module.base) +} +``` + +In addition, modularization of operating systems allows for greater portability of connectors. For +instance, we have now split `memflow-qemu-procfs` into +[`memflow-qemu`](https://github.com/memflow/memflow-qemu), which (optionally) accepts an OS layer. +This way, you can not only analyze QEMU VMs running on your computer, but you can also open them up +in a nested way on a machine that is already being analyzed through DMA. As seen in this chart: + +```asciirend +dynamic_w = true +dynamic_h = false +fov = 4.5 +ortho = true +disable_zoom = true +# Scene: +{ + "camera_props": { + "proj_mode": "Orthographic", + "fov": 1.0, + "near": 0.01, + "far": 100.0 + }, + "camera_controller":{"fov_y":1.0,"focus_point":[0.0,0.0,0.0],"rot":[-0.19996414, -0.08282786, 0.37361234, 0.90197986],"dist":2.0,"in_motion":"None","scroll_sensitivity":0.02,"orbit_sensitivity":1.0,"last_down":false,"pressed":false}, + "objects":[ + { + "transform":[ + 1.0,0.0,0.0,0.0, + 0.0,1.0,0.0,0.0, + 0.0,0.0,1.0,0.0, + 0.0,0.0,0.0,1.0 + ], + "material":0, + "ty":{ + "Primitive":{ + "Line":{ + "start":[-0.75,0.0,1.5,1.0], + "end":[-0.25,0.0,0.5,1.0] + } + } + } + }, + { + "transform":[ + 1.0,0.0,0.0,0.0, + 0.0,1.0,0.0,0.0, + 0.0,0.0,1.0,0.0, + 0.0,0.0,0.0,1.0 + ], + "material":0, + "ty":{ + "Primitive":{ + "Line":{ + "start":[-0.25,0.0,0.5,1.0], + "end":[0.25,0.0,-0.5,1.0] + } + } + } + }, + { + "transform":[ + 1.0,0.0,0.0,0.0, + 0.0,1.0,0.0,0.0, + 0.0,0.0,1.0,0.0, + 0.0,0.0,0.0,1.0 + ], + "material":0, + "ty":{ + "Primitive":{ + "Line":{ + "start":[0.25,0.0,-0.5,1.0], + "end":[0.75,0.0,-1.5,1.0] + } + } + } + }, + { + "transform":[ + 1.0,0.0,0.0,0.0, + 0.0,1.0,0.0,0.0, + 0.0,0.0,1.0,0.0, + -0.75,0.0,1.5,1.0 + ],"material":1,"ty":{"Cube":{"size":[1.0,1.0,0.5]}},"text":"memflow-kvm" + }, + { + "transform":[ + 1.0,0.0,0.0,0.0, + 0.0,1.0,0.0,0.0, + 0.0,0.0,1.0,0.0, + -0.25,0.0,0.5,1.0 + ],"material":1,"ty":{"Cube":{"size":[1.0,1.0,0.5]}},"text":"memflow-win32" + }, + { + "transform":[ + 1.0,0.0,0.0,0.0, + 0.0,1.0,0.0,0.0, + 0.0,0.0,1.0,0.0, + 0.25,0.0,-0.5,1.0 + ],"material":1,"ty":{"Cube":{"size":[1.0,1.0,0.5]}},"text":"memflow-qemu" + }, + { + "transform":[ + 1.0,0.0,0.0,0.0, + 0.0,1.0,0.0,0.0, + 0.0,0.0,1.0,0.0, + 0.75,0.0,-1.5,1.0 + ],"material":1,"ty":{"Cube":{"size":[1.0,1.0,0.5]}},"text":"memflow-win32" + } + ], + "bg":{"color":[0.0,0.0,0.0]}, + "dithering":{"count_frames":false,"frame_cnt":4181} +} +``` + +### 2. Stable ABI + +In 0.1, the Connectors were turned into plugins through use of Rust trait objects. This was an okay +solution at the time, however, we knew that it was not a safe one - changes in Rust versions could +change the layout of those trait objects, leading to crashes or other misbehavior, in case of +mismatch of plugin's `rustc` version and the one of the user's code. While the layout has remained +stable most of the time, the tides started to shift a few years ago, as more effort was put into +trait objects on the compiler front. + +For 0.2, we knew we could not keep the status quo, so, we built `cglue`. The crate allows for +simple and flexible ABI safe code generation, suited for the needs of `memflow`. Throughout the +(very long) beta period, we received 0 crash reports stemming from ABI instability, while 0.1 had +such cases. Therefore, we can conclude that it was a good investment that already made memflow more +stable. + +In `0.2.0-betaX` series, you may have encountered "invalid ABI" errors, well, fear not, because in +stable series, we commit to not breaking the ABI across entirety of `0.2` series, so this problem +should be a thing of the past for most users. + +### 3. Memory constrained vtop + +memflow 0.2 introduces the most scalable virtual address translation backend, period. The backend +is able to walk entire page tree in milliseconds, targeting any modern memory architecture (x86 and +ARM support out-of-the box, sufficient building blocks for RISC-V). In addition, compared to 0.1, +the new backend uses fixed-size buffers, meaning RAM usage will no longer blow up on large +translation ranges. + +### 4. 64-bit and 128-bit address spaces, on all architectures + +We now support analyzing 64-bit operating systems on 32-bit machines. In addition, if there was a +theoretical 128-bit architecture, we would support that as well. However, it's more of a PoC and we +do not expect this to be needed in the foreseeable future. + +The support can be toggled through `64_bit_mem` (default) and `128_bit_mem` features. Do note that +these feature toggles do change memflow's ABI and it should not be possible to mix the plugin +features. + +### 5. Shared `MemoryView` + +In 0.1, we have had a split between physical and virtual memory. The reason for the split is +caching - we wish minimize latency by caching read-only memory in high-latency scenarios. +However, to tell the cache what mode the memory is in (readable/writeable/executable), you must add +metadata with each request. Meanwhile, this metadata may only be filled in by the virtual address +translation backend. + +If user submits an I/O operation - they can't possibly know whether the request is going to a +read-only, or a writeable page, therefore they just submit `UNKNOWN` page flags. This is +complicated, therefore, we have lowered the gap between virtual and physical memory access through +use of `MemoryView` trait. This trait not only removes the need for the user to explicitly submit +the page flags, but also brings all I/O helpers that existed in virtual memory contexts. To use +`MemoryView` on physical memory, just use the `phys_view` function: + +```rust +use memflow::prelude::v1::*; + +fn main() -> Result<()> { + let inventory = Inventory::scan(); + let mut conn = inventory.create_connector("dummy", None, None)?; + + // Create a physical memory view + let mut view = conn.phys_view(); + + // Read from phys addr 0 + let value: u64 = view.read(0.into())?; + + Ok(()) +} +``` + +### 6. C and C++ are now first-class citizen + +The FFI is now automatically generated using `cbindgen` and `cglue-bindgen`. It may initially seem +like a downgrade, however, this way we can ensure entirety of memflow's plugin-focused API surface +can be both accessed, and implemented by foreign languages, such as C and C++. + +The key to using the new FFI, is reading Rust documentation and examples, and then finding the +function equivalents in the headers. There are a few quirks here and there, but after understanding +them, using the FFI should not be hard. For inspiration, see the following: + +- [C examples](https://github.com/memflow/memflow/tree/0.2.0/memflow-ffi/examples/c) +- [C++ examples](https://github.com/memflow/memflow/tree/0.2.0/memflow-ffi/examples/cpp) +- [CMake template](https://github.com/memflow/memflow-cmake-example) + +## Side projects + +`memflow` is as useful as the projects utilizing it. To get started with the new version faster, +you may want to have a look at some of them. Here's the list of first-party releases: + +- [`reflow`](https://github.com/memflow/reflow) - execute code on top of virtual memory. +- [`scanflow`](https://github.com/memflow/scanflow) - basic CheatEngine features in a command line + interface. +- [`cloudflow`](https://github.com/memflow/cloudflow) (WIP) - flexible filesystem based way to + interact with memflow. +- [`memflow-py`](https://github.com/memflow/memflow-py) - python bindings for memflow (courtesy of + emesare). + +## Reflection + +0.2 took way longer than we originally anticipated. This is mostly due to changing living +conditions and the fact that both ko1N and I are only working on the project in hobbyist capacity. +In addition, we pushed for perfection from documentation and implementation front - a feat +infeasible at the current point. We do believe memflow is the framework that is going to bring the +most empowerement to users, however, there are still ways to go. + +## Next up - Async Metamorphosis + +Next, we will work towards integrating [`mfio`](https://github.com/memflow/mfio) into memflow, +which will enable higher scalability and simplicity. The key change is going to be transition from +synchronous to asynchronous API. There are still a lot of open questions regarding this, such as +FFI handling, how much the individual pieces of memflow's code will have to change, and how +multithreading needs to be handled. However, we are confident those questions are not impossible +to solve. Once the metamorphosis is done, we can consider the structure of memflow done. What comes +afterwards, is rapid feature development. It will definitely be an exciting time to be alive. So +let's just get there, shall we? + +\- h33p diff --git a/src/quick_start.md b/src/quick_start.md index e1117b7..fcfa705 100644 --- a/src/quick_start.md +++ b/src/quick_start.md @@ -31,9 +31,9 @@ After setting up cargo properly you can install memflowup via our install script Alternatively you can install memflowup via cargo: ``` -> cargo install memflowup --force --version "=0.1.0-beta11" +> cargo install memflowup --force ... -Installed package `memflowup v0.1.0-beta11` (executable `memflowup.exe`) +Installed package `memflowup v0.1.0` (executable `memflowup.exe`) ``` #### Note @@ -41,7 +41,7 @@ memflowup should __not__ be installed or ran as root or via sudo. By default rus ### 2. Installing plugins -When running `memflowup` for the first time it is recommended to use the interactive mode and install memflow from the 0.2.0-beta branch (development). +When running `memflowup` for the first time it is recommended to use the interactive mode and install memflow from the stable branch. Installing packages system-wide will place all plugins in `/usr/local/lib/memflow`.\ Installing packages per user will place all plugins in `$HOME/.local/lib/memflow`. @@ -99,11 +99,11 @@ In case you installed the plugins like in the example above you can simply use t ### 3. Verify your installation and run an example To test if everything is working properly the easiest method is to simply -use one of the [examples](https://github.com/memflow/memflow/tree/0.2.0-beta11/memflow/examples) provided in memflow. +use one of the [examples](https://github.com/memflow/memflow/tree/stable/memflow/examples) provided in memflow. To run the examples simply check out the memflow repo with the appropiate version: ``` -> git clone --depth 1 --branch 0.2.0-beta11 https://github.com/memflow/memflow +> git clone --depth 1 --branch stable https://github.com/memflow/memflow > cd memflow ``` @@ -147,14 +147,14 @@ info: cleaning up downloads & tmp directories After setting up cargo properly you can install memflowup via cargo: ``` -> cargo install memflowup --force --version "=0.1.0-beta11" +> cargo install memflowup --force ... -Installed package `memflowup v0.1.0-beta11` (executable `memflowup.exe`) +Installed package `memflowup v0.1.0` (executable `memflowup.exe`) ``` ### 2. Installing plugins -When running `memflowup` for the first time it is recommended to use the interactive mode and install memflow from the 0.2.0-beta branch (development). +When running `memflowup` for the first time it is recommended to use the interactive mode and install memflow from the stable branch. Installing packages system-wide will place all plugins in `%ProgramFiles%\memflow\`.\ Installing packages per user will place all plugins in `%UserProfile%\Documents\memflow\`. @@ -210,11 +210,11 @@ In case you installed the plugins like in the example above you can simply use t ### 3. Verify your installation and run an example To test if everything is working properly the easiest method is to simply -use one of the [examples](https://github.com/memflow/memflow/tree/0.2.0-beta11/memflow/examples) provided in memflow. +use one of the [examples](https://github.com/memflow/memflow/tree/stable/memflow/examples) provided in memflow. To run the examples simply check out the memflow repo with the appropiate version: ``` -> git clone --depth 1 --branch 0.2.0-beta11 https://github.com/memflow/memflow +> git clone --depth 1 --branch stable https://github.com/memflow/memflow > cd memflow ``` @@ -242,4 +242,4 @@ If everything went well you should see a list of all open processes: 1484 x86_64 x86_64 services.exe () (Alive) ... -``` \ No newline at end of file +``` diff --git a/src/static/js/draw.js b/src/static/js/draw.js new file mode 100644 index 0000000..5a01b07 --- /dev/null +++ b/src/static/js/draw.js @@ -0,0 +1,239 @@ +import { + new_frame, + set_text, + event_mouse_pos_clear, + scene_from_json, + scene_to_json, + event_scroll, + ProjectionMode, + set_bg_color, + set_dither_count_frames, + event_mouse_pos, + event_mouse_button_state, + event_focus, + new_scene, + add_cube, + render, + set_camera_aspect, + add_line, + set_obj_transform, + color_conv, + TermColorMode, + StandardMaterial, + vec3, + default as init +} from './asciirend/asciirend.js'; + +function uiVec3(x, y, z, w, h) { + return vec3(100 / w * x, z, 100 / h * y) +} + +function updateUiText(scene, obj, text, x, y, w, h) { + const menu_text = set_text(scene, obj, text); + if (text != null) { + set_obj_transform(scene, obj, uiVec3(Math.floor(x + (text.length + 2) / 2), y, 2, w, h), vec3(0, 0, 0), uiVec3(text.length + 2, 3, 1, w, h)); + } else { + set_obj_transform(scene, obj, uiVec3(0, 0, 0), vec3(0, 0, 0), uiVec3(0, 0, 0)); + } + return menu_text +} + +function uiText(scene, text, x, y, w, h) { + const menu_text = add_cube(scene, StandardMaterial.UiText, vec3(1, 1, 1), text); + if (text != null) { + set_obj_transform(scene, menu_text, uiVec3(Math.floor(x + (text.length + 2) / 2), y, 2, w, h), vec3(0, 0, 0), uiVec3(text.length + 2, 3, 1, w, h)); + } + return menu_text +} + +function charWidth(divElement) { + // Currently we hardcode the character ratios. Using canvas, we could measure them exactly, but + // I am not good enough for this. + /* + // Create a canvas element to measure text width + const canvas = document.createElement('canvas'); + const context = canvas.getContext('2d'); + */ + + const computedStyle = window.getComputedStyle(divElement); + const fontFamily = computedStyle.getPropertyValue('font-family'); + const fontSize = computedStyle.getPropertyValue('font-size'); + + /* + // Set the font size and family to match the ASCII art + context.font = `${fontSize}px ${fontFamily}`; + + // Measure the width of a single character + const measurement = context.measureText('#'); + console.log(measurement) + const actualSize = measurement.fontBoundingBoxAscent + measurement.fontBoundingBoxDescent; + */ + const parsedSize = parseInt(fontSize, 10); + + return [parsedSize / 1.65, parsedSize] +} + +export default async function ascii_render(id, scene_json, fw, fh, is_ortho, fov, znear, zfar, showUsage, zoomDisable) { + await init(); + + const conv = color_conv(TermColorMode.SingleCol); + + const scene = scene_from_json(scene_json); + + const dom = document.getElementById(id); + const [cw, ch] = charWidth(dom); + + let w = fw !== null ? fw : Math.floor(dom.clientWidth / cw); + let h = fh !== null ? fh : Math.floor(dom.clientHeight / ch); + + console.log(w, h, cw, ch); + + const usage = showUsage ? uiText(scene, "Click and drag", 0, h - 1, w, h) : null; + + const startTime = performance.now(); + + function mousePos(e) { + if (e.touches != null) { + e.preventDefault(); + e = e.touches[0]; + } + const rect = e.target.getBoundingClientRect(); + const x = (e.clientX - rect.left) / rect.width; + const y = (e.clientY - rect.top) / rect.height; + + event_mouse_pos(scene, x * w, y * h); + } + + function mouseButton(e, down) { + if (down) { + mousePos(e); + } else { + event_mouse_pos_clear(scene); + } + + let isRightMB = false; + + if ("which" in e) // Gecko (Firefox), WebKit (Safari/Chrome) & Opera + isRightMB = e.which == 3; + else if ("button" in e) // IE, Opera + isRightMB = e.button == 2; + + event_mouse_button_state(scene, down, !isRightMB); + } + + let focused = false; + + function focusEnter() { + focused = true; + event_focus(scene, true); + if (usage) { + updateUiText(scene, usage, null, 0, h - 1, w, h); + } + } + + function focusExit() { + focused = false; + event_mouse_button_state(scene, false, true); + event_mouse_button_state(scene, false, false); + event_focus(scene, false) + if (usage) { + updateUiText(scene, usage, "Click and drag", 0, h - 1, w, h); + } + } + + function scrollEvent(e) { + if (focused && !zoomDisable) { + event_scroll(scene, -e.deltaX, -e.deltaY); + e.preventDefault(); + } + } + + let prevDist = null; + + function touchMove(e) { + if (e.touches.length == 1) { + e.preventDefault(); + mousePos(e.touches[0]); + prevDist = null; + } else if (e.touches.length == 2) { + e.preventDefault(); + const distX = e.touches[0].clientX - e.touches[1].clientX; + const distY = e.touches[0].clientY - e.touches[1].clientY; + const dist = Math.sqrt(distX * distX + distY * distY); + + if (prevDist !== null && !zoomDisable) { + const diff = dist - prevDist; + event_scroll(scene, 0, diff); + } + + prevDist = dist; + } else { + prevDist = null; + } + } + + addEventListener("wheel", scrollEvent, { + passive: false + }); + dom.onclick = mousePos; + dom.onmousemove = mousePos; + dom.onmousedown = (e) => mouseButton(e, true); + dom.onmouseup = (e) => mouseButton(e, false); + dom.onmouseout = (e) => focusExit(); + dom.onmouseover = (e) => focusEnter(); + + dom.addEventListener("touchmove", touchMove); + dom.addEventListener("touchstart", (e) => { + mouseButton(e, true); + focusEnter(); + }); + dom.addEventListener("touchend", (e) => { + mouseButton(e, false); + focusExit(); + prevDist = null; + }); + + function nf() { + set_camera_aspect(scene, (w / 2) / h); + new_frame(scene, w, h); + } + + nf() + + const v = setInterval(function() { + w = fw !== null ? fw : Math.floor(dom.clientWidth / cw); + h = fh !== null ? fh : Math.floor(dom.clientHeight / ch); + + const elapsed = performance.now() - startTime; + performance.mark("asciirend-start-frame"); + + const render_res = render(scene, conv, w, h, elapsed / 1000); + performance.mark("asciirend-rendered-frame"); + + let lines = '
';
+
+    for (let y = 0; y < h; y++) {
+      // Can't really do color, because we'd end up with far too many dom elements.
+      // Instead, just render grayscale.
+      const bytes = render_res.slice(y * w, y * w + w).map(v => v.c);
+      const line = String.fromCharCode(...bytes);
+      lines += line;
+      lines += "
";
+    }
+    performance.mark("asciirend-converted-frame");
+
+    const elapsed2 = performance.now() - startTime;
+
+    dom.innerHTML = lines + "
"; + + for (let i = 0; i < render_res.length; i++) { + // Make sure you free the pixels, or else you'll find a nasty surprise! + render_res[i].free(); + } + + performance.mark("asciirend-drawn-frame"); + //console.log(elapsed, elapsed2 - elapsed, performance.measure("frame-time", "asciirend-start-frame", "asciirend-converted-frame"), render_res.length); + + nf() + }, 4); +} diff --git a/templates/main.html b/templates/main.html index dc08237..a05858f 100644 --- a/templates/main.html +++ b/templates/main.html @@ -28,7 +28,7 @@
- {{ content | markdown }} + {{ content | markdown(backlink) }}