Release version 2.0 of dissect.cstruct

.gitignore

@@ -1,7 +1,6 @@
-*~
-.*.swp
 *.pyc
-*.egg-info
-.DS_Store
-.tox
-.pytest_cache
+*.egg-info/
+.tox/
+.pytest_cache/
+.eggs/
+dist/

MANIFEST.in

@@ -0,0 +1,2 @@
+exclude .gitignore
+exclude .gitlab-ci.yml

README.md

@@ -74,6 +74,55 @@ assert cs.uint24[2](b'\x01\x00\x00\x02\x00\x00') == [1, 2]  # You can also parse
 assert cs.char[None](b'hello world!\x00') == b'hello world!'  # A list index of None means null terminated
 ```
 
+### Unions and nested structures
+Unions and nested structures are support, both anonymous and named.
+
+```python
+cdef = """
+struct test_union {
+    char magic[4];
+    union {
+        struct {
+            uint32 a;
+            uint32 b;
+        } a;
+        struct {
+            char   b[8];
+        } b;
+    } c;
+};
+
+struct test_anonymous {
+    char magic[4];
+    struct {
+        uint32 a;
+        uint32 b;
+    };
+    struct {
+        char   c[8];
+    };
+};
+"""
+c = cstruct.cstruct()
+c.load(cdef)
+
+assert len(c.test_union) == 12
+
+a = c.test_union(b'ohaideadbeef')
+assert a.magic == b'ohai'
+assert a.c.a.a == 0x64616564
+assert a.c.a.b == 0x66656562
+assert a.c.b.b == b'deadbeef'
+
+assert a.dumps() == b'ohaideadbeef'
+
+b = c.test_anonymous(b'ohai\x39\x05\x00\x00\x28\x23\x00\x00deadbeef')
+assert b.magic == b'ohai'
+assert b.a == 1337
+assert b.b == 9000
+assert b.c == b'deadbeef'
+```
+
 ### Parse bit fields
 Bit fields are supported as part of structures. They are properly aligned to their boundaries.
 
@@ -109,7 +158,3 @@ You can implement your own types by subclassing `BaseType` or `RawType`, and add
 
 ### Custom definition parsers
 Don't like the C-like definition syntax? Write your own syntax parser!
-
-## Todo
-- Nested structure definitions
-- Unions

dissect/cstruct/__init__.py

@@ -1,33 +1,64 @@
+from dissect.cstruct.compiler import Compiler
+from dissect.cstruct.expression import Expression
+from dissect.cstruct.types.base import Array, BaseType, RawType
+from dissect.cstruct.types.chartype import CharType
+from dissect.cstruct.types.instance import Instance
+from dissect.cstruct.types.structure import Structure, Field, Union
+from dissect.cstruct.types.voidtype import VoidType
+from dissect.cstruct.types.wchartype import WcharType
+from dissect.cstruct.types.packedtype import PackedType
+from dissect.cstruct.types.flag import Flag, FlagInstance
+from dissect.cstruct.types.enum import Enum, EnumInstance
+from dissect.cstruct.types.bytesinteger import BytesInteger
+from dissect.cstruct.types.pointer import Pointer, PointerInstance
+
+from dissect.cstruct.exceptions import (
+    Error,
+    ParserError,
+    ResolveError,
+    NullPointerDereference,
+)
+
 from dissect.cstruct.cstruct import (
     cstruct,
     ctypes,
+)
+
+from dissect.cstruct.utils import (
     dumpstruct,
     hexdump,
-    Instance,
-    PointerInstance,
-    Parser,
-    RawType,
-    BaseType,
-    Error,
-    ParserError,
-    CompilerError,
-    ResolveError,
-    NullPointerDereference,
 )
 
+from dissect.cstruct.bitbuffer import BitBuffer
+
 __all__ = [
-    "cstruct",
-    "ctypes",
-    "dumpstruct",
-    "hexdump",
+    "Compiler",
+    "Array",
+    "Union",
+    "Field",
     "Instance",
+    "Structure",
+    "Expression",
+    "PackedType",
+    "Pointer",
     "PointerInstance",
-    "Parser",
+    "VoidType",
+    "WcharType",
     "RawType",
     "BaseType",
+    "CharType",
+    "Enum",
+    "EnumInstance",
+    "Flag",
+    "FlagInstance",
+    "BytesInteger",
+    "BitBuffer",
+    "cstruct",
+    "ctypes",
+    "dumpstruct",
+    "hexdump",
     "Error",
     "ParserError",
-    "CompilerError",
     "ResolveError",
     "NullPointerDereference",
 ]

dissect/cstruct/bitbuffer.py

@@ -0,0 +1,50 @@
+class BitBuffer(object):
+    """Implements a bit buffer that can read and write bit fields."""
+
+    def __init__(self, stream, endian):
+        self.stream = stream
+        self.endian = endian
+
+        self._type = None
+        self._buffer = 0
+        self._remaining = 0
+
+    def read(self, field_type, bits):
+        if self._remaining < 1 or self._type.size != field_type.size:
+            self._type = field_type
+            self._remaining = field_type.size * 8
+            self._buffer = field_type._read(self.stream)
+
+        if self.endian != '>':
+            v = self._buffer & ((1 << bits) - 1)
+            self._buffer >>= bits
+            self._remaining -= bits
+        else:
+            v = self._buffer & (((1 << (self._remaining - bits)) - 1) ^ ((1 << self._remaining) - 1))
+            v >>= self._remaining - bits
+            self._remaining -= bits
+
+        return v
+
+    def write(self, field_type, data, bits):
+        if self._remaining == 0:
+            self._remaining = field_type.size * 8
+            self._type = field_type
+
+        if self.endian != '>':
+            self._buffer |= data << (self._type.size * 8 - self._remaining)
+        else:
+            self._buffer |= data << (self._remaining - bits)
+
+        self._remaining -= bits
+
+    def flush(self):
+        self._type._write(self.stream, self._buffer)
+        self._type = None
+        self._remaining = 0
+        self._buffer = 0
+
+    def reset(self):
+        self._type = None
+        self._buffer = 0
+        self._remaining = 0