Version 0.4.1

nesm.nim

@@ -65,7 +65,8 @@ proc cleanupTypeDeclaration(declaration: NimNode): NimNode =
            c[1].kind == nnkTableConstr:
         continue
       elif c[last].kind == nnkTupleTy:
-        var newID = c
+        var newID = newNimNode(c.kind)
+        copyChildrenTo(c, newID)
         newID[last] = cleanupTypeDeclaration(c[last])
         children.add(newID)
       else:

nesm.nimble

@@ -1,9 +1,9 @@
 
-version       = "0.4.0"
+version       = "0.4.1"
 author        = "xomachine (Fomichev Dmitriy)"
 description   = "NESM stands for Nim's Easy Serialization Macro. The macro allowing generation of serialization functions by one line of code!"
 license       = "MIT"
-skipDirs      = @["tests"]
+skipDirs      = @["tests", "demos"]
 
 requires "nim >= 0.14.2"
 

nesm/generator.nim

@@ -10,34 +10,14 @@ static:
 from tables import Table, contains, `[]`, `[]=`, initTable, pairs
 from sequtils import mapIt, foldl, toSeq, filterIt
 from strutils import `%`
-from utils import unfold, correct_sum
+from utils import unfold, correct_sum, dig
 from typesinfo import isBasic, estimateBasicSize
 from objects import genObject
 from basics import genBasic
 from periodic import genPeriodic, genCStringDeserialize, genCStringSerialize
 from enums import genEnum
 from sets import genSet
 
-
-proc dig(node: NimNode, depth: Natural): NimNode {.compileTime.} =
-  if depth == 0:
-    return node
-  case node.kind
-  of nnkDotExpr:
-    node.expectMinLen(1)
-    node[0].dig(depth - 1)
-  of nnkCall:
-    node.expectMinLen(2)
-    node[1].dig(depth - 1)
-  of nnkEmpty:
-    node
-  of nnkIdent, nnkSym:
-    error("Too big depth to dig: " & $depth)
-    newEmptyNode()
-  else:
-    error("Unknown symbol: " & node.treeRepr)
-    newEmptyNode()
-
 proc insert_source(length_declaration, source: NimNode,
                    depth: Natural): NimNode  =
   if length_declaration.kind == nnkCurly:
@@ -56,6 +36,30 @@ proc incrementDepth(ctx: Context): Context {.compileTime.} =
   result = ctx
   result.depth += 1
 
+proc handleSizeOption(context: Context, elem: NimNode = newEmptyNode()): TypeChunk =
+  var subcontext = context
+  let capture = subcontext.overrides.size.pop()
+  let size = capture.size
+  let relative_depth = context.depth - capture.depth
+  let len_proc = proc (s: NimNode): NimNode =
+    (quote do: (`s`.len())).unfold()
+    # Parentesis is important because it forces genPeriodic to treat
+    # data as array and do not generate length code for it
+    # (it is not very elegant thougth)
+  result = subcontext.genPeriodic(elem, len_proc)
+  let olddeser = result.deserialize
+  result.deserialize = proc (s: NimNode): NimNode =
+    let origin = size.insert_source(s, relative_depth)
+    let deser = olddeser(s)
+    if elem.kind == nnkEmpty:
+      quote do:
+        `s` = newString(`origin`)
+        `deser`
+    else:
+      quote do:
+        `s` = newSeq[`elem`](`origin`)
+        `deser`
+
 proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk =
   let context = immutableContext.incrementDepth()
   result.has_hidden = false
@@ -108,19 +112,7 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk =
         error("Strings are not allowed in static context")
       assert(context.overrides.size.len in 0..1, "To many 'size' options")
       if context.overrides.size.len > 0:
-        let capture = context.overrides.size[0]
-        let size = capture.size
-        let relative_depth = context.depth - capture.depth
-        let len_proc = proc (s: NimNode): NimNode =
-          (quote do: `s`.len()).unfold()
-        result = context.genPeriodic(newEmptyNode(), len_proc)
-        let olddeser = result.deserialize
-        result.deserialize = proc (s: NimNode): NimNode =
-          let origin = size.insert_source(s, relative_depth)
-          let deser = olddeser(s)
-          quote do:
-            `s` = newString(`origin`)
-            `deser`
+        result = context.handleSizeOption()
       else:
         let len_proc = proc (s: NimNode): NimNode =
             (quote do: len(`s`)).unfold()
@@ -163,20 +155,8 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk =
               " structures")
       let elem = thetype[1]
       if context.overrides.size.len > 0:
-        var subcontext = context
-        let capture = subcontext.overrides.size.pop()
-        let size = capture.size
-        let relative_depth = context.depth - capture.depth
-        let seqLen = proc (s: NimNode): NimNode =
-          (quote do: `s`.len()).unfold()
-        result = subcontext.genPeriodic(elem, seqLen)
-        let olddeser = result.deserialize
-        result.deserialize = proc (s: NimNode): NimNode =
-          let origin = size.insert_source(s, relative_depth)
-          let deser = olddeser(s)
-          quote do:
-            `s` = newSeq[`elem`](`origin`)
-            `deser`
+        result = context.handleSizeOption(elem)
+
       else:
         let seqLen = proc (source: NimNode): NimNode =
           (quote do: len(`source`)).unfold()
@@ -226,5 +206,3 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk =
   else:
     error("Unexpected AST: " & thetype.treeRepr & "\n at " & thetype.lineinfo())
   result.dynamic = not context.is_static
-
-

nesm/utils.nim

@@ -3,6 +3,7 @@ import macros
 proc unfold*(node: NimNode): NimNode {.compileTime.}
 proc correct_sum*(part_size: NimNode): NimNode {.compileTime.}
 proc onlyname*(node: NimNode): NimNode {.compileTime.}
+proc dig*(node: NimNode, depth: Natural): NimNode {.compileTime.}
 
 proc correct_sum(part_size: NimNode): NimNode =
   if part_size.kind == nnkInfix or part_size.len == 0:
@@ -19,3 +20,22 @@ proc onlyname(node: NimNode): NimNode =
   case node.kind
   of nnkIdent, nnkPrefix, nnkPostfix: node.basename
   else: node
+
+proc dig(node: NimNode, depth: Natural): NimNode {.compileTime.} =
+  if depth == 0:
+    return node
+  case node.kind
+  of nnkDotExpr:
+    node.expectMinLen(1)
+    node[0].dig(depth - 1)
+  of nnkCall:
+    node.expectMinLen(2)
+    node[1].dig(depth - 1)
+  of nnkEmpty:
+    node
+  of nnkIdent, nnkSym:
+    error("Too big depth to dig: " & $depth)
+    newEmptyNode()
+  else:
+    error("Unknown symbol: " & node.treeRepr)
+    newEmptyNode()

tests/bugs.nim

@@ -0,0 +1,37 @@
+import nesm
+import helpers.rnw
+import unittest
+import streams
+
+type MNIST_imgs* = object
+  magic_number*: int32
+  n_imgs*: int32
+  n_rows*: int32
+  n_cols*: int32
+  data*: seq[uint8]
+toSerializable(MNIST_imgs, endian: bigEndian)
+suite "Bugs":
+  test "#5":
+    var t: MNIST_imgs
+    let rnw = getReaderNWriter()
+    t.data = randomSeqWith(uint8(rand(100)))
+    t.serialize(rnw)
+    rnw.setPosition(0)
+    let dt = MNIST_imgs.deserialize(rnw)
+    require(dt.data.len == t.data.len)
+
+  test "#7":
+    serializable:
+      type MyType = object
+        kind: byte
+        size: uint32
+        data: seq[byte] as {size: {}.size}
+    let rnw = getReaderNWriter()
+    var t: MyType
+    t.kind = rand(100).byte
+    t.data = randomSeqWith(byte(rand(100)))
+    t.size = uint32(max(1, t.data.len) - 1)
+    t.serialize(rnw)
+    rnw.setPosition(0)
+    let dt = MyType.deserialize(rnw)
+    require(dt.data.len == dt.size.int)

tests/customseqsize.nim

@@ -17,15 +17,14 @@ suite "Custom periodic size":
     let rnw = get_reader_n_writer()
     var o: MySeq
     o.data = random_seq_with(rand(20000).int32)
-    o.size = o.data.len.int32
+    o.size = max(0, o.data.len.int32 - 1)
     o.a = rand(20000).int32
     o.serialize(rnw)
     rnw.setPosition(0)
     let dso = MySeq.deserialize(rnw)
-    require(size(o) == size(dso))
     check(o.a == dso.a)
     require(o.size == dso.size)
-    require(o.data.len == dso.data.len)
+    require(o.size.int == dso.data.len)
     for i in 0..<o.size:
       check(o.data[i] == dso.data[i])
 
@@ -40,15 +39,14 @@ suite "Custom periodic size":
     let rnw = get_reader_n_writer()
     var o: MyString
     o.data = get_random_string()
-    o.size = o.data.len.int32
+    o.size = max(0, o.data.len.int32 - 1)
     o.a = rand(20000).int32
     o.serialize(rnw)
     rnw.setPosition(0)
     let dso = MyString.deserialize(rnw)
-    require(size(o) == size(dso))
     check(o.a == dso.a)
     require(o.size == dso.size)
-    require(o.data.len == dso.data.len)
+    require(o.size.int == dso.data.len)
     for i in 0..<o.size:
       check(o.data[i] == dso.data[i])
 
@@ -62,13 +60,12 @@ suite "Custom periodic size":
     let rnw = get_reader_n_writer()
     var o: MySeqStr
     o.data = random_seq_with(get_random_string())
-    o.s = o.data.len.int32
+    o.s = max(0, o.data.len.int32 - 1)
     o.a = rand(20000).int32
     o.serialize(rnw)
     rnw.setPosition(0)
     let dso = MySeqStr.deserialize(rnw)
-    require(dso.size() == o.size())
-    require(dso.data.len == o.data.len)
+    require(o.s.int == dso.data.len)
     require(dso.s == o.s)
     check(dso.a == o.a)
     for i in 0..<o.s:
@@ -89,15 +86,15 @@ suite "Custom periodic size":
       for i in 0..<o.columns:
         result[i] = rand(20000).int32
     o.data = random_seq_with(get_line())
-    o.lines = o.data.len.int32
+    o.lines = max(0, o.data.len.int32 - 1)
     o.serialize(rnw)
     rnw.setPosition(0)
     let dso = Matrix.deserialize(rnw)
-    require(o.size() == dso.size())
-    require(o.data.len == dso.data.len)
+    require(o.lines.int == dso.data.len)
     require(o.lines == dso.lines)
     require(o.columns == dso.columns)
     for i in 0..<o.lines:
+      check(dso.data[i].len == o.columns.int)
       for j in 0..<o.columns:
         check(o.data[i][j] == dso.data[i][j])
 
@@ -113,15 +110,14 @@ suite "Custom periodic size":
     let rnw = get_reader_n_writer()
     var o: Nester
     o.a.data = random_seq_with(rand(20000).int32)
-    o.a.dsize = o.a.data.len.int32
+    o.a.dsize = max(0, o.a.data.len.int32 - 1)
     o.a.a = rand(20000).int32
     o.serialize(rnw)
     rnw.setPosition(0)
     let dso = Nester.deserialize(rnw)
-    require(size(o) == size(dso))
     check(o.a.a == dso.a.a)
+    require(o.a.dsize.int == dso.a.data.len)
     require(o.a.dsize == dso.a.dsize)
-    require(o.a.data.len == dso.a.data.len)
     for i in 0..<o.a.dsize:
       check(o.a.data[i] == dso.a.data[i])
 
@@ -135,13 +131,12 @@ suite "Custom periodic size":
     var o: NestedTuple
     o.data.name = get_random_string()
     o.data.code = random_seq_with(rand(20000).int32)
-    o.length = o.data.code.len.int32
+    o.length = max(0, o.data.code.len.int32 - 1)
     o.serialize(rnw)
     rnw.setPosition(0)
     let dso = NestedTuple.deserialize(rnw)
-    require(size(o) == size(dso))
     check(o.data.name == dso.data.name)
     require(o.length == dso.length)
-    require(o.data.code.len == dso.data.code.len)
+    require(o.length.int == dso.data.code.len)
     for i in 0..<o.length:
       check(o.data.code[i] == dso.data.code[i])