From 6b08b33ff82a5d5b7eec166c6d7427446ceab75a Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Tue, 24 Jan 2017 22:54:25 +0000 Subject: [PATCH 01/67] NTP packet generation and parsing demo --- demos/ntp.nim | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++ demos/ntp.ntp | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 244 insertions(+) create mode 100644 demos/ntp.nim create mode 100644 demos/ntp.ntp diff --git a/demos/ntp.nim b/demos/ntp.nim new file mode 100644 index 0000000..134f993 --- /dev/null +++ b/demos/ntp.nim @@ -0,0 +1,122 @@ +# +# NTP client demo +# + +import + nativesockets, + net, + os, + sequtils, + strutils, + times + +import NESM + +# 1900 to 1970 in seconds +const unix_time_delta = 2208988800.0 + +proc to_host(x: uint32): uint32 = ntohl(x) + +proc to_net(x: uint32): uint32 = + htonl(x) + +serializable: + static: + type + nuint16 = distinct uint16 + nint16 = distinct int16 + nuint32 = distinct uint32 + nint32 = distinct int32 + nuint64 = distinct uint64 + nint64 = distinct int64 + + FixedPoint16dot16 = uint32 + seconds = distinct nuint32 + fraction = distinct nuint32 + FixedPoint32dot32 = object + seconds: uint32 + fraction: uint32 + + NTPTimeStamp = object + seconds: uint32 + fraction: uint32 + + LeapVersionMode = uint8 + + NTPPacket = object + leap_version_mode: LeapVersionMode + stratum: uint8 + polling_interval: int8 + clock_precision: uint8 + delay: FixedPoint16dot16 + dispersion: FixedPoint16dot16 + reference_id: uint32 + reference_ts: NTPTimeStamp + origin_ts: NTPTimeStamp + receive_ts: NTPTimeStamp + transmit_ts: NTPTimeStamp + + +proc ntp_ts_to_epoch(ts: NTPTimeStamp): float64 = + const twoto32 = 4294967296.0 + let + sec = ts.seconds.to_host.float - unix_time_delta + frac = ts.fraction.to_host.float / twoto32 + sec + frac + +proc epoch_to_ntp_ts(x: float): NTPTimeStamp = + + result = NTPTimeStamp( + seconds: uint32(x).to_net() + ) + +const packet_size = NTPPacket.size() +assert packet_size == 48 + +proc main() = + if paramCount() != 1: + echo "Usage $# " % paramStr(0) + quit(1) + + let target = paramStr(1) + + var b = NTPPacket() + + # example values + b.leap_version_mode = 0x23 + b.stratum = 0x03 + b.polling_interval = 0x06 + b.clock_precision = 0xfa + b.delay = 0x00010000 + b.dispersion = 0x00010000 + b.reference_id = 0xaabbcc + b.origin_ts = epochTime().epoch_to_ntp_ts + + let bytes = cast[string](b.serialize()) + + var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP, true) + # client send time + let t1 = epochTime() + doAssert socket.sendTo(target, Port(123), bytes) == packet_size + + let raw_resp = socket.recv(packet_size, timeout=1000) + + # client receive time + let t4 = epochTime() + + assert raw_resp.len == packet_size + let resp: NTPPacket = deserialize(NTPPacket, raw_resp) + + let t2 = resp.receive_ts.ntp_ts_to_epoch + let t3 = resp.transmit_ts.ntp_ts_to_epoch + + # NTP delay / clock_offset calculation + let delay = (t4 - t1) - (t3 - t2) + let clock_offset = (t2 - t1 + t3 - t4)/2 + + echo "delay: ", delay * 1000, " ms" + echo "clock_offset ", clock_offset * 1000, " ms" + + +when isMainModule: + main() diff --git a/demos/ntp.ntp b/demos/ntp.ntp new file mode 100644 index 0000000..96d1519 --- /dev/null +++ b/demos/ntp.ntp @@ -0,0 +1,122 @@ +# +# NTP client demo +# + +import + nativesockets, + net, + os, + sequtils, + strutils, + times + +import NESM + +# 1900 to 1970 in seconds +const unix_time_delta = 2208988800.0 + +proc to_host(x: uint32): uint32 = ntohl(x) + +proc to_net(x: uint32): uint32 = + htonl(x) + +serializable: + static: + type + nuint16 = distinct uint16 + nint16 = distinct int16 + nuint32 = distinct uint32 + nint32 = distinct int32 + nuint64 = distinct uint64 + nint64 = distinct int64 + + FixedPoint16dot16 = uint32 + seconds = distinct nuint32 + fraction = distinct nuint32 + FixedPoint32dot32 = object + seconds: uint32 + fraction: uint32 + + NTPTimeStamp = object + seconds: uint32 + fraction: uint32 + + LeapVersionMode = uint8 + + NTPPacket = object + leap_version_mode: LeapVersionMode + stratum: uint8 + polling_interval: int8 + clock_precision: uint8 + delay: FixedPoint16dot16 + dispersion: FixedPoint16dot16 + reference_id: uint32 + reference_ts: NTPTimeStamp + origin_ts: NTPTimeStamp + receive_ts: NTPTimeStamp + transmit_ts: NTPTimeStamp + + +proc ntp_ts_to_epoch(ts: NTPTimeStamp): float64 = + const twoto32 = 4294967296.0 + let + sec = ts.seconds.to_host.float - unix_time_delta + frac = ts.fraction.to_host.float / twoto32 + sec + frac + +proc epoch_to_ntp_ts(x: float): NTPTimeStamp = + + result = NTPTimeStamp( + seconds: uint32(x).to_net() + ) + +proc main() = + if paramCount() != 1: + echo "Usage $# " % paramStr(0) + quit(1) + + let target = paramStr(1) + var b = NTPPacket() + + # example values + b.leap_version_mode = 0x23 + b.stratum = 0x03 + b.polling_interval = 0x06 + b.clock_precision = 0xfa + b.delay = 0x00010000 + b.dispersion = 0x00010000 + b.reference_id = 0xaabbcc + b.origin_ts = epochTime().epoch_to_ntp_ts + + var bytes = "" + for i in b.serialize(): + bytes.add chr(i) + + assert bytes.len == 48 + + var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP, true) + # client send time + let t1 = epochTime() + doAssert socket.sendTo(target, Port(123), bytes) == 48 + + let raw_resp = socket.recv(48, timeout=1000) + + # client receive time + let t4 = epochTime() + + assert raw_resp.len == 48 + let resp: NTPPacket = deserialize(NTPPacket, raw_resp) + + let t2 = resp.receive_ts.ntp_ts_to_epoch + let t3 = resp.transmit_ts.ntp_ts_to_epoch + + # NTP delay / clock_offset calculation + let delay = (t4 - t1) - (t3 - t2) + let clock_offset = (t2 - t1 + t3 - t4)/2 + + echo "delay: ", delay * 1000, " ms" + echo "clock_offset ", clock_offset * 1000, " ms" + + +when isMainModule: + main() From 8e36d4562b2094c16523feb9da430bda501bca0a Mon Sep 17 00:00:00 2001 From: xomachine Date: Tue, 23 May 2017 16:17:16 +0300 Subject: [PATCH 02/67] Option parsing moved to a new proc --- nesm/objects.nim | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/nesm/objects.nim b/nesm/objects.nim index e95d8ba..d07b28d 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -17,6 +17,8 @@ proc genCase(context: Context, decl: NimNode): TypeChunk {.compileTime.} proc caseWorkaround(tc: TypeChunk): TypeChunk {.compileTime.} proc evalSize(e: NimNode): BiggestInt {.compileTime.} proc genFields(context: Context, decl: NimNode): FieldChunk {.compileTime.} +proc applyOptions(context: Context, + options: NimNode | seq[NimNode]): Context {.compileTime.} proc caseWorkaround(tc: TypeChunk): TypeChunk = # st - type of field under case @@ -36,6 +38,19 @@ proc caseWorkaround(tc: TypeChunk): TypeChunk = `ods` `s` = `tmpvar` +proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = + result = context + for option in options.items(): + option.expectKind(nnkExprColonExpr) + option.expectMinLen(2) + let key = option[0].repr + let val = option[1].repr + case key + of "endian": + result.swapEndian = cmpIgnoreStyle(val, $cpuEndian) != 0 + else: + error("Unknown setting: " & key) + proc genObject(context: Context, thetype: NimNode): TypeChunk = var elems = newSeq[Field]() var newContext = context @@ -63,16 +78,7 @@ proc genObject(context: Context, thetype: NimNode): TypeChunk = declaration[1].kind == nnkTableConstr: # The set: {key:value} syntax encountered let paramslist = declaration[1] - for param in paramslist.children(): - param.expectKind(nnkExprColonExpr) - param.expectMinLen(2) - let key = param[0].repr - let val = param[1].repr - case key - of "endian": - newContext.swapEndian = cmpIgnoreStyle(val, $cpuEndian) != 0 - else: - error("Unknown setting: " & key) + newContext = newContext.applyOptions(paramslist) else: let fchunk = newContext.genFields(declaration) elems &= fchunk.entries From ad1c91b8781ceea043fe3860a3ac4e50243759c6 Mon Sep 17 00:00:00 2001 From: xomachine Date: Tue, 23 May 2017 16:41:53 +0300 Subject: [PATCH 03/67] Added test for inline options syntax --- tests/endiantest.nim | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/endiantest.nim b/tests/endiantest.nim index fc487f6..e7574b6 100644 --- a/tests/endiantest.nim +++ b/tests/endiantest.nim @@ -1,4 +1,6 @@ from nesm import serializable +from endians import bigEndian32, littleEndian32 +import helpers.rnw import unittest suite "Endianness support": @@ -57,3 +59,21 @@ suite "Endianness support": check(o.set[1] == 20) let so = o.serialize() check(so == teststring) + + test "Inline syntax": + serializable: + static: + type SomeInline = object + set: {endian: bigEndian} + od: int32 + sd: int32 {endian: littleEndian} + od2: int32 + let rnw = get_random_reader_n_writer() + let dso = SomeInline.deserialize(rnw) + var a: SomeInline + bigEndian32(a.od.addr, rnw.buffer[0].addr) + littleEndian32(a.sd.addr, rnw.buffer[4].addr) + bigEndian32(a.od2.addr, rnw.buffer[8].addr) + check(dso.od == a.od) + check(dso.sd == a.sd) + check(dso.od2 == a.od2) From 691d28ce99c64aba72e9f9ccdca511518b5614ae Mon Sep 17 00:00:00 2001 From: xomachine Date: Tue, 23 May 2017 16:58:16 +0300 Subject: [PATCH 04/67] Implementation of inline options syntax --- nesm.nim | 13 +++++++++++-- nesm/objects.nim | 6 +++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/nesm.nim b/nesm.nim index 316a43c..81145b2 100644 --- a/nesm.nim +++ b/nesm.nim @@ -154,10 +154,19 @@ proc cleanupTypeDeclaration(declaration: NimNode): NimNode = for cc in c.children(): children.add(cleanupTypeDeclaration(cc)) of nnkIdentDefs: - if c[^2].repr == "cstring": + let last = if c[^1].kind == nnkEmpty: c.len - 2 else: c.len - 1 + if c[last].kind == nnkCurlyExpr: + # newID need to be a NimNode that contains IdentDefs node + # to utilze recursive call of cleanupTypeDeclaration + var newID = newTree(nnkStmtList, newNimNode(nnkIdentDefs)) + copyChildrenTo(c, newID[0]) + # first element of CurlyExpr is an actual type + newID[0][last] = c[last][0] + children.add(newID.cleanupTypeDeclaration()[0]) + elif c[last].repr == "cstring": var newID = newNimNode(nnkIdentDefs) copyChildrenTo(c, newID) - newID[^2] = newIdentNode("string") + newID[last] = newIdentNode("string") children.add(newID) elif c.len == 3 and c[0] == settingsKeyword and c[1].kind == nnkTableConstr: diff --git a/nesm/objects.nim b/nesm/objects.nim index d07b28d..b69736c 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -136,7 +136,11 @@ proc genFields(context: Context, decl: NimNode): FieldChunk = if decl[^1].kind == nnkEmpty: decl.len - 2 else: decl.len - 1 let subtype = decl[last] - let chunk = context.genTypeChunk(subtype) + let chunk = + if subtype.kind == nnkCurlyExpr: + context.applyOptions(toSeq(subtype.children)[1.. Date: Tue, 23 May 2017 17:15:42 +0300 Subject: [PATCH 05/67] Eliminated code duplication in options parsing --- nesm.nim | 21 ++------------------- nesm/objects.nim | 13 +++++++++++-- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/nesm.nim b/nesm.nim index 81145b2..53daf74 100644 --- a/nesm.nim +++ b/nesm.nim @@ -10,6 +10,7 @@ from streams import Stream, newStringStream when not defined(nimdoc): from nesm.typesinfo import TypeChunk, Context from nesm.generator import genTypeChunk, STREAM_NAME + from nesm.objects import applyOptions else: import endians type TypeChunk = object @@ -194,25 +195,7 @@ macro toSerializable*(typedecl: typed, settings: varargs[untyped]): untyped = when defined(debug): hint(typedecl.symbol.getImpl().treeRepr()) var ast = typedecl.symbol.getImpl() - for arg in settings: - arg.expectKind(nnkExprColonExpr) - arg.expectMinLen(2) - let key = $arg[0] - let value = $arg[1] - case key - of "dynamic": - if value == "false": - ctx.is_static = true - elif value == "true": - ctx.is_static = false - else: - error("'dynamic' property can be only 'true' or 'false', but not: " & - value) - of "endian": - if value != $cpuEndian: - ctx.swapEndian = true - else: - error("Unknown property: " & key) + ctx = ctx.applyOptions(settings) when defined(debug): hint(ast.treeRepr) result.add(ctx.prepare(ast)) diff --git a/nesm/objects.nim b/nesm/objects.nim index b69736c..d251733 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -13,12 +13,12 @@ type has_hidden: bool proc genObject*(context: Context, thetype: NimNode): TypeChunk {.compileTime.} +proc applyOptions*(context: Context, + options: NimNode | seq[NimNode]): Context {.compileTime.} proc genCase(context: Context, decl: NimNode): TypeChunk {.compileTime.} proc caseWorkaround(tc: TypeChunk): TypeChunk {.compileTime.} proc evalSize(e: NimNode): BiggestInt {.compileTime.} proc genFields(context: Context, decl: NimNode): FieldChunk {.compileTime.} -proc applyOptions(context: Context, - options: NimNode | seq[NimNode]): Context {.compileTime.} proc caseWorkaround(tc: TypeChunk): TypeChunk = # st - type of field under case @@ -48,6 +48,15 @@ proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = case key of "endian": result.swapEndian = cmpIgnoreStyle(val, $cpuEndian) != 0 + of "dynamic": + let code = int(cmpIgnoreStyle(val, "true") == 0) + + 2*int(cmpIgnoreStyle(val, "false") == 0) + case code + of 0: error("The dynamic property can be only 'true' or 'false' but not" & + val) + of 1: result.is_static = true + of 2: result.is_static = false + else: error("Unexpected error! dynamic is in superposition! WTF?") else: error("Unknown setting: " & key) From c9c88cf71ffdb6c18770e829f1d12f87d06a31fb Mon Sep 17 00:00:00 2001 From: xomachine Date: Tue, 23 May 2017 17:40:04 +0300 Subject: [PATCH 06/67] Added test for the custom seq size feature --- tests/customseqsize.nim | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/customseqsize.nim diff --git a/tests/customseqsize.nim b/tests/customseqsize.nim new file mode 100644 index 0000000..d32e439 --- /dev/null +++ b/tests/customseqsize.nim @@ -0,0 +1,52 @@ +from nesm import serializable +from streams import setPosition +import unittest +import helpers.rnw + +{.hint[XDeclaredButNotUsed]:off.} + +suite "Custom periodic size": + test "Seq": + serializable: + type + MySeq = object + size: int32 + a: int32 + data: seq[int32] {size: size} + + let rnw = get_reader_n_writer() + var o: MySeq + o.data = random_seq_with(random(20000).int32) + o.size = o.data.len.int32 + o.a = random(20000).int32 + o.serialize(rnw) + rnw.setPosition(0) + let dso = MySeq.deserialize(rnw) + check(o.a == dso.a) + require(o.size == dso.size) + require(o.data.len == dso.data.len) + for i in 0.. Date: Tue, 23 May 2017 19:18:13 +0300 Subject: [PATCH 07/67] The custom seq size feature implementation --- nesm.nim | 1 + nesm/generator.nim | 50 ++++++++++++++++++++++++++++++++++++++++------ nesm/objects.nim | 2 ++ nesm/periodic.nim | 5 +++-- nesm/typesinfo.nim | 3 ++- 5 files changed, 52 insertions(+), 9 deletions(-) diff --git a/nesm.nim b/nesm.nim index 53daf74..69d3588 100644 --- a/nesm.nim +++ b/nesm.nim @@ -78,6 +78,7 @@ when not defined(nimdoc): static: var ctx: Context ctx.declared = initTable[string, TypeChunk]() + ctx.size_override = newSeq[NimIdent]() proc generateProc(pattern: string, name: string, sign: string, body: NimNode = newEmptyNode()): NimNode = diff --git a/nesm/generator.nim b/nesm/generator.nim index 44e38f3..ec33405 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -26,6 +26,14 @@ proc correct_sum(part_size: NimNode): NimNode = else: result_node.infix("+=", part_size) +proc dig_root(source: NimNode): NimNode = + case source.kind + of nnkIdent, nnkSym, nnkEmpty: + source + else: + source.expectMinLen(1) + source[0].dig_root() + proc genTypeChunk(context: Context, thetype: NimNode): TypeChunk = result.has_hidden = false result.nodekind = thetype.kind @@ -56,9 +64,24 @@ proc genTypeChunk(context: Context, thetype: NimNode): TypeChunk = elif thetype.repr == "string": if context.is_static: error("Strings are not allowed in static context") - let len_proc = proc (s: NimNode):NimNode = - (quote do: len(`s`)).last - result = context.genPeriodic(newEmptyNode(), len_proc) + assert(context.size_override.len in 0..1, "To many 'size' options") + if context.size_override.len > 0: + let last = context.size_override[0] + let len_proc = proc (s: NimNode): NimNode = + let origin = s.dig_root() + (quote do: `origin`.`last`).last + result = context.genPeriodic(newEmptyNode(), len_proc) + let olddeser = result.deserialize + result.deserialize = proc (s: NimNode): NimNode = + let origin = s.dig_root() + let deser = olddeser(s) + quote do: + `s` = newString(`origin`.`last`) + `deser` + else: + let len_proc = proc (s: NimNode): NimNode = + (quote do: len(`s`)).last + result = context.genPeriodic(newEmptyNode(), len_proc) elif thetype.repr == "cstring": if context.is_static: error("CStrings are not allowed in static context") @@ -96,9 +119,24 @@ proc genTypeChunk(context: Context, thetype: NimNode): TypeChunk = error("Dynamic types are not supported in static" & " structures") let elem = thetype[1] - let seqLen = proc (source: NimNode): NimNode = - (quote do: len(`source`)).last - result = context.genPeriodic(elem, seqLen) + var subcontext = context + if context.size_override.len > 0: + let last = subcontext.size_override.pop() + let seqLen = proc (s: NimNode): NimNode = + let origin = s.dig_root() + (quote do: `origin`.`last`).last + result = subcontext.genPeriodic(elem, seqLen) + let olddeser = result.deserialize + result.deserialize = proc (s: NimNode): NimNode = + let origin = s.dig_root() + let deser = olddeser(s) + quote do: + `s` = newSeq[`elem`](`origin`.`last`) + `deser` + else: + let seqLen = proc (source: NimNode): NimNode = + (quote do: len(`source`)).last + result = subcontext.genPeriodic(elem, seqLen) of "set": result = context.genSet(thetype) else: diff --git a/nesm/objects.nim b/nesm/objects.nim index d251733..1fe2642 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -57,6 +57,8 @@ proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = of 1: result.is_static = true of 2: result.is_static = false else: error("Unexpected error! dynamic is in superposition! WTF?") + of "size": + result.size_override.add(!val) else: error("Unknown setting: " & key) diff --git a/nesm/periodic.nim b/nesm/periodic.nim index 24a3c60..b588350 100644 --- a/nesm/periodic.nim +++ b/nesm/periodic.nim @@ -50,11 +50,12 @@ proc genPeriodic*(context: Context, elem: NimNode, quote do: if `lens` > 0: `serialization` result.deserialize = proc (s: NimNode): NimNode = - let size = (quote do: `lenvarname` * `eSize`).last + let lens = length(s) + let size = (quote do: `lens` * `eSize`).last let newsource = (quote do: `s`[0]).last let deserialization = genDeserialize(newsource, size) quote do: - if `lenvarname` > 0: `deserialization` + if `lens` > 0: `deserialization` result.dynamic = not context.is_static result.has_hidden = false else: diff --git a/nesm/typesinfo.nim b/nesm/typesinfo.nim index 9779424..f0b4ac1 100644 --- a/nesm/typesinfo.nim +++ b/nesm/typesinfo.nim @@ -1,6 +1,6 @@ from macros import error, warning -from macros import NimNodeKind, nnkEnumTy +from macros import NimNodeKind, nnkEnumTy, NimIdent from tables import Table const basic_types = [ @@ -22,6 +22,7 @@ type Context* = object declared*: Table[string, TypeChunk] + size_override*: seq[NimIdent] is_static*: bool swapEndian*: bool From 6f311170cd6cc2d7d95ba82d0a27bf2bf24f2190 Mon Sep 17 00:00:00 2001 From: xomachine Date: Tue, 23 May 2017 19:42:54 +0300 Subject: [PATCH 08/67] Added size checking --- tests/customseqsize.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/customseqsize.nim b/tests/customseqsize.nim index d32e439..a87e832 100644 --- a/tests/customseqsize.nim +++ b/tests/customseqsize.nim @@ -22,6 +22,7 @@ suite "Custom periodic size": o.serialize(rnw) rnw.setPosition(0) let dso = MySeq.deserialize(rnw) + require(size(o) == size(dso)) check(o.a == dso.a) require(o.size == dso.size) require(o.data.len == dso.data.len) @@ -44,6 +45,7 @@ suite "Custom periodic size": o.serialize(rnw) rnw.setPosition(0) let dso = MyString.deserialize(rnw) + require(size(o) == size(dso)) check(o.a == dso.a) require(o.size == dso.size) require(o.data.len == dso.data.len) From 6d8154bbb351ec8dcdbf8a09bd52c1615ef56753 Mon Sep 17 00:00:00 2001 From: xomachine Date: Tue, 23 May 2017 20:18:01 +0300 Subject: [PATCH 09/67] Added more complex tests for custom seq size feature --- tests/customseqsize.nim | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/customseqsize.nim b/tests/customseqsize.nim index a87e832..0e67863 100644 --- a/tests/customseqsize.nim +++ b/tests/customseqsize.nim @@ -52,3 +52,51 @@ suite "Custom periodic size": for i in 0.. Date: Tue, 23 May 2017 20:44:37 +0300 Subject: [PATCH 10/67] Correct root digging --- nesm/generator.nim | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nesm/generator.nim b/nesm/generator.nim index ec33405..d38017a 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -30,9 +30,15 @@ proc dig_root(source: NimNode): NimNode = case source.kind of nnkIdent, nnkSym, nnkEmpty: source - else: + of nnkDotExpr: source.expectMinLen(1) source[0].dig_root() + of nnkCall: + source.expectMinLen(2) + source[1].dig_root() + else: + error("Unknown symbol: " & source.treeRepr) + newEmptyNode() proc genTypeChunk(context: Context, thetype: NimNode): TypeChunk = result.has_hidden = false From 609261ab66cdc65caf806e9194af3a72d19ebb56 Mon Sep 17 00:00:00 2001 From: xomachine Date: Tue, 23 May 2017 21:09:53 +0300 Subject: [PATCH 11/67] Fixed typo in tests --- tests/customseqsize.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/customseqsize.nim b/tests/customseqsize.nim index 0e67863..c6bfeef 100644 --- a/tests/customseqsize.nim +++ b/tests/customseqsize.nim @@ -71,7 +71,7 @@ suite "Custom periodic size": require(dso.data.len == o.data.len) require(dso.s == o.s) check(dso.a == o.a) - for i in 0.. Date: Tue, 23 May 2017 21:32:15 +0300 Subject: [PATCH 12/67] Intuitive order of 'size' options --- nesm/objects.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nesm/objects.nim b/nesm/objects.nim index 1fe2642..524fe6a 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -58,7 +58,7 @@ proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = of 2: result.is_static = false else: error("Unexpected error! dynamic is in superposition! WTF?") of "size": - result.size_override.add(!val) + result.size_override.insert(!val, 0) else: error("Unknown setting: " & key) From f9d1199770a25a1150bd477539475e14a84067ea Mon Sep 17 00:00:00 2001 From: xomachine Date: Tue, 23 May 2017 21:33:17 +0300 Subject: [PATCH 13/67] Fixed incorrect codegen for the complex custom size seq test --- nesm/periodic.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nesm/periodic.nim b/nesm/periodic.nim index b588350..a0ddeab 100644 --- a/nesm/periodic.nim +++ b/nesm/periodic.nim @@ -66,7 +66,7 @@ proc genPeriodic*(context: Context, elem: NimNode, let periodic_len = length(s) let newsource = (quote do: `s`[`index_letter`]).last let chunk_size = one_chunk.size(newsource) - if is_array and chunk_size.kind != nnkStmtList: + if is_array and periodic_len == lenvarname: periodic_len.infix("*", chunk_size) else: let chunk_expr = correct_sum(chunk_size) @@ -81,10 +81,11 @@ proc genPeriodic*(context: Context, elem: NimNode, for `index_letter` in 0..<(`periodic_len`): `chunk_expr` result.deserialize = proc(s: NimNode): NimNode = + let lens = length(s) let newsource = (quote do: `s`[`index_letter`]).last let chunk_expr = onechunk.deserialize(newsource) quote do: - for `index_letter` in 0..<(`lenvarname`): + for `index_letter` in 0..<(`lens`): `chunk_expr` if not is_array: let size_header_chunk = context.genTypeChunk( From 75add2f2850307dc45d1937eb25421423529968b Mon Sep 17 00:00:00 2001 From: xomachine Date: Tue, 23 May 2017 22:22:30 +0300 Subject: [PATCH 14/67] Correct condition for simple size evaluation --- nesm/periodic.nim | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/nesm/periodic.nim b/nesm/periodic.nim index a0ddeab..690b276 100644 --- a/nesm/periodic.nim +++ b/nesm/periodic.nim @@ -20,7 +20,13 @@ proc genCStringSerialize*(name: NimNode): NimNode {.compileTime.} = `name`.len) `STREAM_NAME`.write('\x00') - +proc findInChilds(a, b: NimNode): bool = + if a == b: return true + else: + for ac in a.children(): + if ac.findInChilds(b): + return true + return false proc genPeriodic*(context: Context, elem: NimNode, length: proc (s:NimNode): NimNode, @@ -66,7 +72,7 @@ proc genPeriodic*(context: Context, elem: NimNode, let periodic_len = length(s) let newsource = (quote do: `s`[`index_letter`]).last let chunk_size = one_chunk.size(newsource) - if is_array and periodic_len == lenvarname: + if not chunk_size.findInChilds(index_letter): periodic_len.infix("*", chunk_size) else: let chunk_expr = correct_sum(chunk_size) From 387c00f3034272dd336942ccb710583f159ed3a1 Mon Sep 17 00:00:00 2001 From: xomachine Date: Tue, 23 May 2017 22:46:24 +0300 Subject: [PATCH 15/67] Added test for nested objects with custom seq size --- tests/customseqsize.nim | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/customseqsize.nim b/tests/customseqsize.nim index c6bfeef..b7cd48a 100644 --- a/tests/customseqsize.nim +++ b/tests/customseqsize.nim @@ -100,3 +100,27 @@ suite "Custom periodic size": for i in 0.. Date: Wed, 24 May 2017 16:05:40 +0300 Subject: [PATCH 16/67] Changed custom size syntax in tests --- tests/customseqsize.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/customseqsize.nim b/tests/customseqsize.nim index b7cd48a..30933d0 100644 --- a/tests/customseqsize.nim +++ b/tests/customseqsize.nim @@ -12,7 +12,7 @@ suite "Custom periodic size": MySeq = object size: int32 a: int32 - data: seq[int32] {size: size} + data: seq[int32] {size: {}.size} let rnw = get_reader_n_writer() var o: MySeq @@ -35,7 +35,7 @@ suite "Custom periodic size": MyString = object size: int32 a: int32 - data: string {size: size} + data: string {size: {}.size} let rnw = get_reader_n_writer() var o: MyString @@ -57,7 +57,7 @@ suite "Custom periodic size": type MySeqStr = object s: int32 a: int32 - data: seq[string] {size: s} + data: seq[string] {size: {}.s} let rnw = get_reader_n_writer() var o: MySeqStr @@ -79,7 +79,7 @@ suite "Custom periodic size": type Matrix = object lines: int32 columns: int32 - data: seq[seq[int32]] {size: lines, size: columns} + data: seq[seq[int32]] {size: {}.lines, size: {}.columns} let rnw = get_reader_n_writer() var o: Matrix @@ -106,7 +106,7 @@ suite "Custom periodic size": type Nested = object dsize: int32 a: int32 - data: seq[int32] {size: dsize} + data: seq[int32] {size: {}.dsize} type Nester = object a: Nested From 2674334c7c4d091a16d5bf09eff4c4ac7cc2700a Mon Sep 17 00:00:00 2001 From: xomachine Date: Wed, 24 May 2017 16:06:20 +0300 Subject: [PATCH 17/67] Fixed Nesting test for custom size syntax --- tests/customseqsize.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/customseqsize.nim b/tests/customseqsize.nim index 30933d0..2241f56 100644 --- a/tests/customseqsize.nim +++ b/tests/customseqsize.nim @@ -110,12 +110,12 @@ suite "Custom periodic size": type Nester = object a: Nested - let rnw = get_random_reader_n_writer() + let rnw = get_reader_n_writer() var o: Nester o.a.data = random_seq_with(random(20000).int32) o.a.dsize = o.a.data.len.int32 o.a.a = random(20000).int32 - o.a.serialize(rnw) + o.serialize(rnw) rnw.setPosition(0) let dso = Nester.deserialize(rnw) require(size(o) == size(dso)) From 054d71170ed3a0e8b7bc2ad049e200e91323afe6 Mon Sep 17 00:00:00 2001 From: xomachine Date: Wed, 24 May 2017 16:08:11 +0300 Subject: [PATCH 18/67] New custom size syntax implementation --- nesm.nim | 8 ++--- nesm/generator.nim | 73 +++++++++++++++++++++++++++++++--------------- nesm/objects.nim | 2 +- nesm/typesinfo.nim | 16 ++++++++-- 4 files changed, 68 insertions(+), 31 deletions(-) diff --git a/nesm.nim b/nesm.nim index 69d3588..14718ac 100644 --- a/nesm.nim +++ b/nesm.nim @@ -4,11 +4,11 @@ when defined(js): import macros from strutils import `%` from sequtils import toSeq -from tables import Table, initTable, contains, `[]`, `[]=` +from tables import contains, `[]`, `[]=` from streams import Stream, newStringStream when not defined(nimdoc): - from nesm.typesinfo import TypeChunk, Context + from nesm.typesinfo import TypeChunk, Context, initContext from nesm.generator import genTypeChunk, STREAM_NAME from nesm.objects import applyOptions else: @@ -76,9 +76,7 @@ const SIZE_DECLARATION = "proc size$1(" & when not defined(nimdoc): static: - var ctx: Context - ctx.declared = initTable[string, TypeChunk]() - ctx.size_override = newSeq[NimIdent]() + var ctx = initContext() proc generateProc(pattern: string, name: string, sign: string, body: NimNode = newEmptyNode()): NimNode = diff --git a/nesm/generator.nim b/nesm/generator.nim index d38017a..3de24bd 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -6,7 +6,8 @@ from tables import Table, contains, `[]`, `[]=`, initTable, from strutils import `%` from sequtils import mapIt, foldl, toSeq, filterIt -proc genTypeChunk*(context: Context, thetype: NimNode): TypeChunk {.compileTime.} +proc genTypeChunk*(immutableContext: Context, + thetype: NimNode): TypeChunk {.compileTime.} proc correct_sum*(part_size: NimNode): NimNode {.compileTime.} static: @@ -26,21 +27,45 @@ proc correct_sum(part_size: NimNode): NimNode = else: result_node.infix("+=", part_size) -proc dig_root(source: NimNode): NimNode = - case source.kind - of nnkIdent, nnkSym, nnkEmpty: - source +proc dig(node: NimNode, depth: Natural): NimNode {.compileTime.} = + if depth == 0: + return node + case node.kind of nnkDotExpr: - source.expectMinLen(1) - source[0].dig_root() + node.expectMinLen(1) + node[0].dig(depth - 1) of nnkCall: - source.expectMinLen(2) - source[1].dig_root() + node.expectMinLen(2) + node[1].dig(depth - 1) + of nnkEmpty: + node + of nnkIdent, nnkSym: + error("Too big depth to dig: " & $depth) + newEmptyNode() else: - error("Unknown symbol: " & source.treeRepr) + error("Unknown symbol: " & node.treeRepr) newEmptyNode() -proc genTypeChunk(context: Context, thetype: NimNode): TypeChunk = +proc insert_source(length_declaration, source: NimNode, + depth: Natural): NimNode = + if length_declaration.kind == nnkCurly: + if length_declaration.len == 1 and length_declaration[0].kind == nnkCurly: + return length_declaration[0].insert_source(source, depth + 1) + elif length_declaration.len == 0: + return source.dig(depth) + if length_declaration.len == 0: + return length_declaration + else: + result = newNimNode(length_declaration.kind) + for child in length_declaration.children(): + result.add(child.insert_source(source, depth)) + +proc incrementDepth(ctx: Context): Context {.compileTime.} = + result = ctx + result.depth += 1 + +proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = + let context = immutableContext.incrementDepth() result.has_hidden = false result.nodekind = thetype.kind case thetype.kind @@ -72,17 +97,18 @@ proc genTypeChunk(context: Context, thetype: NimNode): TypeChunk = error("Strings are not allowed in static context") assert(context.size_override.len in 0..1, "To many 'size' options") if context.size_override.len > 0: - let last = context.size_override[0] + let capture = context.size_override[0] + let size = capture.size + let relative_depth = context.depth - capture.depth let len_proc = proc (s: NimNode): NimNode = - let origin = s.dig_root() - (quote do: `origin`.`last`).last + size.insert_source(s, relative_depth) result = context.genPeriodic(newEmptyNode(), len_proc) let olddeser = result.deserialize result.deserialize = proc (s: NimNode): NimNode = - let origin = s.dig_root() + let origin = size.insert_source(s, relative_depth) let deser = olddeser(s) quote do: - `s` = newString(`origin`.`last`) + `s` = newString(`origin`) `deser` else: let len_proc = proc (s: NimNode): NimNode = @@ -125,24 +151,25 @@ proc genTypeChunk(context: Context, thetype: NimNode): TypeChunk = error("Dynamic types are not supported in static" & " structures") let elem = thetype[1] - var subcontext = context if context.size_override.len > 0: - let last = subcontext.size_override.pop() + var subcontext = context + let capture = subcontext.size_override.pop() + let size = capture.size + let relative_depth = context.depth - capture.depth let seqLen = proc (s: NimNode): NimNode = - let origin = s.dig_root() - (quote do: `origin`.`last`).last + size.insert_source(s, relative_depth) result = subcontext.genPeriodic(elem, seqLen) let olddeser = result.deserialize result.deserialize = proc (s: NimNode): NimNode = - let origin = s.dig_root() + let origin = size.insert_source(s, relative_depth) let deser = olddeser(s) quote do: - `s` = newSeq[`elem`](`origin`.`last`) + `s` = newSeq[`elem`](`origin`) `deser` else: let seqLen = proc (source: NimNode): NimNode = (quote do: len(`source`)).last - result = subcontext.genPeriodic(elem, seqLen) + result = context.genPeriodic(elem, seqLen) of "set": result = context.genSet(thetype) else: diff --git a/nesm/objects.nim b/nesm/objects.nim index 524fe6a..cd16a82 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -58,7 +58,7 @@ proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = of 2: result.is_static = false else: error("Unexpected error! dynamic is in superposition! WTF?") of "size": - result.size_override.insert(!val, 0) + result.size_override.insert((option[1], context.depth), 0) else: error("Unknown setting: " & key) diff --git a/nesm/typesinfo.nim b/nesm/typesinfo.nim index f0b4ac1..ec5498e 100644 --- a/nesm/typesinfo.nim +++ b/nesm/typesinfo.nim @@ -1,7 +1,7 @@ from macros import error, warning from macros import NimNodeKind, nnkEnumTy, NimIdent -from tables import Table +from tables import Table, initTable const basic_types = [ "int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64", @@ -20,12 +20,24 @@ type maxcount*: uint64 else: discard + SizeCapture* = tuple + size: NimNode + depth: Natural + Context* = object declared*: Table[string, TypeChunk] - size_override*: seq[NimIdent] + size_override*: seq[SizeCapture] + depth*: Natural is_static*: bool swapEndian*: bool +proc initContext*(): Context = + result.size_override = newSeq[SizeCapture]() + result.declared = initTable[string, TypeChunk]() + result.depth = 0 + result.is_static = false + result.swapEndian = false + proc isBasic*(thetype: string): bool = thetype in basic_types From 3f4019a2c9686687425628078f5337f9d3b769b7 Mon Sep 17 00:00:00 2001 From: xomachine Date: Thu, 25 May 2017 17:30:48 +0300 Subject: [PATCH 19/67] Added documentation for serialization control syntax --- nesm/documentation.nim | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/nesm/documentation.nim b/nesm/documentation.nim index 75ad1b9..bd0d6b4 100644 --- a/nesm/documentation.nim +++ b/nesm/documentation.nim @@ -174,13 +174,46 @@ ## let ss142004 = newStringStream(cast[string](data)) ## deserialize(type(result), ss142004) ## +## Serialization options +## --------------------- +## The serialization process can be controlled via the special syntax +## **{: , : ,...}**. There are three ways of +## using this syntax: +## +## * From invocation to the end of object or another invocation +## +## .. code-block:: nim +## serializable: +## type MyType = object +## set: {} +## ... # all fields until the end of object will be affected +## ... # another set: {} can override previous one +## +## * For converted structure +## +## .. code-block:: nim +## toSerializable(TheType, ) +## # NOTE: curly braces are not required here +## +## * For particular field (inline) +## +## .. code-block:: nim +## serializable: +## type MyType = object +## typical_field: string +## field_with_special_rules: int32 {} # inline options +## # NOTE: inline options have highest priority +## another_typical_field: float32 +## +## The serialization options themself are described in the paragraphs they are +## related. +## ## Endianness switching ## ----------------- ## There is a way exists to set which endian should be used ## while [de]serialization particular structure or part of -## the structure. A special syntax **set: {endian: }** allows to set -## the endian for all the fields of structure below until the -## end or other **set: {endian: }** will override previous settings. +## the structure. A special keyword **endian** in serialization options +## allows to set the endian. ## E.g.: ## ## .. code-block:: nim From 02d4c314fc706464a2d5d7b0b7dc2a82d325b33d Mon Sep 17 00:00:00 2001 From: xomachine Date: Thu, 25 May 2017 17:31:15 +0300 Subject: [PATCH 20/67] Changed order of sections --- nesm/documentation.nim | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/nesm/documentation.nim b/nesm/documentation.nim index bd0d6b4..5334540 100644 --- a/nesm/documentation.nim +++ b/nesm/documentation.nim @@ -228,23 +228,6 @@ ## The generated code will use the **swapEndian{16|32|64}()** ## calls from the endians module to change endianness. ## -## Usage of int, float, uint types without size specifier -## ------------------------------------------------------ -## By default the serializable macro throws an error when the type -## under the macro contains basic type description without size specification. -## For example, the following code will cause an error: -## -## .. code-block:: nim -## serializable: -## type MyInt = distinct int -## -## To avoid this behaviour one can tell the macro to allow all basic type -## description without size specification -## by using **-d:allow_undefined_type_size** compiler switch. -## You must avoid to use this switch as far as possible because when -## the switch enabled the library can not guarantee proper deserialization -## of objects on devices with different architectures. -## ## Converting existent types to serializable ## ----------------------------------------- ## In case when the type to be serialized cannot be rewriten under the @@ -263,6 +246,27 @@ ## ... ## include oids # Oid's fields are visible only inside the module, so including is necessary ## toSerializable(Oid) +## ... +## toSerializable(Point2d, dynamic: false) # The "dynamic: false" option +## # is equal to "static:" for +## # serializable macro +## +## Usage of int, float, uint types without size specifier +## ------------------------------------------------------ +## By default the serializable macro throws an error when the type +## under the macro contains basic type description without size specification. +## For example, the following code will cause an error: +## +## .. code-block:: nim +## serializable: +## type MyInt = distinct int +## +## To avoid this behaviour one can tell the macro to allow all basic type +## description without size specification +## by using **-d:allow_undefined_type_size** compiler switch. +## You must avoid to use this switch as far as possible because when +## the switch enabled the library can not guarantee proper deserialization +## of objects on devices with different architectures. ## ## Enum correctness checking ## ------------------------- From 919196165555f5d1d87f91e56a5173bef6af2df5 Mon Sep 17 00:00:00 2001 From: xomachine Date: Thu, 25 May 2017 17:34:45 +0300 Subject: [PATCH 21/67] Removed useless 'when's --- nesm.nim | 183 +++++++++++++++++++++++++++---------------------------- 1 file changed, 90 insertions(+), 93 deletions(-) diff --git a/nesm.nim b/nesm.nim index 14718ac..16d8408 100644 --- a/nesm.nim +++ b/nesm.nim @@ -25,30 +25,29 @@ const DESERIALIZE_DECLARATION = """proc deserialize$1""" & """: seq[byte | char | int8 | uint8] | string):""" & """$2 = discard""" -when not defined(nimdoc): - proc makeSerializeStreamDeclaration(typename: string, - is_exported: bool, - body: NimNode): NimNode {.compileTime.} = - let itn = !typename - let fname = - if is_exported: newIdentNode("serialize").postfix("*") - else: newIdentNode("serialize") - let isin = !SERIALIZER_INPUT_NAME - quote do: - proc `fname`(`isin`: `itn`, - `STREAM_NAME`: Stream) = `body` - - proc makeDeserializeStreamDeclaration(typename: string, - is_exported: bool, - body: NimNode): NimNode {.compileTime.} = - let itn = !typename - let fname = - if is_exported: - newIdentNode("deserialize").postfix("*") - else: newIdentNode("deserialize") - quote do: - proc `fname`(thetype: typedesc[`itn`], - `STREAM_NAME`: Stream): `itn` = `body` +proc makeSerializeStreamDeclaration(typename: string, + is_exported: bool, + body: NimNode): NimNode {.compileTime.} = + let itn = !typename + let fname = + if is_exported: newIdentNode("serialize").postfix("*") + else: newIdentNode("serialize") + let isin = !SERIALIZER_INPUT_NAME + quote do: + proc `fname`(`isin`: `itn`, + `STREAM_NAME`: Stream) = `body` + +proc makeDeserializeStreamDeclaration(typename: string, + is_exported: bool, + body: NimNode): NimNode {.compileTime.} = + let itn = !typename + let fname = + if is_exported: + newIdentNode("deserialize").postfix("*") + else: newIdentNode("deserialize") + quote do: + proc `fname`(thetype: typedesc[`itn`], + `STREAM_NAME`: Stream): `itn` = `body` proc makeSerializeStreamConversion(): NimNode {.compileTime.} = let isin = !SERIALIZER_INPUT_NAME @@ -73,75 +72,73 @@ const SIZE_DECLARATION = "proc size$1(" & SERIALIZER_INPUT_NAME & ": $2): int = discard" - -when not defined(nimdoc): - static: - var ctx = initContext() - proc generateProc(pattern: string, name: string, - sign: string, - body: NimNode = newEmptyNode()): NimNode = - result = parseExpr(pattern % [sign, name]) - if body.kind != nnkEmpty: - result.body = body - - proc generateProcs(context: var Context, - obj: NimNode): NimNode {.compileTime.} = - expectKind(obj, nnkTypeDef) - expectMinLen(obj, 3) - expectKind(obj[1], nnkEmpty) - let typename = if obj[0].kind == nnkPragmaExpr: obj[0][0] else: obj[0] - let is_shared = typename.kind == nnkPostfix - let name = if is_shared: $typename.basename else: $typename - let sign = - if is_shared: "*" - else: "" - let body = obj[2] - let info = context.genTypeChunk(body) - let size_node = - info.size(newIdentNode(SERIALIZER_INPUT_NAME)) - context.declared[name] = info - let writer_conversion = makeSerializeStreamConversion() - let serializer = generateProc(SERIALIZE_DECLARATION, - name, sign, - writer_conversion) - let serialize_stream = - makeSerializeStreamDeclaration(name, is_shared, - info.serialize(newIdentNode(SERIALIZER_INPUT_NAME))) - let obtainer_conversion = - if context.is_static: - makeDeserializeStreamConversion("result") - else: newEmptyNode() - let deserializer = - if context.is_static: - generateProc(DESERIALIZE_DECLARATION, name, sign, - obtainer_conversion) - else: newEmptyNode() - let deserialize_stream = - makeDeserializeStreamDeclaration(name, is_shared, - info.deserialize(newIdentNode("result"))) - let size_declaration = - if context.is_static: STATIC_SIZE_DECLARATION - else: SIZE_DECLARATION - let sizeProc = generateProc(size_declaration, name, sign, - size_node) - newStmtList(sizeProc, serialize_stream, serializer, - deserialize_stream, deserializer) - - proc prepare(context: var Context, statements: NimNode - ): NimNode {.compileTime.} = - result = newStmtList() - case statements.kind - of nnkStmtList, nnkTypeSection, nnkStaticStmt: - let oldstatic = context.is_static - context.is_static = context.is_static or - (statements.kind == nnkStaticStmt) - for child in statements.children(): - result.add(context.prepare(child)) - context.is_static = oldstatic - of nnkTypeDef: - result.add(context.generateProcs(statements)) - else: - error("Only type declarations can be serializable") +static: + var ctx = initContext() +proc generateProc(pattern: string, name: string, + sign: string, + body: NimNode = newEmptyNode()): NimNode = + result = parseExpr(pattern % [sign, name]) + if body.kind != nnkEmpty: + result.body = body + +proc generateProcs(context: var Context, + obj: NimNode): NimNode {.compileTime.} = + expectKind(obj, nnkTypeDef) + expectMinLen(obj, 3) + expectKind(obj[1], nnkEmpty) + let typename = if obj[0].kind == nnkPragmaExpr: obj[0][0] else: obj[0] + let is_shared = typename.kind == nnkPostfix + let name = if is_shared: $typename.basename else: $typename + let sign = + if is_shared: "*" + else: "" + let body = obj[2] + let info = context.genTypeChunk(body) + let size_node = + info.size(newIdentNode(SERIALIZER_INPUT_NAME)) + context.declared[name] = info + let writer_conversion = makeSerializeStreamConversion() + let serializer = generateProc(SERIALIZE_DECLARATION, + name, sign, + writer_conversion) + let serialize_stream = + makeSerializeStreamDeclaration(name, is_shared, + info.serialize(newIdentNode(SERIALIZER_INPUT_NAME))) + let obtainer_conversion = + if context.is_static: + makeDeserializeStreamConversion("result") + else: newEmptyNode() + let deserializer = + if context.is_static: + generateProc(DESERIALIZE_DECLARATION, name, sign, + obtainer_conversion) + else: newEmptyNode() + let deserialize_stream = + makeDeserializeStreamDeclaration(name, is_shared, + info.deserialize(newIdentNode("result"))) + let size_declaration = + if context.is_static: STATIC_SIZE_DECLARATION + else: SIZE_DECLARATION + let sizeProc = generateProc(size_declaration, name, sign, + size_node) + newStmtList(sizeProc, serialize_stream, serializer, + deserialize_stream, deserializer) + +proc prepare(context: var Context, statements: NimNode + ): NimNode {.compileTime.} = + result = newStmtList() + case statements.kind + of nnkStmtList, nnkTypeSection, nnkStaticStmt: + let oldstatic = context.is_static + context.is_static = context.is_static or + (statements.kind == nnkStaticStmt) + for child in statements.children(): + result.add(context.prepare(child)) + context.is_static = oldstatic + of nnkTypeDef: + result.add(context.generateProcs(statements)) + else: + error("Only type declarations can be serializable") proc cleanupTypeDeclaration(declaration: NimNode): NimNode = var children = newSeq[NimNode]() From 17646f8f835eed06c78c14bcd263ccd172994504 Mon Sep 17 00:00:00 2001 From: xomachine Date: Thu, 25 May 2017 17:36:21 +0300 Subject: [PATCH 22/67] Removed useless fake type --- nesm.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/nesm.nim b/nesm.nim index 16d8408..a84cae1 100644 --- a/nesm.nim +++ b/nesm.nim @@ -13,7 +13,6 @@ when not defined(nimdoc): from nesm.objects import applyOptions else: import endians - type TypeChunk = object include nesm.documentation const SERIALIZER_INPUT_NAME = "obj" From 8392307d04cf66a30e5ac08138e03b5881dbb0a7 Mon Sep 17 00:00:00 2001 From: xomachine Date: Thu, 25 May 2017 17:46:37 +0300 Subject: [PATCH 23/67] Added 'deadCodeElim' pragma to reduce compilation time --- nesm.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nesm.nim b/nesm.nim index a84cae1..7959001 100644 --- a/nesm.nim +++ b/nesm.nim @@ -1,6 +1,6 @@ when defined(js): error("Non C-like targets non supported yet.") - +{.deadCodeElim: on.} import macros from strutils import `%` from sequtils import toSeq From f8393b346422e05d7ea521f829be50a9b4e27a7f Mon Sep 17 00:00:00 2001 From: xomachine Date: Thu, 25 May 2017 18:45:36 +0300 Subject: [PATCH 24/67] Added description for 'size' keyword --- nesm/documentation.nim | 56 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/nesm/documentation.nim b/nesm/documentation.nim index 5334540..df02f2f 100644 --- a/nesm/documentation.nim +++ b/nesm/documentation.nim @@ -251,6 +251,62 @@ ## # is equal to "static:" for ## # serializable macro ## +## Customizing seq and string serialization schemas +## ------------------------------------------------ +## By default, NESM serializes seq's and string's in a following way: +## 1. Serialize the seq or string length as uint32 +## 2. Serialize the seq or string content as an array[length, type] +## In other words seq and string types can be described in following +## pseudo-code: +## +## .. code-block:: nim +## serializable: +## type seq[T] = object +## length: uint32 +## data: array[length, T] +## type string = object +## length: uint32 +## data: array[length, char] +## +## In some cases this scheme is being not flexible enough, say there is +## a structure: +## +## .. code-block:: nim +## serializable: +## type Matrix = object +## lines: uint32 +## columns: uint32 +## data: array[lines, array[columns, int32]] +## +## This type is impossible in Nim due to static nature of array type. +## But how else the seq size may be controlled outside the common scheme? +## For this case the **size** keyword exists in serializable options: +## +## .. code-block:: nim +## serializable: +## type Matrix = object +## lines: uint32 # the size specifier should be placed before it's usage in the 'size' option +## columns: uint32 +## data: seq[seq[int32]] {size: {}.lines, size: {}.columns} +## # The seq's will be stored like array's but their sizes will be +## # taken from 'lines' and 'columns' fields during deserialization +## +## Note that first 'size' option controls only outer seq, but the second +## one is related to inner seq. Honestly, any valid expression can be used as +## argument for the 'size' option. Empty curly braces mean an object itself +## at the level of invocation. Special case is a double, triple, etc empty +## curly braces. Take a look at the example: +## +## .. code-block:: nim +## serializable: +## type SpecialType = object +## length: uint32 # <- this field will be used as length of subtype.a +## subtype: tuple[length: string, a: seq[int32] {size: {{}}.length}] +## +## The 'size' options invocation located inside the subtype field, and +## the only way to use field 'length' from the outer type without affecting +## inner string 'subtype.length' is the double curly braces notation. +## ## Usage of int, float, uint types without size specifier ## ------------------------------------------------------ ## By default the serializable macro throws an error when the type From 7382892843b2da2f4be1319b448722f215af78f2 Mon Sep 17 00:00:00 2001 From: xomachine Date: Thu, 25 May 2017 19:04:12 +0300 Subject: [PATCH 25/67] Added double curlies test --- tests/customseqsize.nim | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/customseqsize.nim b/tests/customseqsize.nim index 2241f56..5d73f30 100644 --- a/tests/customseqsize.nim +++ b/tests/customseqsize.nim @@ -124,3 +124,24 @@ suite "Custom periodic size": require(o.a.data.len == dso.a.data.len) for i in 0.. Date: Thu, 25 May 2017 19:05:26 +0300 Subject: [PATCH 26/67] Improved the service keywords remover to hanle keywords inside the tuple[] --- nesm.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nesm.nim b/nesm.nim index 7959001..3f869cc 100644 --- a/nesm.nim +++ b/nesm.nim @@ -167,6 +167,10 @@ proc cleanupTypeDeclaration(declaration: NimNode): NimNode = elif c.len == 3 and c[0] == settingsKeyword and c[1].kind == nnkTableConstr: continue + elif c[last].kind == nnkTupleTy: + var newID = c + newID[last] = cleanupTypeDeclaration(c[last]) + children.add(newID) else: children.add(c) else: From f855d4c63b6c997bc14352b2da5f41b3d09f9d2c Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 14 Oct 2017 16:01:55 +0300 Subject: [PATCH 27/67] Improved error message when unexpected AST encountered --- nesm/generator.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nesm/generator.nim b/nesm/generator.nim index 3de24bd..359ccaf 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -212,8 +212,7 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = `deserialization` cast[type(`r`)](`tmp`) else: - discard - error("Unexpected AST: " & thetype.treeRepr) + error("Unexpected AST: " & thetype.treeRepr & "\n at " & thetype.lineinfo()) result.dynamic = not context.is_static From 1f9760ba2018c67e4a039e24d6c9805b0ee06e60 Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 14 Oct 2017 16:18:21 +0300 Subject: [PATCH 28/67] Added test for 'else: discard' in one line issue --- tests/dynamic.nim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/dynamic.nim b/tests/dynamic.nim index e218500..367c8e6 100644 --- a/tests/dynamic.nim +++ b/tests/dynamic.nim @@ -243,3 +243,13 @@ suite "Dynamic structure tests": check(d.d == o.d) check(d.s == o.s) check(o.size() == 9) + + test "else: discard": + serializable: + type ED = object + case a: uint8 + else: discard + let rnw = get_random_reader_n_writer() + let o = ED.deserialize(rnw) + rnw.setPosition(0) + o.serialize(rnw) From 0a27d68c4efbbcd71192ffda7bc92b3fd251d4d6 Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 14 Oct 2017 16:54:54 +0300 Subject: [PATCH 29/67] Fixed 'else: discard' in one line test --- nesm/generator.nim | 2 ++ nesm/objects.nim | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/nesm/generator.nim b/nesm/generator.nim index 359ccaf..d543124 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -211,6 +211,8 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = var `tmp` = cast[`basetype`](`source`) `deserialization` cast[type(`r`)](`tmp`) + of nnkNilLit: + result = context.genObject(newTree(nnkRecList, thetype)) else: error("Unexpected AST: " & thetype.treeRepr & "\n at " & thetype.lineinfo()) result.dynamic = not context.is_static diff --git a/nesm/objects.nim b/nesm/objects.nim index cd16a82..f553d4b 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -162,8 +162,8 @@ proc genFields(context: Context, decl: NimNode): FieldChunk = proc genCase(context: Context, decl: NimNode): TypeChunk = let checkable = decl[0][0].basename let eachbranch = proc(b: NimNode): auto = - let conditions = toSeq(b.children) - .filterIt(it.kind != nnkRecList) + let children = toSeq(b.children) + let conditions = children[0..^2] let branch = context.genTypeChunk(b.last) let size = proc(source: NimNode):NimNode = let casebody = branch.size(source) From 85a3df6f96108ffce68e3e7ee1420b185668739d Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 14 Oct 2017 17:40:37 +0300 Subject: [PATCH 30/67] Added test for {sizeof: x} setting --- tests/sizeof.nim | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/sizeof.nim diff --git a/tests/sizeof.nim b/tests/sizeof.nim new file mode 100644 index 0000000..a6a197b --- /dev/null +++ b/tests/sizeof.nim @@ -0,0 +1,26 @@ +from nesm import serializable +from streams import setPosition +import unittest +import helpers.rnw + + +suite "{sizeof: x}": + test "Basic": + serializable: + type Basic = object + size: int32 {sizeof: {}.data} + a: int32 + data: seq[int32] + + let rnw = get_reader_n_writer() + var o: Basic + o.data = random_seq_with(random(20000).int32) + o.size = 0 + o.a = random(20000).int32 + o.serialize(rnw) + rnw.setPosition(0) + let dso = Basic.deserialize(rnw) + check(o.a == dso.a) + check(o.data == dso.data) + check(o.data.len == dso.size) + From 23d4af0d1ade7ad4b222cdfa498b42b459750370 Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 14 Oct 2017 18:26:07 +0300 Subject: [PATCH 31/67] Changed overrides structure to implement {sizeof: x} support --- nesm/generator.nim | 10 +++++----- nesm/objects.nim | 4 +++- nesm/typesinfo.nim | 9 +++++++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/nesm/generator.nim b/nesm/generator.nim index d543124..f90cbca 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -95,9 +95,9 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = elif thetype.repr == "string": if context.is_static: error("Strings are not allowed in static context") - assert(context.size_override.len in 0..1, "To many 'size' options") - if context.size_override.len > 0: - let capture = context.size_override[0] + assert(context.overrides.size.len in 0..1, "To many 'size' options") + if context.overrides.size.len > 0: + let capture = context.overrides.size[0] let size = capture.size let relative_depth = context.depth - capture.depth let len_proc = proc (s: NimNode): NimNode = @@ -151,9 +151,9 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = error("Dynamic types are not supported in static" & " structures") let elem = thetype[1] - if context.size_override.len > 0: + if context.overrides.size.len > 0: var subcontext = context - let capture = subcontext.size_override.pop() + let capture = subcontext.overrides.size.pop() let size = capture.size let relative_depth = context.depth - capture.depth let seqLen = proc (s: NimNode): NimNode = diff --git a/nesm/objects.nim b/nesm/objects.nim index f553d4b..684b691 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -58,7 +58,9 @@ proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = of 2: result.is_static = false else: error("Unexpected error! dynamic is in superposition! WTF?") of "size": - result.size_override.insert((option[1], context.depth), 0) + result.overrides.size.insert((option[1], context.depth), 0) + of "sizeof": + discard else: error("Unknown setting: " & key) diff --git a/nesm/typesinfo.nim b/nesm/typesinfo.nim index ec5498e..6d79577 100644 --- a/nesm/typesinfo.nim +++ b/nesm/typesinfo.nim @@ -24,15 +24,20 @@ type size: NimNode depth: Natural + Overrides* = tuple + size: seq[SizeCapture] + sizeof: seq[SizeCapture] + Context* = object declared*: Table[string, TypeChunk] - size_override*: seq[SizeCapture] + overrides*: Overrides depth*: Natural is_static*: bool swapEndian*: bool proc initContext*(): Context = - result.size_override = newSeq[SizeCapture]() + result.overrides.size = newSeq[SizeCapture]() + result.overrides.sizeof = newSeq[SizeCapture]() result.declared = initTable[string, TypeChunk]() result.depth = 0 result.is_static = false From 3c604dcd555c0acd45685e2db7a53ba05202b2ba Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 14 Oct 2017 22:11:19 +0300 Subject: [PATCH 32/67] Changed syntax of settings in test files --- tests/customseqsize.nim | 12 ++++++------ tests/endiantest.nim | 2 +- tests/sizeof.nim | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/customseqsize.nim b/tests/customseqsize.nim index 5d73f30..566481c 100644 --- a/tests/customseqsize.nim +++ b/tests/customseqsize.nim @@ -12,7 +12,7 @@ suite "Custom periodic size": MySeq = object size: int32 a: int32 - data: seq[int32] {size: {}.size} + data: seq[int32] as {size: {}.size} let rnw = get_reader_n_writer() var o: MySeq @@ -35,7 +35,7 @@ suite "Custom periodic size": MyString = object size: int32 a: int32 - data: string {size: {}.size} + data: string as {size: {}.size} let rnw = get_reader_n_writer() var o: MyString @@ -57,7 +57,7 @@ suite "Custom periodic size": type MySeqStr = object s: int32 a: int32 - data: seq[string] {size: {}.s} + data: seq[string] as {size: {}.s} let rnw = get_reader_n_writer() var o: MySeqStr @@ -79,7 +79,7 @@ suite "Custom periodic size": type Matrix = object lines: int32 columns: int32 - data: seq[seq[int32]] {size: {}.lines, size: {}.columns} + data: seq[seq[int32]] as {size: {}.lines, size: {}.columns} let rnw = get_reader_n_writer() var o: Matrix @@ -106,7 +106,7 @@ suite "Custom periodic size": type Nested = object dsize: int32 a: int32 - data: seq[int32] {size: {}.dsize} + data: seq[int32] as {size: {}.dsize} type Nester = object a: Nested @@ -129,7 +129,7 @@ suite "Custom periodic size": serializable: type NestedTuple = object length: int32 - data: tuple[name: string, code: seq[int32] {size: {{}}.length}] + data: tuple[name: string, code: seq[int32] as {size: {{}}.length}] let rnw = get_reader_n_writer() var o: NestedTuple diff --git a/tests/endiantest.nim b/tests/endiantest.nim index e7574b6..e1d45a9 100644 --- a/tests/endiantest.nim +++ b/tests/endiantest.nim @@ -66,7 +66,7 @@ suite "Endianness support": type SomeInline = object set: {endian: bigEndian} od: int32 - sd: int32 {endian: littleEndian} + sd: int32 as {endian: littleEndian} od2: int32 let rnw = get_random_reader_n_writer() let dso = SomeInline.deserialize(rnw) diff --git a/tests/sizeof.nim b/tests/sizeof.nim index a6a197b..422436b 100644 --- a/tests/sizeof.nim +++ b/tests/sizeof.nim @@ -8,7 +8,7 @@ suite "{sizeof: x}": test "Basic": serializable: type Basic = object - size: int32 {sizeof: {}.data} + size: int32 as {sizeof: {}.data} a: int32 data: seq[int32] From 86193565787ba966ac688bbead141b9d2dc0c786 Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 14 Oct 2017 22:16:48 +0300 Subject: [PATCH 33/67] New settings syntax implementation --- nesm.nim | 11 ++++++----- nesm/objects.nim | 39 ++++----------------------------------- nesm/settings.nim | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 40 deletions(-) create mode 100644 nesm/settings.nim diff --git a/nesm.nim b/nesm.nim index 3f869cc..3c08819 100644 --- a/nesm.nim +++ b/nesm.nim @@ -10,7 +10,7 @@ from streams import Stream, newStringStream when not defined(nimdoc): from nesm.typesinfo import TypeChunk, Context, initContext from nesm.generator import genTypeChunk, STREAM_NAME - from nesm.objects import applyOptions + from nesm.settings import applyOptions, splitSettingsExpr else: import endians include nesm.documentation @@ -151,13 +151,14 @@ proc cleanupTypeDeclaration(declaration: NimNode): NimNode = children.add(cleanupTypeDeclaration(cc)) of nnkIdentDefs: let last = if c[^1].kind == nnkEmpty: c.len - 2 else: c.len - 1 - if c[last].kind == nnkCurlyExpr: + let (originalType, options) = c[last].splitSettingsExpr() + if options.len > 0: # newID need to be a NimNode that contains IdentDefs node # to utilze recursive call of cleanupTypeDeclaration - var newID = newTree(nnkStmtList, newNimNode(nnkIdentDefs)) - copyChildrenTo(c, newID[0]) + var newID = newTree(nnkStmtList, c) + #copyChildrenTo(c, newID[0]) # first element of CurlyExpr is an actual type - newID[0][last] = c[last][0] + newID[0][last] = originalType children.add(newID.cleanupTypeDeclaration()[0]) elif c[last].repr == "cstring": var newID = newNimNode(nnkIdentDefs) diff --git a/nesm/objects.nim b/nesm/objects.nim index 1623247..86e4622 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -1,7 +1,7 @@ from sequtils import toSeq, filterIt, mapIt -from strutils import cmpIgnoreStyle from nesm.generator import genTypeChunk, correct_sum from nesm.typesinfo import TypeChunk, Context +from nesm.settings import applyOptions, splitSettingsExpr import macros type @@ -13,13 +13,12 @@ type has_hidden: bool proc genObject*(context: Context, thetype: NimNode): TypeChunk {.compileTime.} -proc applyOptions*(context: Context, - options: NimNode | seq[NimNode]): Context {.compileTime.} proc genCase(context: Context, decl: NimNode): TypeChunk {.compileTime.} proc caseWorkaround(tc: TypeChunk): TypeChunk {.compileTime.} proc evalSize(e: NimNode): BiggestInt {.compileTime.} proc genFields(context: Context, decl: NimNode): FieldChunk {.compileTime.} + proc caseWorkaround(tc: TypeChunk): TypeChunk = # st - type of field under case result = tc @@ -38,32 +37,6 @@ proc caseWorkaround(tc: TypeChunk): TypeChunk = `ods` `s` = `tmpvar` -proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = - result = context - for option in options.items(): - option.expectKind(nnkExprColonExpr) - option.expectMinLen(2) - let key = option[0].repr - let val = option[1].repr - case key - of "endian": - result.swapEndian = cmpIgnoreStyle(val, $cpuEndian) != 0 - of "dynamic": - let code = int(cmpIgnoreStyle(val, "true") == 0) + - 2*int(cmpIgnoreStyle(val, "false") == 0) - case code - of 0: error("The dynamic property can be only 'true' or 'false' but not" & - val) - of 1: result.is_static = true - of 2: result.is_static = false - else: error("Unexpected error! dynamic is in superposition! WTF?") - of "size": - result.overrides.size.insert((option[1], context.depth), 0) - of "sizeof": - discard - else: - error("Unknown setting: " & key) - proc genObject(context: Context, thetype: NimNode): TypeChunk = var elems = newSeq[Field]() var newContext = context @@ -149,18 +122,14 @@ proc genFields(context: Context, decl: NimNode): FieldChunk = if decl[^1].kind == nnkEmpty: decl.len - 2 else: decl.len - 1 let subtype = decl[last] - let chunk = - if subtype.kind == nnkCurlyExpr: - context.applyOptions(toSeq(subtype.children)[1.. Date: Sat, 14 Oct 2017 22:19:11 +0300 Subject: [PATCH 34/67] New syntax now reflected in the docs --- nesm/documentation.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nesm/documentation.nim b/nesm/documentation.nim index df02f2f..27a1665 100644 --- a/nesm/documentation.nim +++ b/nesm/documentation.nim @@ -201,7 +201,7 @@ ## serializable: ## type MyType = object ## typical_field: string -## field_with_special_rules: int32 {} # inline options +## field_with_special_rules: int32 as {} # inline options ## # NOTE: inline options have highest priority ## another_typical_field: float32 ## @@ -287,7 +287,7 @@ ## type Matrix = object ## lines: uint32 # the size specifier should be placed before it's usage in the 'size' option ## columns: uint32 -## data: seq[seq[int32]] {size: {}.lines, size: {}.columns} +## data: seq[seq[int32]] as {size: {}.lines, size: {}.columns} ## # The seq's will be stored like array's but their sizes will be ## # taken from 'lines' and 'columns' fields during deserialization ## @@ -301,7 +301,7 @@ ## serializable: ## type SpecialType = object ## length: uint32 # <- this field will be used as length of subtype.a -## subtype: tuple[length: string, a: seq[int32] {size: {{}}.length}] +## subtype: tuple[length: string, a: seq[int32] as {size: {{}}.length}] ## ## The 'size' options invocation located inside the subtype field, and ## the only way to use field 'length' from the outer type without affecting From 85f17315d7fa02f03458eaf6540523267bcf9120 Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 14 Oct 2017 22:33:22 +0300 Subject: [PATCH 35/67] Added Travis CI manifest for testing --- .travis.yml | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c2b2a4a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,52 @@ +language: c +env: + # Build and test against the master and devel branches of Nim + - BRANCH=master + - BRANCH=devel +compiler: + # Build and test using both gcc and clang + - gcc + - clang +matrix: + allow_failures: + # Ignore failures when building against the devel Nim branch + - env: BRANCH=devel + fast_finish: true +install: + - | + if [ ! -x nim-$BRANCH/bin/nim ]; then + git clone -b $BRANCH --depth 1 git://github.com/nim-lang/nim nim-$BRANCH/ + cd nim-$BRANCH + git clone --depth 1 git://github.com/nim-lang/csources csources/ + cd csources + sh build.sh + cd .. + rm -rf csources + bin/nim c koch + ./koch boot -d:release + ./koch nimble + else + cd nim-$BRANCH + git fetch origin + if ! git merge FETCH_HEAD | grep "Already up-to-date"; then + bin/nim c koch + ./koch boot -d:release + ./koch nimble + fi + fi + cd .. +before_script: + - export PATH="nim-$BRANCH/bin${PATH:+:$PATH}" +script: + - nimble tests + # Replace uppercase strings! + #- nim c --cc:$CC --verbosity:0 -r MYFILE.nim + # Optional: build docs. + #- nim doc --docSeeSrcUrl:https://github.com/AUTHOR/MYPROJECT/blob/master --project MYFILE.nim +cache: + directories: + - nim-master + - nim-devel +branches: + except: + - gh-pages \ No newline at end of file From 2bfdc78313a88f17ef954227328f58be9aea3230 Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 14 Oct 2017 23:05:03 +0300 Subject: [PATCH 36/67] Merge backward compatibility commit from 0.17.3-devel-fix branch --- nesm/generator.nim | 11 ++++++++--- nesm/objects.nim | 10 +++++----- nesm/periodic.nim | 18 +++++++++--------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/nesm/generator.nim b/nesm/generator.nim index a8090ed..c317b79 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -9,6 +9,11 @@ from sequtils import mapIt, foldl, toSeq, filterIt proc genTypeChunk*(immutableContext: Context, thetype: NimNode): TypeChunk {.compileTime.} proc correct_sum*(part_size: NimNode): NimNode {.compileTime.} +proc unfold*(node: NimNode): NimNode = + if node.kind == nnkStmtList and node.len == 1: + node.last + else: + node static: let STREAM_NAME* = !"thestream" @@ -112,7 +117,7 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = `deser` else: let len_proc = proc (s: NimNode): NimNode = - (quote do: len(`s`)) + (quote do: len(`s`)).unfold() result = context.genPeriodic(newEmptyNode(), len_proc) elif thetype.repr == "cstring": if context.is_static: @@ -122,7 +127,7 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = result.deserialize = proc (s: NimNode): NimNode = genCStringDeserialize(s) result.size = proc (s: NimNode): NimNode = - (quote do: len(`s`) + 1) + (quote do: len(`s`) + 1).unfold() else: error(("Type $1 is not a basic " % plaintype) & "type nor a complex type under 'serializable'" & @@ -168,7 +173,7 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = `deser` else: let seqLen = proc (source: NimNode): NimNode = - (quote do: len(`source`)) + (quote do: len(`source`)).unfold() result = context.genPeriodic(elem, seqLen) of "set": result = context.genSet(thetype) diff --git a/nesm/objects.nim b/nesm/objects.nim index 86e4622..e1d3499 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -1,5 +1,5 @@ from sequtils import toSeq, filterIt, mapIt -from nesm.generator import genTypeChunk, correct_sum +from nesm.generator import genTypeChunk, correct_sum, unfold from nesm.typesinfo import TypeChunk, Context from nesm.settings import applyOptions, splitSettingsExpr import macros @@ -80,7 +80,7 @@ proc genObject(context: Context, thetype: NimNode): TypeChunk = for i in elems.items(): let n = !i.name let newsource = - if ($n).len > 0: (quote do: `source`.`n`) + if ($n).len > 0: (quote do: `source`.`n`).unfold() else: source let e = i.chunk let part_size = e.size(newsource) @@ -99,7 +99,7 @@ proc genObject(context: Context, thetype: NimNode): TypeChunk = for i in elems.items(): let n = !i.name let newsource = - if ($n).len > 0: (quote do: `source`.`n`) + if ($n).len > 0: (quote do: `source`.`n`).unfold() else: source let e = i.chunk result.add(e.serialize(newsource)) @@ -108,7 +108,7 @@ proc genObject(context: Context, thetype: NimNode): TypeChunk = for i in elems.items(): let n = !i.name let newsource = - if ($n).len > 0: (quote do: `source`.`n`) + if ($n).len > 0: (quote do: `source`.`n`).unfold() else: source let e = i.chunk result &= e.deserialize(newsource) @@ -154,7 +154,7 @@ proc genCase(context: Context, decl: NimNode): TypeChunk = let deserializes = branches.mapIt(it[2]) result.dynamic = true let condition = proc (source: NimNode): NimNode = - (quote do: `source`.`checkable`) + (quote do: `source`.`checkable`).unfold() result.size = proc(source: NimNode):NimNode = let sizenodes:seq[NimNode] = sizes.mapIt(it(source)) if context.is_static: diff --git a/nesm/periodic.nim b/nesm/periodic.nim index 20bad08..1876ce6 100644 --- a/nesm/periodic.nim +++ b/nesm/periodic.nim @@ -1,6 +1,6 @@ from nesm.typesinfo import Context, TypeChunk, estimateBasicSize, isBasic from nesm.basics import genSerialize, genDeserialize -from nesm.generator import genTypeChunk, STREAM_NAME, correct_sum +from nesm.generator import genTypeChunk, STREAM_NAME, correct_sum, unfold from streams import writeData, write, readChar import macros @@ -50,14 +50,14 @@ proc genPeriodic*(context: Context, elem: NimNode, arraylen.infix("*", newIntLitNode(elemSize)) result.serialize = proc (s: NimNode): NimNode = let lens = length(s) - let size = (quote do: `lens` * `eSize`) - let newsource = (quote do: `s`[0]) + let size = (quote do: `lens` * `eSize`).unfold() + let newsource = (quote do: `s`[0]).unfold() let serialization = genSerialize(newsource, size) quote do: if `lens` > 0: `serialization` result.deserialize = proc (s: NimNode): NimNode = let lens = length(s) - let size = (quote do: `lens` * `eSize`) + let size = (quote do: `lens` * `eSize`).unfold() let newsource = (quote do: `s`[0]) let deserialization = genDeserialize(newsource, size) quote do: @@ -70,7 +70,7 @@ proc genPeriodic*(context: Context, elem: NimNode, let index_letter = nskForVar.genSym("index") result.size = proc (s: NimNode): NimNode = let periodic_len = length(s) - let newsource = (quote do: `s`[`index_letter`]) + let newsource = (quote do: `s`[`index_letter`]).unfold() let chunk_size = one_chunk.size(newsource) if not chunk_size.findInChilds(index_letter): periodic_len.infix("*", chunk_size) @@ -81,14 +81,14 @@ proc genPeriodic*(context: Context, elem: NimNode, `chunk_expr` result.serialize = proc(s: NimNode): NimNode = let periodic_len = length(s) - let newsource = (quote do: `s`[`index_letter`]) + let newsource = (quote do: `s`[`index_letter`]).unfold() let chunk_expr = onechunk.serialize(newsource) quote do: for `index_letter` in 0..<(`periodic_len`): `chunk_expr` result.deserialize = proc(s: NimNode): NimNode = let lens = length(s) - let newsource = (quote do: `s`[`index_letter`]) + let newsource = (quote do: `s`[`index_letter`]).unfold() let chunk_expr = onechunk.deserialize(newsource) quote do: for `index_letter` in 0..<(`lens`): @@ -117,8 +117,8 @@ proc genPeriodic*(context: Context, elem: NimNode, `pr_ser` result.deserialize = proc(s:NimNode): NimNode = let init_template = - if elem.kind == nnkEmpty: (quote do: newString) - else: (quote do: newSeq[`elem`]) + if elem.kind == nnkEmpty: (quote do: newString).unfold() + else: (quote do: newSeq[`elem`]).unfold() let sd = size_header_chunk.deserialize(lenvarname) let deserialization = preresult.deserialize(s) quote do: From 814ba554797d7054e312e0092e177537d14c05f6 Mon Sep 17 00:00:00 2001 From: xomachine Date: Sun, 15 Oct 2017 00:47:26 +0300 Subject: [PATCH 37/67] Initial sizeof option implementation --- nesm/generator.nim | 18 ++++++++++++++++++ nesm/settings.nim | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/nesm/generator.nim b/nesm/generator.nim index c317b79..9db920a 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -82,6 +82,24 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = if plaintype.isBasic(): let size = estimateBasicSize(plaintype) result = context.genBasic(size) + case context.overrides.sizeof.len: + of 0: discard + of 1: + assert(plaintype[0..2] in ["uin", "int"], + "The sizeof the seq must be an integer type!") + let capture = context.overrides.sizeof[0] + let relative_depth = context.depth - capture.depth + let tmpsym = nskLet.genSym("tmp") + let preser = result.serialize(tmpsym) + let undertype = newIdentNode("uint" & + $(result.size(newEmptyNode()).intVal * 8)) + result.serialize = proc(source: NimNode): NimNode = + let origin = capture.size.insert_source(source, relative_depth) + quote do: + let `tmpsym`: `undertype` = `origin`.len.`undertype` + `preser` + else: + error("It is impossible to use more than one sizeof options at once!") elif plaintype in context.declared: let declared_type = context.declared[plaintype] if declared_type.dynamic and context.is_static: diff --git a/nesm/settings.nim b/nesm/settings.nim index 7eff7fb..6bd60e1 100644 --- a/nesm/settings.nim +++ b/nesm/settings.nim @@ -32,7 +32,7 @@ proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = of "size": result.overrides.size.insert((option[1], context.depth), 0) of "sizeof": - discard + result.overrides.sizeof.add((option[1], context.depth)) else: error("Unknown setting: " & key) From 2a3784891cd14deff0e7c5aa9bcb1c044565cc4f Mon Sep 17 00:00:00 2001 From: xomachine Date: Sun, 15 Oct 2017 02:16:13 +0300 Subject: [PATCH 38/67] Added test including both of size and sizeof options --- tests/sizeof.nim | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/sizeof.nim b/tests/sizeof.nim index 422436b..11ee8c0 100644 --- a/tests/sizeof.nim +++ b/tests/sizeof.nim @@ -24,3 +24,21 @@ suite "{sizeof: x}": check(o.data == dso.data) check(o.data.len == dso.size) + test "CrossRef": + serializable: + type CR = object + size: int32 as {sizeof: {}.data} + someval: int64 + data: seq[int32] as {size: {}.size} + let rnw = get_reader_n_writer() + var o: CR + o.data = random_seq_with(random(20000).int32) + o.size = 0 + o.someval = random(20000).int64 + o.serialize(rnw) + rnw.setPosition(0) + let dso = CR.deserialize(rnw) + check(size(o) == size(dso)) + check(o.someval == dso.someval) + check(o.data == dso.data) + check(o.data.len == dso.size) From 10bab0f2443c14881bcd2b2a728dd1dcf4470f5c Mon Sep 17 00:00:00 2001 From: xomachine Date: Sun, 15 Oct 2017 02:18:27 +0300 Subject: [PATCH 39/67] Changes to pass CrossRef test Unfortunately it is necessary to make a mutable copy of serializable object --- nesm.nim | 22 ++++++++++++++++------ nesm/generator.nim | 22 ++++++++++++++++------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/nesm.nim b/nesm.nim index 3c08819..53c7eb3 100644 --- a/nesm.nim +++ b/nesm.nim @@ -24,9 +24,17 @@ const DESERIALIZE_DECLARATION = """proc deserialize$1""" & """: seq[byte | char | int8 | uint8] | string):""" & """$2 = discard""" +proc wrapImmutableBody(immutable: string, + action: proc(s:NimNode):NimNode): NimNode = + let mutable_obj = nskVar.genSym(immutable) + let immutable_obj = newIdentNode(immutable) + let body = action(mutable_obj) + quote do: + var `mutable_obj` = `immutable_obj` + `body` + proc makeSerializeStreamDeclaration(typename: string, - is_exported: bool, - body: NimNode): NimNode {.compileTime.} = + is_exported: bool, body: NimNode): NimNode {.compileTime.} = let itn = !typename let fname = if is_exported: newIdentNode("serialize").postfix("*") @@ -34,7 +42,8 @@ proc makeSerializeStreamDeclaration(typename: string, let isin = !SERIALIZER_INPUT_NAME quote do: proc `fname`(`isin`: `itn`, - `STREAM_NAME`: Stream) = `body` + `STREAM_NAME`: Stream) = + `body` proc makeDeserializeStreamDeclaration(typename: string, is_exported: bool, @@ -93,8 +102,6 @@ proc generateProcs(context: var Context, else: "" let body = obj[2] let info = context.genTypeChunk(body) - let size_node = - info.size(newIdentNode(SERIALIZER_INPUT_NAME)) context.declared[name] = info let writer_conversion = makeSerializeStreamConversion() let serializer = generateProc(SERIALIZE_DECLARATION, @@ -102,7 +109,7 @@ proc generateProcs(context: var Context, writer_conversion) let serialize_stream = makeSerializeStreamDeclaration(name, is_shared, - info.serialize(newIdentNode(SERIALIZER_INPUT_NAME))) + wrapImmutableBody(SERIALIZER_INPUT_NAME, info.serialize)) let obtainer_conversion = if context.is_static: makeDeserializeStreamConversion("result") @@ -118,6 +125,9 @@ proc generateProcs(context: var Context, let size_declaration = if context.is_static: STATIC_SIZE_DECLARATION else: SIZE_DECLARATION + let size_node = + if context.is_static: info.size(newIdentNode(SERIALIZER_INPUT_NAME)) + else: wrapImmutableBody(SERIALIZER_INPUT_NAME, info.size) let sizeProc = generateProc(size_declaration, name, sign, size_node) newStmtList(sizeProc, serialize_stream, serializer, diff --git a/nesm/generator.nim b/nesm/generator.nim index 9db920a..cba3db8 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -85,18 +85,28 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = case context.overrides.sizeof.len: of 0: discard of 1: + assert(not context.is_static, + "Sizeof option is not allowed in the static context!") assert(plaintype[0..2] in ["uin", "int"], - "The sizeof the seq must be an integer type!") + "The sizeof field must be an integer type!") + let prev_serialize = result.serialize let capture = context.overrides.sizeof[0] let relative_depth = context.depth - capture.depth - let tmpsym = nskLet.genSym("tmp") - let preser = result.serialize(tmpsym) - let undertype = newIdentNode("uint" & - $(result.size(newEmptyNode()).intVal * 8)) + let prev_size = result.size(newEmptyNode()).intVal + let undertype = newIdentNode("int" & + $(prev_size * 8)) + result.size = proc(source: NimNode): NimNode = + let origin = capture.size.insert_source(source, relative_depth) + let presize = newIntLitNode(prev_size) + let resultnode = !"result" + quote do: + `source` = `origin`.len.`undertype` + `resultnode` += `presize` result.serialize = proc(source: NimNode): NimNode = let origin = capture.size.insert_source(source, relative_depth) + let preser = prev_serialize(source) quote do: - let `tmpsym`: `undertype` = `origin`.len.`undertype` + `source` = `origin`.len.`undertype` `preser` else: error("It is impossible to use more than one sizeof options at once!") From 0673269f1db62143c961741f44a1d362df58e9b3 Mon Sep 17 00:00:00 2001 From: xomachine Date: Sun, 15 Oct 2017 02:25:33 +0300 Subject: [PATCH 40/67] Added documentation remark about sizeof option --- nesm/documentation.nim | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/nesm/documentation.nim b/nesm/documentation.nim index 27a1665..02c0d1e 100644 --- a/nesm/documentation.nim +++ b/nesm/documentation.nim @@ -307,6 +307,17 @@ ## the only way to use field 'length' from the outer type without affecting ## inner string 'subtype.length' is the double curly braces notation. ## +## In oposite to `size` option there is `sizeof` option with can be used to +## set particular fields value to length of other periodic value instead of +## actual one during serialization. The previous example can be rewriten in +## following way to utilize the `sizeof` option: +## +## .. code-block:: nim +## serializable: +## type SpecialType = object +## length: uint32 as {sizeof: {}.subtype.a} +## subtype: tuple[length: string, a: seq[int32] as {size: {{}}.length}] +## ## Usage of int, float, uint types without size specifier ## ------------------------------------------------------ ## By default the serializable macro throws an error when the type From b7a7ae2e4de2ee4ca59eaaee8722af6a86cdaaf6 Mon Sep 17 00:00:00 2001 From: xomachine Date: Wed, 18 Oct 2017 01:13:08 +0300 Subject: [PATCH 41/67] unfold compatibility proc now moved to utils.nim --- nesm/generator.nim | 6 +----- nesm/objects.nim | 3 ++- nesm/periodic.nim | 3 ++- nesm/utils.nim | 8 ++++++++ 4 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 nesm/utils.nim diff --git a/nesm/generator.nim b/nesm/generator.nim index cba3db8..f8fd3c8 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -1,6 +1,7 @@ import macros from nesm.typesinfo import isBasic, estimateBasicSize from nesm.typesinfo import TypeChunk, Context +from nesm.utils import unfold from tables import Table, contains, `[]`, `[]=`, initTable, pairs from strutils import `%` @@ -9,11 +10,6 @@ from sequtils import mapIt, foldl, toSeq, filterIt proc genTypeChunk*(immutableContext: Context, thetype: NimNode): TypeChunk {.compileTime.} proc correct_sum*(part_size: NimNode): NimNode {.compileTime.} -proc unfold*(node: NimNode): NimNode = - if node.kind == nnkStmtList and node.len == 1: - node.last - else: - node static: let STREAM_NAME* = !"thestream" diff --git a/nesm/objects.nim b/nesm/objects.nim index e1d3499..189be02 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -1,5 +1,6 @@ from sequtils import toSeq, filterIt, mapIt -from nesm.generator import genTypeChunk, correct_sum, unfold +from nesm.generator import genTypeChunk, correct_sum +from nesm.utils import unfold from nesm.typesinfo import TypeChunk, Context from nesm.settings import applyOptions, splitSettingsExpr import macros diff --git a/nesm/periodic.nim b/nesm/periodic.nim index 1876ce6..2a00a0a 100644 --- a/nesm/periodic.nim +++ b/nesm/periodic.nim @@ -1,6 +1,7 @@ from nesm.typesinfo import Context, TypeChunk, estimateBasicSize, isBasic from nesm.basics import genSerialize, genDeserialize -from nesm.generator import genTypeChunk, STREAM_NAME, correct_sum, unfold +from nesm.generator import genTypeChunk, STREAM_NAME, correct_sum +from nesm.utils import unfold from streams import writeData, write, readChar import macros diff --git a/nesm/utils.nim b/nesm/utils.nim new file mode 100644 index 0000000..7c91b20 --- /dev/null +++ b/nesm/utils.nim @@ -0,0 +1,8 @@ +import macros + +proc unfold*(node: NimNode): NimNode {.compileTime.} + + +proc unfold(node: NimNode): NimNode = + if node.kind == nnkStmtList and node.len == 1: node.last + else: node From 04710414c27e16fa4dcbb7d9c133e1f73d814f7f Mon Sep 17 00:00:00 2001 From: xomachine Date: Wed, 18 Oct 2017 01:14:18 +0300 Subject: [PATCH 42/67] Serialization procs body generators now have their own named type: BodyGenerator --- nesm/typesinfo.nim | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nesm/typesinfo.nim b/nesm/typesinfo.nim index 6d79577..4fd6bed 100644 --- a/nesm/typesinfo.nim +++ b/nesm/typesinfo.nim @@ -9,10 +9,11 @@ const basic_types = [ ] type + BodyGenerator* = proc (source: NimNode): NimNode TypeChunk* = object - size*: proc(source: NimNode): NimNode - serialize*: proc(source: NimNode): NimNode - deserialize*: proc(source: NimNode): NimNode + size*: BodyGenerator + serialize*: BodyGenerator + deserialize*: BodyGenerator dynamic*: bool has_hidden*: bool case nodekind*: NimNodeKind From cf8b295fb53b93673b90470baa7feb736253d0e9 Mon Sep 17 00:00:00 2001 From: xomachine Date: Wed, 18 Oct 2017 03:31:33 +0300 Subject: [PATCH 43/67] size option now affects only deserialization behaviour --- nesm/generator.nim | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/nesm/generator.nim b/nesm/generator.nim index f8fd3c8..5329e8b 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -88,21 +88,12 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = let prev_serialize = result.serialize let capture = context.overrides.sizeof[0] let relative_depth = context.depth - capture.depth - let prev_size = result.size(newEmptyNode()).intVal - let undertype = newIdentNode("int" & - $(prev_size * 8)) - result.size = proc(source: NimNode): NimNode = - let origin = capture.size.insert_source(source, relative_depth) - let presize = newIntLitNode(prev_size) - let resultnode = !"result" - quote do: - `source` = `origin`.len.`undertype` - `resultnode` += `presize` result.serialize = proc(source: NimNode): NimNode = let origin = capture.size.insert_source(source, relative_depth) - let preser = prev_serialize(source) + let tmpvar = nskLet.genSym("seqLen") + let preser = prev_serialize(tmpvar) quote do: - `source` = `origin`.len.`undertype` + let `tmpvar` = cast[`thetype`](`origin`.len) `preser` else: error("It is impossible to use more than one sizeof options at once!") @@ -130,7 +121,7 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = let size = capture.size let relative_depth = context.depth - capture.depth let len_proc = proc (s: NimNode): NimNode = - size.insert_source(s, relative_depth) + (quote do: `s`.len()).unfold() result = context.genPeriodic(newEmptyNode(), len_proc) let olddeser = result.deserialize result.deserialize = proc (s: NimNode): NimNode = @@ -186,7 +177,7 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = let size = capture.size let relative_depth = context.depth - capture.depth let seqLen = proc (s: NimNode): NimNode = - size.insert_source(s, relative_depth) + (quote do: `s`.len()).unfold() result = subcontext.genPeriodic(elem, seqLen) let olddeser = result.deserialize result.deserialize = proc (s: NimNode): NimNode = From f106af8e07c82843ffe57c591aa16276abd96494 Mon Sep 17 00:00:00 2001 From: xomachine Date: Wed, 18 Oct 2017 03:35:24 +0300 Subject: [PATCH 44/67] A little refactoring of the procs generating code --- nesm.nim | 117 +---------------------------------------------- nesm/procgen.nim | 101 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 116 deletions(-) create mode 100644 nesm/procgen.nim diff --git a/nesm.nim b/nesm.nim index 53c7eb3..dca24c4 100644 --- a/nesm.nim +++ b/nesm.nim @@ -9,130 +9,15 @@ from streams import Stream, newStringStream when not defined(nimdoc): from nesm.typesinfo import TypeChunk, Context, initContext - from nesm.generator import genTypeChunk, STREAM_NAME + from nesm.procgen import generateProcs from nesm.settings import applyOptions, splitSettingsExpr else: import endians include nesm.documentation -const SERIALIZER_INPUT_NAME = "obj" -const DESERIALIZER_DATA_NAME = "data" -const SERIALIZE_DECLARATION = """proc serialize$1(""" & - SERIALIZER_INPUT_NAME & """: $2): string = discard""" -const DESERIALIZE_DECLARATION = """proc deserialize$1""" & - """(thetype: typedesc[$2], """ & DESERIALIZER_DATA_NAME & - """: seq[byte | char | int8 | uint8] | string):""" & - """$2 = discard""" - -proc wrapImmutableBody(immutable: string, - action: proc(s:NimNode):NimNode): NimNode = - let mutable_obj = nskVar.genSym(immutable) - let immutable_obj = newIdentNode(immutable) - let body = action(mutable_obj) - quote do: - var `mutable_obj` = `immutable_obj` - `body` - -proc makeSerializeStreamDeclaration(typename: string, - is_exported: bool, body: NimNode): NimNode {.compileTime.} = - let itn = !typename - let fname = - if is_exported: newIdentNode("serialize").postfix("*") - else: newIdentNode("serialize") - let isin = !SERIALIZER_INPUT_NAME - quote do: - proc `fname`(`isin`: `itn`, - `STREAM_NAME`: Stream) = - `body` - -proc makeDeserializeStreamDeclaration(typename: string, - is_exported: bool, - body: NimNode): NimNode {.compileTime.} = - let itn = !typename - let fname = - if is_exported: - newIdentNode("deserialize").postfix("*") - else: newIdentNode("deserialize") - quote do: - proc `fname`(thetype: typedesc[`itn`], - `STREAM_NAME`: Stream): `itn` = `body` - -proc makeSerializeStreamConversion(): NimNode {.compileTime.} = - let isin = !SERIALIZER_INPUT_NAME - quote do: - let ss = newStringStream() - serialize(`isin`, ss) - ss.data - -proc makeDeserializeStreamConversion(name: string): NimNode {.compileTime.} = - let iname = !name - let ddn = !DESERIALIZER_DATA_NAME - quote do: - assert(`ddn`.len >= type(`iname`).size(), - "Given sequence should contain at least " & - $(type(`iname`).size()) & " bytes!") - let ss = newStringStream(cast[string](`ddn`)) - deserialize(type(`iname`), ss) - -const STATIC_SIZE_DECLARATION = - """proc size$1(thetype: typedesc[$2]): int = discard""" -const SIZE_DECLARATION = "proc size$1(" & - SERIALIZER_INPUT_NAME & - ": $2): int = discard" static: var ctx = initContext() -proc generateProc(pattern: string, name: string, - sign: string, - body: NimNode = newEmptyNode()): NimNode = - result = parseExpr(pattern % [sign, name]) - if body.kind != nnkEmpty: - result.body = body - -proc generateProcs(context: var Context, - obj: NimNode): NimNode {.compileTime.} = - expectKind(obj, nnkTypeDef) - expectMinLen(obj, 3) - expectKind(obj[1], nnkEmpty) - let typename = if obj[0].kind == nnkPragmaExpr: obj[0][0] else: obj[0] - let is_shared = typename.kind == nnkPostfix - let name = if is_shared: $typename.basename else: $typename - let sign = - if is_shared: "*" - else: "" - let body = obj[2] - let info = context.genTypeChunk(body) - context.declared[name] = info - let writer_conversion = makeSerializeStreamConversion() - let serializer = generateProc(SERIALIZE_DECLARATION, - name, sign, - writer_conversion) - let serialize_stream = - makeSerializeStreamDeclaration(name, is_shared, - wrapImmutableBody(SERIALIZER_INPUT_NAME, info.serialize)) - let obtainer_conversion = - if context.is_static: - makeDeserializeStreamConversion("result") - else: newEmptyNode() - let deserializer = - if context.is_static: - generateProc(DESERIALIZE_DECLARATION, name, sign, - obtainer_conversion) - else: newEmptyNode() - let deserialize_stream = - makeDeserializeStreamDeclaration(name, is_shared, - info.deserialize(newIdentNode("result"))) - let size_declaration = - if context.is_static: STATIC_SIZE_DECLARATION - else: SIZE_DECLARATION - let size_node = - if context.is_static: info.size(newIdentNode(SERIALIZER_INPUT_NAME)) - else: wrapImmutableBody(SERIALIZER_INPUT_NAME, info.size) - let sizeProc = generateProc(size_declaration, name, sign, - size_node) - newStmtList(sizeProc, serialize_stream, serializer, - deserialize_stream, deserializer) - proc prepare(context: var Context, statements: NimNode ): NimNode {.compileTime.} = result = newStmtList() diff --git a/nesm/procgen.nim b/nesm/procgen.nim new file mode 100644 index 0000000..4e6493f --- /dev/null +++ b/nesm/procgen.nim @@ -0,0 +1,101 @@ +from typesinfo import BodyGenerator, Context +from streams import newStringStream, Stream +from tables import `[]=` +from nesm.utils import unfold +from nesm.generator import STREAM_NAME, genTypeChunk +import macros +type + ProcType = enum + serialize + deserialize + size + +proc makeDeclaration(typenode: NimNode, kind: ProcType, is_exported: bool, + is_static: bool, bodygen: BodyGenerator): NimNode {.compileTime.} +proc makeSerializeStreamConversion(typenode: NimNode, + is_exported: bool): NimNode {.compileTime.} +proc makeDeserializeStreamConversion(typenode: NimNode, + is_exported: bool): NimNode {.compileTime.} +proc generateProcs*(context: var Context, obj: NimNode): NimNode {.compileTime.} + +proc makeDeclaration(typenode: NimNode, kind: ProcType, is_exported: bool, + is_static: bool, bodygen: BodyGenerator): NimNode = + ## Makes declaration of size, serialize or deserialize procs + let procname = + if is_exported: newIdentNode($kind).postfix("*") + else: newIdentNode($kind) + let bgsource = case kind + of deserialize: newIdentNode("result") + else: nskParam.genSym("obj") + let body = bodygen(bgsource) + case kind + of serialize: + quote do: + proc `procname`(`bgsource`: `typenode`, `STREAM_NAME`: Stream) = `body` + of deserialize: + quote do: + proc `procname`(a: typedesc[`typenode`], + `STREAM_NAME`: Stream): `typenode` = `body` + of size: + if is_static: + quote do: + proc `procname`(`bgsource`: typedesc[`typenode`]): Natural = `body` + else: + quote do: + proc `procname`(`bgsource`: `typenode`): Natural = `body` + +proc makeSerializeStreamConversion(typenode: NimNode, + is_exported: bool): NimNode = + ## Makes overloaded serialize proc which returns ``string`` as input instead + ## of writing data to the ``Stream`` + let rawprocname = newIdentNode("serialize") + let procname = if is_exported: rawprocname.postfix("*") else: rawprocname + quote do: + proc `procname`(obj: `typenode`): string = + let ss = newStringStream() + `rawprocname`(obj, ss) + ss.data + +proc makeDeserializeStreamConversion(typenode: NimNode, + is_exported: bool): NimNode = + ## Makes overloaded deserialize proc which takes ``string`` as input instead + ## of ``Stream`` + let rawprocname = newIdentNode("deserialize") + let sizeident = !("size") + let procname = if is_exported: rawprocname.postfix("*") else: rawprocname + quote do: + proc `procname`(a: typedesc[`typenode`], + data: string | seq[byte|char|uint8|int8]): auto = + assert(data.len >= `typenode`.`sizeident`(), + "Given sequence should contain at least " & + $(`typenode`.`sizeident`()) & " bytes!") + let ss = newStringStream(cast[string](data)) + `rawprocname`(`typenode`, ss) + +proc generateProcs(context: var Context, obj: NimNode): NimNode = + ## Generates ``NimNode``s for serialize, deserialize and size procs related + ## to given type declaration. + expectKind(obj, nnkTypeDef) + expectMinLen(obj, 3) + expectKind(obj[1], nnkEmpty) + let typedeclaration = if obj[0].kind == nnkPragmaExpr: obj[0][0] else: obj[0] + let is_shared = typedeclaration.kind == nnkPostfix + let typenode = if is_shared: typedeclaration.basename else: typedeclaration + let body = obj[2] + let typeinfo = genTypeChunk(context, body) + context.declared[$typenode] = typeinfo + let size_proc = makeDeclaration(typenode, size, is_shared, context.is_static, + typeinfo.size) + let stream_serializer = makeDeclaration(typenode, serialize, is_shared, + context.is_static, typeinfo.serialize) + let stream_deserializer = + makeDeclaration(typenode, deserialize, is_shared, context.is_static, + typeinfo.deserialize) + let string_serializer = makeSerializeStreamConversion(typenode, is_shared) + let string_deserializer = + if context.is_static: makeDeserializeStreamConversion(typenode ,is_shared) + else: newEmptyNode() + newStmtList(size_proc, stream_serializer, string_serializer, + stream_deserializer, string_deserializer) + + From 9537ddcefde900325b355fcf1524bc785d00ae52 Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 21 Oct 2017 18:43:48 +0300 Subject: [PATCH 45/67] Removed 'nesm.' prefix from local imports to stop nimsuggest complining --- nesm/basics.nim | 2 +- nesm/generator.nim | 6 +++--- nesm/objects.nim | 8 ++++---- nesm/periodic.nim | 8 ++++---- nesm/procgen.nim | 3 +-- nesm/sets.nim | 6 +++--- 6 files changed, 16 insertions(+), 17 deletions(-) diff --git a/nesm/basics.nim b/nesm/basics.nim index 6e97a50..ab434e9 100644 --- a/nesm/basics.nim +++ b/nesm/basics.nim @@ -1,4 +1,4 @@ -from nesm.typesinfo import Context, TypeChunk +from typesinfo import Context, TypeChunk from endians import swapEndian16, swapEndian32, swapEndian64 from streams import writeData, readData import macros diff --git a/nesm/generator.nim b/nesm/generator.nim index 5329e8b..8bd36e1 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -1,7 +1,7 @@ import macros -from nesm.typesinfo import isBasic, estimateBasicSize -from nesm.typesinfo import TypeChunk, Context -from nesm.utils import unfold +from typesinfo import isBasic, estimateBasicSize +from typesinfo import TypeChunk, Context +from utils import unfold from tables import Table, contains, `[]`, `[]=`, initTable, pairs from strutils import `%` diff --git a/nesm/objects.nim b/nesm/objects.nim index 189be02..9584ed1 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -1,8 +1,8 @@ from sequtils import toSeq, filterIt, mapIt -from nesm.generator import genTypeChunk, correct_sum -from nesm.utils import unfold -from nesm.typesinfo import TypeChunk, Context -from nesm.settings import applyOptions, splitSettingsExpr +from generator import genTypeChunk, correct_sum +from utils import unfold +from typesinfo import TypeChunk, Context +from settings import applyOptions, splitSettingsExpr import macros type diff --git a/nesm/periodic.nim b/nesm/periodic.nim index 2a00a0a..f0f3b8e 100644 --- a/nesm/periodic.nim +++ b/nesm/periodic.nim @@ -1,7 +1,7 @@ -from nesm.typesinfo import Context, TypeChunk, estimateBasicSize, isBasic -from nesm.basics import genSerialize, genDeserialize -from nesm.generator import genTypeChunk, STREAM_NAME, correct_sum -from nesm.utils import unfold +from typesinfo import Context, TypeChunk, estimateBasicSize, isBasic +from basics import genSerialize, genDeserialize +from generator import genTypeChunk, STREAM_NAME, correct_sum +from utils import unfold from streams import writeData, write, readChar import macros diff --git a/nesm/procgen.nim b/nesm/procgen.nim index 4e6493f..aaab1d9 100644 --- a/nesm/procgen.nim +++ b/nesm/procgen.nim @@ -1,8 +1,7 @@ from typesinfo import BodyGenerator, Context from streams import newStringStream, Stream from tables import `[]=` -from nesm.utils import unfold -from nesm.generator import STREAM_NAME, genTypeChunk +from generator import STREAM_NAME, genTypeChunk import macros type ProcType = enum diff --git a/nesm/sets.nim b/nesm/sets.nim index 0a36d35..96f3fb2 100644 --- a/nesm/sets.nim +++ b/nesm/sets.nim @@ -1,9 +1,9 @@ import macros from strutils import `%` from tables import contains, `[]` -from nesm.typesinfo import TypeChunk, Context -from nesm.typesinfo import isBasic, estimateBasicSize -from nesm.basics import genBasic +from typesinfo import TypeChunk, Context +from typesinfo import isBasic, estimateBasicSize +from basics import genBasic proc genSet*(context: Context, declaration: NimNode): TypeChunk {.compileTime.} From 34ace063447e2a1381bb3857790202a4f90b9d41 Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 21 Oct 2017 19:44:13 +0300 Subject: [PATCH 46/67] Imports now should be placed by following rules: 1. Imports to declare exported functions (e.g. macros or typesinfo) 2. Exported function declarations 3. Other exported stuff declarations (can be places before 2) 4. Other imports Such an order should prevent circular dependences --- nesm/basics.nim | 4 ++-- nesm/enums.nim | 13 +++++++++---- nesm/generator.nim | 29 ++++++++++------------------- nesm/objects.nim | 10 +++++----- nesm/periodic.nim | 24 +++++++++++++++--------- nesm/sets.nim | 7 ++++--- nesm/settings.nim | 4 +++- nesm/typesinfo.nim | 4 ++-- nesm/utils.nim | 7 +++++++ 9 files changed, 57 insertions(+), 45 deletions(-) diff --git a/nesm/basics.nim b/nesm/basics.nim index ab434e9..95145f3 100644 --- a/nesm/basics.nim +++ b/nesm/basics.nim @@ -1,6 +1,4 @@ from typesinfo import Context, TypeChunk -from endians import swapEndian16, swapEndian32, swapEndian64 -from streams import writeData, readData import macros proc genBasic*(context: Context, size: int): TypeChunk {.compileTime.} @@ -8,6 +6,8 @@ proc genSerialize*(name: NimNode, size: NimNode): NimNode {.compileTime.} proc genDeserialize*(name: NimNode, size: NimNode): NimNode {.compileTime.} from nesm.generator import STREAM_NAME +from endians import swapEndian16, swapEndian32, swapEndian64 +from streams import writeData, readData proc genDeserialize*(name: NimNode, size: NimNode): NimNode = quote do: diff --git a/nesm/enums.nim b/nesm/enums.nim index 4ba6d98..81d1704 100644 --- a/nesm/enums.nim +++ b/nesm/enums.nim @@ -1,8 +1,13 @@ import macros -from nesm.typesinfo import TypeChunk, Context -from nesm.basics import genBasic +from typesinfo import TypeChunk, Context -proc getCount*(declaration: NimNode): uint64 {.compileTime.} = + +proc getCount*(declaration: NimNode): uint64 {.compileTime.} +proc genEnum*(context:Context, declaration: NimNode): TypeChunk {.compileTime.} + +from basics import genBasic + +proc getCount(declaration: NimNode): uint64 = for c in declaration.children(): case c.kind of nnkEnumFieldDef: @@ -34,7 +39,7 @@ proc estimateEnumSize(highest: uint64): int {.compileTime.} = else: 0 -proc genEnum*(context:Context, declaration: NimNode): TypeChunk {.compileTime.}= +proc genEnum(context:Context, declaration: NimNode): TypeChunk = let count = getCount(declaration) let estimated = estimateEnumSize(count) if estimated == 0: error("Internal error while estimating enum size") diff --git a/nesm/generator.nim b/nesm/generator.nim index 8bd36e1..dc6ccce 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -1,32 +1,23 @@ import macros -from typesinfo import isBasic, estimateBasicSize from typesinfo import TypeChunk, Context -from utils import unfold -from tables import Table, contains, `[]`, `[]=`, initTable, - pairs -from strutils import `%` -from sequtils import mapIt, foldl, toSeq, filterIt proc genTypeChunk*(immutableContext: Context, thetype: NimNode): TypeChunk {.compileTime.} -proc correct_sum*(part_size: NimNode): NimNode {.compileTime.} static: let STREAM_NAME* = !"thestream" -from nesm.objects import genObject -from nesm.basics import genBasic -from nesm.periodic import genPeriodic, genCStringDeserialize, genCStringSerialize -from nesm.enums import genEnum -from nesm.sets import genSet - +from tables import Table, contains, `[]`, `[]=`, initTable, pairs +from sequtils import mapIt, foldl, toSeq, filterIt +from strutils import `%` +from utils import unfold, correct_sum +from typesinfo import isBasic, estimateBasicSize +from objects import genObject +from basics import genBasic +from periodic import genPeriodic, genCStringDeserialize, genCStringSerialize +from enums import genEnum +from sets import genSet -proc correct_sum(part_size: NimNode): NimNode = - if part_size.kind == nnkInfix or part_size.len == 0: - let result_node = newIdentNode("result") - result_node.infix("+=", part_size) - else: - part_size proc dig(node: NimNode, depth: Natural): NimNode {.compileTime.} = if depth == 0: diff --git a/nesm/objects.nim b/nesm/objects.nim index 9584ed1..a850360 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -1,9 +1,4 @@ -from sequtils import toSeq, filterIt, mapIt -from generator import genTypeChunk, correct_sum -from utils import unfold from typesinfo import TypeChunk, Context -from settings import applyOptions, splitSettingsExpr -import macros type Field = tuple @@ -19,6 +14,11 @@ proc caseWorkaround(tc: TypeChunk): TypeChunk {.compileTime.} proc evalSize(e: NimNode): BiggestInt {.compileTime.} proc genFields(context: Context, decl: NimNode): FieldChunk {.compileTime.} +from sequtils import toSeq, filterIt, mapIt +from generator import genTypeChunk +from utils import unfold, correct_sum +from settings import applyOptions, splitSettingsExpr +import macros proc caseWorkaround(tc: TypeChunk): TypeChunk = # st - type of field under case diff --git a/nesm/periodic.nim b/nesm/periodic.nim index f0f3b8e..92557d8 100644 --- a/nesm/periodic.nim +++ b/nesm/periodic.nim @@ -1,11 +1,18 @@ -from typesinfo import Context, TypeChunk, estimateBasicSize, isBasic +from typesinfo import Context, TypeChunk, estimateBasicSize, isBasic, BodyGenerator +import macros + + +proc genCStringDeserialize*(name: NimNode): NimNode {.compileTime.} +proc genCStringSerialize*(name: NimNode): NimNode {.compileTime.} +proc genPeriodic*(context: Context, elem: NimNode, + length: BodyGenerator): TypeChunk {.compileTime.} + from basics import genSerialize, genDeserialize -from generator import genTypeChunk, STREAM_NAME, correct_sum -from utils import unfold +from generator import genTypeChunk, STREAM_NAME +from utils import unfold, correct_sum from streams import writeData, write, readChar -import macros -proc genCStringDeserialize*(name: NimNode): NimNode {.compileTime.} = +proc genCStringDeserialize(name: NimNode): NimNode = quote do: block: var str = "" & `STREAM_NAME`.readChar() @@ -15,7 +22,7 @@ proc genCStringDeserialize*(name: NimNode): NimNode {.compileTime.} = index += 1 `name` = str[0.. Date: Sun, 22 Oct 2017 21:39:54 +0300 Subject: [PATCH 48/67] Make the 'Nested distinct' test green again --- nesm/generator.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nesm/generator.nim b/nesm/generator.nim index dc6ccce..2f31fbc 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -221,7 +221,7 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = quote do: var `tmp` = cast[`basetype`](`source`) `deserialization` - cast[type(`r`)](`tmp`) + `source` = cast[type(`source`)](`tmp`) of nnkNilLit: result = context.genObject(newTree(nnkRecList, thetype)) else: From aaa751f890635602853e782fa054ee27da25e857 Mon Sep 17 00:00:00 2001 From: xomachine Date: Thu, 2 Nov 2017 00:42:06 +0300 Subject: [PATCH 49/67] Get rid of some dirtyness left from previous changes --- nesm.nim | 2 +- nesm/basics.nim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nesm.nim b/nesm.nim index dca24c4..2cac1b8 100644 --- a/nesm.nim +++ b/nesm.nim @@ -5,7 +5,7 @@ import macros from strutils import `%` from sequtils import toSeq from tables import contains, `[]`, `[]=` -from streams import Stream, newStringStream +from streams import Stream when not defined(nimdoc): from nesm.typesinfo import TypeChunk, Context, initContext diff --git a/nesm/basics.nim b/nesm/basics.nim index 95145f3..50fa4b4 100644 --- a/nesm/basics.nim +++ b/nesm/basics.nim @@ -5,7 +5,7 @@ proc genBasic*(context: Context, size: int): TypeChunk {.compileTime.} proc genSerialize*(name: NimNode, size: NimNode): NimNode {.compileTime.} proc genDeserialize*(name: NimNode, size: NimNode): NimNode {.compileTime.} -from nesm.generator import STREAM_NAME +from generator import STREAM_NAME from endians import swapEndian16, swapEndian32, swapEndian64 from streams import writeData, readData From 4485fa85835cae522955d4f91259f3bdc5f0d5f5 Mon Sep 17 00:00:00 2001 From: xomachine Date: Fri, 3 Nov 2017 23:28:27 +0300 Subject: [PATCH 50/67] Added test for correctness of static status set via options passed to toSerializable --- tests/converter.nim | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/converter.nim b/tests/converter.nim index 01a90f2..eb7e639 100644 --- a/tests/converter.nim +++ b/tests/converter.nim @@ -38,4 +38,14 @@ suite "Converter tests": var b = 0.AInt bigEndian32(b.addr, rnw.buffer[0].addr) check(b == a) - + test "Reusage of static": + toSerializable(MD5Digest, dynamic: false) + serializable: + static: + type Reuser = object + a: MD5Digest + let rnw = get_random_reader_n_writer() + let da = Reuser.deserialize(rnw) + rnw.setPosition(0) + da.serialize(rnw) + check(true) From 81d5aacf0035584ea114bc2db3b6891d0f25019a Mon Sep 17 00:00:00 2001 From: xomachine Date: Fri, 3 Nov 2017 23:31:16 +0300 Subject: [PATCH 51/67] Fixed inproper interpretation of the dynamic option --- nesm/settings.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nesm/settings.nim b/nesm/settings.nim index f61c066..9b3c6dc 100644 --- a/nesm/settings.nim +++ b/nesm/settings.nim @@ -23,8 +23,8 @@ proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = of "endian": result.swapEndian = cmpIgnoreStyle(val, $cpuEndian) != 0 of "dynamic": - let code = int(cmpIgnoreStyle(val, "true") == 0) + - 2*int(cmpIgnoreStyle(val, "false") == 0) + let code = 2*int(cmpIgnoreStyle(val, "true") == 0) + + int(cmpIgnoreStyle(val, "false") == 0) case code of 0: error("The dynamic property can be only 'true' or 'false' but not" & val) From 335bcdaafc975b31b342f029a27b717666668bc0 Mon Sep 17 00:00:00 2001 From: xomachine Date: Fri, 3 Nov 2017 23:31:53 +0300 Subject: [PATCH 52/67] Removed dead code --- nesm/generator.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/nesm/generator.nim b/nesm/generator.nim index 2f31fbc..490b39a 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -217,7 +217,6 @@ proc genTypeChunk(immutableContext: Context, thetype: NimNode): TypeChunk = `serialization` result.deserialize = proc(source: NimNode): NimNode = let deserialization = distincted.deserialize(tmp) - let r = !"result" quote do: var `tmp` = cast[`basetype`](`source`) `deserialization` From 9543ad9e55e083d623062a58b7877154b5fdf401 Mon Sep 17 00:00:00 2001 From: xomachine Date: Mon, 13 Nov 2017 03:32:08 +0300 Subject: [PATCH 53/67] More informative error message when enum is out of range --- nesm/enums.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nesm/enums.nim b/nesm/enums.nim index 81d1704..05eca79 100644 --- a/nesm/enums.nim +++ b/nesm/enums.nim @@ -51,6 +51,6 @@ proc genEnum(context:Context, declaration: NimNode): TypeChunk = result.deserialize = proc (source: NimNode): NimNode = let check = quote do: if $(`source`) == $(ord(`source`)) & " (invalid data!)": - raise newException(ValueError, "Enum value is out of range!") + raise newException(ValueError, "Enum value is out of range: " & $(`source`)) newTree(nnkStmtList, olddeser(source), check) From 5ee3995ea78f5f5e76d5293dee93476b9a9fedb0 Mon Sep 17 00:00:00 2001 From: xomachine Date: Mon, 13 Nov 2017 15:13:39 +0300 Subject: [PATCH 54/67] Added test for case when size of enum is explicitly set --- tests/enumsize.nim | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/enumsize.nim b/tests/enumsize.nim index 1e97386..2554af9 100644 --- a/tests/enumsize.nim +++ b/tests/enumsize.nim @@ -65,3 +65,24 @@ suite "Enum size evaluation with increment": increment_test(4294967295) increment_test(4294967296) increment_test(4294967297) + +from nesm import serializable + +template pragma_test(val: Natural) = + block: + serializable: + static: + type sometype {.size: 1.} = enum + somefield = val + if sometype.sizeof != size(sometype): + echo $sometype.sizeof & " != " & $size(sometype) & " on test with val=" & + $val + check(false) + +suite "Explicitly set size of enum": + test "1 byte enum": + pragma_test(253) + pragma_test(254) + pragma_test(255) + pragma_test(256) + pragma_test(257) From 443a2f53e98d63a0ef6de197b885968a98373168 Mon Sep 17 00:00:00 2001 From: xomachine Date: Mon, 13 Nov 2017 17:33:45 +0300 Subject: [PATCH 55/67] Implementation of the size pragma handling for enums --- nesm/enums.nim | 20 +++++++++++++++++--- nesm/procgen.nim | 9 +++++++-- nesm/settings.nim | 5 +++++ 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/nesm/enums.nim b/nesm/enums.nim index 05eca79..d4add61 100644 --- a/nesm/enums.nim +++ b/nesm/enums.nim @@ -39,10 +39,24 @@ proc estimateEnumSize(highest: uint64): int {.compileTime.} = else: 0 -proc genEnum(context:Context, declaration: NimNode): TypeChunk = +proc genEnum(context: Context, declaration: NimNode): TypeChunk = let count = getCount(declaration) - let estimated = estimateEnumSize(count) - if estimated == 0: error("Internal error while estimating enum size") + let sizeOverrides = len(context.overrides.size) + const intErrorMsg = "Only plain int literals allowed in size pragma " & + "under serializable macro, not " + let estimated = + if sizeOverrides == 0: + estimateEnumSize(count) + elif sizeOverrides == 1: + (let size = context.overrides.size[0][0]; + if size.kind != nnkIntLit: error(intErrorMsg & size.repr, size); + size.intVal.int) + else: + (error("Incorrect amount of size options encountered", declaration); + 0) + #if size.kind != nnkIntLit: + if estimated == 0: + error("Internal error while estimating enum size", declaration) result = context.genBasic(estimated) result.nodekind = nnkEnumTy result.maxcount = count diff --git a/nesm/procgen.nim b/nesm/procgen.nim index aaab1d9..a8da610 100644 --- a/nesm/procgen.nim +++ b/nesm/procgen.nim @@ -2,6 +2,7 @@ from typesinfo import BodyGenerator, Context from streams import newStringStream, Stream from tables import `[]=` from generator import STREAM_NAME, genTypeChunk +from settings import applyOptions import macros type ProcType = enum @@ -77,11 +78,15 @@ proc generateProcs(context: var Context, obj: NimNode): NimNode = expectKind(obj, nnkTypeDef) expectMinLen(obj, 3) expectKind(obj[1], nnkEmpty) - let typedeclaration = if obj[0].kind == nnkPragmaExpr: obj[0][0] else: obj[0] + let (newcontext, typedeclaration) = + if obj[0].kind == nnkPragmaExpr: + (context.applyOptions(obj[0][1]), obj[0][0]) + else: + (context, obj[0]) let is_shared = typedeclaration.kind == nnkPostfix let typenode = if is_shared: typedeclaration.basename else: typedeclaration let body = obj[2] - let typeinfo = genTypeChunk(context, body) + let typeinfo = genTypeChunk(newcontext, body) context.declared[$typenode] = typeinfo let size_proc = makeDeclaration(typenode, size, is_shared, context.is_static, typeinfo.size) diff --git a/nesm/settings.nim b/nesm/settings.nim index 9b3c6dc..14c931e 100644 --- a/nesm/settings.nim +++ b/nesm/settings.nim @@ -14,7 +14,10 @@ proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = ## Options should be a NimNode or seq[NimNode] which contains nnkExprColonExpr ## nodes with key - value pairs. result = context + let pragma = type(options) is NimNode and options.kind == nnkPragma for option in options.items(): + if pragma and option.kind != nnkExprColonExpr: + continue option.expectKind(nnkExprColonExpr) option.expectMinLen(2) let key = option[0].repr @@ -35,6 +38,8 @@ proc applyOptions(context: Context, options: NimNode | seq[NimNode]): Context = result.overrides.size.insert((option[1], context.depth), 0) of "sizeof": result.overrides.sizeof.add((option[1], context.depth)) + elif pragma: + continue else: error("Unknown setting: " & key) From 451f23de788da2fd7099a2385036bac3e3800408 Mon Sep 17 00:00:00 2001 From: xomachine Date: Mon, 13 Nov 2017 17:53:25 +0300 Subject: [PATCH 56/67] Indentation fix for devel Nim version --- nesm/enums.nim | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nesm/enums.nim b/nesm/enums.nim index d4add61..a0a3d61 100644 --- a/nesm/enums.nim +++ b/nesm/enums.nim @@ -52,9 +52,7 @@ proc genEnum(context: Context, declaration: NimNode): TypeChunk = if size.kind != nnkIntLit: error(intErrorMsg & size.repr, size); size.intVal.int) else: - (error("Incorrect amount of size options encountered", declaration); - 0) - #if size.kind != nnkIntLit: + (error("Incorrect amount of size options encountered", declaration); 0) if estimated == 0: error("Internal error while estimating enum size", declaration) result = context.genBasic(estimated) From 6bec87c0420fa8b07698c37798647907ed49db1f Mon Sep 17 00:00:00 2001 From: xomachine Date: Mon, 13 Nov 2017 18:10:36 +0300 Subject: [PATCH 57/67] Enum range check error message improved Now it shows allowed values as well --- nesm/enums.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nesm/enums.nim b/nesm/enums.nim index a0a3d61..ba3c2c9 100644 --- a/nesm/enums.nim +++ b/nesm/enums.nim @@ -60,9 +60,11 @@ proc genEnum(context: Context, declaration: NimNode): TypeChunk = result.maxcount = count when not defined(disableEnumChecks): let olddeser = result.deserialize + let enumdecl = newStrLitNode(declaration.repr) result.deserialize = proc (source: NimNode): NimNode = let check = quote do: if $(`source`) == $(ord(`source`)) & " (invalid data!)": - raise newException(ValueError, "Enum value is out of range: " & $(`source`)) + raise newException(ValueError, "Enum value is out of range: " & + $(`source`) & "\nCorrect values are:\n" & `enumdecl`) newTree(nnkStmtList, olddeser(source), check) From c94dd353829f8f9b631aa14a3688efe6b35daec1 Mon Sep 17 00:00:00 2001 From: xomachine Date: Wed, 15 Nov 2017 17:51:59 +0300 Subject: [PATCH 58/67] Added missing debug output when toSerializable macro invoked --- nesm.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nesm.nim b/nesm.nim index 2cac1b8..1ff4b46 100644 --- a/nesm.nim +++ b/nesm.nim @@ -96,6 +96,8 @@ macro toSerializable*(typedecl: typed, settings: varargs[untyped]): untyped = result.add(ctx.prepare(ast)) ctx.is_static = false ctx.swapEndian = false + when defined(debug): + hint(result.repr) macro serializable*(typedecl: untyped): untyped = ## The main macro that generates code. From 8b2a4cdeb5ccf4dd6331712c8c2268fae61512bf Mon Sep 17 00:00:00 2001 From: xomachine Date: Wed, 15 Nov 2017 20:44:21 +0300 Subject: [PATCH 59/67] Added another failing test Its impossible to obtain pragmas in toSerializable, but possible to double them in options --- tests/enumsize.nim | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/enumsize.nim b/tests/enumsize.nim index 2554af9..368be2d 100644 --- a/tests/enumsize.nim +++ b/tests/enumsize.nim @@ -66,7 +66,7 @@ suite "Enum size evaluation with increment": increment_test(4294967296) increment_test(4294967297) -from nesm import serializable +from nesm import serializable, toSerializable template pragma_test(val: Natural) = block: @@ -86,3 +86,13 @@ suite "Explicitly set size of enum": pragma_test(255) pragma_test(256) pragma_test(257) +# test "toSerializable neverfixed test": +# type myenum {.size: 1.} = enum +# a = 255 +# toSerializable(myenum, dynamic: false) +# check(size(myenum) == 1) + test "toSerializable test": + type myenum {.size: 1.} = enum + a = 255 + toSerializable(myenum, dynamic: false, size: 1) + check(size(myenum) == 1) From 31cdf03707dbae26d1fd89ac50e44e1bcaab1715 Mon Sep 17 00:00:00 2001 From: xomachine Date: Wed, 15 Nov 2017 21:08:57 +0300 Subject: [PATCH 60/67] Added context shadowing test for toSerializable --- tests/converter.nim | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/converter.nim b/tests/converter.nim index eb7e639..8e9bab1 100644 --- a/tests/converter.nim +++ b/tests/converter.nim @@ -49,3 +49,13 @@ suite "Converter tests": rnw.setPosition(0) da.serialize(rnw) check(true) + test "Context shadowing": + type + A = enum + a + B = enum + b + toSerializable(A, size: 1) + toSerializable(B, size: 1, dynamic: false) + check(size(a) == size(B)) + check(size(a) == 1) From 076639c5503a6daf32d31fb3397e95d19e474126 Mon Sep 17 00:00:00 2001 From: xomachine Date: Wed, 15 Nov 2017 21:15:02 +0300 Subject: [PATCH 61/67] Fixed global toSerializable options sharing --- nesm.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/nesm.nim b/nesm.nim index 1ff4b46..4c5d6f9 100644 --- a/nesm.nim +++ b/nesm.nim @@ -90,12 +90,11 @@ macro toSerializable*(typedecl: typed, settings: varargs[untyped]): untyped = when defined(debug): hint(typedecl.symbol.getImpl().treeRepr()) var ast = typedecl.symbol.getImpl() - ctx = ctx.applyOptions(settings) + var newctx = ctx.applyOptions(settings) when defined(debug): hint(ast.treeRepr) - result.add(ctx.prepare(ast)) - ctx.is_static = false - ctx.swapEndian = false + result.add(newctx.prepare(ast)) + ctx.declared = newctx.declared when defined(debug): hint(result.repr) From 50101df49d9a2ed74249e2169101a0b2d219362c Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 17 Feb 2018 16:16:51 +0300 Subject: [PATCH 62/67] Replaced all deprecated `!` calls to `newIdentNode` calls --- nesm/generator.nim | 2 +- nesm/objects.nim | 6 +++--- nesm/periodic.nim | 2 +- nesm/procgen.nim | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nesm/generator.nim b/nesm/generator.nim index 490b39a..1738627 100644 --- a/nesm/generator.nim +++ b/nesm/generator.nim @@ -5,7 +5,7 @@ proc genTypeChunk*(immutableContext: Context, thetype: NimNode): TypeChunk {.compileTime.} static: - let STREAM_NAME* = !"thestream" + let STREAM_NAME* = newIdentNode("thestream") from tables import Table, contains, `[]`, `[]=`, initTable, pairs from sequtils import mapIt, foldl, toSeq, filterIt diff --git a/nesm/objects.nim b/nesm/objects.nim index a850360..5b51f2f 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -79,7 +79,7 @@ proc genObject(context: Context, thetype: NimNode): TypeChunk = result = newIntLitNode(0) var result_list = newSeq[NimNode]() for i in elems.items(): - let n = !i.name + let n = newIdentNode(i.name) let newsource = if ($n).len > 0: (quote do: `source`.`n`).unfold() else: source @@ -98,7 +98,7 @@ proc genObject(context: Context, thetype: NimNode): TypeChunk = result.serialize = proc(source: NimNode): NimNode = result = newStmtList(parseExpr("discard")) for i in elems.items(): - let n = !i.name + let n = newIdentNode(i.name) let newsource = if ($n).len > 0: (quote do: `source`.`n`).unfold() else: source @@ -107,7 +107,7 @@ proc genObject(context: Context, thetype: NimNode): TypeChunk = result.deserialize = proc(source: NimNode): NimNode = result = newStmtList(parseExpr("discard")) for i in elems.items(): - let n = !i.name + let n = newIdentNode(i.name) let newsource = if ($n).len > 0: (quote do: `source`.`n`).unfold() else: source diff --git a/nesm/periodic.nim b/nesm/periodic.nim index 92557d8..323181f 100644 --- a/nesm/periodic.nim +++ b/nesm/periodic.nim @@ -107,7 +107,7 @@ proc genPeriodic(context: Context, elem: NimNode, result.size = proc(s:NimNode):NimNode = let presize = preresult.size(s) let headersize = size_header_chunk.size(s) - let r = !"result" + let r = newIdentNode("result") if presize.kind notin [nnkInfix, nnkIntLit, nnkCall]: quote do: `presize` diff --git a/nesm/procgen.nim b/nesm/procgen.nim index a8da610..45a0fca 100644 --- a/nesm/procgen.nim +++ b/nesm/procgen.nim @@ -61,7 +61,7 @@ proc makeDeserializeStreamConversion(typenode: NimNode, ## Makes overloaded deserialize proc which takes ``string`` as input instead ## of ``Stream`` let rawprocname = newIdentNode("deserialize") - let sizeident = !("size") + let sizeident = newIdentNode("size") let procname = if is_exported: rawprocname.postfix("*") else: rawprocname quote do: proc `procname`(a: typedesc[`typenode`], From 10f5d69b797fd65aac7e20d777ed011af4f9bdbc Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 17 Feb 2018 16:18:11 +0300 Subject: [PATCH 63/67] Replaced all deprecated 'random' calls to 'rand' --- tests/customseqsize.nim | 18 +++++++++--------- tests/dynamic.nim | 8 ++++---- tests/enum.nim | 10 +++++----- tests/helpers/rnw.nim | 8 ++++---- tests/nested.nim | 2 +- tests/sizeof.nim | 8 ++++---- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/tests/customseqsize.nim b/tests/customseqsize.nim index 566481c..77e92dc 100644 --- a/tests/customseqsize.nim +++ b/tests/customseqsize.nim @@ -16,9 +16,9 @@ suite "Custom periodic size": let rnw = get_reader_n_writer() var o: MySeq - o.data = random_seq_with(random(20000).int32) + o.data = random_seq_with(rand(20000).int32) o.size = o.data.len.int32 - o.a = random(20000).int32 + o.a = rand(20000).int32 o.serialize(rnw) rnw.setPosition(0) let dso = MySeq.deserialize(rnw) @@ -41,7 +41,7 @@ suite "Custom periodic size": var o: MyString o.data = get_random_string() o.size = o.data.len.int32 - o.a = random(20000).int32 + o.a = rand(20000).int32 o.serialize(rnw) rnw.setPosition(0) let dso = MyString.deserialize(rnw) @@ -63,7 +63,7 @@ suite "Custom periodic size": var o: MySeqStr o.data = random_seq_with(get_random_string()) o.s = o.data.len.int32 - o.a = random(20000).int32 + o.a = rand(20000).int32 o.serialize(rnw) rnw.setPosition(0) let dso = MySeqStr.deserialize(rnw) @@ -83,11 +83,11 @@ suite "Custom periodic size": let rnw = get_reader_n_writer() var o: Matrix - o.columns = random(1..20).int32 + o.columns = rand(1..20).int32 proc get_line(): seq[int32] = result = newSeq[int32](o.columns) for i in 0.. Date: Sat, 17 Feb 2018 16:33:50 +0300 Subject: [PATCH 64/67] Fixed build for older Nim versions --- tests/helpers/rnw.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/helpers/rnw.nim b/tests/helpers/rnw.nim index 661274f..09747f4 100644 --- a/tests/helpers/rnw.nim +++ b/tests/helpers/rnw.nim @@ -1,6 +1,10 @@ from sequtils import newSeqWith -from random import rand +when NimMajor > 0 or NimMinor > 17 or (NimMinor == 17 and NimPatch > 2): + from random import rand +else: + from random import random + proc rand[T](a: T): int = random(a) from streams import Stream, StringStream, newStringStream export rand From 5ce51302735c65cec500796eb343cf5371d91f9b Mon Sep 17 00:00:00 2001 From: xomachine Date: Sat, 17 Feb 2018 19:13:02 +0300 Subject: [PATCH 65/67] Added simple test case for nonintrusive serializer --- nesm/objects.nim | 6 +++--- nesm/utils.nim | 6 ++++++ tests/nonintrusive.nim | 19 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 tests/nonintrusive.nim diff --git a/nesm/objects.nim b/nesm/objects.nim index 5b51f2f..caf648b 100644 --- a/nesm/objects.nim +++ b/nesm/objects.nim @@ -16,7 +16,7 @@ proc genFields(context: Context, decl: NimNode): FieldChunk {.compileTime.} from sequtils import toSeq, filterIt, mapIt from generator import genTypeChunk -from utils import unfold, correct_sum +from utils import unfold, correct_sum, onlyname from settings import applyOptions, splitSettingsExpr import macros @@ -128,11 +128,11 @@ proc genFields(context: Context, decl: NimNode): FieldChunk = for i in 0.. Date: Sat, 17 Feb 2018 19:14:15 +0300 Subject: [PATCH 66/67] Non-intrusive serializer implementation --- nesm.nim | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/nesm.nim b/nesm.nim index 4c5d6f9..020cecc 100644 --- a/nesm.nim +++ b/nesm.nim @@ -9,6 +9,7 @@ from streams import Stream when not defined(nimdoc): from nesm.typesinfo import TypeChunk, Context, initContext + from nesm.generator import genTypeChunk from nesm.procgen import generateProcs from nesm.settings import applyOptions, splitSettingsExpr else: @@ -73,6 +74,36 @@ proc cleanupTypeDeclaration(declaration: NimNode): NimNode = children.add(cleanupTypeDeclaration(c)) newTree(declaration.kind, children) +macro nonIntrusiveBody(typename: typed, o: untyped, de: static[bool]): untyped = + let typebody = getTypeImpl(typename) + when defined(debug): + hint("Deserialize? " & $de) + hint(typebody.treeRepr) + hint(typebody.repr) + let chunk = ctx.genTypeChunk(typebody) + result = if de: chunk.deserialize(o) else: chunk.serialize(o) + when defined(debug): + hint(result.repr) + +template nonIntrusiveTemplate[S](o: S, de: static[bool]) = + nonIntrusiveBody(S, o, de) + +proc serialize*[T](obj: T, thestream: Stream) = + ## The non-intrusive serialize proc which allows to perform object + ## serialization without special declaration. + ## The negative side-effect of such approach is inability to pass any options + ## (like `endian` or `dynamic`) to the serializer and lack of nested objects + ## support. This proc should be imported directly. + nonIntrusiveTemplate(obj, false) + +proc deserialize*[T](thestream: Stream): T = + ## The non-intrusive deserialize proc which allows to perform object + ## deserialization without special declaration. + ## The negative side-effect of such approach is inability to pass any options + ## (like `endian` or `dynamic`) to the serializer and lack of nested objects + ## support. This proc should be imported directly. + nonIntrusiveTemplate(result, true) + macro toSerializable*(typedecl: typed, settings: varargs[untyped]): untyped = ## Generate [de]serialize procedures for existing type with given settings. ## From 0da16e2eaf59e92b022e36eb81a7c227b7071c68 Mon Sep 17 00:00:00 2001 From: xomachine Date: Fri, 23 Feb 2018 18:40:52 +0300 Subject: [PATCH 67/67] NTP demo rework --- demos/ntp.nim | 98 ++++++++++++++++++---------------------- demos/ntp.ntp | 122 -------------------------------------------------- 2 files changed, 43 insertions(+), 177 deletions(-) delete mode 100644 demos/ntp.ntp diff --git a/demos/ntp.nim b/demos/ntp.nim index 134f993..ee96465 100644 --- a/demos/ntp.nim +++ b/demos/ntp.nim @@ -1,5 +1,8 @@ # # NTP client demo +# --------------- +# Makes a request to ntp server and shows information. +# All serialization/deserialization work is being performed by NESM. # import @@ -12,32 +15,19 @@ import import NESM -# 1900 to 1970 in seconds -const unix_time_delta = 2208988800.0 - -proc to_host(x: uint32): uint32 = ntohl(x) - -proc to_net(x: uint32): uint32 = - htonl(x) serializable: static: + # Types are static, so their size after serialization + # can be calculated at compile time type - nuint16 = distinct uint16 - nint16 = distinct int16 - nuint32 = distinct uint32 - nint32 = distinct int32 - nuint64 = distinct uint64 - nint64 = distinct int64 - FixedPoint16dot16 = uint32 - seconds = distinct nuint32 - fraction = distinct nuint32 - FixedPoint32dot32 = object - seconds: uint32 - fraction: uint32 NTPTimeStamp = object + set: {endian: bigEndian} + # The endian of following fields is set to bigEndian, + # so following values will be swapped while serialization + # and there is no necessity to use htonl/ntohl seconds: uint32 fraction: uint32 @@ -56,67 +46,65 @@ serializable: receive_ts: NTPTimeStamp transmit_ts: NTPTimeStamp +const twoto32 = 4294967296.0 +const packet_size = NTPPacket.size() # Compile-time type size calculation +const unix_time_delta = 2208988800.0 # 1900 to 1970 in seconds -proc ntp_ts_to_epoch(ts: NTPTimeStamp): float64 = - const twoto32 = 4294967296.0 +proc toEpoch(ts: NTPTimeStamp): float64 = let - sec = ts.seconds.to_host.float - unix_time_delta - frac = ts.fraction.to_host.float / twoto32 + sec = ts.seconds.float - unix_time_delta + frac = ts.fraction.float / twoto32 sec + frac -proc epoch_to_ntp_ts(x: float): NTPTimeStamp = - - result = NTPTimeStamp( - seconds: uint32(x).to_net() - ) - -const packet_size = NTPPacket.size() -assert packet_size == 48 +proc fromEpoch(epoch: float64): NTPTimeStamp = + let secs = epoch.uint32 + NTPTimeStamp(seconds: secs, + fraction: uint32((epoch - secs.float) * twoto32)) -proc main() = +when isMainModule: if paramCount() != 1: echo "Usage $# " % paramStr(0) quit(1) let target = paramStr(1) - - var b = NTPPacket() + let b = NTPPacket( # example values - b.leap_version_mode = 0x23 - b.stratum = 0x03 - b.polling_interval = 0x06 - b.clock_precision = 0xfa - b.delay = 0x00010000 - b.dispersion = 0x00010000 - b.reference_id = 0xaabbcc - b.origin_ts = epochTime().epoch_to_ntp_ts + leap_version_mode: 0x23, + stratum: 0x03, + polling_interval: 0x06, + clock_precision: 0xfa, + delay: 0x00010000, + dispersion: 0x00010000, + reference_id: 0xaabbcc, + origin_ts: epochTime().fromEpoch, + ) let bytes = cast[string](b.serialize()) var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP, true) # client send time - let t1 = epochTime() + let sendtime = epochTime() doAssert socket.sendTo(target, Port(123), bytes) == packet_size let raw_resp = socket.recv(packet_size, timeout=1000) # client receive time - let t4 = epochTime() + let resptime = epochTime() assert raw_resp.len == packet_size let resp: NTPPacket = deserialize(NTPPacket, raw_resp) - let t2 = resp.receive_ts.ntp_ts_to_epoch - let t3 = resp.transmit_ts.ntp_ts_to_epoch + let recvtime = resp.receive_ts.toEpoch + let anstime = resp.transmit_ts.toEpoch # NTP delay / clock_offset calculation - let delay = (t4 - t1) - (t3 - t2) - let clock_offset = (t2 - t1 + t3 - t4)/2 - - echo "delay: ", delay * 1000, " ms" - echo "clock_offset ", clock_offset * 1000, " ms" - - -when isMainModule: - main() + let delay = (resptime - sendtime) - (anstime - recvtime) + let clock_offset = (recvtime - sendtime + anstime - resptime)/2 + + echo " Request sent: ", sendtime + echo " Request received: ", recvtime + echo " Answer sent: ", anstime + echo " Answer received: ", resptime + echo " Round-trip delay: ", delay * 1000, " ms" + echo " Clock offset: ", clock_offset * 1000, " ms" diff --git a/demos/ntp.ntp b/demos/ntp.ntp deleted file mode 100644 index 96d1519..0000000 --- a/demos/ntp.ntp +++ /dev/null @@ -1,122 +0,0 @@ -# -# NTP client demo -# - -import - nativesockets, - net, - os, - sequtils, - strutils, - times - -import NESM - -# 1900 to 1970 in seconds -const unix_time_delta = 2208988800.0 - -proc to_host(x: uint32): uint32 = ntohl(x) - -proc to_net(x: uint32): uint32 = - htonl(x) - -serializable: - static: - type - nuint16 = distinct uint16 - nint16 = distinct int16 - nuint32 = distinct uint32 - nint32 = distinct int32 - nuint64 = distinct uint64 - nint64 = distinct int64 - - FixedPoint16dot16 = uint32 - seconds = distinct nuint32 - fraction = distinct nuint32 - FixedPoint32dot32 = object - seconds: uint32 - fraction: uint32 - - NTPTimeStamp = object - seconds: uint32 - fraction: uint32 - - LeapVersionMode = uint8 - - NTPPacket = object - leap_version_mode: LeapVersionMode - stratum: uint8 - polling_interval: int8 - clock_precision: uint8 - delay: FixedPoint16dot16 - dispersion: FixedPoint16dot16 - reference_id: uint32 - reference_ts: NTPTimeStamp - origin_ts: NTPTimeStamp - receive_ts: NTPTimeStamp - transmit_ts: NTPTimeStamp - - -proc ntp_ts_to_epoch(ts: NTPTimeStamp): float64 = - const twoto32 = 4294967296.0 - let - sec = ts.seconds.to_host.float - unix_time_delta - frac = ts.fraction.to_host.float / twoto32 - sec + frac - -proc epoch_to_ntp_ts(x: float): NTPTimeStamp = - - result = NTPTimeStamp( - seconds: uint32(x).to_net() - ) - -proc main() = - if paramCount() != 1: - echo "Usage $# " % paramStr(0) - quit(1) - - let target = paramStr(1) - var b = NTPPacket() - - # example values - b.leap_version_mode = 0x23 - b.stratum = 0x03 - b.polling_interval = 0x06 - b.clock_precision = 0xfa - b.delay = 0x00010000 - b.dispersion = 0x00010000 - b.reference_id = 0xaabbcc - b.origin_ts = epochTime().epoch_to_ntp_ts - - var bytes = "" - for i in b.serialize(): - bytes.add chr(i) - - assert bytes.len == 48 - - var socket = newSocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP, true) - # client send time - let t1 = epochTime() - doAssert socket.sendTo(target, Port(123), bytes) == 48 - - let raw_resp = socket.recv(48, timeout=1000) - - # client receive time - let t4 = epochTime() - - assert raw_resp.len == 48 - let resp: NTPPacket = deserialize(NTPPacket, raw_resp) - - let t2 = resp.receive_ts.ntp_ts_to_epoch - let t3 = resp.transmit_ts.ntp_ts_to_epoch - - # NTP delay / clock_offset calculation - let delay = (t4 - t1) - (t3 - t2) - let clock_offset = (t2 - t1 + t3 - t4)/2 - - echo "delay: ", delay * 1000, " ms" - echo "clock_offset ", clock_offset * 1000, " ms" - - -when isMainModule: - main()