libnixt: add Serialize

libnixt/include/nixt/Serialize.h

@@ -0,0 +1,206 @@
+#pragma once
+
+#include <nix/nixexpr.hh>
+#include <nix/symbol-table.hh>
+#include <nix/value.hh>
+
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <type_traits>
+
+/// \file
+/// This file contains serialization method for nix::Expr
+
+namespace nixt::serialize {
+
+using nix::NixFloat;
+using nix::NixInt;
+
+template <class T, class = std::enable_if_t<std::is_standard_layout_v<T>>,
+          class = std::enable_if_t<std::is_trivial_v<T>>>
+std::size_t encode(char *Dst, const T &Data) {
+  memcpy(Dst, &Data, sizeof(T));
+  return sizeof(T);
+}
+
+template <class T, class = std::enable_if_t<std::is_standard_layout_v<T>>,
+          class = std::enable_if_t<std::is_trivial_v<T>>>
+std::size_t getSize(const T &Data) {
+  return sizeof(T);
+}
+
+template <class T> std::size_t encode(char *Dst, const std::vector<T> &Data) {
+  std::size_t Ret = encode(Dst, Data.size());
+  for (const auto &E : Data) {
+    Ret += encode(Dst + Ret, E);
+  }
+  return Ret;
+}
+
+template <class T> std::size_t getSize(const std::vector<T> &Data) {
+  std::size_t Ret = sizeof(Data.size());
+  for (const auto &E : Data) {
+    Ret += getSize(E);
+  }
+  return Ret;
+}
+
+/// Wrapper around std::vector that we can easily get indexes of added element.
+struct IndexVector {
+  template <class T> std::size_t add(T Elt) {
+    std::size_t EltSize = getSize(Elt);
+    Data.reserve(Data.size() + EltSize);
+    Data.resize(Data.size() + EltSize);
+    return encode(Data.data() + Data.size() - EltSize, Elt);
+  }
+  IndexVector() = default;
+  std::vector<char> Data;
+};
+
+inline std::size_t encode(char *Dst, const IndexVector &Data) {
+  return encode(Dst, Data.Data);
+}
+
+inline std::size_t getSize(const IndexVector &Data) {
+  return getSize(Data.Data);
+}
+
+class StringTable {
+public:
+  using Size = std::size_t;
+  Size add(std::string Str);
+
+  [[nodiscard]] uint32_t getLength() const { return Length; }
+
+  void write(char *Dst) const;
+
+  StringTable() = default;
+
+private:
+  std::map<std::string, Size> Map;
+  std::vector<std::string> Strings;
+  uint32_t Length = 0;
+};
+
+using StringIdx = StringTable::Size;
+using PosIdx = std::size_t;
+using ExprIdx = std::size_t;
+
+struct Position {
+  uint32_t Line, Column;
+  enum OriginKind {
+    OK_None,
+    OK_Stdin,
+    OK_String,
+    OK_SourcePath,
+  } Kind;
+  // For "Source Path"
+  StringIdx Source;
+};
+
+struct AttrPath {
+  uint32_t Size;
+  StringIdx Name;
+};
+
+struct AttrDef {
+  bool Inherited;
+  ExprIdx E;
+  PosIdx Pos;
+  uint32_t Displ;
+};
+
+struct AttrDefEntry {
+  StringIdx Name;
+  AttrDef Def;
+};
+
+struct DynamicAttrDef {
+  ExprIdx NameExpr, ValueExpr;
+  PosIdx Pos;
+};
+
+enum class ExprKind : uint32_t {
+  EK_Int,
+  EK_Float,
+  EK_String,
+  EK_Path,
+  EK_Var,
+
+  // Needs custom encoding & decoding
+  EK_AttrSet,
+};
+
+struct SimpleExpr {
+  ExprKind Kind;
+
+  struct Variable {
+    PosIdx Pos;
+    StringIdx Name;
+
+    bool FromWidth;
+
+    uint32_t Level;
+    uint32_t Displ;
+  };
+
+  struct Select {
+    PosIdx Pos;
+    AttrPath Path;
+    ExprIdx E;
+    ExprIdx Def;
+  };
+
+  struct OpHasAttr {
+    ExprIdx E;
+    AttrPath Path;
+  };
+
+  union {
+    NixInt Int;
+    NixFloat Float;
+    StringIdx String;
+    StringIdx Path;
+    Variable Var;
+    struct Select Select;
+  } Body;
+};
+
+struct ExprAttrSet {
+  ExprKind Kind = ExprKind::EK_AttrSet;
+  PosIdx Pos;
+  std::vector<AttrDefEntry> Attrs;
+  std::vector<DynamicAttrDef> DynamicAttrs;
+};
+
+std::size_t encode(char *Dst, const ExprAttrSet &E);
+
+std::size_t getSize(const ExprAttrSet &E);
+
+struct ASTHeader {
+  unsigned char NixAST[8] = "\x7fNixAST";
+  uint32_t Version;
+  std::size_t ExprTableOffset;
+  std::size_t PosTableOffset;
+  std::size_t StringTableOffset;
+};
+
+class ASTWriter {
+public:
+  ASTWriter(const nix::SymbolTable &STable, const nix::PosTable &PTable)
+      : STable(STable), PTable(PTable) {}
+
+  std::size_t count(const nix::Expr *E);
+
+  void write(char *Dst);
+
+private:
+  const nix::SymbolTable &STable;
+  const nix::PosTable &PTable;
+  IndexVector Exprs;
+  IndexVector Pos;
+  StringTable Strings;
+};
+
+} // namespace nixt::serialize

libnixt/lib/Serialize.cpp

@@ -0,0 +1,116 @@
+#include "nixt/Serialize.h"
+#include "nixt/Visitor.h"
+
+#include <nix/input-accessor.hh>
+#include <nix/nixexpr.hh>
+#include <nix/symbol-table.hh>
+
+#include <cstring>
+#include <type_traits>
+
+namespace nixt::serialize {
+
+StringIdx StringTable::add(std::string Str) {
+  if (Map.contains(Str)) {
+    return Map[std::move(Str)];
+  }
+  Length += Str.size() + 1; // size of string + null terminator
+  StringIdx Ret = Map[Str] = Length;
+  Strings.push_back(std::move(Str));
+  return Ret;
+}
+
+void StringTable::write(char *Dst) const {
+  for (const auto &S : Strings) {
+    memcpy(Dst, S.c_str(), S.size() + 1);
+    Dst = static_cast<char *>(Dst) + S.size() + 1;
+  }
+}
+
+std::size_t ASTWriter::count(const nix::Expr *E) {
+  struct ASTWriteVisitor : public RecursiveASTVisitor<ASTWriteVisitor> {
+    IndexVector &Exprs;
+    IndexVector &Pos;
+    StringTable &Strings;
+    const nix::SymbolTable &STable;
+    const nix::PosTable &PTable;
+    std::map<const nix::Expr *, std::size_t> ExprMap;
+
+    ASTWriteVisitor(IndexVector &Exprs, IndexVector &Pos, StringTable &Strings,
+                    const nix::SymbolTable &STable, const nix::PosTable &PTable)
+        : Exprs(Exprs), Pos(Pos), Strings(Strings), STable(STable),
+          PTable(PTable) {}
+
+    Position getPos(const nix::PosIdx P) {
+      if (P == nix::noPos) {
+        return Position{};
+      }
+      const auto &Pos = PTable[P];
+      Position Ret;
+      Ret.Column = Pos.column;
+      Ret.Line = Pos.line;
+      Position::OriginKind KindTable[] = {Position::OK_None, Position::OK_Stdin,
+                                          Position::OK_String,
+                                          Position::OK_SourcePath};
+      Ret.Kind = KindTable[Pos.origin.index()];
+      if (Ret.Kind == Position::OK_SourcePath)
+        Ret.Source =
+            Strings.add(std::get<nix::SourcePath>(Pos.origin).to_string());
+      return Ret;
+    }
+    bool visitExprAttrs(const nix::ExprAttrs *E) {
+      ExprAttrSet Expr;
+      Expr.Pos = Pos.add(getPos(E->pos));
+      for (const auto &[K, V] : E->attrs) {
+        Expr.Attrs.emplace_back(AttrDefEntry{
+            Strings.add(STable[K]),
+            {V.inherited, ExprMap[V.e], Pos.add(getPos(V.pos)), V.displ}});
+      }
+      ExprMap[E] = Exprs.add(Expr);
+      return true;
+    }
+    bool visitExprVar(const nix::ExprVar *E) {
+      SimpleExpr Expr;
+      Expr.Kind = ExprKind::EK_Var;
+      Expr.Body.Var.Displ = E->displ;
+      Expr.Body.Var.FromWidth = E->fromWith;
+      Expr.Body.Var.Name = Strings.add(STable[E->name]);
+      Expr.Body.Var.Pos = Pos.add(getPos(E->pos));
+      ExprMap[E] = Exprs.add(Expr);
+      return true;
+    }
+
+  } Writer(Exprs, Pos, Strings, STable, PTable);
+  Writer.traverseExpr(E);
+
+  return sizeof(ASTHeader) + getSize(Exprs) + getSize(Pos) +
+         Strings.getLength();
+}
+
+void ASTWriter::write(char *Dst) {
+  ASTHeader Header;
+  Header.Version = 1;
+  Header.ExprTableOffset = sizeof(ASTHeader);
+  Header.PosTableOffset = Header.ExprTableOffset + getSize(Exprs);
+  Header.StringTableOffset = Header.PosTableOffset + getSize(Pos);
+  memcpy(Dst, &Header, sizeof(ASTHeader));
+  Dst += sizeof(ASTHeader);
+  Dst += encode(Dst, Exprs);
+  Dst += encode(Dst, Pos);
+  Strings.write(Dst);
+}
+
+std::size_t encode(char *Dst, const ExprAttrSet &E) {
+  std::size_t Ret = encode(Dst, E.Kind);
+  Ret += encode(Dst + Ret, E.Pos);
+  Ret += encode(Dst + Ret, E.Attrs);
+  Ret += encode(Dst + Ret, E.DynamicAttrs);
+  return Ret;
+}
+
+std::size_t getSize(const ExprAttrSet &E) {
+  return sizeof(E.Kind) + getSize(E.Pos) + getSize(E.Attrs) +
+         getSize(E.DynamicAttrs);
+}
+
+} // namespace nixt::serialize

libnixt/meson.build

@@ -10,6 +10,7 @@ libnixt = library('nixt',
                   'lib/Displacement.cpp',
                   'lib/Name.cpp',
                   'lib/ParentMap.cpp',
+                  'lib/Serialize.cpp',
                   include_directories: libnixdInc,
                   dependencies: libnixtDeps,
                   install: true