Remove token and token? from tree

docs/serialization.md

@@ -31,8 +31,6 @@ Each node's child is then appended to the serialized string. The child node type
 * `node?` - A child node that is optionally present. If the node is not present, then a single `0` byte will be written in its place. If it is present, then it will be structured just as like parent node.
 * `node[]` - A child node that is an array of nodes. This is structured as a variable-length integer length, followed by the child nodes themselves.
 * `string` - A child node that is a string. For example, this is used as the name of the method in a call node, since it cannot directly reference the source string (as in `@-` or `foo=`). This is structured as a variable-length integer byte length, followed by the string itself (_without_ a trailing null byte).
-* `token` - A child node that is a token. This is structured as a single byte type, followed by two variable-length integers that represent offsets into the source string.
-* `token?` - A child node that is a token that is optionally present. If the token is not present, then a single `0` byte will be written in its place. If it is present, then it will be structured just like the `token` child node.
 * `constant` - A variable-length integer that represents an index in the constant pool.
 * `constant[]` - A child node that is an array of constants. This is structured as a variable-length integer length, followed by the child constants themselves.
 * `location` - A child node that is a location. This is structured as a variable-length integer start followed by a variable-length integer length.

templates/ext/yarp/api_node.c.erb

@@ -53,10 +53,6 @@ yp_node_new(yp_parser_t *parser, yp_node_t *node, rb_encoding *encoding, ID *con
             }
             <%- when StringParam -%>
             argv[<%= index %>] = yp_string_new(&cast-><%= param.name %>, encoding);
-            <%- when TokenParam -%>
-            argv[<%= index %>] = yp_token_new(parser, &cast-><%= param.name %>, encoding);
-            <%- when OptionalTokenParam -%>
-            argv[<%= index %>] = cast-><%= param.name %>.type == YP_TOKEN_NOT_PROVIDED ? Qnil : yp_token_new(parser, &cast-><%= param.name %>, encoding);
             <%- when LocationListParam -%>
             argv[<%= index %>] = rb_ary_new();
             for (size_t index = 0; index < cast-><%= param.name %>.size; index++) {

templates/include/yarp/ast.h.erb

@@ -72,7 +72,6 @@ typedef struct yp_<%= node.human %> {
     <%= case param
     in NodeParam | OptionalNodeParam then "struct #{param.c_type} *#{param.name}"
     in NodeListParam then "struct yp_node_list #{param.name}"
-    in TokenParam | OptionalTokenParam then "yp_token_t #{param.name}"
     in LocationListParam then "yp_location_list_t #{param.name}"
     in ConstantParam then "yp_constant_id_t #{param.name}"
     in ConstantListParam then "yp_constant_id_list_t #{param.name}"

templates/java/org/yarp/Loader.java.erb

@@ -78,15 +78,6 @@ public class Loader {
         return string;
     }
 
-    private Nodes.Token loadOptionalToken() {
-        if (buffer.get(buffer.position()) != 0) {
-            return loadToken();
-        } else {
-            buffer.position(buffer.position() + 1); // continue after the 0 byte
-            return null;
-        }
-    }
-
     private Nodes.Node loadOptionalNode() {
         if (buffer.get(buffer.position()) != 0) {
             return loadNode();
@@ -127,12 +118,6 @@ public class Loader {
         return nodes;
     }
 
-    private Nodes.Token loadToken() {
-        int type = buffer.get() & 0xFF;
-        final Nodes.TokenType tokenType = Nodes.TOKEN_TYPES[type];
-        return new Nodes.Token(tokenType, loadVarInt(), loadVarInt());
-    }
-
     private Nodes.Location loadLocation() {
         return new Nodes.Location(loadVarInt(), loadVarInt());
     }
@@ -179,11 +164,9 @@ public class Loader {
               when OptionalNodeParam then "#{param.java_cast}loadOptionalNode()"
               when StringParam then "loadString()"
               when NodeListParam then "loadNodes()"
-              when TokenParam then "loadToken()"
               when LocationListParam then "loadLocations()"
               when ConstantParam then "loadConstant()"
               when ConstantListParam then "loadConstants()"
-              when OptionalTokenParam then "loadOptionalToken()"
               when LocationParam then "loadLocation()"
               when OptionalLocationParam then "loadOptionalLocation()"
               when UInt32Param then "loadVarInt()"

templates/java/org/yarp/Nodes.java.erb

@@ -10,15 +10,6 @@ import java.util.Arrays;
 // @formatter:off
 public abstract class Nodes {
 
-    public enum TokenType {
-        OPTIONAL_TOKEN,
-        <%- tokens.each do |token| -%>
-        <%= token.name %>,
-        <%- end -%>
-    }
-
-    static final TokenType[] TOKEN_TYPES = TokenType.values();
-
 <%- flags.each do |group| -%>
     public static final class <%= group.name %> implements Comparable<<%= group.name %>> {
 
@@ -66,25 +57,6 @@ public abstract class Nodes {
     }
 
 <%- end -%>
-    public static final class Token {
-        public final TokenType type;
-        public final int startOffset;
-        public final int length;
-
-        public Token(TokenType type, int startOffset, int length) {
-            this.type = type;
-            this.startOffset = startOffset;
-            this.length = length;
-        }
-
-        public int length() {
-            return length;
-        }
-
-        public int endOffset() {
-            return startOffset + length;
-        }
-    }
 
     public static final class Location {
         public final int startOffset;

templates/lib/yarp/node.rb.erb

@@ -70,12 +70,5 @@ module YARP
       <%= node.name %>.new(<%= (node.params.map(&:name) + ["0", "0"]).join(", ") %>)
     end
     <%- end -%>
-    <%- tokens.each do |token| -%>
-
-    # Create a new <%= token.name %> token
-    def <%= token.name %>(value, location = Location.null)
-      Token.new(:<%= token.name %>, value, location.start_offset, location.length)
-    end
-    <%- end -%>
   end
 end

templates/lib/yarp/serialize.rb.erb

@@ -54,33 +54,13 @@ module YARP
         io.read(4).unpack1("L")
       end
 
-      def load_token
-        type =
-          case io.getbyte
-          <%- tokens.each_with_index do |token, index| -%>
-          when <%= index + 1 %> then :<%= token.name.inspect %>
-          <%- end -%>
-          end
-
-        start_offset, length = load_varint, load_varint
-        range = source.byteslice(start_offset, length)
-        Token.new(type, range, start_offset, length)
-      end
-
       def load_optional_node
         if io.getbyte != 0
           io.pos -= 1
           load_node
         end
       end
 
-      def load_optional_token
-        if io.getbyte != 0
-          io.pos -= 1
-          load_token
-        end
-      end
-
       def load_string
         io.read(load_varint).force_encoding(encoding)
       end
@@ -126,11 +106,9 @@ module YARP
             when OptionalNodeParam then "load_optional_node"
             when StringParam then "load_string"
             when NodeListParam then "Array.new(load_varint) { load_node }"
-            when TokenParam then "load_token"
             when LocationListParam then "Array.new(load_varint) { load_location }"
             when ConstantParam then "load_constant"
             when ConstantListParam then "Array.new(load_varint) { load_constant }"
-            when OptionalTokenParam then "load_optional_token"
             when LocationParam then "load_location"
             when OptionalLocationParam then "load_optional_location"
             when UInt32Param then "load_varint"

templates/src/node.c.erb

@@ -81,7 +81,7 @@ yp_node_destroy(yp_parser_t *parser, yp_node_t *node) {
         case <%= node.type %>:
             <%- node.params.each do |param| -%>
             <%- case param -%>
-            <%- when TokenParam, OptionalTokenParam, LocationParam, OptionalLocationParam, UInt32Param, ConstantParam -%>
+            <%- when LocationParam, OptionalLocationParam, UInt32Param, ConstantParam -%>
             <%- when NodeParam -%>
             yp_node_destroy(parser, (yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>);
             <%- when OptionalNodeParam -%>
@@ -121,7 +121,7 @@ yp_node_memsize_node(yp_node_t *node, yp_memsize_t *memsize) {
             memsize->memsize += sizeof(yp_<%= node.human %>_t);
             <%- node.params.each do |param| -%>
             <%- case param -%>
-            <%- when TokenParam, OptionalTokenParam, ConstantParam, UInt32Param, LocationParam, OptionalLocationParam -%>
+            <%- when ConstantParam, UInt32Param, LocationParam, OptionalLocationParam -%>
             <%- when NodeParam -%>
             yp_node_memsize_node((yp_node_t *)((yp_<%= node.human %>_t *)node)-><%= param.name %>, memsize);
             <%- when OptionalNodeParam -%>

templates/src/prettyprint.c.erb

@@ -37,14 +37,6 @@ prettyprint_node(yp_buffer_t *buffer, yp_parser_t *parser, yp_node_t *node) {
                 if (index != 0) yp_buffer_append_str(buffer, ", ", 2);
                 prettyprint_node(buffer, parser, (yp_node_t *) ((yp_<%= node.human %>_t *) node)-><%= param.name %>.nodes[index]);
             }
-            <%- when TokenParam -%>
-            prettyprint_token(buffer, &((yp_<%= node.human %>_t *)node)-><%= param.name %>);
-            <%- when OptionalTokenParam -%>
-            if (((yp_<%= node.human %>_t *)node)-><%= param.name %>.type == YP_TOKEN_NOT_PROVIDED) {
-                yp_buffer_append_str(buffer, "nil", 3);
-            } else {
-                prettyprint_token(buffer, &((yp_<%= node.human %>_t *)node)-><%= param.name %>);
-            }
             <%- when LocationListParam -%>
             for (uint32_t index = 0; index < ((yp_<%= node.human %>_t *)node)-><%= param.name %>.size; index++) {
                 if (index != 0) yp_buffer_append_str(buffer, ", ", 2);

templates/src/serialize.c.erb

@@ -63,14 +63,6 @@ yp_serialize_node(yp_parser_t *parser, yp_node_t *node, yp_buffer_t *buffer) {
             for (uint32_t index = 0; index < <%= param.name %>_size; index++) {
                 yp_serialize_node(parser, (yp_node_t *) ((yp_<%= node.human %>_t *)node)-><%= param.name %>.nodes[index], buffer);
             }
-            <%- when TokenParam -%>
-            serialize_token(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer);
-            <%- when OptionalTokenParam -%>
-            if (((yp_<%= node.human %>_t *)node)-><%= param.name %>.type == YP_TOKEN_NOT_PROVIDED) {
-                yp_buffer_append_u8(buffer, 0);
-            } else {
-                serialize_token(parser, &((yp_<%= node.human %>_t *)node)-><%= param.name %>, buffer);
-            }
             <%- when LocationListParam -%>
             uint32_t <%= param.name %>_size = yp_ulong_to_u32(((yp_<%= node.human %>_t *)node)-><%= param.name %>.size);
             yp_buffer_append_u32(buffer, <%= param.name %>_size);

templates/template.rb

@@ -57,19 +57,6 @@ def rbs_class = "Array[Node]"
   def java_type = "Node[]"
 end
 
-# This represents a parameter to a node that is a token. We pass them as
-# references and store them by copying.
-class TokenParam < Param
-  def rbs_class = "Token"
-  def java_type = "Token"
-end
-
-# This represents a parameter to a node that is a token that is optional.
-class OptionalTokenParam < Param
-  def rbs_class = "Token?"
-  def java_type = "Token"
-end
-
 # This represents a parameter to a node that is a list of locations.
 class LocationListParam < Param
   def rbs_class = "Array[Location]"
@@ -119,8 +106,6 @@ def java_type = "int"
   "node?" => OptionalNodeParam,
   "node[]" => NodeListParam,
   "string" => StringParam,
-  "token" => TokenParam,
-  "token?" => OptionalTokenParam,
   "location[]" => LocationListParam,
   "constant" => ConstantParam,
   "constant[]" => ConstantListParam,