picoproto.h

/* Copyright 2016 Pete Warden. All Rights Reserved.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

 http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/

/*
 See the README for full details, but this module lets you read in protobuf
 encoded files with a minimal code footprint.
 
 It doesn't create classes for each kind of message it encounters, it just has a
 single Message interface that lets you access the members of the protobuf as a
 key/value store. This loses a lot of the convenience of type-checked classes,
 but it does mean that very little code is needed to access data from files.
 
 As a simple example, if you had read a `bytes_size` long file into `bytes` that
 contained a TensorFlow GraphDef proto:
 
 Message graph_def;
 graph_def.ParseFromBytes(bytes, bytes_size);
 
 You can then access the different fields of the GraphDef using the field
 numbers assigned in the .proto file:
 
 std::vector<picoproto::Message*> nodes = graph_def.GetMessageArray(1);
 
 One big difference between this minimal approach and normal protobufs is that
 the calling code has to already know the field number and type of any members
 it's trying to access. Here I know that the `node` field is number 1, and that
 it should contain a repeated list of NodeDefs. Since they are not primitive
 types like numbers or strings, they are accessed as an array of Messages.
 
 Here are the design goals of this module:
  - Keep the code size tiny (single-digit kilobytes on most platforms).
  - Minimize memory usage (for example allow in-place references to byte data).
  - Provide a simple, readable implementation that can be ported easily.
  - Deserialize all saved protobuf files into a usable representation.
  - No dependencies other than the standard C++ library.
  - No build-time support (e.g. protoc) required.
 
 Here's what it's explicitly not offering:
  - Providing a readable and transparent way of accessing serialized data.
  - Saving out data to protobuf format.
 
*/

#ifndef INCLUDE_PICOPROTO_H
#define INCLUDE_PICOPROTO_H

#include <cstddef>
#include <cstdint>
#include <iostream>
#include <map>
#include <string>
#include <vector>

// To keep dependencies minimal, some bare-bones macros to make logging easier.
#define PP_LOG(X) PP_LOG_##X
#define PP_LOG_INFO std::cerr << __FILE__ << ":" << __LINE__ << " - INFO: "
#define PP_LOG_WARN std::cerr << __FILE__ << ":" << __LINE__ << " - WARN: "
#define PP_LOG_ERROR std::cerr << __FILE__ << ":" << __LINE__ << " - ERROR: "
#define PP_CHECK(X) \
  if (!(X)) PP_LOG(ERROR) << "PP_CHECK(" << #X << ") failed. "

namespace picoproto {

// These roughly correspond to the wire types used to save data in protobuf
// files. The best reference to understand the full format is:
// https://developers.google.com/protocol-buffers/docs/encoding
// Because we don't know the bit-depth of VarInts, they're always stored as
// uint64 values, which is why there's no specific type for them.
enum FieldType {
  FIELD_UNSET,
  FIELD_UINT32,
  FIELD_UINT64,
  FIELD_BYTES,
};

// Gives a readable name for the field type for logging purposes.
std::string FieldTypeDebugString(enum FieldType type);

// Forward declare the main message class, since fields can contain them.
class Message;

// Fields are the building blocks of messages. They contain the values for each
// data member, and handle all the allocation and deallocation of storage.
// It's unlikely you'll want to access this class directly, since you'll
// normally want to use Message below to pull typed values.
class Field {
 public:
  // You need to specify the type of a Field on creation, so that the right
  // storage can be set up for the values. You also need to indicate whether the
  // underlying memory will be around for the lifetime of the message (in which
  // case no copies are needed) or whether the class should make copies and take
  // ownership in case the data goes away.
  Field(FieldType type, bool owns_data);
  Field(const Field& other);
  ~Field();

  enum FieldType type;
  // I know, this isn't very OOP, but the simplicity of keeping track of a type
  // and deciding how to initialize and access the data based on that persuaded
  // me this was the best approach. The `value` member contains whatever data
  // the field should be holding.
  union {
    std::vector<uint32_t>* v_uint32;
    std::vector<uint64_t>* v_uint64;
    std::vector<std::pair<uint8_t*, size_t>>* v_bytes;
  } value;
  // One of the drawbacks of not requiring .proto files ahead of time is that I
  // don't know if a length-delimited field contains raw bytes, strings, or
  // sub-messages. The only time we know that a field should be interpreted as a
  // message is when client code requests it in that form. Because parsing can
  // be costly, here we cache the results of any such calls for subsequent
  // accesses.
  std::vector<Message*>* cached_messages;
  // If this is set, then the object will allocate its own storage for
  // length-delimited values, and copy from the input stream. If you know the
  // underlying data will be around for the lifetime of the message, you can
  // save memory and copies by leaving this as false.
  bool owns_data;
};

// The main interface for loading and accessing serialized protobuf data.
class Message {
 public:
  // If you're not sure about the lifetime of any binary data you're reading
  // from, just call this default constructor.
  Message();
  // In the case when you're sure the lifetime of the byte stream you'll be
  // decoding is longer than the lifetime of the message, you can set
  // copy_arrays to false. This is especially useful if you have a memory
  // mapped file to read from containing large binary blobs, since you'll skip
  // a lot of copying and extra allocation.
  Message(bool copy_arrays);
  Message(const Message& other);
  ~Message();

  // Populates fields with all of the data from this stream of bytes.
  // You can call this repeatedly with new messages, and the results will be
  // merged together.
  bool ParseFromBytes(uint8_t* binary, size_t binary_size);

  // These are the accessor functions if you're expecting exactly one value in a
  // field. As discussed above, the burden is on the client code to know the
  // field number and type of each member it's trying to access, and so pick the
  // correct accessor function.
  // If the field isn't present, this will raise an error, so if it's optional
  // you should use the array accessors below.
  int32_t GetInt32(int32_t number);
  int64_t GetInt64(int32_t number);
  uint32_t GetUInt32(int32_t number);
  uint64_t GetUInt64(int32_t number);
  int64_t GetInt(int32_t number);
  bool GetBool(int32_t number);
  float GetFloat(int32_t number);
  double GetDouble(int32_t number);
  std::pair<uint8_t*, size_t> GetBytes(int32_t number);
  std::string GetString(int32_t number);
  Message* GetMessage(int32_t number);

  // If you're not sure if a value will be present, or if it is repeated, you
  // should call these array functions. If no such field has been seen, then the
  // result will be an empty vector, otherwise you'll get back one or more
  // entries.
  std::vector<int32_t> GetInt32Array(int32_t number);
  std::vector<int64_t> GetInt64Array(int32_t number);
  std::vector<uint32_t> GetUInt32Array(int32_t number);
  std::vector<uint64_t> GetUInt64Array(int32_t number);
  std::vector<bool> GetBoolArray(int32_t number);
  std::vector<float> GetFloatArray(int32_t number);
  std::vector<double> GetDoubleArray(int32_t number);
  std::vector<std::pair<uint8_t*, size_t>> GetByteArray(int32_t number);
  std::vector<std::string> GetStringArray(int32_t number);
  std::vector<Message*> GetMessageArray(int32_t number);

  // It's unlikely you'll want to access fields directly, but here's an escape
  // hatch in case you do have to manipulate them more directly.
  Field* GetField(int32_t number);

private:

  // Inserts a new field, updating all the internal data structures.
  Field* AddField(int32_t number, enum FieldType type);
  
  Field* GetFieldAndCheckType(int32_t number, enum FieldType type);
  
  // Maps from a field number to an index in the `fields` vector.
  std::map<int32_t, size_t> field_map;
  // The core list of fields that have been parsed.
  std::vector<Field> fields;
  bool copy_arrays;
};

}  // namespace picoproto

#endif  // INCLUDE_PICOPROTO_H