From b0c3129d85cd94652e52043ddab5c90cdd379e17 Mon Sep 17 00:00:00 2001 From: Tao He Date: Thu, 18 Jan 2024 13:12:37 +0800 Subject: [PATCH] Packed nbr_unit_t to decrease the memory footprint when using int32_t as vid_t (#1715) Fixes #1714 Signed-off-by: Tao He --- k8s/cmd/main.go | 2 +- modules/graph/CMakeLists.txt | 17 ++++++++++ modules/graph/README.rst | 12 +++++++ modules/graph/fragment/property_graph_types.h | 34 ++++++++++++++----- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/k8s/cmd/main.go b/k8s/cmd/main.go index a783c306..62d89ca5 100644 --- a/k8s/cmd/main.go +++ b/k8s/cmd/main.go @@ -48,7 +48,7 @@ var cmdLong = util.LongDesc(` var cmd = &cobra.Command{ Use: "vineyardctl [command]", - Version: "v0.19.3", + Version: "v0.20.0", Short: "vineyardctl is the command-line tool for interact with the Vineyard Operator.", Long: cmdLong, } diff --git a/modules/graph/CMakeLists.txt b/modules/graph/CMakeLists.txt index 781b11c6..2529a090 100644 --- a/modules/graph/CMakeLists.txt +++ b/modules/graph/CMakeLists.txt @@ -1,5 +1,18 @@ include(GNUInstallDirs) +# options to control several default behaviors of vineyard graph +option(VINEYARD_GRAPH_MAX_LABEL_ID "Maximum label id value of vineyard graphs, defaults to 128" OFF) + +option(MY_OPTION "Description of My Option" OFF) +set(VINEYARD_GRAPH_MAX_LABEL_ID_DEFAULT_VALUE 128 CACHE STRING "Default value for maximum label if for vineyard graphs") +set_property(CACHE VINEYARD_GRAPH_MAX_LABEL_ID PROPERTY STRINGS "1;2;4;8;16;32;64;128") + +if (VINEYARD_GRAPH_MAX_LABEL_ID) + message(STATUS "Setting VINEYARD_GRAPH_MAX_LABEL_ID to '${VINEYARD_GRAPH_MAX_LABEL_ID}'") +else() + message(STATUS "Setting VINEYARD_GRAPH_MAX_LABEL_ID to default value: '${VINEYARD_GRAPH_MAX_LABEL_ID_DEFAULT_VALUE}'") +endif() + # build vineyard-graph file(GLOB_RECURSE VINEYARD_MOD_SRCS "${CMAKE_CURRENT_SOURCE_DIR}" "*.vineyard-mod") @@ -137,6 +150,10 @@ if(BUILD_VINEYARD_GRAPH_WITH_GAR) endif() endif() +if (VINEYARD_GRAPH_MAX_LABEL_ID) + target_compile_options(vineyard_graph PUBLIC -DVINEYARD_GRAPH_MAX_LABEL_ID=${VINEYARD_GRAPH_MAX_LABEL_ID}) +endif() + target_link_libraries(vineyard_graph PUBLIC vineyard_client vineyard_basic vineyard_io diff --git a/modules/graph/README.rst b/modules/graph/README.rst index eaa23fb5..7407541a 100644 --- a/modules/graph/README.rst +++ b/modules/graph/README.rst @@ -19,6 +19,18 @@ among graph computing engines. * `Read Options <#read-options>`_ * `Global Options <#global-options>`_ +CMake configure options +----------------------- + +- :code:`VINEYARD_GRAPH_MAX_LABEL_ID` + + The internal vertex id (aka. :code:`VID`) in vineyard is encoded as fragment id, vertex label id + and vertex offset. The option :code:`VINEYARD_GRAPH_MAX_LABEL_ID` decides the bit field width of + label id in :code:`VID`. Decreasing this value can be helpful to support larger number of vertices + when using :code:`int32_t` as :code:`VID_T`. + + Defaults to `128`, can be `1`, `2`, `4`, `8`, `16`, `32`, `64`, or `128`. + vineyard-graph-loader --------------------- diff --git a/modules/graph/fragment/property_graph_types.h b/modules/graph/fragment/property_graph_types.h index cec97ac1..3622e78c 100644 --- a/modules/graph/fragment/property_graph_types.h +++ b/modules/graph/fragment/property_graph_types.h @@ -80,8 +80,11 @@ using LABEL_ID_TYPE = int; } // namespace property_graph_types -// Hardcoded the max vertex label num to 128 +#if defined(VINEYARD_GRAPH_MAX_LABEL_ID) +constexpr int MAX_VERTEX_LABEL_NUM = VINEYARD_GRAPH_MAX_LABEL_ID; +#else constexpr int MAX_VERTEX_LABEL_NUM = 128; +#endif static inline int num_to_bitwidth(int num) { if (num <= 2) { @@ -104,13 +107,11 @@ static inline int num_to_bitwidth(int num) { */ template class IdParser { - using LabelIDT = int; // LABEL_ID_TYPE - public: IdParser() {} ~IdParser() {} - void Init(fid_t fnum, LabelIDT label_num) { + void Init(fid_t fnum, property_graph_types::LABEL_ID_TYPE label_num) { CHECK_LE(label_num, MAX_VERTEX_LABEL_NUM); int fid_width = num_to_bitwidth(fnum); fid_offset_ = (sizeof(ID_TYPE) * 8) - fid_width; @@ -125,7 +126,7 @@ class IdParser { fid_t GetFid(ID_TYPE v) const { return (v >> fid_offset_); } - LabelIDT GetLabelId(ID_TYPE v) const { + property_graph_types::LABEL_ID_TYPE GetLabelId(ID_TYPE v) const { return (v & label_id_mask_) >> label_id_offset_; } @@ -133,10 +134,13 @@ class IdParser { ID_TYPE GetLid(ID_TYPE v) const { return v & lid_mask_; } + ID_TYPE GetMaxOffset() const { return offset_mask_; } + /** * @brief Generate the LID */ - ID_TYPE GenerateId(LabelIDT label, int64_t offset) const { + ID_TYPE GenerateId(property_graph_types::LABEL_ID_TYPE label, + int64_t offset) const { return (((ID_TYPE) offset) & offset_mask_) | ((((ID_TYPE) label) << label_id_offset_) & label_id_mask_); } @@ -144,7 +148,8 @@ class IdParser { /** * @brief Generate the GID */ - ID_TYPE GenerateId(fid_t fid, LabelIDT label, int64_t offset) const { + ID_TYPE GenerateId(fid_t fid, property_graph_types::LABEL_ID_TYPE label, + int64_t offset) const { return (((ID_TYPE) offset) & offset_mask_) | ((((ID_TYPE) label) << label_id_offset_) & label_id_mask_) | ((((ID_TYPE) fid) << fid_offset_) & fid_mask_); @@ -170,13 +175,26 @@ struct NbrUnit { VID_T vid; EID_T eid; + NbrUnit() = default; NbrUnit(VID_T v, EID_T e) : vid(v), eid(e) {} grape::Vertex get_neighbor() const { return grape::Vertex(vid); } -}; +} __attribute__((packed, aligned(4))); + +static_assert(alignof(NbrUnit) == 4, + "int32_t vid alignment check"); +static_assert(sizeof(NbrUnit) == + sizeof(int32_t) + sizeof(uint64_t), + "int32_t vid size check"); + +static_assert(alignof(NbrUnit) == 4, + "int64_t vid alignment check"); +static_assert(sizeof(NbrUnit) == + sizeof(int64_t) + sizeof(uint64_t), + "int64_t vid size check"); template using NbrUnitDefault = NbrUnit;