BUILD: fix regcache alignment (openucx#877)

janjust · Jan 31, 2024 · dcf8716 · dcf8716
1 parent 54958ab
commit dcf8716
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 9 deletions.
diff --git a/config/m4/ucx.m4 b/config/m4/ucx.m4
@@ -113,6 +113,11 @@ AS_IF([test "x$ucx_checked" != "xyes"],[
                 [AC_DEFINE([UCS_HAVE_CONFIG_GLOBAL_LIST_ENTRY_FLAGS], [1], [flags for config table])],
                 [],
                 [#include <ucs/config/parser.h>])
+
+            AC_CHECK_MEMBER(ucs_rcache_region_t.alignment,
+                [AC_DEFINE([UCS_HAVE_RCACHE_REGION_ALIGNMENT], [1], [flags for ucs_rcache_get])],
+                [],
+                [#include <ucs/memory/rcache.h>])
         ],
         [
             AS_IF([test "x$with_ucx" != "xguess"],

diff --git a/src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c b/src/components/tl/mlx5/mcast/tl_mlx5_mcast_rcache.c
@@ -19,7 +19,7 @@ static ucs_status_t ucc_tl_mlx5_mcast_coll_reg_mr(ucc_tl_mlx5_mcast_coll_context
         tl_error(ctx->lib, "failed to register MR");
         return UCS_ERR_NO_MEMORY;
     }
-    
+
     return UCS_OK;
 }
 
@@ -33,7 +33,7 @@ static ucc_status_t ucc_tl_mlx5_mcast_coll_dereg_mr(ucc_tl_mlx5_mcast_coll_conte
     }
 
     tl_debug(ctx->lib, "external memory deregister: mr %p", mr);
-    
+
     if (ibv_dereg_mr(mr)) {
         tl_error(ctx->lib, "couldn't destroy mr %p", mr);
         return UCC_ERR_NO_RESOURCE;
@@ -140,12 +140,10 @@ ucc_status_t ucc_tl_mlx5_mcast_setup_rcache(ucc_tl_mlx5_mcast_coll_context_t *ct
 {
     ucc_rcache_params_t rcache_params;
 
-    rcache_params.alignment          = 64;
     rcache_params.ucm_event_priority = 1000;
     rcache_params.max_regions        = ULONG_MAX;
     rcache_params.max_size           = SIZE_MAX;
     rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_mcast_rcache_region_t);
-    rcache_params.max_alignment      = ucc_get_page_size();
     rcache_params.ucm_events         = UCM_EVENT_VM_UNMAPPED |
                                        UCM_EVENT_MEM_TYPE_FREE;
     rcache_params.context            = ctx;

diff --git a/src/components/tl/mlx5/tl_mlx5_rcache.c b/src/components/tl/mlx5/tl_mlx5_rcache.c
@@ -63,8 +63,6 @@ ucc_status_t tl_mlx5_rcache_create(ucc_tl_mlx5_context_t *ctx)
     ucc_rcache_params_t     rcache_params;
 
     rcache_params.region_struct_size = sizeof(ucc_tl_mlx5_rcache_region_t);
-    rcache_params.alignment          = UCS_PGT_ADDR_ALIGN;
-    rcache_params.max_alignment      = ucc_get_page_size();
     rcache_params.ucm_event_priority = 1000;
     rcache_params.context            = (void *)ctx;
     rcache_params.ops                = &ucc_rcache_ops;

diff --git a/src/components/tl/sharp/tl_sharp_context.c b/src/components/tl/sharp/tl_sharp_context.c
@@ -269,12 +269,10 @@ ucc_status_t ucc_tl_sharp_rcache_create(struct sharp_coll_context *context,
 {
     ucc_rcache_params_t rcache_params;
 
-    rcache_params.alignment          = 64;
     rcache_params.ucm_event_priority = 1000;
     rcache_params.max_regions        = ULONG_MAX;
     rcache_params.max_size           = SIZE_MAX;
     rcache_params.region_struct_size = sizeof(ucc_tl_sharp_rcache_region_t);
-    rcache_params.max_alignment      = ucc_get_page_size();
     rcache_params.ucm_events         = UCM_EVENT_VM_UNMAPPED |
         UCM_EVENT_MEM_TYPE_FREE;
     rcache_params.context            = context;

diff --git a/src/utils/ucc_rcache.h b/src/utils/ucc_rcache.h
@@ -8,6 +8,7 @@
 
 #include <ucs/memory/rcache.h>
 #include <ucm/api/ucm.h>
+#include <utils/ucc_sys.h>
 
 //TODO: handle external events
 #define ucc_rcache_t                 ucs_rcache_t
@@ -25,8 +26,17 @@ static inline ucc_status_t
 ucc_rcache_create(const ucc_rcache_params_t *params,
                   const char *name, ucc_rcache_t **rcache_p)
 {
+#ifndef UCS_HAVE_RCACHE_REGION_ALIGNMENT
+    ucc_rcache_params_t params_dup = *params;
+    params_dup.alignment     = UCS_PGT_ADDR_ALIGN;
+    params_dup.max_alignment = ucc_get_page_size();
+
+    return ucs_status_to_ucc_status(ucs_rcache_create(
+                                    &params_dup, name, NULL, rcache_p));
+#else
     return ucs_status_to_ucc_status(ucs_rcache_create(
-                                        params, name, NULL, rcache_p));
+                                    params, name, NULL, rcache_p));
+#endif
 }
 
 /* [arg] parameter allows passing additional information from mem_reg callabck.
@@ -36,9 +46,16 @@ static inline ucc_status_t
 ucc_rcache_get(ucc_rcache_t *rcache, void *address, size_t length,
                void *arg, ucc_rcache_region_t **region_p)
 {
+#ifdef UCS_HAVE_RCACHE_REGION_ALIGNMENT
     return ucs_status_to_ucc_status(ucs_rcache_get(
                                        rcache, address, length,
+                                       ucc_get_page_size(),
                                        PROT_READ | PROT_WRITE, arg, region_p));
+#else
+    return ucs_status_to_ucc_status(ucs_rcache_get(
+                                       rcache, address, length,
+                                       PROT_READ | PROT_WRITE, arg, region_p));
+#endif
 }
 
 #endif