-
Notifications
You must be signed in to change notification settings - Fork 0
/
0.11.1..0a4c530bca5dc2476418a3833a15ab812d636e78.patch
2371 lines (2253 loc) · 87.2 KB
/
0.11.1..0a4c530bca5dc2476418a3833a15ab812d636e78.patch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
diff --git a/.gitignore b/.gitignore
index ee2bfff..f7d108c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -71,3 +71,6 @@ doc/build
# spack files
.spack*
spack.lock
+
+# Code editors
+.vscode
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4880124..a559722 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -57,6 +57,12 @@ IF(ENABLE_LUSTRE)
# - LUSTRE_STAT
ENDIF(ENABLE_LUSTRE)
+OPTION(ENABLE_HPSS "Enable optimization and features for HPSSFS-FUSE" OFF)
+MESSAGE(STATUS "ENABLE_HPSS: ${ENABLE_HPSS}")
+IF(ENABLE_HPSS)
+ ADD_DEFINITIONS(-DHPSS_SUPPORT)
+ENDIF(ENABLE_HPSS)
+
OPTION(ENABLE_GPFS "Enable GFPS/Spectrum Scale support")
MESSAGE(STATUS "ENABLE_GPFS: ${ENABLE_GPFS}")
IF(ENABLE_GPFS)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d7c9a01..4ffcf16 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -3,17 +3,12 @@
*Thank you for taking the time to contribute!*
## Table of Contents
-[Resources](#resources)
-
[How To Contribute](#how-to-contribute)
* [Reporting an Issue & Feature Suggestions](#reporting-an-issue--feature-suggestions)
* [Pull Requests](#pull-requests)
* [License](#license)
* [Contributor's Declaration](#contributors-declaration)
-## Resources
-[mpiFileUtils Google Group](https://groups.google.com/forum/#!forum/mpifileutils)
-
## How To Contribute
### Reporting an Issue & Feature Suggestions
diff --git a/dist/CMakeLists.txt b/dist/CMakeLists.txt
index 3b6c581..102d1b5 100644
--- a/dist/CMakeLists.txt
+++ b/dist/CMakeLists.txt
@@ -57,6 +57,12 @@ IF(ENABLE_LUSTRE)
# - LUSTRE_STAT
ENDIF(ENABLE_LUSTRE)
+OPTION(ENABLE_HPSS "Enable optimization and features for HPSSFS-FUSE" OFF)
+MESSAGE(STATUS "ENABLE_HPSS: ${ENABLE_HPSS}")
+IF(ENABLE_HPSS)
+ ADD_DEFINITIONS(-DHPSS_SUPPORT)
+ENDIF(ENABLE_HPSS)
+
OPTION(ENABLE_GPFS "Enable GFPS/Spectrum Scale support")
MESSAGE(STATUS "ENABLE_GPFS: ${ENABLE_GPFS}")
IF(ENABLE_GPFS)
diff --git a/doc/rst/build.rst b/doc/rst/build.rst
index 047ba87..81bdf76 100644
--- a/doc/rst/build.rst
+++ b/doc/rst/build.rst
@@ -34,7 +34,7 @@ To be certain of compatibility, it is recommended that one install libarchive-3.
mkdir install
installdir=`pwd`/install
- wget https://github.com/libarchive/libarchive/releases/download/3.5.1/libarchive-3.5.1.tar.gz
+ wget https://github.com/libarchive/libarchive/releases/download/v3.5.1/libarchive-3.5.1.tar.gz
tar -zxf libarchive-3.5.1.tar.gz
cd libarchive-3.5.1
./configure --prefix=$installdir
@@ -66,6 +66,7 @@ Additional CMake options:
* :code:`-DENABLE_XATTRS=[ON/OFF]` : use extended attributes and libattr, defaults to :code:`ON`
* :code:`-DENABLE_LUSTRE=[ON/OFF]` : specialization for Lustre, defaults to :code:`OFF`
* :code:`-DENABLE_GPFS=[ON/OFF]` : specialization for GPFS, defaults to :code:`OFF`
+* :code:`-DENABLE_HPSS=[ON/OFF]` : specialization for HPSS, defaults to :code:`OFF`
* :code:`-DENABLE_EXPERIMENTAL=[ON/OFF]` : build experimental tools, defaults to :code:`OFF`
-------------------------------------------
@@ -102,7 +103,7 @@ Spack
To use `Spack <https://github.com/spack/spack>`_, it is recommended that one first create a `packages.yaml` file to list system-provided packages, like MPI.
Without doing this, Spack will fetch and install an MPI library that may not work on your system.
-Make sure that you've set up spack in your shell (see `these instructions <https://spack.readthedocs.io/en/latest/getting_started.html>`_).
+Make sure that you've set up Spack in your shell (see `these instructions <https://spack.readthedocs.io/en/latest/getting_started.html>`_).
Once Spack has been configured, mpiFileUtils can be installed as:
@@ -114,7 +115,7 @@ or to enable all features:
.. code-block:: Bash
- spack install mpifileutils +lustre +gpfs +experimental
+ spack install mpifileutils +lustre +gpfs +hpss +experimental
----------------------------
Development build with CMake
@@ -135,7 +136,7 @@ which can be done with the following commands:
wget https://github.com/hpc/libcircle/releases/download/v0.3/libcircle-0.3.0.tar.gz
wget https://github.com/llnl/lwgrp/releases/download/v1.0.4/lwgrp-1.0.4.tar.gz
wget https://github.com/llnl/dtcmp/releases/download/v1.1.4/dtcmp-1.1.4.tar.gz
- wget https://github.com/libarchive/libarchive/releases/download/3.5.1/libarchive-3.5.1.tar.gz
+ wget https://github.com/libarchive/libarchive/releases/download/v3.5.1/libarchive-3.5.1.tar.gz
tar -zxf libcircle-0.3.0.tar.gz
cd libcircle-0.3.0
@@ -205,5 +206,5 @@ Thus, the commands to build become:
cd ../build
cmake ../mpifileutils
-The other way to use spack is to create a "view" to the installed dependencies.
+The other way to use Spack is to create a "view" to the installed dependencies.
Details on this are coming soon.
diff --git a/doc/rst/dcmp.1.rst b/doc/rst/dcmp.1.rst
index acf51c6..397c301 100644
--- a/doc/rst/dcmp.1.rst
+++ b/doc/rst/dcmp.1.rst
@@ -60,6 +60,10 @@ OPTIONS
Use O_DIRECT to avoid caching file data.
+.. option:: --open-noatime
+
+ Open files with O_NOATIME flag.
+
.. option:: --progress N
Print progress message to stdout approximately every N seconds.
diff --git a/doc/rst/dcp.1.rst b/doc/rst/dcp.1.rst
index 00f3d33..ff3a20b 100644
--- a/doc/rst/dcp.1.rst
+++ b/doc/rst/dcp.1.rst
@@ -49,7 +49,7 @@ OPTIONS
DFS API, and all other containers use the DAOS object API.
Values must be in {DFS, DAOS}.
-.. option:: --daos-preserve FILENAME
+.. option:: --daos-preserve FILENAME
Option to turn on metadata preservation in DAOS. This should
be used in the case that data is being moved to/from DAOS. For instance,
@@ -82,6 +82,10 @@ OPTIONS
Use O_DIRECT to avoid caching file data.
+.. option:: --open-noatime
+
+ Open files with O_NOATIME flag.
+
.. option:: -S, --sparse
Create sparse files when possible.
@@ -92,6 +96,16 @@ OPTIONS
The number of seconds must be a non-negative integer.
A value of 0 disables progress messages.
+.. option:: -G, --gid GID
+
+ Set the effective group ID to perform the copy operation. The copy
+ may fail if the group does not have sufficient privileges.
+
+.. option:: -U, --uid UID
+
+ Set the effective user ID to perform the copy operation. The copy
+ may fail if the user does not have sufficient privileges.
+
.. option:: -v, --verbose
Run in verbose mode.
diff --git a/doc/rst/ddup.1.rst b/doc/rst/ddup.1.rst
index 33e0313..d47bc98 100644
--- a/doc/rst/ddup.1.rst
+++ b/doc/rst/ddup.1.rst
@@ -20,6 +20,10 @@ Multiple sets of duplicate files can be matched using this final reported hash.
OPTIONS
-------
+.. option:: --open-noatime
+
+ Open files with O_NOATIME flag, if possible.
+
.. option:: -d, --debug LEVEL
Set verbosity level. LEVEL can be one of: fatal, err, warn, info, dbg.
diff --git a/doc/rst/dsync.1.rst b/doc/rst/dsync.1.rst
index 402ee05..248bf43 100644
--- a/doc/rst/dsync.1.rst
+++ b/doc/rst/dsync.1.rst
@@ -83,6 +83,10 @@ OPTIONS
Use O_DIRECT to avoid caching file data.
+.. option:: --open-noatime
+
+ Open files with O_NOATIME flag.
+
.. option:: --link-dest DIR
Create hardlink in DEST to files in DIR when file is unchanged
diff --git a/doc/rst/dtar.1.rst b/doc/rst/dtar.1.rst
index 5fa4e19..f732037 100644
--- a/doc/rst/dtar.1.rst
+++ b/doc/rst/dtar.1.rst
@@ -119,6 +119,10 @@ OPTIONS
Apply recorded flags to extracted files.
Default does not record or extract flags.
+.. option:: --open-noatime
+
+ Open source files with O_NOATIME flag when creating archive.
+
.. option:: --fsync
Call fsync before closing files after writing.
diff --git a/doc/rst/proj-design.rst b/doc/rst/proj-design.rst
index b404696..cb18b90 100644
--- a/doc/rst/proj-design.rst
+++ b/doc/rst/proj-design.rst
@@ -32,7 +32,7 @@ Portability
-----------
The tools are intended to support common file systems used in HPC centers, like
-Lustre, GPFS, and NFS. Additionally, methods in the library should be portable
+Lustre, GPFS, NFS, and HPSS. Additionally, methods in the library should be portable
and efficient across multiple file systems. Tool and library users can rely on
mpiFileUtils to provide portable and performant implementations.
diff --git a/mpifileutils.spec b/mpifileutils.spec
index b3a0f81..94619f0 100644
--- a/mpifileutils.spec
+++ b/mpifileutils.spec
@@ -17,15 +17,12 @@ File utilities designed for scalability and performance.
%setup -q
%build
-#topdir=`pwd`
-#installdir=$topdir/install
-
-cmake ./ -DWITH_DTCMP_PREFIX=${installdir} -DWITH_LibCircle_PREFIX=${installdir} -DCMAKE_INSTALL_PREFIX=%{buildroot} -DENABLE_LUSTRE=ON -DENABLE_XATTRS=ON
-make
+%{cmake} ./ -DWITH_DTCMP_PREFIX=${installdir} -DWITH_LibCircle_PREFIX=${installdir} -DCMAKE_INSTALL_PREFIX=%{buildroot} -DENABLE_LUSTRE=ON -DENABLE_XATTRS=ON
+%{cmake_build}
%install
rm -rf %{buildroot}
-make install DESTDIR=%{buildroot}
+%{cmake_install}
%files
diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt
index 509ba35..5178c1a 100644
--- a/src/common/CMakeLists.txt
+++ b/src/common/CMakeLists.txt
@@ -15,6 +15,7 @@ LIST(APPEND libmfu_install_headers
mfu_param_path.h
mfu_path.h
mfu_pred.h
+ mfu_proc.h
mfu_progress.h
mfu_util.h
)
@@ -46,6 +47,7 @@ LIST(APPEND libmfu_srcs
mfu_param_path.c
mfu_path.c
mfu_pred.c
+ mfu_proc.c
mfu_progress.c
mfu_util.c
strmap.c
diff --git a/src/common/mfu.h b/src/common/mfu.h
index 108f0fb..dcf7de8 100644
--- a/src/common/mfu.h
+++ b/src/common/mfu.h
@@ -27,6 +27,7 @@ extern "C" {
#include "mfu_param_path.h"
#include "mfu_flist.h"
#include "mfu_pred.h"
+#include "mfu_proc.h"
#include "mfu_progress.h"
#include "mfu_bz2.h"
diff --git a/src/common/mfu_daos.c b/src/common/mfu_daos.c
index 8f53133..2dbf8cf 100644
--- a/src/common/mfu_daos.c
+++ b/src/common/mfu_daos.c
@@ -210,13 +210,11 @@ static int daos_check_args(
bool have_src_cont = strlen(da->src_cont) ? true : false;
bool have_dst_pool = strlen(da->dst_pool) ? true : false;
bool have_dst_cont = strlen(da->dst_cont) ? true : false;
- bool have_prefix = (da->dfs_prefix != NULL);
/* Determine whether any DAOS arguments are supplied.
* If not, then there is nothing to check. */
*flag_daos_args = 0;
- if (have_src_pool || have_src_cont || have_dst_pool || have_dst_cont
- || have_prefix) {
+ if (have_src_pool || have_src_cont || have_dst_pool || have_dst_cont) {
*flag_daos_args = 1;
}
if ((have_src_path && (strncmp(src_path, "daos:", 5) == 0)) ||
@@ -276,37 +274,6 @@ static int daos_check_args(
return rc;
}
-/* Checks if the prefix is valid.
- * If valid, returns matching string into suffix */
-static bool daos_check_prefix(
- char* path,
- const char* dfs_prefix,
- char** suffix)
-{
- bool is_prefix = false;
- int prefix_len = strlen(dfs_prefix);
- int path_len = strlen(path);
-
- /* ignore trailing '/' on the prefix */
- if (dfs_prefix[prefix_len-1] == '/') {
- prefix_len--;
- }
-
- /* figure out if dfs_prefix is a prefix of the file path */
- if (strncmp(path, dfs_prefix, prefix_len) == 0) {
- /* if equal, assume root */
- if (path_len == prefix_len) {
- *suffix = strdup("/");
- is_prefix = true;
- }
- /* if path is longer, it must start with '/' */
- else if (path_len > prefix_len && path[prefix_len] == '/') {
- *suffix = strdup(path + prefix_len);
- is_prefix = true;
- }
- }
- return is_prefix;
-}
/*
* Parse a path of the format:
@@ -341,6 +308,9 @@ int daos_parse_path(
} else {
strncpy(path, dattr.da_rel_path, path_len);
}
+ } else if (strncmp(path, "daos:", 5) == 0) {
+ /* Actual error, since we expect a daos path */
+ rc = -1;
} else {
/* If basename does not exist yet then duns_resolve_path will fail even
* if dirname is a UNS path */
@@ -387,9 +357,6 @@ int daos_parse_path(
snprintf(*pool_str, DAOS_PROP_LABEL_MAX_LEN + 1, "%s", dattr.da_pool);
snprintf(*cont_str, DAOS_PROP_LABEL_MAX_LEN + 1, "%s", dattr.da_cont);
strncpy(path, dattr.da_rel_path, path_len);
- } else if (strncmp(path, "daos:", 5) == 0) {
- /* Actual error, since we expect a daos path */
- rc = -1;
} else {
/* We didn't parse a daos path,
* but we weren't necessarily looking for one */
@@ -415,106 +382,39 @@ static int daos_set_paths(
{
int rc = 0;
bool have_dst = (numpaths > 1);
- bool prefix_on_src = false;
- bool prefix_on_dst = false;
- char* prefix_path = NULL;
char* src_path = NULL;
char* dst_path = NULL;
- /* find out if a dfs_prefix is being used,
- * if so, then that means that the container
- * is not being copied from the root of the
- * UNS path */
- if (da->dfs_prefix != NULL) {
- char prefix_pool[DAOS_PROP_LABEL_MAX_LEN + 1];
- char prefix_cont[DAOS_PROP_LABEL_MAX_LEN + 1];
- int prefix_rc;
-
- size_t prefix_len = strlen(da->dfs_prefix);
- prefix_path = strndup(da->dfs_prefix, prefix_len);
- if (prefix_path == NULL) {
- MFU_LOG(MFU_LOG_ERR, "Unable to allocate space for DAOS prefix.");
- rc = 1;
- goto out;
- }
-
- memset(prefix_pool, '\0', DAOS_PROP_LABEL_MAX_LEN + 1);
- memset(prefix_cont, '\0', DAOS_PROP_LABEL_MAX_LEN + 1);
-
- /* Get the pool/container uuids from the prefix */
- prefix_rc = daos_parse_path(prefix_path, prefix_len,
- &prefix_pool, &prefix_cont);
- if (prefix_rc != 0) {
- MFU_LOG(MFU_LOG_ERR, "Failed to resolve DAOS Prefix UNS path");
- rc = 1;
- goto out;
- }
-
- /* In case the user tries to give a sub path in the UNS path */
- if (strcmp(prefix_path, "/") != 0) {
- MFU_LOG(MFU_LOG_ERR, "DAOS prefix must be a UNS path");
- rc = 1;
- goto out;
- }
-
- /* Check if the prefix matches the source */
- prefix_on_src = daos_check_prefix(argpaths[0], da->dfs_prefix, &da->src_path);
- if (prefix_on_src) {
- snprintf(da->src_pool, DAOS_PROP_LABEL_MAX_LEN + 1, "%s", prefix_pool);
- snprintf(da->src_cont, DAOS_PROP_LABEL_MAX_LEN + 1, "%s", prefix_cont);
- argpaths[0] = da->src_path;
- }
-
- if (have_dst) {
- /* Check if the prefix matches the destination */
- prefix_on_dst = daos_check_prefix(argpaths[1], da->dfs_prefix, &da->dst_path);
- if (prefix_on_dst) {
- snprintf(da->dst_pool, DAOS_PROP_LABEL_MAX_LEN + 1, "%s", prefix_pool);
- snprintf(da->dst_cont, DAOS_PROP_LABEL_MAX_LEN + 1, "%s", prefix_cont);
- argpaths[1] = da->dst_path;
- }
- }
-
- if (!prefix_on_src && !prefix_on_dst) {
- MFU_LOG(MFU_LOG_ERR, "DAOS prefix does not match source or destination");
- rc = 1;
- goto out;
- }
- }
-
/*
- * For the source and destination paths,
- * if they are not using a prefix,
- * then just directly try to parse a DAOS path.
+ * Try to parse a DAOS source and/or destination path.
*/
- if (!prefix_on_src) {
- size_t src_len = strlen(argpaths[0]);
- src_path = strndup(argpaths[0], src_len);
- if (src_path == NULL) {
- MFU_LOG(MFU_LOG_ERR, "Unable to allocate space for source path.");
+ size_t src_len = strlen(argpaths[0]);
+ src_path = strndup(argpaths[0], src_len);
+ if (src_path == NULL) {
+ MFU_LOG(MFU_LOG_ERR, "Unable to allocate space for source path.");
+ rc = 1;
+ goto out;
+ }
+ int src_rc = daos_parse_path(src_path, src_len, &da->src_pool, &da->src_cont);
+ if (src_rc == 0) {
+ if (strlen(da->src_cont) == 0) {
+ MFU_LOG(MFU_LOG_ERR, "Source pool requires a source container.");
rc = 1;
goto out;
}
- int src_rc = daos_parse_path(src_path, src_len, &da->src_pool, &da->src_cont);
- if (src_rc == 0) {
- if (strlen(da->src_cont) == 0) {
- MFU_LOG(MFU_LOG_ERR, "Source pool requires a source container.");
- rc = 1;
- goto out;
- }
- argpaths[0] = da->src_path = strdup(src_path);
- if (argpaths[0] == NULL) {
- MFU_LOG(MFU_LOG_ERR, "Unable to allocate space for source path.");
- rc = 1;
- goto out;
- }
- } else if (src_rc == -1) {
- MFU_LOG(MFU_LOG_ERR, "Failed to parse DAOS source path: daos://<pool>/<cont>[/<path>]");
+ argpaths[0] = da->src_path = strdup(src_path);
+ if (argpaths[0] == NULL) {
+ MFU_LOG(MFU_LOG_ERR, "Unable to allocate space for source path.");
rc = 1;
goto out;
}
+ } else if (src_rc == -1) {
+ MFU_LOG(MFU_LOG_ERR, "Failed to parse DAOS source path: daos://<pool>/<cont>[/<path>]");
+ rc = 1;
+ goto out;
}
- if (have_dst && !prefix_on_dst) {
+
+ if (have_dst) {
size_t dst_len = strlen(argpaths[1]);
dst_path = strndup(argpaths[1], dst_len);
if (dst_path == NULL) {
@@ -535,9 +435,7 @@ static int daos_set_paths(
rc = 1;
goto out;
}
- }
- if (have_dst) {
int dst_cont_len = strlen(da->dst_cont);
*dst_cont_passed = dst_cont_len > 0 ? true : false;
/* Generate a new container uuid if only a pool was given. */
@@ -547,13 +445,12 @@ static int daos_set_paths(
}
out:
- mfu_free(&prefix_path);
mfu_free(&src_path);
mfu_free(&dst_path);
return rc;
}
-static int daos_get_cont_type(
+static int mfu_daos_get_cont_type(
daos_handle_t coh,
enum daos_cont_props* type)
{
@@ -619,7 +516,7 @@ out:
* Try to set the file type based on the container type,
* using api as a guide.
*/
-static int daos_set_api_cont_type(
+static int mfu_daos_set_api_cont_type(
mfu_file_t* mfu_file,
enum daos_cont_props cont_type,
daos_api_t api)
@@ -745,10 +642,10 @@ static int cont_get_props(daos_handle_t coh, daos_prop_t** _props,
daos_prop_t* prop_acl = NULL;
daos_prop_t* props_merged = NULL;
/* total amount of properties to allocate */
- uint32_t total_props = 15;
+ uint32_t total_props = 16;
/* minimum number of properties that are always allocated/used to start
* count */
- int prop_index = 15;
+ int prop_index = 16;
if (get_oid) {
total_props++;
@@ -790,6 +687,7 @@ static int cont_get_props(daos_handle_t coh, daos_prop_t** _props,
props->dpp_entries[12].dpe_type = DAOS_PROP_CO_DEDUP;
props->dpp_entries[13].dpe_type = DAOS_PROP_CO_DEDUP_THRESHOLD;
props->dpp_entries[14].dpe_type = DAOS_PROP_CO_EC_CELL_SZ;
+ props->dpp_entries[15].dpe_type = DAOS_PROP_CO_SCRUBBER_DISABLED;
/* Conditionally get the OID. Should always be true for serialization. */
if (get_oid) {
@@ -1115,7 +1013,7 @@ void daos_bcast_handle(
}
/* connect to DAOS pool, and then open container */
-int daos_connect(
+int mfu_daos_connect(
int rank,
daos_args_t* da,
char (*pool)[],
@@ -1124,7 +1022,9 @@ int daos_connect(
daos_handle_t* coh,
bool force_serialize,
bool connect_pool,
+ unsigned int pool_connect_flags,
bool create_cont,
+ unsigned int cont_open_flags,
bool require_new_cont,
bool preserve,
mfu_file_t* mfu_src_file,
@@ -1161,12 +1061,12 @@ int daos_connect(
}
#endif
/* Connect to DAOS pool */
- if (connect_pool) {
+ if (connect_pool && !daos_handle_is_valid(*poh)) {
daos_pool_info_t pool_info = {0};
#if DAOS_API_VERSION_MAJOR < 1
- rc = daos_pool_connect(*pool, NULL, NULL, DAOS_PC_RW, poh, &pool_info, NULL);
+ rc = daos_pool_connect(*pool, NULL, NULL, pool_connect_flags, poh, &pool_info, NULL);
#else
- rc = daos_pool_connect(*pool, NULL, DAOS_PC_RW, poh, &pool_info, NULL);
+ rc = daos_pool_connect(*pool, NULL, pool_connect_flags, poh, &pool_info, NULL);
#endif
if (rc != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to connect to pool: "DF_RC, DP_RC(rc));
@@ -1182,9 +1082,11 @@ int daos_connect(
* If nothing is passed in for destination a uuid is always generated
* unless user passed one in, because destination container labels are
* not generated */
- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL);
+ char cont_str[130];
+ uuid_unparse(da->dst_cont_uuid, cont_str);
+ rc = daos_cont_open(*poh, cont_str, cont_open_flags, coh, &co_info, NULL);
} else {
- rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL);
+ rc = daos_cont_open(*poh, *cont, cont_open_flags, coh, &co_info, NULL);
}
if (rc != 0) {
if (rc != -DER_NONEXIST || !create_cont) {
@@ -1246,7 +1148,7 @@ int daos_connect(
/* if nothing is passed in for destination a uuid is always
* generated unless user passed one in, destination container
* labels are not generated */
- rc = dfs_cont_create(*poh, da->dst_cont_uuid, &attr, NULL, NULL);
+ rc = dfs_cont_create(*poh, &da->dst_cont_uuid, &attr, NULL, NULL);
}
if (rc != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to create container: (%d %s)", rc, strerror(rc));
@@ -1256,7 +1158,7 @@ int daos_connect(
if (dst_cont_passed && !is_uuid) {
rc = daos_cont_create_with_label(*poh, *cont, props, &da->dst_cont_uuid, NULL);
} else {
- rc = daos_cont_create(*poh, da->dst_cont_uuid, props, NULL);
+ rc = daos_cont_create(*poh, &da->dst_cont_uuid, props, NULL);
}
if (rc != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to create container: "DF_RC, DP_RC(rc));
@@ -1273,14 +1175,12 @@ int daos_connect(
}
/* try to open it again */
- if (dst_cont_passed) {
- if (is_uuid) {
- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL);
- } else {
- rc = daos_cont_open(*poh, *cont, DAOS_COO_RW, coh, &co_info, NULL);
- }
+ if (dst_cont_passed && !is_uuid) {
+ rc = daos_cont_open(*poh, *cont, cont_open_flags, coh, &co_info, NULL);
} else {
- rc = daos_cont_open(*poh, da->dst_cont_uuid, DAOS_COO_RW, coh, &co_info, NULL);
+ char cont_str[130];
+ uuid_unparse(da->dst_cont_uuid, cont_str);
+ rc = daos_cont_open(*poh, cont_str, cont_open_flags, coh, &co_info, NULL);
}
if (rc != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to open container: "DF_RC, DP_RC(rc));
@@ -1436,7 +1336,6 @@ daos_args_t* daos_args_new(void)
da->dst_poh = DAOS_HDL_INVAL;
da->src_coh = DAOS_HDL_INVAL;
da->dst_coh = DAOS_HDL_INVAL;
- da->dfs_prefix = NULL;
da->src_path = NULL;
da->dst_path = NULL;
@@ -1470,7 +1369,6 @@ void daos_args_delete(daos_args_t** pda)
{
if (pda != NULL) {
daos_args_t* da = *pda;
- mfu_free(&da->dfs_prefix);
mfu_free(&da->src_path);
mfu_free(&da->dst_path);
mfu_free(&da->daos_preserve_path);
@@ -1639,10 +1537,7 @@ int daos_setup(
}
}
- /* Figure out if daos path is the src or dst,
- * using UNS path, then chop off UNS path
- * prefix since the path is mapped to the root
- * of the container in the DAOS DFS mount */
+ /* Figure out if daos path is the src or dst */
if (!local_daos_error) {
tmp_rc = daos_set_paths(rank, argpaths, numpaths, da, &dst_cont_passed);
if (tmp_rc != 0) {
@@ -1661,8 +1556,7 @@ int daos_setup(
}
/* Make sure there weren't any errors before continuing.
- * Since daos_connect has a collective broadcast.
- * we have to make sure same_pool below is valid. */
+ * Since mfu_daos_connect has a collective broadcast. */
if (daos_any_error(rank, local_daos_error, flag_daos_args)) {
tmp_rc = daos_fini();
return 1;
@@ -1676,10 +1570,16 @@ int daos_setup(
bool have_src_cont = strlen(da->src_cont) > 0 ? true : false;
bool have_dst_pool = strlen(da->dst_pool) > 0 ? true : false;
- /* Check if containers are in the same pool */
- bool same_pool = (strcmp(da->src_pool, da->dst_pool) == 0);
+ /* Need W on destination pools for creating a container */
+ unsigned int src_pool_connect_flags = DAOS_PC_RO;
+ unsigned int dst_pool_connect_flags = DAOS_PC_RW;
+
+ /* For POSIX containers, the source only needs read, but the destination needs read and write.
+ * For DAOS (object-level), both containers need read and write.
+ * Open the source with RO first, then elevate to RW if needed below. */
+ unsigned int src_cont_open_flags = DAOS_COO_RO;
+ unsigned int dst_cont_open_flags = DAOS_COO_RW;
- bool connect_pool = true;
bool create_cont = false;
bool require_new_cont = false;
bool preserve = false;
@@ -1694,28 +1594,52 @@ int daos_setup(
local_daos_error = true;
goto out;
}
- tmp_rc = daos_connect(rank, da, &da->src_pool, &da->src_cont,
- &da->src_poh, &da->src_coh, false,
- connect_pool, create_cont, require_new_cont,
- preserve, mfu_src_file, dst_cont_passed);
+ tmp_rc = mfu_daos_connect(rank, da, &da->src_pool, &da->src_cont,
+ &da->src_poh, &da->src_coh, false,
+ true, src_pool_connect_flags, create_cont, src_cont_open_flags, require_new_cont,
+ preserve, mfu_src_file, dst_cont_passed);
if (tmp_rc != 0) {
- /* tmp_rc from daos_connect is collective */
+ /* tmp_rc from mfu_daos_connect is collective */
local_daos_error = true;
goto out;
}
/* Get the container type */
- tmp_rc = daos_get_cont_type(da->src_coh, &da->src_cont_type);
+ tmp_rc = mfu_daos_get_cont_type(da->src_coh, &da->src_cont_type);
if (tmp_rc != 0) {
/* ideally, this should be the same for each process */
local_daos_error = true;
goto out;
}
/* Set the src api based on the container type */
- tmp_rc = daos_set_api_cont_type(mfu_src_file, da->src_cont_type, da->api);
+ tmp_rc = mfu_daos_set_api_cont_type(mfu_src_file, da->src_cont_type, da->api);
if (tmp_rc != 0) {
local_daos_error = true;
goto out;
}
+ /* If using the DAOS API, we need to elevate permissions to RW for creating a snapshot */
+ if (mfu_src_file->type == DAOS) {
+ MPI_Barrier(MPI_COMM_WORLD);
+
+ tmp_rc = daos_cont_close(da->src_coh, NULL);
+ if (tmp_rc != 0) {
+ local_daos_error = true;
+ goto out;
+ }
+ da->src_coh = DAOS_HDL_INVAL;
+
+ /* Re-open with RW */
+ src_cont_open_flags = DAOS_COO_RW;
+ tmp_rc = mfu_daos_connect(rank, da, &da->src_pool, &da->src_cont,
+ &da->src_poh, &da->src_coh, false,
+ true, src_pool_connect_flags, create_cont, src_cont_open_flags, require_new_cont,
+ preserve, mfu_src_file, dst_cont_passed);
+ if (tmp_rc != 0) {
+ /* tmp_rc from mfu_daos_connect is collective */
+ local_daos_error = true;
+ goto out;
+ }
+ }
+
/* Sanity check before we create a new container */
if (da->src_cont_type != DAOS_PROP_CO_LAYOUT_POSIX) {
if (strcmp(da->src_path, "/") != 0) {
@@ -1729,7 +1653,7 @@ int daos_setup(
}
/* If we're using the DAOS API, the destination container cannot
- * exist already, unless overriden by allow_exist_dst_cont. */
+ * exist already, unless overridden by allow_exist_dst_cont. */
if (mfu_src_file->type != POSIX && mfu_src_file->type != DFS && !da->allow_exist_dst_cont) {
require_new_cont = true;
}
@@ -1753,11 +1677,11 @@ int daos_setup(
create_cont = true;
/* do check that src is POSIX and since dst has a pool,
* then the dst *should* always be DFS, but this is not set
- * on the destintaion until after daos_connect is called, and
+ * on the destintaion until after mfu_daos_connect is called, and
* we should read the properties *before* the container is
* created. We do not have the destination container type
* here yet which is why extra check is used then passed
- * to daos_connect */
+ * to mfu_daos_connect */
if (da->daos_preserve) {
if (mfu_src_file->type == POSIX) {
preserve = true;
@@ -1769,26 +1693,17 @@ int daos_setup(
goto out;
}
}
- if (same_pool) {
- connect_pool = false;
- tmp_rc = daos_connect(rank, da, &da->dst_pool, &da->dst_cont,
- &da->src_poh, &da->dst_coh, false,
- connect_pool, create_cont, require_new_cont,
- preserve, mfu_src_file, dst_cont_passed);
- } else {
- connect_pool = true;
- tmp_rc = daos_connect(rank, da, &da->dst_pool, &da->dst_cont,
+ tmp_rc = mfu_daos_connect(rank, da, &da->dst_pool, &da->dst_cont,
&da->dst_poh, &da->dst_coh, false,
- connect_pool, create_cont, require_new_cont,
- preserve, mfu_src_file, dst_cont_passed);
- }
+ true, dst_pool_connect_flags, create_cont, dst_cont_open_flags,
+ require_new_cont, preserve, mfu_src_file, dst_cont_passed);
if (tmp_rc != 0) {
- /* tmp_rc from daos_connect is collective */
+ /* tmp_rc from mfu_daos_connect is collective */
local_daos_error = true;
goto out;
}
/* Get the container type */
- tmp_rc = daos_get_cont_type(da->dst_coh, &da->dst_cont_type);
+ tmp_rc = mfu_daos_get_cont_type(da->dst_coh, &da->dst_cont_type);
if (tmp_rc != 0) {
/* ideally, this should be the same for each process */
local_daos_error = true;
@@ -1796,7 +1711,7 @@ int daos_setup(
}
if (have_dst) {
/* Set the dst api based on the container type */
- tmp_rc = daos_set_api_cont_type(mfu_dst_file, da->dst_cont_type, da->api);
+ tmp_rc = mfu_daos_set_api_cont_type(mfu_dst_file, da->dst_cont_type, da->api);
if (tmp_rc != 0) {
local_daos_error = true;
goto out;
@@ -1823,11 +1738,7 @@ int daos_setup(
if (have_dst) {
/* Mount destination DFS container */
if (mfu_dst_file->type == DFS) {
- if (same_pool) {
- tmp_rc = mfu_dfs_mount(mfu_dst_file, &da->src_poh, &da->dst_coh);
- } else {
- tmp_rc = mfu_dfs_mount(mfu_dst_file, &da->dst_poh, &da->dst_coh);
- }
+ tmp_rc = mfu_dfs_mount(mfu_dst_file, &da->dst_poh, &da->dst_coh);
if (tmp_rc != 0) {
local_daos_error = true;
goto out;
@@ -1874,8 +1785,6 @@ int daos_cleanup(
int rank;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
- bool same_pool = (strcmp(da->src_pool, da->dst_pool) == 0);
-
/* Destroy source snapshot */
if (rank == 0 && da->src_epc != 0) {
tmp_rc = mfu_daos_destroy_snap(da->src_coh, da->src_epc);
@@ -1944,7 +1853,7 @@ int daos_cleanup(
}
/* Close destination pool */
- if (daos_handle_is_valid(da->dst_poh) && !same_pool) {
+ if (daos_handle_is_valid(da->dst_poh)) {
tmp_rc = daos_pool_disconnect(da->dst_poh, NULL);
if (tmp_rc != 0) {
MFU_LOG(MFU_LOG_ERR, "Failed to disconnect from destination pool "DF_RC, DP_RC(tmp_rc));
@@ -2420,7 +2329,7 @@ static int mfu_daos_obj_sync(
/* open handle of object in dst container */
daos_handle_t dst_oh;
- rc = daos_obj_open(dst_coh, oid, DAOS_OO_EXCL, &dst_oh, NULL);
+ rc = daos_obj_open(dst_coh, oid, DAOS_OO_RW, &dst_oh, NULL);
if (rc != 0) {
MFU_LOG(MFU_LOG_ERR, "DAOS object open returned with errors: " MFU_ERRF,
MFU_ERRP(-MFU_ERR_DAOS));
@@ -3857,7 +3766,7 @@ static int cont_serialize_prop_str(struct hdf5_args* hdf5,
hid_t attr_dspace;
hid_t usr_attr;
- if (entry == NULL || entry->dpe_str == NULL) {
+ if (entry == NULL) {
MFU_LOG(MFU_LOG_ERR, "Property %s not found", prop_str);
rc = 1;
goto out;
@@ -3870,7 +3779,7 @@ static int cont_serialize_prop_str(struct hdf5_args* hdf5,
rc = 1;
goto out;
}
- status = H5Tset_size(attr_dtype, strlen(entry->dpe_str) + 1);
+ status = H5Tset_size(attr_dtype, (entry->dpe_str ? strlen(entry->dpe_str) : 0) + 1);
if (status < 0) {
MFU_LOG(MFU_LOG_ERR, "failed to set dtype size");
rc = 1;
@@ -3895,7 +3804,7 @@ static int cont_serialize_prop_str(struct hdf5_args* hdf5,
rc = 1;
goto out;
}
- status = H5Awrite(usr_attr, attr_dtype, entry->dpe_str);
+ status = H5Awrite(usr_attr, attr_dtype, entry->dpe_str ? entry->dpe_str : "");
if (status < 0) {
MFU_LOG(MFU_LOG_ERR, "failed to write attribute");
rc = 1;
@@ -4156,26 +4065,32 @@ int cont_serialize_props(struct hdf5_args *hdf5,
}
entry = &prop_query->dpp_entries[15];
- rc = cont_serialize_prop_uint(hdf5, entry, "DAOS_PROP_CO_ALLOCED_OID");
+ rc = cont_serialize_prop_uint(hdf5, entry, "DAOS_PROP_CO_SCRUBBER_DISABLED");
if (rc != 0) {
goto out;
}
entry = &prop_query->dpp_entries[16];
- rc = cont_serialize_prop_str(hdf5, entry, "DAOS_PROP_CO_LABEL");
+ rc = cont_serialize_prop_uint(hdf5, entry, "DAOS_PROP_CO_ALLOCED_OID");
if (rc != 0) {
goto out;
}
entry = &prop_query->dpp_entries[17];
+ rc = cont_serialize_prop_str(hdf5, entry, "DAOS_PROP_CO_LABEL");
+ if (rc != 0) {
+ goto out;
+ }
+
+ entry = &prop_query->dpp_entries[18];
rc = cont_serialize_prop_roots(hdf5, entry, "DAOS_PROP_CO_ROOTS");
if (rc != 0) {
goto out;
}
/* serialize ACL */
- if (prop_query->dpp_nr > 18) {
- entry = &prop_query->dpp_entries[18];
+ if (prop_query->dpp_nr > 19) {
+ entry = &prop_query->dpp_entries[19];
rc = cont_serialize_prop_acl(hdf5, entry, "DAOS_PROP_CO_ACL");
if (rc != 0) {
goto out;
@@ -4301,7 +4216,7 @@ int daos_cont_serialize_hdlr(int rank, struct hdf5_args *hdf5, char *output_dir,
goto out;
}
} else {
- rc = daos_obj_open(da->src_coh, oid, 0, &oh, NULL);
+ rc = daos_obj_open(da->src_coh, oid, DAOS_OO_RO, &oh, NULL);
if (rc != 0) {
MFU_LOG(MFU_LOG_ERR, "failed to open object: "DF_RC, DP_RC(rc));
goto out;
@@ -5369,7 +5284,7 @@ int cont_deserialize_all_props(struct hdf5_args *hdf5,
{
int rc = 0;
bool deserialize_label = false;
- uint32_t num_props = 18;
+ uint32_t num_props = 19;
daos_prop_t *label = NULL;
daos_prop_t *prop = NULL;
struct daos_prop_entry *entry;
@@ -5394,22 +5309,23 @@ int cont_deserialize_all_props(struct hdf5_args *hdf5,
goto out;
}
- rc = daos_cont_open(poh, label_entry->dpe_str, DAOS_COO_RW, &coh, &cont_info, NULL);
- if (rc == -DER_NONEXIST) {
- /* doesn't exist so ok to deserialize this container label */
- deserialize_label = true;
- } else if (rc != 0) {
- MFU_LOG(MFU_LOG_ERR, "daos_cont_open failed: "DF_RC, DP_RC(rc));
- goto out;
- } else {
- /* if this succeeds then label already exists, close container after
- * checking */
- rc = daos_cont_close(coh, NULL);
- if (rc != 0) {
+ if (label_entry->dpe_str[0]) {
+ rc = daos_cont_open(poh, label_entry->dpe_str, DAOS_COO_RW, &coh, &cont_info, NULL);
+ if (rc == -DER_NONEXIST) {
+ /* doesn't exist so ok to deserialize this container label */
+ deserialize_label = true;
+ } else if (rc != 0) {
+ MFU_LOG(MFU_LOG_ERR, "daos_cont_open failed: "DF_RC, DP_RC(rc));
goto out;
+ } else {
+ /* if this succeeds then label already exists, close container after
+ * checking */
+ rc = daos_cont_close(coh, NULL);
+ if (rc != 0) {
+ goto out;
+ }
}
}
-
if (deserialize_label) {
num_props++;
}
@@ -5434,11 +5350,12 @@ int cont_deserialize_all_props(struct hdf5_args *hdf5,
prop->dpp_entries[12].dpe_type = DAOS_PROP_CO_DEDUP;
prop->dpp_entries[13].dpe_type = DAOS_PROP_CO_DEDUP_THRESHOLD;
prop->dpp_entries[14].dpe_type = DAOS_PROP_CO_EC_CELL_SZ;
- prop->dpp_entries[15].dpe_type = DAOS_PROP_CO_ALLOCED_OID;
- prop->dpp_entries[16].dpe_type = DAOS_PROP_CO_ACL;
- prop->dpp_entries[17].dpe_type = DAOS_PROP_CO_ROOTS;
+ prop->dpp_entries[15].dpe_type = DAOS_PROP_CO_SCRUBBER_DISABLED;
+ prop->dpp_entries[16].dpe_type = DAOS_PROP_CO_ALLOCED_OID;
+ prop->dpp_entries[17].dpe_type = DAOS_PROP_CO_ACL;
+ prop->dpp_entries[18].dpe_type = DAOS_PROP_CO_ROOTS;
if (deserialize_label) {
- prop->dpp_entries[18].dpe_type = DAOS_PROP_CO_LABEL;
+ prop->dpp_entries[19].dpe_type = DAOS_PROP_CO_LABEL;
}
entry = &prop->dpp_entries[0];
@@ -5532,12 +5449,18 @@ int cont_deserialize_all_props(struct hdf5_args *hdf5,
}
entry = &prop->dpp_entries[15];