修改COPY-FROM No.13 distributed

Signed-off-by: jjyaoao <[email protected]>
PaddlePaddle · Jul 19, 2023 · 4ce6d38 · 4ce6d38
1 parent 3354d86
commit 4ce6d38
Show file tree

Hide file tree

Showing 7 changed files with 120 additions and 822 deletions.
diff --git a/docs/api/paddle/distributed/QueueDataset_cn.rst b/docs/api/paddle/distributed/QueueDataset_cn.rst
@@ -14,10 +14,7 @@ QueueyDataset 是流式处理数据使用 Dataset 类。与 InmemoryDataset 继
 代码示例
 ::::::::::::
 
-.. code-block:: python
-
-    import paddle
-    dataset = paddle.distributed.QueueDataset()
+COPY-FROM: paddle.distributed.QueueDataset
 
 方法
 ::::::::::::
@@ -49,56 +46,7 @@ None。
 
 **代码示例**
 
-.. code-block:: python
-
-
-    import paddle
-    import os
-
-    paddle.enable_static()
-
-    with open("test_queue_dataset_run_a.txt", "w") as f:
-        data = "2 1 2 2 5 4 2 2 7 2 1 3\n"
-        data += "2 6 2 2 1 4 2 2 4 2 2 3\n"
-        data += "2 5 2 2 9 9 2 2 7 2 1 3\n"
-        data += "2 7 2 2 1 9 2 3 7 2 5 3\n"
-        f.write(data)
-    with open("test_queue_dataset_run_b.txt", "w") as f:
-        data = "2 1 2 2 5 4 2 2 7 2 1 3\n"
-        data += "2 6 2 2 1 4 2 2 4 2 2 3\n"
-        data += "2 5 2 2 9 9 2 2 7 2 1 3\n"
-        data += "2 7 2 2 1 9 2 3 7 2 5 3\n"
-        f.write(data)
-
-    slots = ["slot1", "slot2", "slot3", "slot4"]
-    slots_vars = []
-    for slot in slots:
-        var = paddle.static.data(
-            name=slot, shape=[None, 1], dtype="int64", lod_level=1)
-        slots_vars.append(var)
-
-    dataset = paddle.distributed.QueueDataset()
-    dataset.init(
-        batch_size=1,
-        thread_num=2,
-        input_type=1,
-        pipe_command="cat",
-        use_var=slots_vars)
-    dataset.set_filelist(
-        ["test_queue_dataset_run_a.txt", "test_queue_dataset_run_b.txt"])
-
-    paddle.enable_static()
-
-    place = paddle.CPUPlace()
-    exe = paddle.static.Executor(place)
-    startup_program = paddle.static.Program()
-    main_program = paddle.static.Program()
-    exe.run(startup_program)
-
-    exe.train_from_dataset(main_program, dataset)
-
-    os.remove("./test_queue_dataset_run_a.txt")
-    os.remove("./test_queue_dataset_run_b.txt")
+COPY-FROM: paddle.distributed.QueueDataset.init
 
 
 set_filelist(filelist)
@@ -108,42 +56,7 @@ set_filelist(filelist)
 
 **代码示例**
 
-.. code-block:: python
-
-    import paddle
-    import os
-
-    paddle.enable_static()
-
-    with open("test_queue_dataset_run_a.txt", "w") as f:
-        data = "2 1 2 2 5 4 2 2 7 2 1 3\n"
-        data += "2 6 2 2 1 4 2 2 4 2 2 3\n"
-        data += "2 5 2 2 9 9 2 2 7 2 1 3\n"
-        data += "2 7 2 2 1 9 2 3 7 2 5 3\n"
-        f.write(data)
-    with open("test_queue_dataset_run_b.txt", "w") as f:
-        data = "2 1 2 2 5 4 2 2 7 2 1 3\n"
-        data += "2 6 2 2 1 4 2 2 4 2 2 3\n"
-        data += "2 5 2 2 9 9 2 2 7 2 1 3\n"
-        data += "2 7 2 2 1 9 2 3 7 2 5 3\n"
-        f.write(data)
-    dataset = paddle.distributed.QueueDataset()
-    slots = ["slot1", "slot2", "slot3", "slot4"]
-    slots_vars = []
-    for slot in slots:
-        var = paddle.static.data(
-            name=slot, shape=[None, 1], dtype="int64", lod_level=1)
-        slots_vars.append(var)
-    dataset.init(
-        batch_size=1,
-        thread_num=2,
-        input_type=1,
-        pipe_command="cat",
-        use_var=slots_vars)
-    filelist = ["a.txt", "b.txt"]
-    dataset.set_filelist(filelist)
-    os.remove("./test_queue_dataset_run_a.txt")
-    os.remove("./test_queue_dataset_run_b.txt")
+COPY-FROM: paddle.distributed.QueueDataset.set_filelist
 
 
 **参数**