From 8307d3c00bbba360a4fecba90f62af8aff155805 Mon Sep 17 00:00:00 2001 From: panxuchen Date: Thu, 18 Apr 2024 14:15:52 +0800 Subject: [PATCH] add psutil to minimal requires --- data_juicer/utils/process_utils.py | 2 +- environments/minimal_requires.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/data_juicer/utils/process_utils.py b/data_juicer/utils/process_utils.py index cd42f6847..2aa60e19f 100644 --- a/data_juicer/utils/process_utils.py +++ b/data_juicer/utils/process_utils.py @@ -1,6 +1,7 @@ import math import subprocess +import psutil from loguru import logger from data_juicer import cuda_device_count, use_cuda @@ -23,7 +24,6 @@ def get_min_cuda_memory(): def calculate_np(num_proc, op, op_name): """Calculate the optimum number of processes for the given OP""" - import psutil if num_proc is None: num_proc = psutil.cpu_count() if use_cuda() and op._accelerator == 'cuda': diff --git a/environments/minimal_requires.txt b/environments/minimal_requires.txt index d7696c75b..7c67a3f65 100644 --- a/environments/minimal_requires.txt +++ b/environments/minimal_requires.txt @@ -24,3 +24,4 @@ streamlit spacy==3.5.0 multiprocess==0.70.12 dill==0.3.4 +psutil