From 88099dd28c05d616d5f35b24fb04e6d775607eb1 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 13 Oct 2020 16:04:04 +0200 Subject: [PATCH 1/4] [core] stats: avoid crash in psutil (windows only) https://github.com/giampaolo/psutil/issues/1763 --- meshroom/core/stats.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/meshroom/core/stats.py b/meshroom/core/stats.py index e9a325381f..e4d5e01483 100644 --- a/meshroom/core/stats.py +++ b/meshroom/core/stats.py @@ -201,15 +201,19 @@ def update(self, proc): data = proc.as_dict(self.dynamicKeys) for k, v in data.items(): self._addKV(k, v) - - files = [f.path for f in proc.open_files()] - if self.lastIterIndexWithFiles != -1: - if set(files) != set(self.openFiles[self.lastIterIndexWithFiles]): - self.openFiles[self.iterIndex] = files - self.lastIterIndexWithFiles = self.iterIndex - elif files: - self.openFiles[self.iterIndex] = files - self.lastIterIndexWithFiles = self.iterIndex + + ## Note: Do not collect stats about open files for now, + # as there is bug in psutil-5.7.2 on Windows which crashes the application. + # https://github.com/giampaolo/psutil/issues/1763 + # + # files = [f.path for f in proc.open_files()] + # if self.lastIterIndexWithFiles != -1: + # if set(files) != set(self.openFiles[self.lastIterIndexWithFiles]): + # self.openFiles[self.iterIndex] = files + # self.lastIterIndexWithFiles = self.iterIndex + # elif files: + # self.openFiles[self.iterIndex] = files + # self.lastIterIndexWithFiles = self.iterIndex self.iterIndex += 1 def toDict(self): From cdee25e26e01d3b3728d4e24663e9fe3bde97ee8 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 13 Oct 2020 16:07:42 +0200 Subject: [PATCH 2/4] [core] stats: minor simplification add timeout on nvidia-smi add try/except remove duplicated code no error is no nvidia-smi available --- meshroom/core/stats.py | 62 ++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/meshroom/core/stats.py b/meshroom/core/stats.py index e4d5e01483..ebc754d614 100644 --- a/meshroom/core/stats.py +++ b/meshroom/core/stats.py @@ -44,7 +44,7 @@ def __init__(self): self.gpuMemoryTotal = 0 self.gpuName = '' self.curves = defaultdict(list) - + self.nvidia_smi = None self._isInit = False def initOnFirstTime(self): @@ -53,40 +53,21 @@ def initOnFirstTime(self): self._isInit = True self.cpuFreq = psutil.cpu_freq().max - self.ramTotal = psutil.virtual_memory().total / 1024/1024/1024 + self.ramTotal = psutil.virtual_memory().total / (1024*1024*1024) if platform.system() == "Windows": from distutils import spawn # If the platform is Windows and nvidia-smi - # could not be found from the environment path, - # try to find it from system drive with default installation path self.nvidia_smi = spawn.find_executable('nvidia-smi') if self.nvidia_smi is None: - self.nvidia_smi = "%s\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe" % os.environ['systemdrive'] + # could not be found from the environment path, + # try to find it from system drive with default installation path + default_nvidia_smi = "%s\\Program Files\\NVIDIA Corporation\\NVSMI\\nvidia-smi.exe" % os.environ['systemdrive'] + if os.path.isfile(default_nvidia_smi): + self.nvidia_smi = default_nvidia_smi else: self.nvidia_smi = "nvidia-smi" - try: - p = subprocess.Popen([self.nvidia_smi, "-q", "-x"], stdout=subprocess.PIPE) - xmlGpu, stdError = p.communicate() - - smiTree = ET.fromstring(xmlGpu) - gpuTree = smiTree.find('gpu') - - try: - self.gpuMemoryTotal = gpuTree.find('fb_memory_usage').find('total').text.split(" ")[0] - except Exception as e: - logging.debug('Failed to get gpuMemoryTotal: "{}".'.format(str(e))) - pass - try: - self.gpuName = gpuTree.find('product_name').text - except Exception as e: - logging.debug('Failed to get gpuName: "{}".'.format(str(e))) - pass - - except Exception as e: - logging.debug('Failed to get information from nvidia_smi at init: "{}".'.format(str(e))) - def _addKV(self, k, v): if isinstance(v, tuple): for ki, vi in v._asdict().items(): @@ -98,18 +79,23 @@ def _addKV(self, k, v): self.curves[k].append(v) def update(self): - self.initOnFirstTime() - self._addKV('cpuUsage', psutil.cpu_percent(percpu=True)) # interval=None => non-blocking (percentage since last call) - self._addKV('ramUsage', psutil.virtual_memory().percent) - self._addKV('swapUsage', psutil.swap_memory().percent) - self._addKV('vramUsage', 0) - self._addKV('ioCounters', psutil.disk_io_counters()) - self.updateGpu() + try: + self.initOnFirstTime() + self._addKV('cpuUsage', psutil.cpu_percent(percpu=True)) # interval=None => non-blocking (percentage since last call) + self._addKV('ramUsage', psutil.virtual_memory().percent) + self._addKV('swapUsage', psutil.swap_memory().percent) + self._addKV('vramUsage', 0) + self._addKV('ioCounters', psutil.disk_io_counters()) + self.updateGpu() + except Exception as e: + logging.debug('Failed to get statistics: "{}".'.format(str(e))) def updateGpu(self): + if not self.nvidia_smi: + return try: - p = subprocess.Popen([self.nvidia_smi, "-q", "-x"], stdout=subprocess.PIPE) - xmlGpu, stdError = p.communicate() + p = subprocess.Popen([self.nvidia_smi, "-q", "-x"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + xmlGpu, stdError = p.communicate(timeout=10) # 10 seconds smiTree = ET.fromstring(xmlGpu) gpuTree = smiTree.find('gpu') @@ -129,7 +115,11 @@ def updateGpu(self): except Exception as e: logging.debug('Failed to get gpuTemperature: "{}".'.format(str(e))) pass - + except subprocess.TimeoutExpired as e: + logging.debug('Timeout when retrieving information from nvidia_smi: "{}".'.format(str(e))) + p.kill() + outs, errs = p.communicate() + return except Exception as e: logging.debug('Failed to get information from nvidia_smi: "{}".'.format(str(e))) return From 12d80ac57bc23e24a1a9a95af5ee07bd7526624a Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 13 Oct 2020 16:11:07 +0200 Subject: [PATCH 3/4] [core] stats: reduce stats refresh precision --- meshroom/core/stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meshroom/core/stats.py b/meshroom/core/stats.py index ebc754d614..784319cdff 100644 --- a/meshroom/core/stats.py +++ b/meshroom/core/stats.py @@ -228,7 +228,7 @@ def __init__(self): self.computer = ComputerStatistics() self.process = ProcStatistics() self.times = [] - self.interval = 5 + self.interval = 10 # refresh interval in seconds def update(self, proc): ''' From fc28636f5caaf305afe56615dae729ac0fe15647 Mon Sep 17 00:00:00 2001 From: Fabien Castan Date: Tue, 13 Oct 2020 22:38:44 +0200 Subject: [PATCH 4/4] [release] Update version to 2020.1.1 Update changelog --- CHANGES.md | 7 +++++++ meshroom/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1b25d84c45..33ea76974c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,13 @@ For algorithmic changes related to the photogrammetric pipeline, please refer to [AliceVision changelog](https://github.com/alicevision/AliceVision/blob/develop/CHANGES.md). +## Release 2020.1.1 (2020.10.14) + +Based on [AliceVision 2.3.1](https://github.com/alicevision/AliceVision/tree/v2.3.1). + + - [core] Fix crashes on process statistics (windows-only) [PR](https://github.com/alicevision/meshroom/pull/1096) + + ## Release 2020.1.0 (2020.10.09) Based on [AliceVision 2.3.0](https://github.com/alicevision/AliceVision/tree/v2.3.0). diff --git a/meshroom/__init__.py b/meshroom/__init__.py index 02bbabcf25..89f5806820 100644 --- a/meshroom/__init__.py +++ b/meshroom/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2020.1.0" +__version__ = "2020.1.1" __version_name__ = __version__ from distutils import util