Skip to content

Commit

Permalink
Fix IPU being blocked by resource limitations
Browse files Browse the repository at this point in the history
First resource limit is maximum number of open file descriptors limit,
second one being limit for maximum writable file size.

Resolves: RHEL-26459 and RHEL-16881
  • Loading branch information
tomasfratrik committed Oct 30, 2024
1 parent a147938 commit 470367d
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 0 deletions.
17 changes: 17 additions & 0 deletions commands/preupgrade/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import resource
import sys
import uuid

Expand Down Expand Up @@ -59,6 +60,22 @@ def preupgrade(args, breadcrumbs):
except LeappError as exc:
raise CommandError(exc.message)

soft_nofile, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
soft_fsize, _ = resource.getrlimit(resource.RLIMIT_FSIZE)
nofile_limit = 1024*16

if soft_nofile < nofile_limit:
try:
resource.setrlimit(resource.RLIMIT_NOFILE, (nofile_limit, nofile_limit))
except OSError as err:
raise CommandError('Failed to set limit for maximum number of open file descriptors: {}'.format(err))

if soft_fsize != resource.RLIM_INFINITY:
try:
resource.setrlimit(resource.RLIMIT_FSIZE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
except OSError as err:
raise CommandError('Failed to set limit for maximum writeable file size: {}'.format(err))

workflow = repositories.lookup_workflow('IPUWorkflow')()
util.warn_if_unsupported(configuration)
util.process_whitelist_experimental(repositories, workflow, configuration, logger)
Expand Down
18 changes: 18 additions & 0 deletions commands/upgrade/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import os
import resource
import sys
import uuid

Expand Down Expand Up @@ -89,6 +90,23 @@ def upgrade(args, breadcrumbs):
repositories = util.load_repositories()
except LeappError as exc:
raise CommandError(exc.message)

soft_nofile, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
soft_fsize, _ = resource.getrlimit(resource.RLIMIT_FSIZE)
nofile_limit = 1024*16

if soft_nofile < nofile_limit:
try:
resource.setrlimit(resource.RLIMIT_NOFILE, (nofile_limit, nofile_limit))
except OSError as err:
raise CommandError('Failed to set limit for maximum number of open file descriptors: {}'.format(err))

if soft_fsize != resource.RLIM_INFINITY:
try:
resource.setrlimit(resource.RLIMIT_FSIZE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
except OSError as err:
raise CommandError('Failed to set limit for maximum writeable file size: {}'.format(err))

workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot)
util.process_whitelist_experimental(repositories, workflow, configuration, logger)
util.warn_if_unsupported(configuration)
Expand Down

0 comments on commit 470367d

Please sign in to comment.