Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Expose the output_type option when streaming, and allow both stdout/stderr to be captured #209

Merged
merged 4 commits into from
Oct 25, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/dev-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
pre-commit
black
black==23.3.0
isort
flake8
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ The client here will eventually be released as "spython" (and eventually to
singularity on pypi), and the versions here will coincide with these releases.

## [master](https://github.com/singularityhub/singularity-cli/tree/master)
- exposed the stream type option, and ability to capture both stdout and stderr when stream=True (0.3.1)
- dropping support for Singularity 2.x (0.3.0)
- add comment out of STOPSIGNAL (0.2.14)
- sudo `-E` flag should not be provided by default (0.2.13)
Expand Down
1 change: 0 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,6 @@ def get_requirements(lookup=None):


if __name__ == "__main__":

INSTALL_REQUIRES = get_requirements(lookup)
TESTS_REQUIRES = get_requirements(lookup)

Expand Down
2 changes: 0 additions & 2 deletions spython/client/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@


def get_parser():

parser = argparse.ArgumentParser(
description="Singularity Client",
formatter_class=argparse.RawTextHelpFormatter,
Expand Down Expand Up @@ -147,7 +146,6 @@ def version():


def main():

parser = get_parser()

def print_help(return_code=0):
Expand Down
2 changes: 0 additions & 2 deletions spython/client/recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@ def main(args, options, parser):
force = True

if args.json:

if outfile is not None:
if not os.path.exists(outfile):
if force:
Expand All @@ -85,7 +84,6 @@ def main(args, options, parser):
print(json.dumps(recipeParser.recipe.json(), indent=4))

else:

# Do the conversion
recipeWriter = writer(recipeParser.recipe)
result = recipeWriter.convert(runscript=entrypoint, force=force)
Expand Down
1 change: 0 additions & 1 deletion spython/instance/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,6 @@ def _update_metadata(self, kwargs=None):

# Add acceptable arguments
for arg in ["pid", "name", "ip_address", "log_err_path", "log_out_path", "img"]:

# Skip over non-iterables:
if arg in kwargs:
setattr(self, arg, kwargs[arg])
Expand Down
1 change: 0 additions & 1 deletion spython/instance/cmd/start.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ def start(

# If an image isn't provided, we have an initialized instance
if image is None:

# Not having this means it was called as a command, without an image
if not hasattr(self, "_image"):
bot.exit("Please provide an image, or create an Instance first.")
Expand Down
2 changes: 0 additions & 2 deletions spython/main/base/command.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,6 @@ def generate_bind_list(self, bindlist=None):
bindlist = bindlist.split(" ")

for bind in bindlist:

# Still cannot be None
if bind:
bot.debug("Adding bind %s" % bind)
Expand Down Expand Up @@ -113,7 +112,6 @@ def run_command(
environ=None,
background=False,
):

"""
Run_command is a wrapper for the global run_command, checking first
for sudo and exiting on error if needed. The message is returned as
Expand Down
1 change: 0 additions & 1 deletion spython/main/base/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@


class RobotNamer:

_descriptors = [
"chunky",
"buttery",
Expand Down
1 change: 0 additions & 1 deletion spython/main/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ def build(
sudo_options=None,
singularity_options=None,
):

"""build a singularity image, optionally for an isolated build
(requires sudo). If you specify to stream, expect the image name
and an iterator to be returned.
Expand Down
7 changes: 5 additions & 2 deletions spython/main/execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ def execute(
sudo_options=None,
quiet=True,
environ=None,
stream_type="stdout",
):
"""execute: send a command to a container

Expand All @@ -51,6 +52,7 @@ def execute(
and message result not (default)
quiet: Do not print verbose output.
environ: extra environment to add.
stream_type: Sets which output stream from the singularity command should be return. Values are 'stdout', 'stderr', 'both'.
"""
from spython.utils import check_install

Expand All @@ -68,7 +70,6 @@ def execute(
image = None

if command is not None:

# No image provided, default to use the client's loaded image
if image is None:
image = self._get_uri()
Expand Down Expand Up @@ -115,7 +116,9 @@ def execute(
quiet=quiet,
environ=environ,
)
return stream_command(cmd, sudo=sudo, sudo_options=sudo_options)
return stream_command(
cmd, sudo=sudo, sudo_options=sudo_options, output_type=stream_type
)

bot.exit("Please include a command (list) to execute.")

Expand Down
1 change: 0 additions & 1 deletion spython/main/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ def export(
sudo=False,
singularity_options=None,
):

"""export will export an image, sudo must be used. Since we have Singularity
versions after 3, export is replaced with building into a sandbox.

Expand Down
2 changes: 0 additions & 2 deletions spython/main/instances.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,13 @@ def list_instances(
# Success, we have instances

if output["return_code"] == 0:

instances = json.loads(output["message"][0]).get("instances", {})

# Does the user want instance objects instead?
listing = []

if not return_json:
for i in instances:

# If the user has provided a name, only add instance matches
if name is not None:
if name != i["instance"]:
Expand Down
2 changes: 0 additions & 2 deletions spython/main/parse/parsers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ def __init__(self, filename, load=True):
self.recipe = {"spython-base": Recipe(self.filename)}

if self.filename:

# Read in the raw lines of the file
self.lines = read_file(self.filename)

Expand All @@ -69,7 +68,6 @@ def _run_checks(self):
attempting parsing.
"""
if self.filename is not None:

# Does the recipe provided exist?
if not os.path.exists(self.filename):
bot.exit("Cannot find %s, is the path correct?" % self.filename)
Expand Down
6 changes: 0 additions & 6 deletions spython/main/parse/parsers/docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@


class DockerParser(ParserBase):

name = "docker"

def __init__(self, filename="Dockerfile", load=True):
Expand Down Expand Up @@ -49,7 +48,6 @@ def parse(self):
previous = None

for line in self.lines:

parser = self._get_mapping(line, parser, previous)

# Parse it, if appropriate
Expand Down Expand Up @@ -147,7 +145,6 @@ def _arg(self, line):

# Try to extract arguments from the line
for arg in line:

# An undefined arg cannot be used
if "=" not in arg:
bot.warning(
Expand Down Expand Up @@ -197,15 +194,13 @@ def parse_env(self, envlist):
exports = []

for env in envlist:

pieces = re.split("( |\\\".*?\\\"|'.*?')", env)
pieces = [p for p in pieces if p.strip()]

while pieces:
current = pieces.pop(0)

if current.endswith("="):

# Case 1: ['A='] --> A=
nextone = ""

Expand Down Expand Up @@ -243,7 +238,6 @@ def _copy(self, lines):
lines = self._setup("COPY", lines)

for line in lines:

# Take into account multistage builds
layer = None
if line.startswith("--from"):
Expand Down
6 changes: 0 additions & 6 deletions spython/main/parse/parsers/singularity.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@


class SingularityParser(ParserBase):

name = "singularity"

def __init__(self, filename="Singularity", load=True):
Expand Down Expand Up @@ -51,7 +50,6 @@ def _setup(self, lines):
bot.warning("SETUP is error prone, please check output.")

for line in lines:

# For all lines, replace rootfs with actual root /
line = re.sub("[$]{?SINGULARITY_ROOTFS}?", "", "$SINGULARITY_ROOTFS")

Expand Down Expand Up @@ -171,7 +169,6 @@ def _run(self, lines):

# Multiple line runscript needs multiple lines written to script
if len(lines) > 1:

bot.warning("More than one line detected for runscript!")
bot.warning("These will be echoed into a single script to call.")
self._write_script("/entrypoint.sh", lines)
Expand Down Expand Up @@ -257,7 +254,6 @@ def _load_section(self, lines, section, layer=None):
members = []

while True:

if not lines:
break
next_line = lines[0]
Expand All @@ -278,7 +274,6 @@ def _load_section(self, lines, section, layer=None):

# Add the list to the config
if members and section is not None:

# Get the correct parsing function
parser = self._get_mapping(section)

Expand Down Expand Up @@ -309,7 +304,6 @@ def load_recipe(self):
comments = []

while lines:

# Clean up white trailing/leading space
line = lines.pop(0)
stripped = line.strip()
Expand Down
1 change: 0 additions & 1 deletion spython/main/parse/recipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ class Recipe:
"""

def __init__(self, recipe=None, layer=1):

self.cmd = None
self.comments = []
self.entrypoint = None
Expand Down
2 changes: 0 additions & 2 deletions spython/main/parse/writers/docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@


class DockerWriter(WriterBase):

name = "docker"

def __init__(self, recipe=None): # pylint: disable=useless-super-delegation
Expand Down Expand Up @@ -161,7 +160,6 @@ def write_lines(label, lines):
result = []
continued = False
for line in lines:

# Skip comments and empty lines
if line.strip() == "" or line.strip().startswith("#"):
continue
Expand Down
4 changes: 0 additions & 4 deletions spython/main/parse/writers/singularity.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@


class SingularityWriter(WriterBase):

name = "singularity"

def __init__(self, recipe=None): # pylint: disable=useless-super-delegation
Expand Down Expand Up @@ -52,7 +51,6 @@ def convert(self, runscript="/bin/bash", force=False):

# Write each layer to new file
for stage, parser in self.recipe.items():

# Set the first and active stage
self.stage = stage

Expand Down Expand Up @@ -111,9 +109,7 @@ def _create_runscript(self, default="/bin/bash", force=False):

# Only look at Docker if not enforcing default
if not force:

if self.recipe[self.stage].entrypoint is not None:

# The provided entrypoint can be a string or a list
if isinstance(self.recipe[self.stage].entrypoint, list):
entrypoint = " ".join(self.recipe[self.stage].entrypoint)
Expand Down
2 changes: 0 additions & 2 deletions spython/main/pull.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ def pull(
quiet=False,
singularity_options=None,
):

"""pull will pull a singularity hub or Docker image

Parameters
Expand Down Expand Up @@ -92,7 +91,6 @@ def pull(

# Option 3: A custom name we can predict (not commit/hash) and can also show
else:

# As of Singularity 3.x (at least 3.8) output goes to stderr
return final_image, stream_command(cmd, sudo=False, output_type="stderr")

Expand Down
1 change: 0 additions & 1 deletion spython/oci/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@


class OciImage(ImageBase):

# Default functions of client don't use sudo
sudo = False

Expand Down
3 changes: 0 additions & 3 deletions spython/oci/cmd/actions.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ def run(
singularity_options=None,
log_format="kubernetes",
):

"""run is a wrapper to create, start, attach, and delete a container.

Equivalent command line example:
Expand Down Expand Up @@ -57,7 +56,6 @@ def create(
log_format="kubernetes",
singularity_options=None,
):

"""use the client to create a container from a bundle directory. The bundle
directory should have a config.json. You must be the root user to
create a runtime.
Expand Down Expand Up @@ -104,7 +102,6 @@ def _run(
log_format="kubernetes",
singularity_options=None,
):

"""_run is the base function for run and create, the only difference
between the two being that run does not have an option for sync_socket.

Expand Down
Loading