Expose programmatic equivalent of CLI commands (#1854)

* export `covalent_start` and `covalent_stop`

* check server stopped

* update changelog

* move commands to main namespace

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* improve docstrings

* fix `covalent_is_running` to return bool

* reorder `covalent_is_running` conditions

* `quiet` mode to suppress stdout; more docstrings

* use poll function instead of while loop

* explain package

* add api docs entry

* update api docs

* restore import from `._programmatic`

* update api docs

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add test for new functions

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add test for `covalent_is_running`

* removing covalent's dependency on dispatcher

* ignore pip reqs in new package

* refactor docstrings

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* update docs

* revert api docs

* include 'Returns' in docstrings so maybe docs will render 🤷 pls

* remove useless 'Returns' from docstrings 🤦‍♂️

* try autofunction refs to main namespace instead

* revert using main namespace refs

* add more logging and edit messages

* refactor hanging tests

* refactor tests into functional tests

* Revert "refactor tests into functional tests"

This reverts commit f308f8e.

* create global var for timeout

* use mock start and stop commands

* renamed server check function and added import error check tests

* None wasn't an acceptable value to redirect_stdout

* refactor to use subprocess

* refactor as multiple tests w/ patched start/stop

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add nopycln inside new tests

* renaming things a bit

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: sankalp <[email protected]>

.github/workflows/requirements.yml

@@ -54,6 +54,7 @@ jobs:
           --ignore-file=covalent/triggers/**
           --ignore-file=covalent/cloud_resource_manager/**
           --ignore-file=covalent/quantum/qserver/**
+          --ignore-file=covalent/_programmatic/**
           covalent
 
       - name: Check missing dispatcher requirements

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+
+### Added
+
+- Programmatic equivalents of CLI commands `covalent start` and `covalent stop`
+
 ### Changed
 
 - Changed the azurebatch.rst banner from default covalent jpg to azure batch's svg file

covalent/__init__.py

@@ -25,6 +25,11 @@
 from ._dispatcher_plugins import local_redispatch as redispatch  # nopycln: import
 from ._dispatcher_plugins import stop_triggers  # nopycln: import
 from ._file_transfer import strategies as fs_strategies  # nopycln: import
+from ._programmatic.commands import (  # nopycln: import
+    covalent_start,
+    covalent_stop,
+    is_covalent_running,
+)
 from ._results_manager.results_manager import (  # nopycln: import
     cancel,
     get_result,

covalent/_programmatic/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2021 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+NOTE: This package exists to avoid circular imports that would be encountered if
+`covalent` imports from `covalent_dispatcher._cli`.
+"""

covalent/_programmatic/commands.py

@@ -0,0 +1,161 @@
+# Copyright 2021 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Functions providing programmatic access to Covalent CLI commands."""
+import subprocess
+from typing import List, Optional
+
+import psutil
+
+from .._shared_files import logger
+from .._shared_files.config import get_config
+
+__all__ = ["is_covalent_running", "covalent_start", "covalent_stop"]
+
+
+app_log = logger.app_log
+
+_MISSING_SERVER_WARNING = "Covalent has not been installed with the server component."
+
+
+def _call_cli_command(cmd: List[str], *, quiet: bool = False) -> subprocess.CompletedProcess:
+    """
+    Call a CLI command with the specified kwargs.
+
+    Args:
+        func: The CLI command to call.
+        quiet: Suppress stdout. Defaults to :code:`False`.
+    """
+
+    if quiet:
+        return subprocess.run(
+            cmd,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+            check=True,
+        )
+
+    return subprocess.run(cmd, check=True)
+
+
+def is_covalent_running() -> bool:
+    """
+    Check if the Covalent server is running.
+
+    Returns:
+        :code:`True` if the Covalent server is in a ready state, :code:`False` otherwise.
+    """
+    try:
+        from covalent_dispatcher._cli.service import _read_pid
+
+        pid = _read_pid(get_config("dispatcher.cache_dir") + "/ui.pid")
+        return (
+            pid != -1
+            and psutil.pid_exists(pid)
+            and get_config("dispatcher.address") != ""
+            and get_config("dispatcher.port") != ""
+        )
+
+    except ModuleNotFoundError:
+        # If the covalent_dispatcher is not installed, assume Covalent is not running.
+        app_log.warning(_MISSING_SERVER_WARNING)
+        return False
+
+
+def covalent_start(
+    develop: bool = False,
+    port: Optional[str] = None,
+    mem_per_worker: Optional[str] = None,
+    workers: Optional[int] = None,
+    threads_per_worker: Optional[int] = None,
+    ignore_migrations: bool = False,
+    no_cluster: bool = False,
+    no_triggers: bool = False,
+    triggers_only: bool = False,
+    *,
+    quiet: bool = False,
+) -> None:
+    """
+    Start the Covalent server. Wrapper for the :code:`covalent start` CLI command.
+    This function returns immediately if the local Covalent server is already running.
+
+    Args:
+        develop: Start local server in develop mode. Defaults to :code:`False`.
+        port: Local server port number. Defaults to :code:`"48008"`.
+        mem_per_worker: Memory limit per worker in GB. Defaults to auto.
+        workers: Number of Dask workers. Defaults to 8.
+        threads_per_worker: Number of threads per Dask worker. Defaults to 1.
+        ignore_migrations: Start server without database migrations. Defaults to :code:`False`.
+        no_cluster: Start server without Dask cluster. Defaults to :code:`False`.
+        no_triggers: Start server without a triggers server. Defaults to :code:`False`.
+        triggers_only: Start only the triggers server. Defaults to :code:`False`.
+        quiet: Suppress stdout. Defaults to :code:`False`.
+    """
+
+    if is_covalent_running():
+        msg = "Covalent server is already running."
+        if not quiet:
+            print(msg)
+
+        app_log.debug(msg)
+        return
+
+    flags = {
+        "--develop": develop,
+        "--ignore-migrations": ignore_migrations,
+        "--no-cluster": no_cluster,
+        "--no-triggers": no_triggers,
+        "--triggers-only": triggers_only,
+    }
+
+    args = {
+        "--port": port or get_config("dispatcher.port"),
+        "--mem-per-worker": mem_per_worker or get_config("dask.mem_per_worker"),
+        "--workers": workers or get_config("dask.num_workers"),
+        "--threads-per-worker": threads_per_worker or get_config("dask.threads_per_worker"),
+    }
+
+    cmd = ["covalent", "start"]
+    cmd.extend(flag for flag, value in flags.items() if value)
+
+    for arg, value in args.items():
+        cmd.extend((arg, str(value)))
+
+    # Run the `covalent start [OPTIONS]` command.
+    app_log.debug("Starting Covalent server programmatically...")
+    _call_cli_command(cmd, quiet=quiet)
+
+
+def covalent_stop(*, quiet: bool = False) -> None:
+    """
+    Stop the Covalent server. Wrapper for the :code:`covalent stop` CLI command.
+    This function returns immediately if the local Covalent server is not running.
+
+    Args:
+        quiet: Suppress stdout. Defaults to :code:`False`.
+    """
+
+    if not is_covalent_running():
+        msg = "Covalent server is not running."
+        if not quiet:
+            print(msg)
+
+        app_log.debug(msg)
+        return
+
+    # Run the `covalent stop` command.
+    app_log.debug("Stopping Covalent server programmatically...")
+    _call_cli_command(["covalent", "stop"], quiet=quiet)

doc/source/api/api.rst

@@ -6,6 +6,7 @@ Covalent API
 
 The following API documentation describes how to use Covalent.
 
+- The :ref:`covalent_server` manages workflow dispatch, orchestration, and metadata
 - :ref:`electrons_api` and :ref:`lattices_api` are used for constructing workflows
 - :ref:`qelectrons_api` are used to customize and track quantum circuit execution
 - :ref:`qclusters_api` are used to distribute Quantum Electrons across multiple quantum backends.
@@ -22,6 +23,29 @@ The following API documentation describes how to use Covalent.
 - :ref:`dispatcher_interface` is used for dispatching workflows and stopping triggered dispatches
 - The :ref:`dispatcher_server_api` is used for interfacing with the Covalent server
 
+.. _covalent_server:
+
+Covalent Server
+"""""""""""""""""""""""""""
+A Covalent server must be running in order to dispatch workflows. The Covalent CLI provides various utilities for starting, stopping, and managing a Covalent server. For more information, see:
+
+.. code-block:: bash
+
+    covalent --help
+
+The Covalent SDK also includes a Python interface for starting and stopping the Covalent server.
+
+.. autofunction:: covalent._programmatic.commands.is_covalent_running
+
+
+.. autofunction:: covalent._programmatic.commands.covalent_start
+
+
+.. autofunction:: covalent._programmatic.commands.covalent_stop
+
+
+----------------------------------------------------------------
+
 .. _electrons_api:
 
 Electron

tests/covalent_tests/programmatic/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2021 Agnostiq Inc.
+#
+# This file is part of Covalent.
+#
+# Licensed under the Apache License 2.0 (the "License"). A copy of the
+# License may be obtained with this software package or at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Use of this file is prohibited except in compliance with the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.