From 993c51135ebc0a23ae78d7482321c606f43eb905 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Mon, 4 Dec 2023 16:12:16 +0100
Subject: [PATCH] Add examples infrastructure and add example for iterative
 computation

---
 docs/requirements.txt                    |  1 +
 examples/README.md                       |  8 ++++
 examples/iterative-computation/README.md | 58 ++++++++++++++++++++++++
 mkdocs.yml                               |  6 +++
 scripts/doc_copy_examples.py             | 15 ++++++
 5 files changed, 88 insertions(+)
 create mode 100644 examples/README.md
 create mode 100644 examples/iterative-computation/README.md
 create mode 100644 scripts/doc_copy_examples.py

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 14fed19e6..9ed08d7c7 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -2,6 +2,7 @@ mkdocs==1.2.3
 mkdocs-material==7.3.2
 mkdocs-minify-plugin==0.5.0
 mkdocs-git-revision-date-localized-plugin==0.10.0
+mkdocs-gen-files==0.5.0
 mike==1.1.2
 requests==2.31.0
 jinja2==3.0.3
diff --git a/examples/README.md b/examples/README.md
new file mode 100644
index 000000000..a22461310
--- /dev/null
+++ b/examples/README.md
@@ -0,0 +1,8 @@
+# Examples
+Here you can find several examples of how HyperQueue can be used for various use-cases, both with the command-line
+interface and also with the Python API.
+
+You can view these examples either in the [documentation](https://it4innovations.github.io/hyperqueue/stable/examples/iterative-computation/)
+or on [GitHub](https://github.com/It4innovations/hyperqueue/tree/main/examples).
+
+- [Iterative computation](iterative-computation)
diff --git a/examples/iterative-computation/README.md b/examples/iterative-computation/README.md
new file mode 100644
index 000000000..0e0b7120b
--- /dev/null
+++ b/examples/iterative-computation/README.md
@@ -0,0 +1,58 @@
+# Iterative computation
+It is a common use-case to perform an iterative computation, e.g. run a randomized simulation until the results are
+stable/accurate enough, or train a machine learning model while the loss keeps dropping.
+
+While there is currently no built-in support in HQ for iteratively submitting new tasks to an existing job, you can perform
+an iterative computation relatively easily with the following approach:
+
+1. Submit a HQ job that performs a computation
+2. Wait for the job to finish
+3. Read the output of the job and decide if computation should continue
+4. If yes, go to 1.
+
+# Python API
+With the Python API, we can simply write the outermost iteration loop in Python, and repeatedly submit jobs, until some
+end criterion has been achieved:
+
+```python
+from hyperqueue import Job, Client
+
+client = Client()
+
+while True:
+    job = Job()
+    job.program(["my-program"], stdout="out.txt")
+
+    # Submit a job
+    submitted = client.submit(job)
+
+    # Wait for it to complete
+    client.wait_for_jobs([submitted])
+
+    # Read the output of the job
+    with open("out.txt") as f:
+        # Check some termination condition and eventually end the loop
+        if f.read().strip() == "done":
+            break
+```
+
+# Command-line interface
+With the command-line interface, you can perform the iterative loop e.g. in Bash.
+
+```bash
+#!/bin/bash
+
+while :
+do
+  # Submit a job and wait for it to complete
+  ./hq submit --wait ./compute.sh
+  
+  # Read the output of the job
+  output=$(./hq job cat last stdout)
+
+  # Decide if we should end or continue
+  if [ "${output}" -eq 0 ]; then
+      break
+  fi
+done
+```
diff --git a/mkdocs.yml b/mkdocs.yml
index 4acfc1699..1415ed5fe 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -13,6 +13,9 @@ nav:
   - Getting Started:
     - Quickstart: quickstart.md
     - Cheatsheet: cheatsheet.md
+  - Examples:
+    - examples/README.md
+    - Iterative computation: examples/iterative-computation/README.md
   - Deployment:
     - deployment/index.md
     - Server: deployment/server.md
@@ -81,6 +84,9 @@ plugins:
       canonical_version: stable
   - nedoc:
       path: python/apidoc
+  - gen-files:
+      scripts:
+        - scripts/doc_copy_examples.py
 
 extra:
   analytics:
diff --git a/scripts/doc_copy_examples.py b/scripts/doc_copy_examples.py
new file mode 100644
index 000000000..b8dce7e6b
--- /dev/null
+++ b/scripts/doc_copy_examples.py
@@ -0,0 +1,15 @@
+"""
+Copy all files from the `examples` directory to `<built-docs>/examples`, so that they can be rendered in the
+documentation.
+"""
+import glob
+import os.path
+
+import mkdocs_gen_files
+
+
+for path in glob.glob("examples/**/*", recursive=True):
+    if os.path.isfile(path):
+        with open(path) as src_file:
+            with mkdocs_gen_files.open(path, "w") as dest_file:
+                dest_file.write(src_file.read())