New Python environment based on FastAPI framework (#388)

* feat: Add python-fastapi-env
* Fix some issues with fastapi environment
* Cosmetic changes to fastapi environment
* Add python-fastapi-env support to github workflows
* Upgrade upload-artifact version to v4
* Fix typos in python-fastapi tests
* Fix python-fastapi CI failure
* Fix python-fastapi tests and examples
* Add sourcepkg example for python-fastapi environment
* Add multifile example for python-fastapi environment
* Add requestdata and statuscode examples for python-fastapi environment
* Add guestbook example for python-fastapi environment
* Remove redis and other packages from python-fastapi core environment

---------

Signed-off-by: Md Soharab Ansari <[email protected]>
Co-authored-by: Jacob Chen <[email protected]>

.github/workflows/environment.yaml

@@ -60,7 +60,7 @@ jobs:
           command -v fission && fission support dump
       - name: Archive fission dump
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: fission-dump
           path: fission-dump/*.zip
@@ -101,7 +101,7 @@ jobs:
           command -v fission && fission support dump
       - name: Archive fission dump
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: fission-dump
           path: fission-dump/*.zip
@@ -191,7 +191,7 @@ jobs:
           command -v fission && fission support dump
       - name: Archive fission dump
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: fission-dump
           path: fission-dump/*.zip
@@ -262,7 +262,7 @@ jobs:
   python:
     runs-on: ubuntu-latest
     needs: check
-    if: contains( needs.check.outputs.packages, 'python' )
+    if: contains( needs.check.outputs, 'python' )
     steps:
       - name: Checkout sources
         uses: actions/checkout@v2
@@ -312,7 +312,62 @@ jobs:
           command -v fission && fission support dump
       - name: Archive fission dump
         if: ${{ failure() }}
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
+        with:
+          name: fission-dump
+          path: fission-dump/*.zip
+          retention-days: 5
+  python-fastapi:
+    runs-on: ubuntu-latest
+    needs: check
+    if: contains( needs.check.outputs.packages, 'python-fastapi' )
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
+      - name: Unit test
+        run: |
+          pip3 install virtualenv
+          sudo apt-get update -y && sudo apt-get install -y libev-dev
+          pushd python-fastapi/
+          export USERFUNCVOL=/tmp
+          export RUNTIME_PORT=8882
+          ./tests/local_test.sh
+          unset RUNTIME_PORT
+          popd
+      - name: Helm
+        uses: Azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
+        with:
+          version: v3.13.0
+      - name: Kind Clutser
+        uses: engineerd/[email protected]
+        with:
+          image: kindest/node:v1.25.16
+          version: v0.23.0
+          config: kind.yaml
+      - name: Base Setup
+        run: |
+          make verify-kind-cluster
+          make install-fission-cli
+          make install-skaffold
+          make create-crds
+      - name: Fission and Test images
+        run: |
+          SKAFFOLD_PROFILE=python-fastapi make skaffold-run
+          make python-fastapi-test-images
+          make router-port-forward
+      - name: python-fastapi-tests
+        run: |
+          ./test_utils/run_test.sh ./python-fastapi/tests/test_python_fastapi_env.sh
+      - name: Collect Fission Dump
+        if: ${{ always() }}
+        run: |
+          command -v fission && fission support dump
+      - name: Archive fission dump
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v4
         with:
           name: fission-dump
           path: fission-dump/*.zip

.github/workflows/filters/filters.yaml

@@ -18,6 +18,8 @@ perl:
     - 'perl/**'
 python:
     - 'python/**'
+python-fastapi:
+    - 'python-fastapi/**'
 ruby: 
     - 'ruby/**'
 tensorflow-serving:

.github/workflows/filters/version_filter.yaml

@@ -18,6 +18,8 @@ perl:
     - 'perl/envconfig.json'
 python:
     - 'python/envconfig.json'
+python-fastapi:
+    - 'python-fastapi/envconfig.json'
 ruby: 
     - 'ruby/envconfig.json'
 tensorflow-serving:

Makefile

@@ -17,6 +17,7 @@
 FISSION_ENVS := nodejs-envs \
 	go-envs \
 	python-envs \
+	python-fastapi-envs \
 	perl-envs \
 	jvm-envs \
 	php7-envs \
@@ -86,5 +87,9 @@ python-test-images:
 	@kind load docker-image python-env
 	@kind load docker-image python-builder
 
+python-fastapi-test-images:
+	@kind load docker-image python-fastapi-env
+	@kind load docker-image python-fastapi-builder
+
 router-port-forward:
 	@kubectl port-forward svc/router 8888:80 -nfission &

python-fastapi/Dockerfile

@@ -0,0 +1,15 @@
+ARG PY_BASE_IMG=3.11-alpine
+
+FROM python:${PY_BASE_IMG}
+
+WORKDIR /app
+
+RUN pip install --no-cache-dir --upgrade pip
+
+COPY requirements.txt /app
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY *.py /app
+
+ENTRYPOINT ["python3"]
+CMD ["server.py"]

python-fastapi/Makefile

@@ -0,0 +1,8 @@
+-include ../rules.mk
+
+.PHONY: all
+all: python-fastapi-env-img
+
+python-fastapi-env-img-buildargs := --build-arg PY_BASE_IMG=3.11-alpine
+
+python-fastapi-env-img: Dockerfile

python-fastapi/README.md

@@ -0,0 +1,51 @@
+# Fission: Python FastAPI Environment
+
+This is the Python environment for Fission based on FastAPI framework.
+
+It's a Docker image containing a Python 3.11 runtime, along with a
+dynamic loader.  A few common dependencies are included in the
+requirements.txt file.
+
+Looking for ready-to-run examples? See the [Python FastAPI examples directory](./examples).
+
+## Customizing this image
+
+To add package dependencies, edit requirements.txt to add what you
+need, and rebuild this image (instructions below).
+
+You also may want to customize what's available to the function in its
+request context.  You can do this by editing server.py (see the
+comment in that file about customizing request context).
+
+## Rebuilding and pushing the image
+
+You'll need access to a Docker registry to push the image: you can
+sign up for Docker hub at hub.docker.com, or use registries from
+gcr.io, quay.io, etc.  Let's assume you're using a docker hub account
+called USER.  Build and push the image to the the registry:
+
+```
+   docker build -t USER/python-fastapi-env --build-arg PY_BASE_IMG=3.11-alpine . && docker push USER/python-fastapi-env
+```
+
+## Using the image in fission
+
+You can add this customized image to fission with "fission env create":
+
+```
+   fission env create --name python --image USER/python-fastapi-env
+```
+
+Or, if you already have an environment, you can update its image:
+
+```
+   fission env update --name python --image USER/python-fastapi-env
+```
+
+After this, fission functions that have the env parameter set to the
+same environment name as this command will use this environment.
+
+## Web Server Framework
+
+Python environment build and start a ASGI server, to support high HTTP
+traffic. It provides Uvicorn server framework.

python-fastapi/builder/Dockerfile

@@ -0,0 +1,12 @@
+ARG BUILDER_IMAGE=fission/builder
+ARG PY_BASE_IMG=3.11-alpine
+
+FROM ${BUILDER_IMAGE}
+FROM python:${PY_BASE_IMG}
+
+COPY --from=0 /builder /builder
+RUN apk add --update --no-cache python3-dev build-base gcc bash
+
+ADD defaultBuildCmd /usr/local/bin/build
+
+EXPOSE 8001

python-fastapi/builder/Makefile

@@ -0,0 +1,8 @@
+-include ../../rules.mk
+
+.PHONY: all
+all: python-fastapi-builder-img
+
+python-fastapi-builder-img-buildargs := --build-arg PY_BASE_IMG=3.11-alpine
+
+python-fastapi-builder-img: Dockerfile

python-fastapi/builder/defaultBuildCmd

@@ -0,0 +1,8 @@
+#!/bin/sh
+set -euxo pipefail
+
+if [ -f ${SRC_PKG}/requirements.txt ]
+then 
+    pip3 install -r ${SRC_PKG}/requirements.txt -t ${SRC_PKG}
+fi
+cp -r ${SRC_PKG} ${DEPLOY_PKG}

python-fastapi/envconfig.json

@@ -0,0 +1,26 @@
+[
+  {
+    "builder": "python-fastapi-builder",
+    "examples": "https://github.com/fission/environments/tree/master/python/examples",
+    "icon": "./logo/Python-logo-notext.png",
+    "image": "python-fastapi-env",
+    "keywords": [],
+    "kind": "environment",
+    "maintainers": [
+      {
+        "link": "https://github.com/sanketsudake",
+        "name": "sanketsudake"
+      },
+      {
+        "link": "https://github.com/vishal-biyani",
+        "name": "vishal-biyani"
+      }
+    ],
+    "name": "Python FastAPI Environment",
+    "readme": "https://github.com/fission/environments/tree/master/python",
+    "runtimeVersion": "3.11",
+    "shortDescription": "Fission Python environment based on FastAPI framework.",
+    "status": "stable",
+    "version": "1.34.2"
+  }
+]

python-fastapi/examples/README.md

@@ -0,0 +1,27 @@
+# Python Examples
+
+This directory contains a Python examples to show different the features of the Fission Python environment:
+- `hello.py` is a simple Pythonic _hello world_ function.
+- `sourcepkg/` is an example of how to use the Fission Python Build environment to resolve (pip) dependencies
+  before deploying the function.
+- `multifile/` shows how to create Fission Python functions with multiple source files.
+- `requestdata.py` shows how you can access the HTTP request fields, such as the body, headers and query.
+- `statuscode.py` is an example of how you can change the response status code.
+- `guestbook/` is a more realistic demonstration of using Python and Fission to create a serverless guestbook.
+
+## Getting Started
+
+Create a Fission Python environment with the default Python runtime image (this does not include the build environment):
+```
+fission environment create --name python --image ghcr.io/fission/python-fastapi-env
+```
+
+Use the `hello.py` to create a Fission Python function:
+```
+fission function create --name hello-py --env python --code hello.py 
+```
+
+Test the function:
+```
+fission function test --name hello-py
+```

python-fastapi/examples/guestbook/add.py

@@ -0,0 +1,20 @@
+#
+# Handles POST /guestbook -- adds item to guestbook
+#
+
+from fastapi import Request, Response
+from fastapi.responses import RedirectResponse
+import redis
+
+# Connect to redis.
+redisConnection = redis.StrictRedis(host='redis.guestbook', port=6379, db=0)
+
+async def main(request: Request):
+    # Read the item from POST params, add it to redis, and redirect
+    # back to the list
+    form = await request.form()
+    book = form.get('text')
+    if book is None:
+        return Response(status_code=400)
+    redisConnection.rpush('guestbook', book)
+    return RedirectResponse('/guestbook', status_code=303)

python-fastapi/examples/guestbook/build.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+pip3 install -r ${SRC_PKG}/requirements.txt -t ${SRC_PKG} && cp -r ${SRC_PKG} ${DEPLOY_PKG}

python-fastapi/examples/guestbook/deploy.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+set -e
+
+kubectl apply -f redis.yaml
+
+if [ -z "$FISSION_URL" ]
+then
+    echo "Need $FISSION_URL set to a fission controller address"
+    exit 1
+fi
+
+# Create python env if it doesn't exist
+fission env get --name python || fission env create --name python --image python-fastapi-env --builder python-fastapi-builder
+
+# Create zip file
+zip -jr guestbook.zip ../guestbook
+
+# Create packages
+fission package create --name guestbook --sourcearchive guestbook.zip --env python --buildcmd "./build.sh"
+
+# Register functions and routes with fission
+fission function create --name guestbook-get --env python --pkg guestbook --entrypoint "get.main" --url /guestbook --method GET
+fission function create --name guestbook-add --env python --pkg guestbook --entrypoint "add.main" --url /guestbook --method POST

python-fastapi/examples/guestbook/get.py

@@ -0,0 +1,29 @@
+#
+# Handles GET /guestbook -- returns a list of items in the guestbook
+# with a form to add more.
+#
+
+from fastapi import Request, Response
+from markupsafe import escape
+import redis
+
+# Connect to redis.  This is run only when this file is loaded; as
+# long as the pod is alive, the connection is reused.
+redisConnection = redis.StrictRedis(host='redis.guestbook', port=6379, db=0)
+
+def main(request: Request):
+    messages = redisConnection.lrange('guestbook', 0, -1)
+
+    items = [("<li>%s</li>" % escape(m.decode('utf-8'))) for m in messages]
+    ul = "<ul>%s</ul>" % "\n".join(items)
+    return Response("""
+      <html><body style="font-family:sans-serif;font-size:2rem;padding:40px">
+          <h1>Guestbook</h1>
+          <form action="/guestbook" method="POST">
+            <input type="text" name="text">
+            <button type="submit">Add</button>
+          </form>
+          <hr/>
+          %s
+      </body></html>
+      """ % ul)