Merge pull request #5 from yonishelach/repo-structure

Add `project_setup.py` and all project repo files

.devcontainer/Dockerfile.txt

@@ -0,0 +1,21 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/python-3/.devcontainer/base.Dockerfile
+
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
+ARG VARIANT="3.8-bullseye"
+FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# [Optional] If your pip requirements rarely change, uncomment this section to add them to the image.
+COPY requirements.txt /tmp/pip-tmp/
+RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt \
+    && rm -rf /tmp/pip-tmp
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+# [Optional] Uncomment this line to install global node packages.
+# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.devcontainer/devcontainer.json

@@ -0,0 +1,71 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.236.0/containers/python-3
+{
+	"name": "Sagemaker Demo",
+	"build": {
+		"dockerfile": "Dockerfile",
+		"context": "..",
+		"args": {
+			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+			// Append -bullseye or -buster to pin to an OS version.
+			// Use -bullseye variants on local on arm64/Apple Silicon.
+			"VARIANT": "3.8",
+			// Options
+			"NODE_VERSION": "none"
+		}
+	},
+	"containerEnv": {
+		"MLRUN_ENV_FILE": "${containerWorkspaceFolder}/mlrun.env",
+		"SHARED_DIR": "~/mlrun-data",
+		"MLRUN_TAG": "1.5.2"
+	},
+	// Configure tool-specific properties.
+	"customizations": {
+		// Configure properties specific to VS Code.
+		"vscode": {
+			// Set *default* container specific settings.json values on container create.
+			"settings": {
+				"python.defaultInterpreterPath": "/usr/local/bin/python",
+				"python.linting.enabled": true,
+				"python.linting.pylintEnabled": true,
+				"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
+				"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
+				"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
+				"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
+				"python.linting.flake8Path": "/usr/local/py-utils/bin/flake8",
+				"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
+				"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
+				"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
+				"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint"
+			},
+
+			// Add the IDs of extensions you want installed when the container is created.
+			"extensions": [
+				"ms-python.python",
+				"ms-python.vscode-pylance"
+			]
+		}
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [8060, 8070, 8080],
+	// "runArgs": [ "--network", "host"],
+	"portsAttributes": {"8060": {"label": "MLRun UI"}, "8070": {"label": "Nuclio UI"}, "8080": {"label": "MLRun API"}},
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "chmod +x /workspaces/tutorials/start.sh",
+	// "postStartCommand": "echo XXX=$(ip route get 1.2.3.4 | awk '{print $7}') > xx.env",
+
+	// Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"features": {
+		"docker-from-docker": "latest",
+		"git": "latest",
+		"jupyterlab": "latest"
+	},
+	"hostRequirements": {
+	   "cpus": 4,
+	   "memory": "8gb",
+	   "storage": "32gb"
+	}
+}

.flake8

@@ -0,0 +1,6 @@
+[flake8]
+max-line-length = 120
+extend-ignore = E203, W503
+
+# exclude these dirs
+exclude = .git,venv,playground

.github/workflows/ci.yaml

@@ -0,0 +1,61 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+    - development
+  push:
+    branches:
+    - main
+
+jobs:
+  lint:
+    name: Lint code (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('**/dev-requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.python-version }}-
+          ${{ runner.os }}-pip-
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip~=22.3.0
+        pip install -r dev-requirements.txt
+    - name: Lint
+      run: make lint
+
+
+  tests:
+    name: Run tests (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - uses: actions/cache@v2
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-${{ matrix.python-version }}-
+          ${{ runner.os }}-pip-
+    - name: Install automation scripts dependencies and add mlrun to dev packages
+      run: pip install -r requirements.txt -r dev-requirements.txt
+    - name: Test package
+      run: make test

.gitignore

@@ -0,0 +1,129 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

Dockerfile

@@ -0,0 +1,3 @@
+FROM mlrun/mlrun
+RUN pip install sagemaker
+RUN pip install xgboost

Makefile

@@ -0,0 +1,80 @@
+
+PYTHON_INTERPRETER = python3
+SHARED_DIR ?= ~/mlrun-data
+MLRUN_TAG ?= 1.4.0
+HOST_IP ?=$$(ip route get 1.2.3.4 | awk '{print $$7}')
+CONDA_ENV ?= mlrun
+SHELL=/bin/bash
+CONDA_PY_VER ?= 3.9
+CONDA_ACTIVATE = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate ; conda activate
+
+#################################################################################
+# COMMANDS                                                                      #
+#################################################################################
+
+.PHONY: help
+help: ## Display available commands
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: all
+all:
+	$(error please pick a target)
+
+.PHONY: install-requirements
+install-requirements: ## Install all requirements needed for development
+	$(PYTHON_INTERPRETER) -m pip install -r requirements.txt -r dev-requirements.txt
+
+
+.PHONY: package-wheel
+package-wheel: clean ## Build python package wheel
+	python setup.py bdist_wheel
+
+.PHONY: clean
+clean: ## Clean python package build artifacts
+	rm -rf build
+	rm -rf dist
+	find . -type f -name "*.py[co]" -delete
+	find . -type d -name "__pycache__" -delete
+
+.PHONY: fmt
+fmt: ## Format the code (using black and isort)
+	@echo "Running black fmt..."
+	$(PYTHON_INTERPRETER) -m black src
+	$(PYTHON_INTERPRETER) -m isort src
+
+.PHONY: lint
+lint: fmt-check flake8 ## Run lint on the code
+
+.PHONY: fmt-check
+fmt-check: ## Format and check the code (using black and isort)
+	@echo "Running black+isort fmt check..."
+	$(PYTHON_INTERPRETER) -m black --check --diff src
+	$(PYTHON_INTERPRETER) -m isort --check --diff src
+
+.PHONY: flake8
+flake8: ## Run flake8 lint
+	@echo "Running flake8 lint..."
+	$(PYTHON_INTERPRETER) -m flake8 src
+
+.PHONY: mlrun-docker
+mlrun-docker: ## Start MLRun & Nuclio containers (using Docker compose)
+	mkdir $(SHARED_DIR) -p
+	@echo "HOST_IP=$(HOST_IP)" > .env
+	SHARED_DIR=$(SHARED_DIR) TAG=$(MLRUN_TAG) docker-compose -f compose.yaml up -d
+	@echo "use docker-compose stop / logs commands to stop or view logs"
+
+.PHONY: mlrun-api
+mlrun-api: ## Run MLRun DB locally (as process)
+	@echo "Installing MLRun API dependencies ..."
+	$(PYTHON_INTERPRETER) -m pip install uvicorn~=0.17.0 dask-kubernetes~=0.11.0 apscheduler~=3.6 sqlite3-to-mysql~=1.4
+	@echo "Starting local mlrun..."
+	MLRUN_ARTIFACT_PATH=$$(realpath ./artifacts) MLRUN_ENV_FILE= mlrun db -b
+
+.PHONY: conda-env
+conda-env: ## Create a conda environment
+	@echo "Creating new conda environment $(CONDA_ENV)..."
+	conda create -n $(CONDA_ENV) -y python=$(CONDA_PY_VER) ipykernel graphviz pip
+	test -s ./mlrun.env && conda env config vars set -n $(CONDA_ENV) MLRUN_ENV_FILE=$$(realpath ./mlrun.env)
+	@echo "Installing requirements.txt..."
+	$(CONDA_ACTIVATE) $(CONDA_ENV); pip install -r requirements.txt
+	@echo -e "\nTo run mlrun API as a local process type:\n  conda activate $(CONDA_ENV) && make mlrun-api"