diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/configure_slc_flavor.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/configure_slc_flavor.ipynb index 31ad9a1b..f9576b5f 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/configure_slc_flavor.ipynb +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/configure_slc_flavor.ipynb @@ -23,39 +23,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "e70ad0a9-7042-4fe8-814b-5c586b9bee6d", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "e04d36f4f245412cba260a28c1831118", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "28e8d00f3a9f4a2687c56750c5ff4762", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Box(children=(Box(children=(Label(value='Configuration Store', layout=Layout(border_bottom='solid 1px', border…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "%run ../utils/access_store_ui.ipynb\n", "display(get_access_store_ui('../'))" @@ -72,59 +43,50 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "224345da-f14e-4a45-bf8a-07e3ba0870dc", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6c363a2a8a9d4c89a661c698c5d329f8", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "%run ./utils/slc_ui.ipynb" ] }, + { + "cell_type": "markdown", + "id": "395d6f4f-6e06-45e3-a9a7-54dc94e73332", + "metadata": {}, + "source": [ + "## Instantiate SLCT Manager\n", + "\n", + "We create an instance of the SlctManager class from the notebook connector" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9f42c7c6-0297-4cb4-bedf-d3a2a58e69fc", + "metadata": {}, + "outputs": [], + "source": [ + "from exasol.nb_connector.slct_manager import SlctManager\n", + "slct_manager = SlctManager(ai_lab_config)" + ] + }, { "cell_type": "markdown", "id": "178b59e9-8f46-43b1-b4f8-9cd6d6bf5076", "metadata": {}, "source": [ - "# Configure the Flavor path\n", - "### Choose the flavor source" + "# Configure the Script-Languages directory\n", + "### Choose the source" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "0f8caa77-35a0-403d-af96-e5f8056ba489", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "0794fd907a2449b4be40e4f59ca0ff1a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Box(children=(Box(children=(Label(value='Flavor choice', layout=Layout(border_bottom='solid 1px', border_left=…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "display(get_slc_source_selection_ui(ai_lab_config))" ] @@ -140,25 +102,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "56e16540-5f4b-4d5c-8c83-d34c1217eb00", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "30fbb9a2d7a449f89ad18f605be32a96", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Box(children=(Box(children=(Label(value='Existing script-languages directory', layout=Layout(border_bottom='so…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "display(get_existing_slc_ui(ai_lab_config))" ] @@ -174,20 +121,10 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "7d84889f-7c3a-4f69-9bf1-c193481867af", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "None" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "display(get_slc_target_dir_ui(ai_lab_config))" ] @@ -202,31 +139,13 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "31132726-9827-4c9b-8fb8-0884d2f36e67", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Ready\n" - ] - } - ], + "outputs": [], "source": [ - "from pathlib import Path\n", - "slc_dir = Path(ai_lab_config.get(slc_target_dir_store_key))\n", "if clone_slc_repo(ai_lab_config):\n", - " from git import Repo\n", - " \n", - " if not slc_dir.is_dir():\n", - " print (f\"Cloning into {slc_dir}...\")\n", - " repo = Repo.clone_from(\"https://github.com/exasol/script-languages-release\", slc_dir)\n", - " print (\"Fetching submodules...\")\n", - " repo.submodule_update(recursive=True)\n", - " else:\n", - " print(f\"Directory '{slc_dir}' already exists. Skipping cloning....\")\n", + " slct_manager.clone_slc_repo()\n", "print(\"Ready\")" ] }, @@ -240,38 +159,30 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "bb6b26b9-cb24-4b66-85c9-e15a0c92ff2c", "metadata": { "scrolled": true }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], + "source": [ + "if not slct_manager.check_slc_repo_complete():\n", + " popup_message(f\"The script-languages repository does not fullfill requirements.\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "21840146-28bd-413f-a9ce-cd22b17939b2", + "metadata": {}, "source": [ - "flavor_base_dir = Path(slc_dir) / \"flavors\"\n", - "flavor_dirs = [flavor_dir.name for flavor_dir in flavor_base_dir.iterdir() if flavor_dir.is_dir()]\n", - "required_flavor = \"template-Exasol-all-python-3.10\"\n", - "filtered_flavors = [flavor for flavor in flavor_dirs if str(flavor) == required_flavor]\n", - "if len(filtered_flavors) == 0:\n", - " popup_message(f\"The script-languages repository at '{slc_dir}' does not contain flavor '{required_flavor}'\")\n", - "assert len(filtered_flavors) == 1\n", - "ai_lab_config.save(slc_source_flavor_store_key, str(required_flavor))" + "## Finish\n", + "Now you can continue with [Using a script-languages-container](./using_a_script_languages_container.ipynb) " ] }, { "cell_type": "code", "execution_count": null, - "id": "64f9a4c4-f370-4f68-8b9b-291dcdf32125", + "id": "7c78e2ab-5050-4b8e-acda-f11bf8a1f178", "metadata": {}, "outputs": [], "source": [] diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/using_a_script_languages_container.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/using_a_script_languages_container.ipynb index 48774a54..c24c5b2f 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/using_a_script_languages_container.ipynb +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/using_a_script_languages_container.ipynb @@ -14,7 +14,7 @@ "id": "39333102-7652-4699-ba0e-d184aeec1753", "metadata": {}, "source": [ - "Before we start we need to configure the script-languages directory and flavor. See [Configure SLC Flavor](./configure_slc_flavor.ipynb)" + "Before we start we need to configure the script-languages directory and flavor. See [Configure SLC repository](./configure_slc_flavor.ipynb)" ] }, { @@ -28,39 +28,10 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "da57e77a-b41f-45e3-9a17-4b474bba898b", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "18d8ee1bf38d48e9a5949126f40c4da1", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "210ed84c68d24907a953d7065bc71686", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Box(children=(Box(children=(Label(value='Configuration Store', layout=Layout(border_bottom='solid 1px', border…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "%run ../utils/access_store_ui.ipynb\n", "display(get_access_store_ui('../'))" @@ -71,32 +42,21 @@ "id": "81fbc9a1-718a-4d93-8753-85b032998c1b", "metadata": {}, "source": [ - "### Utility functions\n", + "### Instantiate SLCT Manager\n", "\n", - "#### Working directory context manager\n", - "Let's create a utility context manager function which enters a directory and jumps back to the previous directory on exit." + "We create an instance of the SlctManager class from the notebook connector.\n", + "This class has some utility function which simplifies the use of the `exaslct` API." ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "27352b75-823d-40a1-b8df-d33218fd7de6", "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import contextlib\n", - "from pathlib import Path\n", - "\n", - "@contextlib.contextmanager\n", - "def working_directory(path: Path):\n", - " \"\"\"Changes working directory and returns to previous on exit.\"\"\"\n", - " prev_cwd = Path.cwd()\n", - " os.chdir(path)\n", - " try:\n", - " yield\n", - " finally:\n", - " os.chdir(prev_cwd)" + "from exasol.nb_connector import slct_manager\n", + "slctmanager = slct_manager.SlctManager(ai_lab_config)" ] }, { @@ -109,25 +69,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "c554d698-d5a2-4f9b-846a-f606a3f183b7", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "23a56fb3057f441d92d26d89ebdf4c0d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "from exasol.nb_connector.utils import upward_file_search\n", "\n", @@ -142,41 +87,22 @@ }, { "cell_type": "markdown", - "id": "ebddfdf9-fdac-4bb9-b8a4-58224cf50bda", + "id": "c88668d2-83e5-4812-a05c-936912068f03", "metadata": {}, "source": [ - "## Let's use the flavor as is\n", - "### Check selected flavor path" + "## Use flavor as is\n", + "\n", + "Currently used flavor is:" ] }, { "cell_type": "code", - "execution_count": 5, - "id": "f03723b8-2229-4e10-b801-ac074c7388cb", + "execution_count": null, + "id": "617380fa-6b14-40a5-bffd-ca8fed37950c", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Script-languages repository path is '/home/jupyter/notebooks/script_languages_release'\n", - "Selected flavor is 'template-Exasol-all-python-3.10'\n" - ] - } - ], + "outputs": [], "source": [ - "from pathlib import Path\n", - "slc_source_flavor_store_key = \"slc_flavor\"\n", - "slc_target_dir_store_key = \"slc_target_dir\"\n", - "\n", - "slc_dir = Path(ai_lab_config.get(slc_target_dir_store_key))\n", - "selected_flavor = ai_lab_config.get(slc_source_flavor_store_key)\n", - "print(f\"Script-languages repository path is '{slc_dir}'\")\n", - "print(f\"Selected flavor is '{selected_flavor}'\")\n", - "if not (Path(slc_dir) / \"flavors\" / selected_flavor).is_dir():\n", - " popup_message(\"Invalid script-languages directory or flavor configuration. Please re-run \")\n", - "\n", - "flavor_path = Path(\"flavors\") / selected_flavor\n" + "slct_manager.REQUIRED_FLAVOR" ] }, { @@ -194,30 +120,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "20665390-4f31-48db-ab78-249a1ef629e9", "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Ready\n" - ] - } - ], + "outputs": [], "source": [ - "from exasol_script_languages_container_tool.lib import api as exaslct_api\n", - "\n", - "export_path = \"/home/jupyter/notebooks/script_languages_container/container\"\n", - "output_path = \"/home/jupyter/notebooks/script_languages_container/output\"\n", - "\n", - "\n", - "with working_directory(slc_dir):\n", - " export_result = exaslct_api.export(flavor_path=(str(flavor_path),), export_path=export_path, output_directory=output_path)\n", - "print(\"Ready\")" + "slctmanager.export()" ] }, { @@ -230,47 +140,13 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "fc142500-d855-4112-bba2-ed3395889b9b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Result file: /home/jupyter/notebooks/script_languages_container/container/template-Exasol-all-python-3.10_release.tar.gz\n", - "Directory content:\n" - ] - }, - { - "data": { - "text/html": [ - "container/template-Exasol-all-python-3.10_release.tar.gz
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_container/container/template-Exasol-all-python-3.10_release.tar.gz" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "container/template-Exasol-all-python-3.10_release.tar.gz.sha512sum
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_container/container/template-Exasol-all-python-3.10_release.tar.gz.sha512sum" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "print(f\"Result file: {export_result.export_infos[str(flavor_path)]['release'].output_file}\")\n", "print(\"Directory content:\")\n", - "show_directory_content(Path(export_path))" + "show_directory_content(slctmanager.working_path.export_path)" ] }, { @@ -288,26 +164,12 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "423b261a-3d99-4893-95dc-224a5794c588", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "output/jobs/2024_07_09_15_12_29_1_ExportContainers/logs/main.log
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_container/output/jobs/2024_07_09_15_12_29_1_ExportContainers/logs/main.log" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "build_output = Path(output_path)\n", - "main_logs = list(build_output.glob('**/main.log'))\n", + "main_logs = list(slctmanager.working_path.output_path.glob('**/main.log'))\n", "show_files(main_logs)" ] }, @@ -321,38 +183,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "b5a816ae-1276-4649-997c-d50f0a3be991", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2024-07-09 15:13:10,550 - luigi.scheduler - DEBUG - Done pruning task graph\n", - "2024-07-09 15:13:10,550 - luigi-interface - DEBUG - Done\n", - "2024-07-09 15:13:10,550 - luigi-interface - DEBUG - There are no more tasks to run at this time\n", - "2024-07-09 15:13:10,550 - luigi-interface - INFO - Worker Worker(salt=636688382, workers=5, host=6563e168dc88, username=root, pid=946, sudo_user=ubuntu) was stopped. Shutting down Keep-Alive thread\n", - "2024-07-09 15:13:10,554 - luigi-interface - INFO - \n", - "===== Luigi Execution Summary =====\n", - "\n", - "Scheduled 12 tasks of which:\n", - "* 12 ran successfully:\n", - " - 1 AnalyzeBuildDeps_2b5d06412b(job_id=2024_07_09_15_12_29_1_ExportContainers, no_cache=False, flavor_path=flavors/template-Exasol-all-python-3.10)\n", - " - 1 AnalyzeBuildRun_2b5d06412b(job_id=2024_07_09_15_12_29_1_ExportContainers, no_cache=False, flavor_path=flavors/template-Exasol-all-python-3.10)\n", - " - 1 AnalyzeFlavorBaseDeps_2b5d06412b(job_id=2024_07_09_15_12_29_1_ExportContainers, no_cache=False, flavor_path=flavors/template-Exasol-all-python-3.10)\n", - " - 1 AnalyzeFlavorCustomization_2b5d06412b(job_id=2024_07_09_15_12_29_1_ExportContainers, no_cache=False, flavor_path=flavors/template-Exasol-all-python-3.10)\n", - " - 1 AnalyzeLanguageDeps_2b5d06412b(job_id=2024_07_09_15_12_29_1_ExportContainers, no_cache=False, flavor_path=flavors/template-Exasol-all-python-3.10)\n", - " ...\n", - "\n", - "This progress looks :) because there were no failed tasks or missing dependencies\n", - "\n", - "===== Luigi Execution Summary =====\n", - "\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "tail_file(main_logs[0], 20)" ] @@ -369,61 +203,12 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "1aa24f25-789c-49b3-bb5e-81d49f4afa69", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "output/jobs/2024_07_09_15_12_29_1_ExportContainers/logs/main.log
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_container/output/jobs/2024_07_09_15_12_29_1_ExportContainers/logs/main.log" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "output/jobs/2024_07_09_15_12_29_1_ExportContainers/logs/.ipynb_checkpoints/main-checkpoint.log
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_container/output/jobs/2024_07_09_15_12_29_1_ExportContainers/logs/.ipynb_checkpoints/main-checkpoint.log" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "output/jobs/2024_07_09_15_12_29_1_ExportContainers/outputs/ExportContainers_7d97d3c8be/ExportFlavorContainer_db967c771c/ExportContainerTask_a2f4c91a06/logs/pack_release_file.log
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_container/output/jobs/2024_07_09_15_12_29_1_ExportContainers/outputs/ExportContainers_7d97d3c8be/ExportFlavorContainer_db967c771c/ExportContainerTask_a2f4c91a06/logs/pack_release_file.log" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "output/jobs/2024_07_09_15_12_29_1_ExportContainers/outputs/ExportContainers_7d97d3c8be/ExportFlavorContainer_db967c771c/ExportContainerTask_a2f4c91a06/logs/extract_release_file.log
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_container/output/jobs/2024_07_09_15_12_29_1_ExportContainers/outputs/ExportContainers_7d97d3c8be/ExportFlavorContainer_db967c771c/ExportContainerTask_a2f4c91a06/logs/extract_release_file.log" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "all_logs = list(build_output.glob('**/*.log'))\n", + "all_logs = list(slctmanager.working_path.output_path.glob('**/*.log'))\n", "show_files(all_logs)" ] }, @@ -454,97 +239,12 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "11f3d8ae-d23b-4968-a5c3-a18de8f434d8", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "../script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_customization/Dockerfile
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_customization/Dockerfile" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "../script_languages_release/flavors/template-Exasol-all-python-3.10/FLAVOR_DESCRIPTION.md
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_release/flavors/template-Exasol-all-python-3.10/FLAVOR_DESCRIPTION.md" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "../script_languages_release/flavors/template-Exasol-all-python-3.10/.ipynb_checkpoints/FLAVOR_DESCRIPTION-checkpoint.md
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_release/flavors/template-Exasol-all-python-3.10/.ipynb_checkpoints/FLAVOR_DESCRIPTION-checkpoint.md" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "../script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_base/testconfig
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_base/testconfig" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "../script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_base/build_steps.py
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_base/build_steps.py" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "../script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_base/language_definition
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_base/language_definition" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "../script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_base/derived_from
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_base/derived_from" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "show_directory_content(slc_dir / flavor_path, 2)" + "show_directory_content(slctmanager.slc_dir.flavor_dir, 2)" ] }, { @@ -565,25 +265,12 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "de268b34-99e3-4af6-a9b3-02a44e803c9f", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "../script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_customization/Dockerfile
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_release/flavors/template-Exasol-all-python-3.10/flavor_customization/Dockerfile" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "show_directory_content(slc_dir / flavor_path / \"flavor_customization\")" + "show_directory_content(slctmanager.slc_dir.flavor_dir / \"flavor_customization\")" ] }, { @@ -591,338 +278,9 @@ "id": "5803c74e-2f7f-4d77-b363-f5968d1937bc", "metadata": {}, "source": [ - "#### Dockerfile\n", "The Dockerfile consists of two parts. The first part installs the packages from the package list and should only be change with care. The second part is free for your changes. Read the description in the Dockerfile carefully to find out what you can and shouldn't do." ] }, - { - "cell_type": "code", - "execution_count": 16, - "id": "28ee0a8b-dc06-41ed-b96a-f270ae4bb862", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
############################################################################################\n",
-       "############################################################################################\n",
-       "# This Dockerfile allows you to extend this flavor by installing packages or adding files. \n",
-       "# IF you didn't change the lines below, you can add packages and their version to the  \n",
-       "# files in ./packages and they get automatically installed.                                \n",
-       "############################################################################################\n",
-       "############################################################################################\n",
-       "\n",
-       "#######################################################################\n",
-       "#######################################################################\n",
-       "# Do not change the following lines unless you know what you are doing \n",
-       "#######################################################################\n",
-       "#######################################################################\n",
-       "\n",
-       "FROM {{flavor_base_deps}}\n",
-       "\n",
-       "RUN mkdir -p /build_info/packages/flavor_customization\n",
-       "\n",
-       "COPY flavor_customization/packages/apt_get_packages /build_info/packages/flavor_customization\n",
-       "RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_customization/apt_get_packages --with-versions\n",
-       "\n",
-       "COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization\n",
-       "RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary python3 --with-versions --allow-no-version\n",
-       "\n",
-       "\n",
-       "##########################################################################\n",
-       "##########################################################################\n",
-       "# Below this text you can add any Dockerfile commands except of FROM.            \n",
-       "# However only commands which change the filesystem will be              \n",
-       "# reflected to the final script-language container.                      \n",
-       "# We recommend to use only RUN, COPY and ADD. For example, WORKDIR, USER \n",
-       "# and ENV will be not carried over into the final container. For more \n",
-       "# information about Dockerfile commands, please check the reference \n",
-       "# https://docs.docker.com/engine/reference/builder/#dockerfile-reference \n",
-       "##########################################################################\n",
-       "##########################################################################\n",
-       "\n",
-       "\n",
-       "###########\n",
-       "###########\n",
-       "# Examples:\n",
-       "###########\n",
-       "###########\n",
-       "\n",
-       "#################################################\n",
-       "# Execute a command during the build of the image\n",
-       "#################################################\n",
-       "\n",
-       "# With RUN you can excute any command you usally would run in a bash shell.\n",
-       "# It usually is a good idea to cleanup downloaded files or packages \n",
-       "# caches in the same in the same RUN statement to keep the image size\n",
-       "# small. You can execute multiple shell commands in one RUN statement\n",
-       "# by combine them with \n",
-       "# - `&&` (only execute if the previous command was successfull), \n",
-       "# - `||` (only execute if the previous command was unsuccesfull)\n",
-       "# - `;` (execute regardless of the success of the previous command)\n",
-       "# To format multiple command per RUN statement you can add a line break with `\\`.\n",
-       "# If you need to execute complex sequences of commands you should execute\n",
-       "# scripts which you can copy into the image.\n",
-       "# -------------------------------------------------------------------------------\n",
-       "\n",
-       "# RUN git clone https://github.com/exasol/data-science-examples\n",
-       "# RUN curl -o data-science-examples.zip https://codeload.github.com/exasol/data-science-examples/zip/master && \\\n",
-       "#     unzip data-science-examples.zip && \\\n",
-       "#     rm data-science-examples.zip\n",
-       "\n",
-       "#####################################\n",
-       "# Copy a file from ./ into the image. \n",
-       "#####################################\n",
-       "\n",
-       "# In case of exaslct it is important to prefix source files or directories with `flavor_customization`\n",
-       "# if you want to copy files or directories located in the same directory as this Dockerfile\n",
-       "# -----------------------------------------------------------------------------------------------------\n",
-       "\n",
-       "# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization\n",
-       "# ADD flavor_customization/code.tar.gz /code\n",
-       "
\n" - ], - "text/latex": [ - "\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{} This Dockerfile allows you to extend this flavor by installing packages or adding files. }\n", - "\\PY{c}{\\PYZsh{} IF you didn\\PYZsq{}t change the lines below, you can add packages and their version to the }\n", - "\\PY{c}{\\PYZsh{} files in ./packages and they get automatically installed. }\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{} Do not change the following lines unless you know what you are doing }\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\n", - "\\PY{k}{FROM}\\PY{+w}{ }\\PY{l+s}{\\PYZob{}\\PYZob{}flavor\\PYZus{}base\\PYZus{}deps\\PYZcb{}\\PYZcb{}}\n", - "\n", - "\\PY{k}{RUN}\\PY{+w}{ }mkdir\\PY{+w}{ }\\PYZhy{}p\\PY{+w}{ }/build\\PYZus{}info/packages/flavor\\PYZus{}customization\n", - "\n", - "\\PY{k}{COPY}\\PY{+w}{ }flavor\\PYZus{}customization/packages/apt\\PYZus{}get\\PYZus{}packages\\PY{+w}{ }/build\\PYZus{}info/packages/flavor\\PYZus{}customization\n", - "\\PY{k}{RUN}\\PY{+w}{ }/scripts/install\\PYZus{}scripts/install\\PYZus{}via\\PYZus{}apt.pl\\PY{+w}{ }\\PYZhy{}\\PYZhy{}file\\PY{+w}{ }/build\\PYZus{}info/packages/flavor\\PYZus{}customization/apt\\PYZus{}get\\PYZus{}packages\\PY{+w}{ }\\PYZhy{}\\PYZhy{}with\\PYZhy{}versions\n", - "\n", - "\\PY{k}{COPY}\\PY{+w}{ }flavor\\PYZus{}customization/packages/python3\\PYZus{}pip\\PYZus{}packages\\PY{+w}{ }/build\\PYZus{}info/packages/flavor\\PYZus{}customization\n", - "\\PY{k}{RUN}\\PY{+w}{ }/scripts/install\\PYZus{}scripts/install\\PYZus{}via\\PYZus{}pip.pl\\PY{+w}{ }\\PYZhy{}\\PYZhy{}file\\PY{+w}{ }/build\\PYZus{}info/packages/flavor\\PYZus{}customization/python3\\PYZus{}pip\\PYZus{}packages\\PY{+w}{ }\\PYZhy{}\\PYZhy{}python\\PYZhy{}binary\\PY{+w}{ }python3\\PY{+w}{ }\\PYZhy{}\\PYZhy{}with\\PYZhy{}versions\\PY{+w}{ }\\PYZhy{}\\PYZhy{}allow\\PYZhy{}no\\PYZhy{}version\n", - "\n", - "\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{} Below this text you can add any Dockerfile commands except of FROM. }\n", - "\\PY{c}{\\PYZsh{} However only commands which change the filesystem will be }\n", - "\\PY{c}{\\PYZsh{} reflected to the final script\\PYZhy{}language container. }\n", - "\\PY{c}{\\PYZsh{} We recommend to use only RUN, COPY and ADD. For example, WORKDIR, USER }\n", - "\\PY{c}{\\PYZsh{} and ENV will be not carried over into the final container. For more }\n", - "\\PY{c}{\\PYZsh{} information about Dockerfile commands, please check the reference }\n", - "\\PY{c}{\\PYZsh{} https://docs.docker.com/engine/reference/builder/\\PYZsh{}dockerfile\\PYZhy{}reference }\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\n", - "\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{} Examples:}\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{} Execute a command during the build of the image}\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\n", - "\\PY{c}{\\PYZsh{} With RUN you can excute any command you usally would run in a bash shell.}\n", - "\\PY{c}{\\PYZsh{} It usually is a good idea to cleanup downloaded files or packages }\n", - "\\PY{c}{\\PYZsh{} caches in the same in the same RUN statement to keep the image size}\n", - "\\PY{c}{\\PYZsh{} small. You can execute multiple shell commands in one RUN statement}\n", - "\\PY{c}{\\PYZsh{} by combine them with }\n", - "\\PY{c}{\\PYZsh{} \\PYZhy{} `\\PYZam{}\\PYZam{}` (only execute if the previous command was successfull), }\n", - "\\PY{c}{\\PYZsh{} \\PYZhy{} `||` (only execute if the previous command was unsuccesfull)}\n", - "\\PY{c}{\\PYZsh{} \\PYZhy{} `;` (execute regardless of the success of the previous command)}\n", - "\\PY{c}{\\PYZsh{} To format multiple command per RUN statement you can add a line break with `\\PYZbs{}`.}\n", - "\\PY{c}{\\PYZsh{} If you need to execute complex sequences of commands you should execute}\n", - "\\PY{c}{\\PYZsh{} scripts which you can copy into the image.}\n", - "\\PY{c}{\\PYZsh{} \\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}}\n", - "\n", - "\\PY{c}{\\PYZsh{} RUN git clone https://github.com/exasol/data\\PYZhy{}science\\PYZhy{}examples}\n", - "\\PY{c}{\\PYZsh{} RUN curl \\PYZhy{}o data\\PYZhy{}science\\PYZhy{}examples.zip https://codeload.github.com/exasol/data\\PYZhy{}science\\PYZhy{}examples/zip/master \\PYZam{}\\PYZam{} \\PYZbs{}}\n", - "\\PY{c}{\\PYZsh{} unzip data\\PYZhy{}science\\PYZhy{}examples.zip \\PYZam{}\\PYZam{} \\PYZbs{}}\n", - "\\PY{c}{\\PYZsh{} rm data\\PYZhy{}science\\PYZhy{}examples.zip}\n", - "\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\\PY{c}{\\PYZsh{} Copy a file from ./ into the image. }\n", - "\\PY{c}{\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}\\PYZsh{}}\n", - "\n", - "\\PY{c}{\\PYZsh{} In case of exaslct it is important to prefix source files or directories with `flavor\\PYZus{}customization`}\n", - "\\PY{c}{\\PYZsh{} if you want to copy files or directories located in the same directory as this Dockerfile}\n", - "\\PY{c}{\\PYZsh{} \\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}\\PYZhy{}}\n", - "\n", - "\\PY{c}{\\PYZsh{} COPY flavor\\PYZus{}customization/packages/python3\\PYZus{}pip\\PYZus{}packages /build\\PYZus{}info/packages/flavor\\PYZus{}customization}\n", - "\\PY{c}{\\PYZsh{} ADD flavor\\PYZus{}customization/code.tar.gz /code}\n", - "\\end{Verbatim}\n" - ], - "text/plain": [ - "############################################################################################\n", - "############################################################################################\n", - "# This Dockerfile allows you to extend this flavor by installing packages or adding files. \n", - "# IF you didn't change the lines below, you can add packages and their version to the \n", - "# files in ./packages and they get automatically installed. \n", - "############################################################################################\n", - "############################################################################################\n", - "\n", - "#######################################################################\n", - "#######################################################################\n", - "# Do not change the following lines unless you know what you are doing \n", - "#######################################################################\n", - "#######################################################################\n", - "\n", - "FROM {{flavor_base_deps}}\n", - "\n", - "RUN mkdir -p /build_info/packages/flavor_customization\n", - "\n", - "COPY flavor_customization/packages/apt_get_packages /build_info/packages/flavor_customization\n", - "RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_customization/apt_get_packages --with-versions\n", - "\n", - "COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization\n", - "RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary python3 --with-versions --allow-no-version\n", - "\n", - "\n", - "##########################################################################\n", - "##########################################################################\n", - "# Below this text you can add any Dockerfile commands except of FROM. \n", - "# However only commands which change the filesystem will be \n", - "# reflected to the final script-language container. \n", - "# We recommend to use only RUN, COPY and ADD. For example, WORKDIR, USER \n", - "# and ENV will be not carried over into the final container. For more \n", - "# information about Dockerfile commands, please check the reference \n", - "# https://docs.docker.com/engine/reference/builder/#dockerfile-reference \n", - "##########################################################################\n", - "##########################################################################\n", - "\n", - "\n", - "###########\n", - "###########\n", - "# Examples:\n", - "###########\n", - "###########\n", - "\n", - "#################################################\n", - "# Execute a command during the build of the image\n", - "#################################################\n", - "\n", - "# With RUN you can excute any command you usally would run in a bash shell.\n", - "# It usually is a good idea to cleanup downloaded files or packages \n", - "# caches in the same in the same RUN statement to keep the image size\n", - "# small. You can execute multiple shell commands in one RUN statement\n", - "# by combine them with \n", - "# - `&&` (only execute if the previous command was successfull), \n", - "# - `||` (only execute if the previous command was unsuccesfull)\n", - "# - `;` (execute regardless of the success of the previous command)\n", - "# To format multiple command per RUN statement you can add a line break with `\\`.\n", - "# If you need to execute complex sequences of commands you should execute\n", - "# scripts which you can copy into the image.\n", - "# -------------------------------------------------------------------------------\n", - "\n", - "# RUN git clone https://github.com/exasol/data-science-examples\n", - "# RUN curl -o data-science-examples.zip https://codeload.github.com/exasol/data-science-examples/zip/master && \\\n", - "# unzip data-science-examples.zip && \\\n", - "# rm data-science-examples.zip\n", - "\n", - "#####################################\n", - "# Copy a file from ./ into the image. \n", - "#####################################\n", - "\n", - "# In case of exaslct it is important to prefix source files or directories with `flavor_customization`\n", - "# if you want to copy files or directories located in the same directory as this Dockerfile\n", - "# -----------------------------------------------------------------------------------------------------\n", - "\n", - "# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization\n", - "# ADD flavor_customization/code.tar.gz /code\n" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "show_docker_file(slc_dir / flavor_path / \"flavor_customization\" / \"Dockerfile\")" - ] - }, { "cell_type": "markdown", "id": "562eac65-b205-49d4-971e-9ed0090796b4", @@ -934,130 +292,12 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "ce88261d-efc4-470e-a875-447a28c0125c", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
# This file specifies the package list which gets installed via pip for python3.\n",
-       "# You must specify the the package and its version separated by a |.\n",
-       "# We recommend here the usage of package versions, to ensure that the container \n",
-       "# builds are reproducible. However, we allow also packages without version.\n",
-       "# As you can see, this file can contain comments which start with #.\n",
-       "# If a line starts with # the whole line is a comment, however you can\n",
-       "# also start a comment after the package definition.\n",
-       "\n",
-       "#tensorflow-probability|0.9.0\n",
-       "
\n" - ], - "text/latex": [ - "\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n", - "\\PY{c+c1}{\\PYZsh{} This file specifies the package list which gets installed via pip for python3.}\n", - "\\PY{c+c1}{\\PYZsh{} You must specify the the package and its version separated by a |.}\n", - "\\PY{c+c1}{\\PYZsh{} We recommend here the usage of package versions, to ensure that the container }\n", - "\\PY{c+c1}{\\PYZsh{} builds are reproducible. However, we allow also packages without version.}\n", - "\\PY{c+c1}{\\PYZsh{} As you can see, this file can contain comments which start with \\PYZsh{}.}\n", - "\\PY{c+c1}{\\PYZsh{} If a line starts with \\PYZsh{} the whole line is a comment, however you can}\n", - "\\PY{c+c1}{\\PYZsh{} also start a comment after the package definition.}\n", - "\n", - "\\PY{c+c1}{\\PYZsh{}tensorflow\\PYZhy{}probability|0.9.0}\n", - "\\end{Verbatim}\n" - ], - "text/plain": [ - "# This file specifies the package list which gets installed via pip for python3.\n", - "# You must specify the the package and its version separated by a |.\n", - "# We recommend here the usage of package versions, to ensure that the container \n", - "# builds are reproducible. However, we allow also packages without version.\n", - "# As you can see, this file can contain comments which start with #.\n", - "# If a line starts with # the whole line is a comment, however you can\n", - "# also start a comment after the package definition.\n", - "\n", - "#tensorflow-probability|0.9.0" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "custom_pip_file = slc_dir / flavor_path / \"flavor_customization\" / \"packages\" / \"python3_pip_packages\"\n", - "show_pip_file(custom_pip_file)" + "show_files([slctmanager.custom_pip_file])\n" ] }, { @@ -1065,143 +305,23 @@ "id": "5857a6fb-f0b7-413f-8ce5-88ced8645abb", "metadata": {}, "source": [ - "We are now going to append the \"xgboost\" Python package to one of the package lists by adding `xgboost|2.0.3` and `scikit-learn|1.5.0` to the `flavor_customization/packages/python3_pip_packages` file. (Note that running the following the command multiple times will iteratively append the packages)" + "We are now going to append the \"xgboost\" Python package to one of the package lists by adding `xgboost|2.0.3` and `scikit-learn|1.5.0` to the `flavor_customization/packages/python3_pip_packages` file. \n", + "Notes:\n", + " - running the following the command multiple times will iteratively append the packages)\n", + " - you can also click on the link and modify the file directly\n" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "fe3db39f-5588-44a9-9e64-d4ec495423ee", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
# This file specifies the package list which gets installed via pip for python3.\n",
-       "# You must specify the the package and its version separated by a |.\n",
-       "# We recommend here the usage of package versions, to ensure that the container \n",
-       "# builds are reproducible. However, we allow also packages without version.\n",
-       "# As you can see, this file can contain comments which start with #.\n",
-       "# If a line starts with # the whole line is a comment, however you can\n",
-       "# also start a comment after the package definition.\n",
-       "\n",
-       "#tensorflow-probability|0.9.0\n",
-       "xgboost|2.0.3\n",
-       "scikit-learn|1.5.0\n",
-       "
\n" - ], - "text/latex": [ - "\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n", - "\\PY{c+c1}{\\PYZsh{} This file specifies the package list which gets installed via pip for python3.}\n", - "\\PY{c+c1}{\\PYZsh{} You must specify the the package and its version separated by a |.}\n", - "\\PY{c+c1}{\\PYZsh{} We recommend here the usage of package versions, to ensure that the container }\n", - "\\PY{c+c1}{\\PYZsh{} builds are reproducible. However, we allow also packages without version.}\n", - "\\PY{c+c1}{\\PYZsh{} As you can see, this file can contain comments which start with \\PYZsh{}.}\n", - "\\PY{c+c1}{\\PYZsh{} If a line starts with \\PYZsh{} the whole line is a comment, however you can}\n", - "\\PY{c+c1}{\\PYZsh{} also start a comment after the package definition.}\n", - "\n", - "\\PY{c+c1}{\\PYZsh{}tensorflow\\PYZhy{}probability|0.9.0}\n", - "\\PY{n}{xgboost}\\PY{err}{|}\\PY{n}{2}\\PY{p}{.}\\PY{n}{0}\\PY{p}{.}\\PY{n}{3}\n", - "\\PY{n}{scikit\\PYZhy{}learn}\\PY{err}{|}\\PY{n}{1}\\PY{p}{.}\\PY{n}{5}\\PY{p}{.}\\PY{n}{0}\n", - "\\end{Verbatim}\n" - ], - "text/plain": [ - "# This file specifies the package list which gets installed via pip for python3.\n", - "# You must specify the the package and its version separated by a |.\n", - "# We recommend here the usage of package versions, to ensure that the container \n", - "# builds are reproducible. However, we allow also packages without version.\n", - "# As you can see, this file can contain comments which start with #.\n", - "# If a line starts with # the whole line is a comment, however you can\n", - "# also start a comment after the package definition.\n", - "\n", - "#tensorflow-probability|0.9.0\n", - "xgboost|2.0.3\n", - "scikit-learn|1.5.0" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "with open(custom_pip_file, \"a\") as f:\n", - " print(\"xgboost|2.0.3\", file=f)\n", - " print(\"scikit-learn|1.5.0\", file=f)\n", - "show_pip_file(custom_pip_file)" + "xgboost_pkg = slct_manager.PipPackageDefinition(pkg=\"xgboost\", version=\"2.0.3\")\n", + "scikit_learn_pkg = slct_manager.PipPackageDefinition(pkg=\"scikit-learn\", version=\"1.5.0\")\n", + "slctmanager.append_custom_packages([xgboost_pkg, scikit_learn_pkg])\n", + "show_files([slctmanager.custom_pip_file])" ] }, { @@ -1215,23 +335,12 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "2e7e9bbb-bf77-474f-949f-9521e9e3fd86", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING - AnalyzeFlavorCustomization_ad071a264b(job_id=2024_07_09_15_16_24_2_ExportContainers, no_cache=False, flavor_path=flavors/template-Exasol-all-python-3.10): Image exasol/script-language-container:template-Exasol-all-python-3.10-flavor_customization_VLB2GXA4ZGMJSNHG6GIKPTTJJ6ISQ7JMS2P3IPUKNEUZ5WI44V5A not in registry, got exception 404 Client Error for http+docker://localhost/v1.46/images/create?tag=template-Exasol-all-python-3.10-flavor_customization_VLB2GXA4ZGMJSNHG6GIKPTTJJ6ISQ7JMS2P3IPUKNEUZ5WI44V5A&fromImage=exasol%2Fscript-language-container: Not Found (\"manifest for exasol/script-language-container:template-Exasol-all-python-3.10-flavor_customization_VLB2GXA4ZGMJSNHG6GIKPTTJJ6ISQ7JMS2P3IPUKNEUZ5WI44V5A not found: manifest unknown: manifest unknown\")\n", - "WARNING - AnalyzeRelease_ad071a264b(job_id=2024_07_09_15_16_24_2_ExportContainers, no_cache=False, flavor_path=flavors/template-Exasol-all-python-3.10): Image exasol/script-language-container:template-Exasol-all-python-3.10-release_QLGFA5R27PSMHZRJWXWA6AD6B6BRFIZYDL5ZAJI4NL2E6WZMYCDA not in registry, got exception 404 Client Error for http+docker://localhost/v1.46/images/create?tag=template-Exasol-all-python-3.10-release_QLGFA5R27PSMHZRJWXWA6AD6B6BRFIZYDL5ZAJI4NL2E6WZMYCDA&fromImage=exasol%2Fscript-language-container: Not Found (\"manifest for exasol/script-language-container:template-Exasol-all-python-3.10-release_QLGFA5R27PSMHZRJWXWA6AD6B6BRFIZYDL5ZAJI4NL2E6WZMYCDA not found: manifest unknown: manifest unknown\")\n", - "Ready\n" - ] - } - ], + "outputs": [], "source": [ - "with working_directory(slc_dir):\n", - " export_result = exaslct_api.export(flavor_path=(str(flavor_path),), export_path=export_path)\n", + "slctmanager.export()\n", "print(\"Ready\")" ] }, @@ -1245,37 +354,12 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "id": "f683dec3-101d-451d-b590-0518a8fdfdc9", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "output/cache/exports/template-Exasol-all-python-3.10-release-BFRSH344TDRPT7LK2FBOJK4KBIDW6A253FFPYEUYT4O2ERFMTCNA.tar.gz
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_container/output/cache/exports/template-Exasol-all-python-3.10-release-BFRSH344TDRPT7LK2FBOJK4KBIDW6A253FFPYEUYT4O2ERFMTCNA.tar.gz" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "output/cache/exports/template-Exasol-all-python-3.10-release-BFRSH344TDRPT7LK2FBOJK4KBIDW6A253FFPYEUYT4O2ERFMTCNA.tar.gz.sha512sum
" - ], - "text/plain": [ - "/home/jupyter/notebooks/script_languages_container/output/cache/exports/template-Exasol-all-python-3.10-release-BFRSH344TDRPT7LK2FBOJK4KBIDW6A253FFPYEUYT4O2ERFMTCNA.tar.gz.sha512sum" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ - "show_directory_content(build_output / \"cache\" / \"exports\")" + "show_directory_content(slctmanager.working_path.output_path / \"cache\" / \"exports\")" ] }, { @@ -1285,27 +369,7 @@ "source": [ "### Testing the new Script-Language Container\n", "\n", - "Now, that we have an updated container, we need to check if our changes were successful. For that we are going to upload the container to an Exasol Database and have a look into it.\n", - "First we get the necessary information from the Secret Store." - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "edbe432c-5440-40f5-b9be-fd4aef89d5c3", - "metadata": {}, - "outputs": [], - "source": [ - "from exasol.nb_connector.ai_lab_config import AILabConfig as CKey, StorageBackend\n", - "bucketfs_name = ai_lab_config.get(CKey.bfs_service)\n", - "bucket_name = ai_lab_config.get(CKey.bfs_bucket)\n", - "database_host = ai_lab_config.get(CKey.bfs_host_name)\n", - "bucketfs_port = ai_lab_config.get(CKey.bfs_port)\n", - "bucketfs_username = ai_lab_config.get(CKey.bfs_user)\n", - "bucketfs_password = ai_lab_config.get(CKey.bfs_password)\n", - "\n", - "PATH_IN_BUCKET = \"container\"\n", - "RELEASE_NAME = \"current\"" + "Now, that we have an updated container, we need to check if our changes were successful. For that we are going to upload the container to an Exasol Database and have a look into it." ] }, { @@ -1321,45 +385,12 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "id": "28f7f45c-8d38-49c0-860a-132b4e8f12c4", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Uploaded .build_output/cache/exports/template-Exasol-all-python-3.10-release-QLGFA5R27PSMHZRJWXWA6AD6B6BRFIZYDL5ZAJI4NL2E6WZMYCDA.tar.gz to\n", - "http://172.17.0.3:2580/default/container/template-Exasol-all-python-3.10-release-current.tar.gz\n", - "\n", - "\n", - "In SQL, you can activate the languages supported by the template-Exasol-all-python-3.10\n", - "flavor by using the following statements:\n", - "\n", - "\n", - "To activate the flavor only for the current session:\n", - "\n", - "ALTER SESSION SET SCRIPT_LANGUAGES='PYTHON3=localzmq+protobuf:///bfsdefault/default/container/template-Exasol-all-python-3.10-release-current?lang=python#buckets/bfsdefault/default/container/template-Exasol-all-python-3.10-release-current/exaudf/exaudfclient_py3';\n", - "\n", - "\n", - "To activate the flavor on the system:\n", - "\n", - "ALTER SYSTEM SET SCRIPT_LANGUAGES='PYTHON3=localzmq+protobuf:///bfsdefault/default/container/template-Exasol-all-python-3.10-release-current?lang=python#buckets/bfsdefault/default/container/template-Exasol-all-python-3.10-release-current/exaudf/exaudfclient_py3';\n", - "\n", - "=================================================\n", - "\n" - ] - } - ], + "outputs": [], "source": [ - "with working_directory(slc_dir):\n", - " upload_result = exaslct_api.upload(flavor_path=(str(flavor_path),), database_host=database_host, bucketfs_name=bucketfs_name, \n", - " bucket_name=bucket_name, bucketfs_port=bucketfs_port, bucketfs_username=bucketfs_username, \n", - " bucketfs_password=bucketfs_password, path_in_bucket=PATH_IN_BUCKET, release_name=RELEASE_NAME)\n", - " \n", - " # The returned string contains the reference to output file containing information about how to use the uploaded container\n", - " tail_file(slc_dir / str(upload_result), 100)\n" + "slctmanager.upload()" ] }, { @@ -1367,53 +398,45 @@ "id": "30014673-f03f-4f38-bd3f-5f9f27ad10b0", "metadata": {}, "source": [ - "#### Getting the language container activation statement without upload\n", - "Sometimes you can't use the upload command to upload your container to the BucketFS. To get the language activation statement regardless of that, you can use the generate-language-activation command. The API returns a tuple with 3 items:\n", - "- The ALTER SESSION SQL command to activate the container\n", - "- The ALTER SYSTEM SQL command to activate the container" + "This command also stores the activation statement in the ai-lab-config.\n", + "We can verify it with:" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "id": "d2f3130b-70f5-44ce-8d27-aa1efe6aa545", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "\n", - "In SQL, you can activate the languages supported by the template-Exasol-all-python-3.10\n", - "flavor by using the following statements:\n", - "\n", - "\n", - "To activate the flavor only for the current session:\n", - "\n", - "ALTER SESSION SET SCRIPT_LANGUAGES='PYTHON3=localzmq+protobuf:///bfsdefault/default/container/template-Exasol-all-python-3.10-release-current?lang=python#buckets/bfsdefault/default/container/template-Exasol-all-python-3.10-release-current/exaudf/exaudfclient_py3';\n", - "\n", - "\n", - "To activate the flavor on the system:\n", - "\n", - "ALTER SYSTEM SET SCRIPT_LANGUAGES='PYTHON3=localzmq+protobuf:///bfsdefault/default/container/template-Exasol-all-python-3.10-release-current?lang=python#buckets/bfsdefault/default/container/template-Exasol-all-python-3.10-release-current/exaudf/exaudfclient_py3';\n", - "\n" - ] - } - ], + "outputs": [], "source": [ - "from IPython.display import JSON\n", - "import json\n", - "PATH_IN_BUCKET = \"container\"\n", - "with working_directory(slc_dir):\n", - " container_name = f\"{selected_flavor}-release-{RELEASE_NAME}\"\n", - " result = exaslct_api.generate_language_activation(flavor_path=flavor_path, bucketfs_name=bucketfs_name, \n", - " bucket_name=bucket_name, container_name=container_name,\n", - " path_in_bucket=PATH_IN_BUCKET)\n", - " alter_session_cmd = result[0]\n", - " alter_system_cmd = result[1]\n", - " explanation = result[2]\n", - " print(explanation)" + "ai_lab_config.get(slct_manager.ACTIVATION_KEY)" + ] + }, + { + "cell_type": "markdown", + "id": "c5e340f7-8061-4113-88f0-15e16009782a", + "metadata": {}, + "source": [ + "The syntax is: `alias=url`. The alias is used when defining an UDF, the URL only for the language activation.\n", + "For simplicity the `SlctManager` provides easy access to the alias, as we need it later:" + ] + }, + { + "cell_type": "markdown", + "id": "e661b9fe-11a4-46bb-b98b-3c6d887a3d1d", + "metadata": {}, + "source": [ + "For simplicity the `SlctManager` provides access to the alias, as we need it later to define our UDF's:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd1b2729-cf64-4cb7-a373-84c7db2a0000", + "metadata": {}, + "outputs": [], + "source": [ + "slctmanager.language_alias" ] }, { @@ -1422,35 +445,21 @@ "metadata": {}, "source": [ "#### Connecting to the database and activate the container\n", - "Once we have a connection to the database we run the ALTER SESSION statement or ALTER SYSTEM statement (if you want to activate the container permanently and globally) we got from `generate_language_activation`." + "Once we have a connection to the database we run the ALTER SESSION statement or ALTER SYSTEM statement (if you want to activate the container permanently and globally) we got from `generate_language_activation`.\n", + "The `notebook` connector package provides a utility method, for creating an `pyexasol` connection and activating all registered languages:" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "357caa4f-74ca-4d41-9157-04608a1a3c8c", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 25, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "from exasol.nb_connector.connections import open_pyexasol_connection\n", - "\n", + "from exasol.nb_connector.language_container_activation import open_pyexasol_connection_with_lang_definitions\n", "\n", - "# Create an Exasol connection\n", - "conn = open_pyexasol_connection(ai_lab_config, compression=True)\n", - "conn.execute(\"DROP SCHEMA IF EXISTS SLC_TUTORIAL\")\n", - "conn.execute(\"CREATE SCHEMA SLC_TUTORIAL\")\n", - "conn.execute(alter_session_cmd)\n" + "conn = open_pyexasol_connection_with_lang_definitions(ai_lab_config, compression=True)\n", + "conn.execute(\"CREATE SCHEMA SLC_TUTORIAL\")" ] }, { @@ -1464,26 +473,15 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": null, "id": "2036fa12-bd31-4526-a9d6-9b618e1e8c58", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 26, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import textwrap\n", "\n", - "conn.execute(textwrap.dedent(\"\"\"\n", - "CREATE OR REPLACE PYTHON3 SCALAR SCRIPT execute_shell_command_py3(command VARCHAR(2000000), split_output boolean)\n", + "conn.execute(textwrap.dedent(f\"\"\"\n", + "CREATE OR REPLACE {slctmanager.language_alias} SCALAR SCRIPT execute_shell_command_py3(command VARCHAR(2000000), split_output boolean)\n", "EMITS (lines VARCHAR(2000000)) AS\n", "import subprocess\n", "\n", @@ -1521,34 +519,10 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "id": "51d61556-f9f1-4749-b70c-8fc21e141662", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Package Version\n", - "--------------- -----------\n", - "joblib 1.4.2\n", - "numpy 1.26.4\n", - "pandas 2.2.2\n", - "pip 21.3.1\n", - "pyarrow 16.0.0\n", - "python-dateutil 2.9.0.post0\n", - "pytz 2024.1\n", - "scikit-learn 1.5.0\n", - "scipy 1.14.0\n", - "setuptools 70.0.0\n", - "six 1.16.0\n", - "threadpoolctl 3.5.0\n", - "tzdata 2024.1\n", - "wheel 0.43.0\n", - "xgboost 2.0.3\n" - ] - } - ], + "outputs": [], "source": [ "rs=conn.execute(\"\"\"select execute_shell_command_py3('python3 -m pip list', true)\"\"\")\n", "for r in rs: \n", @@ -1568,52 +542,10 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "id": "92b4f6d4-9235-4e51-a3f4-1ee73aae90ef", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/build_info\n", - "/build_info/dockerfiles\n", - "/build_info/dockerfiles/template-Exasol-all-python-3.10-udfclient_deps\n", - "/build_info/dockerfiles/template-Exasol-all-python-3.10-release\n", - "/build_info/dockerfiles/template-Exasol-all-python-3.10-flavor_customization\n", - "/build_info/dockerfiles/template-Exasol-all-python-3.10-build_run\n", - "/build_info/dockerfiles/template-Exasol-all-python-3.10-language_deps\n", - "/build_info/dockerfiles/template-Exasol-all-python-3.10-build_deps\n", - "/build_info/dockerfiles/template-Exasol-all-python-3.10-flavor_base_deps\n", - "/build_info/packages\n", - "/build_info/packages/flavor_customization\n", - "/build_info/packages/flavor_customization/python3_pip_packages\n", - "/build_info/packages/flavor_customization/apt_get_packages\n", - "/build_info/packages/language_deps\n", - "/build_info/packages/language_deps/python3_pip_packages\n", - "/build_info/packages/language_deps/apt_get_packages\n", - "/build_info/packages/udfclient_deps\n", - "/build_info/packages/udfclient_deps/apt_get_packages\n", - "/build_info/packages/build_deps\n", - "/build_info/packages/build_deps/apt_get_packages\n", - "/build_info/packages/flavor_base_deps\n", - "/build_info/packages/flavor_base_deps/python3_pip_packages\n", - "/build_info/packages/flavor_base_deps/apt_get_packages\n", - "/build_info/actual_installed_packages\n", - "/build_info/actual_installed_packages/release\n", - "/build_info/actual_installed_packages/release/python3_pip_packages\n", - "/build_info/actual_installed_packages/release/apt_get_packages\n", - "/build_info/image_info\n", - "/build_info/image_info/template-Exasol-all-python-3.10-udfclient_deps\n", - "/build_info/image_info/template-Exasol-all-python-3.10-release\n", - "/build_info/image_info/template-Exasol-all-python-3.10-flavor_customization\n", - "/build_info/image_info/template-Exasol-all-python-3.10-build_run\n", - "/build_info/image_info/template-Exasol-all-python-3.10-language_deps\n", - "/build_info/image_info/template-Exasol-all-python-3.10-build_deps\n", - "/build_info/image_info/template-Exasol-all-python-3.10-flavor_base_deps\n" - ] - } - ], + "outputs": [], "source": [ "rs=conn.execute(\"\"\"select execute_shell_command_py3('find /build_info', true)\"\"\")\n", "for r in rs: \n", @@ -1630,32 +562,10 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "4f716612-fa6e-496f-9ec9-39fefb647b60", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "joblib|1.4.2\n", - "numpy|1.26.4\n", - "pandas|2.2.2\n", - "pip|21.3.1\n", - "pyarrow|16.0.0\n", - "python-dateutil|2.9.0.post0\n", - "pytz|2024.1\n", - "scikit-learn|1.5.0\n", - "scipy|1.14.0\n", - "setuptools|70.0.0\n", - "six|1.16.0\n", - "threadpoolctl|3.5.0\n", - "tzdata|2024.1\n", - "wheel|0.43.0\n", - "xgboost|2.0.3\n" - ] - } - ], + "outputs": [], "source": [ "rs=conn.execute(\"\"\"select execute_shell_command_py3('cat /build_info/actual_installed_packages/release/python3_pip_packages', true)\"\"\")\n", "for r in rs: \n", @@ -1672,28 +582,10 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "aefe0c99-32d5-4357-bc2e-1131c7b38b9a", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "# This file specifies the package list which gets installed via pip for python3.\n", - "# You must specify the the package and its version separated by a |.\n", - "# We recommend here the usage of package versions, to ensure that the container \n", - "# builds are reproducible. However, we allow also packages without version.\n", - "# As you can see, this file can contain comments which start with #.\n", - "# If a line starts with # the whole line is a comment, however you can\n", - "# also start a comment after the package definition.\n", - "\n", - "#tensorflow-probability|0.9.0\n", - "xgboost|2.0.3\n", - "scikit-learn|1.5.0\n" - ] - } - ], + "outputs": [], "source": [ "rs=conn.execute(\"\"\"select execute_shell_command_py3('cat /build_info/packages/flavor_customization/python3_pip_packages', true)\"\"\")\n", "for r in rs:\n", @@ -1715,24 +607,13 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "id": "91b3c817-dbd4-411e-ab45-3ac5008624fc", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[('success',)]" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "conn.execute(textwrap.dedent(\"\"\"\n", - "CREATE OR REPLACE PYTHON3 SET SCRIPT test_xgboost(i integer)\n", + "conn.execute(textwrap.dedent(f\"\"\"\n", + "CREATE OR REPLACE {slctmanager.language_alias} SET SCRIPT test_xgboost(i integer)\n", "EMITS (o VARCHAR(2000000)) AS\n", "\n", "def run(ctx):\n", @@ -1749,260 +630,10 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "id": "c3b18d63-22f8-4e33-9701-7e4f6759f970", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
O1O2O3
00.0056380.9775520.016810
10.9825470.0139580.003495
20.0036370.0061520.990210
30.0056470.9791700.015182
40.0054010.9364090.058190
50.9860800.0104130.003508
60.0050480.9897580.005194
70.0156950.0665310.917774
80.0063380.9487720.044891
90.0043900.9904130.005198
100.0178500.0756660.906483
110.9905420.0056190.003840
120.9825470.0139580.003495
130.9908550.0056200.003525
140.9908550.0056200.003525
150.0043570.9863810.009262
160.0028350.0054570.991708
170.0050690.9875320.007398
180.0043550.9826590.012985
190.0028370.0048480.992315
200.9908550.0056200.003525
210.0117750.0854630.902761
220.9908550.0056200.003525
230.0028370.0048480.992315
240.0056170.0108120.983570
250.0036340.0069950.989370
260.0036370.0061520.990210
270.0056170.0108120.983570
280.9905420.0056190.003840
290.9908550.0056200.003525
\n", - "
" - ], - "text/plain": [ - " O1 O2 O3\n", - "0 0.005638 0.977552 0.016810\n", - "1 0.982547 0.013958 0.003495\n", - "2 0.003637 0.006152 0.990210\n", - "3 0.005647 0.979170 0.015182\n", - "4 0.005401 0.936409 0.058190\n", - "5 0.986080 0.010413 0.003508\n", - "6 0.005048 0.989758 0.005194\n", - "7 0.015695 0.066531 0.917774\n", - "8 0.006338 0.948772 0.044891\n", - "9 0.004390 0.990413 0.005198\n", - "10 0.017850 0.075666 0.906483\n", - "11 0.990542 0.005619 0.003840\n", - "12 0.982547 0.013958 0.003495\n", - "13 0.990855 0.005620 0.003525\n", - "14 0.990855 0.005620 0.003525\n", - "15 0.004357 0.986381 0.009262\n", - "16 0.002835 0.005457 0.991708\n", - "17 0.005069 0.987532 0.007398\n", - "18 0.004355 0.982659 0.012985\n", - "19 0.002837 0.004848 0.992315\n", - "20 0.990855 0.005620 0.003525\n", - "21 0.011775 0.085463 0.902761\n", - "22 0.990855 0.005620 0.003525\n", - "23 0.002837 0.004848 0.992315\n", - "24 0.005617 0.010812 0.983570\n", - "25 0.003634 0.006995 0.989370\n", - "26 0.003637 0.006152 0.990210\n", - "27 0.005617 0.010812 0.983570\n", - "28 0.990542 0.005619 0.003840\n", - "29 0.990855 0.005620 0.003525" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "conn.execute(textwrap.dedent(\"\"\"\n", "CREATE OR REPLACE PYTHON3 SET SCRIPT test_xgboost(i integer)\n", diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/utils/slc_ui.ipynb b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/utils/slc_ui.ipynb index 3b220204..ea8f6b4f 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/utils/slc_ui.ipynb +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook/script_languages_container/utils/slc_ui.ipynb @@ -19,7 +19,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "7d5b5aab56a44f02af835eeccc0ba1af", + "model_id": "9a723961b7434468b2124780cf364706", "version_major": 2, "version_minor": 0 }, @@ -43,33 +43,24 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "id": "a2e90ebe-37d6-43e6-9677-dace6c06d942", "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name 'Secrets' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[6], line 8\u001b[0m\n\u001b[1;32m 4\u001b[0m slc_source_flavor_store_key \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mslc_flavor_path\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mipyfilechooser\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m FileChooser\n\u001b[0;32m----> 8\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21muse_slc_repo\u001b[39m(conf: \u001b[43mSecrets\u001b[49m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mbool\u001b[39m:\n\u001b[1;32m 9\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m conf\u001b[38;5;241m.\u001b[39mget(slc_source_store_key) \u001b[38;5;241m==\u001b[39m slc_source_slc_repo_option_value\n\u001b[1;32m 11\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_slc_source_selection_ui\u001b[39m(conf: Secrets) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m widgets\u001b[38;5;241m.\u001b[39mWidget:\n", - "\u001b[0;31mNameError\u001b[0m: name 'Secrets' is not defined" - ] - } - ], + "outputs": [], "source": [ - "slc_source_store_key = \"slc_source\"\n", + "from exasol.nb_connector.ai_lab_config import AILabConfig\n", + "\n", "slc_source_slc_repo_option_value = \"Clone script languages release repository\"\n", - "slc_target_dir_store_key = \"slc_target_dir\"\n", - "slc_source_flavor_store_key = \"slc_flavor\"\n", "\n", "from ipyfilechooser import FileChooser\n", + "from pathlib import Path\n", + "from exasol.nb_connector.secret_store import Secrets\n", + "\n", + "\n", + "NOTEBOOKS_DIRECTORY = str(Path().parent.parent)\n", "\n", - "NOTEBOOKS_DIRECTORY = \"/home/jupyter/notebooks\"\n", "def clone_slc_repo(conf: Secrets) -> bool:\n", - " return conf.get(slc_source_store_key) == slc_source_slc_repo_option_value\n", + " return conf.get(AILabConfig.slc_source) == slc_source_slc_repo_option_value\n", "\n", "def get_slc_source_selection_ui(conf: Secrets) -> widgets.Widget:\n", " \"\"\"\n", @@ -80,7 +71,7 @@ " ui_look.input_layout.max_width = \"500px\"\n", " \n", " slc_sources = [slc_source_slc_repo_option_value, \"Use existing repository\"]\n", - " default_value = conf.get(slc_source_store_key)\n", + " default_value = conf.get(AILabConfig.slc_source)\n", "\n", " flavor_source_selector = widgets.RadioButtons(options=slc_sources, value=default_value, \n", " layout=ui_look.input_layout, style=ui_look.input_style)\n", @@ -89,7 +80,7 @@ "\n", "\n", " def select_flavor_source(btn):\n", - " conf.save(slc_source_store_key, flavor_source_selector.value)\n", + " conf.save(AILabConfig.slc_source, flavor_source_selector.value)\n", " btn.icon = 'check'\n", "\n", " def on_value_change(change):\n", @@ -114,7 +105,7 @@ " \n", " inputs = [\n", " [\n", - " ('Target Base Directory', target_dir_chooser_widget, slc_target_dir_store_key),\n", + " ('Target Base Directory', target_dir_chooser_widget, AILabConfig.slc_target_dir),\n", " ]\n", " ]\n", "\n", @@ -126,7 +117,7 @@ "\n", " def save_configuration(btn):\n", " target_dir = Path(target_dir_chooser_widget.selected) / \"script_languages_release\"\n", - " conf.save(slc_target_dir_store_key, str(target_dir))\n", + " conf.save(AILabConfig.slc_target_dir, str(target_dir))\n", " btn.icon = 'check'\n", "\n", " def on_value_change(change):\n", @@ -149,15 +140,15 @@ " \"\"\"\n", " Creates a UI form for choosing the existing script-languages repository.\n", " \"\"\"\n", - " default_src_dir = conf.get(slc_target_dir_store_key, '')\n", - " select_default = True if slc_target_dir_store_key else False\n", + " default_src_dir = conf.get(AILabConfig.slc_target_dir, '')\n", + " select_default = True if default_src_dir else False\n", " src_dir_chooser_widget = FileChooser(path=default_src_dir, select_default=select_default)\n", " src_dir_chooser_widget.show_only_dirs = True\n", " src_dir_chooser_widget.sandbox_path = NOTEBOOKS_DIRECTORY\n", " \n", " inputs = [\n", " [\n", - " ('Flavor source directory', src_dir_chooser_widget, slc_target_dir_store_key),\n", + " ('Flavor source directory', src_dir_chooser_widget, AILabConfig.slc_target_dir),\n", " ]\n", " ]\n", "\n", @@ -172,7 +163,7 @@ " target_dir = Path(src_dir_chooser_widget.selected)\n", " if not (target_dir / \"flavors\").is_dir():\n", " popup_message(\"Invalid directory. You need to choose a valid script-languages repository.\")\n", - " conf.save(slc_target_dir_store_key, str(src_dir_chooser_widget.selected))\n", + " conf.save(AILabConfig.slc_target_dir, str(src_dir_chooser_widget.selected))\n", " btn.icon = 'check'\n", "\n", " def on_value_change(change):\n", @@ -189,59 +180,7 @@ " \n", "\n", " if not clone_slc_repo(conf):\n", - " return ui\n", - "\n", - "\n", - "def get_flavor_selection_ui(conf: Secrets) -> widgets.Widget:\n", - " \"\"\"\n", - " Creates a UI form for choosing from a list of flavors.\n", - " \"\"\"\n", - "\n", - " from os import listdir\n", - " from os.path import isdir\n", - " from pathlib import Path\n", - "\n", - " slc_dir = conf.get(slc_target_dir_store_key)\n", - " \n", - "\n", - "\n", - " flavor_base_dir = Path(slc_dir) / \"flavors\"\n", - " flavor_dirs = [flavor_dir.name for flavor_dir in flavor_base_dir.iterdir() if flavor_dir.is_dir()]\n", - "\n", - " ui_look = get_config_styles()\n", - " ui_look.input_layout.max_width = \"500px\"\n", - "\n", - " #Setup default value\n", - " #First we check if we can match the value from the Secret store\n", - " default_src_dir = conf.get(slc_source_flavor_store_key, '')\n", - " default_src_dir_value = ''\n", - " if default_src_dir:\n", - " matching_flavor_dirs = [flavor_dir.name for flavor_dir in flavor_base_dir.iterdir() if flavor_dir.is_dir() and str(flavor_dir) == default_src_dir]\n", - " if len(matching_flavor_dirs) == 1:\n", - " default_src_dir_value = matching_flavor_dirs[0]\n", - " #If not found in the secret store, we choose the first path which matches the template-\n", - " if not default_src_dir_value:\n", - " default_src_dir_value = \"template-Exasol-all-python-3.10\"\n", - " flavor_selector = widgets.RadioButtons(options=flavor_dirs, value=default_src_dir_value, \n", - " layout=ui_look.input_layout, style=ui_look.input_style)\n", - " select_btn = widgets.Button(description='Select', style=ui_look.button_style, layout=ui_look.button_layout)\n", - " header_lbl = widgets.Label(value='Flavor choice', style=ui_look.header_style, layout=ui_look.header_layout)\n", - "\n", - "\n", - " def select_flavor(btn):\n", - " conf.save(slc_source_flavor_store_key, str(flavor_selector.value))\n", - " btn.icon = 'check'\n", - "\n", - " def on_value_change(change):\n", - " select_btn.icon = 'pen'\n", - "\n", - " select_btn.on_click(select_flavor)\n", - " flavor_selector.observe(on_value_change, names=['value'])\n", - "\n", - " group_items = [header_lbl, widgets.Box([flavor_selector], layout=ui_look.row_layout)]\n", - " items = [widgets.Box(group_items, layout=ui_look.group_layout), select_btn]\n", - " ui = widgets.Box(items, layout=ui_look.outer_layout)\n", - " return ui" + " return ui\n" ] }, { diff --git a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt index d74377ab..163cbcd3 100644 --- a/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt +++ b/exasol/ds/sandbox/runtime/ansible/roles/jupyter/files/notebook_requirements.txt @@ -7,8 +7,6 @@ jupysql==0.10.10 sqlalchemy_exasol==4.6.3 stopwatch.py==2.0.1 boto3==1.26.163 -exasol-notebook-connector @ git+https://github.com/exasol/notebook-connector.git@main +exasol-notebook-connector @ git+https://github.com/exasol/notebook-connector.git@feature/19_move_slc_notebook pickleshare==0.7.5 # See https://github.com/exasol/ai-lab/issues/291 for details. -exasol-script-languages-container-tool==0.19.0 -GitPython>=2.1.0 ipyfilechooser==0.6.0 \ No newline at end of file diff --git a/test/notebooks/nbtest_script_languages_container.py b/test/notebooks/nbtest_script_languages_container.py new file mode 100644 index 00000000..2bbb1f94 --- /dev/null +++ b/test/notebooks/nbtest_script_languages_container.py @@ -0,0 +1,40 @@ +import os +import textwrap +from pathlib import Path + +import pytest + +from notebook_test_utils import (access_to_temp_secret_store, + access_to_temp_saas_secret_store, + run_notebook, + uploading_hack) +from exasol.nb_connector.ai_lab_config import AILabConfig as CKey, StorageBackend +from exasol.nb_connector.secret_store import Secrets + + +def _store_slc_config(store_path: Path, store_password: str, target_dir: Path): + + conf = Secrets(store_path, store_password) + conf.connection() + conf.save(CKey.slc_source, "Clone script languages release repository") + conf.save(CKey.slc_target_dir, str(target_dir)) + +@pytest.mark.parametrize( + "notebook_file", + [ + 'using_a_script_languages_container.ipynb', + ] +) +@pytest.mark.parametrize('access_to_temp_secret_store', [StorageBackend.onprem], indirect=True) +def test_script_languages_container(access_to_temp_secret_store, uploading_hack, notebook_file) -> None: + current_dir = os.getcwd() + store_path, store_password = access_to_temp_secret_store + store_file = str(store_path) + try: + + run_notebook('main_config.ipynb', store_file, store_password) + os.chdir('./script_languages_container') + run_notebook('configure_slc_flavor.ipynb', store_file, store_password) + run_notebook('using_a_script_language_container.ipynb', store_file, store_password) + finally: + os.chdir(current_dir)