update_reqs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f14ee4d3-2d1f-4da8-b293-f4f27eb9f89b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/Caskroom/miniforge/base/envs/sklearn-env/lib/python3.11/site-packages/_distutils_hack/__init__.py:33: UserWarning: Setuptools is replacing distutils.\n",
      "  warnings.warn(\"Setuptools is replacing distutils.\")\n"
     ]
    }
   ],
   "source": [
    "import nbformat\n",
    "import os\n",
    "import re\n",
    "from itertools import chain\n",
    "import ast\n",
    "import pipdeptree"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d6d0d99-a36c-4188-a036-0f6b4e83c55c",
   "metadata": {},
   "source": [
    "## Find all packages that are currently being pip or conda installed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "aab7e8e4-7db6-4d7e-a3a8-13f195196cf2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Packages being manually installed in notebooks in the 'content' folder:\n",
      "______________________________________________________________________________________________________\n",
      "\n",
      "All packages being manually installed:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_required_packages(notebook_path):\n",
    "    with open(notebook_path, 'r', encoding='utf-8') as notebook_file:\n",
    "        notebook_content = nbformat.read(notebook_file, as_version=4)\n",
    "\n",
    "    required_packages = set()\n",
    "\n",
    "    for cell in notebook_content['cells']:\n",
    "        if cell['cell_type'] == 'code':\n",
    "            source_code = cell['source']\n",
    "            lines = source_code.split('\\n')\n",
    "            for line in lines:\n",
    "                if 'pip install' in line or 'conda install' in line:\n",
    "                    # Extracting the package name from the installation command\n",
    "                    package_name = line.split('install')[1]\n",
    "                    required_packages.add(package_name.strip())\n",
    "\n",
    "    return required_packages\n",
    "\n",
    "def get_notebooks_in_folder(folder_path):\n",
    "    notebook_files = []\n",
    "    for root, dirs, files in os.walk(folder_path):\n",
    "        for file in files:\n",
    "            if file.endswith('.ipynb'):\n",
    "                notebook_files.append(os.path.join(root, file))\n",
    "    return notebook_files\n",
    "\n",
    "def get_required_packages_from_folder(folder_path):\n",
    "    all_packages = dict()\n",
    "    notebook_files = get_notebooks_in_folder(folder_path)\n",
    "\n",
    "    for notebook_file in notebook_files:\n",
    "        if '-checkpoint' not in str(notebook_file):\n",
    "            packages = get_required_packages(notebook_file)\n",
    "            if len(packages)> 0:\n",
    "                all_packages[notebook_file] = packages\n",
    "\n",
    "    return all_packages\n",
    "\n",
    "chapters_packages = get_required_packages_from_folder('content')\n",
    "\n",
    "print(\"Packages being manually installed in notebooks in the 'content' folder:\")\n",
    "for i in chapters_packages.items():\n",
    "    print(i)\n",
    "\n",
    "def flatten_chain(matrix):\n",
    "    return set(chain.from_iterable(matrix))\n",
    "\n",
    "print(\"______________________________________________________________________________________________________\")\n",
    "print(\"\\nAll packages being manually installed:\")\n",
    "flatten_chain(chapters_packages.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "363d7f9d-36d2-4004-bab0-fd03901e8c82",
   "metadata": {},
   "source": [
    "## Find all packages that are imported"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "54a8fb02-68b8-4756-96cc-2e619ef19356",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Packages being imported in notebooks in the 'content' folder:\n",
      "('content/05-regex/regex.ipynb', {'regex', 'pandas'})\n",
      "('content/02-prereqs/numpy.ipynb', {'numpy'})\n",
      "('content/07-sql/sql.ipynb', {'numpy', 'pathlib', 'sqlalchemy', 'pandas'})\n",
      "('content/04-visualizations/types.ipynb', {'seaborn', 'matplotlib', 'plotly', 'pandas', 'numpy'})\n",
      "('content/04-visualizations/advanced.ipynb', {'numpy', 'seaborn', 'pandas', 'matplotlib'})\n",
      "('content/06-api/api.ipynb', {'urllib', 'requests', 'pandas'})\n",
      "('content/08-regression/regressions.ipynb', {'statsmodels', 'pandas'})\n",
      "('content/03-pandas/merge.ipynb', {'pandas'})\n",
      "('content/03-pandas/groupby_pivot.ipynb', {'numpy', 'pandas'})\n",
      "('content/03-pandas/loading-looking.ipynb', {'pandas'})\n",
      "('content/03-pandas/select_modify.ipynb', {'numpy', 'pandas'})\n",
      "('content/03-pandas/misc_funcs.ipynb', {'datetime', 'numpy', 'pandas'})\n",
      "______________________________________________________________________________________________________\n",
      "\n",
      "All packages being imported:\n",
      "urllib\n",
      "statsmodels\n",
      "pathlib\n",
      "seaborn\n",
      "matplotlib\n",
      "plotly\n",
      "requests\n",
      "sqlalchemy\n",
      "pandas\n",
      "datetime\n",
      "regex\n",
      "numpy\n"
     ]
    }
   ],
   "source": [
    "def get_notebooks_in_folder(folder_path):\n",
    "    notebook_files = []\n",
    "    for root, dirs, files in os.walk(folder_path):\n",
    "        for file in files:\n",
    "            if file.endswith('.ipynb'):\n",
    "                notebook_files.append(os.path.join(root, file))\n",
    "    return notebook_files\n",
    "\n",
    "def get_imported_packages(notebook_path):\n",
    "    with open(notebook_path, 'r', encoding='utf-8') as notebook_file:\n",
    "        notebook_content = nbformat.read(notebook_file, as_version=4)\n",
    "\n",
    "    required_packages = set()\n",
    "\n",
    "    for cell in notebook_content['cells']:\n",
    "        if cell['cell_type'] == 'code':\n",
    "            source_code = cell['source']\n",
    "            lines = source_code.split('\\n')\n",
    "            for line in lines:\n",
    "                if not line.startswith('#') and 'import' in line:\n",
    "                    if 'from' in line:\n",
    "                        package_name = line.split('from')[1].split('import')[0].split(' as ')[0].split('.')[0]\n",
    "                    else:\n",
    "                        package_name = line.split('import')[1].split(' as ')[0].split('.')[0]\n",
    "                    required_packages.add(package_name.strip())\n",
    "\n",
    "    return required_packages\n",
    "\n",
    "def get_imported_packages_from_folder(folder_path):\n",
    "    all_imported_packages = dict()\n",
    "    notebook_files = get_notebooks_in_folder(folder_path)\n",
    "\n",
    "    for notebook_file in notebook_files:\n",
    "        if '-checkpoint' not in str(notebook_file):\n",
    "            packages = get_imported_packages(notebook_file)\n",
    "            if len(packages)> 0:\n",
    "                    all_imported_packages[notebook_file] = packages\n",
    "\n",
    "    return all_imported_packages\n",
    "\n",
    "chapters_imported_packages = get_imported_packages_from_folder('content')\n",
    "\n",
    "print(\"Packages being imported in notebooks in the 'content' folder:\")\n",
    "for i in chapters_imported_packages.items():\n",
    "    print(i)\n",
    "\n",
    "\n",
    "def flatten_chain(matrix):\n",
    "    return set(chain.from_iterable(matrix))\n",
    "\n",
    "all_packs_imported = flatten_chain(chapters_imported_packages.values())\n",
    "\n",
    "print(\"______________________________________________________________________________________________________\")\n",
    "print(\"\\nAll packages being imported:\")\n",
    "for i in all_packs_imported:\n",
    "    print(i)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "598403cc-0deb-46a6-89cd-f5a0504bcf86",
   "metadata": {},
   "source": [
    "## Generate streamlined requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dc1f1cfc-d6b4-46d7-a91e-ad599a86e836",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Warning!!! Possibly conflicting dependencies found:',\n",
       " '* econtools==0.3.2',\n",
       " ' - scipy [required: Any, installed: ?]',\n",
       " '------------------------------------------------------------------------',\n",
       " 'No packages matched using the following patterns: ydata-profiling']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = \"ydata-profiling\" #example pipdeptree for a package\n",
    "out = !pipdeptree -p $x\n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "06cd6045-0738-4c0b-bb69-524739c869ce",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('numpy', '1.23.2')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = re.compile(\"- (.*) \\[.*installed: (.*)]\") # regex wooo\n",
    "m = p.match( '- numpy [required: >=1.18.5,<1.25.0, installed: 1.23.2]' )\n",
    "m.group(1), m.group(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "dc7e1a2a-0436-4df0-981e-86fbe372509d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Warning!!! Possibly conflicting dependencies found:',\n",
       " '* econtools==0.3.2',\n",
       " ' - scipy [required: Any, installed: ?]',\n",
       " '------------------------------------------------------------------------',\n",
       " 'No packages matched using the following patterns: statsmodels']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pck='statsmodels'\n",
    "out = !pipdeptree -p $pck #get pipdeptree of pck\n",
    "    \n",
    "out #= [y.strip() for y in out if (y.lower().startswith(pck)) or (\"installed\" in y)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5e10fe03-194a-4f02-967c-ee848d4eed5e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'scipy': '',\n",
       " 'seaborn': '0.12.2',\n",
       " 'matplotlib': '3.7.1',\n",
       " 'plotly': '5.13.0',\n",
       " 'requests': '2.28.1',\n",
       " 'SQLAlchemy': '1.4.46',\n",
       " 'pandas': '1.5.3',\n",
       " 'regex': '2023.5.5',\n",
       " 'numpy': '1.25.0'}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# loop through main packages and collect dependencies in dictionary called pack\n",
    "# buggy cases go into buggy_pcks\n",
    "# i have manually removed jl4 and all it's dependencies from buggy cases for now (otherwise they would appear there)\n",
    "pack = {}\n",
    "buggy_pcks = {}\n",
    "for pck in all_packs_imported:\n",
    "    out = !pipdeptree -p $pck #get pipdeptree of pck\n",
    "    \n",
    "    out = [y.strip() for y in out if (y.lower().startswith(pck)) or (\"installed\" in y)] #take out unrelated text\n",
    "    \n",
    "    for depend in out: \n",
    "        if (pck+\"=\") in depend.lower(): #for including the actual pck itself\n",
    "            package = depend.split(\"==\")[0]\n",
    "            version = depend.split(\"==\")[1]\n",
    "            pack[package] = version\n",
    "            continue\n",
    "            \n",
    "        else:\n",
    "            try:\n",
    "                m = p.match(depend) #for including the dependencies of pck\n",
    "                package = m.group(1)\n",
    "                version = m.group(2)\n",
    "                if version == '?':\n",
    "                    version = ''    \n",
    "                pack[package] = version\n",
    "            except: #for debugging :)\n",
    "                if not str(depend).startswith('- jupyterlab') and \\\n",
    "                not str(depend).strip().startswith('└──') and not str(depend).strip().startswith('├──') and \\\n",
    "                not str(depend).strip().startswith('│'):\n",
    "                    print(depend)\n",
    "                    if pck not in buggy_pcks:\n",
    "                        buggy_pcks[pck] = [depend]\n",
    "                    else:\n",
    "                        buggy_pcks[pck].append(depend)\n",
    "pack    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9ab1cb8d-348c-4e2e-9c94-4b7a577d8066",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "buggy_pcks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "83ecaf6d-47a8-4fc7-981c-c497e8a3a164",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "pk = list(pack.keys()) #list of all packages for requirements.txt\n",
    "pk.sort() #sorted by capitals first then small letters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5dc19747-2053-4260-9788-6ccb103fe4b4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SQLAlchemy==1.4.46\n",
      "matplotlib==3.7.1\n",
      "numpy==1.25.0\n",
      "pandas==1.5.3\n",
      "plotly==5.13.0\n",
      "regex==2023.5.5\n",
      "requests==2.28.1\n",
      "scipy\n",
      "seaborn==0.12.2\n"
     ]
    }
   ],
   "source": [
    "for pck in pk: #the actual info in requirements.txt :)\n",
    "    if pack[pck] != \"\":\n",
    "        print(pck + \"==\" + pack[pck]) \n",
    "    else:\n",
    "        print(pck)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6e6cdf6-0465-401b-9604-05054ef3d296",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:sklearn-env] *",
   "language": "python",
   "name": "conda-env-sklearn-env-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}