Feedback

.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

.ipynb_checkpoints/Cian_parser-checkpoint.ipynb

@@ -0,0 +1,109 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import requests\n",
+    "from bs4 import BeautifulSoup\n",
+    "from fake_useragent import UserAgent\n",
+    "from tqdm.notebook import trange, tqdm\n",
+    "import time\n",
+    "import random\n",
+    "import json"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def url_search(iterable):\n",
+    "    urls = []\n",
+    "    errors = []\n",
+    "    for i in tqdm(sorted(iterable, key=lambda w: random.random())):\n",
+    "        try:\n",
+    "            url = f'https://www.cian.ru/cat.php?deal_type=rent&engine_version=2&offer_type=flat&p={i}&region=1&room3=1&type=4'\n",
+    "            response = requests.get(url, headers={'User-Agent': UserAgent().chrome})\n",
+    "            soup = BeautifulSoup(response.content)\n",
+    "            print(response)\n",
+    "            for item in soup.find('div', {'data-name': 'Offers'}).find_all('a'):\n",
+    "                url = item.get('href')\n",
+    "                if url != None and 'https://www.cian.ru/rent/flat/' in url:\n",
+    "                    urls.append(url)\n",
+    "            if i in errors:\n",
+    "                errors.remove(i)\n",
+    "            time.sleep(random.randint(1, 3))\n",
+    "        except:\n",
+    "            print('error')\n",
+    "            errors.append(i)\n",
+    "    if len(errors) > 0:\n",
+    "        return urls.extend(url_search(errors)) \n",
+    "    else:\n",
+    "        return urls"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "urls = url_search(range(1, 55))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "len(urls)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "urls"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('urls.json', 'w', encoding='utf8') as f:\n",
+    "    json.dump(urls, f)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}