Microsoft19thCenturyBooks/maps_from_Flickr_BLPhotos_19cBooks_Collection.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "maps_from_Flickr_BLPhotos_19cBooks_Collection.ipynb",
      "provenance": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HgUqbmCFe9EK"
      },
      "source": [
        "<div class=\"jumbotron jumbotron-fluid\">\n",
        "  <div class=\"container\">\n",
        "    <h1 class=\"display-4\">Maps from Flickr BL's Photos Collection</h1>\n",
        "    <p class=\"lead\">53,367 images extracted from a collection of 65,000 digitised books (largely from the 19th Century), that were identified as maps.</p>\n",
        "  </div>\n",
        "</div>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4X6xCoFJe9EM"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/BL-Labs/Jupyter-notebooks-projects-using-BL-Sources/blob/master/Microsoft19thCenturyBooks/maps_from_Flickr_BLPhotos_19cBooks_Collection.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rRA5_biUe9EN"
      },
      "source": [
        "### In this Notebook:\n",
        "\n",
        " * Load metadata from a JSON file (compressed inside a zip file)\n",
        "     * 53,367 records inside a \"rows\" array at top level (root of the JSON file)\n",
        "     * Available properties/keys (fields): photoid, title, description, alephSysNumber, page, geoRefId, url_c, url_o, tags, machine_tags, views\n",
        " * Generate the relevant links to the several services\n",
        " * Display the map (image)\n",
        " * Next: add metadata to a DataFrame (Pandas) and do some search and processing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IUjjy3HLe9EO"
      },
      "source": [
        "# imports\n",
        "\n",
        "import zipfile\n",
        "import urllib\n",
        "import json\n",
        "import io"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7JtbTAIse9EW"
      },
      "source": [
        "# RUN this CELL if using BINDER or a LOCAL Jupyter Notebooks SERVER (see bellow for Google Colab)\n",
        "# read JSON metadata file (311MB) inside Flickr_BLPhotos20200330_maps.zip (20.8MB))\n",
        "\n",
        "filename = \"../DataBL/19cbooks/Flickr_BLPhotos20200330_maps.zip\"\n",
        "\n",
        "# unzips the files Flickr_BLPhotos20200330_maps.zip\n",
        "z = zipfile.ZipFile(filename)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Im31NsGOe9Eb"
      },
      "source": [
        "# RUN this CELL if using Google COLAB\n",
        "# read JSON metadata file (311MB) inside Flickr_BLPhotos20200330_maps.zip (20.8MB))\n",
        "\n",
        "zip_file = \"https://github.com/BL-Labs/Jupyter-notebooks-projects-using-BL-Sources/raw/master/DataBL/19cbooks/Flickr_BLPhotos20200330_maps.zip\"\n",
        "\n",
        "# OR test loading it from data.bl.uk\n",
        "# zip_file = \"http://data.bl.uk/19cbooks/Flickr_BLPhotos20200330_maps.zip\"\n",
        "\n",
        "import urllib.request\n",
        "\n",
        "access_url = urllib.request.urlopen(zip_file)\n",
        "\n",
        "# unzips the files inside book_data.zip\n",
        "z = zipfile.ZipFile(io.BytesIO(access_url.read()))"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k2_dgteqe9Eg"
      },
      "source": [
        "# loads JSON from the first (only) file inside into a list \n",
        "meta = json.loads(z.read(z.infolist()[0]).decode('utf-8'))"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LeIyug_1e9En",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b4e2fbad-ad62-454e-fc6e-46067b1cac98"
      },
      "source": [
        "# number of records inside this list\n",
        "\n",
        "len(meta['rows'])"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "53367"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6KBfg3pee9Es",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a8c61807-ecae-4b8e-cf0d-f4fa09d1221e"
      },
      "source": [
        "# checking a certain record (0 to 53366)\n",
        "print(\"\\nWhich record you would like to view?\")\n",
        "i = int(input(\"(0 to \" + str(len(meta['rows'])-1) + \"): \"))\n",
        "\n",
        "print(\"\\n\")\n",
        "meta['rows'][i]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Which record you would like to view?\n",
            "(0 to 53366): 373\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'alephSysNumber': '000490438',\n",
              " 'description': 'Image taken from:\\r\\n\\t\\t\\r\\n<strong>Title</strong>: &quot;A Voyage of Discovery to the North Pacific Ocean ... Performed in His Majesty´s Sloop Providence, and her tender, in the years 1795, 1796, 1797, 1798 [With plates, including maps.]&quot;\\r\\n<strong>Author(s)</strong>: Broughton, William Robert [person]\\r\\n<strong>British Library shelfmark</strong>: &quot;Digital Store 981.g.22&quot;\\r\\n<strong>Page</strong>: 127 (scanned page number - not necessarily the actual page number in the publication)\\r\\n<strong>Place of publication</strong>: London (England)\\r\\n<strong>Date of publication</strong>: 1804\\r\\n<strong>Publisher</strong>: T. Cadell & W. Davies\\r\\n<strong>Type of resource</strong>: Monograph\\r\\n<strong>Language(s)</strong>: English\\r\\n<strong>Physical description</strong>: xx, 393 pages (4°)\\t\\t\\r\\n\\t\\t\\r\\n<strong>Explore this item</strong> in the British Library’s catalogue:  \\r\\n<a href=\"http://explore.bl.uk/primo_library/libweb/action/search.do?cs=frb&amp;doc=BLL01000490438&amp;dscnt=1&amp;scp.scps=scope:(BLCONTENT)&amp;frbg=&amp;tab=local_tab&amp;srt=rank&amp;ct=search&amp;mode=Basic&amp;dum=true&amp;tb=t&amp;indx=1&amp;vl(freeText0)=000490438&amp;fn=search&amp;vid=BLVU1\" rel=\"nofollow\">000490438</a> (physical copy) and <a href=\"http://explore.bl.uk/primo_library/libweb/action/search.do?cs=frb&amp;doc=BLL01014846952&amp;dscnt=1&amp;scp.scps=scope:(BLCONTENT)&amp;frbg=&amp;tab=local_tab&amp;srt=rank&amp;ct=search&amp;mode=Basic&amp;dum=true&amp;tb=t&amp;indx=1&amp;vl(freeText0)=014846952&amp;fn=search&amp;vid=BLVU1\" rel=\"nofollow\">014846952</a> (digitised copy)\\r\\n(numbers are British Library identifiers)\\r\\n\\r\\n<strong>Other links related to this image:</strong>\\r\\n- <strong>View</strong> this image as a scanned publication on the <a href=\"http://access.bl.uk/item/viewer/ark:/81055/vdc_0000000065E2#?cv=126\" alt=\"Open the book scan page in the British Library’s Universal Viewer\">British Library’s online viewer</a> (you can download the image, selected pages or the whole book)\\r\\n- <strong>View</strong> this <a href=\"http://britishlibrary.georeferencer.com/id/10999570306\" rel=\"noreferrer nofollow\">digitised map overlaid on a modern map</a> using the <strong>British Library’s Georeferencer service</strong>\\r\\n- <strong>Order</strong> a <a href=\"http://tiny.cc/BL-Digitisation-Services\" alt=\"The British Library: Digitisation Services\">higher quality scanned version of this image</a> from the British Library\\r\\n\\r\\n<strong>Other links related to this publication:</strong>\\r\\n- <strong>View</strong> all the <a href=\"http://www.flickr.com/photos/britishlibrary/tags/sysnum000490438\">illustrations found in this publication</a>\\r\\n- <strong>View</strong> all the <a href=\"http://www.flickr.com/photos/britishlibrary/tags/date1804\">illustrations in publications from the same year</a> (1804)\\r\\n- <strong>Download</strong> the Optical Character Recognised (OCR) <a href=\"https://data.bl.uk/19cbooks/json/0004/000490438_01_text.json\">derived text</a> for this publication as JavaScript Object Notation (JSON)\\r\\n\\r\\n- <strong>Explore</strong> and experiment</strong> with the British Library’s <a href=\"https://data.bl.uk/\" alt=\"British Library’s digital collections\">digital collections</a>\\r\\n\\r\\nThe British Library community is able to flourish online thanks to freely available resources such as this. \\r\\n\\r\\nYou can help support our mission to continue making our collection accessible to everyone, for research, inspiration and enjoyment, by donating on the British Library supporter <a href=\"http://tiny.cc/BL-Donate\" alt=\"Donate to The British Library\">webpage here</a>. \\r\\n\\r\\nThank you for supporting the <a href=\"https://www.bl.uk\" alt=\"The British Library\">British Library</a>.',\n",
              " 'geoRefId': '10999570306',\n",
              " 'machine_tags': 'otherhalf:pair=1099960467610999570306 geo:osmscale=12 geo:continent=asia geo:country=jp geo:country=japan geo:state=hokkaidōregion geo:statedistrict=北海道hokkaidō geo:county=usudistrict geo:city=muroran sherlocknet:tag=import sherlocknet:tag=manufacture sherlocknet:tag=port sherlocknet:tag=high sherlocknet:tag=water sherlocknet:tag=direct sherlocknet:tag=island sherlocknet:tag=france sherlocknet:tag=general sherlocknet:tag=land sherlocknet:tag=portion sherlocknet:tag=rise sherlocknet:tag=western sherlocknet:tag=fine sherlocknet:tag=coast sherlocknet:tag=consider sherlocknet:category=maps',\n",
              " 'page': 127,\n",
              " 'photoid': '10999570306',\n",
              " 'tags': 'bldigital date1804 pubplacelondon publicdomain sysnum000490438 broughtonwilliamrobert large vol0 page127 mechanicalcurator imagesfrombook000490438 imagesfromvolume0004904380 map split otherhalf:pair=1099960467610999570306 righthalf hasgeoref geo:osmscale=12 geo:continent=asia geo:country=jp geo:country=japan geo:state=hokkaidōregion geo:statedistrict=北海道hokkaidō geo:county=usudistrict geo:city=muroran sherlocknet:tag=import sherlocknet:tag=manufacture sherlocknet:tag=port sherlocknet:tag=high sherlocknet:tag=water sherlocknet:tag=direct sherlocknet:tag=island sherlocknet:tag=france sherlocknet:tag=general sherlocknet:tag=land sherlocknet:tag=portion sherlocknet:tag=rise sherlocknet:tag=western sherlocknet:tag=fine sherlocknet:tag=coast sherlocknet:tag=consider sherlocknet:category=maps',\n",
              " 'title': 'British Library digitised image from page 127 of \"A Voyage of Discovery to the North Pacific Ocean ... Performed in His Majesty´s Sloop Providence, and her tender, in the years 1795, 1796, 1797, 1798 [With plates, including maps.]\"',\n",
              " 'url_c': 'https://live.staticflickr.com/3830/10999570306_245843fb39_c.jpg',\n",
              " 'url_o': 'https://live.staticflickr.com/3830/10999570306_122cb0ddb3_o.jpg',\n",
              " 'views': 1517}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mlhSMKwfe9Ey",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f4f91589-5ceb-436b-a9b0-bbe71b39785c"
      },
      "source": [
        "print('\\nTitle:')\n",
        "print(meta['rows'][i]['title']+'\\n')\n",
        "print('* Flickr record:')\n",
        "print('https://www.flickr.com/photos/britishlibrary/'+meta['rows'][i]['geoRefId']+'\\n')\n",
        "print('* This book\\'s catalog record at BL explorer:')\n",
        "print('http://explore.bl.uk/primo_library/libweb/action/search.do?cs=frb&doc=BLL'+ meta['rows'][i]['alephSysNumber'] + '&dscnt=1&scp.scps=scope:(BLCONTENT)&frbg=&tab=local_tab&srt=rank&ct=search&mode=Basic&dum=true&tb=t&indx=1&vl(freeText0)='+ meta['rows'][i]['alephSysNumber'] + '&fn=search&vid=BLVU1'+'\\n')\n",
        "print('* This map on BL\\'s GeoReferencer Service:')\n",
        "print('http://britishlibrary.georeferencer.com/id/'+ meta['rows'][i]['geoRefId']+'\\n')\n",
        "print('\\nAs of 30th March 2020, this image / map had '+ str(meta['rows'][i]['views']) +' views.\\n')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Title:\n",
            "British Library digitised image from page 127 of \"A Voyage of Discovery to the North Pacific Ocean ... Performed in His Majesty´s Sloop Providence, and her tender, in the years 1795, 1796, 1797, 1798 [With plates, including maps.]\"\n",
            "\n",
            "* Flickr record:\n",
            "https://www.flickr.com/photos/britishlibrary/10999570306\n",
            "\n",
            "* This book's catalog record at BL explorer:\n",
            "http://explore.bl.uk/primo_library/libweb/action/search.do?cs=frb&doc=BLL000490438&dscnt=1&scp.scps=scope:(BLCONTENT)&frbg=&tab=local_tab&srt=rank&ct=search&mode=Basic&dum=true&tb=t&indx=1&vl(freeText0)=000490438&fn=search&vid=BLVU1\n",
            "\n",
            "* This map on BL's GeoReferencer Service:\n",
            "http://britishlibrary.georeferencer.com/id/10999570306\n",
            "\n",
            "\n",
            "As of 30th March 2020, this image / map had 1517 views.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0IPmxN1me9FA",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 646
        },
        "outputId": "19fffc9e-30ea-449c-af6b-00f10ae576f2"
      },
      "source": [
        "print('\\nImage Description:\\n')\n",
        "from IPython.core.display import HTML\n",
        "HTML(meta['rows'][i]['description'].replace('\\r\\n','<br>'))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Image Description:\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "Image taken from:<br>\t\t<br><strong>Title</strong>: &quot;A Voyage of Discovery to the North Pacific Ocean ... Performed in His Majesty´s Sloop Providence, and her tender, in the years 1795, 1796, 1797, 1798 [With plates, including maps.]&quot;<br><strong>Author(s)</strong>: Broughton, William Robert [person]<br><strong>British Library shelfmark</strong>: &quot;Digital Store 981.g.22&quot;<br><strong>Page</strong>: 127 (scanned page number - not necessarily the actual page number in the publication)<br><strong>Place of publication</strong>: London (England)<br><strong>Date of publication</strong>: 1804<br><strong>Publisher</strong>: T. Cadell & W. Davies<br><strong>Type of resource</strong>: Monograph<br><strong>Language(s)</strong>: English<br><strong>Physical description</strong>: xx, 393 pages (4°)\t\t<br>\t\t<br><strong>Explore this item</strong> in the British Library’s catalogue:  <br><a href=\"http://explore.bl.uk/primo_library/libweb/action/search.do?cs=frb&amp;doc=BLL01000490438&amp;dscnt=1&amp;scp.scps=scope:(BLCONTENT)&amp;frbg=&amp;tab=local_tab&amp;srt=rank&amp;ct=search&amp;mode=Basic&amp;dum=true&amp;tb=t&amp;indx=1&amp;vl(freeText0)=000490438&amp;fn=search&amp;vid=BLVU1\" rel=\"nofollow\">000490438</a> (physical copy) and <a href=\"http://explore.bl.uk/primo_library/libweb/action/search.do?cs=frb&amp;doc=BLL01014846952&amp;dscnt=1&amp;scp.scps=scope:(BLCONTENT)&amp;frbg=&amp;tab=local_tab&amp;srt=rank&amp;ct=search&amp;mode=Basic&amp;dum=true&amp;tb=t&amp;indx=1&amp;vl(freeText0)=014846952&amp;fn=search&amp;vid=BLVU1\" rel=\"nofollow\">014846952</a> (digitised copy)<br>(numbers are British Library identifiers)<br><br><strong>Other links related to this image:</strong><br>- <strong>View</strong> this image as a scanned publication on the <a href=\"http://access.bl.uk/item/viewer/ark:/81055/vdc_0000000065E2#?cv=126\" alt=\"Open the book scan page in the British Library’s Universal Viewer\">British Library’s online viewer</a> (you can download the image, selected pages or the whole book)<br>- <strong>View</strong> this <a href=\"http://britishlibrary.georeferencer.com/id/10999570306\" rel=\"noreferrer nofollow\">digitised map overlaid on a modern map</a> using the <strong>British Library’s Georeferencer service</strong><br>- <strong>Order</strong> a <a href=\"http://tiny.cc/BL-Digitisation-Services\" alt=\"The British Library: Digitisation Services\">higher quality scanned version of this image</a> from the British Library<br><br><strong>Other links related to this publication:</strong><br>- <strong>View</strong> all the <a href=\"http://www.flickr.com/photos/britishlibrary/tags/sysnum000490438\">illustrations found in this publication</a><br>- <strong>View</strong> all the <a href=\"http://www.flickr.com/photos/britishlibrary/tags/date1804\">illustrations in publications from the same year</a> (1804)<br>- <strong>Download</strong> the Optical Character Recognised (OCR) <a href=\"https://data.bl.uk/19cbooks/json/0004/000490438_01_text.json\">derived text</a> for this publication as JavaScript Object Notation (JSON)<br><br>- <strong>Explore</strong> and experiment</strong> with the British Library’s <a href=\"https://data.bl.uk/\" alt=\"British Library’s digital collections\">digital collections</a><br><br>The British Library community is able to flourish online thanks to freely available resources such as this. <br><br>You can help support our mission to continue making our collection accessible to everyone, for research, inspiration and enjoyment, by donating on the British Library supporter <a href=\"http://tiny.cc/BL-Donate\" alt=\"Donate to The British Library\">webpage here</a>. <br><br>Thank you for supporting the <a href=\"https://www.bl.uk\" alt=\"The British Library\">British Library</a>."
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t2F5DT9Ve9FE",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 872
        },
        "outputId": "23f58eaa-fdc0-4c6d-eb2a-1f27e5067019"
      },
      "source": [
        "print('\\nAnd the image / map:\\n')\n",
        "HTML('<img src=\"'+meta['rows'][i]['url_c']+'\">')"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "And the image / map:\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<img src=\"https://live.staticflickr.com/3830/10999570306_245843fb39_c.jpg\">"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jSFZxFThe9FL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2e680763-6303-4e0b-9a9c-0ee8cd890165"
      },
      "source": [
        "print('\\nDownload the original image / map:\\n')\n",
        "print(meta['rows'][i]['url_o']+'\\n')"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Download the original image / map:\n",
            "\n",
            "https://live.staticflickr.com/3830/10999570306_122cb0ddb3_o.jpg\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}