Update and improve schlib/autogen script for STM32

schlib/autogen/stm32/.gitignore

@@ -0,0 +1,3 @@
+stm32_datasheets/
+db/
+mcu/

schlib/autogen/stm32/README.md

@@ -6,15 +6,18 @@ STM32 devices.
 ## Prerequisites
 
 * XML files, taken from STM32CubeMX install (from db/mcu folder).
-* Datasheet PDFs, downloaded from ST's website.  A current list as of
-  2018-02-06 can be found in datasheets.txt.  These PDFs can be downloaded all
-  at once with `wget -P datasheets -ci datasheets.txt`.
+* Datasheet PDFs, downloaded from ST's website.  The helper tool
+  `stm32_get_datasheets.py` will download all available STM32 datasheets into
+  `./stm32_datasheets/`.
 * [pdfminer](https://github.com/euske/pdfminer) tool.
 
 ## Running
 
 If you have the correct XML files, just run
 `./stm32_generator.py xmldir pdfdir`, where `xmldir` is a directory containing
 all the necessary XML files, and `pdfdir` is a directory containing all the PDF
-files.  Make sure the XML files have the correct format as there is no error
-checking.
+files (e.g. `./stm32_datasheets/`).  Make sure the XML files have the correct
+format as there is no error checking.
+
+Mining data from the pdf filestakes time, expect about 1-2 minutes per
+datasheet and CPU core.

schlib/autogen/stm32/stm32_generator.py

@@ -6,6 +6,8 @@
 import os
 import re
 import sys
+import multiprocessing
+from itertools import repeat
 
 from lxml import etree
 
@@ -670,6 +672,13 @@ def draw_symbol(self):
                 visibility=SymbolField.FieldVisibility.INVISIBLE)
 
 
+def run_pdf2txt(pdffile, pdfdir):
+    pdffile = os.path.join(pdfdir, pdffile)
+    pdfparsedfile = pdffile + ".par"
+    if not os.path.isfile(pdfparsedfile) and pdffile.endswith(".pdf"):
+        logging.info(f"Converting: {pdffile}")
+        os.system("pdf2txt.py -o " + pdfparsedfile + " " + pdffile)
+
 def main():
     parser = argparse.ArgumentParser(
             description='Generator for STM32 microcontroller symbols')
@@ -696,28 +705,25 @@ def main():
 
     # Parse text from PDFs
     for _, _, filenames in os.walk(args.pdfdir):
-        for pdffile in filenames:
-            pdffile = os.path.join(args.pdfdir, pdffile)
-            pdfparsedfile = pdffile + ".par"
-            if not os.path.isfile(pdfparsedfile) and pdffile.endswith(".pdf"):
-                logging.info(f"Converting: {pdffile}")
-                os.system("pdf2txt.py -o " + pdfparsedfile + " " + pdffile)
-        break
+        filenames.sort()
+        with multiprocessing.Pool() as pool:
+            pool.starmap(run_pdf2txt, zip(filenames, repeat(args.pdfdir)))
 
     # Load devices from XML, sorted by family
     libraries = {}
     for _, _, filenames in os.walk(args.xmldir):
         filenames.sort()
         for xmlfile in filenames:
-            # Load information about the part(s)
-            mcu = Device(os.path.join(args.xmldir, xmlfile), args.pdfdir)
-            # If there isn't a SymbolGenerator for this family yet, make one
-            if mcu.family not in libraries:
-                libraries[mcu.family] = SymbolGenerator(
-                        lib_name=f"MCU_ST_{mcu.family}")
-            # If the part has a datasheet PDF, make a symbol for it
-            if mcu.pdf is not None:
-                mcu.create_symbol(libraries[mcu.family])
+            if xmlfile.startswith("STM"):
+                # Load information about the part(s)
+                mcu = Device(os.path.join(args.xmldir, xmlfile), args.pdfdir)
+                # If there isn't a SymbolGenerator for this family yet, make one
+                if mcu.family not in libraries:
+                    libraries[mcu.family] = SymbolGenerator(
+                            lib_name=f"MCU_ST_{mcu.family}")
+                # If the part has a datasheet PDF, make a symbol for it
+                if mcu.pdf is not None:
+                    mcu.create_symbol(libraries[mcu.family])
         break
 
     # Write libraries

schlib/autogen/stm32/stm32_get_datasheets.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+
+import json
+import urllib.request
+import shutil
+import os
+import multiprocessing
+
+
+class Document:
+    def __init__(self, url, filename):
+        self.url = url
+        self.filename = filename
+
+class DocumentManager:
+    ds_urls = [
+        'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-high-performance-mcus.cxst-rs-grid.html/SC2154.technical_literature.datasheet.json',
+        'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-mainstream-mcus.cxst-rs-grid.html/SC2155.technical_literature.datasheet.json',
+        'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-ultra-low-power-mcus.cxst-rs-grid.html/SC2157.technical_literature.datasheet.json',
+        'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-wireless-mcus.cxst-rs-grid.html/SC2156.technical_literature.datasheet.json',
+        #'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-arm-cortex-mpus.cxst-rs-grid.html/SC2230.technical_literature.datasheet.json',
+    ]
+    hdr = {
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
+        'Accept-Encoding': 'none',
+        'Accept-Language': 'en-US,en;q=0.8',
+        'Connection': 'keep-alive'}
+
+    ds_list = list()
+
+    _data_document_dir = 'stm32_datasheets/'
+
+    def update_ds_list(self):
+        tmp_ds_list = list()
+        for url in self.ds_urls:
+            with urllib.request.urlopen(urllib.request.Request(url, headers=self.hdr)) as url:
+                data = json.loads(url.read().decode())
+                rows = data['rows']
+                for row in rows:
+                    tmp_ds_list.append(Document(
+                        url='https://www.st.com' + row['localizedLinks']['en'],
+                        filename=row['localizedLinks']['en'].split('/')[-1],
+                    )
+                    )
+        self.ds_list = tmp_ds_list
+
+    def download_pdf(self, d):
+        os.makedirs(self._data_document_dir, exist_ok=True)
+        if not os.path.isfile(self._data_document_dir + d.filename):
+            print('Downloading file {} ...'.format(d.url))
+            with urllib.request.urlopen(urllib.request.Request(d.url, headers=self.hdr)) as response,\
+                    open(self._data_document_dir +d.filename, 'wb') as out_file:
+                shutil.copyfileobj(response, out_file)
+        else:
+            print('Document {} already exists.'.format(d.filename))
+
+    def download(self):
+        with multiprocessing.Pool(10) as pool:
+            pool.map(self.download_pdf, self.ds_list)
+
+if __name__ == "__main__":
+    manager = DocumentManager()
+    manager.update_ds_list()
+    print("Found {} datasheets:".format(len(manager.ds_list)))
+    for ds in manager.ds_list:
+        print(ds.filename)
+    print("Downloading...")
+    manager.download()