diff --git a/schlib/autogen/stm32/.gitignore b/schlib/autogen/stm32/.gitignore new file mode 100644 index 00000000..277681f4 --- /dev/null +++ b/schlib/autogen/stm32/.gitignore @@ -0,0 +1,3 @@ +stm32_datasheets/ +db/ +mcu/ diff --git a/schlib/autogen/stm32/README.md b/schlib/autogen/stm32/README.md index aa7ab2d3..87f2ba97 100644 --- a/schlib/autogen/stm32/README.md +++ b/schlib/autogen/stm32/README.md @@ -6,9 +6,9 @@ STM32 devices. ## Prerequisites * XML files, taken from STM32CubeMX install (from db/mcu folder). -* Datasheet PDFs, downloaded from ST's website. A current list as of - 2018-02-06 can be found in datasheets.txt. These PDFs can be downloaded all - at once with `wget -P datasheets -ci datasheets.txt`. +* Datasheet PDFs, downloaded from ST's website. The helper tool + `stm32_get_datasheets.py` will download all available STM32 datasheets into + `./stm32_datasheets/`. * [pdfminer](https://github.com/euske/pdfminer) tool. ## Running @@ -16,5 +16,8 @@ STM32 devices. If you have the correct XML files, just run `./stm32_generator.py xmldir pdfdir`, where `xmldir` is a directory containing all the necessary XML files, and `pdfdir` is a directory containing all the PDF -files. Make sure the XML files have the correct format as there is no error -checking. +files (e.g. `./stm32_datasheets/`). Make sure the XML files have the correct +format as there is no error checking. + +Mining data from the pdf filestakes time, expect about 1-2 minutes per +datasheet and CPU core. diff --git a/schlib/autogen/stm32/datasheets.txt b/schlib/autogen/stm32/datasheets.txt deleted file mode 100644 index ad0b72f3..00000000 --- a/schlib/autogen/stm32/datasheets.txt +++ /dev/null @@ -1,111 +0,0 @@ -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00161561.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00161566.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00191174.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00191185.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00210831.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00210833.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00210837.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00210843.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00212417.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00220364.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00237391.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00251732.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00253739.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00253742.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00263874.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/CD00277537.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00034689.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00035129.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00037051.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00039193.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00039232.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00046749.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00048356.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00049732.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00058181.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00059126.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00071990.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00077036.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00078075.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00078689.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00081611.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00086815.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00088500.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00090183.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00090510.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00092070.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00093332.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00093333.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00094064.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00097745.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00098321.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00098745.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00100431.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00101621.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00102166.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00102435.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00104043.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00105814.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00105960.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00106442.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108217.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108218.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108219.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108661.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108832.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00108833.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00109263.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00109264.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00110868.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00111457.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00112716.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00115237.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00115249.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00115350.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00116561.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00118585.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00133117.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00135027.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00140359.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00140762.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141036.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141132.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141133.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141136.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141306.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00141386.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00149404.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00149412.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00150658.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00150671.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00152023.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00162467.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00166114.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00166116.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00172872.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00206508.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00206858.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00208574.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00213872.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00214043.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00219980.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00225424.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00254865.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00257192.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00257195.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00257205.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00257211.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00273119.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00282247.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00282249.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00284207.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00284211.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00330506.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00330507.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00340475.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00340549.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00340637.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00366448.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00366449.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00387108.pdf -http://www.st.com/st-web-ui/static/active/en/resource/technical/document/datasheet/DM00388325.pdf diff --git a/schlib/autogen/stm32/stm32_generator.py b/schlib/autogen/stm32/stm32_generator.py index ceafb967..d23fa6a9 100755 --- a/schlib/autogen/stm32/stm32_generator.py +++ b/schlib/autogen/stm32/stm32_generator.py @@ -6,6 +6,8 @@ import os import re import sys +import multiprocessing +from itertools import repeat from lxml import etree @@ -670,6 +672,13 @@ def draw_symbol(self): visibility=SymbolField.FieldVisibility.INVISIBLE) +def run_pdf2txt(pdffile, pdfdir): + pdffile = os.path.join(pdfdir, pdffile) + pdfparsedfile = pdffile + ".par" + if not os.path.isfile(pdfparsedfile) and pdffile.endswith(".pdf"): + logging.info(f"Converting: {pdffile}") + os.system("pdf2txt.py -o " + pdfparsedfile + " " + pdffile) + def main(): parser = argparse.ArgumentParser( description='Generator for STM32 microcontroller symbols') @@ -696,28 +705,25 @@ def main(): # Parse text from PDFs for _, _, filenames in os.walk(args.pdfdir): - for pdffile in filenames: - pdffile = os.path.join(args.pdfdir, pdffile) - pdfparsedfile = pdffile + ".par" - if not os.path.isfile(pdfparsedfile) and pdffile.endswith(".pdf"): - logging.info(f"Converting: {pdffile}") - os.system("pdf2txt.py -o " + pdfparsedfile + " " + pdffile) - break + filenames.sort() + with multiprocessing.Pool() as pool: + pool.starmap(run_pdf2txt, zip(filenames, repeat(args.pdfdir))) # Load devices from XML, sorted by family libraries = {} for _, _, filenames in os.walk(args.xmldir): filenames.sort() for xmlfile in filenames: - # Load information about the part(s) - mcu = Device(os.path.join(args.xmldir, xmlfile), args.pdfdir) - # If there isn't a SymbolGenerator for this family yet, make one - if mcu.family not in libraries: - libraries[mcu.family] = SymbolGenerator( - lib_name=f"MCU_ST_{mcu.family}") - # If the part has a datasheet PDF, make a symbol for it - if mcu.pdf is not None: - mcu.create_symbol(libraries[mcu.family]) + if xmlfile.startswith("STM"): + # Load information about the part(s) + mcu = Device(os.path.join(args.xmldir, xmlfile), args.pdfdir) + # If there isn't a SymbolGenerator for this family yet, make one + if mcu.family not in libraries: + libraries[mcu.family] = SymbolGenerator( + lib_name=f"MCU_ST_{mcu.family}") + # If the part has a datasheet PDF, make a symbol for it + if mcu.pdf is not None: + mcu.create_symbol(libraries[mcu.family]) break # Write libraries diff --git a/schlib/autogen/stm32/stm32_get_datasheets.py b/schlib/autogen/stm32/stm32_get_datasheets.py new file mode 100755 index 00000000..8def4169 --- /dev/null +++ b/schlib/autogen/stm32/stm32_get_datasheets.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 + +import json +import urllib.request +import shutil +import os +import multiprocessing + + +class Document: + def __init__(self, url, filename): + self.url = url + self.filename = filename + +class DocumentManager: + ds_urls = [ + 'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-high-performance-mcus.cxst-rs-grid.html/SC2154.technical_literature.datasheet.json', + 'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-mainstream-mcus.cxst-rs-grid.html/SC2155.technical_literature.datasheet.json', + 'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-ultra-low-power-mcus.cxst-rs-grid.html/SC2157.technical_literature.datasheet.json', + 'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-32-bit-arm-cortex-mcus/stm32-wireless-mcus.cxst-rs-grid.html/SC2156.technical_literature.datasheet.json', + #'https://www.st.com/content/st_com/en/products/microcontrollers-microprocessors/stm32-arm-cortex-mpus.cxst-rs-grid.html/SC2230.technical_literature.datasheet.json', + ] + hdr = { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', + 'Accept-Encoding': 'none', + 'Accept-Language': 'en-US,en;q=0.8', + 'Connection': 'keep-alive'} + + ds_list = list() + + _data_document_dir = 'stm32_datasheets/' + + def update_ds_list(self): + tmp_ds_list = list() + for url in self.ds_urls: + with urllib.request.urlopen(urllib.request.Request(url, headers=self.hdr)) as url: + data = json.loads(url.read().decode()) + rows = data['rows'] + for row in rows: + tmp_ds_list.append(Document( + url='https://www.st.com' + row['localizedLinks']['en'], + filename=row['localizedLinks']['en'].split('/')[-1], + ) + ) + self.ds_list = tmp_ds_list + + def download_pdf(self, d): + os.makedirs(self._data_document_dir, exist_ok=True) + if not os.path.isfile(self._data_document_dir + d.filename): + print('Downloading file {} ...'.format(d.url)) + with urllib.request.urlopen(urllib.request.Request(d.url, headers=self.hdr)) as response,\ + open(self._data_document_dir +d.filename, 'wb') as out_file: + shutil.copyfileobj(response, out_file) + else: + print('Document {} already exists.'.format(d.filename)) + + def download(self): + with multiprocessing.Pool(10) as pool: + pool.map(self.download_pdf, self.ds_list) + +if __name__ == "__main__": + manager = DocumentManager() + manager.update_ds_list() + print("Found {} datasheets:".format(len(manager.ds_list))) + for ds in manager.ds_list: + print(ds.filename) + print("Downloading...") + manager.download()