From 753bf730b35fd1869529617f70bba8d3e7da2ea2 Mon Sep 17 00:00:00 2001 From: Juan Sanchez Date: Sun, 10 Sep 2023 01:28:57 -0500 Subject: [PATCH] matplotlib recipe (#858) * matplotlib kiwisolver cpplink * fix flake8 issues * fix startswith issue * fix manifest for cpplink and refactor code * minor tweak in removing stale so files * fix message * need to include templates in the manifest * remove numpy direct dependency * add setup.py patch * add patch file * find special numpy include using a recursive glob --- MANIFEST.in | 6 +- kivy_ios/recipes/kiwisolver/__init__.py | 38 +++++ kivy_ios/recipes/matplotlib/__init__.py | 122 ++++++++++++++ kivy_ios/recipes/matplotlib/_tri.cpp.patch | 15 ++ kivy_ios/recipes/matplotlib/_tri.h.patch | 17 ++ .../recipes/matplotlib/_tri_wrapper.cpp.patch | 12 ++ .../matplotlib/libfreetype.pc.template | 10 ++ .../recipes/matplotlib/setup.cfg.template | 38 +++++ kivy_ios/recipes/matplotlib/setup.py.patch | 20 +++ kivy_ios/recipes/matplotlib/setupext.py.patch | 55 ++++++ kivy_ios/tools/cpplink | 157 ++++++++++++++++++ 11 files changed, 487 insertions(+), 3 deletions(-) create mode 100644 kivy_ios/recipes/kiwisolver/__init__.py create mode 100644 kivy_ios/recipes/matplotlib/__init__.py create mode 100644 kivy_ios/recipes/matplotlib/_tri.cpp.patch create mode 100644 kivy_ios/recipes/matplotlib/_tri.h.patch create mode 100644 kivy_ios/recipes/matplotlib/_tri_wrapper.cpp.patch create mode 100644 kivy_ios/recipes/matplotlib/libfreetype.pc.template create mode 100644 kivy_ios/recipes/matplotlib/setup.cfg.template create mode 100644 kivy_ios/recipes/matplotlib/setup.py.patch create mode 100644 kivy_ios/recipes/matplotlib/setupext.py.patch create mode 100755 kivy_ios/tools/cpplink diff --git a/MANIFEST.in b/MANIFEST.in index 0bcb1877..92611cf7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ recursive-include kivy_ios *.py -recursive-include kivy_ios/recipes *.py *.patch *.diff *.rst ModulesSetup ModulesSetup.mobile *.m *.h *.pyx *.so -recursive-include kivy_ios/tools *.py biglink liblink +recursive-include kivy_ios/recipes *.py *.patch *.diff *.rst ModulesSetup ModulesSetup.mobile *.m *.h *.pyx *.so *.template +recursive-include kivy_ios/tools *.py biglink liblink cpplink recursive-include kivy_ios/tools/templates * -prune .git \ No newline at end of file +prune .git diff --git a/kivy_ios/recipes/kiwisolver/__init__.py b/kivy_ios/recipes/kiwisolver/__init__.py new file mode 100644 index 00000000..d71f7d3a --- /dev/null +++ b/kivy_ios/recipes/kiwisolver/__init__.py @@ -0,0 +1,38 @@ +''' +This file is derived from the p4a recipe for kiwisolver. +It is a dependency of matplotlib. + +It is a C++ library, and it utilizes the cpplink script to handle +creating the library files needed for inclusion in an iOS project. + +It also depends on the headers from the cppy package. +''' + +from kivy_ios.toolchain import CythonRecipe +from os.path import join + + +class KiwiSolverRecipe(CythonRecipe): + + site_packages_name = 'kiwisolver' + version = '1.3.2' + url = 'https://github.com/nucleic/kiwi/archive/{version}.zip' + depends = ["python"] + hostpython_prerequisites = ["cppy"] + cythonize = False + library = "libkiwisolver.a" + + def get_recipe_env(self, arch=None, with_flags_in_cc=True): + env = super().get_recipe_env(arch) + + # cpplink setup + env['CXX_ORIG'] = env['CXX'] + env['CXX'] = join(self.ctx.root_dir, "tools", "cpplink") + + # setuptools uses CC for compiling and CXX for linking + env['CC'] = env['CXX'] + env['CFLAGS'] += ' -isysroot {}'.format(env['IOSSDKROOT']) + return env + + +recipe = KiwiSolverRecipe() diff --git a/kivy_ios/recipes/matplotlib/__init__.py b/kivy_ios/recipes/matplotlib/__init__.py new file mode 100644 index 00000000..628055dd --- /dev/null +++ b/kivy_ios/recipes/matplotlib/__init__.py @@ -0,0 +1,122 @@ +''' +This file is derived from the p4a recipe for matplotlib. +It is a dependency of matplotlib. + +It is a C++ library, and it utilizes the cpplink script to handle +creating the library files needed for inclusion in an iOS project. + +In addition to the original patch files for p4a, additional patch files +are necessary to prevent duplicate symbols from appearing in the final +link of a kivy-ios application. +''' + +from kivy_ios.toolchain import CythonRecipe, ensure_dir +from os.path import join, abspath, dirname +import shutil +import sh + + +class MatplotlibRecipe(CythonRecipe): + version = '3.5.2' + url = 'https://github.com/matplotlib/matplotlib/archive/v{version}.zip' + library = 'libmatplotlib.a' + depends = ['kiwisolver', 'numpy', 'pillow', 'freetype'] + pre_build_ext = True + python_depends = ['cycler', 'fonttools', 'packaging', + 'pyparsing', 'python-dateutil'] + cythonize = False + + def generate_libraries_pc_files(self, arch): + """ + Create *.pc files for libraries that `matplotib` depends on. + + Because, for unix platforms, the mpl install script uses `pkg-config` + to detect libraries installed in non standard locations (our case... + well...we don't even install the libraries...so we must trick a little + the mlp install). + """ + pkg_config_path = self.get_recipe_env(arch)['PKG_CONFIG_PATH'] + ensure_dir(pkg_config_path) + + lib_to_pc_file = { + # `pkg-config` search for version freetype2.pc, our current + # version for freetype, but we have our recipe named without + # the version...so we add it in here for our pc file + 'freetype': 'freetype2.pc', + } + + for lib_name in {'freetype'}: + pc_template_file = join( + abspath(self.recipe_dir), + f'lib{lib_name}.pc.template' + ) + # read template file into buffer + with open(pc_template_file) as template_file: + text_buffer = template_file.read() + # set the library absolute path and library version + lib_recipe = self.get_recipe(lib_name, self.ctx) + text_buffer = text_buffer.replace( + 'path_to_built', lib_recipe.get_build_dir(arch.arch), + ) + text_buffer = text_buffer.replace( + 'library_version', lib_recipe.version, + ) + + # write the library pc file into our defined dir `PKG_CONFIG_PATH` + pc_dest_file = join(pkg_config_path, lib_to_pc_file[lib_name]) + with open(pc_dest_file, 'w') as pc_file: + pc_file.write(text_buffer) + + def prebuild_arch(self, arch): + if self.has_marker("patched"): + return + shutil.copyfile( + join(abspath(self.recipe_dir), "setup.cfg.template"), + join(self.get_build_dir(arch.arch), "mplsetup.cfg"), + ) + self.generate_libraries_pc_files(arch) + self.apply_patch('_tri.cpp.patch') + self.apply_patch('_tri.h.patch') + self.apply_patch('_tri_wrapper.cpp.patch') + self.apply_patch('setupext.py.patch') + self.apply_patch('setup.py.patch') + self.set_marker("patched") + + def get_recipe_env(self, arch=None, with_flags_in_cc=True): + env = super().get_recipe_env(arch) + + # we make use of the same directory than `XDG_CACHE_HOME`, for our + # custom library pc files, so we have all the install files that we + # generate at the same place + env['XDG_CACHE_HOME'] = join( + self.get_build_dir(arch.arch), + 'p4a_files' + ) + env['PKG_CONFIG_PATH'] = env['XDG_CACHE_HOME'] + + # creating proper *.pc files for our libraries does not seem enough to + # success with our build (without depending on system development + # libraries), but if we tell the compiler where to find our libraries + # and includes, then the install success :) + freetype = self.get_recipe('freetype', self.ctx) + free_inc_dir = join(freetype.get_build_dir(arch.arch), 'include') + + numpytype = self.get_recipe('numpy', self.ctx) + numpy_inc_dir = join(numpytype.get_build_dir(arch.arch), + 'build', 'src.macosx-13.5-arm64-3.10', + 'numpy', 'core', 'include', 'numpy') + + # this numpy include directory is not in the dist directory + numpy_inc_dir = dirname(sh.glob(numpytype.get_build_dir(arch.arch) + '/**/_numpyconfig.h', recursive=True)[0]) + + env['CFLAGS'] += f' -I{free_inc_dir} -I{numpy_inc_dir}' + env['CXX_ORIG'] = env['CXX'] + env['CXX'] = join(self.ctx.root_dir, "tools", "cpplink") + + # setuptools uses CC for compiling and CXX for linking + env['CFLAGS'] += ' -isysroot {}'.format(env['IOSSDKROOT']) + + return env + + +recipe = MatplotlibRecipe() diff --git a/kivy_ios/recipes/matplotlib/_tri.cpp.patch b/kivy_ios/recipes/matplotlib/_tri.cpp.patch new file mode 100644 index 00000000..1aa7db01 --- /dev/null +++ b/kivy_ios/recipes/matplotlib/_tri.cpp.patch @@ -0,0 +1,15 @@ +--- matplotlib-3.5.2.orig/src/tri/_tri.cpp 2022-05-02 22:49:57 ++++ matplotlib-3.5.2/src/tri/_tri.cpp 2023-09-08 13:17:37 +@@ -13,6 +13,7 @@ + #include + #include + ++namespace tri { + + TriEdge::TriEdge() + : tri(-1), edge(-1) +@@ -2069,3 +2070,4 @@ + _seed = (_seed*_a + _c) % _m; + return (_seed*max_value) / _m; + } ++} diff --git a/kivy_ios/recipes/matplotlib/_tri.h.patch b/kivy_ios/recipes/matplotlib/_tri.h.patch new file mode 100644 index 00000000..1191a2c0 --- /dev/null +++ b/kivy_ios/recipes/matplotlib/_tri.h.patch @@ -0,0 +1,17 @@ +--- matplotlib-3.5.2.orig/src/tri/_tri.h 2022-05-02 22:49:57 ++++ matplotlib-3.5.2/src/tri/_tri.h 2023-09-08 13:17:12 +@@ -72,6 +72,7 @@ + #include + + ++namespace tri { + + /* An edge of a triangle consisting of an triangle index in the range 0 to + * ntri-1 and an edge index in the range 0 to 2. Edge i goes from the +@@ -814,5 +815,6 @@ + const unsigned long _m, _a, _c; + unsigned long _seed; + }; ++} + + #endif diff --git a/kivy_ios/recipes/matplotlib/_tri_wrapper.cpp.patch b/kivy_ios/recipes/matplotlib/_tri_wrapper.cpp.patch new file mode 100644 index 00000000..4cd87f61 --- /dev/null +++ b/kivy_ios/recipes/matplotlib/_tri_wrapper.cpp.patch @@ -0,0 +1,12 @@ +--- matplotlib-3.5.2.orig/src/tri/_tri_wrapper.cpp 2022-05-02 22:49:57 ++++ matplotlib-3.5.2/src/tri/_tri_wrapper.cpp 2023-09-08 13:23:47 +@@ -1,7 +1,9 @@ + #include "_tri.h" + #include "../mplutils.h" + #include "../py_exceptions.h" ++ + ++using namespace tri; + + /* Triangulation */ + diff --git a/kivy_ios/recipes/matplotlib/libfreetype.pc.template b/kivy_ios/recipes/matplotlib/libfreetype.pc.template new file mode 100644 index 00000000..df5ef288 --- /dev/null +++ b/kivy_ios/recipes/matplotlib/libfreetype.pc.template @@ -0,0 +1,10 @@ +prefix=path_to_built +exec_prefix=${prefix} +includedir=${prefix}/include +libdir=${exec_prefix}/objs/.libs + +Name: freetype2 +Description: The freetype2 library +Version: library_version +Cflags: -I${includedir} +Libs: -L${libdir} -lfreetype diff --git a/kivy_ios/recipes/matplotlib/setup.cfg.template b/kivy_ios/recipes/matplotlib/setup.cfg.template new file mode 100644 index 00000000..96ef80d4 --- /dev/null +++ b/kivy_ios/recipes/matplotlib/setup.cfg.template @@ -0,0 +1,38 @@ +# Rename this file to mplsetup.cfg to modify Matplotlib's build options. + +[libs] +# By default, Matplotlib builds with LTO, which may be slow if you re-compile +# often, and don't need the space saving/speedup. +enable_lto = False +# By default, Matplotlib downloads and builds its own copies of FreeType and of +# Qhull. You may set the following to True to instead link against a system +# FreeType/Qhull. As an exception, Matplotlib defaults to the system version +# of FreeType on AIX. +system_freetype = True +#system_qhull = False + +[packages] +# There are a number of data subpackages from Matplotlib that are +# considered optional. All except 'tests' data (meaning the baseline +# image files) are installed by default, but that can be changed here. +#tests = False + +[gui_support] +# Matplotlib supports multiple GUI toolkits, known as backends. +# The MacOSX backend requires the Cocoa headers included with XCode. +# You can select whether to build it by uncommenting the following line. +# It is never built on Linux or Windows, regardless of the config value. +# +macosx = False + +[rc_options] +# User-configurable options +# +# Default backend, one of: Agg, Cairo, GTK3Agg, GTK3Cairo, GTK4Agg, GTK4Cairo, +# MacOSX, Pdf, Ps, QtAgg, QtCairo, SVG, TkAgg, WX, WXAgg. +# +# The Agg, Ps, Pdf and SVG backends do not require external dependencies. Do +# not choose MacOSX if you have disabled the relevant extension modules. The +# default is determined by fallback. +# +#backend = Agg \ No newline at end of file diff --git a/kivy_ios/recipes/matplotlib/setup.py.patch b/kivy_ios/recipes/matplotlib/setup.py.patch new file mode 100644 index 00000000..84c197f7 --- /dev/null +++ b/kivy_ios/recipes/matplotlib/setup.py.patch @@ -0,0 +1,20 @@ +--- matplotlib-3.5.2.orig/setup.py 2022-05-02 22:49:57 ++++ matplotlib-3.5.2/setup.py 2023-09-09 22:22:57 +@@ -315,7 +315,7 @@ + python_requires='>={}'.format('.'.join(str(n) for n in py_min_version)), + setup_requires=[ + "certifi>=2020.06.20", +- "numpy>=1.17", ++# "numpy>=1.17", + "setuptools_scm>=4", + "setuptools_scm_git_archive", + ], +@@ -323,7 +323,7 @@ + "cycler>=0.10", + "fonttools>=4.22.0", + "kiwisolver>=1.0.1", +- "numpy>=1.17", ++# "numpy>=1.17", + "packaging>=20.0", + "pillow>=6.2.0", + "pyparsing>=2.2.1", diff --git a/kivy_ios/recipes/matplotlib/setupext.py.patch b/kivy_ios/recipes/matplotlib/setupext.py.patch new file mode 100644 index 00000000..83d8cdd5 --- /dev/null +++ b/kivy_ios/recipes/matplotlib/setupext.py.patch @@ -0,0 +1,55 @@ +--- matplotlib-3.5.2.orig/setupext.py 2023-09-08 14:01:18 ++++ matplotlib-3.5.2/setupext.py 2023-09-09 22:23:24 +@@ -404,7 +404,7 @@ + "matplotlib._contour", [ + "src/_contour.cpp", + "src/_contour_wrapper.cpp", +- "src/py_converters.cpp", ++# "src/py_converters.cpp", + ]) + add_numpy_flags(ext) + add_libagg_flags(ext) +@@ -414,7 +414,7 @@ + "matplotlib.ft2font", [ + "src/ft2font.cpp", + "src/ft2font_wrapper.cpp", +- "src/py_converters.cpp", ++# "src/py_converters.cpp", + ]) + FreeType.add_flags(ext) + add_numpy_flags(ext) +@@ -424,19 +424,19 @@ + ext = Extension( + "matplotlib._image", [ + "src/_image_wrapper.cpp", +- "src/py_converters.cpp", ++# "src/py_converters.cpp", + ]) + add_numpy_flags(ext) +- add_libagg_flags_and_sources(ext) ++ add_libagg_flags(ext) + yield ext + # path + ext = Extension( + "matplotlib._path", [ +- "src/py_converters.cpp", ++# "src/py_converters.cpp", + "src/_path_wrapper.cpp", + ]) + add_numpy_flags(ext) +- add_libagg_flags_and_sources(ext) ++ add_libagg_flags(ext) + yield ext + # qhull + ext = Extension( +@@ -499,8 +499,8 @@ + + + def add_numpy_flags(ext): +- import numpy as np +- ext.include_dirs.append(np.get_include()) ++# import numpy as np ++# ext.include_dirs.append(np.get_include()) + ext.define_macros.extend([ + # Ensure that PY_ARRAY_UNIQUE_SYMBOL is uniquely defined for each + # extension. diff --git a/kivy_ios/tools/cpplink b/kivy_ios/tools/cpplink new file mode 100755 index 00000000..dac8a671 --- /dev/null +++ b/kivy_ios/tools/cpplink @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +''' +C++ libraries are linked using environ['CXX'] compiler. This is different +from C libraries, which are linked using environ['LDSHARED']. + +This script behaves like a C++ compiler or a linker, depending on +whether its output file ends with '.so'. + +If for compiling, this script forwards all arguments in a call to the +compiler specified as environ['CXX_ORIG']. + +If for linking, this script collects the same object and dependent +library data. It then generates the same files as liblink would for an +ld target. The linker called is specified in environ['ARM_LD']. +''' + +import sys +import subprocess +from os import environ, remove + + +# this section is to quickly bypass the full script for cases of c++ compiling + +def get_output(args): + ''' + gets output file + ''' + output = None + get_next = False + + for arg in args: + if get_next: + output = arg + break + elif arg.startswith('-o'): + if arg == '-o': + get_next = True + else: + output = arg[2:] + break + + return output + + +def call_cpp(args): + ''' + call the c++ compiler and return error code + throws a RuntimeError if there is an exception in processing + ''' + result = subprocess.run([environ['CXX_ORIG'], *args]) + if result.returncode != 0: + raise RuntimeError("Compiling C++ failed") + + +def parse_linker_args(args): + ''' + parse arguments to the linker + ''' + libs = [] + objects = [] + i = 0 + while i < len(args): + opt = args[i] + i += 1 + + if opt == "-o": + i += 1 + continue + elif opt.startswith("-l") or opt.startswith("-L"): + libs.append(opt) + continue + elif opt in ("-r", "-pipe", "-no-cpp-precomp"): + continue + elif opt in ( + "--sysroot", "-isysroot", "-framework", "-undefined", + "-macosx_version_min" + ): + i += 1 + continue + elif opt.startswith(("-I", "-m", "-f", "-O", "-g", "-D", "-arch", "-Wl", "-W", "-stdlib=")): + continue + elif opt.startswith("-"): + raise RuntimeError(str(args) + "\nUnknown option: " + opt) + elif not opt.endswith('.o'): + continue + + objects.append(opt) + + if not objects: + raise RuntimeError('C++ Linker arguments contain no object files') + + return libs, objects + + +def call_linker(objects, output): + ''' + calls linker (environ['ARM_LD']) and returns error code + throws a RuntimeError if there is an exception in processing + ''' + print('cpplink redirect linking with', objects) + ld = environ.get('ARM_LD') + arch = environ.get('ARCH', 'arm64') + if 'arm' in arch: + min_version_flag = '-ios_version_min' + else: + min_version_flag = '-ios_simulator_version_min' + call = [ld, '-r', '-o', output + '.o', min_version_flag, '9.0', '-arch', arch] + if min_version_flag == "-ios_version_min": + call += ["-bitcode_bundle"] + call += objects + print("Linking: {}".format(" ".join(call))) + result = subprocess.run(call) + + if result.returncode != 0: + raise RuntimeError("C++ Linking failed") + + +def delete_so_files(output): + ''' + delete shared object files needed for proper module loading + ''' + try: + remove(output) + except FileNotFoundError: + pass + + try: + remove(output + '.libs') + except FileNotFoundError: + pass + + +def write_so_files(output, libs): + ''' + Writes empty .so and .so.libs file which is needed for proper module loading + ''' + with open(output, "w") as f: + f.write('') + + with open(output + ".libs", "w") as f: + f.write(" ".join(libs)) + + +# command line arguments to the C++ Compiler/Linker +args = sys.argv[1:] +# get the output files +output = get_output(args) + +if not output.endswith('.so'): + # C++ Compiling + subprocess.run([environ['CXX_ORIG'], *args]) +else: + # C++ Linking + libs, objects = parse_linker_args(args) + delete_so_files(output) + call_linker(objects, output) + write_so_files(output, libs)