PICARD-2751: Avoid deprecated importlib APIs, full Python 3.12 compatibility

.github/workflows/pypi-release.yml

@@ -81,7 +81,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-11, windows-2019]
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12.0-rc.1']
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}

.github/workflows/run-tests.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest, windows-2019]
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12.0-rc.1']
         include:
         - os: macos-11
           python-version: '3.7'

picard/pluginmanager.py

@@ -25,11 +25,12 @@
 
 
 from functools import partial
-import imp
 import importlib
+from importlib.abc import MetaPathFinder
 import json
 import os.path
 import shutil
+import sys
 import tempfile
 import zipfile
 import zipimport
@@ -46,6 +47,7 @@
     PLUGINS_API,
     USER_PLUGIN_DIR,
 )
+from picard.const.sys import IS_FROZEN
 from picard.plugin import (
     _PLUGIN_MODULE_PREFIX,
     PluginData,
@@ -156,37 +158,61 @@ def _plugin_name_from_path(path):
         return None
 
 
-def load_manifest(archive_path):
-    archive = zipfile.ZipFile(archive_path)
-    manifest_data = None
-    with archive.open('MANIFEST.json') as f:
-        manifest_data = json.loads(str(f.read().decode()))
-    return manifest_data
+def load_zip_manifest(archive_path):
+    if is_zipped_package(archive_path):
+        try:
+            archive = zipfile.ZipFile(archive_path)
+            with archive.open('MANIFEST.json') as f:
+                return json.loads(str(f.read().decode()))
+        except Exception as why:
+            log.warning("Failed to load manifest data from json: %s", why)
+            return None
 
 
 def zip_import(path):
     if (not is_zip(path) or not os.path.isfile(path)):
-        return (None, None, None)
+        return None
     try:
-        zip_importer = zipimport.zipimporter(path)
-        plugin_name = _plugin_name_from_path(path)
-        manifest_data = None
-        if is_zipped_package(path):
-            try:
-                manifest_data = load_manifest(path)
-            except Exception as why:
-                log.warning("Failed to load manifest data from json: %s", why)
-        return (zip_importer, plugin_name, manifest_data)
+        return zipimport.zipimporter(path)
     except zipimport.ZipImportError as why:
         log.error("ZIP import error: %s", why)
-        return (None, None, None)
+        return None
 
 
 def _compatible_api_versions(api_versions):
     versions = [Version.from_string(v) for v in list(api_versions)]
     return set(versions) & set(picard.api_versions_tuple)
 
 
+_plugin_dirs = []
+
+
+def plugin_dirs():
+    yield from _plugin_dirs
+
+
+def init_default_plugin_dirs():
+    # Add user plugin dir first
+    if not os.path.exists(USER_PLUGIN_DIR):
+        os.makedirs(USER_PLUGIN_DIR)
+    register_plugin_dir(USER_PLUGIN_DIR)
+
+    # Register system wide plugin dir
+    if IS_FROZEN:
+        toppath = sys.argv[0]
+    else:
+        toppath = os.path.abspath(__file__)
+
+    topdir = os.path.dirname(toppath)
+    plugin_dir = os.path.join(topdir, "plugins")
+    register_plugin_dir(plugin_dir)
+
+
+def register_plugin_dir(path):
+    if path not in _plugin_dirs:
+        _plugin_dirs.append(path)
+
+
 class PluginManager(QtCore.QObject):
 
     plugin_installed = QtCore.pyqtSignal(PluginWrapper, bool)
@@ -201,6 +227,7 @@ def __init__(self, plugins_directory=None):
         if plugins_directory is None:
             plugins_directory = USER_PLUGIN_DIR
         self.plugins_directory = os.path.normpath(plugins_directory)
+        init_default_plugin_dirs()
 
     @property
     def available_plugins(self):
@@ -273,32 +300,44 @@ def _get_plugin_index_by_name(self, name):
         return (None, None)
 
     def _load_plugin_from_directory(self, name, plugindir):
-        module_file = None
-        zipfilename = os.path.join(plugindir, name + '.zip')
-        (zip_importer, module_name, manifest_data) = zip_import(zipfilename)
-        if zip_importer:
-            name = module_name
-            if not zip_importer.find_module(name):
-                errorfmt = _('Failed loading zipped plugin "%(plugin)s" from "%(filename)s"')
-                self.plugin_error(name, errorfmt, params={
-                    'plugin': name,
-                    'filename': zipfilename,
-                })
-                return None
-            module_pathname = zip_importer.get_filename(name)
-        else:
-            try:
-                info = imp.find_module(name, [plugindir])
-                module_file = info[0]
-                module_pathname = info[1]
-            except ImportError:
+        spec = None
+        module_pathname = None
+        zip_importer = None
+        manifest_data = None
+        full_module_name = _PLUGIN_MODULE_PREFIX + name
+
+        # Legacy loading of ZIP plugins. In Python >= 3.10 this is all handled
+        # by PluginMetaPathFinder. Remove once Python 3.9 is no longer supported.
+        if not hasattr(zipimport.zipimporter, 'find_spec'):
+            zipfilename = os.path.join(plugindir, name + '.zip')
+            zip_importer = zip_import(zipfilename)
+            if zip_importer:
+                if not zip_importer.find_module(name):
+                    errorfmt = _('Failed loading zipped plugin "%(plugin)s" from "%(filename)s"')
+                    self.plugin_error(name, errorfmt, params={
+                        'plugin': name,
+                        'filename': zipfilename,
+                    })
+                    return None
+                module_pathname = zip_importer.get_filename(name)
+                manifest_data = load_zip_manifest(zip_importer.archive)
+
+        if not module_pathname:
+            spec = PluginMetaPathFinder().find_spec(full_module_name, [plugindir])
+            if not spec or not spec.loader:
                 errorfmt = _('Failed loading plugin "%(plugin)s" in "%(dirname)s"')
                 self.plugin_error(name, errorfmt, params={
                     'plugin': name,
                     'dirname': plugindir,
                 })
                 return None
 
+            module_pathname = spec.origin
+            if isinstance(spec.loader, zipimport.zipimporter):
+                manifest_data = load_zip_manifest(spec.loader.archive)
+            if module_pathname.endswith("__init__.py"):
+                module_pathname = os.path.dirname(module_pathname)
+
         plugin = None
         try:
             existing_plugin, existing_plugin_index = self._get_plugin_index_by_name(name)
@@ -310,11 +349,12 @@ def _load_plugin_from_directory(self, name, plugindir):
                             existing_plugin.version,
                             existing_plugin.file)
                 _unregister_module_extensions(name)
-            full_module_name = _PLUGIN_MODULE_PREFIX + name
             if zip_importer:
                 plugin_module = zip_importer.load_module(full_module_name)
             else:
-                plugin_module = imp.load_module(full_module_name, *info)
+                plugin_module = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(plugin_module)
+
             plugin = PluginWrapper(plugin_module, plugindir,
                                    file=module_pathname, manifest_data=manifest_data)
             compatible_versions = _compatible_api_versions(plugin.api_versions)
@@ -345,8 +385,6 @@ def _load_plugin_from_directory(self, name, plugindir):
             errorfmt = _('Plugin "%(plugin)s"')
             self.plugin_error(name, errorfmt, log_func=log.exception,
                               params={'plugin': name})
-        if module_file is not None:
-            module_file.close()
         return plugin
 
     def _get_existing_paths(self, plugin_name, fileexts):
@@ -553,3 +591,24 @@ def _display_update():
             self.query_available_plugins(_display_update)
         else:
             _display_update()
+
+
+class PluginMetaPathFinder(MetaPathFinder):
+    def find_spec(self, fullname, path, target=None):
+        if not fullname.startswith(_PLUGIN_MODULE_PREFIX):
+            return None
+        plugin_name = fullname[len(_PLUGIN_MODULE_PREFIX):]
+        spec = None
+        if hasattr(zipimport.zipimporter, 'find_spec'):  # Python >= 3.10
+            for plugin_dir in _plugin_dirs:
+                zipfilename = os.path.join(plugin_dir, plugin_name + '.zip')
+                zip_importer = zip_import(zipfilename)
+                if zip_importer:
+                    spec = zip_importer.find_spec(fullname)
+                    break
+        if not spec:
+            spec = importlib.machinery.PathFinder().find_spec(fullname, _plugin_dirs)
+        return spec if spec and spec.loader else None
+
+
+sys.meta_path.append(PluginMetaPathFinder())

picard/tagger.py

@@ -94,12 +94,8 @@
     setup_config,
 )
 from picard.config_upgrade import upgrade_config
-from picard.const import (
-    USER_DIR,
-    USER_PLUGIN_DIR,
-)
+from picard.const import USER_DIR
 from picard.const.sys import (
-    IS_FROZEN,
     IS_HAIKU,
     IS_MACOS,
     IS_WIN,
@@ -114,7 +110,10 @@
 from picard.file import File
 from picard.formats import open_ as open_file
 from picard.i18n import setup_gettext
-from picard.pluginmanager import PluginManager
+from picard.pluginmanager import (
+    PluginManager,
+    plugin_dirs,
+)
 from picard.releasegroup import ReleaseGroup
 from picard.track import (
     NonAlbumTrack,
@@ -175,21 +174,6 @@ def _patched_shutil_copystat(src, dst, *, follow_symlinks=True):
 shutil.copystat = _patched_shutil_copystat
 
 
-def plugin_dirs():
-    if IS_FROZEN:
-        toppath = sys.argv[0]
-    else:
-        toppath = os.path.abspath(__file__)
-
-    topdir = os.path.dirname(toppath)
-    plugin_dir = os.path.join(topdir, "plugins")
-    yield plugin_dir
-
-    if not os.path.exists(USER_PLUGIN_DIR):
-        os.makedirs(USER_PLUGIN_DIR)
-    yield USER_PLUGIN_DIR
-
-
 class ParseItemsToLoad:
 
     WINDOWS_DRIVE_TEST = re.compile(r"^[a-z]\:", re.IGNORECASE)