From 38a7b9e609772da3c8914ac40294097d5bd17cde Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Fri, 12 Jan 2024 20:03:03 +0100 Subject: [PATCH 1/6] Fix plugin management bugs in nikola plugin and nikola import_wordpress --- CHANGES.txt | 2 +- nikola/plugin_categories.py | 1 + nikola/plugin_manager.py | 8 +- nikola/plugins/command/import_wordpress.py | 35 +++-- nikola/plugins/command/plugin.py | 148 +++++++++------------ nikola/utils.py | 32 ++--- 6 files changed, 108 insertions(+), 118 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 50f5b6ebac..f6e0582a82 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -7,7 +7,7 @@ Features Bugfixes -------- -* Fix the ``nikola plugin`` command not working (Issue #3736) +* Fix the ``nikola plugin`` command not working (Issue #3736, #3737) New in v8.3.0 ============= diff --git a/nikola/plugin_categories.py b/nikola/plugin_categories.py index b9eee3af00..7b650a6c7d 100644 --- a/nikola/plugin_categories.py +++ b/nikola/plugin_categories.py @@ -61,6 +61,7 @@ class BasePlugin: """Base plugin class.""" logger = None + site: 'nikola.nikola.Nikola' def set_site(self, site): """Set site, which is a Nikola instance.""" diff --git a/nikola/plugin_manager.py b/nikola/plugin_manager.py index 5c1a29a103..e2e97f22a8 100644 --- a/nikola/plugin_manager.py +++ b/nikola/plugin_manager.py @@ -74,6 +74,7 @@ class PluginInfo: category: str compiler: Optional[str] source_dir: Path + py_file_location: Path module_name: str module_object: object plugin_object: BasePlugin @@ -147,9 +148,10 @@ def locate_plugins(self) -> List[PluginCandidate]: ) return self.candidates - def load_plugins(self, candidates: List[PluginCandidate]) -> None: + def load_plugins(self, candidates: List[PluginCandidate]) -> List[PluginInfo]: """Load selected candidate plugins.""" plugins_root = Path(__file__).parent.parent + new_plugins = [] for candidate in candidates: name = candidate.name @@ -223,11 +225,13 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: category=candidate.category, compiler=candidate.compiler, source_dir=source_dir, + py_file_location=py_file_location, module_name=module_name, module_object=module_object, plugin_object=plugin_object, ) self.plugins.append(info) + new_plugins.append(info) self._plugins_by_category = {category: [] for category in CATEGORY_NAMES} for plugin_info in self.plugins: @@ -240,6 +244,8 @@ def load_plugins(self, candidates: List[PluginCandidate]) -> None: self.logger.warning("Waiting 2 seconds before continuing.") time.sleep(2) + return new_plugins + def get_plugins_of_category(self, category: str) -> List[PluginInfo]: """Get loaded plugins of a given category.""" return self._plugins_by_category.get(category, []) diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index e6ec45b334..45cf218e5f 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -38,7 +38,7 @@ import requests from lxml import etree -from nikola.plugin_categories import Command +from nikola.plugin_categories import Command, CompilerExtension from nikola import utils, hierarchy_utils from nikola.nikola import DEFAULT_TRANSLATIONS_PATTERN from nikola.utils import req_missing @@ -68,12 +68,8 @@ def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False) # Get hold of the 'plugin' plugin plugin_installer_info = site.plugin_manager.get_plugin_by_name('plugin', 'Command') if plugin_installer_info is None: - LOGGER.error('Internal error: cannot find the "plugin" plugin which is supposed to come with Nikola!') + LOGGER.error('Internal error: cannot find the "plugin" plugin which is supposed to come with Nikola - it might be disabled in conf.py') return False - if not plugin_installer_info.is_activated: - # Someone might have disabled the plugin in the `conf.py` used - site.plugin_manager.activatePluginByName(plugin_installer_info.name) - plugin_installer_info.plugin_object.set_site(site) plugin_installer = plugin_installer_info.plugin_object # Try to install the requested plugin options = {} @@ -85,9 +81,16 @@ def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False) if plugin_installer.execute(options=options) > 0: return False # Let the plugin manager find newly installed plugins - site.plugin_manager.collectPlugins() - # Re-scan for compiler extensions - site.compiler_extensions = site._activate_plugins_of_category("CompilerExtension") + old_candidates = set(site.plugin_manager.candidates.copy()) + new_candidates = set(site.plugin_manager.locate_plugins()) + missing_candidates = list(new_candidates - old_candidates) + new_plugins = site.plugin_manager.load_plugins(missing_candidates) + + # Activate new plugins + for p in new_plugins: + site._activate_plugin(p) + if isinstance(p.plugin_object, CompilerExtension): + site.compiler_extensions.append(p) return True @@ -248,12 +251,16 @@ def _find_wordpress_compiler(self): """Find WordPress compiler plugin.""" if self.wordpress_page_compiler is not None: return + plugin_info = self.site.plugin_manager.get_plugin_by_name('wordpress', 'PageCompiler') - if plugin_info is not None: - if not plugin_info.is_activated: - self.site.plugin_manager.activatePluginByName(plugin_info.name) - plugin_info.plugin_object.set_site(self.site) - self.wordpress_page_compiler = plugin_info.plugin_object + if plugin_info is None: + candidates = self.site.plugin_manager.locate_plugins() + wordpress_candidates = [c for c in candidates if c.name == "wordpress"] + if wordpress_candidates: + new_plugins = self.site.plugin_manager.load_plugins(wordpress_candidates) + for p in new_plugins: + self.site._activate_plugin(p) + self.wordpress_page_compiler = p def _read_options(self, options, args): """Read command-line options.""" diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py index 4fd8e8f805..50afbc571b 100644 --- a/nikola/plugins/command/plugin.py +++ b/nikola/plugins/command/plugin.py @@ -28,11 +28,13 @@ import io import json.decoder -import os +import pathlib import sys import shutil import subprocess import time +import typing + import requests import pygments @@ -52,8 +54,9 @@ class CommandPlugin(Command): name = "plugin" doc_usage = "[-u url] [--user] [-i name] [-r name] [--upgrade] [-l] [--list-installed]" doc_purpose = "manage plugins" - output_dir = None + output_dir: pathlib.Path = None needs_config = False + never_upgrade = {'emoji'} # plugin with the same name is shipped with Nikola cmd_options = [ { 'name': 'install', @@ -133,16 +136,16 @@ def _execute(self, options, args): return 2 if options.get('output_dir') is not None: - self.output_dir = options.get('output_dir') + self.output_dir = pathlib.Path(options.get('output_dir')) else: if not self.site.configured and not user_mode and install: LOGGER.warning('No site found, assuming --user') user_mode = True if user_mode: - self.output_dir = os.path.expanduser(os.path.join('~', '.nikola', 'plugins')) + self.output_dir = pathlib.Path.home() / ".nikola" / "plugins" else: - self.output_dir = 'plugins' + self.output_dir = pathlib.Path("plugins") if list_available: return self.list_available(url) @@ -166,14 +169,7 @@ def list_available(self, url): def list_installed(self): """List installed plugins.""" - plugins = [] - for plugin in self.site.plugin_manager.getAllPlugins(): - p = plugin.path - if os.path.isdir(p): - p = p + os.sep - else: - p = p + '.py' - plugins.append([plugin.name, p]) + plugins = self.get_plugins() plugins.sort() print('Installed Plugins:') @@ -196,25 +192,18 @@ def do_upgrade(self, url): """Upgrade all installed plugins.""" LOGGER.warning('This is not very smart, it just reinstalls some plugins and hopes for the best') data = self.get_json(url) - plugins = [] - for plugin in self.site.plugin_manager.getAllPlugins(): - p = plugin.path - if os.path.isdir(p): - p = p + os.sep - else: - p = p + '.py' - if plugin.name in data: - plugins.append([plugin.name, p]) - print('Will upgrade {0} plugins: {1}'.format(len(plugins), ', '.join(n for n, _ in plugins))) + plugins = [(n, p) for n, p in self.get_plugins() if n in data and n not in self.never_upgrade] + LOGGER.info('Will upgrade {0} plugins: {1}'.format(len(plugins), ', '.join(n for n, _ in plugins))) for name, path in plugins: - print('Upgrading {0}'.format(name)) + path: pathlib.Path + LOGGER.info('Upgrading {0}'.format(name)) p = path while True: - tail, head = os.path.split(path) + tail, head = path.parent, path.name if head == 'plugins': self.output_dir = path break - elif tail == '': + elif path == tail: LOGGER.error("Can't find the plugins folder for path: {0}".format(p)) return 1 else: @@ -229,104 +218,84 @@ def do_install(self, url, name, show_install_notes=True): utils.makedirs(self.output_dir) url = data[name] LOGGER.info("Downloading '{0}'".format(url)) - try: - zip_data = requests.get(url).content - except requests.exceptions.SSLError: - LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") - time.sleep(1) - url = url.replace('https', 'http', 1) - zip_data = requests.get(url).content + zip_data = requests.get(url).content zip_file = io.BytesIO() zip_file.write(zip_data) LOGGER.info('Extracting: {0} into {1}/'.format(name, self.output_dir)) utils.extract_all(zip_file, self.output_dir) - dest_path = os.path.join(self.output_dir, name) + dest_path = self.output_dir / name else: LOGGER.error("Can't find plugin " + name) return 1 - reqpath = os.path.join(dest_path, 'requirements.txt') - if os.path.exists(reqpath): + requirements_path = dest_path / 'requirements.txt' + if requirements_path.exists(): LOGGER.warning('This plugin has Python dependencies.') LOGGER.info('Installing dependencies with pip...') try: - subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', reqpath)) + subprocess.check_call((sys.executable, '-m', 'pip', 'install', '-r', str(requirements_path))) except subprocess.CalledProcessError: LOGGER.error('Could not install the dependencies.') print('Contents of the requirements.txt file:\n') - with io.open(reqpath, 'r', encoding='utf-8-sig') as fh: - print(utils.indent(fh.read(), 4 * ' ')) - print('You have to install those yourself or through a ' - 'package manager.') + print(utils.indent(requirements_path.read_text(), 4 * ' ')) + print('You have to install those yourself or through a package manager.') else: LOGGER.info('Dependency installation succeeded.') - reqnpypath = os.path.join(dest_path, 'requirements-nonpy.txt') - if os.path.exists(reqnpypath): - LOGGER.warning('This plugin has third-party ' - 'dependencies you need to install ' - 'manually.') + requirements_nonpy_path = dest_path / 'requirements-nonpy.txt' + if requirements_nonpy_path.exists(): + LOGGER.warning('This plugin has third-party dependencies you need to install manually.') print('Contents of the requirements-nonpy.txt file:\n') - with io.open(reqnpypath, 'r', encoding='utf-8-sig') as fh: - for l in fh.readlines(): - i, j = l.split('::') - print(utils.indent(i.strip(), 4 * ' ')) - print(utils.indent(j.strip(), 8 * ' ')) - print() + for l in requirements_nonpy_path.read_text().strip().split('\n'): + i, j = l.split('::') + print(utils.indent(i.strip(), 4 * ' ')) + print(utils.indent(j.strip(), 8 * ' ')) + print() print('You have to install those yourself or through a package ' 'manager.') - req_plug_path = os.path.join(dest_path, 'requirements-plugins.txt') - if os.path.exists(req_plug_path): + requirements_plugins_path = dest_path / 'requirements-plugins.txt' + if requirements_plugins_path.exists(): LOGGER.info('This plugin requires other Nikola plugins.') LOGGER.info('Installing plugins...') plugin_failure = False try: - with io.open(req_plug_path, 'r', encoding='utf-8-sig') as inf: - for plugname in inf.readlines(): - plugin_failure = self.do_install(url, plugname.strip(), show_install_notes) != 0 + for plugin_name in requirements_plugins_path.read_text().strip().split('\n'): + plugin_failure = self.do_install(url, plugin_name.strip(), show_install_notes) != 0 except Exception: plugin_failure = True if plugin_failure: LOGGER.error('Could not install a plugin.') print('Contents of the requirements-plugins.txt file:\n') - with io.open(req_plug_path, 'r', encoding='utf-8-sig') as fh: - print(utils.indent(fh.read(), 4 * ' ')) + print(utils.indent(requirements_plugins_path.read_text(), 4 * ' ')) print('You have to install those yourself manually.') else: LOGGER.info('Dependency installation succeeded.') - confpypath = os.path.join(dest_path, 'conf.py.sample') - if os.path.exists(confpypath) and show_install_notes: + confpy_path = dest_path / 'conf.py.sample' + if confpy_path.exists() and show_install_notes: LOGGER.warning('This plugin has a sample config file. Integrate it with yours in order to make this plugin work!') print('Contents of the conf.py.sample file:\n') - with io.open(confpypath, 'r', encoding='utf-8-sig') as fh: - if self.site.colorful: - print(pygments.highlight(fh.read(), PythonLexer(), TerminalFormatter())) - else: - print(fh.read()) + if self.site.colorful: + print(pygments.highlight(confpy_path.read_text(), PythonLexer(), TerminalFormatter())) + else: + print(confpy_path.read_text()) return 0 def do_uninstall(self, name): """Uninstall a plugin.""" - for plugin in self.site.plugin_manager.getAllPlugins(): # FIXME: this is repeated thrice - if name == plugin.name: # Uninstall this one - p = plugin.path - if os.path.isdir(p): - # Plugins that have a package in them need to delete parent - # Issue #2356 - p = p + os.sep - p = os.path.abspath(os.path.join(p, os.pardir)) - else: - p = os.path.dirname(p) + for found_name, path in self.get_plugins(): + path: pathlib.Path + if name == found_name: # Uninstall this one + to_delete = path.parent # Delete parent of .py file or parent of package LOGGER.warning('About to uninstall plugin: {0}'.format(name)) - LOGGER.warning('This will delete {0}'.format(p)) + LOGGER.warning('This will delete {0}'.format(to_delete)) sure = utils.ask_yesno('Are you sure?') if sure: - LOGGER.warning('Removing {0}'.format(p)) - shutil.rmtree(p) + LOGGER.warning('Removing {0}'.format(to_delete)) + shutil.rmtree(to_delete) return 0 return 1 LOGGER.error('Unknown plugin: {0}'.format(name)) @@ -336,19 +305,24 @@ def get_json(self, url): """Download the JSON file with all plugins.""" if self.json is None: try: - try: - self.json = requests.get(url).json() - except requests.exceptions.SSLError: - LOGGER.warning("SSL error, using http instead of https (press ^C to abort)") - time.sleep(1) - url = url.replace('https', 'http', 1) - self.json = requests.get(url).json() + self.json = requests.get(url).json() except json.decoder.JSONDecodeError as e: LOGGER.error("Failed to decode JSON data in response from server.") LOGGER.error("JSON error encountered: " + str(e)) - LOGGER.error("This issue might be caused by server-side issues, or by to unusual activity in your " + LOGGER.error("This issue might be caused by server-side issues, or by unusual activity in your " "network (as determined by CloudFlare). Please visit https://plugins.getnikola.com/ in " "a browser.") sys.exit(2) return self.json + + def get_plugins(self) -> typing.List[typing.Tuple[str, pathlib.Path]]: + """Get currently installed plugins in site.""" + plugins = [] + for plugin in self.site.plugin_manager.plugins: + if plugin.py_file_location.name == "__init__.py": + path = plugin.py_file_location.parent + else: + path = plugin.py_file_location + plugins.append((plugin.name, path)) + return plugins diff --git a/nikola/utils.py b/nikola/utils.py index 20e35529ce..cba00614e3 100644 --- a/nikola/utils.py +++ b/nikola/utils.py @@ -49,7 +49,7 @@ from urllib.parse import quote as urlquote from urllib.parse import unquote as urlunquote from urllib.parse import urlparse, urlunparse -from zipfile import ZipFile as zipf +from zipfile import ZipFile import babel.dates import dateutil.parser @@ -904,20 +904,22 @@ def extract_all(zipfile, path='themes'): """Extract all files from a zip file.""" pwd = os.getcwd() makedirs(path) - os.chdir(path) - z = zipf(zipfile) - namelist = z.namelist() - for f in namelist: - if f.endswith('/') and '..' in f: - raise UnsafeZipException('The zip file contains ".." and is ' - 'not safe to expand.') - for f in namelist: - if f.endswith('/'): - makedirs(f) - else: - z.extract(f) - z.close() - os.chdir(pwd) + try: + os.chdir(path) + z = ZipFile(zipfile) + namelist = z.namelist() + for f in namelist: + if f.endswith('/') and '..' in f: + raise UnsafeZipException('The zip file contains ".." and is ' + 'not safe to expand.') + for f in namelist: + if f.endswith('/'): + makedirs(f) + else: + z.extract(f) + z.close() + finally: + os.chdir(pwd) def to_datetime(value, tzinfo=None): From e1403f5eb06a54a5aab56f4bd628a73c2a9ec877 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Fri, 12 Jan 2024 20:06:20 +0100 Subject: [PATCH 2/6] flake8 --- nikola/plugins/command/plugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py index 50afbc571b..d84a14ed09 100644 --- a/nikola/plugins/command/plugin.py +++ b/nikola/plugins/command/plugin.py @@ -32,7 +32,6 @@ import sys import shutil import subprocess -import time import typing import requests @@ -289,7 +288,7 @@ def do_uninstall(self, name): for found_name, path in self.get_plugins(): path: pathlib.Path if name == found_name: # Uninstall this one - to_delete = path.parent # Delete parent of .py file or parent of package + to_delete = path.parent # Delete parent of .py file or parent of package LOGGER.warning('About to uninstall plugin: {0}'.format(name)) LOGGER.warning('This will delete {0}'.format(to_delete)) sure = utils.ask_yesno('Are you sure?') From 5c57f196eeb6403e8b992e7470decaff7f6c9b24 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Mon, 29 Apr 2024 12:23:14 +0200 Subject: [PATCH 3/6] Apply suggestions from code review Co-authored-by: Felix Fontein --- nikola/plugins/command/plugin.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nikola/plugins/command/plugin.py b/nikola/plugins/command/plugin.py index d84a14ed09..39add955d1 100644 --- a/nikola/plugins/command/plugin.py +++ b/nikola/plugins/command/plugin.py @@ -246,7 +246,7 @@ def do_install(self, url, name, show_install_notes=True): if requirements_nonpy_path.exists(): LOGGER.warning('This plugin has third-party dependencies you need to install manually.') print('Contents of the requirements-nonpy.txt file:\n') - for l in requirements_nonpy_path.read_text().strip().split('\n'): + for l in requirements_nonpy_path.read_text().strip().splitlines(): i, j = l.split('::') print(utils.indent(i.strip(), 4 * ' ')) print(utils.indent(j.strip(), 8 * ' ')) @@ -261,7 +261,7 @@ def do_install(self, url, name, show_install_notes=True): LOGGER.info('Installing plugins...') plugin_failure = False try: - for plugin_name in requirements_plugins_path.read_text().strip().split('\n'): + for plugin_name in requirements_plugins_path.read_text().strip().splitlines(): plugin_failure = self.do_install(url, plugin_name.strip(), show_install_notes) != 0 except Exception: plugin_failure = True @@ -286,7 +286,6 @@ def do_install(self, url, name, show_install_notes=True): def do_uninstall(self, name): """Uninstall a plugin.""" for found_name, path in self.get_plugins(): - path: pathlib.Path if name == found_name: # Uninstall this one to_delete = path.parent # Delete parent of .py file or parent of package LOGGER.warning('About to uninstall plugin: {0}'.format(name)) From c317976f0df0e149e4ab73f761537ce0cd825811 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Mon, 29 Apr 2024 12:23:55 +0200 Subject: [PATCH 4/6] Code review fixes in import_wordpress --- nikola/plugins/command/import_wordpress.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index 45cf218e5f..b69b52cf6a 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -81,7 +81,7 @@ def install_plugin(site, plugin_name, output_dir=None, show_install_notes=False) if plugin_installer.execute(options=options) > 0: return False # Let the plugin manager find newly installed plugins - old_candidates = set(site.plugin_manager.candidates.copy()) + old_candidates = set(site.plugin_manager.candidates) new_candidates = set(site.plugin_manager.locate_plugins()) missing_candidates = list(new_candidates - old_candidates) new_plugins = site.plugin_manager.load_plugins(missing_candidates) @@ -255,7 +255,7 @@ def _find_wordpress_compiler(self): plugin_info = self.site.plugin_manager.get_plugin_by_name('wordpress', 'PageCompiler') if plugin_info is None: candidates = self.site.plugin_manager.locate_plugins() - wordpress_candidates = [c for c in candidates if c.name == "wordpress"] + wordpress_candidates = [c for c in candidates if c.name == "wordpress" and c.category == "PageCompiler"] if wordpress_candidates: new_plugins = self.site.plugin_manager.load_plugins(wordpress_candidates) for p in new_plugins: From 21bfeb62e71e0133fd1cb8752906f4893b70f17b Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Mon, 29 Apr 2024 12:32:05 +0200 Subject: [PATCH 5/6] Mention in HTTP fallback removal in CHANGES.txt --- CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.txt b/CHANGES.txt index 430fee7621..edffc76e56 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ Features Bugfixes -------- +* Remove insecure HTTP fallback from ``nikola plugin`` * Fix the ``nikola plugin`` command not working (Issue #3736, #3737) * Fix ``nikola new_post --available-formats`` crashing with TypeError (Issue #3750) From 2090eb68166ab74d5112725f41043d5664476ad0 Mon Sep 17 00:00:00 2001 From: Chris Warrick Date: Mon, 29 Apr 2024 12:48:19 +0200 Subject: [PATCH 6/6] Fix setting of wordpress_page_compiler --- nikola/plugins/command/import_wordpress.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nikola/plugins/command/import_wordpress.py b/nikola/plugins/command/import_wordpress.py index b69b52cf6a..bc162a5aaf 100644 --- a/nikola/plugins/command/import_wordpress.py +++ b/nikola/plugins/command/import_wordpress.py @@ -260,7 +260,9 @@ def _find_wordpress_compiler(self): new_plugins = self.site.plugin_manager.load_plugins(wordpress_candidates) for p in new_plugins: self.site._activate_plugin(p) - self.wordpress_page_compiler = p + self.wordpress_page_compiler = p.plugin_object + else: + self.wordpress_page_compiler = plugin_info.plugin_object def _read_options(self, options, args): """Read command-line options."""