reckless update: enable install @ commit and add python virtual environments

doc/reckless.7.md

@@ -14,12 +14,12 @@ installation involves: finding the source plugin, copying,
 installing dependencies, testing, activating, and updating the
 lightningd config file. Reckless does all of these by invoking:
 
-**reckless** **install** *plugin\_name*
+**reckless** **install**[@*commit/tag*] *plugin\_name*
 
 reckless will exit early in the event that:
 
 - the plugin is not found in any available source repositories
-- dependencies are not sucessfully installed
+- dependencies are not successfully installed
 - the plugin fails to execute
 
 Reckless-installed plugins reside in the 'reckless' subdirectory
@@ -74,11 +74,14 @@ Available option flags:
 **-v**, **--verbose**
 	request additional debug output
 
+**--network**=*network*
+	specify bitcoin, regtest, liquid, liquid-regtest, litecoin, signet,
+	or testnet networks. (default: bitcoin)
+
 NOTES
 -----
 
-Reckless currently supports python plugins only. Additional language
-support will be provided in future releases.
+Reckless currently supports python and javascript plugins.
 
 Running the first time will prompt the user that their lightningd's
 bitcoin config will be appended (or created) to inherit the reckless
@@ -97,6 +100,10 @@ the option flag **reckless -d=<my\_alternate\_dir>** may be used to
 relocate the reckless directory from its default. Consider creating a
 permanent alias in this case.
 
+Python plugins are installed to their own virtual environments. The
+environment is activated by a wrapper (named the same as the plugin)
+which then imports and executes the actual plugin entrypoint.
+
 For Plugin Developers:
 
 To make your plugin compatible with reckless install:

tests/data/recklessrepo/lightningd/testplugpass/requirements.txt

@@ -0,0 +1,2 @@
+pyln-client
+

tests/data/recklessrepo/lightningd/testplugpass/testplugpass.py

@@ -3,6 +3,8 @@
 
 plugin = Plugin()
 
+__version__ = 'v1'
+
 
 @plugin.init()
 def init(options, configuration, plugin, **kwargs):
@@ -14,4 +16,10 @@ def testmethod(plugin):
     return ("I live.")
 
 
+@plugin.method("gettestplugversion")
+def gettestplugversion(plugin):
+    "to test commit/tag checkout"
+    return __version__
+
+
 plugin.run()

tests/data/recklessrepo/lightningd/testplugpyproj/pyproject.toml

@@ -0,0 +1,17 @@
+[project]
+dependencies = ["pyln-client"]
+
+[tool.poetry]
+name = "testplugpyproj"
+version = "0.1.0"
+description = "testing poetry installation of python plugins"
+authors = ["Alex Myers <[email protected]>"]
+
+[tool.poetry.dependencies]
+# Build dependencies belong here
+python = "^3.8"
+pyln-client = "^23.11"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

tests/data/recklessrepo/lightningd/testplugpyproj/testplugpyproj.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+from pyln.client import Plugin
+
+plugin = Plugin()
+
+
+@plugin.init()
+def init(options, configuration, plugin, **kwargs):
+    plugin.log("testplug initialized")
+
+
+@plugin.method("testmethod")
+def testmethod(plugin):
+    return ("I live.")
+
+
+plugin.run()

tests/data/recklessrepo/rkls_api_lightningd_plugins.json

@@ -16,5 +16,14 @@
     "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugfail",
     "download_url": null,
     "type": "dir"
+  },
+  {
+    "name": "testplugpyproj",
+    "path": "testplugpyproj",
+    "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master",
+    "html_url": "https://github.com/lightningd/plugins/tree/master/testplugpyproj",
+    "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugpyproj",
+    "download_url": null,
+    "type": "dir"
   }
 ]

tests/test_reckless.py

@@ -2,10 +2,12 @@
 import subprocess
 from pathlib import PosixPath, Path
 import socket
+from pyln.testing.utils import VALGRIND
 import pytest
 import os
 import shutil
 import time
+import unittest
 
 
 @pytest.fixture(autouse=True)
@@ -59,8 +61,15 @@ def canned_github_server(directory):
     repo_initialization = (f'cp -r {plugins_path}/* .;'
                            'git add --all;'
                            'git commit -m "initial commit - autogenerated by test_reckless.py";')
+    tag_and_update = ('git tag v1;'
+                      "sed -i 's/v1/v2/g' testplugpass/testplugpass.py;"
+                      'git add testplugpass/testplugpass.py;'
+                      'git commit -m "update to v2";'
+                      'git tag v2;')
     subprocess.check_output([repo_initialization], env=my_env, shell=True,
                             cwd=repo_dir)
+    subprocess.check_output([tag_and_update], env=my_env,
+                            shell=True, cwd=repo_dir)
     del my_env['HOME']
     del my_env['GIT_DIR']
     del my_env['GIT_WORK_TREE']
@@ -101,7 +110,8 @@ def get_reckless_node(node_factory):
 
 def check_stderr(stderr):
     def output_okay(out):
-        for warning in ['[notice]', 'WARNING:', 'npm WARN', 'npm notice']:
+        for warning in ['[notice]', 'WARNING:', 'npm WARN',
+                        'npm notice', 'DEPRECATION:', 'Creating virtualenv']:
             if out.startswith(warning):
                 return True
         return False
@@ -135,6 +145,35 @@ def test_sources(node_factory):
     n = get_reckless_node(node_factory)
     r = reckless(["source", "-h"], dir=n.lightning_dir)
     assert r.returncode == 0
+    r = reckless(["source", "list"], dir=n.lightning_dir)
+    print(r.stdout)
+    assert r.returncode == 0
+    print(n.lightning_dir)
+    reckless_dir = Path(n.lightning_dir) / 'reckless'
+    print(dir(reckless_dir))
+    assert (reckless_dir / '.sources').exists()
+    print(os.listdir(reckless_dir))
+    print(reckless_dir / '.sources')
+    r = reckless([f"--network={NETWORK}", "-v", "source", "add",
+                  "tests/data/recklessrepo/lightningd/testplugfail"],
+                 dir=n.lightning_dir)
+    r = reckless([f"--network={NETWORK}", "-v", "source", "add",
+                  "tests/data/recklessrepo/lightningd/testplugpass"],
+                 dir=n.lightning_dir)
+    with open(reckless_dir / '.sources') as sources:
+        contents = [c.strip() for c in sources.readlines()]
+        print('contents:', contents)
+        assert 'https://github.com/lightningd/plugins' in contents
+        assert "tests/data/recklessrepo/lightningd/testplugfail" in contents
+        assert "tests/data/recklessrepo/lightningd/testplugpass" in contents
+    r = reckless([f"--network={NETWORK}", "-v", "source", "remove",
+                  "tests/data/recklessrepo/lightningd/testplugfail"],
+                 dir=n.lightning_dir)
+    with open(reckless_dir / '.sources') as sources:
+        contents = [c.strip() for c in sources.readlines()]
+        print('contents:', contents)
+        assert "tests/data/recklessrepo/lightningd/testplugfail" not in contents
+        assert "tests/data/recklessrepo/lightningd/testplugpass" in contents
 
 
 def test_search(node_factory):
@@ -159,6 +198,26 @@ def test_install(node_factory):
     assert os.path.exists(plugin_path)
 
 
+@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection")
+def test_poetry_install(node_factory):
+    """test search, git clone, and installation to folder."""
+    n = get_reckless_node(node_factory)
+    r = reckless([f"--network={NETWORK}", "-v", "install", "testplugpyproj"], dir=n.lightning_dir)
+    assert r.returncode == 0
+    assert 'dependencies installed successfully' in r.stdout
+    assert 'plugin installed:' in r.stdout
+    assert 'testplugpyproj enabled' in r.stdout
+    check_stderr(r.stderr)
+    plugin_path = Path(n.lightning_dir) / 'reckless/testplugpyproj'
+    print(plugin_path)
+    assert os.path.exists(plugin_path)
+    n.start()
+    print(n.rpc.testmethod())
+    assert n.daemon.is_in_log(r'plugin-manager: started\([0-9].*\) /tmp/ltests-[a-z0-9_].*/test_poetry_install_1/lightning-1/reckless/testplugpyproj/testplugpyproj.py')
+    assert n.rpc.testmethod() == 'I live.'
+
+
+@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection")
 def test_local_dir_install(node_factory):
     """Test search and install from local directory source."""
     n = get_reckless_node(node_factory)
@@ -175,6 +234,7 @@ def test_local_dir_install(node_factory):
     assert os.path.exists(plugin_path)
 
 
+@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection")
 def test_disable_enable(node_factory):
     """test search, git clone, and installation to folder."""
     n = get_reckless_node(node_factory)
@@ -202,4 +262,42 @@ def test_disable_enable(node_factory):
                    'active': True, 'dynamic': True}
     time.sleep(1)
     print(n.rpc.plugin_list()['plugins'])
-    assert(test_plugin in n.rpc.plugin_list()['plugins'])
+    assert test_plugin in n.rpc.plugin_list()['plugins']
+
+
+@unittest.skipIf(VALGRIND, "virtual environment triggers memleak detection")
+def test_tag_install(node_factory):
+    "install a plugin from a specific commit hash or tag"
+    node = get_reckless_node(node_factory)
+    node.start()
+    r = reckless([f"--network={NETWORK}", "-v", "install", "testPlugPass"],
+                 dir=node.lightning_dir)
+    assert r.returncode == 0
+    metadata = node.lightning_dir / "reckless/testplugpass/.metadata"
+    with open(metadata, "r") as md:
+        header = ''
+        for line in md.readlines():
+            line = line.strip()
+            if header == 'requested commit':
+                assert line == 'None'
+            header = line
+    # should install v2 (latest) without specifying
+    version = node.rpc.gettestplugversion()
+    assert version == 'v2'
+    r = reckless([f"--network={NETWORK}", "-v", "uninstall", "testplugpass"],
+                 dir=node.lightning_dir)
+    r = reckless([f"--network={NETWORK}", "-v", "install", "testplugpass@v1"],
+                 dir=node.lightning_dir)
+    assert r.returncode == 0
+    # v1 should now be checked out.
+    version = node.rpc.gettestplugversion()
+    assert version == 'v1'
+    installed_path = Path(node.lightning_dir) / 'reckless/testplugpass'
+    assert installed_path.is_dir()
+    with open(metadata, "r") as md:
+        header = ''
+        for line in md.readlines():
+            line = line.strip()
+            if header == 'requested commit':
+                assert line == 'v1'
+            header = line