Include the assets to build a topsail Python package

This commit includes the resource required for
the creation of a Python package called topsail
with the same features and without introducing
any directory structure change change to the current
interfaces calling diretly the run_toolbox.py script.
This commit includes the scripts and jobs so with any
change to the package version there will be a new
released pypi package.

The list of what this commit intends:
- Create the pypi workflow for releasing topsail as a python package in pypi.
- Including the setuptools dependencies to build a python package.
- Organizing the files so the dependencies can be reached from within the
package (running run_toolbox.py <params> and topsail <params> should be equivalent).

.github/workflows/build_package.yml

@@ -0,0 +1,49 @@
+---
+name: Build package
+on:
+  push:
+  pull_request:
+  # Run the functional tests every 8 hours.
+  # This will help to identify faster if
+  # there is a CI failure related to a
+  # change in any dependency.
+  schedule:
+    - cron: '0 */8 * * *'
+jobs:
+  pypibuild:
+    if: github.repository_owner == 'openshift-psap'
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          sudo apt update -y
+          sudo apt install jq -y
+          sudo apt-get purge python3-yaml ansible -y
+          sudo locale-gen en_US.UTF-8
+          sudo dpkg-reconfigure locales
+          sudo apt install build-essential findutils -y
+          sudo apt-get --purge autoremove python3-pip
+          sudo apt install python3-pip -y
+          sudo python3 -m pip install --upgrade pip
+          sudo python3 -m pip install --upgrade virtualenv
+          sudo python3 -m pip install --upgrade setuptools
+          sudo python3 -m pip install --upgrade pbr
+          sudo python3 -m pip install -r requirements.txt
+      - name: Build the package
+        run: |
+          sudo python3 -m pip uninstall -y topsail
+          sudo rm -rf topsail.egg-info
+          sudo rm -rf dist
+          sudo python3 setup.py sdist
+          sudo python3 -m pip install --force dist/*
+          twine check dist/*
+          topsail get_info

.gitignore

@@ -1,2 +1,6 @@
 __pycache__
 .env*
+nohup.out
+*.pyc
+*topsail.egg-info*
+dist*

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "subprojects/matrix-benchmarking"]
-	path = subprojects/matrix-benchmarking
-	url = https://github.com/openshift-psap/matrix-benchmarking
 [submodule "subprojects/llm-load-test"]
 	path = subprojects/llm-load-test
 	url = https://github.com/openshift-psap/llm-load-test

MANIFEST.in

@@ -0,0 +1,7 @@
+include *
+recursive-include topsail *
+global-exclude *pyc
+global-exclude .git
+global-exclude .github
+global-exclude .project
+global-exclude .pydevproject

projects

@@ -0,0 +1 @@
+topsail/projects

requirements.txt

@@ -8,6 +8,7 @@ pylint
 awscli
 numpy
 joblib
+twine
 
 jsonpath_ng
 state-signals==0.5.2

setup.cfg

@@ -0,0 +1 @@
+[bdist_wheel]

setup.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+"""
+Copyright TOPSAIL contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may
+not use this file except in compliance with the License. You may obtain
+a copy of the License at:
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+License for the specific language governing permissions and limitations
+under the License.
+"""
+
+import os
+from sys import version_info
+import pathlib
+import itertools
+from setuptools import find_packages, setup
+from topsail.__info__ import version
+
+if version_info < (3, 9):
+    raise RuntimeError(
+        'Python 3.9 or greater is required'
+    )
+
+_NAME = 'topsail'
+_DESCRIPTION = 'topsail CLI'
+_REVISION = str(version)
+
+topsail_revision = os.environ.get('TOPSAIL_REVISION', "")
+if (topsail_revision != ""):
+    _REVISION = _REVISION + "." + topsail_revision
+
+if os.path.isfile('.README.md'):
+    with open('.README.md') as f:
+        long_description = f.read()
+else:
+    long_description = _DESCRIPTION
+
+with open('requirements.txt') as f:
+    requirements = f.read().splitlines()
+
+setup(
+    name=_NAME,
+    version=_REVISION,
+    description=_DESCRIPTION,
+    long_description_content_type='text/markdown',
+    long_description=long_description,
+    url='https://github.com/openshift-psap/topsail',
+    packages=find_packages(),
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'License :: OSI Approved :: Apache Software License',
+        'Natural Language :: English',
+        'Environment :: Console',
+        'Environment :: Web Environment',
+        'Intended Audience :: Developers',
+        'Intended Audience :: System Administrators',
+        'Topic :: Scientific/Engineering',
+        'Topic :: Software Development :: Testing',
+    ],
+    author='TOPSAIL',
+    include_package_data=True,
+    install_requires=requirements,
+    py_modules=['run_toolbox'],
+    entry_points={
+        'console_scripts': [
+            # TODO:FIXME: The CLI entrypoint
+            # should be part of the topsail package,
+            # for backwards compatibility this is
+            # not possible unless run_toolbox.py is
+            # inside the topsail package folder
+            '{0} = {0}.__main__:main'.format(_NAME),
+        ]
+    }
+)

topsail/.pypirc

@@ -0,0 +1,16 @@
+[distutils]
+index-servers =
+    pypi
+    testpypi
+    topsail
+
+[pypi]
+username = __token__
+
+[testpypi]
+username = __token__
+
+[topsail]
+repository = https://upload.pypi.org/legacy/
+username = __token__
+password = $KARG

topsail/__info__.py

@@ -0,0 +1 @@
+version= "0.0.2"

topsail/__init__.py

@@ -3,6 +3,7 @@
 import importlib
 import itertools
 import logging
+import pkg_resources
 
 class Toolbox:
     """
@@ -11,13 +12,30 @@ class Toolbox:
 
     def __init__(self):
 
-        top_dir = pathlib.Path(__file__).resolve().parent.parent
-
+        top_dir = pathlib.Path(__file__).resolve().parent
+        topsail_package_location = ''
+        try:
+            topsail_package_location = pkg_resources.resource_filename('topsail', '')
+            logging.info(f"Topsail package location: {topsail_package_location}")
+        except pkg_resources.ResolutionError:
+            logging.info("Topsail is not installed as a package")
+
+        import_prefix=''
+        if "dist-packages" in str(top_dir):
+            logging.info("Running from the package binary")
+            import_prefix='topsail.'
+        else:
+            logging.info("Running from the run_toolbox.py script")
 
         for toolbox_file in itertools.chain((top_dir / "projects").glob("*/toolbox/*.py"), (top_dir / "topsail").glob("*.py")):
 
-            project_toolbox_module = str(toolbox_file.relative_to(top_dir).with_suffix("")).replace(os.path.sep, ".")
-            mod = importlib.import_module(project_toolbox_module)
+            project_toolbox_module = import_prefix + str(toolbox_file.relative_to(top_dir).with_suffix("")).replace(os.path.sep, ".")
+            try:
+                mod = importlib.import_module(project_toolbox_module)
+            except ModuleNotFoundError as e:
+                logging.fatal(str(e))
+                sys.exit(1)
+
             toolbox_name = toolbox_file.with_suffix("").name
 
             if toolbox_name.startswith("_"): continue

topsail/__main__.py

@@ -0,0 +1,4 @@
+from run_toolbox import main
+
+if __name__ == "__main__":
+    main()

topsail/projects/core/toolbox/get_info.py

@@ -0,0 +1,9 @@
+from topsail.__info__ import version
+
+class Get_Info:
+    """
+    Command for retrieving information about the current topsail
+    """
+
+    def __new__(self):
+        print('topsail version: ' + str(version))