Better biosphere install/update handling

.github/workflows/comment-milestoned-issues.yaml

@@ -0,0 +1,35 @@
+name: comment when milestone is closed
+on:
+  milestone:
+    types: [closed]
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Comment on issue
+        uses: actions/github-script@v5
+        with:
+          script: |
+            const milestone_number = context.payload.milestone.number;
+            const milestone_title = context.payload.milestone.title;
+
+            // Get all issues associated with the milestone
+            const issues = await github.rest.issues.listForRepo({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              milestone: milestone_number,
+              state: 'all',
+            });
+
+            // Loop through the issues and if the issue was closed, post a comment referring to the new release
+            for (const issue of issues.data) {
+              if (issue.state === 'closed') {
+                await github.rest.issues.createComment({
+                  owner: context.repo.owner,
+                  repo: context.repo.repo,
+                  issue_number: issue.number,
+                  body: `This issue has been implemented in the [new release of Activity Browser 🚀 (version ${milestone_title})](https://github.com/LCA-ActivityBrowser/activity-browser/releases/tag/${milestone_title}), you can get the new version by [updating Activity Browser](https://github.com/LCA-ActivityBrowser/activity-browser#updating-the-ab).\n\n_🤖beep boop! I'm a bot and this message was an automated action. If updating does not make sense for this issue, just ignore this._`,
+                });
+              }
+            }

.github/workflows/request-review-instructions.yaml

@@ -0,0 +1,32 @@
+name: notify reviewer
+on:
+  pull_request:
+    types: [review_requested]
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Comment on PR
+        uses: actions/github-script@v5
+        with:
+          script: |
+            const pr_number = context.payload.pull_request.number;
+            const reviewer = context.payload.review.user.login;
+
+            const fs = require('fs');
+            const path = require('path');
+
+            // Read CONTRIBUTING.md
+            const markdown = fs.readFileSync(path.join(process.env.GITHUB_WORKSPACE, '../tree/master/CONTRIBUTING.md'), 'utf8');
+
+            // Extract Reviewing guidelines
+            const section = markdown.split('### Reviewing pull-requests')[1].split('###')[0];
+
+            // Post a comment tagging the reviewer asking them to review following the guidelines
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: pr_number,
+              body: `> [!NOTE]\n> @${reviewer}, your review was requested for this Pull Request. Please review this PR following these guidelines:\n\n___\n${section}\n\n\n🤖 _beep boop! I'm a bot and this was an automated message._`,
+            });

activity_browser/bwutils/ecoinvent_biosphere_versions/compatible_ei_versions.txt

@@ -0,0 +1,3 @@
+{
+'2.9': ['3.4', '3.5', '3.6', '3.7', '3.7.1', '3.8', '3.9', '3.9.1'],
+}

activity_browser/bwutils/ecoinvent_biosphere_versions/ecospold2biosphereimporter.py

@@ -0,0 +1,75 @@
+from bw2io.importers import Ecospold2BiosphereImporter
+from bw2io.importers.ecospold2_biosphere import EMISSIONS_CATEGORIES
+from bw2data.utils import recursive_str_to_unicode
+from zipfile import ZipFile
+
+from ...info import __ei_versions__
+from ...utils import sort_semantic_versions
+
+import os
+from lxml import objectify
+
+import logging
+from activity_browser.logger import ABHandler
+
+logger = logging.getLogger('ab_logs')
+log = ABHandler.setup_with_logger(logger, __name__)
+
+def create_default_biosphere3(version) -> None:
+    """Reimplementation of bw.create_default_biosphere3 to allow import from older biosphere versions."""
+    # format version number to only Major/Minor
+    version = version[:3]
+
+    if version == sort_semantic_versions(__ei_versions__)[0]:
+        # most recent version
+        eb = Ecospold2BiosphereImporter()
+    else:
+        # not most recent version, import legacy biosphere from AB
+        eb = ABEcospold2BiosphereImporter(version=version)
+    eb.apply_strategies()
+    eb.write_database()
+
+
+class ABEcospold2BiosphereImporter(Ecospold2BiosphereImporter):
+    """Reimplementation of bw2io.importers Ecospold2BiosphereImporter to import legacy biosphere from AB data"""
+
+    def extract(self, version):
+        def extract_flow_data(o):
+            ds = {
+                "categories": (
+                    o.compartment.compartment.text,
+                    o.compartment.subcompartment.text,
+                ),
+                "code": o.get("id"),
+                "CAS number": o.get("casNumber"),
+                "name": o.name.text,
+                "database": self.db_name,
+                "exchanges": [],
+                "unit": o.unitName.text,
+            }
+            ds["type"] = EMISSIONS_CATEGORIES.get(
+                ds["categories"][0], ds["categories"][0]
+            )
+            return ds
+
+        lci_dirpath = os.path.join(os.path.dirname(__file__), 'legacy_biosphere')
+
+        # find the most recent legacy biosphere that is equal to or older than chosen version
+        for ei_version in sort_semantic_versions(__ei_versions__):
+            use_version = ei_version
+            fp = os.path.join(lci_dirpath, f'ecoinvent elementary flows {use_version}.xml.zip')
+            if sort_semantic_versions([version, ei_version])[0] == version and os.path.isfile(fp):
+                # this version is equal/lower and available
+                break
+
+        # extract the xml from the zip
+        with ZipFile(fp) as zipped_file:
+            with zipped_file.open(f'ecoinvent elementary flows {use_version}.xml') as file:
+                root = objectify.parse(file).getroot()
+
+        log.debug(f'Installing biosphere {use_version} for chosen version {version}')
+        flow_data = recursive_str_to_unicode(
+            [extract_flow_data(ds) for ds in root.iterchildren()]
+        )
+
+        return flow_data

activity_browser/controllers/database.py

@@ -9,13 +9,15 @@
 from ..bwutils.strategies import relink_exchanges_existing_db
 from ..ui.widgets import (
     CopyDatabaseDialog, DatabaseLinkingDialog, DefaultBiosphereDialog,
-    BiosphereUpdater, DatabaseLinkingResultsDialog
+    BiosphereUpdater, DatabaseLinkingResultsDialog, EcoinventVersionDialog
 )
 from ..ui.wizards.db_export_wizard import DatabaseExportWizard
 from ..ui.wizards.db_import_wizard import DatabaseImportWizard
 from ..settings import project_settings
 from ..signals import signals
 from .project import ProjectController
+from ..info import __ei_versions__
+from ..utils import sort_semantic_versions
 
 import logging
 from activity_browser.logger import ABHandler
@@ -66,23 +68,43 @@ def ensure_sqlite_indices(self):
 
     @Slot(name="bw2Setup")
     def install_default_data(self) -> None:
-        dialog = DefaultBiosphereDialog(self.window)
+
+        # let user choose version
+        version_dialog = EcoinventVersionDialog(self.window)
+        if version_dialog.exec_() != EcoinventVersionDialog.Accepted: return
+        version = version_dialog.options.currentText()
+
+        dialog = DefaultBiosphereDialog(version[:3], self.window)  # only read Major/Minor part of version
         dialog.show()
 
     @Slot(name="updateBiosphereDialog")
     def update_biosphere(self) -> None:
         """ Open a popup with progression bar and run through the different
         functions for adding ecoinvent biosphere flows.
         """
-        ok = QtWidgets.QMessageBox.question(
+        # warn user of consequences of updating
+        warn_dialog = QtWidgets.QMessageBox.question(
             self.window, "Update biosphere3?",
-            "Updating the biosphere3 database cannot be undone!",
+            'Newer versions of the biosphere database may not\n'
+            'always be compatible with older ecoinvent versions.\n'
+            '\nUpdating the biosphere3 database cannot be undone!\n',
             QtWidgets.QMessageBox.Ok | QtWidgets.QMessageBox.Abort,
             QtWidgets.QMessageBox.Abort
         )
-        if ok == QtWidgets.QMessageBox.Ok:
-            dialog = BiosphereUpdater(self.window)
-            dialog.show()
+        if warn_dialog is not QtWidgets.QMessageBox.Ok: return
+
+        # let user choose version
+        version_dialog = EcoinventVersionDialog(self.window)
+        if version_dialog.exec_() != EcoinventVersionDialog.Accepted: return
+        version = version_dialog.options.currentText()
+
+        # reduce biosphere update list up to the selected version
+        sorted_versions = sort_semantic_versions(__ei_versions__, highest_to_lowest=False)
+        ei_versions = sorted_versions[:sorted_versions.index(version) + 1]
+
+        # show updating dialog
+        dialog = BiosphereUpdater(ei_versions, self.window)
+        dialog.show()
 
     @Slot(name="addDatabase")
     def add_database(self):

activity_browser/info.py

@@ -1,6 +1,62 @@
+import ast
+import os.path
 from importlib.metadata import version, PackageNotFoundError
+from .utils import safe_link_fetch, sort_semantic_versions
 
+import logging
+from .logger import ABHandler
+logger = logging.getLogger('ab_logs')
+log = ABHandler.setup_with_logger(logger, __name__)
+
+# get AB version
 try:
     __version__ = version(__package__)
 except PackageNotFoundError:
     __version__ = "0.0.0"
+
+
+def get_compatible_versions() -> list:
+    """Get compatible versions of ecoinvent for this AB version.
+
+    Reads this file on github repo: activity-browser/better_biosphere_handling/compatible_ei_versions.txt'.
+    Converts file content to available ecoinvent versions for each version of AB.
+    Finds the correct available versions for this AB version, if failing to read version,
+    the lowest version in the file is chosen.
+    """
+    try:
+        # read versions
+        versions_URL = 'https://raw.githubusercontent.com/marc-vdm/activity-browser/better_biosphere_handling/activity_browser/bwutils/ecoinvent_biosphere_versions/compatible_ei_versions.txt'
+        error, page = safe_link_fetch(versions_URL)
+        if not error:
+            file = page.text
+        else:
+            # silently try a local fallback:
+            log.debug(f'Reading online compatible ecoinvent versions failed '
+                      f'-attempting local fallback- with this error: {error[:500]}')
+            file_path = os.path.join(os.path.dirname(__file__),
+                                     'bwutils',
+                                     'ecoinvent_biosphere_versions',
+                                     'compatible_ei_versions.txt')
+            with open(file_path, 'r') as f:
+                file = f.read()
+        all_versions = ast.literal_eval(file)
+
+        # select either the latest lower version available or if none available the lowest version for safety
+        sorted_versions = sort_semantic_versions(all_versions.keys())
+        for ab_version in sorted_versions:
+            if sort_semantic_versions([__version__, ab_version])[0] == __version__:
+                # current version is higher than or equal to tested AB version:
+                ei_versions = all_versions[ab_version]
+                break
+        else:
+            ei_versions = all_versions[sorted_versions[-1]]
+
+        log.debug(f'Following versions of ecoinvent are compatible with AB {__version__}: {ei_versions}')
+        return ei_versions
+
+    except Exception as error:
+        log.debug(f'Reading local fallback failed with: {error[:500]}')
+        return ['3.4', '3.5', '3.6', '3.7', '3.7.1', '3.8', '3.9', '3.9.1']
+
+
+__ei_versions__ = get_compatible_versions()

activity_browser/ui/tables/models/lca_setup.py

@@ -94,6 +94,7 @@ def activities(self) -> list:
         return [{x["key"]: x["Amount"]} for x in selection]
 
     def check_activities(self, db):
+        if db == 'biosphere3': return  # if biosphere changed, we don't need to check as it doesn't have activities.
         databases = [list(k.keys())[0][0] for k in self.activities]
         if db in databases:
             self.sync()

activity_browser/ui/widgets/__init__.py

@@ -9,7 +9,7 @@
     DatabaseLinkingDialog, DefaultBiosphereDialog,
     DatabaseLinkingResultsDialog, ActivityLinkingDialog,
     ActivityLinkingResultsDialog, ProjectDeletionDialog,
-    ScenarioDatabaseDialog, LocationLinkingDialog
+    ScenarioDatabaseDialog, LocationLinkingDialog, EcoinventVersionDialog
 )
 from .line_edit import (SignalledPlainTextEdit, SignalledComboEdit,
                         SignalledLineEdit)

activity_browser/ui/widgets/biosphere_update.py

@@ -14,14 +14,14 @@
 
 
 class BiosphereUpdater(QtWidgets.QProgressDialog):
-    def __init__(self, parent=None):
+    def __init__(self, ei_versions, parent=None):
         super().__init__(parent=parent)
         self.setWindowTitle("Updating '{}' database".format(bw.config.biosphere))
         self.setLabelText("Adding new flows to biosphere database")
         self.setRange(0, 0)
         self.show()
 
-        self.thread = UpdateBiosphereThread(self)
+        self.thread = UpdateBiosphereThread(ei_versions, self)
         self.setMaximum(self.thread.total_patches)
         self.thread.progress.connect(self.update_progress)
         self.thread.finished.connect(self.finished)
@@ -43,16 +43,21 @@ def update_progress(self, current: int):
 class UpdateBiosphereThread(QtCore.QThread):
     PATCHES = [patch for patch in dir(data) if patch.startswith('add_ecoinvent') and patch.endswith('biosphere_flows')]
     progress = Signal(int)
-    def __init__(self, parent=None):
+    def __init__(self, ei_versions, parent=None):
         super().__init__(parent)
+
+        # reduce the patches list to only compatible versions for this AB version
+        self.PATCHES = [p for p in self.PATCHES if any(v.replace('.', '') in p for v in ei_versions)]
+
         self.total_patches = len(self.PATCHES)
 
     def run(self):
         try:
             for i, patch in enumerate(self.PATCHES):
                 self.progress.emit(i)
+                log.debug(f'Applying biosphere patch: {patch}')
                 update_bio = getattr(data, patch)
                 update_bio()
         except ValidityError as e:
-            log.error("Could not patch biosphere: {}".format(str(e)))
+            log.error(f'Could not patch biosphere: {str(e)}')
             self.exit(1)