layer_group_filter.py

# -*- coding: utf-8 -*-
"""
/***************************************************************************
 LayerGroupFilter
                                 A QGIS plugin
 Manage your Layer Filters (data subset) using layer-groups
 Generated by Plugin Builder: http://g-sherman.github.io/Qgis-Plugin-Builder/
                              -------------------
        begin                : 2018-05-16
        git sha              : $Format:%H$
        copyright            : (C) 2018 by Peter Bontinck
        email                : bontinck.p@gmail.com
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/
"""
from qgis.core import *
from PyQt5.QtCore import QSettings, QTranslator, qVersion, QCoreApplication, Qt
from PyQt5.QtGui import QIcon
from PyQt5.QtWidgets import QAction, QMessageBox, QFileDialog

# Initialize Qt resources from file resources.py
from .resources import *
import xml.etree.ElementTree as ET

# Import the code for the DockWidget
from .layer_group_filter_dockwidget import LayerGroupFilterDockWidget
import os.path


class valueRelationException(Exception):
    pass


class LayerGroupFilter:
    """QGIS Plugin Implementation."""

    def __init__(self, iface):
        """Constructor.

        :param iface: An interface instance that will be passed to this class
            which provides the hook by which you can manipulate the QGIS
            application at run time.
        :type iface: QgsInterface
        """
        # Save reference to the QGIS interface
        self.iface = iface

        # initialize plugin directory
        self.plugin_dir = os.path.dirname(__file__)

        # initialize locale
        locale = QSettings().value('locale/userLocale')[0:2]
        locale_path = os.path.join(
            self.plugin_dir, 'i18n', 'LayerGroupFilter_{}.qm'.format(locale)
        )

        if os.path.exists(locale_path):
            self.translator = QTranslator()
            self.translator.load(locale_path)

            if qVersion() > '4.3.3':
                QCoreApplication.installTranslator(self.translator)

        # Declare instance attributes
        self.actions = []
        self.menu = self.tr(u'&Layer Group Filter')
        # TODO: We are going to let the user set this up in a future iteration
        self.toolbar = self.iface.addToolBar(u'LayerGroupFilter')
        self.toolbar.setObjectName(u'LayerGroupFilter')

        # print "** INITIALIZING LayerGroupFilter"

        self.pluginIsActive = False
        self.dockwidget = None

    # noinspection PyMethodMayBeStatic
    def tr(self, message):
        """Get the translation for a string using Qt translation API.

        We implement this ourselves since we do not inherit QObject.

        :param message: String for translation.
        :type message: str, QString

        :returns: Translated version of message.
        :rtype: QString
        """
        # noinspection PyTypeChecker,PyArgumentList,PyCallByClass
        return QCoreApplication.translate('LayerGroupFilter', message)

    def add_action(
        self,
        icon_path,
        text,
        callback,
        enabled_flag=True,
        add_to_menu=True,
        add_to_toolbar=True,
        status_tip=None,
        whats_this=None,
        parent=None,
    ):
        """Add a toolbar icon to the toolbar.

        :param icon_path: Path to the icon for this action. Can be a resource
            path (e.g. ':/plugins/foo/bar.png') or a normal file system path.
        :type icon_path: str

        :param text: Text that should be shown in menu items for this action.
        :type text: str

        :param callback: Function to be called when the action is triggered.
        :type callback: function

        :param enabled_flag: A flag indicating if the action should be enabled
            by default. Defaults to True.
        :type enabled_flag: bool

        :param add_to_menu: Flag indicating whether the action should also
            be added to the menu. Defaults to True.
        :type add_to_menu: bool

        :param add_to_toolbar: Flag indicating whether the action should also
            be added to the toolbar. Defaults to True.
        :type add_to_toolbar: bool

        :param status_tip: Optional text to show in a popup when mouse pointer
            hovers over the action.
        :type status_tip: str

        :param parent: Parent widget for the new action. Defaults None.
        :type parent: QWidget

        :param whats_this: Optional text to show in the status bar when the
            mouse pointer hovers over the action.

        :returns: The action that was created. Note that the action is also
            added to self.actions list.
        :rtype: QAction
        """

        icon = QIcon(icon_path)
        action = QAction(icon, text, parent)
        action.triggered.connect(callback)
        action.setEnabled(enabled_flag)

        if status_tip is not None:
            action.setStatusTip(status_tip)

        if whats_this is not None:
            action.setWhatsThis(whats_this)

        if add_to_toolbar:
            self.toolbar.addAction(action)

        if add_to_menu:
            self.iface.addPluginToMenu(self.menu, action)

        self.actions.append(action)

        return action

    def initGui(self):
        """Create the menu entries and toolbar icons inside the QGIS GUI."""

        icon_path = ':/plugins/layer_group_filter/icon.png'
        self.add_action(
            icon_path,
            text=self.tr(u'Layer group filters'),
            callback=self.run,
            parent=self.iface.mainWindow(),
        )

    # --------------------------------------------------------------------------

    def onClosePlugin(self):
        """Cleanup necessary items here when plugin dockwidget is closed"""

        # print "** CLOSING LayerGroupFilter"

        # disconnects
        self.dockwidget.closingPlugin.disconnect(self.onClosePlugin)

        # remove this statement if dockwidget is to remain
        # for reuse if plugin is reopened
        # Commented next statement since it causes QGIS crashe
        # when closing the docked window:
        # self.dockwidget = None

        self.pluginIsActive = False

    def unload(self):
        """Removes the plugin menu item and icon from QGIS GUI."""

        # print "** UNLOAD LayerGroupFilter"

        for action in self.actions:
            self.iface.removePluginMenu(self.tr(u'&Layer Group Filter'), action)
            self.iface.removeToolBarIcon(action)
        # remove the toolbar
        del self.toolbar

    # --------------------------------------------------------------------------

    def run(self):
        """Run method that loads and starts the plugin"""

        if not self.pluginIsActive:
            self.pluginIsActive = True

            # print "** STARTING LayerGroupFilter"

            # dockwidget may not exist if:
            #    first run of plugin
            #    removed on close (see self.onClosePlugin method)
            if self.dockwidget == None:
                # Create the dockwidget (after translation) and keep reference
                self.dockwidget = LayerGroupFilterDockWidget()
                self.dockwidget.replacePushButton.clicked.connect(self.replaceFilters)
                self.dockwidget.addPushButton.clicked.connect(self.addFilter)
                self.dockwidget.clearPushButton.clicked.connect(self.clearFilters)
                self.dockwidget.loadXMLButton.clicked.connect(self.loadXml)
                self.dockwidget.loadValuesButton.clicked.connect(
                    self.manualUpdateDistinctValues
                )

                self.dockwidget.presetComboBox.currentIndexChanged.connect(
                    self.presetChanged
                )
                self.dockwidget.layerGroupComboBox.currentIndexChanged.connect(
                    self.layerGroupChanged
                )
                self.dockwidget.filterValueComboBox.currentIndexChanged.connect(
                    self.updateGUI
                )

            # connect to provide cleanup on closing of dockwidget
            self.dockwidget.closingPlugin.connect(self.onClosePlugin)

            # show the dockwidget
            # TODO: fix to allow choice of dock location
            self.iface.addDockWidget(Qt.LeftDockWidgetArea, self.dockwidget)
            self.dockwidget.show()
            self.reloadProject()

    def reloadProject(self):
        """(re)intitiation of the Plugin."""

        # save the layerTreeRoot
        self.root = QgsProject.instance().layerTreeRoot()

        # create emty list for alle LayerGroupes and there Layers to be saved in.
        # And fill with current LayerGroupes and corresponding Layers
        self.allGroups = []
        self.discoverGroups()

        # initate layerGroupComboBox
        self.dockwidget.layerGroupComboBox.clear()
        self.dockwidget.layerGroupComboBox.addItems(i[0] for i in self.allGroups)

        # check if the current QGIS project has a "groupFilterPresets" XML as project variable
        if QgsExpressionContextUtils.projectScope(QgsProject.instance()).variable(
            "groupFilterPresets"
        ):

            # pharse the XML string  to XML object and further initialize the plugin.
            self.filterXmlRoot = ET.fromstring(
                QgsExpressionContextUtils.projectScope(QgsProject.instance()).variable(
                    "groupFilterPresets"
                )
            )
            self.loadFilterPresets()
        else:
            QMessageBox.warning(
                self.dockwidget,
                "No groupFilterPresets Set ",
                "No Filter Presets Set for this Project yet",
            )

    def discoverGroups(self):
        """save all layer Tree Groups of the project in a 2D list, self.allGroups :
        [[layerGroup name,[list of ALL layersID's in the group]],[...,[...]],...]
            remark: for sub-groups we add a prefix to the  saved layerGroup name
        """

        self.allGroups = []
        self.discoverBranch(self.root.children(), "")

    def discoverBranch(self, parrent, groupNamePrefix):
        """discover recursively a list of Nodes , a Group or a layer) to the bottoms and append all LayerTreeGroups to self.allGroups with all their layerID's"""

        if isinstance(parrent, QgsLayerTreeGroup):

            # append the found group
            self.allGroups.append(
                [
                    groupNamePrefix + parrent.name(),
                    [treeLayer.layerId() for treeLayer in parrent.findLayers()],
                ]
            )

            # go look for sub_groups
            # to the name of sub groups we add a prefix, to distict them in the GUI
            newGroupNamePrefix = groupNamePrefix + "__ "
            self.discoverBranch(parrent.children(), newGroupNamePrefix)

        elif isinstance(parrent, QgsLayerTreeLayer):

            # layers do not need further inverigation , there are al ready saved by ID with there parrents (and grand parrents)
            pass

        else:  # parrent is list of nodes as reslut of layerTreeNode.children()

            for child in parrent:
                self.discoverBranch(child, groupNamePrefix)

    def loadFilterPresets(self):
        """save 2dimentional List of all filterpresets and load the filterPresetNames in the GUI (presetComboBox)
        self.filterPresets = [[filterPresetName,filterPresetXMLobject],[...,...],...]
        """
        self.filterPresets = [
            [preset.get('name'), preset]
            for preset in self.filterXmlRoot.findall('preset')
        ]

        # Load in GUI
        self.dockwidget.presetComboBox.clear()
        self.dockwidget.presetComboBox.addItems(
            presetName[0] for presetName in self.filterPresets
        )

    def GetValueRelation(self, presetXml):
        """check if a Filter Preset XML object has a Value relation
        if:  no value Relation -> rerurn = None
        else: return Value Relation XML object """
        vr = presetXml.findall("valueRelation")
        if not len(vr) == 0:
            return vr[0]  # only retrun first ValueRelation, others will be ignored
        else:
            return None

    def GetValue(self, presetXml):
        """check if a Filter Preset XML object has a Value List
        if:  no value Relation -> return = None
        else: return Value Relation XML object """
        vr = presetXml.findall("value")
        if not len(vr) == 0:
            return vr[0]  # only retrun first Value, others will be ignored
        else:
            return None

    def changeSubsetStrings(
        self, presetIndex=None, groupIndex=None, values=None, clearAll=False
    ):
        """Append to  (using an XmlPreset) or clear the SubsetStrings for all vectorLayers of a Layer Group
            - xmlPresets:  is indexed from self.filterPresets (defaut by GUI)
            - layerGroup: is indexed from self.allGroups (defaut by GUI)
            - list of values, used te set the filter , multiple values will have OR relation in SubsetString (default from GUI)
            - clearAll : if True all SubsetStrings of all Layers in the layerGroup are cleared

            Value relations are also and taken in account.
            - If inverted valueRelation retuns muliple values ( unique Key, with non unique values):
                all key's will be found with OR relation in the subsetString
            - if valueRelation: values argument needs to be a list of len() =1
            """
        if not presetIndex:  # set default filterIndex from GUI
            presetIndex = self.dockwidget.presetComboBox.currentIndex()

        if not groupIndex:  # set default groupIndex from GUI
            groupIndex = self.dockwidget.layerGroupComboBox.currentIndex()

        if not values:  # set default value from GUI as list with single value
            values = []
            fieldtext = self.dockwidget.filterValueComboBox.currentText()
            if fieldtext is not '':
                values.append(fieldtext)

        # look up presetXml (XML object)
        presetXml = self.filterPresets[presetIndex][1]

        # save filter operator
        filterOper = presetXml.find('operator').text

        # save filterColums as list of attibure name strings
        filterColumns = [col.text for col in presetXml.findall('column')]

        # check if preset has ValueRelation, if so replace the value with a list of corresponing key's
        vr = self.GetValueRelation(presetXml)
        if vr is not None:
            values = self.getValueRelation(values[0], vr, invert=True, getAll=True)

        vl = self.GetValue(presetXml)
        if vl is not None:
            values.append(vl.text)

        # loop true ALL layers of current group
        for layer in self.allGroups[groupIndex][1]:
            maplayer = QgsProject.instance().mapLayer(layer)
            fields = maplayer.fields()
            fieldNames = [field.name() for field in fields]
            subSetstring = maplayer.subsetString()

            if clearAll:
                subSetstring = ""
            else:
                if filterOper == 'in':
                    opensep = "("
                    closesep = ")"
                else:
                    opensep = closesep = "'"

                for filterColumn in filterColumns:
                    if filterColumn in fieldNames:
                        firstValue = True
                        for value in values:
                            # if muliple values create 'OR' relations
                            if firstValue:
                                addSubSetstring = "\"{0}\" {1} {2}{3}{4}".format(
                                    filterColumn, filterOper, opensep, value, closesep
                                )
                                firstValue = False
                            else:
                                addSubSetstring = (
                                    addSubSetstring
                                    + " or "
                                    + "\"{0}\" {1} {2}{3}{4}".format(
                                        filterColumn,
                                        filterOper,
                                        opensep,
                                        value,
                                        closesep,
                                    )
                                )

                            # append the new SubSetstring with 'AND'relation to the current SubSetstring
                        if subSetstring == "":
                            subSetstring = addSubSetstring
                        else:
                            subSetstring = subSetstring + " and " + addSubSetstring

            # replace the subsetstring of the Layer
            maplayer.setSubsetString(subSetstring)

    def getValueRelation(self, value, valueRelationXml, invert=False, getAll=False):
        """
        returns the value(s) found in the value column , corresponding to a Key column of a other layer/table
            - value : value to look for in the key column
            - valueRelationXml : Layer, Key column, value column are given as a XMLobject
            - invert : if True, the inveted value relation is returned
            - getAll: if False, only first value is returned, if false a LIST of all values is returned
        """

        # initialize warnings
        warning = False
        warningText = ""

        # parse valueRelationXml in dictionary
        vr = {"Layer": "", "keyColumn": "", "valueColumn": ""}
        vr["Layer"] = valueRelationXml.find("Layer").text
        vr["keyColumn"] = valueRelationXml.find("keyColumn").text
        vr["valueColumn"] = valueRelationXml.find("valueColumn").text

        # inverting valueRelation if necessary
        if invert:
            temp = vr["keyColumn"]
            vr["keyColumn"] = vr["valueColumn"]
            vr["valueColumn"] = temp

        # save the lookup layer in list with len() = 1
        layers = QgsProject.instance().mapLayersByName(vr["Layer"])

        # create and save the expression to look up the key value
        exp = '"{0}" = \'{1}\''.format(vr["keyColumn"], value)
        exp = exp.replace(
            "\\", "\\\\"
        )  # work around in case a \ appears in value string

        try:
            if len(layers) > 1:  # set warning: layername not unique
                warning = True
                warningText = (
                    warningText
                    + "layer : {0} is not unique, only first instance is used \n".format(
                        vr["Layer"]
                    )
                )

            if len(layers) < 1:  # set warning: layername not found, result will be None
                warning = True
                warningText = warningText + "layer : {0} is not found \n".format(
                    vr["Layer"]
                )

                raise valueRelationException

            # look up the corresponding values
            subset = layers[0].getFeatures(QgsFeatureRequest().setFilterExpression(exp))

            resultCounter = 0

            if getAll:
                result = []

            for feature in subset:
                value = feature.attribute(vr["valueColumn"])

                if getAll:
                    result.append(value)
                    resultCounter = resultCounter + 1
                else:

                    if resultCounter > 1:  # set warning: to many results
                        warning = True
                        warningText = (
                            warningText
                            + "More than one, matiching value found for : {0} , only first is used \n".format(
                                exp
                            )
                        )

                        return  # GOTO finally , loving this spaghetti :)

                    result = value
                    resultCounter = resultCounter + 1

            if resultCounter == 0:
                warning = True
                warningText = (
                    warningText + "No matiching value found for : {0} \n".format(exp)
                )

                raise valueRelationException

            result = subset[0].attribute(vr["valueColumn"])

        except valueRelationException:
            result = None

        finally:
            if warning:
                print(warningText)
            return result

    def updateDistinctValues(self, findAll=False, manual=False):
        """
        updates self.distinctValues and updates GUI
        self.distinctValues: is a list of all distinct values found in the FilterColumns of the current filter preset.
        if the filter preset hasValueRelation then the relationValues are listed.
        in the current preset we look for a element "autoValueList"
            if this is set to True: the project is iterated and self.distinctValues is updated,
            of not: the self.distinctValues is cleared
        FindAll: if false, we look only in the layers of the current layer Group.
        manual: if true the project is iterated for distinct values, regardless a "autoValueList" XML element.
        """
        self.distinctValues = []
        presetXML = self.filterPresets[self.dockwidget.presetComboBox.currentIndex()][1]

        # set the var autoUpdate as list of XML elements 'autoValueList'
        autoUpdate = presetXML.findall('autoValueList')

        # if we found a element 'autoValueList':
        #   change the var autoUpdate to a string representing the UpperCase text of the XMLElement.
        if len(autoUpdate) > 0:
            autoUpdate = str(autoUpdate[0].text).upper()
        # print ('au:'+ str(autoUpdate) + 'm :' + str(manual))

        # clear the filterValueComboBox and exit the method if necessary
        if not (
            autoUpdate == "TRUE" or manual
        ):  # autoUpdate: can be list of XML or String
            self.dockwidget.filterValueComboBox.clear()
            return

        # print("updating your values")
        FilterColumns = [c.text for c in presetXML.findall('column')]

        # defining the layer groups to invesigate: groups = [[layerGroup name,[list of ALL layersID's in the group]],[...,[...]],...] see discoverGroups(self)
        if findAll:
            groups = self.allGroups
        else:
            groups = []
            groups.append(
                self.allGroups[self.dockwidget.layerGroupComboBox.currentIndex()]
            )

        for group in groups:
            for layer in group[1]:
                maplayer = QgsProject.instance().mapLayer(layer)

                if isinstance(
                    maplayer, QgsVectorLayer
                ):  # only vectorLayers have fields
                    for FilterColumn in FilterColumns:
                        if FilterColumn in [
                            field.name() for field in maplayer.fields()
                        ]:
                            for feature in maplayer.getFeatures():
                                value = str(feature.attribute(FilterColumn))
                                if not value in self.distinctValues:
                                    self.distinctValues.append(value)

        vr = self.GetValueRelation(presetXML)
        if (
            vr
        ):  # if has ValueRelation, we replace the found key's values with the values
            newUniqueValues = []
            for value in self.distinctValues:
                newUniqueValues.append(self.getValueRelation(value, vr))
            self.distinctValues = newUniqueValues

        # update GUI
        self.dockwidget.filterValueComboBox.clear()
        self.dockwidget.filterValueComboBox.addItems(self.distinctValues)
        self.dockwidget.filterValueComboBox.setCurrentIndex(-1)

    def checkInputs(self, hasValueRelation=False):
        """check the inputs before a filter can be set"""
        if self.dockwidget.presetComboBox.currentIndex() == -1:
            return False
        if self.dockwidget.layerGroupComboBox.currentIndex() == -1:
            return False
        if hasValueRelation:
            if self.dockwidget.filterValueComboBox.currentIndex() == -1:
                return False
        return True

    def updateGUI(self):
        if not self.checkInputs():
            self.dockwidget.replacePushButton.setEnabled(False)
            self.dockwidget.addPushButton.setEnabled(False)
            self.dockwidget.clearPushButton.setEnabled(False)
            return
        presetXml = self.filterPresets[self.dockwidget.presetComboBox.currentIndex()][1]
        vr = presetXml.findall("valueRelation")
        if not len(vr) == 0:
            self.dockwidget.filterValueComboBox.setEditable(False)
            self.dockwidget.clearPushButton.setEnabled(True)
            if self.checkInputs(hasValueRelation=True):
                self.dockwidget.replacePushButton.setEnabled(True)
                self.dockwidget.addPushButton.setEnabled(True)
            else:
                self.dockwidget.replacePushButton.setEnabled(False)
                self.dockwidget.addPushButton.setEnabled(False)
            return

        self.dockwidget.filterValueComboBox.setEditable(True)
        self.dockwidget.replacePushButton.setEnabled(True)
        self.dockwidget.addPushButton.setEnabled(True)
        self.dockwidget.clearPushButton.setEnabled(True)

    def presetChanged(self):
        if not self.dockwidget.layerGroupComboBox.currentIndex() == -1:
            self.updateDistinctValues()

        self.updateGUI()

    def layerGroupChanged(self):
        if not self.dockwidget.presetComboBox.currentIndex() == -1:
            self.updateDistinctValues()
        self.updateGUI()

    def manualUpdateDistinctValues(self):
        self.updateDistinctValues(manual=True)

    def replaceFilters(self):
        self.changeSubsetStrings(clearAll=True)
        self.changeSubsetStrings()

    def addFilter(self):
        self.changeSubsetStrings()

    def clearFilters(self):
        self.changeSubsetStrings(clearAll=True)

    def loadXml(self):
        fileName = QFileDialog.getOpenFileName(
            self.dockwidget, "Select XML filter Preset XML ", "", '*.xml'
        )[0]
        xmlFile = open(fileName, 'r')
        presetXml = xmlFile.read().strip()
        xmlFile.close
        QgsExpressionContextUtils.setProjectVariable(
            QgsProject.instance(), 'groupFilterPresets', presetXml
        )
        self.reloadProject()