Merge remote-tracking branch 'upstream/master' into mbencer/FixAutoPa…

…dding

.github/org_control/check_org.py

@@ -0,0 +1,51 @@
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Check GitHub organization and invite members
+"""
+
+# pylint: disable=fixme,no-member
+
+from argparse import ArgumentParser
+
+import github_api
+from configs import Config
+
+
+def main():
+    """The main entry point function"""
+    arg_parser = ArgumentParser()
+    arg_parser.add_argument("--cfg-file", metavar="PATH", default=Config.default_cfg_path,
+                            help=f"Path to json configuration file, e.g. {Config.default_cfg_path}")
+    arg_parser.add_argument("--teams", action="store_true", help="Check GitHub teams")
+    args, unknown_args = arg_parser.parse_known_args()
+
+    Config(args.cfg_file, unknown_args)
+    gh_api = github_api.GithubOrgApi()
+
+    if args.teams:
+        gh_api.get_org_teams()
+    else:
+        dev_emails = github_api.get_dev_emails()
+        print(f'\nDeveloper emails {len(dev_emails)}:', '; '.join(dev_emails))
+
+        org_emails = gh_api.get_org_emails()
+        print(f'\nOrg emails {len(org_emails)}:', '; '.join(org_emails))
+
+        org_pendig_invitation_emails = gh_api.get_org_invitation_emails()
+
+        invite_emails = dev_emails.difference(org_emails).difference(org_pendig_invitation_emails)
+        print(f'\nInvite emails {len(invite_emails)}:', '; '.join(invite_emails))
+
+        no_in_dev_emails = org_emails.difference(dev_emails)
+        print(f'\nOrg members - no in developers list {len(no_in_dev_emails)}:',
+              '; '.join(no_in_dev_emails))
+
+        valid_github_users = gh_api.get_valid_github_users(invite_emails)
+
+        gh_api.invite_users(valid_github_users)
+
+
+if __name__ == '__main__':
+    main()

.github/org_control/check_pr.py

@@ -0,0 +1,138 @@
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Check GitHub PRs and set labels by type and categories, e.g. 'ExternalPR', 'category: ci'
+"""
+
+# pylint: disable=fixme,no-member
+
+import re
+from argparse import ArgumentParser
+from enum import Enum
+
+import github_api
+from configs import Config
+
+
+class PrType(Enum):
+    """Constants for type of GitHub pull request by author membership"""
+    EXTERNAL = 'ExternalPR'
+    INTEL = 'IntelDevPR'
+    ORG = 'OpenvinoDevPR'
+    BAD = 'BadPR'
+
+
+def get_pr_labels(pull):
+    """Gets PR labels as set"""
+    pr_lables = set()
+    for label in pull.labels:
+        pr_lables.add(label.name)
+    return pr_lables
+
+
+def set_pr_label(pull, labels):
+    """Sets PR labels"""
+    if not labels or Config().DRY_RUN:
+        return
+    print(f'Set PR labels:', labels)
+    # TODO: Review labels and enable. Check setting existing labels
+    #pull.set_labels(labels)
+
+
+def get_pr_type(pull):
+    """Gets PR type using labels"""
+    pr_lables = get_pr_labels(pull)
+    pr_types = set(type.value for type in PrType)
+    pr_types_labels = pr_lables & pr_types
+    if not pr_types_labels:
+        return None
+    if len(pr_types_labels) > 1:
+        print(f'Duplicated labels: {pr_types_labels}')
+        return PrType.BAD
+    return PrType(PrType(pr_types_labels.pop()))
+
+
+def get_label_by_team_name(team_name):
+    """Generates label by PR reviwer team name"""
+    if 'admins' in team_name:
+        return 'category: ci'
+    re_compile_label = re.compile(rf'{Config().GITHUB_REPO}-(.+)-maintainers')
+    re_label = re_compile_label.match(team_name)
+    if re_label:
+        return f'category: {re_label.group(1).strip()}'
+    return None
+
+
+def get_category_labels(pull):
+    """Gets list of category labels by all PR reviwer teams"""
+    labels = []
+    pr_lables = get_pr_labels(pull)
+    for reviewer_team in pull.get_review_requests()[1]:
+        reviewer_label = get_label_by_team_name(reviewer_team.name)
+        if reviewer_label and reviewer_label not in pr_lables:
+            labels.append(reviewer_label)
+    return labels
+
+
+def main():
+    """The main entry point function"""
+    arg_parser = ArgumentParser()
+    arg_parser.add_argument("--cfg-file", metavar="PATH", default=Config.default_cfg_path,
+                            help=f"Path to json configuration file, e.g. {Config.default_cfg_path}")
+    arg_parser.add_argument("--pr", metavar="NUMBER",
+                            help="Get GitHub pull request with the number")
+    arg_parser.add_argument("--pr-state", default="open", choices=["open", "closed"],
+                            help="Set GitHub pull request state")
+    args, unknown_args = arg_parser.parse_known_args()
+
+    Config(args.cfg_file, unknown_args)
+    gh_api = github_api.GithubOrgApi()
+
+    if args.pr:
+        pulls = [gh_api.repo.get_pull(int(args.pr))]
+    else:
+        pulls = gh_api.repo.get_pulls(state=args.pr_state)
+        print(f'PRs count ({args.pr_state}):', pulls.totalCount)
+    non_org_intel_pr_users = set()
+    non_org_pr_users = set()
+    for pull in pulls:
+        pr_lables = get_pr_labels(pull)
+        pr_type = get_pr_type(pull)
+        set_labels = []
+        print('\n', pull, f'- Labels: {pr_lables} -', f'Type: {pr_type}', end='')
+
+        # Checks PR source type
+        if gh_api.is_org_user(pull.user):
+            print(' - Org user')
+            if pr_type is not PrType.ORG:
+                print(f'NO "{PrType.ORG.value}" label: ', end='')
+                github_api.print_users(pull.user)
+                set_labels.append(PrType.ORG.value)
+        elif github_api.is_intel_email(pull.user.email) or \
+             github_api.is_intel_company(pull.user.company):
+            print(' - Non org user with Intel email or company')
+            non_org_intel_pr_users.add(pull.user)
+            if pr_type is not PrType.INTEL:
+                print(f'NO "{PrType.INTEL.value}" label: ', end='')
+                github_api.print_users(pull.user)
+                set_labels.append(PrType.INTEL.value)
+        else:
+            print(f' - Non org user with NO Intel email or company')
+            non_org_pr_users.add(pull.user)
+            if pr_type is not PrType.EXTERNAL:
+                print(f'NO "{PrType.EXTERNAL.value}" label: ', end='')
+                github_api.print_users(pull.user)
+                set_labels.append(PrType.EXTERNAL.value)
+
+        set_labels += get_category_labels(pull)
+        set_pr_label(pull, set_labels)
+
+    print(f'\nNon org user with Intel email or company:')
+    github_api.print_users(non_org_intel_pr_users)
+    print(f'\nNon org user with NO Intel email or company:')
+    github_api.print_users(non_org_pr_users)
+
+
+if __name__ == '__main__':
+    main()

.github/org_control/config.json

@@ -0,0 +1,18 @@
+{
+    "GITHUB_TOKEN": "<Put token here or set as arg or as env variable>",
+    "GITHUB_ORGANIZATION": "openvinotoolkit",
+    "GITHUB_REPO": "openvino",
+    "IGNORE_LOGINS": [
+        "openvino-ci",
+        "openvino-pushbot",
+        "lab-nerval",
+        "lab-nerval-onnx-ci"
+    ],
+    "EMAILS_FILE_PATH": "dev_emails-test.txt",
+    "PROXIES": {
+        "HTTP_PROXY": null,
+        "HTTPS_PROXY": null,
+        "NO_PROXY": "localhost,127.0.0.1,.intel.com"
+    },
+    "DRY_RUN": false
+}

.github/org_control/configs.py

@@ -0,0 +1,113 @@
+# Copyright (C) 2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Configurations management
+"""
+
+# pylint: disable=fixme,broad-except
+
+import os
+import sys
+import ast
+import json
+from pathlib import Path
+
+
+if sys.hexversion < 0x3060000:
+    raise Exception('Python version must be >= 3.6')
+
+
+class ConfigException(Exception):
+    """Base configuration exception"""
+
+
+class Config:
+    """Configuration wrapper"""
+    _instance = None
+    properties = None
+    default_cfg_path = Path(__file__).resolve().parent / 'config.json'
+
+    def __new__(cls, *_args, **_kwargs):
+        if not Config._instance:
+            Config._instance = super(Config, cls).__new__(cls)
+        return Config._instance
+
+    def __init__(self, file_path=None, cli_args=None):
+        """
+        :param file_path: Path to json configuration file
+        :type file_path: String
+
+        :param args: List of argparse arguments with patterns: 'name=value' or 'name'
+        :type args: list
+        """
+        if Config.properties:
+            return
+
+        self._file_path = file_path or Config.default_cfg_path
+        self._cli_args = cli_args or []
+
+        self._json_cfg = {}
+        self._args = {}
+
+        self._load_cfg()
+        self._parse_cli_args()
+
+        Config.properties = {}
+        for name, value in self._json_cfg.items():
+            if hasattr(self, name):
+                raise ConfigException(f'Duplicating prosperity: {name}')
+            prosperity_value = self._args.get(name) or os.getenv(name)
+            if prosperity_value:
+                # Try to set prosperity_value as Python literal structures, e.g. DRY_RUN=False
+                try:
+                    prosperity_value = ast.literal_eval(prosperity_value)
+                except Exception:
+                    pass
+                if not isinstance(prosperity_value, type(value)):
+                    raise ConfigException(f'Python type of {name} parameter must be {type(value)}')
+            else:
+                prosperity_value = value
+            setattr(self, name, prosperity_value)
+            Config.properties[name] = prosperity_value
+
+        self.set_proxy()
+
+    def _load_cfg(self):
+        """Load the json configuration file"""
+        try:
+            with open(self._file_path) as conf:
+                self._json_cfg = json.load(conf)
+        except:
+            print('Failed to load configuration from:', self._file_path)
+            raise
+
+    def _parse_cli_args(self):
+        """Parse argparse arguments with patterns: 'name=value' or 'name'"""
+        for cli_arg in self._cli_args:
+            arg = cli_arg.split('=')
+            if arg[0] not in self._json_cfg:
+                raise ConfigException(f'Unsupported argument: {arg}')
+            self._args[arg[0]] = True if len(arg) == 1 else '='.join(arg[1:])
+
+    def get_properties(self):
+        """Get all properties as Dict"""
+        return self.properties
+
+    def set_proxy(self):
+        """Set proxies"""
+        for proxy_name, url in self.properties['PROXIES'].items():
+            if url is not None:
+                print(f'Set proxy: {proxy_name}={url}')
+                os.environ[proxy_name] = url
+
+
+def _test():
+    """Test and debug"""
+    print('Config.default_cfg_path:', Config.default_cfg_path)
+    cfg = Config(cli_args=['DRY_RUN=True'])
+    print('Config.properties:', cfg.get_properties())
+
+
+if __name__ == '__main__':
+    _test()

.github/org_control/dev_emails-test.txt

@@ -0,0 +1,9 @@
+# good comment
+Last_name, First_name <[email protected]>
+[email protected]
+[email protected]
+
+# Wrong emails
+[email protected]
+  foo1  foo2
+foo1 [email protected]