CI: Update org control scripts

.github/org_control/__init__.py

@@ -1,3 +0,0 @@
-# Copyright (C) 2018-2021 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-

.github/org_control/check_org.py

@@ -5,12 +5,13 @@
 Check GitHub organization and invite members
 """
 
-# pylint: disable=fixme,no-member
+# pylint: disable=fixme,no-member,too-many-locals
 
 from argparse import ArgumentParser
 
-import github_api
 from configs import Config
+from github_api import GithubOrgApi, get_dev_emails
+from ldap_api import LdapApi, print_user_info, InfoLevel
 
 
 def main():
@@ -19,32 +20,74 @@ def main():
     arg_parser.add_argument("--cfg-file", metavar="PATH", default=Config.default_cfg_path,
                             help=f"Path to json configuration file, e.g. {Config.default_cfg_path}")
     arg_parser.add_argument("--teams", action="store_true", help="Check GitHub teams")
+    arg_parser.add_argument("--no-ldap", action="store_true", help="Don't use LDAP info")
     args, unknown_args = arg_parser.parse_known_args()
 
     Config(args.cfg_file, unknown_args)
-    gh_api = github_api.GithubOrgApi()
+    gh_api = GithubOrgApi()
 
     if args.teams:
         gh_api.get_org_teams()
-    else:
-        dev_emails = github_api.get_dev_emails()
-        print(f'\nDeveloper emails {len(dev_emails)}:', '; '.join(dev_emails))
+        return
 
-        org_emails = gh_api.get_org_emails()
-        print(f'\nOrg emails {len(org_emails)}:', '; '.join(org_emails))
+    cfg_emails = get_dev_emails()
+    print(f'\nCfg developer emails {len(cfg_emails)}:', '; '.join(sorted(cfg_emails)))
 
-        org_pendig_invitation_emails = gh_api.get_org_invitation_emails()
+    dev_emails = set()
+    dev_emails.update(cfg_emails)
 
-        invite_emails = dev_emails.difference(org_emails).difference(org_pendig_invitation_emails)
-        print(f'\nInvite emails {len(invite_emails)}:', '; '.join(invite_emails))
+    if not args.no_ldap:
+        ldap_api = LdapApi()
+        ldap_emails = ldap_api.get_user_emails()
+        dev_emails.update(ldap_emails)
+        print(f'\nLDAP developer emails {len(ldap_emails)}:', '; '.join(sorted(ldap_emails)))
 
-        no_in_dev_emails = org_emails.difference(dev_emails)
-        print(f'\nOrg members - no in developers list {len(no_in_dev_emails)}:',
-              '; '.join(no_in_dev_emails))
+        cfg_emails_no_in_ldap = ldap_api.get_absent_emails(cfg_emails)
+        print(f'\nCfg developer emails - absent in LDAP at all {len(cfg_emails_no_in_ldap)}:',
+              '; '.join(sorted(cfg_emails_no_in_ldap)))
 
-        valid_github_users = gh_api.get_valid_github_users(invite_emails)
+        cfg_ldap_inters = cfg_emails.intersection(ldap_emails)
+        print(f'\nCfg developer emails - present in LDAP developers {len(cfg_ldap_inters)}:',
+              '; '.join(sorted(cfg_ldap_inters)))
 
-        gh_api.invite_users(valid_github_users)
+    org_emails, org_logins_no_intel_email = gh_api.get_org_emails()
+    print(f'\nOrg emails {len(org_emails)}:', '; '.join(sorted(org_emails)))
+
+    org_emails_no_in_ldap = set()
+    if not args.no_ldap:
+        org_ldap_diff = org_emails.difference(ldap_emails)
+        print(f'\nOrg member emails - absent in LDAP developers {len(org_ldap_diff)}:',
+              '; '.join(sorted(org_ldap_diff)))
+
+        for email in org_ldap_diff:
+            user_info = ldap_api.get_user_info_by_email(email)
+            if user_info:
+                print_user_info(user_info, InfoLevel.PDL)
+            else:
+                org_emails_no_in_ldap.add(email)
+
+    org_pendig_invitation_emails = gh_api.get_org_invitation_emails()
+    invite_emails = dev_emails.difference(org_emails).difference(org_pendig_invitation_emails)
+    print(f'\nInvite emails {len(invite_emails)}:', '; '.join(sorted(invite_emails)))
+
+    valid_github_users = gh_api.get_valid_github_users(invite_emails)
+    gh_api.invite_users(valid_github_users)
+
+    print('\nCheck accounts below and remove from the GitHub organization and cfg list')
+
+    cfg_emails_no_in_org = sorted(cfg_emails.difference(org_emails))
+    print(f'\nCfg developer emails - absent in GitHub organization {len(cfg_emails_no_in_org)}:',
+          '; '.join(cfg_emails_no_in_org))
+
+    org_emails_no_in_dev = sorted(org_emails.difference(dev_emails))
+    print(f'\nOrg member emails - absent in cfg and LDAP developers {len(org_emails_no_in_dev)}:',
+          '; '.join(org_emails_no_in_dev))
+
+    print(f'\nOrg member emails - absent in LDAP at all {len(org_emails_no_in_ldap)}:',
+          '; '.join(sorted(org_emails_no_in_ldap)))
+
+    print(f'\nOrg member logins - absent Intel email {len(org_logins_no_intel_email)}:',
+          '; '.join(sorted(org_logins_no_intel_email)))
 
 
 if __name__ == '__main__':

.github/org_control/check_pr.py

@@ -33,13 +33,23 @@ def get_pr_labels(pull):
 
 
 def set_pr_labels(pull, labels):
-    """Sets PR labels"""
+    """Sets new PR labels (all previously set labels are removed)"""
     if not labels or Config().DRY_RUN:
         return
-    print(f'Set PR labels:', labels)
+    print('Set PR labels:', labels)
+    # set_labels() should accept list but fails with empty "AssertionError:"
     pull.set_labels(labels)
 
 
+def add_pr_labels(pull, labels):
+    """Adds PR labels"""
+    if not labels or Config().DRY_RUN:
+        return
+    print('Add PR labels:', labels)
+    for label in labels:
+        pull.add_to_labels(label)
+
+
 def get_pr_type_by_labels(pull):
     """Gets PR type using labels"""
     pr_lables = get_pr_labels(pull)
@@ -80,6 +90,17 @@ def get_category_labels(pull):
     return labels
 
 
+def get_pr_info_str(pull):
+    """Gets info about PR using a few workarounds"""
+    pr_title = pull.title.encode("ASCII", "ignore").decode()
+
+    # Workaround for PyGithub issue: https://github.com/PyGithub/PyGithub/issues/512
+    pr_created_at = pull.created_at.replace(tzinfo=datetime.timezone.utc).astimezone()
+
+    return f'PR: {pull.number} - {pr_title} - Created: {pr_created_at} - ' \
+           f'Labels: {get_pr_labels(pull)} - Type: {get_pr_type_by_labels(pull)}'
+
+
 def main():
     """The main entry point function"""
     arg_parser = ArgumentParser()
@@ -103,19 +124,19 @@ def main():
         print(f'\nPRs count ({args.pr_state}):', pulls.totalCount)
 
     if args.newer:
-        pr_created_after = datetime.datetime.now() - datetime.timedelta(minutes=int(args.newer))
-        print('PRs created after:', pr_created_after)
+        pr_created_after = (datetime.datetime.now() -
+                            datetime.timedelta(minutes=int(args.newer))).astimezone()
+        print('Checking PRs created after:', pr_created_after)
     non_org_intel_pr_users = set()
     non_org_pr_users = set()
     for pull in pulls:
-        if args.newer and pull.created_at <= pr_created_after:
-            print(f'\nIGNORE: {pull} - Created: {pull.created_at}')
+        pr_created_at = pull.created_at.replace(tzinfo=datetime.timezone.utc).astimezone()
+        if args.newer and pr_created_at <= pr_created_after:
+            print(f'\nIGNORE: {get_pr_info_str(pull)}')
             continue
-        pr_lables = get_pr_labels(pull)
         pr_type_by_labels = get_pr_type_by_labels(pull)
-        set_labels = []
-        print(f'\n{pull} - Created: {pull.created_at} - Labels: {pr_lables} -',
-              f'Type: {pr_type_by_labels}', end='')
+        add_labels = []
+        print(f'\n{get_pr_info_str(pull)}', end='')
 
         # Checks PR source type
         if gh_api.is_org_user(pull.user):
@@ -127,21 +148,23 @@ def main():
             if pr_type_by_labels is not PrType.INTEL:
                 print(f'NO "{PrType.INTEL.value}" label: ', end='')
                 github_api.print_users(pull.user)
-                set_labels.append(PrType.INTEL.value)
+                add_labels.append(PrType.INTEL.value)
+        elif github_api.is_user_ignored(pull.user):
+            print(' - IGNORED non org user with NO Intel email or company')
         else:
-            print(f' - Non org user with NO Intel email or company')
+            print(' - Non org user with NO Intel email or company')
             non_org_pr_users.add(pull.user)
             if pr_type_by_labels is not PrType.EXTERNAL:
                 print(f'NO "{PrType.EXTERNAL.value}" label: ', end='')
                 github_api.print_users(pull.user)
-                set_labels.append(PrType.EXTERNAL.value)
+                add_labels.append(PrType.EXTERNAL.value)
 
-        set_labels += get_category_labels(pull)
-        set_pr_labels(pull, set_labels)
+        add_labels += get_category_labels(pull)
+        add_pr_labels(pull, add_labels)
 
-    print(f'\nNon org user with Intel email or company:')
+    print('\nNon org user with Intel email or company:')
     github_api.print_users(non_org_intel_pr_users)
-    print(f'\nNon org user with NO Intel email or company:')
+    print('\nNon org user with NO Intel email or company:')
     github_api.print_users(non_org_pr_users)
 
 

.github/org_control/config.json

@@ -6,7 +6,9 @@
         "openvino-ci",
         "openvino-pushbot",
         "lab-nerval",
-        "lab-nerval-onnx-ci"
+        "lab-nerval-onnx-ci",
+        "onnx-watchdog-agent",
+        "dependabot"
     ],
     "EMAILS_FILE_PATH": "dev_emails-test.txt",
     "PROXIES": {

.github/org_control/configs.py

@@ -57,19 +57,19 @@ def __init__(self, file_path=None, cli_args=None):
         for name, value in self._json_cfg.items():
             if hasattr(self, name):
                 raise ConfigException(f'Duplicating prosperity: {name}')
-            prosperity_value = self._args.get(name) or os.getenv(name)
-            if prosperity_value:
+            property_value = self._args.get(name) or os.getenv(name)
+            if property_value:
                 # Try to set prosperity_value as Python literal structures, e.g. DRY_RUN=False
                 try:
-                    prosperity_value = ast.literal_eval(prosperity_value)
+                    property_value = ast.literal_eval(property_value)
                 except Exception:
                     pass
-                if not isinstance(prosperity_value, type(value)):
+                if not isinstance(property_value, type(value)):
                     raise ConfigException(f'Python type of {name} parameter must be {type(value)}')
             else:
-                prosperity_value = value
-            setattr(self, name, prosperity_value)
-            Config.properties[name] = prosperity_value
+                property_value = value
+            setattr(self, name, property_value)
+            Config.properties[name] = property_value
 
         self.set_proxy()
 
@@ -78,7 +78,7 @@ def _load_cfg(self):
         try:
             with open(self._file_path) as conf:
                 self._json_cfg = json.load(conf)
-        except:
+        except Exception:
             print('Failed to load configuration from:', self._file_path)
             raise
 
@@ -105,7 +105,7 @@ def set_proxy(self):
 def _test():
     """Test and debug"""
     print('Config.default_cfg_path:', Config.default_cfg_path)
-    cfg = Config(cli_args=['DRY_RUN=True'])
+    cfg = Config(cli_args=['DRY_RUN', 'PROXIES={"NO_PROXY": "localhost"}'])
     print('Config.properties:', cfg.get_properties())
 
 

.github/org_control/github_api.py

@@ -11,7 +11,6 @@
 import time
 
 from github import Github, GithubException, RateLimitExceededException, IncompletableObject
-from github import UnknownObjectException
 from github.PaginatedList import PaginatedList
 
 from configs import Config
@@ -110,17 +109,13 @@ def __init__(self):
     def is_org_user(self, user):
         """Checks that user is a member of GitHub organization"""
         if is_valid_user(user):
-            try:
-                membership = user.get_organization_membership(self.github_org)
-                # membership.role can be 'member' or 'admin'
-                if membership.state == 'active' and membership.role:
-                    return True
-            except UnknownObjectException:
-                pass
+            # user.get_organization_membership(self.github_org) doesn't work with org members
+            # permissions, GITHUB_TOKEN must be org owner now
+            return self.github_org.has_in_members(user)
         return False
 
     def get_org_emails(self):
-        """Gets and prints all emails of GitHub organization members"""
+        """Gets and prints emails of all GitHub organization members"""
         org_members = self.github_org.get_members()
         org_emails = set()
         org_members_fix = set()
@@ -146,7 +141,7 @@ def get_org_emails(self):
               '; '.join(org_logins_fix_intel_email))
         print(f'\nOrg members - no real name {len(org_emails_fix_name)}:',
               '; '.join(org_emails_fix_name))
-        return org_emails
+        return (org_emails, org_logins_fix_intel_email)
 
     def get_org_invitation_emails(self):
         """Gets GitHub organization teams prints info"""