update fork

.ci/azure/linux.yml

@@ -29,6 +29,7 @@ jobs:
     MODELS_PATH: $(REPO_DIR)/../testdata
     WORK_DIR: $(Pipeline.Workspace)/_w
     BUILD_DIR: $(WORK_DIR)/build
+    BUILD_SAMPLES_DIR: $(WORK_DIR)/build_samples
     BIN_DIR: $(REPO_DIR)/bin/intel64/$(BUILD_TYPE)
     INSTALL_DIR: $(WORK_DIR)/install_pkg
     SETUPVARS: $(INSTALL_DIR)/bin/setupvars.sh
@@ -56,6 +57,7 @@ jobs:
   - script: |
       rm -rf $(WORK_DIR) ; mkdir $(WORK_DIR)
       rm -rf $(BUILD_DIR) ; mkdir $(BUILD_DIR)
+      rm -rf $(BUILD_SAMPLES_DIR) ; mkdir $(BUILD_SAMPLES_DIR)
       echo TargetBranch: $(System.PullRequest.TargetBranch)
       echo SourceBranch: $(Build.SourceBranch)
     displayName: 'Make dir'
@@ -117,12 +119,23 @@ jobs:
     displayName: 'Build Lin'
 
   - script: ls -alR $(REPO_DIR)/bin/
-    displayName: 'List files'
+    displayName: 'List bin files'
 
   - script: cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P cmake_install.cmake
     workingDirectory: $(BUILD_DIR)
     displayName: 'Install'
 
+  - script: ls -alR $(INSTALL_DIR)
+    displayName: 'List install files'
+
+  - script: $(INSTALL_DIR)/deployment_tools/inference_engine/samples/cpp/build_samples.sh
+    workingDirectory: $(BUILD_SAMPLES_DIR)
+    displayName: 'Build cpp samples'
+
+  - script: $(INSTALL_DIR)/deployment_tools/inference_engine/samples/c/build_samples.sh
+    workingDirectory: $(BUILD_SAMPLES_DIR)
+    displayName: 'Build c samples'
+
   - script: $(BIN_DIR)/unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml
     displayName: 'nGraph UT'
     continueOnError: false

.ci/azure/windows.yml

@@ -29,6 +29,7 @@ jobs:
     MODELS_PATH: $(REPO_DIR)\..\testdata
     WORK_DIR: $(Pipeline.Workspace)\_w
     BUILD_DIR: D:\build
+    BUILD_SAMPLES_DIR: D:\build_samples
     BIN_DIR: $(REPO_DIR)\bin\intel64
     MSVS_VARS_PATH: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat
     MSVC_COMPILER_PATH: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Tools\MSVC\14.24.28314\bin\Hostx64\x64\cl.exe
@@ -56,10 +57,11 @@ jobs:
   - script: |
       rd /Q /S $(WORK_DIR) & mkdir $(WORK_DIR)
       rd /Q /S $(BUILD_DIR) & mkdir $(BUILD_DIR)
+      rd /Q /S $(BUILD_SAMPLES_DIR) & mkdir $(BUILD_SAMPLES_DIR)
     displayName: 'Make dir'
 
   - script: |
-      certutil -urlcache -split -f https://incredibuilddiag1wu2.blob.core.windows.net/incredibuild/install_ib_console.bat install_ib_console.bat
+      certutil -urlcache -split -f https://openvinoweb.z5.web.core.windows.net/incredibuild/install_ib_console.bat install_ib_console.bat
       call install_ib_console.bat
     workingDirectory: $(WORK_DIR)
     displayName: 'Install IncrediBuild'
@@ -101,12 +103,23 @@ jobs:
     displayName: 'Build Win'
 
   - script: dir $(REPO_DIR)\bin\ /s
-    displayName: 'List files'
+    displayName: 'List bin files'
 
   - script: cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_DIR) -P cmake_install.cmake
     workingDirectory: $(BUILD_DIR)
     displayName: 'Install'
 
+  - script: dir $(INSTALL_DIR) /s
+    displayName: 'List install files'
+
+  - script: $(INSTALL_DIR)\deployment_tools\inference_engine\samples\cpp\build_samples_msvc.bat
+    workingDirectory: $(BUILD_SAMPLES_DIR)
+    displayName: 'Build cpp samples'
+
+  - script: $(INSTALL_DIR)\deployment_tools\inference_engine\samples\c\build_samples_msvc.bat
+    workingDirectory: $(BUILD_SAMPLES_DIR)
+    displayName: 'Build c samples'
+
   - script: |
       set PATH=$(TEST_ENV_PATH)
       $(BIN_DIR)\unit-test --gtest_print_time=1 --gtest_filter=-backend_api.config_unsupported:*IE_GPU* --gtest_output=xml:TEST-NGraphUT.xml

.ci/azure/windows_conditional_compilation.yml

@@ -47,7 +47,7 @@ jobs:
     displayName: 'Make dir'
 
   - script: |
-      certutil -urlcache -split -f https://incredibuilddiag1wu2.blob.core.windows.net/incredibuild/install_ib_console.bat install_ib_console.bat
+      certutil -urlcache -split -f https://openvinoweb.z5.web.core.windows.net/incredibuild/install_ib_console.bat install_ib_console.bat
       call install_ib_console.bat
     workingDirectory: $(WORK_DIR)
     displayName: 'Install IncrediBuild'

.github/org_control/__init__.py

@@ -1,3 +0,0 @@
-# Copyright (C) 2018-2021 Intel Corporation
-# SPDX-License-Identifier: Apache-2.0
-

.github/org_control/check_org.py

@@ -5,12 +5,13 @@
 Check GitHub organization and invite members
 """
 
-# pylint: disable=fixme,no-member
+# pylint: disable=fixme,no-member,too-many-locals
 
 from argparse import ArgumentParser
 
-import github_api
 from configs import Config
+from github_api import GithubOrgApi, get_dev_emails
+from ldap_api import LdapApi, print_user_info, InfoLevel
 
 
 def main():
@@ -19,32 +20,74 @@ def main():
     arg_parser.add_argument("--cfg-file", metavar="PATH", default=Config.default_cfg_path,
                             help=f"Path to json configuration file, e.g. {Config.default_cfg_path}")
     arg_parser.add_argument("--teams", action="store_true", help="Check GitHub teams")
+    arg_parser.add_argument("--no-ldap", action="store_true", help="Don't use LDAP info")
     args, unknown_args = arg_parser.parse_known_args()
 
     Config(args.cfg_file, unknown_args)
-    gh_api = github_api.GithubOrgApi()
+    gh_api = GithubOrgApi()
 
     if args.teams:
         gh_api.get_org_teams()
-    else:
-        dev_emails = github_api.get_dev_emails()
-        print(f'\nDeveloper emails {len(dev_emails)}:', '; '.join(dev_emails))
+        return
 
-        org_emails = gh_api.get_org_emails()
-        print(f'\nOrg emails {len(org_emails)}:', '; '.join(org_emails))
+    cfg_emails = get_dev_emails()
+    print(f'\nCfg developer emails {len(cfg_emails)}:', '; '.join(sorted(cfg_emails)))
 
-        org_pendig_invitation_emails = gh_api.get_org_invitation_emails()
+    dev_emails = set()
+    dev_emails.update(cfg_emails)
 
-        invite_emails = dev_emails.difference(org_emails).difference(org_pendig_invitation_emails)
-        print(f'\nInvite emails {len(invite_emails)}:', '; '.join(invite_emails))
+    if not args.no_ldap:
+        ldap_api = LdapApi()
+        ldap_emails = ldap_api.get_user_emails()
+        dev_emails.update(ldap_emails)
+        print(f'\nLDAP developer emails {len(ldap_emails)}:', '; '.join(sorted(ldap_emails)))
 
-        no_in_dev_emails = org_emails.difference(dev_emails)
-        print(f'\nOrg members - no in developers list {len(no_in_dev_emails)}:',
-              '; '.join(no_in_dev_emails))
+        cfg_emails_no_in_ldap = ldap_api.get_absent_emails(cfg_emails)
+        print(f'\nCfg developer emails - absent in LDAP at all {len(cfg_emails_no_in_ldap)}:',
+              '; '.join(sorted(cfg_emails_no_in_ldap)))
 
-        valid_github_users = gh_api.get_valid_github_users(invite_emails)
+        cfg_ldap_inters = cfg_emails.intersection(ldap_emails)
+        print(f'\nCfg developer emails - present in LDAP developers {len(cfg_ldap_inters)}:',
+              '; '.join(sorted(cfg_ldap_inters)))
 
-        gh_api.invite_users(valid_github_users)
+    org_emails, org_logins_no_intel_email = gh_api.get_org_emails()
+    print(f'\nOrg emails {len(org_emails)}:', '; '.join(sorted(org_emails)))
+
+    org_emails_no_in_ldap = set()
+    if not args.no_ldap:
+        org_ldap_diff = org_emails.difference(ldap_emails)
+        print(f'\nOrg member emails - absent in LDAP developers {len(org_ldap_diff)}:',
+              '; '.join(sorted(org_ldap_diff)))
+
+        for email in org_ldap_diff:
+            user_info = ldap_api.get_user_info_by_email(email)
+            if user_info:
+                print_user_info(user_info, InfoLevel.PDL)
+            else:
+                org_emails_no_in_ldap.add(email)
+
+    org_pendig_invitation_emails = gh_api.get_org_invitation_emails()
+    invite_emails = dev_emails.difference(org_emails).difference(org_pendig_invitation_emails)
+    print(f'\nInvite emails {len(invite_emails)}:', '; '.join(sorted(invite_emails)))
+
+    valid_github_users = gh_api.get_valid_github_users(invite_emails)
+    gh_api.invite_users(valid_github_users)
+
+    print('\nCheck accounts below and remove from the GitHub organization and cfg list')
+
+    cfg_emails_no_in_org = sorted(cfg_emails.difference(org_emails))
+    print(f'\nCfg developer emails - absent in GitHub organization {len(cfg_emails_no_in_org)}:',
+          '; '.join(cfg_emails_no_in_org))
+
+    org_emails_no_in_dev = sorted(org_emails.difference(dev_emails))
+    print(f'\nOrg member emails - absent in cfg and LDAP developers {len(org_emails_no_in_dev)}:',
+          '; '.join(org_emails_no_in_dev))
+
+    print(f'\nOrg member emails - absent in LDAP at all {len(org_emails_no_in_ldap)}:',
+          '; '.join(sorted(org_emails_no_in_ldap)))
+
+    print(f'\nOrg member logins - absent Intel email {len(org_logins_no_intel_email)}:',
+          '; '.join(sorted(org_logins_no_intel_email)))
 
 
 if __name__ == '__main__':

.github/org_control/check_pr.py

@@ -33,13 +33,23 @@ def get_pr_labels(pull):
 
 
 def set_pr_labels(pull, labels):
-    """Sets PR labels"""
+    """Sets new PR labels (all previously set labels are removed)"""
     if not labels or Config().DRY_RUN:
         return
-    print(f'Set PR labels:', labels)
+    print('Set PR labels:', labels)
+    # set_labels() should accept list but fails with empty "AssertionError:"
     pull.set_labels(labels)
 
 
+def add_pr_labels(pull, labels):
+    """Adds PR labels"""
+    if not labels or Config().DRY_RUN:
+        return
+    print('Add PR labels:', labels)
+    for label in labels:
+        pull.add_to_labels(label)
+
+
 def get_pr_type_by_labels(pull):
     """Gets PR type using labels"""
     pr_lables = get_pr_labels(pull)
@@ -80,6 +90,17 @@ def get_category_labels(pull):
     return labels
 
 
+def get_pr_info_str(pull):
+    """Gets info about PR using a few workarounds"""
+    pr_title = pull.title.encode("ASCII", "ignore").decode()
+
+    # Workaround for PyGithub issue: https://github.com/PyGithub/PyGithub/issues/512
+    pr_created_at = pull.created_at.replace(tzinfo=datetime.timezone.utc).astimezone()
+
+    return f'PR: {pull.number} - {pr_title} - Created: {pr_created_at} - ' \
+           f'Labels: {get_pr_labels(pull)} - Type: {get_pr_type_by_labels(pull)}'
+
+
 def main():
     """The main entry point function"""
     arg_parser = ArgumentParser()
@@ -103,19 +124,19 @@ def main():
         print(f'\nPRs count ({args.pr_state}):', pulls.totalCount)
 
     if args.newer:
-        pr_created_after = datetime.datetime.now() - datetime.timedelta(minutes=int(args.newer))
-        print('PRs created after:', pr_created_after)
+        pr_created_after = (datetime.datetime.now() -
+                            datetime.timedelta(minutes=int(args.newer))).astimezone()
+        print('Checking PRs created after:', pr_created_after)
     non_org_intel_pr_users = set()
     non_org_pr_users = set()
     for pull in pulls:
-        if args.newer and pull.created_at <= pr_created_after:
-            print(f'\nIGNORE: {pull} - Created: {pull.created_at}')
+        pr_created_at = pull.created_at.replace(tzinfo=datetime.timezone.utc).astimezone()
+        if args.newer and pr_created_at <= pr_created_after:
+            print(f'\nIGNORE: {get_pr_info_str(pull)}')
             continue
-        pr_lables = get_pr_labels(pull)
         pr_type_by_labels = get_pr_type_by_labels(pull)
-        set_labels = []
-        print(f'\n{pull} - Created: {pull.created_at} - Labels: {pr_lables} -',
-              f'Type: {pr_type_by_labels}', end='')
+        add_labels = []
+        print(f'\n{get_pr_info_str(pull)}', end='')
 
         # Checks PR source type
         if gh_api.is_org_user(pull.user):
@@ -127,21 +148,23 @@ def main():
             if pr_type_by_labels is not PrType.INTEL:
                 print(f'NO "{PrType.INTEL.value}" label: ', end='')
                 github_api.print_users(pull.user)
-                set_labels.append(PrType.INTEL.value)
+                add_labels.append(PrType.INTEL.value)
+        elif github_api.is_user_ignored(pull.user):
+            print(' - IGNORED non org user with NO Intel email or company')
         else:
-            print(f' - Non org user with NO Intel email or company')
+            print(' - Non org user with NO Intel email or company')
             non_org_pr_users.add(pull.user)
             if pr_type_by_labels is not PrType.EXTERNAL:
                 print(f'NO "{PrType.EXTERNAL.value}" label: ', end='')
                 github_api.print_users(pull.user)
-                set_labels.append(PrType.EXTERNAL.value)
+                add_labels.append(PrType.EXTERNAL.value)
 
-        set_labels += get_category_labels(pull)
-        set_pr_labels(pull, set_labels)
+        add_labels += get_category_labels(pull)
+        add_pr_labels(pull, add_labels)
 
-    print(f'\nNon org user with Intel email or company:')
+    print('\nNon org user with Intel email or company:')
     github_api.print_users(non_org_intel_pr_users)
-    print(f'\nNon org user with NO Intel email or company:')
+    print('\nNon org user with NO Intel email or company:')
     github_api.print_users(non_org_pr_users)
 
 

.github/org_control/config.json

@@ -6,7 +6,9 @@
         "openvino-ci",
         "openvino-pushbot",
         "lab-nerval",
-        "lab-nerval-onnx-ci"
+        "lab-nerval-onnx-ci",
+        "onnx-watchdog-agent",
+        "dependabot"
     ],
     "EMAILS_FILE_PATH": "dev_emails-test.txt",
     "PROXIES": {

.github/org_control/configs.py

@@ -57,19 +57,19 @@ def __init__(self, file_path=None, cli_args=None):
         for name, value in self._json_cfg.items():
             if hasattr(self, name):
                 raise ConfigException(f'Duplicating prosperity: {name}')
-            prosperity_value = self._args.get(name) or os.getenv(name)
-            if prosperity_value:
+            property_value = self._args.get(name) or os.getenv(name)
+            if property_value:
                 # Try to set prosperity_value as Python literal structures, e.g. DRY_RUN=False
                 try:
-                    prosperity_value = ast.literal_eval(prosperity_value)
+                    property_value = ast.literal_eval(property_value)
                 except Exception:
                     pass
-                if not isinstance(prosperity_value, type(value)):
+                if not isinstance(property_value, type(value)):
                     raise ConfigException(f'Python type of {name} parameter must be {type(value)}')
             else:
-                prosperity_value = value
-            setattr(self, name, prosperity_value)
-            Config.properties[name] = prosperity_value
+                property_value = value
+            setattr(self, name, property_value)
+            Config.properties[name] = property_value
 
         self.set_proxy()
 
@@ -78,7 +78,7 @@ def _load_cfg(self):
         try:
             with open(self._file_path) as conf:
                 self._json_cfg = json.load(conf)
-        except:
+        except Exception:
             print('Failed to load configuration from:', self._file_path)
             raise
 
@@ -105,7 +105,7 @@ def set_proxy(self):
 def _test():
     """Test and debug"""
     print('Config.default_cfg_path:', Config.default_cfg_path)
-    cfg = Config(cli_args=['DRY_RUN=True'])
+    cfg = Config(cli_args=['DRY_RUN', 'PROXIES={"NO_PROXY": "localhost"}'])
     print('Config.properties:', cfg.get_properties())