Add watchdog of OpenVino ONNX CI (openvinotoolkit#2550)

.ci/openvino-onnx/watchdog/Jenkinsfile

@@ -0,0 +1,65 @@
+// Copyright (C) 2018-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+
+timeout(30)
+{
+    node(LABEL) {
+
+        BUILD_WORKSPACE = "$WORKSPACE/$BUILD_NUMBER"
+        WATCHDOG_ROOT = "$BUILD_WORKSPACE/.ci/openvino-onnx/watchdog"
+        VENV_PATH = "${BUILD_WORKSPACE}/.wdvenv"
+
+        try {
+                stage("Clone repository") {
+                    dir ("$BUILD_WORKSPACE") {
+                        checkout([$class: 'GitSCM', branches: [[name: "*/$BRANCH"]],
+                                doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'CloneOption', timeout: 30]],  submoduleCfg: [],
+                                userRemoteConfigs: [[credentialsId: "${GITHUB_KEY}", url: '[email protected]:rblaczkowski/openvino.git']]])
+                    }
+                }
+                stage("Prepare environment") {
+                    sh """#!/bin/bash
+                        if [ ! -d ${VENV_PATH} ]; then
+                            python3 -m venv ${VENV_PATH}
+                            source ${VENV_PATH}/bin/activate
+                            pip install -r ${WATCHDOG_ROOT}/requirements.txt
+                        fi
+                    """
+                }
+                stage("Run script") {
+                    withCredentials([
+                        usernamePassword(credentialsId: '7157091e-bc04-42f0-99fd-dc4da2922a55',
+                        usernameVariable: 'username',
+                        passwordVariable: 'password')])
+                    {
+                        dir ("$BUILD_WORKSPACE") {
+                            sh """#!/bin/bash
+                                source ${VENV_PATH}/bin/activate
+                                export PYTHONHTTPSVERIFY=0
+                                python ${WATCHDOG_ROOT}/src/main.py \
+                                    --msteams-url=${MSTEAMS_URL_FILE} \
+                                    --github-credentials '${username}' '${password}' \
+                                    --github-org=${GITHUB_ORG} \
+                                    --github-project=${GITHUB_PROJECT} \
+                                    --jenkins-token=${JENKINS_TOKEN_FILE} \
+                                    --jenkins-server=${JENKINS_SERVER} \
+                                    --jenkins-user=${JENKINS_USER} \
+                                    --ci-job=${CI_JOB_NAME} \
+                                    --watchdog-job=${WATCHDOG_JOB_NAME}
+                            """
+                        }
+                    }
+                }
+        } catch (e) {
+            echo "$e"
+            currentBuild.result = "FAILURE"
+        } finally {
+            stage("Cleanup") {
+                sh """
+                    cd $BUILD_WORKSPACE
+                    rm -rf ..?* .[!.]* *
+                """
+            }
+        }
+    }
+}

.ci/openvino-onnx/watchdog/requirements.txt

@@ -0,0 +1,6 @@
+python-jenkins==1.7.0
+retrying==1.3.3
+pygithub==1.51
+timeout-decorator==0.4.1
+requests==2.23.0
+wheel

.ci/openvino-onnx/watchdog/src/git_wrapper.py

@@ -0,0 +1,117 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2018-2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import timeout_decorator
+from datetime import datetime
+from retrying import retry
+from github import Github, GithubException
+
+# Logging
+logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+_RETRY_LIMIT = 3
+_RETRY_COOLDOWN_MS = 2000
+_REQUEST_TIMEOUT_S = 10
+
+
+class GitWrapper:
+    """Class wrapping PyGithub API.
+
+    The purpose of this class is to wrap methods from PyGithub API used in Watchdog, for less error-prone and
+    more convenient use. Docs for used API, including wrapped methods can be found at:
+    https://pygithub.readthedocs.io/en/latest/introduction.html
+
+    :param github_credentials:       Credentials used for GitHub
+    :param repository:            GitHub repository name
+    :param project:               GitHub project name
+    :type github_credentials:        String
+    :type repository:             String
+    :type project:                String
+    """
+
+    def __init__(self, github_credentials, repository, project):
+        self.git = Github(*github_credentials)
+        self.repository = repository
+        self.project = project
+        self.github_credentials = github_credentials
+
+    @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
+    def get_git_time(self):
+        """Retrieve time from GitHub.
+
+        Used to reliably determine time during Watchdog run.
+
+        :return:                    Datetime object describing current time
+        :rtype:                     datetime
+        """
+        try:
+            datetime_object = self._get_git_time()
+        except ValueError as e:
+            raise GitWrapperError(str(e))
+        except GithubException as e:
+            message = 'GitHub Exception during API status retrieval. Exception: {}'.format(str(e))
+            raise GitWrapperError(message)
+        except timeout_decorator.TimeoutError:
+            message = 'GitHub Exception during API status retrieval. Timeout during API request.'
+            raise GitWrapperError(message)
+        return datetime_object
+
+    @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
+    def get_pull_requests(self):
+        """Retrieve paginated list of pull requests from GitHub.
+
+        :return:                    Paginated list of Pull Requests in GitHub repo
+        :rtype:                     github.PaginatedList.PaginatedList of github.PullRequest.PullRequest
+        """
+        try:
+            prs = self._get_pull_requests()
+        except GithubException as e:
+            message = 'GitHub Exception during API status retrieval. Exception: {}'.format(str(e))
+            raise GitWrapperError(message)
+        return prs
+
+    @timeout_decorator.timeout(_REQUEST_TIMEOUT_S)
+    def _get_git_time(self):
+        """Private method retrieving time from GitHub.
+
+        :return:                    Datetime object describing current time
+        :rtype:                     datetime
+        """
+        datetime_string = self.git.get_api_status().raw_headers.get('date', '')
+        datetime_format = '%a, %d %b %Y %H:%M:%S %Z'
+        datetime_object = datetime.strptime(datetime_string, datetime_format)
+        return datetime_object
+
+    @timeout_decorator.timeout(_REQUEST_TIMEOUT_S)
+    def _get_pull_requests(self):
+        """Private method retrieving pull requests from GitHub.
+
+        :return:                    Paginated list of Pull Requests in GitHub repo
+        :rtype:                     github.PaginatedList.PaginatedList of github.PullRequest.PullRequest
+        """
+        message = "git: {}".format(self.git)
+        message2 = "repository: {}".format(self.repository)
+        message3 = "project: {}".format(self.project)
+        message4 = "github_credentials: {}".format(self.github_credentials)
+        log.warning(message)
+        log.warning(message2)
+        log.warning(message3)
+        log.warning(message4)
+
+        return self.git.get_organization(self.repository).get_repo(self.project).get_pulls()
+
+
+class GitWrapperError(Exception):
+    """Base class for exceptions raised in GitWrapper.
+
+    :param message                   Explanation of the error
+    """
+
+    def __init__(self, message):
+        self.message = message
+        log.exception(message)

.ci/openvino-onnx/watchdog/src/jenkins_wrapper.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2018-2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import requests
+import jenkins
+import logging
+from retrying import retry
+
+# Logging
+logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s')
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
+_RETRY_LIMIT = 3
+_RETRY_COOLDOWN_MS = 5000
+
+
+class JenkinsWrapper:
+    """Class wrapping Python-Jenkins API.
+
+    The purpose of this class is to wrap methods from Python-Jenkins API used in Watchdog, for less error-prone and
+    more convenient use. Docs for used API, including wrapped methods can be found at:
+    https://python-jenkins.readthedocs.io/en/latest/
+
+        :param jenkins_token:       Token used for Jenkins
+        :param jenkins_user:        Username used to connect to Jenkins
+        :param jenkins_server:      Jenkins server address
+        :type jenkins_token:        String
+        :type jenkins_user:         String
+        :type jenkins_server:       String
+    """
+
+    def __init__(self, jenkins_token, jenkins_user, jenkins_server):
+        self.jenkins_server = jenkins_server
+        self.jenkins = jenkins.Jenkins(jenkins_server, username=jenkins_user,
+                                       password=jenkins_token)
+
+    @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
+    def get_build_console_output(self, job_name, build_number):
+        return self.jenkins.get_build_console_output(job_name, build_number)
+
+    @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
+    def get_job_info(self, job_name):
+        return self.jenkins.get_job_info(job_name)
+
+    @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
+    def get_build_info(self, job_name, build_number):
+        return self.jenkins.get_build_info(job_name, build_number)
+
+    @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
+    def get_queue_item(self, queue_id):
+        """Attempt to retrieve Jenkins job queue item.
+
+        Exception communicating queue doesn't exist is expected,
+        in that case method returns empty dict.
+
+            :param queue_id:            Jenkins job queue ID number
+            :type queue_id:             int
+            :return:                    Dictionary representing Jenkins job queue item
+            :rtype:                     dict
+        """
+        try:
+            return self.jenkins.get_queue_item(queue_id)
+        except Exception as e:
+            # Exception 'queue does not exist' is expected behaviour when job is running
+            if 'queue' in str(e) and 'does not exist' in str(e):
+                return {}
+            else:
+                raise
+
+    @retry(stop_max_attempt_number=_RETRY_LIMIT, wait_fixed=_RETRY_COOLDOWN_MS)
+    def get_idle_ci_hosts(self):
+        """Query Jenkins for idle servers.
+
+        Send GET request to Jenkins server, querying for idle servers labeled
+        for OpenVino-ONNX CI job.
+
+            :return:     Number of idle hosts delegated to OpenVino-ONNX CI
+            :rtype:      int
+        """
+        jenkins_request_url = self.jenkins_server + 'label/ci&&onnx/api/json?pretty=true'
+        try:
+            log.info('Sending request to Jenkins: %s', jenkins_request_url)
+            r = requests.Request(method='GET', url=jenkins_request_url, verify=False)
+            response = self.jenkins.jenkins_request(r).json()
+            return int(response['totalExecutors']) - int(response['busyExecutors'])
+        except Exception as e:
+            log.exception('Failed to send request to Jenkins!\nException message: %s', str(e))
+            raise

.ci/openvino-onnx/watchdog/src/main.py

@@ -0,0 +1,89 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2018-2020 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+import argparse
+import sys
+from watchdog import Watchdog
+
+DEFAULT_MSTEAMS_URL_FILE = '/home/lab_nerval/tokens/msteams_url'
+DEFAULT_GITHUB_ORGANIZATION = 'openvinotoolkit'
+DEFAULT_GITHUB_PROJECT = 'openvino'
+DEFAULT_JENKINS_TOKEN_FILE = '/home/lab_nerval/tokens/crackerjack'
+DEFAULT_JENKINS_SERVER = 'https://crackerjack.intel.com/'
+DEFAULT_JENKINS_USER = 'lab_nerval'
+DEFAULT_CI_JOB_NAME = 'onnx/OpenVino_CI'
+DEFAULT_WATCHDOG_JOB_NAME = 'onnx/ci_watchdog'
+
+
+def main(args):
+    """
+    Read args passed to script, load tokens and run watchdog.
+
+    Keyword arguments:
+    :param args:    arguments parsed by argparse ArgumentParser
+
+    :return:        returns status code 0 on successful completion
+
+    """
+    jenkins_server = args.jenkins_server.strip()
+    jenkins_user = args.jenkins_user.strip()
+    jenkins_token = open(args.jenkins_token).read().replace('\n', '').strip()
+    msteams_url = open(args.msteams_url).read().replace('\n', '').strip()
+    github_credentials = args.github_credentials
+    github_org = args.github_org
+    github_project = args.github_project
+    ci_job = args.ci_job.strip()
+    watchdog_job = args.watchdog_job.strip()
+    quiet = args.quiet
+
+    wd = Watchdog(jenkins_token=jenkins_token,
+                  jenkins_server=jenkins_server,
+                  jenkins_user=jenkins_user,
+                  github_credentials=github_credentials,
+                  git_org=github_org,
+                  git_project=github_project,
+                  msteams_url=msteams_url,
+                  ci_job_name=ci_job,
+                  watchdog_job_name=watchdog_job)
+    wd.run(quiet=quiet)
+
+    return 0
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--msteams-url', help='Path to MS Teams channel url to communicate messages.',
+                        default=DEFAULT_MSTEAMS_URL_FILE, action='store', required=False)
+
+    parser.add_argument('--github-credentials', help='GitHub user credentials to access repo.',
+                        nargs="+", required=True)
+
+    parser.add_argument('--github-org', help='Name of organization on GitHub.',
+                        default=DEFAULT_GITHUB_ORGANIZATION, action='store', required=False)
+
+    parser.add_argument('--github-project', help='Name of project on GitHub.',
+                        default=DEFAULT_GITHUB_PROJECT, action='store', required=False)
+
+    parser.add_argument('--jenkins-token', help='Path to Jenkins user token to access build info.',
+                        default=DEFAULT_JENKINS_TOKEN_FILE, action='store', required=False)
+
+    parser.add_argument('--jenkins-server', help='Jenkins server address.',
+                        default=DEFAULT_JENKINS_SERVER, action='store', required=False)
+
+    parser.add_argument('--jenkins-user', help='Jenkins user used to log in.',
+                        default=DEFAULT_JENKINS_USER, action='store', required=False)
+
+    parser.add_argument('--ci-job', help='Jenkins CI job name.',
+                        default=DEFAULT_CI_JOB_NAME, action='store', required=False)
+
+    parser.add_argument('--watchdog-job', help='Jenkins CI Watchdog job name.',
+                        default=DEFAULT_WATCHDOG_JOB_NAME, action='store', required=False)
+
+    parser.add_argument('--quiet', help="Quiet mode - doesn\'t send message to communicator.",
+                        action='store_true', required=False)
+
+    args = parser.parse_args()
+    sys.exit(main(args))