[build][infra] Improve build functions. (#6296)

Important functional changes involve mostly improvements to
the command line scripts (this doesn't affect the build infra, only
local use):

1. Make sure scripts use the same builder as builds requested by infra, otherwise builds
will be very slow and will fail for larger projects.
2. Allow users to specify --test-images to use base images with suffix "-testing"
3. Allow script users to specify --parallel for parallel builds.
4. Allow script users to specify --testing so that builds are uploaded to testing buckets.
5. Allow script users to specify --branch so that builds use specified branch instead of master.
6. Clone oss-fuzz with depth 1 for improved speed and space usage.
7. Use logging instead of writing to stderr or print.
8. Allow scripts to accept multiple projects.
9. Allow script to keep executing after failure to get build steps.
10. Change scripts to use python3.
11. Tag more so builds are easier to query.
12. Log the gcb page for each build.

Other changes include major refactoring:

1. Don't construct image names from scratch using format strings each time they are used.
Provide a helper function for this.
2. Provide a helper function,  get_env instead of constructing the env from scratch each time.
3. Move compile step into its own function: get_compile_step.
4. Move upload steps into their own helper function get_upload_steps.
5. Don't misuse the name image_project when we really mean cloud project.
6. Move cleanup step into its own helper function: get_cleanup_step.
7. Exit with returncode of main function from build_project.
8. Add unittests for build_project.
9. Make request_build share run_build code with build_project.
10. Use proper spacing in comments.
11. Test builds other than libfuzzer-ASAN-x86_64. Test other sanitizers, fuzzers and architectures
12. Make build_and_run_coverage share more code with build_project.
13. Move tests for build_and_run_coverage_test.py out of requst_coverage_test.py into their own file.
14. Use single quotes for strings.
15. Store state for a build in Build object instead of passing it everywhere.
16. Don't abuse project_yaml dict for storing project state. Use a Project object instead.
17. Better variable naming.
18. Use more classes instead of passing around arguments.
19. Use more f-strings.
20. Make scripts share main function.
21. Begin comments with uppercase and end with period.
22. Don't import functions or classes as dictated by style guide.
23. Share more test code in test_utils

Related: #6180.

infra/build/functions/base_images.py

@@ -32,8 +32,6 @@
 BASE_PROJECT = 'oss-fuzz-base'
 TAG_PREFIX = f'gcr.io/{BASE_PROJECT}/'
 
-BASE_SANITIZER_LIBS_IMAGE = TAG_PREFIX + 'base-sanitizer-libs-builder'
-
 
 def _get_base_image_steps(images, tag_prefix=TAG_PREFIX):
   """Returns build steps for given images."""

infra/build/functions/build_and_run_coverage.py

@@ -13,11 +13,11 @@
 # limitations under the License.
 #
 ################################################################################
-#!/usr/bin/python2
+#!/usr/bin/env python3
 """Starts and runs coverage build on Google Cloud Builder.
-Usage: build_and_run_coverage.py <project_dir>
+
+Usage: build_and_run_coverage.py <project>.
 """
-import datetime
 import json
 import logging
 import os
@@ -27,117 +27,104 @@
 import build_project
 
 SANITIZER = 'coverage'
-CONFIGURATION = ['FUZZING_ENGINE=libfuzzer', 'SANITIZER=%s' % SANITIZER]
+FUZZING_ENGINE = 'libfuzzer'
+ARCHITECTURE = 'x86_64'
+
 PLATFORM = 'linux'
 
-COVERAGE_BUILD_TAG = 'coverage'
+COVERAGE_BUILD_TYPE = 'coverage'
 
 # Where code coverage reports need to be uploaded to.
 COVERAGE_BUCKET_NAME = 'oss-fuzz-coverage'
 
-# Link to the code coverage report in HTML format.
-HTML_REPORT_URL_FORMAT = (build_lib.GCS_URL_BASENAME + COVERAGE_BUCKET_NAME +
-                          '/{project}/reports/{date}/{platform}/index.html')
-
 # This is needed for ClusterFuzz to pick up the most recent reports data.
-LATEST_REPORT_INFO_URL = ('/' + COVERAGE_BUCKET_NAME +
-                          '/latest_report_info/{project}.json')
-LATEST_REPORT_INFO_CONTENT_TYPE = 'application/json'
 
-# Link where to upload code coverage report files to.
-UPLOAD_URL_FORMAT = 'gs://' + COVERAGE_BUCKET_NAME + '/{project}/{type}/{date}'
+LATEST_REPORT_INFO_CONTENT_TYPE = 'application/json'
 
 # Languages from project.yaml that have code coverage support.
 LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'jvm', 'rust']
 
 
-def usage():
-  """Exit with code 1 and display syntax to use this file."""
-  sys.stderr.write("Usage: " + sys.argv[0] + " <project_dir>\n")
-  sys.exit(1)
+class Bucket:  # pylint: disable=too-few-public-methods
+  """Class representing the coverage GCS bucket."""
+
+  def __init__(self, project, date, platform, testing):
+    self.coverage_bucket_name = 'oss-fuzz-coverage'
+    if testing:
+      self.coverage_bucket_name += '-testing'
+
+    self.date = date
+    self.project = project
+    self.html_report_url = (
+        f'{build_lib.GCS_URL_BASENAME}{self.coverage_bucket_name}/{project}'
+        f'/reports/{date}/{platform}/index.html')
+    self.latest_report_info_url = (f'/{COVERAGE_BUCKET_NAME}'
+                                   f'/latest_report_info/{project}.json')
 
+  def get_upload_url(self, upload_type):
+    """Returns an upload url for |upload_type|."""
+    return (f'gs://{self.coverage_bucket_name}/{self.project}'
+            f'/{upload_type}/{self.date}')
 
-# pylint: disable=too-many-locals
-def get_build_steps(project_name, project_yaml_file, dockerfile_lines,
-                    image_project, base_images_project):
+
+def get_build_steps(  # pylint: disable=too-many-locals, too-many-arguments
+    project_name, project_yaml_contents, dockerfile_lines, image_project,
+    base_images_project, config):
   """Returns build steps for project."""
-  project_yaml = build_project.load_project_yaml(project_name,
-                                                 project_yaml_file,
-                                                 image_project)
-  if project_yaml['disabled']:
-    logging.info('Project "%s" is disabled.', project_name)
+  project = build_project.Project(project_name, project_yaml_contents,
+                                  dockerfile_lines, image_project)
+  if project.disabled:
+    logging.info('Project "%s" is disabled.', project.name)
     return []
 
-  if project_yaml['language'] not in LANGUAGES_WITH_COVERAGE_SUPPORT:
+  if project.fuzzing_language not in LANGUAGES_WITH_COVERAGE_SUPPORT:
     logging.info(
         'Project "%s" is written in "%s", coverage is not supported yet.',
-        project_name, project_yaml['language'])
+        project.name, project.fuzzing_language)
     return []
 
-  name = project_yaml['name']
-  image = project_yaml['image']
-  language = project_yaml['language']
-  report_date = datetime.datetime.now().strftime('%Y%m%d')
-
-  build_steps = build_lib.project_image_steps(name, image, language)
+  report_date = build_project.get_datetime_now().strftime('%Y%m%d')
+  bucket = Bucket(project.name, report_date, PLATFORM, config.testing)
 
-  env = CONFIGURATION[:]
-  out = '/workspace/out/' + SANITIZER
-  env.append('OUT=' + out)
-  env.append('FUZZING_LANGUAGE=' + language)
+  build_steps = build_lib.project_image_steps(project.name,
+                                              project.image,
+                                              project.fuzzing_language,
+                                              branch=config.branch,
+                                              test_images=config.test_images)
 
-  workdir = build_project.workdir_from_dockerfile(dockerfile_lines)
-  if not workdir:
-    workdir = '/src'
-
-  failure_msg = ('*' * 80 + '\nCoverage build failed.\nTo reproduce, run:\n'
-                 f'python infra/helper.py build_image {name}\n'
-                 'python infra/helper.py build_fuzzers --sanitizer coverage '
-                 f'{name}\n' + '*' * 80)
-
-  # Compilation step.
-  build_steps.append({
-      'name':
-          image,
-      'env':
-          env,
-      'args': [
-          'bash',
-          '-c',
-          # Remove /out to make sure there are non instrumented binaries.
-          # `cd /src && cd {workdir}` (where {workdir} is parsed from the
-          # Dockerfile). Container Builder overrides our workdir so we need
-          # to add this step to set it back.
-          (f'rm -r /out && cd /src && cd {workdir} && mkdir -p {out} && '
-           f'compile || (echo "{failure_msg}" && false)'),
-      ],
-  })
-
-  download_corpora_steps = build_lib.download_corpora_steps(project_name)
+  build = build_project.Build('libfuzzer', 'coverage', 'x86_64')
+  env = build_project.get_env(project.fuzzing_language, build)
+  build_steps.append(
+      build_project.get_compile_step(project, build, env, config.parallel))
+  download_corpora_steps = build_lib.download_corpora_steps(
+      project.name, testing=config.testing)
   if not download_corpora_steps:
-    logging.info('Skipping code coverage build for %s.', project_name)
+    logging.info('Skipping code coverage build for %s.', project.name)
     return []
 
   build_steps.extend(download_corpora_steps)
 
   failure_msg = ('*' * 80 + '\nCode coverage report generation failed.\n'
                  'To reproduce, run:\n'
-                 f'python infra/helper.py build_image {name}\n'
+                 f'python infra/helper.py build_image {project.name}\n'
                  'python infra/helper.py build_fuzzers --sanitizer coverage '
-                 f'{name}\n'
-                 f'python infra/helper.py coverage {name}\n' + '*' * 80)
+                 f'{project.name}\n'
+                 f'python infra/helper.py coverage {project.name}\n' + '*' * 80)
 
   # Unpack the corpus and run coverage script.
   coverage_env = env + [
       'HTTP_PORT=',
-      'COVERAGE_EXTRA_ARGS=%s' % project_yaml['coverage_extra_args'].strip(),
+      f'COVERAGE_EXTRA_ARGS={project.coverage_extra_args.strip()}',
   ]
-  if 'dataflow' in project_yaml['fuzzing_engines']:
+  if 'dataflow' in project.fuzzing_engines:
     coverage_env.append('FULL_SUMMARY_PER_TARGET=1')
 
   build_steps.append({
-      'name': f'gcr.io/{base_images_project}/base-runner',
-      'env': coverage_env,
+      'name':
+          build_project.get_runner_image_name(base_images_project,
+                                              config.testing),
+      'env':
+          coverage_env,
       'args': [
           'bash', '-c',
           ('for f in /corpus/*.zip; do unzip -q $f -d ${f%%.*} || ('
@@ -156,9 +143,7 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines,
   })
 
   # Upload the report.
-  upload_report_url = UPLOAD_URL_FORMAT.format(project=project_name,
-                                               type='reports',
-                                               date=report_date)
+  upload_report_url = bucket.get_upload_url('reports')
 
   # Delete the existing report as gsutil cannot overwrite it in a useful way due
   # to the lack of `-T` option (it creates a subdir in the destination dir).
@@ -170,15 +155,14 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines,
           '-m',
           'cp',
           '-r',
-          os.path.join(out, 'report'),
+          os.path.join(build.out, 'report'),
           upload_report_url,
       ],
   })
 
   # Upload the fuzzer stats. Delete the old ones just in case.
-  upload_fuzzer_stats_url = UPLOAD_URL_FORMAT.format(project=project_name,
-                                                     type='fuzzer_stats',
-                                                     date=report_date)
+  upload_fuzzer_stats_url = bucket.get_upload_url('fuzzer_stats')
+
   build_steps.append(build_lib.gsutil_rm_rf_step(upload_fuzzer_stats_url))
   build_steps.append({
       'name':
@@ -187,15 +171,13 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines,
           '-m',
           'cp',
           '-r',
-          os.path.join(out, 'fuzzer_stats'),
+          os.path.join(build.out, 'fuzzer_stats'),
           upload_fuzzer_stats_url,
       ],
   })
 
   # Upload the fuzzer logs. Delete the old ones just in case
-  upload_fuzzer_logs_url = UPLOAD_URL_FORMAT.format(project=project_name,
-                                                    type='logs',
-                                                    date=report_date)
+  upload_fuzzer_logs_url = bucket.get_upload_url('logs')
   build_steps.append(build_lib.gsutil_rm_rf_step(upload_fuzzer_logs_url))
   build_steps.append({
       'name':
@@ -204,15 +186,13 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines,
           '-m',
           'cp',
           '-r',
-          os.path.join(out, 'logs'),
+          os.path.join(build.out, 'logs'),
           upload_fuzzer_logs_url,
       ],
   })
 
   # Upload srcmap.
-  srcmap_upload_url = UPLOAD_URL_FORMAT.format(project=project_name,
-                                               type='srcmap',
-                                               date=report_date)
+  srcmap_upload_url = bucket.get_upload_url('srcmap')
   srcmap_upload_url = srcmap_upload_url.rstrip('/') + '.json'
   build_steps.append({
       'name': 'gcr.io/cloud-builders/gsutil',
@@ -225,15 +205,13 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines,
 
   # Update the latest report information file for ClusterFuzz.
   latest_report_info_url = build_lib.get_signed_url(
-      LATEST_REPORT_INFO_URL.format(project=project_name),
+      bucket.latest_report_info_url,
       content_type=LATEST_REPORT_INFO_CONTENT_TYPE)
   latest_report_info_body = json.dumps({
       'fuzzer_stats_dir':
           upload_fuzzer_stats_url,
       'html_report_url':
-          HTML_REPORT_URL_FORMAT.format(project=project_name,
-                                        date=report_date,
-                                        platform=PLATFORM),
+          bucket.html_report_url,
       'report_date':
           report_date,
       'report_summary_path':
@@ -249,25 +227,10 @@ def get_build_steps(project_name, project_yaml_file, dockerfile_lines,
 
 def main():
   """Build and run coverage for projects."""
-  if len(sys.argv) != 2:
-    usage()
-
-  image_project = 'oss-fuzz'
-  base_images_project = 'oss-fuzz-base'
-  project_dir = sys.argv[1].rstrip(os.path.sep)
-  project_name = os.path.basename(project_dir)
-  dockerfile_path = os.path.join(project_dir, 'Dockerfile')
-  project_yaml_path = os.path.join(project_dir, 'project.yaml')
-
-  with open(dockerfile_path) as docker_file:
-    dockerfile_lines = docker_file.readlines()
-
-  with open(project_yaml_path) as project_yaml_file:
-    steps = get_build_steps(project_name, project_yaml_file, dockerfile_lines,
-                            image_project, base_images_project)
-
-  build_project.run_build(steps, project_name, COVERAGE_BUILD_TAG)
+  return build_project.build_script_main(
+      'Generates coverage report for project.', get_build_steps,
+      COVERAGE_BUILD_TYPE)
 
 
-if __name__ == "__main__":
-  main()
+if __name__ == '__main__':
+  sys.exit(main())