Migrate from python-gflags to absl-py.

CHANGES.next.md

@@ -1,4 +1,5 @@
 Breaking changes:
+- Replaced gflags with absl-py. (GH-1518)
 
 New features:
 

perfkitbenchmarker/__init__.py

@@ -12,4 +12,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import gflags as flags  # NOQA
+from absl import flags  # NOQA

perfkitbenchmarker/configs/__init__.py

@@ -84,7 +84,7 @@
                     'a path relative to the current working directory, '
                     'an absolute path, or just the name of a file in the '
                     'configs/ directory.')
-flags.DEFINE_multistring(
+flags.DEFINE_multi_string(
     'config_override', None,
     'This flag can be used to override any config value. It is applied after '
     'the user config (specified via --benchmark_config_file_path), so it has '

perfkitbenchmarker/configs/benchmark_config_spec.py

@@ -73,7 +73,7 @@ def Decode(self, value, component_full_name, flag_values):
         if not merged_flag_values[key].present:
           try:
             merged_flag_values[key].parse(value)
-          except flags.IllegalFlagValue as e:
+          except flags.IllegalFlagValueError as e:
             raise errors.Config.InvalidValue(
                 'Invalid {0}.{1} value: "{2}" (of type "{3}").{4}{5}'.format(
                     self._GetOptionFullName(component_full_name), key, value,

perfkitbenchmarker/data/__init__.py

@@ -36,10 +36,10 @@
 
 FLAGS = flags.FLAGS
 
-flags.DEFINE_multistring('data_search_paths', ['.'],
-                         'Additional paths to search for data files. '
-                         'These paths will be searched prior to using files '
-                         'bundled with PerfKitBenchmarker.')
+flags.DEFINE_multi_string('data_search_paths', ['.'],
+                          'Additional paths to search for data files. '
+                          'These paths will be searched prior to using files '
+                          'bundled with PerfKitBenchmarker.')
 
 _RESOURCES = 'resources'
 

perfkitbenchmarker/flag_util.py

@@ -334,7 +334,7 @@ def DEFINE_units(name, default, help, convertible_to,
         specifications, where each unit specification is either a string (e.g.
         'byte') or a units.Unit. The flag value must be convertible to at least
         one of the specified Units to be considered valid.
-    flag_values: the gflags.FlagValues object to define the flag in.
+    flag_values: the absl.flags.FlagValues object to define the flag in.
   """
   parser = UnitsParser(convertible_to=convertible_to)
   serializer = UnitsSerializer()
@@ -455,8 +455,8 @@ def DEFINE_yaml(name, default, help, flag_values=flags.FLAGS, **kwargs):
     name: string. The name of the flag.
     default: object. The default value of the flag.
     help: string. A help message for the user.
-    flag_values: the gflags.FlagValues object to define the flag in.
-    kwargs: extra arguments to pass to gflags.DEFINE().
+    flag_values: the absl.flags.FlagValues object to define the flag in.
+    kwargs: extra arguments to pass to absl.flags.DEFINE().
   """
 
   parser = YAMLParser()

perfkitbenchmarker/linux_benchmarks/object_storage_service_benchmark.py

@@ -1187,7 +1187,7 @@ def CLIThroughputBenchmark(output_results, metadata, vm, command_builder,
 
 def PrepareVM(vm, service):
   vm.Install('pip')
-  vm.RemoteCommand('sudo pip install python-gflags==2.0')
+  vm.RemoteCommand('sudo pip install absl-py')
   vm.RemoteCommand('sudo pip install pyyaml')
 
   vm.Install('openssl')
@@ -1213,7 +1213,7 @@ def PrepareVM(vm, service):
 
 def CleanupVM(vm, service):
   service.CleanupVM(vm)
-  vm.RemoteCommand('/usr/bin/yes | sudo pip uninstall python-gflags')
+  vm.RemoteCommand('/usr/bin/yes | sudo pip uninstall absl-py')
   vm.RemoteCommand('sudo rm -rf /tmp/run/')
   objects_written_file = posixpath.join(vm_util.VM_TMP_DIR,
                                         OBJECTS_WRITTEN_FILE)

perfkitbenchmarker/linux_packages/netperf.py

@@ -45,7 +45,7 @@
 def _Install(vm):
   """Installs the netperf package on the VM."""
   vm.Install('pip')
-  vm.RemoteCommand('sudo pip install python-gflags==2.0')
+  vm.RemoteCommand('sudo pip install absl-py')
   vm.Install('build_tools')
   _CopyTar(vm)
   vm.RemoteCommand('cd %s && tar xvzf %s' % (INSTALL_DIR, NETPERF_TAR))

perfkitbenchmarker/managed_relational_db.py

@@ -47,11 +47,10 @@
                     'Defaults to the client vm\'s zone.')
 
 BACKUP_TIME_REGULAR_EXPRESSION = '^\d\d\:\d\d$'
-flags.RegisterValidator('managed_db_backup_start_time',
-                        lambda value: re.search(BACKUP_TIME_REGULAR_EXPRESSION,
-                                                value) is not None,
-                        message=('--database_backup_start_time must be in the '
-                                 'form HH:MM'))
+flags.register_validator(
+    'managed_db_backup_start_time',
+    lambda value: re.search(BACKUP_TIME_REGULAR_EXPRESSION, value) is not None,
+    message=('--database_backup_start_time must be in the form HH:MM'))
 
 MYSQL = 'mysql'
 POSTGRES = 'postgres'

perfkitbenchmarker/pkb.py

@@ -282,7 +282,7 @@ def _ParseFlags(argv=sys.argv):
   """Parses the command-line flags."""
   try:
     argv = FLAGS(argv)
-  except flags.FlagsError as e:
+  except flags.Error as e:
     logging.error(e)
     logging.info('For usage instructions, use --helpmatch={module_name}')
     logging.info('For example, ./pkb.py --helpmatch=benchmarks.fio')

perfkitbenchmarker/providers/gcp/flags.py

@@ -43,7 +43,7 @@
 flags.DEFINE_string('gce_remote_access_firewall_rule', None, 'The name of an '
                     'already created firewall rule which allows remote access '
                     'instead of creating a new one.')
-flags.DEFINE_multistring(
+flags.DEFINE_multi_string(
     'gcp_instance_metadata_from_file',
     [],
     'A colon separated key-value pair that will be added to the '
@@ -52,7 +52,7 @@
     'each pair by commas. This option can be repeated multiple times. For '
     'information about GCP instance metadata, see: --metadata-from-file from '
     '`gcloud help compute instances create`.')
-flags.DEFINE_multistring(
+flags.DEFINE_multi_string(
     'gcp_instance_metadata',
     [],
     'A colon separated key-value pair that will be added to the '

perfkitbenchmarker/providers/kubernetes/flags.py

@@ -43,9 +43,9 @@
 flags.DEFINE_boolean('kubernetes_anti_affinity', True,
                      'If set to True, PKB pods will not be scheduled on the '
                      'same nodes as other PKB pods.')
-flags.DEFINE_multistring('k8s_volume_parameters', None,
-                         'A colon separated key-value pair that will be '
-                         'added to Kubernetes storage class parameters.')
+flags.DEFINE_multi_string('k8s_volume_parameters', None,
+                          'A colon separated key-value pair that will be '
+                          'added to Kubernetes storage class parameters.')
 _K8S_PROVISIONERS = [
     'kubernetes.io/azure-disk', 'kubernetes.io/gce-pd', 'kubernetes.io/aws-ebs'
 ]

perfkitbenchmarker/providers/openstack/swift.py

@@ -76,7 +76,7 @@ def PrepareVM(self, vm):
 
   def CleanupVM(self, vm):
     vm.Uninstall('swift_client')
-    vm.RemoteCommand('/usr/bin/yes | sudo pip uninstall python-gflags')
+    vm.RemoteCommand('/usr/bin/yes | sudo pip uninstall absl-py')
 
   def CLIUploadDirectory(self, vm, directory, file_names, bucket):
     return vm.RemoteCommand(

perfkitbenchmarker/publisher.py

@@ -110,7 +110,7 @@
 
 flags.DEFINE_string('es_type', 'result', 'Elasticsearch document type')
 
-flags.DEFINE_multistring(
+flags.DEFINE_multi_string(
     'metadata',
     [],
     'A colon separated key-value pair that will be added to the labels field '
@@ -891,7 +891,7 @@ def RepublishJSONSamples(path):
 
   try:
     argv = FLAGS(sys.argv)
-  except flags.FlagsError as e:
+  except flags.Error as e:
     logging.error(e)
     logging.info('Flag error. Usage: publisher.py <flags> path-to-json-file')
     sys.exit(1)

perfkitbenchmarker/scripts/database_scripts/launch_mysql_service.py

@@ -49,7 +49,7 @@
 import subprocess
 import sys
 import time
-import gflags
+from absl import flags
 
 import plot_sysbench_results
 
@@ -101,45 +101,45 @@
 CLEANUP = 'cleanup'
 TEARDOWN = 'teardown'
 
-FLAGS = gflags.FLAGS
-gflags.DEFINE_bool(PER_SECOND_GRAPHS, False,
-                   'Indicator for using per second data collection.'
-                   'To enable set True.')
-gflags.DEFINE_integer(SYSBENCH_RUN_SECONDS, 480,
-                      'The duration, in seconds, of each run phase with varying'
-                      'thread count.')
-gflags.DEFINE_integer(SYSBENCH_WARMUP_SECONDS, 0,
-                      'The duration, in seconds, of the warmup run in which '
-                      'results are discarded.')
-gflags.DEFINE_list(THREAD_COUNT_LIST, [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
-                   'The number of test threads on the client side.')
-gflags.DEFINE_integer(SYSBENCH_REPORT_INTERVAL, 1,
-                      'The interval, in seconds, we ask sysbench to report '
-                      'results.')
-gflags.DEFINE_string(RUN_URI, None,
-                     'Run identifier, if provided, only run phase '
-                     'will be completed.')
-gflags.DEFINE_string(RUN_STAGE, None,
-                     'List of phases to be executed. For example:'
-                     '"--run_uri=provision,prepare". Available phases:'
-                     'prepare, provision, run, cleanup, teardown.')
-gflags.DEFINE_string(GCE_BOOT_DISK_SIZE, '1000',
-                     'The boot disk size in GB for GCP VMs..')
-gflags.DEFINE_string(GCE_BOOT_DISK_TYPE, 'pd-ssd',
-                     'The boot disk type for GCP VMs.')
-gflags.DEFINE_string(MACHINE_TYPE, 'n1-standard-16',
-                     'Machine type for GCE Virtual machines.')
-gflags.DEFINE_enum(MYSQL_SVC_DB_INSTANCE_CORES, '4', ['1', '4', '8', '16'],
-                   'The number of cores to be provisioned for the DB instance.')
-gflags.DEFINE_string(MYSQL_SVC_OLTP_TABLES_COUNT, '4',
-                     'The number of tables used in sysbench oltp.lua tests')
-gflags.DEFINE_string(MYSQL_SVC_OLTP_TABLE_SIZE, '100000',
-                     'The number of rows of each table used in the oltp tests')
-gflags.DEFINE_string(MYSQL_INSTANCE_STORAGE_SIZE, '300',
-                     'Storage size (in GB) for SQL instance.')
-gflags.DEFINE_list(ADDITIONAL_FLAGS, None,
-                   'List of additional PKB mysql_service valid flags (strings).'
-                   'For example: "--cloud_storage_bucket=bucket_name".')
+FLAGS = flags.FLAGS
+flags.DEFINE_bool(PER_SECOND_GRAPHS, False,
+                  'Indicator for using per second data collection.'
+                  'To enable set True.')
+flags.DEFINE_integer(SYSBENCH_RUN_SECONDS, 480,
+                     'The duration, in seconds, of each run phase with varying'
+                     'thread count.')
+flags.DEFINE_integer(SYSBENCH_WARMUP_SECONDS, 0,
+                     'The duration, in seconds, of the warmup run in which '
+                     'results are discarded.')
+flags.DEFINE_list(THREAD_COUNT_LIST, [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
+                  'The number of test threads on the client side.')
+flags.DEFINE_integer(SYSBENCH_REPORT_INTERVAL, 1,
+                     'The interval, in seconds, we ask sysbench to report '
+                     'results.')
+flags.DEFINE_string(RUN_URI, None,
+                    'Run identifier, if provided, only run phase '
+                    'will be completed.')
+flags.DEFINE_string(RUN_STAGE, None,
+                    'List of phases to be executed. For example:'
+                    '"--run_uri=provision,prepare". Available phases:'
+                    'prepare, provision, run, cleanup, teardown.')
+flags.DEFINE_string(GCE_BOOT_DISK_SIZE, '1000',
+                    'The boot disk size in GB for GCP VMs..')
+flags.DEFINE_string(GCE_BOOT_DISK_TYPE, 'pd-ssd',
+                    'The boot disk type for GCP VMs.')
+flags.DEFINE_string(MACHINE_TYPE, 'n1-standard-16',
+                    'Machine type for GCE Virtual machines.')
+flags.DEFINE_enum(MYSQL_SVC_DB_INSTANCE_CORES, '4', ['1', '4', '8', '16'],
+                  'The number of cores to be provisioned for the DB instance.')
+flags.DEFINE_string(MYSQL_SVC_OLTP_TABLES_COUNT, '4',
+                    'The number of tables used in sysbench oltp.lua tests')
+flags.DEFINE_string(MYSQL_SVC_OLTP_TABLE_SIZE, '100000',
+                    'The number of rows of each table used in the oltp tests')
+flags.DEFINE_string(MYSQL_INSTANCE_STORAGE_SIZE, '300',
+                    'Storage size (in GB) for SQL instance.')
+flags.DEFINE_list(ADDITIONAL_FLAGS, None,
+                  'List of additional PKB mysql_service valid flags (strings).'
+                  'For example: "--cloud_storage_bucket=bucket_name".')
 
 # TODO: Implement flag for STDOUT/STDERR file paths.
 
@@ -167,7 +167,7 @@ def driver(argv):
   """
   try:  # Parse command line flags
     argv = FLAGS(argv)
-  except gflags.FlagsError as e:
+  except flags.Error as e:
     logging.error('%s\nUsage: %s ARGS\n%s', e, sys.argv[0], FLAGS)
     sys.exit(1)
   run_uri = FLAGS.run_uri

perfkitbenchmarker/scripts/netperf_test_scripts/netperf_test.py

@@ -18,7 +18,7 @@
 import logging
 import subprocess
 import sys
-import gflags as flags
+from absl import flags
 
 FLAGS = flags.FLAGS
 
@@ -35,7 +35,7 @@ def Main(argv=sys.argv):
   # Parse command-line flags
   try:
     argv = FLAGS(argv)
-  except flags.FlagsError as e:
+  except flags.Error as e:
     logging.error('%s\nUsage: %s ARGS\n%s', e, sys.argv[0], FLAGS)
     sys.exit(1)
 

perfkitbenchmarker/scripts/object_storage_api_test_scripts/azure_flags.py

@@ -14,7 +14,7 @@
 
 """Flags for the Azure Blob Storage interface."""
 
-import gflags as flags
+from absl import flags
 
 flags.DEFINE_string('azure_account', None,
                     'The name of the storage account for Azure.')

perfkitbenchmarker/scripts/object_storage_api_test_scripts/azure_service.py

@@ -17,7 +17,7 @@
 import logging
 import time
 
-import gflags as flags
+from absl import flags
 
 import azure.storage.blob
 

perfkitbenchmarker/scripts/object_storage_api_test_scripts/gcs.py

@@ -16,7 +16,7 @@
 
 import time
 
-import gflags as flags
+from absl import flags
 import gcs_oauth2_boto_plugin  # noqa
 
 import boto_service

perfkitbenchmarker/scripts/object_storage_api_test_scripts/object_storage_api_tests.py

@@ -35,7 +35,7 @@
 
 import yaml
 
-import gflags as flags
+from absl import flags
 
 import azure_flags  # noqa
 import s3_flags  # noqa
@@ -1055,7 +1055,7 @@ def Main(argv=sys.argv):
 
   try:
     argv = FLAGS(argv)  # parse flags
-  except flags.FlagsError as e:
+  except flags.Error as e:
     logging.error(
         '%s\nUsage: %s ARGS\n%s', e, sys.argv[0], FLAGS)
     sys.exit(1)

perfkitbenchmarker/scripts/object_storage_api_test_scripts/s3.py

@@ -17,7 +17,7 @@
 import logging
 import time
 
-import gflags as flags
+from absl import flags
 
 import boto_service
 

perfkitbenchmarker/scripts/object_storage_api_test_scripts/s3_flags.py

@@ -14,6 +14,6 @@
 
 """Flags for the S3 provider interface."""
 
-import gflags as flags
+from absl import flags
 
 flags.DEFINE_string('host', None, 'The hostname of the storage endpoint.')

perfkitbenchmarker/scripts/object_storage_api_test_scripts/validate_service.py

@@ -20,7 +20,7 @@
 import logging
 import sys
 
-import gflags as flags
+from absl import flags
 
 import object_storage_api_tests
 

requirements.txt

@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-python-gflags==3.1.1
+absl-py
 jinja2>=2.7
 setuptools
 colorlog[windows]==2.6.0

setup.py

@@ -20,6 +20,6 @@
     license='Apache 2.0',
     packages=find_packages(exclude=['tests']),
     scripts=['pkb.py'],
-    install_requires=['python-gflags==3.1.1',
+    install_requires=['absl-py',
                       'jinja2>=2.7',
                       'setuptools'])

tests/__init__.py

@@ -12,12 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
 from perfkitbenchmarker import flags
 
 
 # Many places in the PKB codebase directly reference the global FLAGS. Several
 # tests were written using python-gflags==2.0, which allows accessing flag
-# values before they are parsed. python-gflags>=3.0.4 discourages this behavior,
-# so parse program name + an empty list of flags before any tests are executed.
-flags.FLAGS([sys.argv[0]])
+# values before they are parsed. Abseil forbids this behavior, so mark flags
+# as parsed before any tests are executed.
+flags.FLAGS.mark_as_parsed()