Ignore properties with no valid characters. (GoogleCloudPlatform#577)

- Drop resource properties that don't contain any BigQuery valid characters from the schema. Prevoiusly such properties caused an 'string index out of range' errors during import. - Upgrade to latest beam sdk. Co-authored-by: Jacob Ferriero <[email protected]>
hyuatpc · Nov 4, 2020 · d57d4f9 · d57d4f9
1 parent f244a36
commit d57d4f9
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 6 deletions.
diff --git a/tools/asset-inventory/asset_inventory/bigquery_schema.py b/tools/asset-inventory/asset_inventory/bigquery_schema.py
@@ -261,6 +261,10 @@ def _sanitize_property(property_name, parent, depth, num_properties):
 
     # enforce column name requirements (condition #2).
     new_property_name = CLEAN_UP_REGEX.sub('', property_name)
+    # contains only non-word characters. just drop the property.
+    if not new_property_name:
+        parent.pop(property_name)
+        return
     first_character = new_property_name[0]
     if not first_character.isalpha() and first_character != '_':
         new_property_name = '_' + new_property_name

diff --git a/tools/asset-inventory/requirements.txt b/tools/asset-inventory/requirements.txt
@@ -1,3 +1,3 @@
-apache-beam[gcp]==2.17.0
+apache-beam[gcp]==2.25.0
 mock==2.0.0
 pytest
diff --git a/tools/asset-inventory/setup.py b/tools/asset-inventory/setup.py
@@ -38,12 +38,8 @@
     keywords='gcp asset inventory',
     packages=['asset_inventory'],
     setup_requires=['pytest-runner'],
-    # https://beam.apache.org/get-started/downloads
-    # Can't use 2.18, 2.19
-    # due to https://issues.apache.org/jira/browse/BEAM-9218
-    # should be fixed in 2.20 when released.
     extras_require = {
-        'testing': ['mock==2.0.0', 'pytest==4.6.6', 'apache-beamn[gcp]==2.17.0'],
+        'testing': ['mock==2.0.0', 'pytest==4.6.6', 'apache-beamn[gcp]==2.25.0'],
     },
     include_package_data=True,
     # https://pypi.org/project/google-cloud-asset/#history

diff --git a/tools/asset-inventory/tests/test_bigquery_schema.py b/tools/asset-inventory/tests/test_bigquery_schema.py
@@ -156,6 +156,7 @@ def test_sanitize_property_value(self):
             'empyty_dict': {},
             'empyty_dict_list': [{}, {}],
             'a' * 200: 'value0',
+            '@!@': 'deleteme',
             '@2_3': 'value1',
             'invalid_numeric': 9.300000191734863,
             'labels': {