add file_type 'other'

dagster-io · Aug 13, 2020 · fe550db · fe550db
1 parent 6d3f793
commit fe550db
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/python_modules/libraries/dagster-pyspark/dagster_pyspark/types.py b/python_modules/libraries/dagster-pyspark/dagster_pyspark/types.py
@@ -317,6 +317,7 @@ def dict_without_keys(ddict, *keys):
                     ),
                 }
             ),
+            'other': Permissive(),
         }
     )
 )
@@ -344,6 +345,8 @@ def dataframe_materializer(_context, config, spark_df):
     elif file_type == 'text':
         spark_df.write.text(**file_options)
         return AssetMaterialization.file(file_options['path'])
+    elif file_type == 'other':
+        return spark_df.write.save(**file_options)
     else:
         raise DagsterInvariantViolationError('Unsupported file_type {}'.format(file_type))
 
@@ -920,6 +923,7 @@ def dataframe_materializer(_context, config, spark_df):
                     ),
                 }
             ),
+            'other': Permissive(),
         },
     ),
     required_resource_keys={'pyspark'},
@@ -943,6 +947,8 @@ def dataframe_loader(_context, config):
         return spark_read.table(**file_options)
     elif file_type == 'text':
         return spark_read.text(path, **dict_without_keys(file_options, 'path'))
+    elif file_type == 'other':
+        return spark_read.load(**file_options)
     else:
         raise DagsterInvariantViolationError(
             'Unsupported file_type {file_type}'.format(file_type=file_type)