Skip to content

Commit

Permalink
fix(ingest): fix lingering demo-data source issues
Browse files Browse the repository at this point in the history
- renames dummy-data -> demo-data
- makes the config empty
  • Loading branch information
hsheth2 committed Dec 6, 2022
1 parent 43c566e commit 910a827
Show file tree
Hide file tree
Showing 5 changed files with 37 additions and 36 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
source:
type: demo-data
config: {}

This file was deleted.

2 changes: 1 addition & 1 deletion metadata-ingestion/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,7 @@ def get_long_description():
"presto-on-hive = datahub.ingestion.source.sql.presto_on_hive:PrestoOnHiveSource",
"pulsar = datahub.ingestion.source.pulsar:PulsarSource",
"salesforce = datahub.ingestion.source.salesforce:SalesforceSource",
"dummy-data = datahub.ingestion.source.dummy_source.DummySource",
"demo-data = datahub.ingestion.source.demo_data.DemoDataSource",
"unity-catalog = datahub.ingestion.source.unity.source:UnityCatalogSource",
],
"datahub.ingestion.sink.plugins": [
Expand Down
33 changes: 33 additions & 0 deletions metadata-ingestion/src/datahub/ingestion/source/demo_data.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
from datahub.configuration.common import ConfigModel
from datahub.ingestion.api.common import PipelineContext
from datahub.ingestion.api.decorators import (
SupportStatus,
config_class,
platform_name,
support_status,
)
from datahub.ingestion.source.file import FileSourceConfig, GenericFileSource
from datahub.utilities.sample_data import download_sample_data


class DemoDataConfig(ConfigModel):
# The demo data source does not accept any configuration.
pass


@platform_name("Demo Data")
@config_class(ConfigModel)
@support_status(SupportStatus.UNKNOWN)
class DemoDataSource(GenericFileSource):
"""
This source loads sample data into DataHub. It is intended for demo and testing purposes only.
"""

def __init__(self, ctx: PipelineContext, config: DemoDataConfig):
file_config = FileSourceConfig(filename=download_sample_data())
super().__init__(ctx, file_config)

@classmethod
def create(cls, config_dict, ctx):
config = DemoDataConfig.parse_obj(config_dict or {})
return cls(ctx, config)
29 changes: 0 additions & 29 deletions metadata-ingestion/src/datahub/ingestion/source/dummy_source.py

This file was deleted.

0 comments on commit 910a827

Please sign in to comment.