Skip to content

Commit

Permalink
Merge pull request #15934 from DrFaust92/r/glue_crawler_mongodb_target
Browse files Browse the repository at this point in the history 
r/glue_crawler - support mongodb target
  • Loading branch information
@breathingdust
breathingdust authored Nov 4, 2020
2 parents 81da040 + 62a6154 commit ed409e5
Show file tree
Hide file tree
Showing 3 changed files with 371 additions and 36 deletions.
140 changes: 104 additions & 36 deletions aws/resource_aws_glue_crawler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,9 +101,10 @@ func resourceAwsGlueCrawler() *schema.Resource {
Optional: true,
},
"s3_target": {
Type: schema.TypeList,
Optional: true,
MinItems: 1,
Type: schema.TypeList,
Optional: true,
MinItems: 1,
AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target"},
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"connection_name": {
Expand All @@ -123,9 +124,10 @@ func resourceAwsGlueCrawler() *schema.Resource {
},
},
"dynamodb_target": {
Type: schema.TypeList,
Optional: true,
MinItems: 1,
Type: schema.TypeList,
Optional: true,
MinItems: 1,
AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target"},
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"path": {
Expand All @@ -145,10 +147,34 @@ func resourceAwsGlueCrawler() *schema.Resource {
},
},
},
"mongodb_target": {
Type: schema.TypeList,
Optional: true,
MinItems: 1,
AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target"},
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"connection_name": {
Type: schema.TypeString,
Required: true,
},
"path": {
Type: schema.TypeString,
Required: true,
},
"scan_all": {
Type: schema.TypeBool,
Optional: true,
Default: true,
},
},
},
},
"jdbc_target": {
Type: schema.TypeList,
Optional: true,
MinItems: 1,
Type: schema.TypeList,
Optional: true,
MinItems: 1,
AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target"},
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"connection_name": {
Expand All @@ -168,9 +194,10 @@ func resourceAwsGlueCrawler() *schema.Resource {
},
},
"catalog_target": {
Type: schema.TypeList,
Optional: true,
MinItems: 1,
Type: schema.TypeList,
Optional: true,
MinItems: 1,
AtLeastOneOf: []string{"s3_target", "dynamodb_target", "mongodb_target", "jdbc_target", "catalog_target"},
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"database_name": {
Expand Down Expand Up @@ -240,16 +267,12 @@ func resourceAwsGlueCrawlerCreate(d *schema.ResourceData, meta interface{}) erro
}

func createCrawlerInput(crawlerName string, d *schema.ResourceData) (*glue.CreateCrawlerInput, error) {
crawlerTargets, err := expandGlueCrawlerTargets(d)
if err != nil {
return nil, err
}
crawlerInput := &glue.CreateCrawlerInput{
Name: aws.String(crawlerName),
DatabaseName: aws.String(d.Get("database_name").(string)),
Role: aws.String(d.Get("role").(string)),
Tags: keyvaluetags.New(d.Get("tags").(map[string]interface{})).IgnoreAws().GlueTags(),
Targets: crawlerTargets,
Targets: expandGlueCrawlerTargets(d),
}
if description, ok := d.GetOk("description"); ok {
crawlerInput.Description = aws.String(description.(string))
Expand Down Expand Up @@ -286,15 +309,11 @@ func createCrawlerInput(crawlerName string, d *schema.ResourceData) (*glue.Creat
}

func updateCrawlerInput(crawlerName string, d *schema.ResourceData) (*glue.UpdateCrawlerInput, error) {
crawlerTargets, err := expandGlueCrawlerTargets(d)
if err != nil {
return nil, err
}
crawlerInput := &glue.UpdateCrawlerInput{
Name: aws.String(crawlerName),
DatabaseName: aws.String(d.Get("database_name").(string)),
Role: aws.String(d.Get("role").(string)),
Targets: crawlerTargets,
Targets: expandGlueCrawlerTargets(d),
}
if description, ok := d.GetOk("description"); ok {
crawlerInput.Description = aws.String(description.(string))
Expand Down Expand Up @@ -352,24 +371,32 @@ func expandGlueSchemaChangePolicy(v []interface{}) *glue.SchemaChangePolicy {
return schemaPolicy
}

func expandGlueCrawlerTargets(d *schema.ResourceData) (*glue.CrawlerTargets, error) {
func expandGlueCrawlerTargets(d *schema.ResourceData) *glue.CrawlerTargets {
crawlerTargets := &glue.CrawlerTargets{}

dynamodbTargets, dynamodbTargetsOk := d.GetOk("dynamodb_target")
jdbcTargets, jdbcTargetsOk := d.GetOk("jdbc_target")
s3Targets, s3TargetsOk := d.GetOk("s3_target")
catalogTargets, catalogTargetsOk := d.GetOk("catalog_target")
if !dynamodbTargetsOk && !jdbcTargetsOk && !s3TargetsOk && !catalogTargetsOk {
return nil, fmt.Errorf("One of the following configurations is required: dynamodb_target, jdbc_target, s3_target, catalog_target")
log.Print("[DEBUG] Creating crawler target")

if v, ok := d.GetOk("dynamodb_target"); ok {
crawlerTargets.DynamoDBTargets = expandGlueDynamoDBTargets(v.([]interface{}))
}

log.Print("[DEBUG] Creating crawler target")
crawlerTargets.DynamoDBTargets = expandGlueDynamoDBTargets(dynamodbTargets.([]interface{}))
crawlerTargets.JdbcTargets = expandGlueJdbcTargets(jdbcTargets.([]interface{}))
crawlerTargets.S3Targets = expandGlueS3Targets(s3Targets.([]interface{}))
crawlerTargets.CatalogTargets = expandGlueCatalogTargets(catalogTargets.([]interface{}))
if v, ok := d.GetOk("jdbc_target"); ok {
crawlerTargets.JdbcTargets = expandGlueJdbcTargets(v.([]interface{}))
}

return crawlerTargets, nil
if v, ok := d.GetOk("s3_target"); ok {
crawlerTargets.S3Targets = expandGlueS3Targets(v.([]interface{}))
}

if v, ok := d.GetOk("catalog_target"); ok {
crawlerTargets.CatalogTargets = expandGlueCatalogTargets(v.([]interface{}))
}

if v, ok := d.GetOk("mongodb_target"); ok {
crawlerTargets.MongoDBTargets = expandGlueMongoDBTargets(v.([]interface{}))
}

return crawlerTargets
}

func expandGlueDynamoDBTargets(targets []interface{}) []*glue.DynamoDBTarget {
Expand Down Expand Up @@ -473,13 +500,36 @@ func expandGlueCatalogTarget(cfg map[string]interface{}) *glue.CatalogTarget {
return target
}

func expandGlueMongoDBTargets(targets []interface{}) []*glue.MongoDBTarget {
if len(targets) < 1 {
return []*glue.MongoDBTarget{}
}

perms := make([]*glue.MongoDBTarget, len(targets))
for i, rawCfg := range targets {
cfg := rawCfg.(map[string]interface{})
perms[i] = expandGlueMongoDBTarget(cfg)
}
return perms
}

func expandGlueMongoDBTarget(cfg map[string]interface{}) *glue.MongoDBTarget {
target := &glue.MongoDBTarget{
ConnectionName: aws.String(cfg["connection_name"].(string)),
Path: aws.String(cfg["path"].(string)),
ScanAll: aws.Bool(cfg["scan_all"].(bool)),
}

return target
}

func resourceAwsGlueCrawlerUpdate(d *schema.ResourceData, meta interface{}) error {
glueConn := meta.(*AWSClient).glueconn
name := d.Get("name").(string)

if d.HasChanges(
"catalog_target", "classifiers", "configuration", "description", "dynamodb_target", "jdbc_target", "role",
"s3_target", "schedule", "schema_change_policy", "security_configuration", "table_prefix") {
"s3_target", "schedule", "schema_change_policy", "security_configuration", "table_prefix", "mongodb_target") {
updateCrawlerInput, err := updateCrawlerInput(name, d)
if err != nil {
return err
Expand Down Expand Up @@ -595,6 +645,10 @@ func resourceAwsGlueCrawlerRead(d *schema.ResourceData, meta interface{}) error
if err := d.Set("catalog_target", flattenGlueCatalogTargets(crawlerOutput.Crawler.Targets.CatalogTargets)); err != nil {
return fmt.Errorf("error setting catalog_target: %s", err)
}

if err := d.Set("mongodb_target", flattenGlueMongoDBTargets(crawlerOutput.Crawler.Targets.MongoDBTargets)); err != nil {
return fmt.Errorf("error setting mongodb_target: %w", err)
}
}

tags, err := keyvaluetags.GlueListTags(glueConn, crawlerARN)
Expand Down Expand Up @@ -665,6 +719,20 @@ func flattenGlueJdbcTargets(jdbcTargets []*glue.JdbcTarget) []map[string]interfa
return result
}

func flattenGlueMongoDBTargets(mongoDBTargets []*glue.MongoDBTarget) []map[string]interface{} {
result := make([]map[string]interface{}, 0)

for _, mongoDBTarget := range mongoDBTargets {
attrs := make(map[string]interface{})
attrs["connection_name"] = aws.StringValue(mongoDBTarget.ConnectionName)
attrs["path"] = aws.StringValue(mongoDBTarget.Path)
attrs["scan_all"] = aws.BoolValue(mongoDBTarget.ScanAll)

result = append(result, attrs)
}
return result
}

func resourceAwsGlueCrawlerDelete(d *schema.ResourceData, meta interface{}) error {
glueConn := meta.(*AWSClient).glueconn

Expand Down
Loading

0 comments on commit ed409e5

Please sign in to comment.