From d9a8d5302325bd1d3e0b66dc3975963a15e5275f Mon Sep 17 00:00:00 2001 From: Pat Nadolny Date: Thu, 17 Oct 2024 14:01:56 -0400 Subject: [PATCH] remove issue custom fields (#90) --- tap_jira/streams.py | 692 +------------------------------------------- 1 file changed, 1 insertion(+), 691 deletions(-) diff --git a/tap_jira/streams.py b/tap_jira/streams.py index 229c550..0b809f5 100644 --- a/tap_jira/streams.py +++ b/tap_jira/streams.py @@ -384,7 +384,7 @@ class IssueStream(JiraStream): records_jsonpath = "$[issues][*]" # Or override `parse_response`. instance_name = "issues" - __content_schema = __content_schema = ArrayType( + __content_schema = ArrayType( ObjectType( Property("version", IntegerType), Property("text", StringType), @@ -538,173 +538,6 @@ class IssueStream(JiraStream): ) ) - base_content_schema = ObjectType( - Property( - "content", - ArrayType( - ObjectType( - Property("version", IntegerType), - Property("text", StringType), - Property("type", StringType), - Property("content", __content_schema), - Property( - "attrs", - ObjectType( - Property("href", StringType), - Property("colspan", IntegerType), - Property("alt", StringType), - Property("timestamp", StringType), - Property( - "colwidth", - ArrayType(IntegerType), - ), - Property("language", StringType), - Property("background", StringType), - Property( - "isNumberColumnEnabled", - BooleanType, - ), - Property("localId", StringType), - Property("color", StringType), - Property("panelType", StringType), - Property("level", IntegerType), - Property("accessLevel", StringType), - Property("style", StringType), - Property("order", IntegerType), - Property("text", StringType), - Property("shortName", StringType), - Property("url", StringType), - Property("layout", StringType), - Property("id", StringType), - Property("type", StringType), - Property("collection", StringType), - Property("width", NumberType), - Property("height", NumberType), - Property("occurrenceKey", StringType), - ), - ), - Property( - "marks", - ArrayType( - ObjectType( - Property("type", StringType), - Property( - "attrs", - ObjectType( - Property( - "href", - StringType, - ), - Property( - "colspan", - IntegerType, - ), - Property( - "alt", - StringType, - ), - Property( - "timestamp", - StringType, - ), - Property( - "colwidth", - ArrayType(IntegerType), - ), - Property( - "language", - StringType, - ), - Property( - "background", - StringType, - ), - Property( - "isNumberColumnEnabled", - BooleanType, - ), - Property( - "localId", - StringType, - ), - Property( - "color", - StringType, - ), - Property( - "panelType", - StringType, - ), - Property( - "level", - IntegerType, - ), - Property( - "accessLevel", - StringType, - ), - Property( - "style", - StringType, - ), - Property( - "order", - IntegerType, - ), - Property( - "text", - StringType, - ), - Property( - "shortName", - StringType, - ), - Property( - "url", - StringType, - ), - Property( - "layout", - StringType, - ), - Property("id", StringType), - Property( - "type", - StringType, - ), - Property( - "collection", - StringType, - ), - Property( - "width", - NumberType, - ), - Property( - "height", - NumberType, - ), - Property( - "occurrenceKey", - StringType, - ), - ), - ), - ), - ), - ), - ) - ), - ), - Property("type", StringType), - Property("version", IntegerType), - ) - base_item_schema = ObjectType( - Property("id", StringType), - Property("self", StringType), - Property("value", StringType), - ) - schema = PropertiesList( Property("expand", StringType), Property("id", StringType), @@ -786,8 +619,6 @@ class IssueStream(JiraStream): ), ), Property("timespent", IntegerType), - Property("customfield_10030", ArrayType(StringType)), - Property("customfield_10031", ArrayType(StringType)), Property( "project", ObjectType( @@ -808,7 +639,6 @@ class IssueStream(JiraStream): ), ), ), - Property("customfield_10032", StringType), Property( "fixVersions", ArrayType( @@ -821,10 +651,7 @@ class IssueStream(JiraStream): ) ), ), - Property("customfield_10033", StringType), - Property("customfield_10034", StringType), Property("aggregatetimespent", IntegerType), - Property("customfield_10035", StringType), Property( "resolution", ObjectType( @@ -834,8 +661,6 @@ class IssueStream(JiraStream): Property("self", StringType), ), ), - Property("customfield_10036", StringType), - Property("customfield_10037", StringType), Property("resolutiondate", StringType), Property("workratio", IntegerType), Property( @@ -849,33 +674,6 @@ class IssueStream(JiraStream): Property("issuerestriction", StringType), Property("lastViewed", StringType), Property("created", StringType), - Property( - "customfield_10020", - ArrayType( - ObjectType( - Property("boardId", IntegerType), - Property("completeDate", StringType), - Property("endDate", StringType), - Property("goal", StringType), - Property("id", IntegerType), - Property("name", StringType), - Property("startDate", StringType), - Property("state", StringType), - ), - ), - ), - Property( - "customfield_10021", - ArrayType( - ObjectType( - Property("id", StringType), - Property("self", StringType), - Property("value", StringType), - ), - ), - ), - Property("customfield_10022", StringType), - Property("customfield_10023", StringType), Property( "priority", ObjectType( @@ -885,26 +683,7 @@ class IssueStream(JiraStream): Property("id", StringType), ), ), - Property("customfield_10024", StringType), - Property("customfield_10025", StringType), Property("labels", ArrayType(StringType)), - Property("customfield_10016", NumberType), - Property("customfield_10017", StringType), - Property( - "customfield_10018", - ObjectType( - Property("hasEpicLinkFieldDependency", BooleanType), - Property("showField", BooleanType), - Property( - "nonEditableReason", - ObjectType( - Property("reason", StringType), - Property("message", StringType), - ), - ), - ), - ), - Property("customfield_10019", StringType), Property("timeestimate", IntegerType), Property("aggregatetimeoriginalestimate", IntegerType), Property("versions", ArrayType(StringType)), @@ -1665,30 +1444,9 @@ class IssueStream(JiraStream): ), ), ), - Property("customfield_10010", ArrayType(base_item_schema)), - Property("customfield_10014", StringType), Property("timetracking", StringType), - Property("customfield_10015", StringType), - Property( - "customfield_10005", - ArrayType( - ObjectType( - Property("id", IntegerType), - Property("boardId", IntegerType), - Property("name", StringType), - Property("state", StringType), - Property("goal", DateTimeType), - Property("startDate", DateTimeType), - Property("completeDate", DateTimeType), - ) - ), - ), - Property("customfield_10006", StringType), - Property("customfield_10007", StringType), Property("security", StringType), - Property("customfield_10008", base_item_schema), Property("aggregatetimeestimate", IntegerType), - Property("customfield_10009", StringType), Property("attachment", ArrayType(StringType)), Property("summary", StringType), Property( @@ -1771,7 +1529,6 @@ class IssueStream(JiraStream): ), ), ), - Property("customfield_10041", StringType), Property( "reporter", ObjectType( @@ -1793,8 +1550,6 @@ class IssueStream(JiraStream): Property("accountType", StringType), ), ), - Property("customfield_10043", StringType), - Property("customfield_10044", StringType), Property( "aggregateprogress", ObjectType( @@ -1803,17 +1558,6 @@ class IssueStream(JiraStream): Property("percent", IntegerType), ), ), - Property("customfield_10045", StringType), - Property("customfield_10001", ArrayType(StringType)), - Property("customfield_10002", NumberType), - Property("customfield_10003", NumberType), - Property("customfield_10004", StringType), - Property("customfield_10038", StringType), - Property("customfield_10039", StringType), - Property("customfield_10000", ArrayType(base_item_schema)), - Property("customfield_10042", StringType), - Property("customfield_10046", StringType), - Property("customfield_10047", StringType), Property( "environment", ObjectType( @@ -1861,408 +1605,6 @@ class IssueStream(JiraStream): Property("id", IntegerType), Property("editmeta", StringType), Property("histories", StringType), - Property("customfield_11394", StringType), - Property("customfield_11395", StringType), - Property("customfield_11397", StringType), - Property("customfield_11396", base_item_schema), - Property("customfield_11399", StringType), - Property("customfield_11398", DateType), - Property("customfield_11384", base_item_schema), - Property("customfield_11385", StringType), - Property("customfield_10600", base_item_schema), - Property("customfield_11490", base_item_schema), - Property("customfield_11492", StringType), - Property( - "customfield_11371", - base_item_schema, - ), - Property( - "customfield_11370", - base_item_schema, - ), - Property("customfield_11491", base_item_schema), - Property( - "customfield_11373", - base_item_schema, - ), - Property( - "customfield_11494", - base_item_schema, - ), - Property( - "customfield_11493", - base_item_schema, - ), - Property( - "customfield_11372", - base_item_schema, - ), - Property( - "customfield_11496", - base_item_schema, - ), - Property( - "customfield_11375", - base_item_schema, - ), - Property( - "customfield_11495", - base_item_schema, - ), - Property( - "customfield_11374", - base_item_schema, - ), - Property( - "customfield_11498", - base_item_schema, - ), - Property("customfield_11377", StringType), - Property("customfield_11497", NumberType), - Property("customfield_11376", StringType), - Property("customfield_11379", ArrayType(base_item_schema)), - Property("customfield_11378", ArrayType(base_item_schema)), - Property( - "customfield_11369", - base_item_schema, - ), - Property("customfield_11481", ArrayType(base_item_schema)), - Property("customfield_11482", base_item_schema), - Property("customfield_11485", StringType), - Property("customfield_11000", StringType), - Property("customfield_11487", StringType), - Property("customfield_11486", StringType), - Property("customfield_11489", base_item_schema), - Property("customfield_11368", DateType), - Property("customfield_11488", StringType), - Property("customfield_11359", DateType), - Property("customfield_11358", StringType), - Property("customfield_10701", StringType), - Property("customfield_11470", NumberType), - Property("customfield_11591", base_item_schema), - Property("customfield_11472", NumberType), - Property("customfield_11350", base_item_schema), - Property("customfield_11353", base_item_schema), - Property("customfield_11594", StringType), - Property("customfield_11473", StringType), - Property("customfield_11355", base_item_schema), - Property("customfield_11596", NumberType), - Property("customfield_11354", base_item_schema), - Property("customfield_11475", base_item_schema), - Property("customfield_11357", StringType), - Property("customfield_11356", StringType), - Property("customfield_11598", StringType), - Property("customfield_11469", base_item_schema), - Property("customfield_11468", NumberType), - Property("customfield_11347", StringType), - Property("customfield_11582", base_item_schema), - Property("customfield_11461", NumberType), - Property("customfield_11340", StringType), - Property("customfield_11460", StringType), - # Property("customfield_11584", StringType), - Property("customfield_11463", NumberType), - Property("customfield_11341", base_item_schema), - Property("customfield_11583", base_item_schema), - Property( - "customfield_11100", - ArrayType( - ObjectType( - Property("_link", StringType), - Property("id", StringType), - Property("name", StringType), - ) - ), - ), - Property("customfield_11586", StringType), - Property("customfield_11344", NumberType), - Property("customfield_11465", DateType), - Property("customfield_11585", StringType), - Property("customfield_11464", NumberType), - Property("customfield_11346", StringType), - Property("customfield_11467", base_item_schema), - Property("customfield_11466", NumberType), - Property("customfield_11345", StringType), - Property("customfield_11587", StringType), - # Property("customfield_11458", StringType), - Property("customfield_11336", NumberType), - Property("customfield_11457", base_item_schema), - Property("customfield_11339", StringType), - Property("customfield_11459", NumberType), - Property( - "customfield_10800", - ObjectType( - Property("hasEpicLinkFieldDependency", BooleanType), - Property( - "nonEditableReason", - ObjectType( - Property("message", StringType), - Property("reason", StringType), - ), - Property("showField", BooleanType), - ), - Property("showField", BooleanType), - ), - ), - Property("customfield_11338", base_item_schema), - Property("customfield_11450", ArrayType(base_item_schema)), - Property( - "customfield_11331", - ObjectType( - Property("displayName", StringType), - Property("languageCode", StringType), - ), - ), - Property("customfield_11452", base_item_schema), - Property( - "customfield_11330", - ObjectType(Property("errorMessage", StringType)), - ), - Property("customfield_11451", StringType), - Property("customfield_11454", base_item_schema), - Property("customfield_11333", base_item_schema), - Property("customfield_11575", base_item_schema), - Property("customfield_11453", DateType), - Property("customfield_11332", StringType), - Property("customfield_11335", base_content_schema), - Property("customfield_11576", StringType), - Property( - "customfield_11334", - base_item_schema, - ), - Property("customfield_11342", base_item_schema), - Property("customfield_11455", ArrayType(base_item_schema)), - Property("customfield_11326", ArrayType(base_item_schema)), - Property("customfield_11447", ArrayType(base_item_schema)), - Property("customfield_11568", NumberType), - Property("customfield_11567", NumberType), - Property("customfield_11446", ArrayType(base_item_schema)), - Property("customfield_11325", ArrayType(base_item_schema)), - Property( - "customfield_11328", - ObjectType(Property("errorMessage", StringType)), - ), - Property("customfield_11449", ArrayType(base_item_schema)), - Property( - "customfield_11327", - ObjectType(Property("errorMessage", StringType)), - ), - Property("customfield_11448", ArrayType(base_item_schema)), - Property( - "customfield_11329", - ObjectType(Property("errorMessage", StringType)), - ), - Property("customfield_11560", StringType), - Property("customfield_11441", StringType), - Property( - "customfield_11562", - base_item_schema, - ), - Property("customfield_11320", base_item_schema), - Property("customfield_11561", base_content_schema), - Property("customfield_11443", StringType), - Property("customfield_11200", base_item_schema), - Property("customfield_11321", base_item_schema), - Property("customfield_11564", base_content_schema), - Property("customfield_11442", StringType), - Property("customfield_11322", base_item_schema), - Property("customfield_11563", base_content_schema), - Property("customfield_11323", StringType), - Property("customfield_11566", NumberType), - Property("customfield_11445", StringType), - Property("customfield_11324", StringType), - Property("customfield_11565", NumberType), - Property("customfield_11444", StringType), - Property("customfield_11557", StringType), - Property("customfield_11314", base_item_schema), - Property("customfield_11436", StringType), - Property("customfield_11435", StringType), - Property("customfield_11556", StringType), - Property("customfield_11315", DateTimeType), - Property("customfield_11559", base_item_schema), - Property("customfield_11316", DateTimeType), - Property("customfield_11438", StringType), - Property("customfield_11437", StringType), - Property("customfield_11558", base_item_schema), - Property("customfield_11317", base_item_schema), - Property("customfield_11318", base_item_schema), - Property("customfield_11319", base_item_schema), - Property("customfield_11439", StringType), - Property("customfield_11430", base_item_schema), - Property("customfield_11310", ArrayType(base_item_schema)), - Property("customfield_10100", StringType), - Property("customfield_11431", ArrayType(base_item_schema)), - Property("customfield_11311", base_item_schema), - Property("customfield_11434", NumberType), - Property("customfield_11312", base_item_schema), - Property("customfield_11555", base_item_schema), - Property("customfield_11433", StringType), - Property("customfield_11313", base_item_schema), - Property("customfield_11425", NumberType), - Property("customfield_11303", DateType), - Property("customfield_11667", StringType), - Property("customfield_11666", ArrayType(base_item_schema)), - Property("customfield_11545", base_content_schema), - Property("customfield_11424", NumberType), - Property( - "customfield_11305", - ObjectType( - Property( - "_links", - ObjectType( - Property("agent", StringType), - Property("jiraRest", StringType), - Property("self", StringType), - Property("web", StringType), - ), - ), - Property( - "currentStatus", - ObjectType( - Property("status", StringType), - Property("statusCategory", StringType), - Property( - "statusDate", - ObjectType( - Property("epochMillis", IntegerType), - Property("friendly", DateTimeType), - Property("iso8601", DateTimeType), - Property("jira", DateTimeType), - ), - ), - ), - ), - Property( - "requestType", - ObjectType( - Property("_expands", ArrayType(StringType)), - Property( - "_links", ObjectType(Property("self", StringType)) - ), - Property("description", StringType), - Property("groupIds", ArrayType(StringType)), - Property("id", StringType), - Property("issueTypeId", StringType), - Property("name", StringType), - Property("portalId", StringType), - Property("serviceDeskId", StringType), - ), - ), - ), - ), - Property("customfield_11615", StringType), - Property("customfield_11427", NumberType), - Property("customfield_11548", base_item_schema), - Property("customfield_11668", base_item_schema), - Property("customfield_11306", ArrayType(base_item_schema)), - Property("customfield_11307", NumberType), - Property("customfield_11429", base_item_schema), - Property("customfield_11308", base_item_schema), - Property("customfield_11428", NumberType), - Property("customfield_11549", StringType), - Property("customfield_11309", DateTimeType), - Property("customfield_11661", base_item_schema), - Property("customfield_11660", base_content_schema), - Property("customfield_11421", NumberType), - Property("customfield_11663", NumberType), - Property("customfield_11420", NumberType), - Property("customfield_11300", StringType), - Property("customfield_11665", StringType), - Property("customfield_11301", DateType), - Property("customfield_11423", NumberType), - Property("customfield_11302", base_item_schema), - Property("customfield_11422", NumberType), - Property("customfield_11664", StringType), - Property("customfield_11414", DateTimeType), - Property("customfield_11656", StringType), - Property("customfield_11413", base_item_schema), - Property("customfield_11658", StringType), - Property("customfield_11415", DateType), - Property("customfield_11657", base_item_schema), - Property("customfield_11418", DateType), - Property("customfield_11659", StringType), - Property("customfield_11652", base_item_schema), - Property("customfield_11410", StringType), - Property("customfield_11530", base_content_schema), - Property("customfield_11411", StringType), - Property("customfield_11524", base_item_schema), - Property("customfield_11403", DateType), - Property("customfield_11645", NumberType), - Property("customfield_11402", ArrayType(base_item_schema)), - Property("customfield_11644", NumberType), - Property("customfield_11523", DateType), - Property("customfield_11647", StringType), - Property("customfield_11405", StringType), - Property("customfield_11526", StringType), - Property("customfield_11404", StringType), - Property("customfield_11525", StringType), - Property("customfield_11528", base_content_schema), - Property("customfield_11649", StringType), - Property("customfield_11648", ArrayType(base_item_schema)), - Property("customfield_11527", StringType), - Property("customfield_11529", StringType), - Property("customfield_11641", StringType), - Property("customfield_11520", base_item_schema), - Property("customfield_11640", StringType), - Property("customfield_11643", NumberType), - Property("customfield_11522", DateType), - Property("customfield_11401", StringType), - Property("customfield_11400", StringType), - Property("customfield_11642", NumberType), - Property("customfield_11521", base_item_schema), - Property("customfield_11513", DateType), - Property("customfield_10302", DateTimeType), - Property("customfield_11512", ArrayType(base_item_schema)), - Property("customfield_11515", ArrayType(base_item_schema)), - Property("customfield_11514", DateType), - Property("customfield_11638", base_item_schema), - Property("customfield_11517", ArrayType(StringType)), - Property("customfield_11516", base_item_schema), - Property("customfield_11637", base_item_schema), - Property("customfield_11519", base_item_schema), - Property("customfield_11639", base_item_schema), - Property("customfield_11518", StringType), - Property("customfield_10300", StringType), - Property("customfield_11511", StringType), - Property("customfield_10301", StringType), - Property("customfield_11510", base_item_schema), - Property("customfield_11502", NumberType), - Property("customfield_11501", StringType), - Property("customfield_11503", base_item_schema), - Property("customfield_11505", base_item_schema), - Property("customfield_11508", DateType), - Property("customfield_11507", base_item_schema), - Property("customfield_11509", StringType), - Property("customfield_11618", base_item_schema), - Property("customfield_11610", NumberType), - Property("customfield_11600", base_item_schema), - Property("customfield_11380", StringType), - Property("customfield_11382", StringType), - Property("customfield_11381", StringType), - Property("customfield_11480", base_item_schema), - Property("customfield_11479", StringType), - Property("customfield_11590", NumberType), - Property("customfield_11478", StringType), - Property("customfield_11477", StringType), - Property("customfield_11348", NumberType), - Property("customfield_11349", base_item_schema), - Property("customfield_11343", base_item_schema), - Property("customfield_11571", base_item_schema), - Property("customfield_11573", base_item_schema), - Property("customfield_11572", NumberType), - Property("customfield_11574", base_item_schema), - Property("customfield_11577", base_item_schema), - Property("customfield_11569", StringType), - Property("customfield_11551", NumberType), - Property("customfield_11550", StringType), - Property("customfield_11553", NumberType), - Property("customfield_11552", StringType), - Property("customfield_11554", NumberType), - Property("customfield_11540", base_item_schema), - Property("customfield_11541", base_item_schema), - Property("customfield_11543", StringType), - Property("customfield_11617", StringType), - Property("customfield_11504", base_item_schema), - Property("customfield_11616", NumberType), ), ), Property("created", StringType), @@ -2308,38 +1650,6 @@ def get_child_context(self, record: dict, context: Optional[dict]) -> dict: """Return a context dictionary for child streams.""" return {"issue_id": record["id"]} - def post_process(self, row: dict, context: dict | None = None) -> dict | None: - # dafault value for array, would remove once handled at SDK level - for key_set_default in [ - "customfield_10010", - "customfield_10005", - "customfield_10001", - "customfield_10000", - "customfield_11379", - "customfield_11378", - "customfield_11481", - "customfield_11481", - "customfield_11100", - "customfield_11450", - "customfield_11455", - "customfield_11326", - "customfield_11447", - "customfield_11446", - "customfield_11325", - "customfield_11449", - "customfield_11448", - "customfield_11310", - "customfield_11431", - "customfield_11666", - "customfield_11402", - "customfield_11648", - "customfield_11512", - "customfield_11515", - ]: - if row["fields"].get(key_set_default) is None: - row["fields"][key_set_default] = [] - return row - class PermissionStream(JiraStream): """