github immediate team members

Signed-off-by: Daniel Brauer <[email protected]>

cartography/intel/github/teams.py

@@ -17,7 +17,10 @@
 
 logger = logging.getLogger(__name__)
 
+# A team's permission on a repo: https://docs.github.com/en/graphql/reference/enums#repositorypermission
 RepoPermission = namedtuple('RepoPermission', ['repo_url', 'permission'])
+# A team member's role: https://docs.github.com/en/graphql/reference/enums#teammemberrole
+UserRole = namedtuple('UserRole', ['user_url', 'role'])
 
 
 @timeit
@@ -35,6 +38,9 @@ def get_teams(org: str, api_url: str, token: str) -> Tuple[PaginatedGraphqlData,
                         repositories(first: 100) {
                             totalCount
                         }
+                        members(first: 100, membership: IMMEDIATE) {
+                            totalCount
+                        }
                     }
                     pageInfo{
                         endCursor
@@ -142,10 +148,106 @@ def _get_team_repos(org: str, api_url: str, token: str, team: str) -> PaginatedG
     return team_repos
 
 
+def _get_team_users_for_multiple_teams(
+        team_raw_data: list[dict[str, Any]],
+        org: str,
+        api_url: str,
+        token: str,
+) -> dict[str, list[UserRole]]:
+    result: dict[str, list[UserRole]] = {}
+    for team in team_raw_data:
+        team_name = team['slug']
+        user_count = team['members']['totalCount']
+
+        if user_count == 0:
+            # This team has no users so let's move on
+            result[team_name] = []
+            continue
+
+        user_urls = []
+        user_roles = []
+
+        max_tries = 5
+
+        for current_try in range(1, max_tries + 1):
+            team_users = _get_team_users(org, api_url, token, team_name)
+
+            try:
+                # The `or []` is because `.nodes` can be None. See:
+                # https://docs.github.com/en/graphql/reference/objects#teammemberconnection
+                for user in team_users.nodes or []:
+                    user_urls.append(user['url'])
+
+                # The `or []` is because `.edges` can be None.
+                for edge in team_users.edges or []:
+                    user_roles.append(edge['role'])
+                # We're done! Break out of the retry loop.
+                break
+
+            except TypeError:
+                # Handles issue #1334
+                logger.warning(
+                    f"GitHub returned None when trying to find user or role data for team {team_name}.",
+                    exc_info=True,
+                )
+                if current_try == max_tries:
+                    raise RuntimeError(f"GitHub returned a None member url for team {team_name}, retries exhausted.")
+                sleep(current_try ** 2)
+
+        # Shape = [(user_url, 'MAINTAINER'), ...]]
+        result[team_name] = [UserRole(url, role) for url, role in zip(user_urls, user_roles)]
+    return result
+
+
+@timeit
+def _get_team_users(org: str, api_url: str, token: str, team: str) -> PaginatedGraphqlData:
+    team_users_gql = """
+    query($login: String!, $team: String!, $cursor: String) {
+        organization(login: $login) {
+            url
+            login
+            team(slug: $team) {
+                slug
+                members(first: 100, after: $cursor, membership: IMMEDIATE) {
+                    totalCount
+                    nodes {
+                        url
+                    }
+                    edges {
+                        role
+                    }
+                    pageInfo {
+                        endCursor
+                        hasNextPage
+                    }
+                }
+            }
+        }
+        rateLimit {
+            limit
+            cost
+            remaining
+            resetAt
+        }
+    }
+    """
+    team_users, _ = fetch_all(
+        token,
+        api_url,
+        org,
+        team_users_gql,
+        'team',
+        resource_inner_type='members',
+        team=team,
+    )
+    return team_users
+
+
 def transform_teams(
         team_paginated_data: PaginatedGraphqlData,
         org_data: Dict[str, Any],
         team_repo_data: dict[str, list[RepoPermission]],
+        team_user_data: dict[str, list[UserRole]],
 ) -> list[dict[str, Any]]:
     result = []
     for team in team_paginated_data.nodes:
@@ -155,19 +257,29 @@ def transform_teams(
             'url': team['url'],
             'description': team['description'],
             'repo_count': team['repositories']['totalCount'],
+            'member_count': team['members']['totalCount'],
             'org_url': org_data['url'],
             'org_login': org_data['login'],
         }
         repo_permissions = team_repo_data[team_name]
-        if not repo_permissions:
+        user_roles = team_user_data[team_name]
+
+        if not repo_permissions and not user_roles:
             result.append(repo_info)
             continue
 
-        # `permission` can be one of ADMIN, READ, WRITE, TRIAGE, or MAINTAIN
-        for repo_url, permission in repo_permissions:
-            repo_info_copy = repo_info.copy()
-            repo_info_copy[permission] = repo_url
-            result.append(repo_info_copy)
+        if repo_permissions:
+            # `permission` can be one of ADMIN, READ, WRITE, TRIAGE, or MAINTAIN
+            for repo_url, permission in repo_permissions:
+                repo_info_copy = repo_info.copy()
+                repo_info_copy[permission] = repo_url
+                result.append(repo_info_copy)
+        if user_roles:
+            # `role` can be one of MAINTAINER, MEMBER
+            for user_url, role in user_roles:
+                repo_info_copy = repo_info.copy()
+                repo_info_copy[role] = user_url
+                result.append(repo_info_copy)
     return result
 
 
@@ -203,7 +315,8 @@ def sync_github_teams(
 ) -> None:
     teams_paginated, org_data = get_teams(organization, github_url, github_api_key)
     team_repos = _get_team_repos_for_multiple_teams(teams_paginated.nodes, organization, github_url, github_api_key)
-    processed_data = transform_teams(teams_paginated, org_data, team_repos)
+    team_users = _get_team_users_for_multiple_teams(teams_paginated.nodes, organization, github_url, github_api_key)
+    processed_data = transform_teams(teams_paginated, org_data, team_repos, team_users)
     load_team_repos(neo4j_session, processed_data, common_job_parameters['UPDATE_TAG'], org_data['url'])
     common_job_parameters['org_url'] = org_data['url']
     cleanup(neo4j_session, common_job_parameters)

cartography/models/github/teams.py

@@ -80,6 +80,33 @@ class GitHubTeamWriteRepoRel(CartographyRelSchema):
     properties: GitHubTeamToRepoRelProperties = GitHubTeamToRepoRelProperties()
 
 
+@dataclass(frozen=True)
+class GitHubTeamToUserRelProperties(CartographyRelProperties):
+    lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
+
+
+@dataclass(frozen=True)
+class GitHubTeamMaintainerUserRel(CartographyRelSchema):
+    target_node_label: str = 'GitHubUser'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('MAINTAINER')},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "MAINTAINER"
+    properties: GitHubTeamToUserRelProperties = GitHubTeamToUserRelProperties()
+
+
+@dataclass(frozen=True)
+class GitHubTeamMemberUserRel(CartographyRelSchema):
+    target_node_label: str = 'GitHubUser'
+    target_node_matcher: TargetNodeMatcher = make_target_node_matcher(
+        {'id': PropertyRef('MEMBER')},
+    )
+    direction: LinkDirection = LinkDirection.INWARD
+    rel_label: str = "MEMBER"
+    properties: GitHubTeamToUserRelProperties = GitHubTeamToUserRelProperties()
+
+
 @dataclass(frozen=True)
 class GitHubTeamToOrganizationRelProperties(CartographyRelProperties):
     lastupdated: PropertyRef = PropertyRef('lastupdated', set_in_kwargs=True)
@@ -107,6 +134,8 @@ class GitHubTeamSchema(CartographyNodeSchema):
             GitHubTeamReadRepoRel(),
             GitHubTeamTriageRepoRel(),
             GitHubTeamWriteRepoRel(),
+            GitHubTeamMaintainerUserRel(),
+            GitHubTeamMemberUserRel(),
         ],
     )
     sub_resource_relationship: GitHubTeamToOrganizationRel = GitHubTeamToOrganizationRel()

docs/root/modules/github/schema.md

@@ -129,6 +129,12 @@ A GitHubTeam [organization object](https://docs.github.com/en/graphql/reference/
     (GitHubOrganization)-[RESOURCE]->(GitHubTeam)
     ```
 
+- GitHubUsers may be ['immediate'](https://docs.github.com/en/graphql/reference/enums#teammembershiptype) members of a team (as opposed to being members via membership in a child team), with their membership [role](https://docs.github.com/en/graphql/reference/enums#teammemberrole) being MEMBER or MAINTAINER.
+
+    ```
+    (GitHubUser)-[MEMBER|MAINTAINER]->(GitHubTeam)
+    ```
+
 ### GitHubUser
 
 Representation of a single GitHubUser [user object](https://developer.github.com/v4/object/user/). This node contains minimal data for the GitHub User.
@@ -178,6 +184,13 @@ WRITE, MAINTAIN, TRIAGE, and READ ([Reference](https://docs.github.com/en/graphq
     (GitHubUser)-[MEMBER_OF|UNAFFILIATED]->(GitHubOrganization)
     ```
 
+- GitHubUsers may be ['immediate'](https://docs.github.com/en/graphql/reference/enums#teammembershiptype) members of a team (as opposed to being members via membership in a child team), with their membership [role](https://docs.github.com/en/graphql/reference/enums#teammemberrole) being MEMBER or MAINTAINER.
+
+    ```
+    (GitHubUser)-[MEMBER|MAINTAINER]->(GitHubTeam)
+    ```
+
+
 ### GitHubBranch
 
 Representation of a single GitHubBranch [ref object](https://developer.github.com/v4/object/ref). This node contains minimal data for a repository branch.

tests/data/github/teams.py

@@ -8,30 +8,35 @@
                 'url': 'https://github.com/orgs/example_org/teams/team-a',
                 'description': None,
                 'repositories': {'totalCount': 0},
+                'members': {'totalCount': 0},
             },
             {
                 'slug': 'team-b',
                 'url': 'https://github.com/orgs/example_org/teams/team-b',
                 'description': None,
                 'repositories': {'totalCount': 3},
+                'members': {'totalCount': 0},
             },
             {
                 'slug': 'team-c',
                 'url': 'https://github.com/orgs/example_org/teams/team-c',
                 'description': None,
                 'repositories': {'totalCount': 0},
+                'members': {'totalCount': 3},
             },
             {
                 'slug': 'team-d',
                 'url': 'https://github.com/orgs/example_org/teams/team-d',
                 'description': 'Team D',
                 'repositories': {'totalCount': 0},
+                'members': {'totalCount': 0},
             },
             {
                 'slug': 'team-e',
                 'url': 'https://github.com/orgs/example_org/teams/team-e',
                 'description': 'some description here',
                 'repositories': {'totalCount': 0},
+                'members': {'totalCount': 0},
             },
         ],
         edges=[],
@@ -53,3 +58,16 @@
         {'permission': 'READ'},
     ],
 )
+
+GH_TEAM_USERS = PaginatedGraphqlData(
+    nodes=[
+        {'url': 'https://example.com/hjsimpson'},
+        {'url': 'https://example.com/lmsimpson'},
+        {'url': 'https://example.com/mbsimpson'},
+    ],
+    edges=[
+        {'role': 'MEMBER'},
+        {'role': 'MAINTAINER'},
+        {'role': 'MAINTAINER'},
+    ],
+)

tests/integration/cartography/intel/github/test_teams.py

@@ -4,7 +4,9 @@
 from cartography.intel.github.teams import sync_github_teams
 from tests.data.github.teams import GH_TEAM_DATA
 from tests.data.github.teams import GH_TEAM_REPOS
-from tests.integration.cartography.intel.github.test_repos import _ensure_local_neo4j_has_test_data
+from tests.data.github.teams import GH_TEAM_USERS
+from tests.integration.cartography.intel.github import test_repos
+from tests.integration.cartography.intel.github import test_users
 from tests.integration.util import check_nodes
 from tests.integration.util import check_rels
 
@@ -14,11 +16,13 @@
 FAKE_API_KEY = 'asdf'
 
 
+@patch.object(cartography.intel.github.teams, '_get_team_users', return_value=GH_TEAM_USERS)
 @patch.object(cartography.intel.github.teams, '_get_team_repos', return_value=GH_TEAM_REPOS)
 @patch.object(cartography.intel.github.teams, 'get_teams', return_value=GH_TEAM_DATA)
-def test_sync_github_teams(mock_teams, mock_team_repos, neo4j_session):
+def test_sync_github_teams(mock_teams, mock_team_repos, mock_team_users, neo4j_session):
     # Arrange
-    _ensure_local_neo4j_has_test_data(neo4j_session)
+    test_repos._ensure_local_neo4j_has_test_data(neo4j_session)
+    test_users._ensure_local_neo4j_has_test_data(neo4j_session)
     # Arrange: Add another org to make sure we don't attach a node to the wrong org
     neo4j_session.run('''
         MERGE (g:GitHubOrganization{id: "this should have no attachments"})
@@ -116,3 +120,22 @@ def test_sync_github_teams(mock_teams, mock_team_repos, neo4j_session):
     ) == {
         ('https://github.com/orgs/example_org/teams/team-b', 'https://github.com/lyft/cartography'),
     }
+    assert check_rels(
+        neo4j_session,
+        'GitHubTeam', 'id',
+        'GitHubUser', 'id',
+        'MEMBER',
+        rel_direction_right=False,
+    ) == {
+        ('https://github.com/orgs/example_org/teams/team-c', 'https://example.com/hjsimpson'),
+    }
+    assert check_rels(
+        neo4j_session,
+        'GitHubTeam', 'id',
+        'GitHubUser', 'id',
+        'MAINTAINER',
+        rel_direction_right=False,
+    ) == {
+        ('https://github.com/orgs/example_org/teams/team-c', 'https://example.com/lmsimpson'),
+        ('https://github.com/orgs/example_org/teams/team-c', 'https://example.com/mbsimpson'),
+    }

tests/integration/cartography/intel/github/test_users.py

@@ -1,6 +1,7 @@
 from unittest.mock import patch
 
 import cartography.intel.github.users
+from cartography.models.github.users import GitHubOrganizationUserSchema
 from tests.data.github.users import GITHUB_ENTERPRISE_OWNER_DATA
 from tests.data.github.users import GITHUB_ORG_DATA
 from tests.data.github.users import GITHUB_USER_DATA
@@ -12,6 +13,24 @@
 FAKE_API_KEY = 'asdf'
 
 
+def _ensure_local_neo4j_has_test_data(neo4j_session):
+    """
+    Not needed for this test file, but used to set up users for other tests that need them
+    """
+    processed_affiliated_user_data, _ = (
+        cartography.intel.github.users.transform_users(
+            GITHUB_USER_DATA[0], GITHUB_ENTERPRISE_OWNER_DATA[0], GITHUB_ORG_DATA,
+        )
+    )
+    cartography.intel.github.users.load_users(
+        neo4j_session,
+        GitHubOrganizationUserSchema(),
+        processed_affiliated_user_data,
+        GITHUB_ORG_DATA,
+        TEST_UPDATE_TAG,
+    )
+
+
 @patch.object(cartography.intel.github.users, 'get_users', return_value=GITHUB_USER_DATA)
 @patch.object(cartography.intel.github.users, 'get_enterprise_owners', return_value=GITHUB_ENTERPRISE_OWNER_DATA)
 def test_sync(mock_owners, mock_users, neo4j_session):