Forbedringer og oppgraderinger.

Nå henter alle pagene for sårbarheter pr repository. Før hentet vi bare 100, men nå følger vi altså pager helt ut. Forberinger i logging slik at info-logging kan slåes av for å hente ut bittelitt ytelse. Oppgraderte biblioteker og Kotlin Litt bedre konstanter for styring av f.eks owner i appen. Vi har ikke propertystyring i denne appen.
digipost · Dec 7, 2023 · 9c00895 · 9c00895
1 parent f852db8
commit 9c00895
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 51 deletions.
diff --git a/README.md b/README.md
@@ -10,3 +10,7 @@ kan du laste ned her: [Public Schema](https://docs.github.com/en/graphql/overvie
 
 For å teste ut spørringer, så har GitHub en fin interaktiv "explorer".
 Den finner du her: [GraphQL API Explorer](https://docs.github.com/en/graphql/overview/explorer)
+
+
+### DISCLAIMER
+Dette er laget veldig spesifikt for Digipost.  
diff --git a/pom.xml b/pom.xml
@@ -15,9 +15,9 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <kotlin.code.style>official</kotlin.code.style>
         <kotlin.compiler.jvmTarget>21</kotlin.compiler.jvmTarget>
-        <apollo.version>3.7.0</apollo.version>
-        <jackson.version>2.14.3</jackson.version>
-        <kotlin.version>1.9.20-RC</kotlin.version>
+        <apollo.version>3.8.2</apollo.version>
+        <jackson.version>2.16.0</jackson.version>
+        <kotlin.version>1.9.21</kotlin.version>
         <digipost.base.image>digipost.azurecr.io/infrastructure/docker-openjdk@sha256:92e1c6b7b5fb9b5e2459595ef6b1be9ea5a835ca44914c1c558a8c025da78651</digipost.base.image>
     </properties>
 
@@ -30,24 +30,24 @@
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
-            <version>2.0.3</version>
+            <version>2.0.5</version>
         </dependency>
         <dependency>
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
             <scope>runtime</scope>
-            <version>1.4.4</version>
+            <version>1.4.14</version>
         </dependency>
         <dependency>
             <groupId>org.codehaus.janino</groupId>
             <artifactId>janino</artifactId>
-            <version>3.1.8</version>
+            <version>3.1.9</version>
             <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>net.logstash.logback</groupId>
             <artifactId>logstash-logback-encoder</artifactId>
-            <version>5.2</version>
+            <version>7.4</version>
             <scope>runtime</scope>
         </dependency>
         <dependency>
@@ -77,7 +77,7 @@
         <dependency>
             <groupId>org.jetbrains.kotlin</groupId>
             <artifactId>kotlin-stdlib-jdk8</artifactId>
-            <version>1.7.20</version>
+            <version>1.9.21</version>
         </dependency>
         <dependency>
             <groupId>com.apollographql.apollo3</groupId>
@@ -92,7 +92,7 @@
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
-            <version>2.10</version>
+            <version>2.10.1</version>
         </dependency>
     </dependencies>
 
@@ -103,7 +103,7 @@
             <plugin>
                 <groupId>com.github.aoudiamoncef</groupId>
                 <artifactId>apollo-client-maven-plugin</artifactId>
-                <version>5.0.0</version>
+                <version>7.0.0</version>
                 <executions>
                     <execution>
                         <goals>

diff --git a/src/main/graphql/github/GetVulnerabilityAlertsForRepo.graphql b/src/main/graphql/github/GetVulnerabilityAlertsForRepo.graphql
@@ -1,12 +1,13 @@
-query GetVulnerabilityAlertsForRepo($name: String!, $owner: String!)
+query GetVulnerabilityAlertsForRepo($name: String!, $owner: String!, $after: String)
 {
     repository(name: $name, owner: $owner) {
         languages(first: 1, orderBy: {field: SIZE, direction: DESC}) {
             nodes {
                 name
             }
         }
-        vulnerabilityAlerts(first: 100, states: OPEN) {
+        vulnerabilityAlerts(first: 100, after:$after, states: OPEN) {
+            pageInfo {hasNextPage, endCursor}
             nodes {
                 createdAt
                 dismissedAt

diff --git a/src/main/kotlin/no/digipost/github/monitoring/GithubApiClient.kt b/src/main/kotlin/no/digipost/github/monitoring/GithubApiClient.kt
@@ -18,10 +18,9 @@ class GithubApiClient(private val githubToken: String) {
 
     val client: HttpClient = HttpClient.newBuilder().build()
     private val formatter: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd")
-    private var cache: HashMap<String, Boolean> = hashMapOf()
 
     fun fetchWorkflowRuns(repo: Repository, days: Int): List<WorkflowRun> {
-        return if (cache.getOrPut(repo.name) { hasContainerScanWorkflow(repo) }) {
+        return if (hasContainerScanWorkflow(repo)) {
             fetchRuns(repo, days)
         } else {
             emptyList()

diff --git a/src/main/kotlin/no/digipost/github/monitoring/GithubGraphql.kt b/src/main/kotlin/no/digipost/github/monitoring/GithubGraphql.kt
@@ -7,11 +7,11 @@ import com.github.graphql.client.QueryRepositoriesQuery
 import kotlinx.coroutines.channels.Channel
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
+import okhttp3.internal.immutableListOf
 import okhttp3.internal.toImmutableList
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import java.io.IOException
-import java.time.ZonedDateTime
 
 data class Repos(val all: List<Repository>)
 
@@ -39,7 +39,7 @@ fun fetchAllReposWithVulnerabilities(apolloClient: ApolloClient, githubApiClient
                     val vulnerabilities = getVulnerabilitiesForRepo(apolloClient, r.name)
                     if (vulnerabilities.isNotEmpty()) {
                         r.copy(vulnerabilities = vulnerabilities).let {
-                            logger.info("${vulnerabilities.size} sårbarheter i ${r.name}")
+                            logger.info("{} sårbarheter i {}", vulnerabilities.size, r.name)
                             vulnRepositories[it.name] = vulnerabilities
                         }
                     }
@@ -51,11 +51,13 @@ fun fetchAllReposWithVulnerabilities(apolloClient: ApolloClient, githubApiClient
                             val containerScanStats = getContainerScanStats(githubApiClient, r)
                             if (containerScanStats != null) {
                                 r.copy(containerScanStats = containerScanStats).let {
-                                    logger.info("${r.name} ${if (containerScanStats.passes) "passerer" else "feiler"} containerscan, ${containerScanStats.passPercentage}% suksess siste ${daysToCount} dager (${containerScanStats.numberOfRuns} kjøringer)")
+                                    if(logger.isInfoEnabled) {
+                                        logger.info("${r.name} ${if (containerScanStats.passes) "passerer" else "feiler"} containerscan, ${containerScanStats.passPercentage}% suksess siste ${daysToCount} dager (${containerScanStats.numberOfRuns} kjøringer)")
+                                    }
                                    containerScanRepositories[it.name] = containerScanStats
                                 }
                             } else {
-                                logger.info("${r.name} har ikke containerscan-workflow, skipper")
+                                logger.info("{} har ikke containerscan-workflow, skipper", r.name)
                             }
                         } catch (e: IOException) {
                             logger.warn("IOException ved henting av container scans", e)
@@ -86,29 +88,44 @@ private suspend fun getVulnerabilitiesForRepo(
     apolloClient: ApolloClient,
     name: String
 ): List<Vulnerability> {
-    if (logger.isDebugEnabled) logger.debug("henter sårbarheter for repo $name")
-    val response = apolloClient.query(GetVulnerabilityAlertsForRepoQuery(name, "digipost")).execute()
-
-    val vulnerabilityAlerts = response.data?.repository?.vulnerabilityAlerts?.nodes ?: emptyList()
-    val vulnerabilities = vulnerabilityAlerts.mapNotNull {
-        it?.let {
-            Vulnerability(
-                it.securityVulnerability!!.severity.name,
-                it.createdAt.toString().substring(0, 10),
-                it.securityVulnerability.`package`.name,
-                it.securityVulnerability.advisory.cvss.score,
-                it.securityVulnerability.advisory.identifiers.firstOrNull { identifier -> "CVE" == identifier.type }?.value
-                    ?: "ukjent CVE"
-            )
-        }
-    }.toImmutableList()
+    logger.debug("henter sårbarheter for repo {}", name)
+
+    var cursor: String? = null
+    var hasNext = true
+
+    var allVulnerabilities: List<Vulnerability> = immutableListOf();
+
+    while (hasNext) {
+
+        val response = apolloClient.query(GetVulnerabilityAlertsForRepoQuery(name, GITHUB_OWNER, after = Optional.present(cursor))).execute()
+
+        val vulnerabilityAlerts = response.data?.repository?.vulnerabilityAlerts?.nodes ?: emptyList()
+        val vulnerabilities = vulnerabilityAlerts.mapNotNull {
+            it?.let {
+                Vulnerability(
+                    it.securityVulnerability!!.severity.name,
+                    it.createdAt.toString().substring(0, 10),
+                    it.securityVulnerability.`package`.name,
+                    it.securityVulnerability.advisory.cvss.score,
+                    it.securityVulnerability.advisory.identifiers.firstOrNull { identifier -> "CVE" == identifier.type }?.value
+                        ?: "ukjent CVE"
+                )
+            }
+        }.toImmutableList()
+
+        allVulnerabilities = allVulnerabilities + vulnerabilities
+
+        hasNext = response.data?.repository?.vulnerabilityAlerts?.pageInfo?.hasNextPage ?: false
+
+        cursor = response.data?.repository?.vulnerabilityAlerts?.pageInfo?.endCursor
+    }
 
-    return vulnerabilities
+    return allVulnerabilities
 }
 
 
 private suspend fun listRepos(apolloClient: ApolloClient, repositoryChannel: Channel<Repository>) {
-    if (logger.isDebugEnabled) logger.debug("Henter repoer med owner 'digipost' som ikke er arkiverte og har språk i listen $LANGUAGES")
+    if (logger.isDebugEnabled) logger.debug("Henter repoer med owner '${GITHUB_OWNER}' som ikke er arkiverte og har språk i listen $LANGUAGES")
 
     var cursor: String? = null
     var hasNext = true
@@ -119,7 +136,7 @@ private suspend fun listRepos(apolloClient: ApolloClient, repositoryChannel: Cha
         val response = apolloClient.query(QueryRepositoriesQuery(Optional.Present(cursor))).execute()
 
         response.data?.viewer?.repositories?.nodes
-            ?.filter { "digipost" == it?.owner?.login }
+            ?.filter { GITHUB_OWNER == it?.owner?.login }
             ?.filter { LANGUAGES.contains(it?.languages?.nodes?.firstOrNull()?.name) }
             ?.forEach {
                 it?.let {

diff --git a/src/main/kotlin/no/digipost/github/monitoring/Main.kt b/src/main/kotlin/no/digipost/github/monitoring/Main.kt
@@ -25,6 +25,9 @@ import kotlin.system.measureTimeMillis
 
 val LANGUAGES = setOf("JavaScript", "Java", "TypeScript", "C#", "Kotlin", "Go")
 val POSSIBLE_CONTAINER_SCAN = setOf("JavaScript", "Java", "TypeScript", "Kotlin")
+const val GITHUB_OWNER = "digipost";
+const val TIMOUT_PUBLISH_VULNS = 1000L * 60 * 2
+const val DELAY_BETWEEN_PUBLISH_VULNS = 1000L * 60 * 5
 
 suspend fun main(): Unit = coroutineScope {
     val env = System.getenv("env")
@@ -38,15 +41,15 @@ suspend fun main(): Unit = coroutineScope {
     ApplicationInfoMetrics().bindTo(prometheusMeterRegistry)
 
     val multiGaugeRepoVulnCount = MultiGauge.builder("repository_vulnerability_count")
-        .tags("owner", "digipost")
+        .tags("owner", GITHUB_OWNER)
         .register(prometheusMeterRegistry)
 
     val multiGaugeContainerScan = MultiGauge.builder("repository_container_scan")
-        .tags("owner", "digipost")
+        .tags("owner", GITHUB_OWNER)
         .register(prometheusMeterRegistry)
 
     val multiGaugeInfoScore = MultiGauge.builder("vulnerability_info_score")
-        .tags("owner", "digipost")
+        .tags("owner", GITHUB_OWNER)
         .register(prometheusMeterRegistry)
 
     val apolloClientFactory = cachedApolloClientFactory(token)
@@ -55,7 +58,7 @@ suspend fun main(): Unit = coroutineScope {
     launch {
         while (isActive) {
             try {
-                withTimeout(60_000) {
+                withTimeout(TIMOUT_PUBLISH_VULNS) {
                     val timeMillis = measureTimeMillis {
                         publish(apolloClientFactory.invoke(), githubApiClient, multiGaugeRepoVulnCount, multiGaugeContainerScan, multiGaugeInfoScore)
                     }
@@ -64,7 +67,7 @@ suspend fun main(): Unit = coroutineScope {
             } catch (e: TimeoutCancellationException) {
                 logger.warn("Henting av repos med sårbarheter brukte for lang tid (timeout)")
             }
-            delay(1000 * 60 * 5)
+            delay(DELAY_BETWEEN_PUBLISH_VULNS)
         }
         logger.warn("Hovedjobben er ikke aktiv lenger og avslutter")
     }
@@ -112,11 +115,11 @@ suspend fun publish(apolloClient: ApolloClient, githubApiClient: GithubApiClient
             val onlyVulnerable = all.filter { it.vulnerabilities.isNotEmpty() }
             val onlyContainerScan = all.filter { it.containerScanStats != null }
 
-            logger.info("Antall repos: ${all.size}")
-            logger.info("Antall med sårbarheter: ${onlyVulnerable.size}")
-            logger.info("Antall sårbarheter å rette: ${onlyVulnerable.flatMap { it.vulnerabilities }.count()}")
-            logger.info("Antall som feiler containerscan: ${onlyContainerScan.map { it.containerScanStats?.passes }.count()}")
-            logger.info("Gjennomsnittlig suksess for containerscan: ${onlyContainerScan.mapNotNull { it.containerScanStats?.passPercentage }.average()}%")
+            logger.info("Antall repos: {}", all.size)
+            logger.info("Antall med sårbarheter: {}", onlyVulnerable.size)
+            logger.info("Antall sårbarheter å rette: {}", onlyVulnerable.flatMap { it.vulnerabilities }.count())
+            logger.info("Antall som feiler containerscan: {}", onlyContainerScan.map { it.containerScanStats?.passes }.count())
+            logger.info("Gjennomsnittlig suksess for containerscan: {}%", onlyContainerScan.mapNotNull { it.containerScanStats?.passPercentage }.average())
 
             all.map { repo ->
                 MultiGauge.Row.of(Tags.of("name", repo.name, "language", repo.language), repo.vulnerabilities.size)
@@ -129,10 +132,10 @@ suspend fun publish(apolloClient: ApolloClient, githubApiClient: GithubApiClient
                             "name", repo.name,
                             "language", repo.language,
                             "created", vuln.createdAt,
-                            "CVE", vuln.CVE ?: "",
-                            "packagename", vuln.packageName ?: "UNKNOWN",
-                            "severity", vuln.severity ?: "UNKNOWN",
-                        ), vuln.score ?: 0.0
+                            "CVE", vuln.CVE,
+                            "packagename", vuln.packageName,
+                            "severity", vuln.severity,
+                        ), vuln.score
                     )
                 }
             }.flatMap { it.toList() }.let { registerVulnerabilites.register(it) }