CID-3153: Enhance Synchronisation Logging

config/detekt/detekt.yml

@@ -6,7 +6,7 @@ complexity:
     active: true
     excludes: [ '**/test/**' ]
   LongParameterList:
-    active: true
+    active: false
     excludes: [ '**/test/**' ]
 
 naming:

src/main/kotlin/net/leanix/githubagent/GitHubAgentApplication.kt

@@ -1,5 +1,11 @@
 package net.leanix.githubagent
 
+import jakarta.annotation.PreDestroy
+import net.leanix.githubagent.dto.LogLevel
+import net.leanix.githubagent.dto.SynchronizationProgress
+import net.leanix.githubagent.services.CachingService
+import net.leanix.githubagent.services.SyncLogService
+import org.slf4j.LoggerFactory
 import org.springframework.boot.autoconfigure.SpringBootApplication
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.boot.runApplication
@@ -13,7 +19,33 @@ import org.springframework.cloud.openfeign.EnableFeignClients
         net.leanix.githubagent.config.LeanIXProperties::class
     ]
 )
-class GitHubAgentApplication
+class GitHubAgentApplication(
+    private val syncLogService: SyncLogService,
+    private val cachingService: CachingService
+) {
+
+    private val logger = LoggerFactory.getLogger(GitHubAgentApplication::class.java)
+
+    @PreDestroy
+    fun onShutdown() {
+        sendShutdownLog()
+    }
+
+    private fun sendShutdownLog() {
+        val message = "Agent shutdown."
+        val synchronizationProgress = if (cachingService.get("runId") != null) {
+            SynchronizationProgress.ABORTED
+        } else {
+            SynchronizationProgress.FINISHED
+        }
+        syncLogService.sendSyncLog(
+            message = message,
+            logLevel = LogLevel.INFO,
+            synchronizationProgress = synchronizationProgress
+        )
+        logger.info(message)
+    }
+}
 
 fun main() {
     runApplication<GitHubAgentApplication>()

src/main/kotlin/net/leanix/githubagent/controllers/advice/GlobalExceptionHandler.kt

@@ -19,7 +19,7 @@ class GlobalExceptionHandler(
 
     @ExceptionHandler(InvalidEventSignatureException::class)
     fun handleInvalidEventSignatureException(exception: InvalidEventSignatureException): ProblemDetail {
-        val detail = "Received event with an invalid signature"
+        val detail = "Received an event with an invalid signature."
         val problemDetail = ProblemDetail.forStatusAndDetail(HttpStatus.UNAUTHORIZED, detail)
         problemDetail.title = exception.message
         exceptionLogger.warn(exception.message)
@@ -29,10 +29,21 @@ class GlobalExceptionHandler(
 
     @ExceptionHandler(WebhookSecretNotSetException::class)
     fun handleWebhookSecretNotSetException(exception: WebhookSecretNotSetException): ProblemDetail {
-        val detail = "Unable to process GitHub event. Webhook secret not set"
+        val detail = "Unable to process GitHub event. Webhook secret is not set. " +
+            "Please configure the webhook secret in the agent settings."
         val problemDetail = ProblemDetail.forStatusAndDetail(HttpStatus.BAD_REQUEST, detail)
         problemDetail.title = exception.message
         syncLogService.sendErrorLog(detail)
         return problemDetail
     }
+
+    @ExceptionHandler(Exception::class)
+    fun handleUncaughtException(exception: Exception): ProblemDetail {
+        val detail = "An unexpected error occurred ${exception.message}"
+        val problemDetail = ProblemDetail.forStatusAndDetail(HttpStatus.INTERNAL_SERVER_ERROR, detail)
+        problemDetail.title = exception.message
+        exceptionLogger.error("Uncaught exception: ${exception.message}", exception)
+        syncLogService.sendErrorLog(detail)
+        return problemDetail
+    }
 }

src/main/kotlin/net/leanix/githubagent/dto/SyncLogDto.kt

@@ -6,14 +6,13 @@ data class SyncLogDto(
     val runId: UUID?,
     val trigger: Trigger,
     val logLevel: LogLevel,
-    val message: String?
+    val synchronizationProgress: SynchronizationProgress,
+    val message: String
 )
 
 enum class Trigger {
-    START_FULL_SYNC,
-    PROGRESS_FULL_SYNC,
-    FINISH_FULL_SYNC,
-    SYSTEM
+    FULL_SCAN,
+    WEB_HOOK
 }
 
 enum class LogLevel {
@@ -22,3 +21,11 @@ enum class LogLevel {
     INFO,
     ERROR
 }
+
+enum class SynchronizationProgress {
+    PENDING,
+    RUNNING,
+    ABORTION_PENDING,
+    ABORTED,
+    FINISHED
+}

src/main/kotlin/net/leanix/githubagent/runners/PostStartupRunner.kt

@@ -1,17 +1,21 @@
 package net.leanix.githubagent.runners
 
+import net.leanix.githubagent.dto.LogLevel
+import net.leanix.githubagent.dto.SynchronizationProgress
 import net.leanix.githubagent.handler.BrokerStompSessionHandler
 import net.leanix.githubagent.services.CachingService
 import net.leanix.githubagent.services.GitHubAuthenticationService
 import net.leanix.githubagent.services.GitHubEnterpriseService
 import net.leanix.githubagent.services.GitHubScanningService
+import net.leanix.githubagent.services.SyncLogService
 import net.leanix.githubagent.services.WebSocketService
 import net.leanix.githubagent.shared.APP_NAME_TOPIC
 import org.slf4j.LoggerFactory
 import org.springframework.boot.ApplicationArguments
 import org.springframework.boot.ApplicationRunner
 import org.springframework.context.annotation.Profile
 import org.springframework.stereotype.Component
+import java.util.*
 
 @Component
 @Profile("!test")
@@ -21,7 +25,8 @@ class PostStartupRunner(
     private val gitHubScanningService: GitHubScanningService,
     private val gitHubEnterpriseService: GitHubEnterpriseService,
     private val cachingService: CachingService,
-    private val brokerStompSessionHandler: BrokerStompSessionHandler
+    private val brokerStompSessionHandler: BrokerStompSessionHandler,
+    private val syncLogService: SyncLogService
 ) : ApplicationRunner {
 
     private val logger = LoggerFactory.getLogger(PostStartupRunner::class.java)
@@ -32,6 +37,27 @@ class PostStartupRunner(
             logger.error("Stopping the application as the WebSocket connection could not be established.")
             return
         }
+        kotlin.runCatching {
+            startFullScan()
+            scanResources()
+        }.onSuccess {
+            fullScanSuccess()
+        }.onFailure {
+            fullScanFailure(it.message)
+        }
+    }
+
+    private fun startFullScan() {
+        cachingService.set("runId", UUID.randomUUID(), null)
+        logger.info("Starting full sync")
+        syncLogService.sendSyncLog(
+            logLevel = LogLevel.INFO,
+            synchronizationProgress = SynchronizationProgress.PENDING,
+            message = "Starting synchronization"
+        )
+    }
+
+    private fun scanResources() {
         githubAuthenticationService.generateAndCacheJwtToken()
         val jwt = cachingService.get("jwtToken") as String
         webSocketService.sendMessage(
@@ -40,4 +66,26 @@ class PostStartupRunner(
         )
         gitHubScanningService.scanGitHubResources()
     }
+
+    private fun fullScanSuccess() {
+        syncLogService.sendSyncLog(
+            logLevel = LogLevel.INFO,
+            synchronizationProgress = SynchronizationProgress.FINISHED,
+            message = "Synchronization finished."
+        )
+        cachingService.remove("runId")
+        logger.info("Full sync finished")
+    }
+
+    private fun fullScanFailure(errorMessage: String?) {
+        val message = "Synchronization aborted. " +
+            "An error occurred while scanning GitHub resources. Error: $errorMessage"
+        syncLogService.sendSyncLog(
+            logLevel = LogLevel.ERROR,
+            synchronizationProgress = SynchronizationProgress.ABORTED,
+            message = message
+        )
+        cachingService.remove("runId")
+        logger.error(message)
+    }
 }

src/main/kotlin/net/leanix/githubagent/services/GitHubAuthenticationService.kt

@@ -6,7 +6,6 @@ import net.leanix.githubagent.client.GitHubClient
 import net.leanix.githubagent.config.GitHubEnterpriseProperties
 import net.leanix.githubagent.dto.Installation
 import net.leanix.githubagent.exceptions.FailedToCreateJWTException
-import net.leanix.githubagent.exceptions.GitHubAppInsufficientPermissionsException
 import org.bouncycastle.jce.provider.BouncyCastleProvider
 import org.slf4j.LoggerFactory
 import org.springframework.core.io.ResourceLoader
@@ -28,8 +27,7 @@ class GitHubAuthenticationService(
     private val githubEnterpriseProperties: GitHubEnterpriseProperties,
     private val resourceLoader: ResourceLoader,
     private val gitHubEnterpriseService: GitHubEnterpriseService,
-    private val gitHubClient: GitHubClient,
-    private val syncLogService: SyncLogService,
+    private val gitHubClient: GitHubClient
 ) {
 
     companion object {
@@ -59,12 +57,6 @@ class GitHubAuthenticationService(
             gitHubEnterpriseService.verifyJwt(jwt.getOrThrow())
             cachingService.set("jwtToken", jwt.getOrThrow(), JWT_EXPIRATION_DURATION)
         }.onFailure {
-            logger.error("Failed to generate/validate JWT token", it)
-            if (it is GitHubAppInsufficientPermissionsException) {
-                syncLogService.sendSystemErrorLog(it.message.toString())
-            } else {
-                syncLogService.sendSystemErrorLog("Failed to generate/validate JWT token")
-            }
             if (it is InvalidKeySpecException) {
                 throw IllegalArgumentException("The provided private key is not in a valid PKCS8 format.", it)
             } else {

src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt

@@ -3,20 +3,17 @@ package net.leanix.githubagent.services
 import net.leanix.githubagent.client.GitHubClient
 import net.leanix.githubagent.dto.Installation
 import net.leanix.githubagent.dto.ItemResponse
-import net.leanix.githubagent.dto.LogLevel
 import net.leanix.githubagent.dto.ManifestFileDTO
 import net.leanix.githubagent.dto.ManifestFilesDTO
 import net.leanix.githubagent.dto.Organization
 import net.leanix.githubagent.dto.OrganizationDto
 import net.leanix.githubagent.dto.RepositoryDto
-import net.leanix.githubagent.dto.Trigger
 import net.leanix.githubagent.exceptions.JwtTokenNotFound
 import net.leanix.githubagent.exceptions.ManifestFileNotFoundException
 import net.leanix.githubagent.shared.MANIFEST_FILE_NAME
 import net.leanix.githubagent.shared.fileNameMatchRegex
 import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Service
-import java.util.UUID
 
 @Service
 class GitHubScanningService(
@@ -31,40 +28,17 @@ class GitHubScanningService(
     private val logger = LoggerFactory.getLogger(GitHubScanningService::class.java)
 
     fun scanGitHubResources() {
-        cachingService.set("runId", UUID.randomUUID(), null)
-        runCatching {
-            syncLogService.sendSyncLog(
-                trigger = Trigger.START_FULL_SYNC,
-                logLevel = LogLevel.INFO,
-            )
-            val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound()
-            val installations = getInstallations(jwtToken.toString())
-            fetchAndSendOrganisationsData(installations)
-            installations.forEach { installation ->
-                fetchAndSendRepositoriesData(installation)
-                    .forEach { repository ->
-                        fetchManifestFilesAndSend(installation, repository)
-                    }
-                syncLogService.sendInfoLog(
-                    "Finished initial full scan for organization ${installation.account.login}"
-                )
-            }
-            syncLogService.sendInfoLog("Finished full scan for all available organizations")
-            syncLogService.sendSyncLog(
-                trigger = Trigger.FINISH_FULL_SYNC,
-                logLevel = LogLevel.INFO,
-            )
-        }.onFailure {
-            val message = "Error while scanning GitHub resources"
-            syncLogService.sendSyncLog(
-                trigger = Trigger.FINISH_FULL_SYNC,
-                logLevel = LogLevel.ERROR,
-                message = message
-            )
-            cachingService.remove("runId")
-            logger.error(message)
-            throw it
+        val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound()
+        val installations = getInstallations(jwtToken.toString())
+        fetchAndSendOrganisationsData(installations)
+        installations.forEach { installation ->
+            fetchAndSendRepositoriesData(installation)
+                .forEach { repository ->
+                    fetchManifestFilesAndSend(installation, repository)
+                }
+            syncLogService.sendInfoLog("Finished initial full scan for organization ${installation.account.login}.")
         }
+        syncLogService.sendInfoLog("Finished full scan for all available organizations.")
     }
 
     private fun getInstallations(jwtToken: String): List<Installation> {
@@ -93,9 +67,8 @@ class GitHubScanningService(
         logger.info("Sending organizations data")
         syncLogService.sendInfoLog(
             "The connector found ${organizations.filter { it.installed }.size} " +
-                "organizations with GitHub application installed."
+                "organizations with GitHub application installed, out of possible ${organizations.size}."
         )
-        syncLogService.sendInfoLog("The connector found ${organizations.size} available organizations.")
         webSocketService.sendMessage("${cachingService.get("runId")}/organizations", organizations)
     }
 
@@ -151,7 +124,7 @@ class GitHubScanningService(
         repositoryName: String
     ) = runCatching {
         val installationToken = cachingService.get("installationToken:${installation.id}").toString()
-        syncLogService.sendInfoLog("Scanning repository $repositoryName for manifest files")
+        syncLogService.sendInfoLog("Scanning repository $repositoryName for manifest files.")
         var numOfManifestFilesFound = 0
         items.map { manifestFile ->
             val content = gitHubGraphQLService.getManifestFileContent(
@@ -162,7 +135,9 @@ class GitHubScanningService(
             )
             if (content != null) {
                 numOfManifestFilesFound++
-                syncLogService.sendInfoLog("Fetched manifest file ${manifestFile.path} from repository $repositoryName")
+                syncLogService.sendInfoLog(
+                    "Fetched manifest file '${manifestFile.path}' from repository '$repositoryName'."
+                )
                 ManifestFileDTO(
                     path = fileNameMatchRegex.replace(manifestFile.path, ""),
                     content = content
@@ -171,7 +146,7 @@ class GitHubScanningService(
                 throw ManifestFileNotFoundException()
             }
         }.also {
-            syncLogService.sendInfoLog("Found $numOfManifestFilesFound manifest files in repository $repositoryName")
+            syncLogService.sendInfoLog("Found $numOfManifestFilesFound manifest files in repository $repositoryName.")
         }
     }
 }

src/main/kotlin/net/leanix/githubagent/services/SyncLogService.kt

@@ -2,6 +2,7 @@ package net.leanix.githubagent.services
 
 import net.leanix.githubagent.dto.LogLevel
 import net.leanix.githubagent.dto.SyncLogDto
+import net.leanix.githubagent.dto.SynchronizationProgress
 import net.leanix.githubagent.dto.Trigger
 import net.leanix.githubagent.shared.LOGS_TOPIC
 import org.springframework.stereotype.Service
@@ -13,23 +14,25 @@ class SyncLogService(
     private val cachingService: CachingService
 ) {
     fun sendErrorLog(message: String) {
-        sendSyncLog(message, LOGS_TOPIC, Trigger.PROGRESS_FULL_SYNC, LogLevel.ERROR)
-    }
-
-    fun sendSystemErrorLog(message: String) {
-        sendSyncLog(message, LOGS_TOPIC, Trigger.SYSTEM, LogLevel.ERROR)
+        sendSyncLog(message, LOGS_TOPIC, LogLevel.ERROR, SynchronizationProgress.RUNNING)
     }
 
     fun sendInfoLog(message: String) {
-        sendSyncLog(message, LOGS_TOPIC, Trigger.PROGRESS_FULL_SYNC, LogLevel.INFO)
+        sendSyncLog(message, LOGS_TOPIC, LogLevel.INFO, SynchronizationProgress.RUNNING)
     }
 
-    fun sendSyncLog(message: String? = null, topic: String = LOGS_TOPIC, trigger: Trigger, logLevel: LogLevel) {
+    fun sendSyncLog(
+        message: String,
+        topic: String = LOGS_TOPIC,
+        logLevel: LogLevel,
+        synchronizationProgress: SynchronizationProgress
+    ) {
         val runId = cachingService.get("runId")?.let { it as UUID }
         val syncLogDto = SyncLogDto(
             runId = runId,
-            trigger = trigger,
+            trigger = if (runId != null) Trigger.FULL_SCAN else Trigger.WEB_HOOK,
             logLevel = logLevel,
+            synchronizationProgress = synchronizationProgress,
             message = message
         )
         webSocketService.sendMessage(constructTopic(topic), syncLogDto)