How to Fetch Logs Using Graylog REST API with Kotlin and Spring Boot

Overview

• Graylog is an open-source log monitoring solution with a long history. While the Web Interface is commonly used, utilizing the API allows for various purposes such as secondary processing of log data, aggregation, and alerting. This post summarizes how to retrieve logs using the Graylog REST API in Kotlin and Spring Boot.

build.gradle.kts

• Create a Spring Boot-based project and add the following libraries:

dependencies {
    implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
    implementation("com.squareup.okhttp3:okhttp:5.0.0-alpha.14")
}

Creating JsonConfig

• Create an ObjectMapper bean that will convert responses from the Graylog REST API into DTOs.

@Configuration
class JsonConfig {

    @Bean("objectMapper")
    @Primary
    fun objectMapper(): ObjectMapper {

        return Jackson2ObjectMapperBuilder
            .json()
            .serializationInclusion(JsonInclude.Include.ALWAYS)
            .failOnEmptyBeans(false)
            .failOnUnknownProperties(false)
            .featuresToDisable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
            .modulesToInstall(kotlinModule(), JavaTimeModule())
            .build()
    }
}

Creating OkHttpConfig

• Create an OkHttpClient bean to make requests to the Graylog REST API.

@Configuration
class OkHttpConfig {

    @Bean("okHttpClient")
    fun okHttpClient(): OkHttpClient {

        return OkHttpClient()
            .newBuilder().apply {
                // Use virtual threads for better performance
                dispatcher(Dispatcher(Executors.newVirtualThreadPerTaskExecutor()))
                // Configure connection specs for both cleartext and TLS
                connectionSpecs(
                    listOf(
                        ConnectionSpec.CLEARTEXT,
                        ConnectionSpec.Builder(ConnectionSpec.MODERN_TLS)
                            .allEnabledTlsVersions()
                            .allEnabledCipherSuites()
                            .build()
                    )
                )
                // Set timeouts
                connectTimeout(10, TimeUnit.SECONDS)
                writeTimeout(10, TimeUnit.SECONDS)
                readTimeout(10, TimeUnit.SECONDS)
            }.build()
    }
}

Creating GraylogSearchService

• Create a GraylogSearchService to query log lists from Graylog.

/**
 * Service class for interacting with the Graylog REST API.
 * Provides functionality to fetch both metrics and message logs.
 */
@Service
class GraylogSearchService(
    private val objectMapper: ObjectMapper,
    private val okHttpClient: OkHttpClient
) {
    /**
     * Fetches metrics from Graylog using the Views API.
     * Supports different metric types (COUNT, MIN, MAX, AVG) with time-based grouping.
     *
     * @param from Start time for the search
     * @param to End time for the search
     * @param metricRequest Contains metric type, field, and interval settings
     * @param query Elasticsearch query string
     * @param graylogUrl Base URL of the Graylog server
     * @param username Graylog username for authentication
     * @param password Graylog password for authentication
     * @return GraylogMetricResponseDTO containing the metric results
     */
    fun fetchMetrics(
        from: Instant,
        to: Instant,
        metricRequest: GraylogMetricRequestDTO,
        query: String = "",
        graylogUrl: String,
        username: String,
        password: String
    ): GraylogMetricResponseDTO {
        val dateTimeFormatter: DateTimeFormatter = DateTimeFormatter
            .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
            .withZone(ZoneOffset.UTC)

        // Construct series JSON based on metric type
        val seriesJson = when (metricRequest.metricType) {
            GraylogMetricType.COUNT -> """
                {
                    "type": "count",
                    "id": "count"
                }
            """.trimIndent()
            else -> """
                {
                    "type": "${metricRequest.metricType.name.lowercase()}",
                    "field": "${metricRequest.field}",
                    "id": "${metricRequest.metricType.name.lowercase()}"
                }
            """.trimIndent()
        }

        // Construct the request body for the Views API
        val requestBody = """
            {
              "queries": [{
                "timerange": {
                  "type": "absolute",
                  "from": "${dateTimeFormatter.format(from)}",
                  "to": "${dateTimeFormatter.format(to)}"
                },
                "query": {
                  "type": "elasticsearch",
                  "query_string": "$query"
                },
                "search_types": [{
                  "type": "pivot",
                  "id": "metric_result",
                  "series": [$seriesJson],
                  "rollup": true,
                  "row_groups": [{
                    "type": "time",
                    "field": "timestamp",
                    "interval": "${metricRequest.interval}"
                  }]
                }]
              }]
            }
        """.trimIndent()

        val request = Request.Builder()
            .url("$graylogUrl/api/views/search/sync")
            .header("Content-Type", "application/json")
            .header("X-Requested-By", "kotlin-client")
            .header("Authorization", Credentials.basic(username, password))
            .post(requestBody.toRequestBody("application/json".toMediaType()))
            .build()

        val response = okHttpClient.newCall(request).execute()
        if (!response.isSuccessful) {
            throw RuntimeException("Failed to fetch metrics: ${response.code}")
        }

        return objectMapper.readValue(response.body.string(), GraylogMetricResponseDTO::class.java)
    }

    /**
     * Fetches log messages from Graylog using the Search API.
     *
     * @param from Start time for the search
     * @param to End time for the search
     * @param query Elasticsearch query string
     * @param limit Maximum number of messages to return
     * @param graylogUrl Base URL of the Graylog server
     * @param username Graylog username for authentication
     * @param password Graylog password for authentication
     * @return GraylogMessageDTO containing the search results
     */
    fun fetchMessages(
        from: Instant,
        to: Instant,
        query: String,
        limit: Int = 100,
        graylogUrl: String,
        username: String,
        password: String,
    ): GraylogMessageDTO {
        val url = buildUrl(graylogUrl, from, to, query, limit)
        val request = buildRequest(url, username, password)

        val response = okHttpClient.newCall(request).execute()
        val responseBody = response.body.string()

        if (!response.isSuccessful) {
            throw RuntimeException("Graylog API request failed: ${response.code}")
        }

        return objectMapper.readValue(responseBody, GraylogMessageDTO::class.java)
    }

    /**
     * Builds the URL for the Graylog Search API request
     */
    private fun buildUrl(
        graylogUrl: String,
        from: Instant = Instant.now().minusSeconds(60),
        to: Instant = Instant.now(),
        query: String,
        limit: Int
    ): String {
        val dateTimeFormatter: DateTimeFormatter = DateTimeFormatter
            .ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")
            .withZone(ZoneOffset.UTC)

        return "$graylogUrl/api/search/universal/absolute?" +
                "from=${dateTimeFormatter.format(from)}&" +
                "to=${dateTimeFormatter.format(to)}&" +
                "query=$query&" +
                "limit=$limit&" +
                "pretty=true"
    }

    /**
     * Builds the HTTP request with appropriate headers and authentication
     */
    private fun buildRequest(url: String, username: String, password: String): Request {
        return Request.Builder()
            .url(url)
            .header("Accept", "application/json")
            .header("Authorization", Credentials.basic(username, password))
            .build()
    }
}

/**
 * Supported metric types for Graylog queries
 */
enum class GraylogMetricType {
    COUNT, MIN, MAX, AVG
}

/**
 * DTO for metric request parameters
 */ data class GraylogMetricRequestDTO(
     val field: String,
     val metricType: GraylogMetricType,
     val interval: String = "1h" // Default 1 hour
 )

 data class GraylogMetricResponseDTO(
     val execution: ExecutionInfo,
     val results: Map<String, SearchResult>,
     val id: String,
     @JsonProperty("search_id")
     val searchId: String,
     val owner: String,
     @JsonProperty("executing_node")
     val executingNode: String
 ) {
     fun extractTimeValuePairs(): List<Pair<String, Double>> {
         return results.values
             .firstOrNull()
             ?.searchTypes
             ?.get("metric_result")
             ?.rows
             ?.filter { it.source == "leaf" }
             ?.map { row ->
                 Pair(
                     row.key.firstOrNull() ?: "",
                     row.values.firstOrNull()?.value ?: 0.0
                 )
             }
             ?: emptyList()
     }

     data class ExecutionInfo(
         val done: Boolean,
         val cancelled: Boolean,
         @JsonProperty("completed_exceptionally")
         val completedExceptionally: Boolean
     )

     data class SearchResult(
         val query: Query,
         @JsonProperty("execution_stats")
         val executionStats: ExecutionStats?,
         @JsonProperty("search_types")
         val searchTypes: Map<String, SearchTypeResult>,
         val errors: List<Any>,
         val state: String
     )

     data class ExecutionStats(
         val duration: Long,
         val timestamp: String,
         @JsonProperty("effective_timerange")
         val effectiveTimerange: TimeRange
     )

     data class Query(
         val id: String,
         val timerange: TimeRange,
         val filter: Filter,
         val filters: List<Any>,
         val query: QueryInfo,
         @JsonProperty("search_types")
         val searchTypes: List<SearchType>?
     )

     data class Filter(
         val type: String,
         val filters: List<StreamFilter>
     )

     data class StreamFilter(
         val type: String,
         val id: String
     )

     data class QueryInfo(
         val type: String?,
         @JsonProperty("query_string")
         val queryString: String?
     )

     data class SearchType(
         val timerange: TimeRange?,
         val query: QueryInfo?,
         val streams: List<Any>,
         val id: String,
         val name: String?,
         val series: List<Series>,
         val sort: List<Any>,
         val rollup: Boolean,
         val type: String,
         @JsonProperty("row_groups")
         val rowGroups: List<RowGroup>,
         @JsonProperty("column_groups")
         val columnGroups: List<Any>,
         val filter: Any?,
         val filters: List<Any>
     )

     data class Series(
         val type: String,
         val id: String,
         val field: String?,
         @JsonProperty("whole_number")
         val wholeNumber: Boolean?
     )

     data class RowGroup(
         val type: String,
         val fields: List<String>,
         val interval: Interval
     )

     data class Interval(
         val type: String,
         val timeunit: String
     )

     data class TimeRange(
         val from: String,
         val to: String,
         val type: String
     )

     data class SearchTypeResult(
         val id: String,
         val rows: List<Row>,
         val total: Long,
         val type: String,
         @JsonProperty("effective_timerange")
         val effectiveTimerange: TimeRange
     )

     data class Row(
         val key: List<String>,
         val values: List<Value>,
         val source: String
     ) {
         data class Value(
             val key: List<String>,
             val value: Double,
             val rollup: Boolean,
             val source: String
         )
     }
}

data class GraylogMessageDTO(
    val query: String?,
    val builtQuery: String?,
    val usedIndices: List<String>?,
    val messages: List<Message>,
    val fields: List<String>,
    val time: Long?,
    val totalResults: Long?,
    val from: String?,
    val to: String?
) {
    data class Message(
        val highlightRanges: Map<String, Any>?,
        val message: Map<String, Any>,
        val index: String?,
        val decorationStats: Any?
    )
}

Usage Example

• You can use the GraylogSearchService#fetchMessages method to query logs at the application level as follows:

// Retrieve error logs from the last minute
val log = graylogSearchService.fetchMessages(
    from = Instant.now().minusSeconds(60),
    to = Instant.now(),
    query = "log_level:ERROR",
    graylogUrl = "https://{your-graylog-domain}",
    username = "{your-graylog-username}",
    password = "{your-graylog-password}"
)

// Print log messages
log.messages.forEach {
    println(it)
}