How to search JSON text and objects using Redis, Ktor and Kotlin
4 min readAug 25, 2023
This short article will show how to implement text searches in Kotlin using the Ktor web framework. Most of the information is in the comments and in the code.
Here is the github project for reference: https://github.com/realityexpander/com.realityexpander.ktor-redis-text-search
This assumes you have `redis-stack` already installed & running on your machine. Here’s a quick start:
# in terminal:
brew tap redis-stack/redis-stack
brew install redis-stack
redis-stack-server Note: redis-stack-servermust be running for the Ktor App to work.
Dependencies
- Add these to your Ktor project (default Ktor project will work)
// build.gradle.kts
dependencies {
// ... normal Ktor gradle dependencies ...
// Lettuce for Redis
implementation("io.lettuce:lettuce-core:6.2.6.RELEASE")
implementation("com.redis:lettucemod:3.6.3") // Extension library
}Application — uses the Ktor framework
// Application.kt
fun Application.module() {
val redisClient: RedisModulesClient = RedisModulesClient.create("redis://localhost:6379")
val redisConnection: StatefulRedisModulesConnection<String, String> = redisClient.connect()
val redisSyncCommand: RedisModulesCommands<String, String> = redisConnection.sync()
val redisCoroutineCommand = redisConnection.coroutines()
val redisReactiveCommand = redisConnection.reactive()
// setup the search commands not included in libraries
val redisSearchCommands = RedisCommandFactory(redisConnection).getCommands(RedisSearchCommands::class.java)
// allow one character strings for FT.SEARCH
redisSearchCommands.ftConfigSet("MINPREFIX", "1")
// Build the JSON Text search indexes
try {
// check if index exists
val result = redisSyncCommand.ftInfo("users_index")
} catch (e: Exception) {
// setup json text search index
val result = redisSyncCommand.ftCreate(
"users_index",
CreateOptions.builder<String, String>()
.prefix("user:")
.on(CreateOptions.DataType.JSON)
.build(),
Field.tag("$.id") // note: TAGs do not separate words/special characters
.`as`("id")
.build(),
Field.tag("$.email")
.`as`("email")
.build(),
Field.text("$.name")
.`as`("name")
.sortable()
.withSuffixTrie() // for improved search (go -> going, goes, gone)
.build()
)
if (result != "OK") {
ktorLogger.error("Error creating index: $result")
}
}
// ...application continues below...Run some basic tests and show operations
val resultRedisAdd1 = redisSyncCommand.jsonSet(
"user:1",
"$", // path
"""
{
"id": "00000000-0000-0000-0000-000000000001",
"email": "chris@alpha.com",
"name": "Chris"
}
""".trimIndent()
)
println("resultRedisAdd1: $resultRedisAdd1")
val resultRedisAdd2 = redisSyncCommand.jsonSet(
"user:2",
"$",
"""
{
"id": "00000000-0000-0000-0000-000000000002",
"email": "billy@beta.com",
"name": "Billy"
}
""".trimIndent()
)
println("resultRedisAdd2: $resultRedisAdd2")
val escapedSearchId = "0000-000000000001".escapeRedisSearchSpecialCharacters()
val resultIdSearch = redisSyncCommand.ftSearch(
"users_index",
"@id:{*$escapedSearchId*}" // search for '0000-000000000001' in id
)
println("resultIdSearch: $resultIdSearch")
val resultTagSearch = redisSyncCommand.ftSearch(
"users_index",
"@email:{*ch*}" // search for 'ch' in email, note use curly-braces for TAG type
)
println("resultTagSearch: $resultTagSearch")
val resultTextSearch = redisSyncCommand.ftSearch(
"users_index",
"@name:*bi*" // search for 'bi' in name, note NO curly-braces for TEXT type
)
println("resultTextSearch: $resultTextSearch")
@Serializable
data class UserSearchResult(
val id: String,
val email: String,
val name: String,
)
val resultArray = resultTagSearch.map { resultMap ->
val resultValue = resultMap.get("$") as String
jsonConfig.decodeFromString<UserSearchResult>(resultValue)
}
println("resultArray: $resultArray")
// ...application continues below...Setup the Ktor application web server to respond to http requests for Redis content
routing {
route("/redis") {
get("/keys") {
val keys = redisCoroutineCommand.keys("*")
val output: ArrayList<String> = arrayListOf()
keys.collect { key ->
output += key
}
call.respondJson(mapOf("keys" to output.toString()))
}
get("/jsonGet") {
val key = call.request.queryParameters["key"]
key ?: run {
call.respondJson(mapOf("error" to "Missing key"), HttpStatusCode.BadRequest)
return@get
}
val paths = call.request.queryParameters["paths"] ?: "$"
val value = redisReactiveCommand.jsonGet(key, paths) ?: run {
call.respondJson(mapOf("error" to "Key not found"), HttpStatusCode.NotFound)
return@get
}
call.respondJson(mapOf("key" to key, "value" to (value.block()?.toString() ?: "null")))
}
get("/jsonSet") {
val key = call.request.queryParameters["key"] ?: run {
call.respondJson(mapOf("error" to "Missing key"), HttpStatusCode.BadRequest)
return@get
}
val paths = call.request.queryParameters["paths"] ?: "$"
val value = call.request.queryParameters["value"] ?: run {
call.respondJson(mapOf("error" to "Missing value"), HttpStatusCode.BadRequest)
return@get
}
val result = redisReactiveCommand.jsonSet( key, paths, value) ?: run {
call.respondJson(mapOf("error" to "Failed to set key"), HttpStatusCode.InternalServerError)
return@get
}
call.respondJson(mapOf("success" to Json.encodeToString(result.block())))
}
get("/jsonFind") {
val index = call.request.queryParameters["index"] ?: run {
call.respondJson(mapOf("error" to "Missing index"), HttpStatusCode.BadRequest)
return@get
}
val query = call.request.queryParameters["query"] ?: run {
call.respondJson(mapOf("error" to "Missing query"), HttpStatusCode.BadRequest)
return@get
}
val result =
redisSyncCommand.ftSearch(
index,
query,
SearchOptions.builder<String, String>()
.limit(0, 100)
.withSortKeys(true)
.build()
) ?: run {
call.respondJson(mapOf("error" to "Failed to find key"), HttpStatusCode.InternalServerError)
return@get
}
val searchResults: List<Map<String, String>> =
result.map { document ->
document.keys.map { key ->
key to document.get(key).toString()
}.toMap()
}
call.respondJson(mapOf("success" to Json.encodeToString(searchResults)))
}
}
}
}
fun String.escapeRedisSearchSpecialCharacters(): String {
val escapeChars =
"""
,.<>{}[]"':;!@#$%^&*()-+=~"
""".trimIndent()
var result = this
escapeChars.forEach {
result = result.replace(it.toString(), "\\$it")
}
return result
}
suspend fun ApplicationCall.respondJson(
map: Map<String, String> = mapOf(),
status: HttpStatusCode = HttpStatusCode.OK
) {
respondText(jsonConfig.encodeToString(map), ContentType.Application.Json, status)
}
// end of application.This configures the mapping from JVM to Redis Text commands. Its a strange syntax, but it’s how it works.
// RedisSearchCommands.kt
// Define the Redis Search Commands
// Yes, its odd that we have to define the commands this way, but it's how it works.
interface RedisSearchCommands : Commands {
@Command("FT.CREATE")
@CommandNaming(strategy = CommandNaming.Strategy.DOT)
fun ftCreate(index: String, vararg args: String): String
@Command("FT.CONFIG SET")
@CommandNaming(strategy = CommandNaming.Strategy.DOT)
fun ftConfigSet(key: String, value: String): String
}Start the Ktor server and using a browser, the app will respond to these commands:
In Browser:
- To get json object in redis database
- http://localhost:8081/redis/jsonGet?key={keyId}&paths={paths}
- Example:
- http://localhost:8081/redis/jsonGet?key=user:1&paths=$
- http://localhost:8081/redis/jsonGet?key=user:1&paths=.name
- To set json object in redis database
- http://localhost:8081/redis/jsonSet?key={keyId}&paths={paths}&value={value}
- Example
- http://localhost:8081/redis/jsonSet?key=user:1&paths=$.name&value=Jimmy
- To query json object fields
- http://localhost:8081/redis/jsonFind?index={index}&query=@{field}:{searchQuery}
- Example
- http://localhost:8081/redis/jsonFind?index=users_index&query=%27@name:bil*%27
- To Dump all keys in Redis database
- http://localhost:8081/redis/keysExample code on GitHub:
https://github.com/realityexpander/com.realityexpander.ktor-redis-text-search
