Using Kotlin kscript for Preprocessing Data

Ken Yee
Ken Yee
Apr 6, 2018 · 6 min read
{
"events" : {
"-L4b_dz4i6Wh94qh1kkr": {
"description" : "<HTML text>",
"duration" : "PT45M",
"name" : "<title>",
"roomIds": {
"-L5_PsZvWSAv4Lrf-Yo9" : true
},
"speakerIds" : {
"-L4NL8eGr2K_XjDFHiTJ" : true
},
"startTime" : "2018-03-26T15:00:00.000Z",
"trackId" : "-L5___NDDDtAjf0dEO7W"
},
"tracks" : {
"-L4bbBGSqk7ZVv5lfKNV" : {
"description" : "<p><br></p>",
"name" : "Workshops",
"sortOrder" : 2
},
...
},
"speakers" : {
"-L4CAY7WXpAXJm0PYTwH" : {
"bio" : "<p>Lora Kulm is a designer turned developer who has been working with Android for a little over 5 years. She is mostly self-taught with a supplementary Comp Sci 101 class in college, and has worked everywhere from a children’s education app startup to city government. She currently works as an Android developer and is passionate about graphics, animations, and her two dogs.</p>",
"name" : "Lora Kulm",
"org" : "Phunware",
"pictureUrl" : "https://firebasestorage.googleapis.com/v0/b/slidesup-8b9d6.appspot.com/o/confs%2Fdetail%2Fdroidcon-boston-2018%2Fspeakers%2Ff32b0416-59b3-451b-8bd2-879cd8cce082.png?alt=media&token=bb202012-8824-427e-a13f-967fa3f5dbd8",
"socialProfiles" : {
"github" : "senojl",
"linkedIn" : "lora-kulm-a4614569",
"twitter" : "loraj_k"
},
"title" : "Software Engineer"
},
...
},
"rooms" : {
"-L5_PsZvWSAv4Lrf-Yo9" : {
"name" : "Virginia Wimberly Theatre",
"updatedAt" : 1518898744018,
"updatedBy" : "y556KVJSEqPQ4s1isGkLnEc9HlD3"
},
...
}
}
data class ConferenceDataModel(
val events: Map<String, EventModel>,
val rooms: Map<String, RoomModel>,
val speakers: Map<String, SpeakerModel>,
val tracks: Map<String, TrackModel>
)
data class EventModel(
val name: String,
val description: String,
val duration: Duration,
val startTime: Date,
var endTime: Date?,
val roomIds: Map<String, Boolean>?,
val speakerIds: Map<String, Boolean>?,
val trackId: String?,
var roomNames: Map<String, Boolean>?,
var primarySpeakerName: String?,
var speakerNames: Map<String, Boolean>?,
var speakerNameToPhotoUrl: Map<String, String>?,
var speakerNameToOrg: Map<String, String>?,
var trackName: String?,
var trackSortOrder: Int?
)
...
confData?.events?.forEach {
// denormalize speakers
val speakerNames = HashMap<String, Boolean>()
val speakerNameToPhotoUrl = HashMap<String, String>()
val speakerNameToOrg = HashMap<String, String>()
it.value.speakerIds?.forEach {
confData.speakers.get(it.key)?.let {
speakerNames.put(it.name, true)
if (it.pictureUrl != null) {
speakerNameToPhotoUrl.put(it.name, it.pictureUrl)
}
if (it.org != null) {
speakerNameToOrg.put(it.name, it.org)
}
}
}
if
(speakerNames.size > 0) {
it.value.speakerNames = speakerNames
it.value.speakerNameToPhotoUrl = speakerNameToPhotoUrl
it.value.speakerNameToOrg = speakerNameToOrg
...
}
// denormalize rooms
val roomNames = HashMap<String, Boolean>()
it.value.roomIds?.forEach {
confData.rooms.get(it.key)?.let {
roomNames.put(it.name, true)
}
}
if
(roomNames.size > 0) {
it.value.roomNames = roomNames
}
// look up track name
it.value.trackId?.apply {
val
trackInfo = confData.tracks.get(it.value.trackId!!)
trackInfo?.let { track ->
it
.value.trackName = track.name
it
.value.trackSortOrder = track.sortOrder
}
}
// calculate the end time
it.value.endTime = Date.from(it.value.startTime.toInstant().plus(it.value.duration))
}
[
{
"HS": "X",
"First Name": "Stephen",
"Last Name": "Vance",
"Position": "CTO / Innovation Lead (advising)",
"Twitter": "StephenRVance",
"PhotoUrl": "http://www.droidcon-boston.com/wp-content/uploads/2018/02/stephen-vance-520x324.jpg"
},
...
data class CsvVolunteerModel(
@Json(name = "First Name")
val firstName: String,
@Json(name = "Last Name")
val lastName: String,
@Json(name = "Position")
val position: String,
@Json(name = "Email")
val email: String,
@Json(name = "Twitter")
val twitter: String,
@Json(name = "PhotoUrl")
val photoUrl: String?
)
data class VolunteerModel(
val firstName: String,
val lastName: String,
val position: String,
val email: String,
val twitter: String?,
val pictureUrl: String?
) {
constructor(csvModel: CsvVolunteerModel): this(
csvModel.firstName,
csvModel.lastName,
csvModel.position,
csvModel.email,
if ("".equals(csvModel.twitter) || "---".equals(csvModel.twitter)) null else csvModel.twitter,
csvModel.photoUrl
)
}
fun processData(data: List<CsvVolunteerModel>?): Map<Integer, VolunteerModel> {
val volunteerModels = HashMap<Integer, VolunteerModel>()
var volunteerIndex = 0
data?.forEach {
val
volunteerModel = VolunteerModel(it)
volunteerModels.put(Integer(volunteerIndex++), volunteerModel)
}
return volunteerModels
}
#!/usr/bin/env kscript
//DEPS com.squareup.moshi:moshi:1.5.0,com.squareup.moshi:moshi-adapters:1.5.0,com.squareup.moshi:moshi-kotlin:1.5.0
//INCLUDE VolunteerDataModels.kt
//INCLUDE VolunteerDataUtils.kt
import okio.Okio
import java.io.FileInputStream
import com.squareup.moshi.Types
if (args.size != 1) {
System.err.println("Usage: processVolunteers <jsonfile>")
kotlin.system.exitProcess(-1)
}
val inputStream = FileInputStream(args.get(0))
val csvJsonAdapter = VolunteerDataUtils.getCsvVolunteerAdapter()
val csvData = csvJsonAdapter.fromJson(Okio.buffer(Okio.source(inputStream)))
val data = VolunteerDataUtils.processData(csvData)val jsonAdapter = VolunteerDataUtils.getVolunteerAdapter()
val fixedJson = jsonAdapter.toJson(data)
println(fixedJson)

Ken Yee

Written by

Ken Yee

Mobile (Native Android), Backend (Kotlin, Java, Python, Node.js, .Net), Devops (Jenkins, Vagrant, Docker) software engineer. Currently Kotlin All The Things!

Welcome to a place where words matter. On Medium, smart voices and original ideas take center stage - with no ads in sight. Watch
Follow all the topics you care about, and we’ll deliver the best stories for you to your homepage and inbox. Explore
Get unlimited access to the best stories on Medium — and support writers while you’re at it. Just $5/month. Upgrade