Microservice Usage Logging with Openresty and Google BigQuery

Justin Poehnelt
Apr 21, 2017 · 3 min read
log_by_lua_block {
local payload = {
bytes = tonumber(ngx.var.bytes_sent),
status = tonumber(ngx.var.status),
timestamp = ngx.now()
... -- more values
}
-- post payload to favorite backend for timeseries analysis
}
lua_shared_dict usage_logging 10m;init_worker_by_lua_block {
local Logging = require "descarteslabs.logging"
l = Logging.new()
l:watch(ngx.shared.usage_logging)
}
location = /test {
proxy_pass service;
log_by_lua_block {
local payload = {
bytes = tonumber(ngx.var.bytes_sent),
status = tonumber(ngx.var.status),
timestamp = ngx.now()
... -- more values
}
local Logging = require "descarteslabs.logging"
l = Logging.save(ngx.shared.usage_logging, payload)
}
}
local check_function
check_function = function(premature)
if not premature then
while
true do
local
requests = self:get(size)
pcall(_M.save, self, rows)

if #rows < size then
break
end
end

local
ok, err = ngx.timer.at(delay(), check_function)
if not ok then
log(ERR, "failed to create timer: ", err)
return
end
end
end
SELECT 
COUNT(*) as calls,
SUM(bytes) as bytes,
DATETIME_TRUNC(DATETIME(timestamp), `second` ) as w
FROM `project.dataset.table`
WHERE
status=200
GROUP BY
w
ORDER BY
w desc
LIMIT 100
Google BigQuery Results Using Vegeta for Load Testing

Thanks to Marguerite Oneto

Justin Poehnelt

Written by

Software Engineer at Descartes Labs

Welcome to a place where words matter. On Medium, smart voices and original ideas take center stage - with no ads in sight. Watch
Follow all the topics you care about, and we’ll deliver the best stories for you to your homepage and inbox. Explore
Get unlimited access to the best stories on Medium — and support writers while you’re at it. Just $5/month. Upgrade