Serverless Dynamic Real-Time Dashboard with AWS DynamoDB, S3 and Cognito

The near-real time dashboard in the RAVEN (Reporting, Analytics, Visualization, Experimental, Networks) platform on Amazon Web Services.

1. Website and Application Hosting

Static Website Hosting on AWS S3

Dependencies for the near-real time table dashboard

[ ... ]
<script src="https://sdk.amazonaws.com/js/aws-sdk-2.2.19.min.js"></script>
<script src="js/raven-table-dashboard.js"></script>
[ ... ]

Dependencies for the near-real time line chart dashboard

[ ... ]
<script src="https://sdk.amazonaws.com/js/aws-sdk-2.2.19.min.js"></script>
<script src="js/Chart.js"></script>
<script src="js/raven-chart-dashboard.js"></script>
[ ... ]
<canvas id="canvas" width="400" height="400"></canvas>
[ ... ]
AWS S3 Permissions
aws s3 cp $source_file s3://$target_bucket/$target_path
aws s3api put-object-acl — bucket $target_bucket — key $target_path — grant-read uri=http://acs.amazonaws.com/groups/global/AuthenticatedUsers

2. Security Layer

S3 Bucket IP Restrictions in AWS Console

{
"Version": "2012-10-17",
"Id": "S3PolicyIPRestrict",
"Statement": [
{
"Sid": "IPAllow",
"Effect": "Allow",
"Principal": {
"AWS": "*"
},
"Action": [
"s3:List*",
"s3:GetObject"
],
"Resource": "arn:aws:s3:::myjgbucket/*",
"Condition": {
"IpAddress": {
"aws:SourceIp": [
"1.1.1.1/32",
"2.2.2.2/32"
]
}
}
}
]
}
<?xml version="1.0" encoding="UTF-8"?>
<CORSConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<CORSRule>
<AllowedOrigin>https://s3-eu-west-1.amazonaws.com</AllowedOrigin>
<AllowedMethod>GET</AllowedMethod>
<MaxAgeSeconds>3000</MaxAgeSeconds>
<AllowedHeader>Authorization</AllowedHeader>
</CORSRule>
</CORSConfiguration>

Cognito Setup in Console

{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"mobileanalytics:PutEvents",
"cognito-sync:*"
],
"Resource": [
"*"
]
},
{
"Sid": "DynamoDBAccess",
"Effect": "Allow",
"Action": [
"dynamodb:BatchGetItem",
"dynamodb:DescribeStream",
"dynamodb:DescribeTable",
"dynamodb:GetItem",
"dynamodb:GetRecords",
"dynamodb:GetShardIterator",
"dynamodb:ListStreams",
"dynamodb:Query",
"dynamodb:Scan"
],
"Resource": [
"arn:aws:dynamodb:eu-west-1:111111111111:table/poc-raven-counters-event-minute"
]
}
]
}
// Initialize the Amazon Cognito credentials provider
AWS.config.region = 'eu-west-1'; // Region
AWS.config.credentials = new AWS.CognitoIdentityCredentials({
IdentityPoolId: 'eu-west-1:11111111-1111-1111-1111-111111111111',
});
[ ...]
AWS.config.credentials.get(function(err) {
if (err) {
console.log("Error: "+err);
return;
}
console.log("Cognito Identity Id: " + AWS.config.credentials.identityId);
var cognitoSyncClient = new AWS.CognitoSync();
cognitoSyncClient.listDatasets({
IdentityId: AWS.config.credentials.identityId,
IdentityPoolId: "eu-west-1:11111111-1111-1111-1111-111111111111"
}, function(err, data) {
if ( !err ) {
console.log(JSON.stringify(data));
[...]

//you can now check that you can describe the DynamoDB table
var params = {TableName: tableName };
var dynamodb = new AWS.DynamoDB({apiVersion: '2012-08-10'});
dynamodb.describeTable(params, function(err, data){
console.log(JSON.stringify(data));
}

3. Data Layer

//Generating a string of the last X hours back
var ts = new Date().getTime();
var tsYesterday = (ts - (hoursBack * 3600) * 1000);
var d = new Date(tsYesterday);
var yesterdayDateString = d.getFullYear() + '-'
+ ('0' + (d.getMonth()+1)).slice(-2) + '-'
+ ('0' + d.getDate()).slice(-2) + 'T'
+ ('0' + (d.getHours()+1)).slice(-2) + ':'
//+ ('0' + (d.getMinutes()+1)).slice(-2) + ':'
//+ ('0' + (d.getSeconds()+1)).slice(-2);

//Forming the DynamoDB Query
var params = {
TableName: tableName,
Limit: maxItems,
ConsistentRead: false,
ScanIndexForward: true,
ExpressionAttributeValues:{
":start_date":yesterdayDateString,
":event_to_find":eventToFind
},
KeyConditionExpression :
"EventName = :event_to_find AND DateHour >= :start_date"
}

4. Presentation Layer

Displaying the data as an HTML table:

//Query DynamoDB using the new documentClient
var docClient = new AWS.DynamoDB.DocumentClient();
docClient.query(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else{
document.write("<table style=\"width:100%\" class=\"pure-table\">\n");
document.write("<tr><th>Date Hour</th><th>Event Name</th><th>Event Count</th></tr>");
data.Items.forEach(function(item) {
document.write("<tr><td>", item.DateHour, '</td><td>', item.EventName,'</td><td>',item.EventCount, '</td></tr>');
});
document.write("</table>");
}
});

Drawing the data in charts.js

var docClient = new AWS.DynamoDB.DocumentClient();
docClient.query(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else{
var recentEventsDateTime = [];
var recentEventsCounter = [];
var dateHour;
data.Items.forEach(function(item) {
dateHour = item.DateHour.toString();
recentEventsDateTime.push(dateHour.slice(0, -6));
recentEventsCounter.push(item.EventCount.toString());
});
//Chart.js code
var lineChartData = {
labels : recentEventsDateTime,
datasets : [
{
label: "JustGiving Pages Views",
fillColor : "rgba(151,187,205,0.2)",
strokeColor : "rgba(151,187,205,1)",
pointColor : "rgba(151,187,205,1)",
pointStrokeColor : "#fff",
pointHighlightFill : "#fff",
pointHighlightStroke : "rgba(151,187,205,1)",
data : recentEventsCounter
}
]}
var ctx = document.getElementById("canvas").getContext("2d");
window.myLine = new Chart(ctx).Line(lineChartData, {
responsive: true,
pointDot: false
});
}});
Example chart generated using from synthetic data stored in DynamoDB, X-Axis is time in hour resolution running counts, and Y-Axis the count for page view and impressions.

Monitoring

  • S3 Logs — a set of raw access logs that can be setup under S3 > Select bucket Properties > Logging
  • Cognito Dashboard which shows the different identifies metrics
  • DynamoDB > Tables > poc-raven-counters-event-minute > Metrics > Read Capacity will show some spikes of activity
  • CloudWatch > DynamoDB > Table Metrics > Consumed Read Capacity offers a more flexible view of the metrics

What next

  • Now that you have a dashboard and static site, you may want to also build a serverless Data API for internal or external services. I go into full details and provide source code in my Building a Scalable Serverless Microservice REST Data API video course.
  • Increase security, e.g.add an authentication provider for users in Cognito so that they need to login, lock down S3 CORS further
  • Reduce latency and increase security using Amazon CloudFront
  • Use CSS to make the table and page look pretty
  • Allow the user to change the parameters via the URL, fields or buttons
  • Run more than one query and add more than one line to the chart (as shown in our example line chart — be aware of the async nature of the queries)
  • In Charts.js override existing functions to better cope with a large number of data points (e.g. limit the number x-axis labels)
  • Try other more advanced charting JavaScript frameworks

Summary

  • DynamoDB allows us to have a persistent store for the data
  • Cognito and IAM role deals with the authentication and authorisation
  • S3 deals with the hosting the HMTL, JavaScript and node.js code, making it low cost, scalable and secure
  • Charts.js deals with drawing the responsive line charts
  • JavaScript and node.js for authentication, querying DynamoDB, generating a tabular representation of the data and charting the data

References:

My newly released Video course: Implementing Serverless Microservices Architecture Patterns with over 15+ patterns such as CI/CD and CQRS, with complete code, and over 7h of video content!
My course: Building a Scalable Serverless Microservice REST Data API video course
  • Code samples are available in the public JustGiving GitHub Repository https://github.com/JustGiving/ . Copyright (c) 2014-2016 Giving.com Ltd, trading as JustGiving, or its affiliates. All Rights Reserved. Licensed under the Apache License, Version 2.0 license.

--

--

--

Author, Advisor, Co-founder and CTO for Architecture & Data Science at Vamstar

Love podcasts or audiobooks? Learn on the go with our new app.

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Richard Freeman, PhD

Richard Freeman, PhD

Author, Advisor, Co-founder and CTO for Architecture & Data Science at Vamstar

More from Medium

INTERACTIVE WEBPAGE THROUGH APIGATEWAY USING DYNAMODB

Using AWS S3 as a simple cache service

New Feature: AWS Introduced Lambda Function URLs

Make it Easy to Retrieve, Search, Visualize, and Analyze Your Data: Amazon OpenSearch Service