Using the Redis multi object in Node.js for fun and profit

This is part 3 of my Node/Redis series. You can read Part 1, Dancing around strings in Node.js and Redis, Part 2, Store Javascript objects in Redis with Node.js the right way, Part 4 Keeping track of account subscriptions with Redis and Node.js, Part 5 Managing modularity and Redis connections in Node.js, Part 6 Redis, Express and Streaming with Node.js and Classic Literature(?) Part 7 Untangling Redis sort results with Node.js and lodash and Part 8 Redis, set, node.

Hey! If you’re reading this, maybe you want to go to RedisConf 2018 for free?

Getting my head around atomic operations in Redis was a bit of a leap initially. The most accessible way to approach this is often with the use of the multi / exec commands — you essentially package up all your commands, node_redis runs them in order and as one unit then you get the results.

I had used the multi for adding chunks of data to Redis for a while, but a break through for me was looking at the source for Reds — it did something neat with multi that I hadn’t considered:

• Sort / Store
• Retrieve the stored results
• Delete the stored results

All this was in the multi and thus accomplished atomically. It was a lightbulb moment for me. You can get the result of something in the middle of the operation and continue after the result atomically. It really changed how I approached redis.

Sometimes, however, it is less than clear where in the result the values you want are stored. Let’s say you have a multi operation where you need to do n operations, get a result and do another n number of operations. How do you find your desired results?

The Veterinary Office Example

Let’s start with a little example.

var
  redis = require(‘redis’),
  client = redis.createClient(),
  clientAnimals = [
    { name : ‘Button’, species: ‘Canis familiaris’, lastVisit: 1430456400000 },
    { name : ‘Wilberforce’, species: ‘Felis catus’, lastVisit: 1413694800000 },
    { name : ‘Spot’, species: ‘Canis familiaris’, lastVisit: 1394686800000 },
    { name : ‘TardarSauce’, species: ‘Felis catus’, lastVisit: 1424844000000 },
    { name : ‘Muffin’, species: ‘Capra hircus’, lastVisit: 1359266400000, notes : “Unusual colouration” }
  ],
  importMulti;
//see “Dancing around strings in Node.js and Redis” https://medium.com/@stockholmux/dancing-around-strings-in-node-js-and-redis-2a8f91ebe0bf
function rk() {
  return Array.prototype.slice.call(arguments).join(‘:’)
}
 
importMulti = client.multi();
clientAnimals.forEach(function(anAnimal){
  importMulti.hmset(rk(‘animal’,’details’,anAnimal.name),anAnimal);
});
importMulti.exec(function(err,results){
  if (err) { throw err; } else {
 
    //this will log the results of the all hmsets:
    //[ ‘OK’, ‘OK’, ‘OK’, ‘OK’, ‘OK’ ]
    //Not very useful… yet!
    console.log(results);
 
 
    client.quit();
   }
});

This doesn’t do much exciting — we’re just adding a bunch of hashes. Notice the results, just a bunch of “OK” values. By the way, Button, from the example is my real-life dog.

Button is an Italian Greyhound and the best dog ever.

Now, let’s use zadd to “index” the hashes by the lastVisit value. By the way, lastVisit is just a Javascript timestamp.

....
clientAnimals.forEach(function(anAnimal){
  var
    detailHashKey = rk(‘animal’,’details’,anAnimal.name),
    visitIndex = rk(‘animal’,’lastVisit’);
 
  importMulti.hmset(detailHashKey,anAnimal);
 
  //adding an “index” by the last visit
  importMulti.zadd(visitIndex,anAnimal.lastVisit,detailHashKey);
});
importMulti.exec(function(err,results){
  if (err) { throw err; } else {
    //this will log the results of the all hmsets AND the zadd
    // note the pattern of hmset result followed by zadd result
    //[ ‘OK’, 1, ‘OK’, 1, ‘OK’, 1, ‘OK’, 1, ‘OK’, 1 ]
    //Perhaps even worse than the first example.
 
    console.log(results);
 
    client.quit();
  }
});

Real simple. We will now have a nice zset with the Javascript timestamp on the left and the key for the hash on the right. Again, we have a bunch of junk that you’re probably not going to use as a result.

Now, lets do something with the data more fun. Lets say we immediately want to get some values after a certain date — we’ll define it in dateMin. Then, after all our hmsets and zadds, we’re going to call a zrangebyscore.

var
  redis = require(‘redis’),
  client = redis.createClient(),
  clientAnimals = [
    { name : ‘Button’, species: ‘Canis familiaris’, lastVisit: 1430456400000 },
    { name : ‘Wilberforce’, species: ‘Felis catus’, lastVisit: 1413694800000 },
    { name : ‘Spot’, species: ‘Canis familiaris’, lastVisit: 1394686800000 },
    { name : ‘TardarSauce’, species: ‘Felis catus’, lastVisit: 1424844000000 },
    { name : ‘Muffin’, species: ‘Capra hircus’, lastVisit: 1359266400000, notes : “Unusual colouration” }
  ],
  importMulti,
  dateMin = new Date(2015,1,1).getTime(),
  visitIndex = rk(‘animal’,’lastVisit’);
 
//see “Dancing around strings in Node.js and Redis” https://medium.com/@stockholmux/dancing-around-strings-in-node-js-and-redis-2a8f91ebe0bf
function rk() {
  return Array.prototype.slice.call(arguments).join(‘:’)
}
 
importMulti = client.multi();
clientAnimals.forEach(function(anAnimal){
  var
    detailHashKey = rk(‘animal’,’details’,anAnimal.name);
 
  importMulti.hmset(detailHashKey,anAnimal);
 
  //adding an “index” by the last visit
  importMulti.zadd(visitIndex,anAnimal.lastVisit,detailHashKey);
});
//now lets get any animals that are due for a checkup
importMulti.zrangebyscore(
  visitIndex,
  ‘-inf’,
  dateMin
);
importMulti.exec(function(err,results){
  if (err) { throw err; } else {
    //this will log the results of the all hmsets AND the zadd AND the zrangebyscore
   //[ ‘OK’,
   // 0,
   // ‘OK’,
   // 0,
   // ‘OK’,
   // 0,
   // ‘OK’,
   // 0,
   // ‘OK’,
   // 0,
   // [ ‘animal:details:Muffin’,
   // ‘animal:details:Spot’,
   // ‘animal:details:Wilberforce’ ] ]
 
   console.log(results);
 
   client.quit();
  }
});

This is straight forward. The last result has the values that you’re looking for. But, what if it wasn’t the last element of the array? Let’s say you needed to do more operations atomically but the the valuable data was somewhere in the middle of your result. How do you find it?

You could keep track of how many commands you’ve sent manually, or, if your script has a fixed number of commands you could just use that. But, let’s say, in this example the array clientAnimals had a varying number of elements — could be 2 could be 200 — where in the results array will your data be?

The answer lies in node_redis’s multi object. The multi object containes two objects, _client and queue. Queue sounds interesting, let’s do a log right before running the zrangebyscore command. Here are the results:

[ [ ‘MULTI’ ],
  [ ‘hmset’,
  ‘animal:details:Button’,
  ‘name’,
  ‘Button’,
  ‘species’,
  ‘Canis familiaris’,
  ‘lastVisit’,
  1430456400000 ],
  [ ‘zadd’,
  ‘animal:lastVisit’,
  1430456400000,
  ‘animal:details:Button’ ],
  [ ‘hmset’,
  ‘animal:details:Wilberforce’,
  ‘name’,
  ‘Wilberforce’,
  ‘species’,
  ‘Felis catus’,
  ‘lastVisit’,
  1413694800000 ],
  [ ‘zadd’,
  ‘animal:lastVisit’,
  1413694800000,
  ‘animal:details:Wilberforce’ ],
  [ ‘hmset’,
  ‘animal:details:Spot’,
  ‘name’,
  ‘Spot’,
  ‘species’,
  ‘Canis familiaris’,
  ‘lastVisit’,
  1394686800000 ],
  [ ‘zadd’,
  ‘animal:lastVisit’,
  1394686800000,
  ‘animal:details:Spot’ ],
  [ ‘hmset’,
  ‘animal:details:TardarSauce’,
  ‘name’,
  ‘TardarSauce’,
  ‘species’,
  ‘Felis catus’,
  ‘lastVisit’,
  1424844000000 ],
  [ ‘zadd’,
  ‘animal:lastVisit’,
  1424844000000,
  ‘animal:details:TardarSauce’ ],
  [ ‘hmset’,
  ‘animal:details:Muffin’,
  ‘name’,
  ‘Muffin’,
  ‘species’,
  ‘Capra hircus’,
  ‘lastVisit’,
  1359266400000,
  ‘notes’,
  ‘Unusual colouration’ ],
  [ ‘zadd’,
  ‘animal:lastVisit’,
  1359266400000,
  ‘animal:details:Muffin’ ] ]

Interesting, right? You can see the pipeline of commands and arguments being sent to redis. Nothing really magical, but it does have a handy use— you know where you are at in your multi command by getting the length of the array right after calling your specific command. Small trick though: subtract 2 from the length (one for zero-based counting and one for the “MULTI”). Store the result in a variable and reference it after you get your results.

var
  resultArrayIndex;
...
importMulti.zrangebyscore(
 visitIndex,
 ‘-inf’,
 dateMin
);
resultArrayIndex = importMulti.queue.length — 2; 
...
importMulti.exec(function(err,results){
  if (err) { throw err; } else {
    //your desired results every time!
    console.log(results[resultArrayIndex]); 
 
    client.quit();
  }
});

It doesn’t matter how many commands you stuff into your multi you can get the data from right in the middle.

Speaking of fun and profit…

Are you building a Node / Redis project? Looking to take it to the next level? I can help.