Firebase Admin SDK Bulk Import

Ron Royston
Jul 1, 2018 · 3 min read

This article demonstrates using the Firebase Admin SDK to import a very large dataset from a CSV file.

Challenges

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed — JavaScript heap out of memory

Setup and Installation

This article assumes a large CSV file as the native dataset format so the csv-parse module is used to read this data. Install this module now.

npm install csv-parse

Running Node with Additional Memory

node — max_old_space_size=8000 app.js

Node Script

var admin = require("firebase-admin");
var serviceAccount = require("./your-firebase-project-service-account-key.json");
var fs = require('fs');
var csvFile = "./my-huge-file.csv"
var parse = require('csv-parse');
require('should');
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
databaseURL: "https://your-project.firebaseio.com"
});
var firestore = admin.firestore();
var thisRef;
var obj = {};
var counter = 0;
var commitCounter = 0;
var batches = [];
batches[commitCounter] = firestore.batch();
fs.createReadStream(csvFile)
.pipe(
parse({delimiter: '|',relax_column_count:true,quote: ''})
)
.on('data', function(csvrow) {
if(counter <= 498){
if(csvrow[1]){
obj.family = csvrow[1];
}
if(csvrow[2]){
obj.series = csvrow[2];
}
if(csvrow[3]){
obj.sku = csvrow[3];
}
if(csvrow[4]){
obj.description = csvrow[4];
}
if(csvrow[6]){
obj.price = csvrow[6];
}
thisRef = firestore.collection("your-collection-name").doc();
batches[commitCounter].set(thisRef, obj);
counter = counter + 1;
} else {
counter = 0;
commitCounter = commitCounter + 1;
batches[commitCounter] = firestore.batch();
}
})
.on('end',function() {
writeToDb(batches);
});
function oneSecond() {
return new Promise(resolve => {
setTimeout(() => {
resolve('resolved');
}, 1010);
});
}
async function writeToDb(arr) {
console.log("beginning write");
for (var i = 0; i < arr.length; i++) {
await oneSecond();
arr[i].commit().then(function () {
console.log("wrote batch " + i);
});
}
console.log("done.");
}

Troubleshooting

Ron Royston

Written by

Dev @ hightechtele.com. Tulane Graduate (Philosophy '97), Cisco Certified Internetworking Expert# 6824, USPA Master Skydiver (D# 20776).