Making CircleCI build fails when test coverage from the current branch is smaller than the mainline.

Photo by Markus Spiske on Unsplash

Tests are critical, and even though checking only the coverage it’s not enough to make sure you’ve good tests, if you’ve poor coverage, there’s a good chance that you’re not testing everything you should.

This article is about the initial version of a script (made in Javascript) that I created to force the CircleCI build to fail if the coverage from the current branch is smaller than the coverage from the mainline branch.

1. Create a script that will:

• Clone the project in a folder to get the results;
• Install the dependencies;
• Prune the dependencies;
• Build the project;
• Run the coverage command;
• Saves the results;
• Do the previous steps for the current branch;
• Compare the results and make the CircleCI build fail if the current coverage is smaller than the coverage from the mainline.

Here is my first version (which can be better) of the script using Jest coverage command.

var chalk = require('chalk');
var fs = require('fs');
var spawn = require('child_process').spawn;
var DOT_ENV_FILE = './.env';
var dotEnvFileExists = fs.existsSync(DOT_ENV_FILE);
var results = []
if(dotEnvFileExists) {
  require('dotenv').config();
}
var BRANCH_NAME = process.argv[2]
var REPOSITORY_URL = process.env.REPOSITORY_URL
var CHECK_COVERAGE_FOLDER = './checkCoverage'
var COVERAGE_FILE = 'coverage/lcov-report/index.html'
var checkCoverageFolderExists = fs.existsSync(CHECK_COVERAGE_FOLDER)
if(!checkCoverageFolderExists) {
  gitClone().then(setupBranchAndUp);
} else {
  setupBranchAndUp();
}
function setupBranchAndUp() {
  process.chdir(CHECK_COVERAGE_FOLDER);
gitUpdate()
    .then(gitResetMaster)
    .then(installDependencies)
    .then(pruneDependencies)
    .then(build)
    .then(jestCoverage)
    .then(getCoverageValues)
    .then(gitUpdate)
    .then(gitReset)
    .then(installDependencies)
    .then(pruneDependencies)
    .then(build)
    .then(jestCoverage)
    .then(getCoverageValues)
    .then(checkCoverageValues)
    .then(() => { process.exit(); })
    .catch(() => { process.exit(1); })
}
function gitClone() {
  return runCommand(
    'git',
    ['clone', REPOSITORY_URL, CHECK_COVERAGE_FOLDER],
    'Cloning repository...'
  );
}
function gitResetMaster() {
  return runCommand(
    'git',
    ['reset', '--hard', 'origin/master'],
    'Reseting repository...'
  );
}
function gitReset() {
  return runCommand(
    'git',
    ['reset', '--hard', 'origin/' + BRANCH_NAME],
    'Reseting repository...'
  );
}
function gitUpdate() {
  return runCommand(
    'git',
    ['remote', 'update', '-p'],
    'Update repository...'
  );
}
function installDependencies() {
  return runCommand(
    'npm',
    ['install', '--save'],
    'Installing dependencies. This may take a few minutes...'
  );
}
function pruneDependencies() {
  return runCommand(
    'npm',
    ['prune'],
    'Pruning dependencies...'
  );
}
function build() {
  return runCommand(
    'npm',
    ['run', 'build'],
    'Building...'
  );
}
function jestCoverage() {
  return runCommand(
    'npm',
    ['test', '--', '--coverage'],
    'Running tests...'
  );
}
function getCoverageValues() {
  return new Promise((resolve, reject) => {
    console.log('Getting coverage values...')
var arrayFile = fs.readFileSync(COVERAGE_FILE).toString().split('\n');
    var coverageValues = 0;
if(arrayFile.length <= 0) {
  reject();
}
for(i in arrayFile) {
      if(arrayFile[i].includes('strong')) {
        var result = arrayFile[i].trim().replace('<span class="strong">', '').replace('% </span>', '')
        results.push(result)
        coverageValues++;
      if(coverageValues === 4) {
          i = arrayFile.length
        }
      }
    }
    return resolve();
  })
}
function checkCoverageValues() {
  return new Promise((resolve, reject) => {
    console.log('Checking coverage values...')
    var statements = shouldFails(0, 4)
    var branches = shouldFails(1, 5)
    var functions = shouldFails(2, 6)
    var lines = shouldFails(3, 7)
if(statements || branches || functions || lines) {
      console.error('The current coverage is smaller than the master coverage');
      return reject();
    }
    console.log();
    console.log('Everything is alright in your tests');
    return resolve();
  })
}
function shouldFails(master, currentBranch) {
  return (results[master] - results[currentBranch]) > 2
}
function runCommand(command, commandOptions, startMessage) {
  return new Promise((resolve, reject) => {
    console.log(startMessage);
    var proc = spawn(command, commandOptions, { stdio: 'inherit' });
     proc.on('close', function(code) {
      if (code !== 0) {
        console.error('Error - `' + command + ' ' + commandOptions.join(' ') + '` failed');
        return reject();
      }
      return resolve();
    })
  })
}

2. Add to your circle.yml the command to run the script we created in the first step.

test:
  override:
    - npm test
  post:
    - npm run coverage $CIRCLE_BRANCH

3. You can put a threshold as well for the coverage results in your package.json, so if your code coverage is smaller than the threshold (%), the build will fail

"jest":{
  "coverageThreshold": {
     "global": {
       "branches": 70,
       "functions": 80,
       "lines": 80,
       "statements": 80
     }
  }
}

Note: I’m using the variable REPOSITORY_URL that I take from the .env file, but you have to add this environment variable in CircleCi to this script to work.

I hope that this can give you an idea of how to do this in your project :]

You can check this article and others on my website: https://www.marinahaack.com/articles/Making-CircleCI-build-fails-when-test-coverage-from-the-current-branch-is-smaller-than-the-mainline