Sending bash commands to an EC2 cluster with AWS Lambda and SSM

Lambda Command Flow

As you scale up your application, you may run into the growing pains of maintaining a cluster of application servers.

At such times there might be day-to-day maintenance tasks that could become quite lengthy as the size of your cluster increases.

To combat such difficulties, we have created a Lambda routine that takes the list of each instance running in an ELB target group, and individually send commands to each instance via AWS SSM.

Here’s how we did it:

Pre-requisites:

• Basic understanding of AWS Lambda, EC2, ELB
• Knowledge of Node.js
• An environment running EC2 with Elastic Load Balancers

Create a lambda function and set up its permissions

• ELB read-only permissions
• SSM.sendCommand permissions

**Make sure to give Lambda permissions to run the official AWS SSM documents!

Get list of running instances from ELB

Getting the list of instances running in an ELB target group is a two-step process, since the describeTargetGroups provided by the AWS SDK does not return a list of running instances.

First, we need to get the ARN value of the target group where our EC2 instances are running in.

Here’s a copy-pasteable function that returns data on a given target group from its name:

function getTargetGroups(tgName = 'test1-webapp-tg') {
  return new Promise(function(resolve, reject) {
    const params = {
      Names: [tgName],
    };
    ELBv2.describeTargetGroups(params, function(err, data) {
      if (err) {
        console.log(err)
        reject(err);
      }
            
      resolve(data);
    });
  });
}

⚠️ This method requires you to have a standard of Target Group naming to be able to! (e.g. test01-webapp-tg️) ⚠️

From this we may extract the ARN of the requested target group, to be used in a separate function that can return

And then we will have to use the describeTargetHealth method to get a list of instances running on a target group

function getInstanceIds(targetGroupArn) {
  return new Promise(function(resolve, reject) {
    const params = {
      TargetGroupArn: targetGroupArn,
    };
    ELBv2.describeTargetHealth(params, async function(err, data) {
      if (err) {
        console.log(err);
        reject(err);
      }
      const instanceIdList = [];
      data.TargetHealthDescriptions.forEach(function(healthData) {
        const name = healthData.Target.Id;
        instanceIdList.push(name);
      });
      resolve(instanceIdList);
    });
  });
}

Use the instance ID list to send commands to each instance

At this point, you may use the instance ID list retrieved in the previous step to send bash commands directly to one of your EC2 instances.

function sendCommands(instanceIdList) {
  return new Promise(function(resolve, reject) {
    //TODO: Place the command you wish to run over here! 
    // or better yet, send it as an argument to your Lambda function
    const cmd = 'ls -al';
    const params = {
      InstanceIds: instanceIdList,
      DocumentName: 'AWS-RunShellScript',
      Parameters: {
        commands: [cmd],
      },
    };
    SSM.sendCommand(params, function(err, data) {
      if (err) {
        console.log(err);
        reject(err);
      }
      resolve(data);
    });
  });
}

Putting it all together

Now that you have all the setup done, all you have to do it use them in the handler function inside Lambda:

const AWS = require('aws-sdk');
const ELBv2 = new AWS.ELBv2({ apiVersion: '2015-12-01' });
const SSM = new AWS.SSM();
function getTargetGroups(tgName = 'test1-webapp-tg') {
  return new Promise(function(resolve, reject) {
    const params = {
      Names: [tgName],
    };
    ELBv2.describeTargetGroups(params, function(err, data) {
      if (err) {
        console.log(err)
        reject(err);
      }
            
      resolve(data);
    });
  });
}
function getInstanceIds(targetGroupArn) {
  return new Promise(function(resolve, reject) {
    const params = {
      TargetGroupArn: targetGroupArn,
    };
    ELBv2.describeTargetHealth(params, async function(err, data) {
      if (err) {
        console.log(err);
        reject(err);
      }
      const instanceIdList = [];
      data.TargetHealthDescriptions.forEach(function(healthData) {
        const name = healthData.Target.Id;
        instanceIdList.push(name);
      });
      resolve(instanceIdList);
    });
  });
}
function sendCommands(instanceIdList) {
  return new Promise(function(resolve, reject) {
    //TODO: Place the command you wish to run over here 
    // or better, send it as an argument to your Lambda function!
    const cmd = 'ls -al';
    const params = {
      InstanceIds: instanceIdList,
      DocumentName: 'AWS-RunShellScript',
      Parameters: {
        commands: [cmd],
      },
    };
    SSM.sendCommand(params, function(err, data) {
      if (err) {
        console.log(err);
        reject(err);
      }
      resolve(data);
    });
  });
}
exports.handler = async (event) => {
  const EXAMPLE_TG = 'test1-webapp-tg';
  const targetGroupData = await getTargetGroups(EXAMPLE_TG);
  if (targetGroupData) {
    const instanceIdList = await getInstanceIds(
      targetGroupData.TargetGroups[0].TargetGroupArn,
    );
    console.log('Sending commands to instances:');
    console.log(instanceIdList);
    const commandResults = await sendCommands(instanceIdList);
    return {
      statusCode: 200,
      message: commandResults,
    };
  }
}

Conclusion

By leveraging the flexibility of Lambda functions with the vast offerings of the AWS SDK, we’ve been able to automate away fairly complex problems with only about 100 lines of code!

References

• Lambda — Getting Started: https://aws.amazon.com/lambda/getting-started/
• Systems Manager (SSM) — Getting Started: https://aws.amazon.com/systems-manager/getting-started/
• AWS SDK: https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/