AWS serverless scraping: randomising your ip.

Pablo Voorvaart
Sep 1 · 3 min read

The code:

import boto3import os, jsonimport time
client = boto3.client('sqs')natClient = boto3.client('ec2')lambdaClient = boto3.client('lambda')def reset(event, context): vpcId = "" subnetId = "" routetableId = "" # block lambda functions response = lambdaClient.put_function_concurrency( FunctionName='feiScraperEvent--dev', ReservedConcurrentExecutions=0 ) response = lambdaClient.put_function_concurrency( FunctionName='feiScraperContest--dev', ReservedConcurrentExecutions=0 ) response = lambdaClient.put_function_concurrency( FunctionName='feiScraperResult--dev', ReservedConcurrentExecutions=0 ) #delete NAt gateway nat_gateway_id = natClient.describe_nat_gateways( Filters=[{ 'Name': 'vpcId', 'Values': [vpcId] }] ) for gateway in nat_gateway_id['NatGateways']: try: natClient.delete_nat_gateway( NatGatewayId=gateway['NatGatewayId'] ) natClient.release_address( AllocationId=gateway['NatGatewayAddresses'] [0]['AllocationId'] ) except: pass #Create eip and nat eip = natClient.allocate_address( Domain = vpcId )
natGateway = natClient.create_nat_gateway(
AllocationId=eip['AllocationId'], SubnetId= subnetId ) response = natClient.describe_nat_gateways( Filters=[{ 'Name': 'nat-gateway-id', 'Values': [natGateway['NatGateway']['NatGatewayId']] }]
)
while response['NatGateways'][0]['State'] != 'available': time.sleep(10) response = natClient.describe_nat_gateways( Filters=[{ 'Name': 'nat-gateway-id', 'Values': [natGateway['NatGateway']['NatGatewayId']] }] )
# Edit route tables
natClient.delete_route( DestinationCidrBlock='0.0.0.0/0', RouteTableId = routetableId ) response = natClient.create_route( DestinationCidrBlock='0.0.0.0/0', NatGatewayId=natGateway['NatGateway']['NatGatewayId'], RouteTableId = routetableId )
# reset lambda functions
response = lambdaClient.put_function_concurrency( FunctionName='feiScraperEvent--dev', ReservedConcurrentExecutions=1 ) response = lambdaClient.put_function_concurrency( FunctionName='feiScraperContest--dev', ReservedConcurrentExecutions=1 ) response = lambdaClient.put_function_concurrency( FunctionName='feiScraperResult--dev', ReservedConcurrentExecutions=1 )

The explanation:

Conclusion:

Welcome to a place where words matter. On Medium, smart voices and original ideas take center stage - with no ads in sight. Watch
Follow all the topics you care about, and we’ll deliver the best stories for you to your homepage and inbox. Explore
Get unlimited access to the best stories on Medium — and support writers while you’re at it. Just $5/month. Upgrade