boto3 aws api - Listing available instance types

Question

Instance types: (t2.micro, t2.small, c4.large...) those listed here: http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html

I want to access a list of these through boto3. something like:

conn.get_all_instance_types()

or even

conn.describe_instance_types()['InstanceTypes'][0]['Name']

which everything seems to look like in this weird api.

I've looked through the docs for client and ServiceResource, but i can't find anything that seems to come close. I haven't even found a hacky solution that lists something else that happen to represent all the instance types.

Anyone with more experience of boto3?

Answer 1

There's now boto3.client('ec2').describe_instance_types() and corresponding aws-cli command aws ec2 describe-instance-types:

'''EC2 describe_instance_types usage example'''

import boto3

def ec2_instance_types(region_name):
    '''Yield all available EC2 instance types in region <region_name>'''
    ec2 = boto3.client('ec2', region_name=region_name)
    describe_args = {}
    while True:
        describe_result = ec2.describe_instance_types(**describe_args)
        yield from [i['InstanceType'] for i in describe_result['InstanceTypes']]
        if 'NextToken' not in describe_result:
            break
        describe_args['NextToken'] = describe_result['NextToken']

for ec2_type in ec2_instance_types('us-east-1'):
    print(ec2_type)

Expect about 3s of running time.

Answer 2

The EC2 API does not provide a way to get a list of all EC2 instance types. I wish it did. Some people have cobbled together their own lists of valid types by scraping sites like this but for now that is the only way.

Answer 3

Try this

'''
Created on Mar 22, 2017

@author: ijessop
'''

import boto3
import urllib2
from bs4 import BeautifulSoup as soup

class EnumEc2():

    def __init__(self, region):

        self.client = boto3.client(
                                   'ec2',
                                   aws_access_key_id = 'YOUR_KEY' ,  
                                   aws_secret_access_key='YOUR_SECRET',
                                   region_name = region
                                   )
        self.instance_types = None
        self.instance_table_headers = None
        self.max_col_width = {}


    def getInstanceTypes(self):
        mp = soup(urllib2.urlopen('https://aws.amazon.com/ec2/instance-types').read(),'html.parser')
        imx = mp.find(id="instance-type-matrix")
        trs = imx.parent.parent.parent.next_sibling.next_sibling.find_all('tr')

        rt = []
        first_row = True
        for trow in trs:
            td_strs = []

            for td in trow.find_all("td"):
                td_nested = []
                for s in td.strings:
                    s.strip()
                    td_nested.append(s)

                td_all = " ".join(td_nested).strip()
                td_strs.append(td_all)

            if first_row is True:
                header_row = td_strs
                for head in header_row:
                    self.max_col_width.update({head:(len(head) + 2)})
                first_row = False

            else:
                dr = dict(zip(header_row,td_strs))
                for k,v in dr.items():
                    cw = len(v)
                    if k in self.max_col_width.keys():
                        if cw >= self.max_col_width.get(k):
                            self.max_col_width.update({k:(cw +2)})

                    else:
                        self.max_col_width.update({k:cw})

                rt.append(dr)

        self.instance_table_headers = header_row
        self.instance_types = rt



if __name__ == '__main__':

    myen = EnumEc2('us-west-2')
    myen.getInstanceTypes()
    heads_I_want_to_see = ['Instance Type', u'vCPU', u'Memory (GiB)', u'Storage (GB)','Physical Processor', u'Clock Speed (GHz)']
    out_str ="|"
    for h in heads_I_want_to_see:
        out_str = "%s%s|" % (out_str,h.ljust(myen.max_col_width.get(h)))
    print "%s" % "-" * len(out_str)
    print "%s" % out_str
    print "%s" % "-" * len(out_str)
    for i in myen.instance_types:
        out_str ="|"
        for k in myen.instance_table_headers: # to preserve the table column order
            if k in heads_I_want_to_see:
                out_str = "%s%s|" % (out_str, i.get(k).ljust(myen.max_col_width.get(k)))
        print "%s" % out_str
        print "%s" % "-" * len(out_str)

Answer 4

I need it too, however, there are no suitable codes for this purpose. I modify one by myself. Enjoy! May someone need it also.

Following code is modified from libcloud/contrib/scrape-ec2-prices.py And this program will generate a dict about available instance types

#!/usr/bin/env python

import os
import re
import json
import time
from collections import defaultdict, OrderedDict

import requests
import demjson

LINUX_PRICING_URLS = [
    # Deprecated instances (JSON format)
    'https://aws.amazon.com/ec2/pricing/json/linux-od.json',
    # Previous generation instances (JavaScript file)
    'https://a0.awsstatic.com/pricing/1/ec2/previous-generation/linux-od.min.js',
    # New generation instances (JavaScript file)
    'https://a0.awsstatic.com/pricing/1/ec2/linux-od.min.js'
]

EC2_REGIONS = [
    'us-east-1',
    'us-east-2',
    'us-west-1',
    'us-west-2',
    'us-gov-west-1',
    'eu-west-1',
    'eu-west-2',
    'eu-central-1',
    'ca-central-1',
    'ap-southeast-1',
    'ap-southeast-2',
    'ap-northeast-1',
    'ap-northeast-2',
    'ap-south-1',
    'sa-east-1',
    'cn-north-1',
]

INSTANCE_SIZES = [
    'micro',
    'small',
    'medium',
    'large',
    'xlarge',
    'x-large',
    'extra-large'
]

RE_NUMERIC_OTHER = re.compile(r'(?:([0-9]+)|([-A-Z_a-z]+)|([^-0-9A-Z_a-z]+))')

PRICING_FILE_PATH = './price.json'
PRICING_FILE_PATH = os.path.abspath(PRICING_FILE_PATH)


def scrape_ec2_pricing():
    result = {}
    result['regions'] = []
    result['prices'] = defaultdict(OrderedDict)
    result['models'] = defaultdict(OrderedDict)

    for url in LINUX_PRICING_URLS:
        response = requests.get(url)

        if re.match('.*?\.json$', url):
            data = response.json()
        elif re.match('.*?\.js$', url):
            data = response.content
            match = re.match('^.*callback\((.*?)\);?$', data,
                             re.MULTILINE | re.DOTALL)
            data = match.group(1)
            # demjson supports non-strict mode and can parse unquoted objects
            data = demjson.decode(data)

        regions = data['config']['regions']

        for region_data in regions:

            region_name = region_data['region']

            if region_name not in result['regions']:
                result['regions'].append(region_name)

            libcloud_region_name = region_name
            instance_types = region_data['instanceTypes']

            for instance_type in instance_types:
                sizes = instance_type['sizes']
                for size in sizes:

                    price = size['valueColumns'][0]['prices']['USD']
                    if str(price).lower() == 'n/a':
                        # Price not available
                        continue

                    if not result['models'][libcloud_region_name].has_key(size['size']):
                        result['models'][libcloud_region_name][size['size']] = {}
                        result['models'][libcloud_region_name][size['size']]['CPU'] = int(size['vCPU'])

                        if size['ECU'] == 'variable':
                            ecu = 0
                        else:
                            ecu = float(size['ECU'])

                        result['models'][libcloud_region_name][size['size']]['ECU'] = ecu

                        result['models'][libcloud_region_name][size['size']]['memoryGiB'] = float(size['memoryGiB'])

                        result['models'][libcloud_region_name][size['size']]['storageGB'] = size['storageGB']

                    result['prices'][libcloud_region_name][size['size']] = float(price)

    return result


def update_pricing_file(pricing_file_path, pricing_data):
    ##    with open(pricing_file_path, 'r') as fp:
    #        content = fp.read()

    data = {'compute': {}}  # json.loads(content)
    data['updated'] = int(time.time())
    data['compute'].update(pricing_data)

    # Always sort the pricing info
    data = sort_nested_dict(data)

    content = json.dumps(data, indent=4)
    lines = content.splitlines()
    lines = [line.rstrip() for line in lines]
    content = '\n'.join(lines)

    with open(pricing_file_path, 'w') as fp:
        fp.write(content)


def sort_nested_dict(value):
    """
    Recursively sort a nested dict.
    """
    result = OrderedDict()

    for key, value in sorted(value.items(), key=sort_key_by_numeric_other):
        if isinstance(value, (dict, OrderedDict)):
            result[key] = sort_nested_dict(value)
        else:
            result[key] = value

    return result


def sort_key_by_numeric_other(key_value):
    """
    Split key into numeric, alpha and other part and sort accordingly.
    """
    return tuple((
                     int(numeric) if numeric else None,
                     INSTANCE_SIZES.index(alpha) if alpha in INSTANCE_SIZES else alpha,
                     other
                 ) for (numeric, alpha, other) in RE_NUMERIC_OTHER.findall(key_value[0]))


def main():
    print('Scraping EC2 pricing data')

    pricing_data = scrape_ec2_pricing()
    update_pricing_file(pricing_file_path=PRICING_FILE_PATH,
                        pricing_data=pricing_data)

    print('Pricing data updated')


if __name__ == '__main__':
    main()

Answer 5

This information can be retrieved in the JSON provided by the recently-announced AWS Price List API. As a simple example using the Python requests module:

#!/usr/bin/env python
# List EC2 Instance Types
# see: https://aws.amazon.com/blogs/aws/new-aws-price-list-api/

import requests

offers = requests.get(
    'https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/index.json'
)
ec2_offer_path = offers.json()['offers']['AmazonEC2']['currentVersionUrl']
ec2offer = requests.get(
    'https://pricing.us-east-1.amazonaws.com%s' % ec2_offer_path
).json()

uniq = set()
for sku, data in ec2offer['products'].items():
    if data['productFamily'] != 'Compute Instance':
        # skip anything that's not an EC2 Instance
        continue
    uniq.add(data['attributes']['instanceType'])
for itype in sorted(uniq):
    print(itype)

Note that this might take a while... as of today, the current EC2 Offers JSON file ( https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/index.json ) is 173MB, so it takes a while both to retrieve and to parse. The current result is 99 distinct instance types.