Буквально недавно появилось время написать статью и поделится моей автоматизацией. Задача заключалась в следующем, — нужно создать терраформ модуль(и) для провиженинга AWS EC2 с использованием AWS ASG + AWS EBS. Если кто-то работал с AWS ASG + AWS EBS volumes, то знают что терраформ не позволяет создавать и прикреплять волюмы (разделы) к автоскейленг-группе. На самом-то деле, решений несколько:
- Написать модуль для Terraform и подружить AWS ASG + AWS EBS volumes. Идея очень крутая и даже разумная, но, увы — я не знаю GO. На освоение потребуется довольно много времени. Сразу отпадает. Конечно, я очень хочу начать писать на данном языке именно для того, чтобы запилить недостающий функционал в Terraform. Ребята конечно, пишут хорошо модули, но, не так быстро как хотелось…..
- Использовать python. Да, я питон знаю довольно хорошо и могу писать хорошие автоматизации. По этому, выбор пал именно на python3 + boto3.
В общим, нам понадобится, — Terraform:
Установка terraform в Unix/Linux
Модули для терраформа, можно взять мои с гитгаба:
$ git clone https://github.com/SebastianUA/terraform.gitПриведу пример моего терраформ примера и main файл выглядит:
module "sg" {
source = "../../../modules/sg"
name = "test"
environment = "NonPROD"
enable_sg_creating = true
vpc_id = "vpc-09f1d36e"
enable_all_egress_ports = true
allowed_ports = ["22", "7199", "7000", "7001", "9160", "9042"]
allow_cidrs_for_allowed_ports = {
"22" = [
"159.224.217.0/24",
"10.0.0.0/8",
"172.16.0.0/12"
],
"7199" = [
"10.0.0.0/8",
"172.16.0.0/12"
],
"7000" = [
"10.0.0.0/8",
"172.16.0.0/12"
],
"7001" = [
"10.0.0.0/8",
"172.16.0.0/12"
],
"9160" = [
"10.0.0.0/8",
"172.16.0.0/12"
],
"9042" = [
"10.0.0.0/8",
"172.16.0.0/12"
]
}
enable_custom_sg_rule = true
sg_rule_type = "ingress"
sg_rule_from_port = "1"
sg_rule_to_port = "65535"
sg_rule_protocol = "all"
#sg_rule_cidr_blocks = []
}
module "asg" {
source = "../../../modules/asg"
name = "test"
region = "us-west-2"
environment = "NonPROD"
security_groups = ["${module.sg.security_group_id}"]
root_block_device = [
{
volume_size = "8"
volume_type = "gp2"
},
]
placement_tenancy = "default"
ami = {
us-west-2 = "ami-09c6e771"
}
# Auto scaling group; NOTE: Use vpc_zone_identifier or availability_zones or will be got error!
vpc_zone_identifier = ["subnet-ca0a9a83", "subnet-f2027b95"]
# NOTE: If will be used availability_zones than enable_associate_public_ip_address = false!
enable_associate_public_ip_address = false
#
enable_asg_azs = false
health_check_type = "EC2"
asg_min_size = 0
asg_max_size = 1
desired_capacity = 1
wait_for_capacity_timeout = 0
monitoring = true
# Set up the autoscaling schedule
enable_autoscaling_schedule = false
asg_recurrence_scale_up = "0 7 * * MON-FRI"
asg_recurrence_scale_down = "0 19 * * MON-FRI"
# Define SSH key pair for our instances
key_path = "additional_files/nonprod-cassandra.pub"
# Set up launch configuration
ec2_instance_type = "t3.medium"
user_data = "./additional_files/bootstrap.tpl"
}Файл (./additional_files/bootstrap.tpl) выглядит следующим образом:
#!/bin/bash
#----------------------------------------------------------------
# Updates; Install additional tools
#----------------------------------------------------------------
sudo yum update -y;
sudo yum install -y epel-release.noarch;
sudo yum update -y;
sudo yum install -y htop \
telnet \
wget \
curl \
python34 \
python34-pip \
net-tools \
vim \
git \
screen
sudo yum update -y;
sudo echo "Test user_data file" >> ~/tmp.txt
#----------------------------------------------------------------
# Install AWS CLI tool
#----------------------------------------------------------------
sudo ln -s /usr/bin/pip-3.4 /usr/bin/pip3
sudo pip3 install awscli --upgrade --user
sudo pip3 install argparse boto3 awscli ec2-metadata --force-reinstall --upgrade
#----------------------------------------------------------------
# Download and run py-script to attach Volume(s) to the node(s)
#----------------------------------------------------------------
git clone REPO_with_SCRIPT
python3 python-scripts/ebs_volumes.py --vol-name=test --pname=default --vol-size=6 --vol-env=nonprod --create
python3 python-scripts/ebs_volumes.py --vol-name=test --pname=default --attach
sudo rm -rf python-scriptsREPO_with_SCRIPT — Репозиторий где лежит скрипт который я приведу ниже:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import argparse
import boto3
import time
import re
from ec2_metadata import ec2_metadata
class bgcolors:
def __init__(self):
self.colors = {
'PURPURE': '\033[95m',
'BLUE': '\033[94m',
'GREEN': '\033[92m',
'YELLOW': '\033[93m',
'RED': '\033[91m',
'ENDC': '\033[0m',
'BOLD': '\033[1m',
'UNDERLINE': '\033[4m'
}
def ec2_connector(b3_client, region, prof_name):
try:
session = boto3.session.Session(profile_name=prof_name)
ec2 = session.client(b3_client, region_name=region)
return ec2
except Exception as err:
print("Failed to create a boto3 client connection to EC2:\n", bgcolors().colors['RED'] + str(err),
bgcolors().colors['ENDC'])
return False
def ec2_describe_parameters():
instance_id = ec2_metadata.instance_id
availability_zone = ec2_metadata.availability_zone
parameters = {'Instance_ID': instance_id, 'Availability_Zone': availability_zone}
return parameters
def ec2_describe_volumes(b3_client, region, prof_name):
volumes_array = []
ec2 = ec2_connector(b3_client, region, prof_name)
if ec2:
volumes = ec2.describe_volumes(Filters=[])
for volume in volumes['Volumes']:
volume_id = volume['VolumeId']
if 'Tags' in volume:
tag_keys = volume['Tags']
for tag_key in tag_keys:
tk = tag_key['Key']
if tk == "Name":
tv = tag_key['Value']
data_tags = {'Volume_Name': tv, 'Volume_ID': volume_id}
volumes_array.append(data_tags)
else:
data_tags = {'Volume_Name': 'None', 'Volume_ID': volume_id}
volumes_array.append(data_tags)
else:
exit(-1)
return volumes_array
def ec2_create_volume(b3_client, region, prof_name, vol_name, vol_env, vol_size):
ec2 = ec2_connector(b3_client, region, prof_name)
if ec2:
volumes = ec2_describe_volumes(b3_client, region, prof_name)
availability_zone = ec2_describe_parameters()['Availability_Zone']
# availability_zone = 'us-west-2a'
if not volumes:
print('None, I will create a new volume!')
try:
new_volume = ec2.create_volume(Size=int(vol_size),
Encrypted=False,
AvailabilityZone='{0}'.format(availability_zone),
VolumeType='gp2',
TagSpecifications = [
{
"ResourceType": 'volume',
'Tags': [
{
'Key': 'Name',
'Value': '{0}-ebs-{1}'.format(vol_name, vol_env)
},
{
'Key': 'Environment',
'Value': '{0}'.format(vol_env)
},
{
'Key': 'Orchestration',
'Value': 'py-script'
},
{
'Key': 'Createdby',
'Value': 'Vitaliy Natarov'
}
],
},
]
)
ec2.get_waiter('volume_available').wait(VolumeIds=[new_volume['VolumeId']])
except Exception as err:
print(
"I can not create a [{0}] volume:\n".format(vol_name), bgcolors().colors['RED'] + str(err),
bgcolors().colors['ENDC'])
else:
circle = 0
founded_volumes = []
while circle < len(volumes):
if re.search(r'{0}-ebs-{1}'.format(vol_name, vol_env), volumes[circle]['Volume_Name']):
founded_volumes.append(volumes[circle])
circle += 1
if not founded_volumes:
print('I will create a new volume!')
try:
new_volume = ec2.create_volume(Size=int(vol_size),
Encrypted=False,
AvailabilityZone='{0}'.format(availability_zone),
VolumeType='gp2',
TagSpecifications=[
{
"ResourceType": 'volume',
'Tags': [
{
'Key': 'Name',
'Value': '{0}-ebs-{1}'.format(vol_name, vol_env)
},
{
'Key': 'Environment',
'Value': '{0}'.format(vol_env)
},
{
'Key': 'Orchestration',
'Value': 'py-script'
},
{
'Key': 'Createdby',
'Value': 'Vitaliy Natarov'
}
],
},
]
)
ec2.get_waiter('volume_available').wait(VolumeIds=[new_volume['VolumeId']])
except Exception as err:
print(
"I can not create a [{0}] volume:\n".format(vol_name), bgcolors().colors['RED'] + str(err),
bgcolors().colors['ENDC'])
else:
print('Woops.... I cant create a new volume. Please use another name to it!!!!!')
else:
exit(-1)
return ec2_create_volume
def ec2_attaching_volumes(b3_client, region, prof_name, vol_name):
ec2 = ec2_connector(b3_client, region, prof_name)
if ec2:
instance_id = ec2_describe_parameters()['Instance_ID']
volumes = ec2_describe_volumes(b3_client, region, prof_name)
#
# instance_id = 'i-0f6f70b38d1ccf0c1'
#
# volumes = [{'Volume_Name': 'test-ebs-nonprod', 'Volume_ID': 'vol-040d07848d558e1da'},
# {'Volume_Name': 'test2-ebs-nonprod', 'Volume_ID': 'vol-040d07848d558e1db'}]
if not volumes:
print('None')
exit(0)
else:
circle = 0
founded_volumes = []
while circle < len(volumes):
if re.search(r'{0}'.format(vol_name), volumes[circle]['Volume_Name']):
founded_volumes.append(volumes[circle])
circle += 1
symbols_for_volumes = ['b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n']
circle2 = 0
while circle2 < len(founded_volumes):
volume_device_name = '/dev/xvd{0}'.format(symbols_for_volumes[circle2])
try:
print('I will try attach!')
print(volumes[circle2]['Volume_ID'], instance_id, volume_device_name)
ec2.attach_volume(
VolumeId='{0}'.format(volumes[circle2]['Volume_ID']),
InstanceId='{0}'.format(instance_id),
Device='{0}'.format(volume_device_name))
except Exception as err:
print("I can not attach [{0}] volume to [{1}] node:\n".format(volumes[circle2]['Volume_ID'], instance_id),
bgcolors().colors['RED'] + str(err), bgcolors().colors['ENDC'])
circle2 += 1
else:
exit(-1)
return ec2_attaching_volumes
def ec2_delete_volume(b3_client, region, prof_name, vol_name, vol_env):
ec2 = ec2_connector(b3_client, region, prof_name)
if ec2:
volumes = ec2_describe_volumes(b3_client, region, prof_name)
circle = 0
if volumes:
while circle < len(volumes):
if re.search(r'{0}-ebs-{1}'.format(vol_name, vol_env), volumes[circle]['Volume_Name']):
volume_id = volumes[circle]['Volume_ID']
try:
delete_volume = ec2.delete_volume(VolumeId='{0}'.format(volume_id))
# ec2.get_waiter('delete_complete').wait(VolumeIds=[delete_volume['VolumeId']])
except Exception as err:
print(
"I can not delete a [{0}] volume:\n".format(vol_name), bgcolors().colors['RED'] + str(err),
bgcolors().colors['ENDC'])
circle += 1
else:
print('I dont have needed volume name to delete! Please use another one....')
else:
exit(-1)
return ec2_delete_volume
def main():
start__time = time.time()
parser = argparse.ArgumentParser(prog='python3 script_name.py -h',
usage='python3 script_name.py {ARGS}',
add_help=True,
prefix_chars='--/',
epilog='''created by Vitalii Natarov''')
parser.add_argument('--version', action='version', version='v0.5.0')
parser.add_argument('--bclient', '--boto3-client', dest='boto3_client', help='Set boto3 client', default='ec2')
parser.add_argument('--region', dest='region', help='Set AWS region for boto3', default='us-west-2')
parser.add_argument('--pname', '--profile-name', dest='profile_name', help='Set profile name',
default='default')
parser.add_argument('--vol-name', '--volume-name', dest='volume_name', help='Set volume name to find|attach it',
default='test')
parser.add_argument('--vol-env', '--volume-env', dest='volume_env', help='Set env for volume', default='nonprod')
parser.add_argument('--vol-size', '--volume-size', dest='volume_size', help='Set size for volume', default=6)
group = parser.add_mutually_exclusive_group(required=False)
group.add_argument('--d', dest='describe', help='Describe volumes', action='store_true')
group.add_argument('--describe', dest='describe', help='Describe volumes', action='store_true')
group2 = parser.add_mutually_exclusive_group(required=False)
group2.add_argument('--c', dest='create', help='Create volume', action='store_true', default=argparse.SUPPRESS)
group2.add_argument('--create', dest='create', help='Create volume', action='store_true')
group3 = parser.add_mutually_exclusive_group(required=False)
group3.add_argument('--a', dest='attach', help='Attach volume(s)', action='store_true', default=argparse.SUPPRESS)
group3.add_argument('--attach', dest='attach', help='Attach volume(s)', action='store_true')
group4 = parser.add_mutually_exclusive_group(required=False)
group4.add_argument('--del', dest='delete', help='Delete volume', action='store_true', default=argparse.SUPPRESS)
group4.add_argument('--delete', dest='delete', help='Delete volume', action='store_true')
results = parser.parse_args()
boto3_client = results.boto3_client
region = results.region
profile_name = results.profile_name
volume_name = results.volume_name
volume_env = results.volume_env
volume_size = results.volume_size
if results.describe:
print(ec2_describe_volumes(boto3_client, region, profile_name))
elif results.create:
ec2_create_volume(boto3_client, region, profile_name, volume_name, volume_env, volume_size)
elif results.attach:
ec2_attaching_volumes(boto3_client, region, profile_name, volume_name)
elif results.delete:
ec2_delete_volume(boto3_client, region, profile_name, volume_name, volume_env)
else:
print(bgcolors().colors['GREEN'],
'Please add [--describe] for describe or [--create] for create or [--attach] for attach',
bgcolors().colors['ENDC'])
print(bgcolors().colors['RED'], 'For help, use: script_name.py -h', bgcolors().colors['ENDC'])
exit(0)
end__time = round(time.time() - start__time, 2)
print("--- %s seconds ---" % end__time)
print(bgcolors().colors['GREEN'], "============================================================",
bgcolors().colors['ENDC'])
print(bgcolors().colors['GREEN'], "==========================FINISHED==========================",
bgcolors().colors['ENDC'])
print(bgcolors().colors['GREEN'], "============================================================",
bgcolors().colors['ENDC'])
if __name__ == '__main__':
main()Вот такой вот скрипт. Чтобы вызвать помощь, выполните:
$ python3 ebs_volumes.py --helpПолучаем вывод:
usage: python3 script_name.py {ARGS}
optional arguments:
-h, --help show this help message and exit
--version show program's version number and exit
--bclient BOTO3_CLIENT, --boto3-client BOTO3_CLIENT
Set boto3 client
--region REGION Set AWS region for boto3
--pname PROFILE_NAME, --profile-name PROFILE_NAME
Set profile name
--vol-name VOLUME_NAME, --volume-name VOLUME_NAME
Set volume name to find|attach it
--vol-env VOLUME_ENV, --volume-env VOLUME_ENV
Set env for volume
--vol-size VOLUME_SIZE, --volume-size VOLUME_SIZE
Set size for volume
--d Describe volumes
--describe Describe volumes
--c Create volume
--create Create volume
--a Attach volume(s)
--attach Attach volume(s)
--del Delete volume
--delete Delete volume
created by Vitalii NatarovДля тех кто не очень понимает, то скрипт умеет создавать, проверять какие томы имеются в AWS, прикреплять томы к нодам и удалять волюмы.
Приведу пару примеров использования:
$ python3 ebs_volumes.py --vol-name=test --pname=default --vol-size=66 --vol-env=nonprod --create
$ python3 ebs_volumes.py --vol-name=test --pname=default --describe
$ python3 ebs_volumes.py --vol-name=test --pname=default --deleteКак-то так. А у меня пока все, статья «Автоматизация AWS EBS через python скрипт и Terraform в Unix/Linux».