catching up
This commit is contained in:
parent
3517f4e6de
commit
cca144d583
|
|
@ -0,0 +1,53 @@
|
|||
#!/bin/bash
|
||||
|
||||
### change image id and naming suffix then run this bitch
|
||||
image=ami-076369466766d94fa
|
||||
naming_suffix="_test_with_fixed_regex"
|
||||
|
||||
function drive_count_check {
|
||||
instance_type=$1
|
||||
echo $instance_type | grep -qE d
|
||||
if [[ $? == 0 ]]; then
|
||||
drive_count=23
|
||||
else
|
||||
drive_count=24
|
||||
fi
|
||||
|
||||
echo $drive_count
|
||||
}
|
||||
|
||||
function launch {
|
||||
instance_types="m4 c4 r4"
|
||||
#instance_types=m5
|
||||
for i in $instance_types; do
|
||||
instance_type=$i".xlarge"
|
||||
name=$i$naming_suffix
|
||||
drive_count=$(drive_count_check $i)
|
||||
response=$(exec aws emr create-cluster --auto-scaling-role EMR_AutoScaling_DefaultRole --applications Name=Hadoop --bootstrap-actions '[{"Path":"s3://ty-emr/XRR/bootstrap/install_sift.sh","Name":"Custom action"}]' --ebs-root-volume-size 10 --ec2-attributes '{"KeyName":"tylebean","AdditionalSlaveSecurityGroups":["sg-3ed30a59"],"InstanceProfile":"EMR_EC2_DefaultRole","SubnetId":"subnet-3db49e4a","EmrManagedSlaveSecurityGroup":"sg-7f523e05","EmrManagedMasterSecurityGroup":"sg-7e523e04","AdditionalMasterSecurityGroups":["sg-3ed30a59","sg-5279fd35"]}' --service-role EMR_DefaultRole --release-label emr-5.17.0 --name "$name" --instance-groups '[{"InstanceCount":1,"EbsConfiguration":{"EbsBlockDeviceConfigs":[{"VolumeSpecification":{"SizeInGB":10,"VolumeType":"gp2"},"VolumesPerInstance":23},{"VolumeSpecification":{"SizeInGB":32,"VolumeType":"gp2"},"VolumesPerInstance":1}],"EbsOptimized":true},"InstanceGroupType":"MASTER","InstanceType":"'$instance_type'","Name":"Master - 1"}]' --scale-down-behavior TERMINATE_AT_TASK_COMPLETION --region us-east-1 --additional-info "{ami64: \"$image\", amiHvm64: \"$image\", hadoopVersion: \"2.7.2\", hadoopConfigurationVersion: \"5.0\"}")
|
||||
id=$(echo $response | jq -r '.ClusterId')
|
||||
url="https://aegir-us-east-1.amazon.com/Search?searchText=$id&searchType=Guess&domain=prod&realm=us-east-1"
|
||||
echo "$name :: $id :: $url"
|
||||
done
|
||||
}
|
||||
|
||||
function launch_test {
|
||||
instance_types="m5 c5 r5 c5d m5d"
|
||||
for i in $instance_types; do
|
||||
instance_type=$i".xlarge"
|
||||
name=$i$naming_suffix
|
||||
drive_count=$(drive_count_check $i)
|
||||
response=$(echo "{\"ClusterId\": \"j-$instance_type\"}")
|
||||
id=$(echo $response | jq -r '.ClusterId')
|
||||
url="https://aegir-us-east-1.amazon.com/Search?searchText=$id&searchType=Guess&domain=prod&realm=us-east-1"
|
||||
echo "$name :: $id :: $url"
|
||||
echo $url
|
||||
done
|
||||
}
|
||||
|
||||
function url_test {
|
||||
id="j-2BGP1R9D3NTD6"
|
||||
url="https://aegir-us-east-1.amazon.com/Search?searchText=$id&searchType=Guess&domain=prod&realm=us-east-1"
|
||||
open $url
|
||||
}
|
||||
|
||||
launch
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# Usage: lsapp application_1537193066945_0202
|
||||
# Ex. Output:
|
||||
# Application ID: application_1537193066945_0202
|
||||
# AM Node: ip-172-28-115-72 - i-0dc1a1053af30c34c
|
||||
# AM Container: container_1537193066945_0202_01_000001
|
||||
# Application Runtime: 2018-09-27 12:48:00,390 - 2018-09-27 12:48:53,201
|
||||
|
||||
function lsapp() {
|
||||
appID=$1
|
||||
appLogs=$(sift -riz --no-filename --ipath=resourcemanager -e "$appID")
|
||||
baseLog=$(echo $appLogs | sift -riz --no-filename --ipath=resourcemanager -e "$appID.*MasterContainer")
|
||||
AMContainerID=$(echo $baseLog | sift --only-matching -e "container_(\d|_)+")
|
||||
AMNode=$(echo $baseLog | sift --only-matching -e "ip-(\d|-)+")
|
||||
ICLog=$(sift -rizl --limit=1 --file-matches=InstanceJointStatusMap --ipath=daemons/instance-controller $AMNode | sort | tail -1)
|
||||
AMInstanceID=$(sift -riz --limit=1 $AMNode $ICLog | awk '{print $1}')
|
||||
startTime=$(echo $appLogs | sift -riz --no-filename --ipath=resourcemanager -e "$appID.*to submitted" | awk '{print $1, $2}')
|
||||
endTime=$(echo $appLogs | sift -riz --no-filename --ipath=resourcemanager -e "application removed.*$appID.* queue" | awk '{print $1, $2}')
|
||||
echo Application ID: $appID
|
||||
echo AM Node: $AMNode - $AMInstanceID
|
||||
echo AM Container: $AMContainerID
|
||||
echo Application Runtime: $startTime - $endTime
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
# Usage: lscontainer container_1537193066945_0202_01_000002
|
||||
# Ex. Output:
|
||||
# Hostname : ip-172-28-115-100 || InstanceID : i-09da76008179f44ff
|
||||
|
||||
function lscontainer() {
|
||||
containerID=$1
|
||||
containerHostname=$(sift -riz --no-filename --ipath=resourcemanager -e "assigned container $containerID" | sift --only-matching -e "ip-(\d|-)+")
|
||||
ICLog=$(sift -rizl --limit=1 --file-matches=InstanceJointStatusMap --ipath=daemons/instance-controller $containerHostname | sort | tail -1)
|
||||
containerInstanceID=$(sift -riz --limit=1 $containerHostname $ICLog | awk '{print $1}')
|
||||
echo Hostname : $containerHostname \|\| InstanceID : $containerInstanceID
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
# Usage: lshost ip-172-28-115-41
|
||||
# Ex. Output:
|
||||
# Hostname : ip-172-28-115-41 || InstanceID : i-0cf1a535723e25e56
|
||||
|
||||
function lshost() {
|
||||
Hostname=$1
|
||||
ICLog=$(sift -rizl --limit=1 --file-matches=InstanceJointStatusMap --ipath=daemons/instance-controller $Hostname | sort | tail -1)
|
||||
InstanceID=$(sift -riz --limit=1 $Hostname $ICLog | awk '{print $1}')
|
||||
echo Hostname : $Hostname \|\| InstanceID : $InstanceID
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
unhealthy nodes
|
||||
---------------
|
||||
sift -riz 'reported unhealthy'
|
||||
sift -riz 'local-dirs are bad'
|
||||
sift -riz 'killing container.'
|
||||
|
||||
sift -riz 'application removed - appid: application_1532193836323_78176'
|
||||
sift -riz -e 'application finished -.*application_1532193836323_78176'
|
||||
|
||||
Loading…
Reference in New Issue