catching up
This commit is contained in:
parent
3517f4e6de
commit
cca144d583
|
|
@ -0,0 +1,53 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
### change image id and naming suffix then run this bitch
|
||||||
|
image=ami-076369466766d94fa
|
||||||
|
naming_suffix="_test_with_fixed_regex"
|
||||||
|
|
||||||
|
function drive_count_check {
|
||||||
|
instance_type=$1
|
||||||
|
echo $instance_type | grep -qE d
|
||||||
|
if [[ $? == 0 ]]; then
|
||||||
|
drive_count=23
|
||||||
|
else
|
||||||
|
drive_count=24
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo $drive_count
|
||||||
|
}
|
||||||
|
|
||||||
|
function launch {
|
||||||
|
instance_types="m4 c4 r4"
|
||||||
|
#instance_types=m5
|
||||||
|
for i in $instance_types; do
|
||||||
|
instance_type=$i".xlarge"
|
||||||
|
name=$i$naming_suffix
|
||||||
|
drive_count=$(drive_count_check $i)
|
||||||
|
response=$(exec aws emr create-cluster --auto-scaling-role EMR_AutoScaling_DefaultRole --applications Name=Hadoop --bootstrap-actions '[{"Path":"s3://ty-emr/XRR/bootstrap/install_sift.sh","Name":"Custom action"}]' --ebs-root-volume-size 10 --ec2-attributes '{"KeyName":"tylebean","AdditionalSlaveSecurityGroups":["sg-3ed30a59"],"InstanceProfile":"EMR_EC2_DefaultRole","SubnetId":"subnet-3db49e4a","EmrManagedSlaveSecurityGroup":"sg-7f523e05","EmrManagedMasterSecurityGroup":"sg-7e523e04","AdditionalMasterSecurityGroups":["sg-3ed30a59","sg-5279fd35"]}' --service-role EMR_DefaultRole --release-label emr-5.17.0 --name "$name" --instance-groups '[{"InstanceCount":1,"EbsConfiguration":{"EbsBlockDeviceConfigs":[{"VolumeSpecification":{"SizeInGB":10,"VolumeType":"gp2"},"VolumesPerInstance":23},{"VolumeSpecification":{"SizeInGB":32,"VolumeType":"gp2"},"VolumesPerInstance":1}],"EbsOptimized":true},"InstanceGroupType":"MASTER","InstanceType":"'$instance_type'","Name":"Master - 1"}]' --scale-down-behavior TERMINATE_AT_TASK_COMPLETION --region us-east-1 --additional-info "{ami64: \"$image\", amiHvm64: \"$image\", hadoopVersion: \"2.7.2\", hadoopConfigurationVersion: \"5.0\"}")
|
||||||
|
id=$(echo $response | jq -r '.ClusterId')
|
||||||
|
url="https://aegir-us-east-1.amazon.com/Search?searchText=$id&searchType=Guess&domain=prod&realm=us-east-1"
|
||||||
|
echo "$name :: $id :: $url"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
function launch_test {
|
||||||
|
instance_types="m5 c5 r5 c5d m5d"
|
||||||
|
for i in $instance_types; do
|
||||||
|
instance_type=$i".xlarge"
|
||||||
|
name=$i$naming_suffix
|
||||||
|
drive_count=$(drive_count_check $i)
|
||||||
|
response=$(echo "{\"ClusterId\": \"j-$instance_type\"}")
|
||||||
|
id=$(echo $response | jq -r '.ClusterId')
|
||||||
|
url="https://aegir-us-east-1.amazon.com/Search?searchText=$id&searchType=Guess&domain=prod&realm=us-east-1"
|
||||||
|
echo "$name :: $id :: $url"
|
||||||
|
echo $url
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
function url_test {
|
||||||
|
id="j-2BGP1R9D3NTD6"
|
||||||
|
url="https://aegir-us-east-1.amazon.com/Search?searchText=$id&searchType=Guess&domain=prod&realm=us-east-1"
|
||||||
|
open $url
|
||||||
|
}
|
||||||
|
|
||||||
|
launch
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
# Usage: lsapp application_1537193066945_0202
|
||||||
|
# Ex. Output:
|
||||||
|
# Application ID: application_1537193066945_0202
|
||||||
|
# AM Node: ip-172-28-115-72 - i-0dc1a1053af30c34c
|
||||||
|
# AM Container: container_1537193066945_0202_01_000001
|
||||||
|
# Application Runtime: 2018-09-27 12:48:00,390 - 2018-09-27 12:48:53,201
|
||||||
|
|
||||||
|
function lsapp() {
|
||||||
|
appID=$1
|
||||||
|
appLogs=$(sift -riz --no-filename --ipath=resourcemanager -e "$appID")
|
||||||
|
baseLog=$(echo $appLogs | sift -riz --no-filename --ipath=resourcemanager -e "$appID.*MasterContainer")
|
||||||
|
AMContainerID=$(echo $baseLog | sift --only-matching -e "container_(\d|_)+")
|
||||||
|
AMNode=$(echo $baseLog | sift --only-matching -e "ip-(\d|-)+")
|
||||||
|
ICLog=$(sift -rizl --limit=1 --file-matches=InstanceJointStatusMap --ipath=daemons/instance-controller $AMNode | sort | tail -1)
|
||||||
|
AMInstanceID=$(sift -riz --limit=1 $AMNode $ICLog | awk '{print $1}')
|
||||||
|
startTime=$(echo $appLogs | sift -riz --no-filename --ipath=resourcemanager -e "$appID.*to submitted" | awk '{print $1, $2}')
|
||||||
|
endTime=$(echo $appLogs | sift -riz --no-filename --ipath=resourcemanager -e "application removed.*$appID.* queue" | awk '{print $1, $2}')
|
||||||
|
echo Application ID: $appID
|
||||||
|
echo AM Node: $AMNode - $AMInstanceID
|
||||||
|
echo AM Container: $AMContainerID
|
||||||
|
echo Application Runtime: $startTime - $endTime
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
# Usage: lscontainer container_1537193066945_0202_01_000002
|
||||||
|
# Ex. Output:
|
||||||
|
# Hostname : ip-172-28-115-100 || InstanceID : i-09da76008179f44ff
|
||||||
|
|
||||||
|
function lscontainer() {
|
||||||
|
containerID=$1
|
||||||
|
containerHostname=$(sift -riz --no-filename --ipath=resourcemanager -e "assigned container $containerID" | sift --only-matching -e "ip-(\d|-)+")
|
||||||
|
ICLog=$(sift -rizl --limit=1 --file-matches=InstanceJointStatusMap --ipath=daemons/instance-controller $containerHostname | sort | tail -1)
|
||||||
|
containerInstanceID=$(sift -riz --limit=1 $containerHostname $ICLog | awk '{print $1}')
|
||||||
|
echo Hostname : $containerHostname \|\| InstanceID : $containerInstanceID
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,10 @@
|
||||||
|
# Usage: lshost ip-172-28-115-41
|
||||||
|
# Ex. Output:
|
||||||
|
# Hostname : ip-172-28-115-41 || InstanceID : i-0cf1a535723e25e56
|
||||||
|
|
||||||
|
function lshost() {
|
||||||
|
Hostname=$1
|
||||||
|
ICLog=$(sift -rizl --limit=1 --file-matches=InstanceJointStatusMap --ipath=daemons/instance-controller $Hostname | sort | tail -1)
|
||||||
|
InstanceID=$(sift -riz --limit=1 $Hostname $ICLog | awk '{print $1}')
|
||||||
|
echo Hostname : $Hostname \|\| InstanceID : $InstanceID
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
unhealthy nodes
|
||||||
|
---------------
|
||||||
|
sift -riz 'reported unhealthy'
|
||||||
|
sift -riz 'local-dirs are bad'
|
||||||
|
sift -riz 'killing container.'
|
||||||
|
|
||||||
|
sift -riz 'application removed - appid: application_1532193836323_78176'
|
||||||
|
sift -riz -e 'application finished -.*application_1532193836323_78176'
|
||||||
|
|
||||||
Loading…
Reference in New Issue