175 lines
4.3 KiB
Bash
175 lines
4.3 KiB
Bash
#!/bin/bash
|
|
|
|
set -x
|
|
|
|
install_jupyter=false
|
|
|
|
# get input parameters
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--jupyter)
|
|
install_jupyter=true
|
|
;;
|
|
-*)
|
|
error_msg "unrecognized option: $1"
|
|
;;
|
|
*)
|
|
break;
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
s3_utils='https://s3.amazonaws.com/ty-emr/XRR/utils'
|
|
build_vim=false
|
|
|
|
if [ "$USER" == "root" ]; then
|
|
util_path="/root/.utils"
|
|
home="/root"
|
|
else
|
|
if [ "$platform" == "Darwin" ]; then
|
|
users_dir="Users"
|
|
fi
|
|
if [ "$platform" == "Linux" ]; then
|
|
users_dir="home"
|
|
fi
|
|
util_path="/$users_dir/$USER/.utils"
|
|
fi
|
|
vim_check=`vim --version`
|
|
if [[ $vim_check != *"+lua"* ]]; then
|
|
build_vim=true
|
|
fi
|
|
mkdir -p $util_path
|
|
|
|
release=`cat /etc/*release* | tr '[:upper:]' '[:lower:]'`
|
|
if [[ $release == *"rhel fedora"* ]]; then
|
|
echo "Looks like we're running on something that is kinda like RHEL..."
|
|
sudo yum groupinstall -y "Development Tools"
|
|
sudo yum install -y \
|
|
tmux \
|
|
wget \
|
|
htop \
|
|
mlocate \
|
|
git \
|
|
rake \
|
|
zsh \
|
|
jq \
|
|
at \
|
|
bind-utils \
|
|
strace \
|
|
lua \
|
|
lua-devel \
|
|
ncurses \
|
|
ncurses-devel \
|
|
gmp \
|
|
gmp-devel \
|
|
ctags \
|
|
tcl-devel \
|
|
perl \
|
|
perl-devel \
|
|
perl-ExtUtils-ParseXS \
|
|
perl-ExtUtils-CBuilder \
|
|
perl-ExtUtils-Embed
|
|
|
|
if [[ $build_vim == true ]]; then
|
|
cd /tmp
|
|
git clone http://luajit.org/git/luajit-2.0.git
|
|
cd luajit-2.0
|
|
make
|
|
sudo make install
|
|
|
|
cd /tmp
|
|
git clone https://github.com/vim/vim.git
|
|
cd vim
|
|
./configure \
|
|
--with-features=huge \
|
|
--enable-cscope \
|
|
--enable-pythoninterp \
|
|
--enable-luainterp \
|
|
--enable-multibyte \
|
|
--enable-fontset \
|
|
--disable-gui \
|
|
--without-x \
|
|
--disable-netbeans \
|
|
--enable-largefile
|
|
make
|
|
sudo make install
|
|
|
|
if [ -e /usr/bin/vi ]; then
|
|
sudo rm /usr/bin/vi
|
|
fi
|
|
sudo ln -s /usr/local/bin/vim /usr/bin/vi
|
|
rm -rf /tmp/vim
|
|
fi
|
|
fi
|
|
|
|
|
|
wget https://bootstrap.pypa.io/get-pip.py
|
|
sudo python2.7 ./get-pip.py
|
|
sudo env "PATH=$PATH" pip install awscli
|
|
wget "https://raw.githubusercontent.com/o0beaner/dotfiles/master/install.sh"
|
|
chmod +x install.sh
|
|
./install.sh
|
|
sudo chmod 644 /usr/bin/chsh
|
|
sudo chmod +x /usr/bin/chsh
|
|
sudo /usr/bin/chsh -s /bin/zsh $USER
|
|
sudo updatedb
|
|
cd $util_path
|
|
wget --no-check-certificate $s3_utils/suntracker.sh
|
|
chmod +x $util_path/suntracker.sh
|
|
(crontab -l ; echo "0 3 * * * $util_path/suntracker.sh") | crontab -
|
|
$util_path/suntracker.sh
|
|
# AWS Instance customization
|
|
if [ -e /usr/bin/cloud-init ]; then
|
|
|
|
# Install SSM Agent
|
|
cd /tmp
|
|
sudo yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm
|
|
|
|
# Am I running EMR?
|
|
instance=`aws ec2 describe-instances --instance-ids $(curl -s 169.254.169.254/latest/meta-data/instance-id)`
|
|
tags=`echo $instance | jq -r '.Reservations[0].Instances[0].Tags[]'`
|
|
cluster_id=`echo $tags | jq -r '. | select(.Key=="aws:elasticmapreduce:job-flow-id") | .Value'`
|
|
if [ -n "$cluster_id" ]; then
|
|
echo "$cluster_id" > ~/.cluster_id
|
|
role=`echo $tags | jq -r '. | select(.Key=="aws:elasticmapreduce:instance-group-role") | .Value'`
|
|
|
|
if [ "$role" == "MASTER" ]; then
|
|
# ToDo: Incorporate Hue?
|
|
|
|
cd $util_path
|
|
wget --no-check-certificate $s3_utils/configure_zeppelin_s3.sh
|
|
chmod +x $util_path/configure_zeppelin_s3.sh
|
|
aws emr add-steps --cluster-id $cluster_id --steps Type=CUSTOM_JAR,Name="Configure Zeppelin for S3",Jar="command-runner.jar",Args=[$util_path/configure_zeppelin_s3.sh]
|
|
fi
|
|
|
|
# install jupyter
|
|
if [ $install_jupyter == true ]; then
|
|
cd $util_path
|
|
wget --no-check-certificate https://s3.amazonaws.com/aws-bigdata-blog/artifacts/aws-blog-emr-jupyter/install-jupyter-emr5.sh
|
|
chmod +x $util_path/install-jupyter-emr5.sh
|
|
$util_path/install-jupyter-emr5.sh \
|
|
--r \
|
|
--julia \
|
|
--toree \
|
|
--torch \
|
|
--ruby \
|
|
--ds-packages \
|
|
--ml-packages \
|
|
--python-packages ggplot nilearn \
|
|
--port 8002 \
|
|
--password jupyter \
|
|
--jupyterhub \
|
|
--jupyterhub-port 8001 \
|
|
--cached-install \
|
|
--notebook-dir s3://ty-emr/XRR/jupyter/notebooks/ \
|
|
--copy-samples \
|
|
--s3fs
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
touch ~/.zsh.prompts
|
|
mkdir ~/.zsh.after/
|
|
echo "prompt agnoster" > ~/.zsh.after/prompt.zsh
|