SparkEMRBootstrap/install.sh.template at master · ianoc/SparkEMRBootstrap · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
mkdir /tmp/deb
cd /tmp/deb
curl -O ${DEB_HOST}/${NAME}.deb

sudo dpkg -i ${NAME}.deb
sudo dpkg -i /tmp/mesos_0.9.0-1_amd64.deb

sudo chown -R hadoop:hadoop /home/hadoop/{scala,spark}
sudo ldconfig

CPUS=`grep processor /proc/cpuinfo | wc -l`
MEM_KB=`cat /proc/meminfo | grep MemTotal | awk '{print $2}'`
MEM=$[(MEM_KB - 1024 * 1024) / 1024]
MEMG=$[MEM / 1024]
RESOURCES="cpus:$CPUS;mem:$MEM"
MEM_SPARK=$[(MEM-500)]

MASTER=$(grep -i "job.tracker<" /home/hadoop/conf/mapred-site.xml | grep -o '[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}\.[0-9]\{1,3\}')

BASE_SPARK_CONF=$( cat <<EOF
export MESOS_NATIVE_LIBRARY=/usr/local/lib/libmesos.so\n
export SCALA_HOME=/home/hadoop/scala\n
export SPARK_LIBRARY_PATH=/home/hadoop/native/Linux-amd64-64\n
export SPARK_MEM=${MEM_SPARK}m\n
export SPARK_HOME=/home/hadoop/spark\n
export LD_LIBRARY_PATH=/home/hadoop/.versions/0.20.205/native/Linux-amd64-64\n
export MASTER=master@${MASTER}:5050\n
export SPARK_CLASSPATH=\$SPARK_CLASSPATH:/home/hadoop/hadoop-core-0.20.205.jar:/home/hadoop/lib/aws-java-sdk-1.2.16.jar
EOF
)
echo -e $BASE_SPARK_CONF > /home/hadoop/spark/conf/spark-env.sh
echo -e $BASE_SPARK_CONF >> /home/hadoop/.bashrc

sudo chown hadoop /usr/local/var/mesos/conf/mesos.conf
echo "resources=$RESOURCES" >> /usr/local/var/mesos/conf/mesos.conf

if [ -d "/mnt2" ]
   then
        echo "export SPARK_JAVA_OPTS=\"-Dspark.local.dir=/mnt1/spark,/mnt2/spark,/mnt3/spark -Dspark.kryoserializer.buffer.mb=10 -XX:+UseCompressedOops\"" >> /home/hadoop/spark/conf/spark-env.sh
elif [ -d "/mnt/tmp" ]
   then
        echo "export SPARK_JAVA_OPTS=\"-Dspark.local.dir=/mnt/tmp -Dspark.kryoserializer.buffer.mb=10\"" >> /home/hadoop/spark/conf/spark-env.sh
else
        echo "export SPARK_JAVA_OPTS=\"-Dspark.local.dir=/tmp -Dspark.kryoserializer.buffer.mb=10\"" >> /home/hadoop/spark/conf/spark-env.sh
fi


sudo sed -i "s/master=/master=master@$MASTER:5050/" /usr/local/var/mesos/conf/mesos.conf
ln -s /home/hadoop/.versions/0.20.205/conf/core-site.xml /home/hadoop/spark/conf/

sudo chown hadoop:hadoop /etc/init.d/mesos-master.sh /etc/init.d/mesos-slave.sh
sudo chmod a+rx /etc/init.d/mesos-master.sh /etc/init.d/mesos-slave.sh
sudo chown hadoop:hadoop /usr/local/var/mesos/conf/mesos.conf
sudo chmod u+rw /usr/local/var/mesos/conf/mesos.conf

sudo chown -R hadoop:hadoop /home/hadoop/{scala,spark}
grep -Fq '"isMaster":true' /mnt/var/lib/info/instance.json
if [ $? -eq 0 ];
then
	/etc/init.d/mesos-master.sh start
else
	/etc/init.d/mesos-slave.sh start
fi