shell script to install spark 2.1.1

#! /bin/bash
echo download scala
wget http://downloads.typesafe.com/scala/2.12.2/scala-2.12.2.tgz
tar xvf scala-2.12.2.tgz
mv scala-2.12.2 /usr/lib
ln -s /usr/lib/scala-2.12.2 /usr/lib/scala
export PATH=$PATH:/usr/lib/scala/bin
scala -version

echo download spark
wget http://d3kbcqa49mib13.cloudfront.net/spark-2.1.1-bin-hadoop2.7.tgz
tar xvf spark-2.1.1-bin-hadoop2.7.tgz

export SPARK_HOME=$HOME/spark-2.1.1-bin-hadoop2.7
export PATH=$PATH:$SPARK_HOME/bin

firewall-cmd –permanent –zone=public –add-port=6066/tcp
firewall-cmd –permanent –zone=public –add-port=7077/tcp
firewall-cmd –permanent –zone=public –add-port=8080-8081/tcp
firewall-cmd –reload

echo ‘export PATH=$PATH:/usr/lib/scala/bin’ >> .bash_profile
echo ‘export SPARK_HOME=$HOME/spark-2.12.2-bin-hadoop2.7’ >> .bash_profile
echo ‘export PATH=$PATH:$SPARK_HOME/bin’ >> .bash_profile

a script to install basic Java dev tools for Vagrant provision

#! /bin/bash
echo install jdk1.8
wget –no-cookies –no-check-certificate –header “Cookie: oraclelicense=accept-securebackup-cookie” “http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.rpm”
yum localinstall jdk-8u131-linux-x64.rpm -y
touch /etc/profile.d/java.sh
cat <<EOT >> /etc/profile.d/java.sh
export JAVA_HOME=”/usr/java/jdk1.8.0_131″
export JAVA_PATH=”$JAVA_HOME”
export PATH=”$PATH:$JAVA_HOME”
EOT
rm jdk-8u131-linux-x64.rpm

echo install ant
wget http://www.us.apache.org/dist/ant/binaries/apache-ant-1.10.1-bin.tar.gz
tar xvfz apache-ant-1.10.1-bin.tar.gz -C /opt
ln -s /opt/apache-ant-1.10.1 /opt/ant
sh -c ‘echo ANT_HOME=/opt/ant>>/etc/environment’
ln -s /opt/ant/bin/ant /usr/bin/ant
rm apache-ant-1.10.1-bin.tar.gz

echo install python3.6
yum -y groupinstall development
yum -y install https://centos7.iuscommunity.org/ius-release.rpm
yum -y install python36u python36u-pip python36u-devel
# yum -y install python36u-pip
# yum -y install python36u-devel

echo install Ansible
yum -y install ansible

echo set basic vim style
touch /etc/vimrc
cat <<EOT >> /etc/vimrc
filetype plugin indent on
set tabstop=2
set shiftwidth=2
set expandtab
set softtabstop=2
set nowrap
set number
color blue
EOT

echo install maven
wget http://apache.mirror.gtcomm.net/maven/maven-3/3.5.0/binaries/apache-maven-3.5.0-bin.tar.gz
tar xvf apache-maven-3.5.0-bin.tar.gz -C /usr/local
rm apache-maven-3.5.0-bin.tar.gz
ln -s /usr/local/apache-maven-3.5.0/ /usr/local/maven
touch /etc/profile.d/maven.sh
cat <<EOT >> /etc/profile.d/maven.sh
#! /bin/bash

export M2_HOME=/usr/local/maven
export PATH=$PATH:/usr/local/maven/bin
EOT

su -l vagrant -c “mkdir /home/vagrant/.m2”
touch /home/vagrant/.m2/settings.xml
cat <<EOT >> /home/vagrant/.m2/settings.xml
<settings>
<proxies>
<proxy>
<active>true</active>
<protocol>http</protocol>
<host>localhost</host>
<port>3128</port>
<nonProxyHosts>127.0.0.1</nonProxyHosts>
</proxy>
</proxies>
</settings>
EOT

chown vagrant /home/vagrant/.m2/settings.xml

echo install SDKMAN
yum install -y zip unzip
su -l vagrant -c ‘curl -s “https://get.sdkman.io&#8221; | bash’
su -l vagrant -c ‘source /home/vagrant/.sdkman/bin/sdkman-init.sh’

echo install Spring Boot
su -l vagrant -c ‘sdk install springboot’

echo install Intellij
sudo yum install -y xauth libXtst
wget http://download.jetbrains.com/idea/ideaIC-2017.1.3.tar.gz
tar xvf ideaIC-2017.1.3.tar.gz
mv idea-IC-171.4424.56 /usr/lib
ln -sf /usr/lib/idea-IC-171.4424.56 /usr/lib/idea
ln -sf /usr/lib/idea/bin/idea.sh /usr/bin/idea
sed -i ‘s/Xm\(s\|x\)[[:digit:]]*m/Xmx\12048m/g’ /usr/lib/idea/bin/idea.vmoptions /usr/lib/idea/bin/idea64.vmoptions
sed -i ‘s/ReservedCodeCacheSize=[[:digit:]]*m/ReservedCodeCacheSize=700m/g’ /usr/lib/idea/bin/idea.vmoptions /usr/lib/idea/bin/idea64.vmoptions
sed -i ‘4i -XX:MaxPermSize=512m’ /usr/lib/idea/bin/idea.vmoptions /usr/lib/idea/bin/idea64.vmoptions
cat <<EOT >> /etc
sysctl -p

How to create a headless Selenium Server vagrant box

Vagrantfile:

Vagrant.configure(“2”) do |config|
config.vm.box = “bento/ubuntu-16.04”
config.ssh.forward_agent = true

config.vm.network “forwarded_port”, guest: 4444, host: 4444

config.vm.provision ‘setup’, type: ‘shell’, path: ‘setup.sh’
config.vm.provision ‘cleanup’, type: ‘shell’, path: ‘cleanup’

end

setup.sh

#!/bin/sh
set -e

if [ -e /.installed ]; then
echo ‘Already installed.’

else
echo ”
echo ‘INSTALLING’
echo ‘———-‘

# Add Google public key to apt
wget -q -O – “https://dl-ssl.google.com/linux/linux_signing_key.pub&#8221; | sudo apt-key add –

# Add Google to the apt-get source list
echo “deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main” > /etc/apt/sources.list.d/google-chrome.list

# Update app-get
apt-get update

# Install Java, Chrome, Xvfb, and unzip
add-apt-repository ppa:webupd8team/java -y
apt-get update -y
echo “oracle-java8-installer shared/accepted-oracle-license-v1-1 select true” | debconf-set-selections
apt-get install -y oracle-java8-installer
apt-get -y install google-chrome-stable xvfb unzip

# Download and copy the ChromeDriver to /usr/local/bin
cd /tmp
wget -N http://chromedriver.storage.googleapis.com/2.29/chromedriver_linux64.zip
wget http://selenium-release.storage.googleapis.com/3.4/selenium-server-standalone-3.4.0.jar
unzip chromedriver_linux64.zip
mv chromedriver /usr/local/bin
mv selenium-server-standalone-3.4.0.jar /usr/local/bin

# So that running `vagrant provision` doesn’t redownload everything
touch /.installed
fi

echo Create xvfb startup script
touch /etc/systemd/system/xvfb.service
chmod 644 /etc/systemd/system/xvfb.service
cat <<EOT >> /etc/systemd/system/xvfb.service
[Unit]
Description=Xvfb
After=syslog.target network.target

[Service]
Type=oneshot
RemainAfterExit=yes
User=vagrant
ExecStart=/bin/xvfb.sh
Environment=DISPLAY=:10

[Install]
WantedBy=multi-user.target
EOT
touch /bin/xvfb.sh
chown :vagrant /bin/xvfb.sh
chmod 750 /bin/xvfb.sh
cat <<EOT >> /bin/xvfb.sh
#! /bin/bash

Xvfb :10 -screen 0 1366x768x24 -ac &
EOT

echo Create google-chrome startup script
touch /etc/systemd/system/google-chrome.service
chmod 644 /etc/systemd/system/google-chrome.service
cat <<EOT >> /etc/systemd/system/google-chrome.service
[Unit]
Description=Google Chrome
After=xvfb.service

[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/bin/google-chrome.sh
Environment=DISPLAY=:10

[Install]
WantedBy=multi-user.target
EOT
touch /bin/google-chrome.sh
chmod 750 /bin/google-chrome.sh
cat <<EOT >> /bin/google-chrome.sh
#! /bin/bash

google-chrome –remote-debugging-port=92222 &
EOT

echo Create selenium-server startup script
touch /etc/systemd/system/selenium-server.service
chmod 644 /etc/systemd/system/selenium-server.service
cat <<EOT >> /etc/systemd/system/selenium-server.service
[Unit]
Description=Selenium Standalone Server
After=google-chrome

[Service]
Type=oneshot
RemainAfterExit=yes
User=vagrant
ExecStart=/bin/selenium-server.sh

[Install]
WantedBy=multi-user.target
EOT
touch /bin/selenium-server.sh
chown :vagrant /bin/selenium-server.sh
chmod 750 /bin/selenium-server.sh
cat <<EOT >> /bin/selenium-server.sh
#! /bin/bash

export DISPLAY=:10
cd /usr/local/bin
nohup java -jar ./selenium-server-standalone-3.4.0.jar &
EOT

systemctl daemon-reload
systemctl enable xvfb
systemctl enable google-chrome
systemctl enable selenium-server
systemctl start xvfb
systemctl start google-chrome
systemctl start selenium-server

cleanup.sh

#! /bin/bash

apt-get autoclean
apt-get clean
apt-get autoremove

Enable X11 forwarding in Vagrantbox running in Cygwin

References:

http://x.cygwin.com/docs/ug/setup.html

https://coderwall.com/p/ozhfva/run-graphical-programs-within-vagrantboxes

https://superuser.com/questions/310197/how-do-i-fix-a-cannot-open-display-error-when-opening-an-x-program-after-sshi

  1. First, you need to install Cygwin/X packages as below:
    xorg-server
    xinit
    xorg-docs (optional)
    xlaunch (optional)
  2. In your Vagrantfile, add the following:
    Vagrant.config(2) do |vb|
    config.ssh.forward_agent = true
    config.ssh.forward_x11 = true
    end
    reboot your vagrant box by running ‘vagrant reload’
  3. In your vagrant box, install xauth ‘sudo yum install -y xauth’ (assume you are using CentOS)
  4. Start X Server in Cygwin:
    startxwin — -multiwindow -listen tcp &
  5. Pay attention to the output of startxwin, set up DISPLAY environment variable in Cygwin, ex. ‘export DISPLAY=’localhost:0.0’
  6. In your Vagrant box, make sure /etc/ssh/sshd_config has ‘X11Forwarding yes’ set up.
  7. Run a GUI app in Vagrant box such as:
    ssh -Y -p 2222 -i vagrant@localhost xclock
    (You need to make sure your private key has a mode of 600)

Note:

if you get error ‘Permission denied (publickey,gssapi-keyex,gssapi-with-mic)’, you need to make sure ‘config.ssh.forward_agent=true’ is added to your vagrant file and ‘Vagrant reload’ is executed.

A sample script installing jdk, ansible, and ant in Vagrant box

#! /bin/bash
echo install jdk1.8
wget –no-cookies –no-check-certificate –header “Cookie: oraclelicense=accept-securebackup-cookie” “http://download.oracle.com/otn-pub/java/jdk/8u131-b11/d54c1d3a095b4ff2b6607d096fa80163/jdk-8u131-linux-x64.rpm&#8221;
yum localinstall jdk-8u131-linux-x64.rpm -y
touch /etc/profile.d/java.sh
cat <<EOT >> /etc/profile.d/java.sh
export JAVA_HOME=”/usr/java/jdk1.8.0_131″
export JAVA_PATH=”$JAVA_HOME”
export PATH=”$PATH:$JAVA_HOME”
EOT
rm jdk-8u131-linux-x64.rpm

echo install ant
wget http://www.us.apache.org/dist/ant/binaries/apache-ant-1.10.1-bin.tar.gz
tar xvfz apache-ant-1.10.1-bin.tar.gz -C /opt
ln -s /opt/apache-ant-1.10.1 /opt/ant
sh -c ‘echo ANT_HOME=/opt/ant>>/etc/environment’
ln -s /opt/ant/bin/ant /usr/bin/ant
rm apache-ant-1.10.1-bin.tar.gz

echo install python3.6
yum -y groupinstall development
yum -y install https://centos7.iuscommunity.org/ius-release.rpm
yum -y install python36u
yum -y install python36u-pip
yum -y install python36u-devel

echo install Ansible
yum -y install ansible

echo set basic vim style
touch /etc/vimrc
cat <<EOT >> /etc/vimrc
filetype plugin indent on
set tabstop=2
set shiftwidth=2
set expandtab
set softtabstop=2
set nowrap
set number
color blue
EOT

Fix Vagrant 1.9.4 issue: An established connection was aborted by the software in your host machine

from: https://github.com/mitchellh/vagrant/issues/8520

Copy en.yml, errors.rb and communicator.rb and replace the corresponding files in:

  • C:/HashiCorp/Vagrant/embedded/gems/gems/vagrant-1.9.4/templates/locales/en.yml
  • C:/HashiCorp/Vagrant/embedded/gems/gems/vagrant-1.9.4/lib/vagrant/errors.rb
  • C:/HashiCorp/Vagrant/embedded/gems/gems/vagrant-1.9.4/plugins/communicators/ssh/communicator.rb

This issue will be fixed in vagrant 1.9.5

Enable GUI on Vagrant Unbuntu 16.04 box

from: http://stackoverflow.com/questions/18878117/using-vagrant-to-run-virtual-machines-with-desktop-environment

  1. Get a basic Ubuntu image working. You should be able to boot it and vagrant ssh.
  2. Next, enable the VirtualBox display, which is off by default. Halt the VM and uncomment these lines in Vagrantfile:
    config.vm.provider :virtualbox do |vb|
      vb.gui = true
    end
  3. Boot the VM and observe the new display window. Now you just need to install and start xfce4. Use vagrant ssh and:
    sudo apt-get install xfce4
    sudo startxfce4&
    

That’s it, you should be landed in a xfce4 session.

Update: For a better experience, I recommend these improvements:

  1. Don’t start the GUI as root. You really want to stay the vagrant user. To do this you need to permit anyone to start the GUI: sudo vim /etc/X11/Xwrapper.config and edit it to allowed_users=anybody.
  2. Next, install the VirtualBox guest tools before starting the GUI. This will give you a healthy screen resolution, integrated mouse, etc.
    $ sudo apt-get install -y xfce4 virtualbox-guest-dkms virtualbox-guest-utils virtualbox-guest-x11
    $ sudo VBoxClient-all
  3. Only now should you start the GUI as the vagrant user, with $ startxfce4&.

Update 2: Tried this today and the VBoxClient-all script isn’t always installed. If it’s missing, you can replace with the equivalent:

sudo VBoxClient --clipboard
sudo VBoxClient --draganddrop
sudo VBoxClient --display
sudo VBoxClient --checkhostversion
sudo VBoxClient --seamless

 

Vagrantfile:

Vagrant.configure(2) do |config|
  # Ubuntu 15.10
  config.vm.box = "ubuntu/wily64"

  config.vm.provider "virtualbox" do |vb|
    # Display the VirtualBox GUI when booting the machine
    vb.gui = true
  end

  # Install xfce and virtualbox additions
  config.vm.provision "shell", inline: "sudo apt-get update"
  config.vm.provision "shell", inline: "sudo apt-get install -y xfce4 virtualbox-guest-dkms virtualbox-guest-utils virtualbox-guest-x11"
  # Permit anyone to start the GUI
  config.vm.provision "shell", inline: "sudo sed -i 's/allowed_users=.*$/allowed_users=anybody/' /etc/X11/Xwrapper.config"
end

To start the vm

vagrant up

Login with username: vagrant, password: vagrant via the login prompt on the virtualbox GUI.

Start xfce

startx

Use Vagrant vm as Ansible control machine

1. Create Vagrantfile as below:

Vagrant.configure(“2”) do |config|

config.ssh.insert_key = “false”

config.vm.define “ansible” do |ansible|
ansible.vm.box = “centos7”
ansible.vm.provision “shell”, path: “provision.sh”
end

config.vm.define “vagrant1” do |vagrant1|
vagrant1.vm.box = “ubuntu/trusty64”
vagrant1.vm.network “private_network”, ip: “192.168.33.11”
end

end

2. The content of provision.sh

#!/usr/bin/env bash

echo “Add extra packages for pip”
sudo rpm -iUvh http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-9.noarch.rpm
echo “update packages”
sudo yum -y update
echo “Install libraries”
sudo yum -y install python-pip python-devel python-lxml openssl-devel
echo “Update pip”
sudo pip install –upgrade pip
echo “Install ansible”
sudo pip install ansible
3. The ansible machine will be used to control the vagrant1 machine

4. After vagrant up, login into ansible machine by: vagrant ssh ansible
create private/public key and upload the public key to vagrant1 (make sure you can ping it first)

5. Add an entry to your /etc/hosts file: (optional)
[target server id] [hostname]
ex: 192.168.33.11 testserv

6. create an Ansible inventory file hosts with the below content:
localhost ansible_connection=local (this line is added so that Ansible can provision local machine)
testserver ansible_host=testsrv ansible_user=vagrant

(Assume ‘testsrv’ is the hostname you add in your /etc/hosts file, if you didn’t add a hostname for the target server, then use its ip instead)

7. Run the following command to make sure Ansible can talk to both local and remote machine
ansible all -i hosts -m ping