From 70a57151a4ac56c50fad4604bea006a7db4a7097 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Thu, 26 Jan 2017 09:06:07 -0800 Subject: [PATCH 01/20] adding ingest install.sh --- spot-ingest/install.sh | 122 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100755 spot-ingest/install.sh diff --git a/spot-ingest/install.sh b/spot-ingest/install.sh new file mode 100755 index 00000000..e43b95cf --- /dev/null +++ b/spot-ingest/install.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +nfdump_vers=1.1 +wshark_vers=2.2.3 +local_path=`pwd` +source_path=${local_path}/src +dependencies=(tar wget screen python) +missing_dep=() +wget_cmd="wget -nc --no-check-certificate" +untar_cmd="tar -xvf -k" +host_os="" + +if [ ! -d ${source_path} ]; then + mkdir ${source_path} +fi + +# functions + +log_cmd () { + + echo "" + echo "****SPOT.INGEST.Install.sh****" + date +"%y-%m-%d %H:%M:%S" + echo "$1" + echo "" +} + +check_pkg () { + for item in "$@"; do + if type ${item} >/dev/null 2>&1; then + log_cmd "${item} found" + else + missing_dep+=(${item}) + fi + done +} + +install_pkg () { + if [[ "${missing_dep[@]}" ]]; then + log_cmd "installing ${missing_dep[@]}" + ${install_cmd} ${missing_dep[@]} + unset missing_dep[*] + fi +} + +install_tshark () { + if [ "${host_os}" == "debian" ]; then + log_cmd "installing dependencies for tshark installation" + check_pkg make bzip2 pkg-config libpcap-dev heimdal-dev libc-ares-dev libsmi flex bison byacc + install_pkg + elif [ "${host_os}" == "rhel" ]; then + check_pkg make bzip2 gcc bison glib2-devel flex-devel libsmi-devel libpcap-devel + install_pkg + fi + ${wget_cmd} https://1.na.dl.wireshark.org/src/wireshark-${wshark_vers}.tar.bz2 -P ${source_path}/ + ${untar_cmd} ${source_path}/wireshark-${wshark_vers}.tar.bz2 -C ${source_path}/ + cd ${source_path}/wireshark-${wshark_vers} + log_cmd "compiling tshark" + ./configure --enable-wireshark=no + make + make install + cd .. + + log_cmd "tshark build complete" + tshark -v +} + +install_nfdump () { + log_cmd "installing spot-nfdump" + ${wget_cmd} https://github.com/Open-Network-Insight/spot-nfdump/archive/${nfdump_vers}.tar.gz -P ${source_path}/ + ${untar_cmd} ${source_path}/${nfdump_vers}.tar.gz -C ${source_path}/ + cd ${source_path}/spot-nfdump-*/ + source ./install_nfdump.sh + cd ${local_path} +} + +# detect distribution +# to add other distribution simply create a test case with installation commands + +if [ -f /etc/redhat-release ]; then + install_cmd="yum -y install" + log_cmd "installation command: $install_cmd" + host_os="rhel" +elif [ -f /etc/debian_version ]; then + install_cmd="apt-get install -yq" + log_cmd "installation command: $install_cmd" + host_os="debian" + apt-get update +fi + +# check dependencies +check_pkg ${dependencies[@]} +install_pkg + +if type tshark >/dev/null 2>&1; then + log_cmd "tshark found" + else + log_cmd "tshark missing" + install_tshark +fi + +if type nfdump >/dev/null 2>&1; then + echo "nfdump found" +else + log_cmd "missing nfdump" + install_nfdump +fi + +if type pip >/dev/null 2>&1; then + echo "pip found" +else + log_cmd "missing pip" + ${wget_cmd} https://bootstrap.pypa.io/get-pip.py -P ${source_path}/ + python ${source_path}/get-pip.py + log_cmd "pip installed" +fi + +if [ -z ${local_path}/requirements.txt ]; then + pip install -r requirements.txt +fi + +log_cmd "spot-ingest dependencies installed" From 31b677b2e48565abebb69e9afd73bafc2e522097 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Thu, 26 Jan 2017 16:03:51 -0800 Subject: [PATCH 02/20] added local_setup.sh to Spot-Setup --- spot-setup/README.md | 10 +++++++ spot-setup/local_setup.sh | 56 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100755 spot-setup/local_setup.sh diff --git a/spot-setup/README.md b/spot-setup/README.md index ad98553f..5083f336 100644 --- a/spot-setup/README.md +++ b/spot-setup/README.md @@ -20,6 +20,12 @@ To collaborate and run spot-setup, it is required the following prerequisites: The main script in the repository is **hdfs_setup.sh** which is responsible of loading environment variables, creating folders in Hadoop for the different use cases (flow, DNS or Proxy), create the Hive database, and finally execute hive query scripts that creates Hive tables needed to access netflow, dns and proxy data. +**local_setup.sh** will add some default directories such as /opt/spot/ for installing known dependencies, when running `local_setup.sh` you should do so with root access or via `sudo` +The first parameter will be the default user who owns /opt/spot. +Example: `sudo local_setup.sh [USER]` + +a. [USER] will be the default user for the spot installation + ## Environment Variables **spot.conf** is the file storing the variables needed during the installation process including node assignment, User interface, Machine Learning and Ingest gateway nodes. @@ -28,6 +34,10 @@ This file also contains sources desired to be installed as part of Apache Spot, To read more about these variables, please review the [wiki] (https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration). +## Quick Start + +1. run local_setup.sh with sudo providing the default user as the first parameter `sudo local_setup.sh $USER` + ## Database Query Scripts spot-setup contains a script per use case, as of today, there is a table creation script for each DNS, flow and Proxy data. diff --git a/spot-setup/local_setup.sh b/spot-setup/local_setup.sh new file mode 100755 index 00000000..c47977d7 --- /dev/null +++ b/spot-setup/local_setup.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Create local directories for Apache Spot (Incubating) +# This script needs to be run with root priviliges or it will fail to create the /opt/ directories + +spot_dir=/opt/spot +spot_bin=${spot_dir}/bin +spot_env="export PATH=\$PATH:${spot_bin}" +spot_env_file=/etc/profile.d/spot.sh +spot_user=$1 + +# functions + +log_cmd () { + + printf "\n****SPOT.SETUP.local_setup.sh****\n" + date +"%y-%m-%d %H:%M:%S" + printf "$1\n\n" + +} + +check_dir () { + if [[ -d "${1}" ]]; then + log_cmd "${1} already exists" + else + log_cmd "Creating ${1}" + mkdir ${1} + fi +} + +# check for root +if [[ "$EUID" -ne 0 ]]; then + + log_cmd "Non root user detected, exiting now" + exit 1 + +fi + +# make spot directories +check_dir ${spot_dir} +check_dir ${spot_bin} + +# set permissions +log_cmd "Setting permissions on ${spot_dir}" +chown -R ${spot_user}:${spot_user} ${spot_dir} +chmod 0755 ${spot_bin} + +# add directory to env +if [[ -f "${spot_env_file}" ]]; then + log_cmd "${spot_env_file} already exists" +else + log_cmd "Creating ${spot_env_file}" + echo ${spot_env} >> ${spot_env_file} +fi + +log_cmd "Spot local setup complete" From 5938eaab14c886de07dbba7627081024469b71d5 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Thu, 26 Jan 2017 16:29:22 -0800 Subject: [PATCH 03/20] change install dir --- spot-ingest/install.sh | 43 +++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/spot-ingest/install.sh b/spot-ingest/install.sh index e43b95cf..f425b4c6 100755 --- a/spot-ingest/install.sh +++ b/spot-ingest/install.sh @@ -3,11 +3,12 @@ nfdump_vers=1.1 wshark_vers=2.2.3 local_path=`pwd` -source_path=${local_path}/src +source_path=/tmp/ingest_src +install_path=/opt/spot/bin dependencies=(tar wget screen python) missing_dep=() wget_cmd="wget -nc --no-check-certificate" -untar_cmd="tar -xvf -k" +untar_cmd="tar -xvf" host_os="" if [ ! -d ${source_path} ]; then @@ -18,14 +19,19 @@ fi log_cmd () { - echo "" - echo "****SPOT.INGEST.Install.sh****" + printf "\n****SPOT.INGEST.Install.sh****\n" date +"%y-%m-%d %H:%M:%S" - echo "$1" - echo "" + printf "$1\n\n" } -check_pkg () { +cleanup () { + log_cmd "executing cleanup" + rm -rf ${source_path} +} + +check_bin () { + # check_bin can be used to verify if a certain binary is already installed + for item in "$@"; do if type ${item} >/dev/null 2>&1; then log_cmd "${item} found" @@ -36,6 +42,15 @@ check_pkg () { } install_pkg () { + # if no parameters this will simply installany $missing_deps + # if any parameters provided they will be added $missing_dep + + if [[ "$@" ]]; then + for item in "$@"; do + missing_dep+=(${item}) + done + fi + if [[ "${missing_dep[@]}" ]]; then log_cmd "installing ${missing_dep[@]}" ${install_cmd} ${missing_dep[@]} @@ -46,17 +61,18 @@ install_pkg () { install_tshark () { if [ "${host_os}" == "debian" ]; then log_cmd "installing dependencies for tshark installation" - check_pkg make bzip2 pkg-config libpcap-dev heimdal-dev libc-ares-dev libsmi flex bison byacc - install_pkg + check_bin make bzip2 pkg-config libsmi flex bison byacc + install_pkg libpcap-dev heimdal-dev libc-ares-dev elif [ "${host_os}" == "rhel" ]; then - check_pkg make bzip2 gcc bison glib2-devel flex-devel libsmi-devel libpcap-devel - install_pkg + check_bin make bzip2 gcc bison + install_pkg glib2-devel flex-devel libsmi-devel libpcap-devel fi + ${wget_cmd} https://1.na.dl.wireshark.org/src/wireshark-${wshark_vers}.tar.bz2 -P ${source_path}/ ${untar_cmd} ${source_path}/wireshark-${wshark_vers}.tar.bz2 -C ${source_path}/ cd ${source_path}/wireshark-${wshark_vers} log_cmd "compiling tshark" - ./configure --enable-wireshark=no + ./configure --prefix=${install_path} --enable-wireshark=no make make install cd .. @@ -89,7 +105,7 @@ elif [ -f /etc/debian_version ]; then fi # check dependencies -check_pkg ${dependencies[@]} +check_bin ${dependencies[@]} install_pkg if type tshark >/dev/null 2>&1; then @@ -120,3 +136,4 @@ if [ -z ${local_path}/requirements.txt ]; then fi log_cmd "spot-ingest dependencies installed" +cleanup From 6767a0e94e081764e2e3057c4d3b43487065c5f7 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Wed, 8 Feb 2017 10:25:36 -0800 Subject: [PATCH 04/20] removing permissions change --- spot-setup/local_setup.sh | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/spot-setup/local_setup.sh b/spot-setup/local_setup.sh index c47977d7..555ffcce 100755 --- a/spot-setup/local_setup.sh +++ b/spot-setup/local_setup.sh @@ -19,7 +19,7 @@ log_cmd () { } -check_dir () { +create_dir () { if [[ -d "${1}" ]]; then log_cmd "${1} already exists" else @@ -28,23 +28,17 @@ check_dir () { fi } -# check for root +check_root () { +# checking for root as many of these functions interact with system owned directories if [[ "$EUID" -ne 0 ]]; then - log_cmd "Non root user detected, exiting now" + log_cmd "Non root user detected, Please run as root or with sudo" exit 1 fi +} -# make spot directories -check_dir ${spot_dir} -check_dir ${spot_bin} - -# set permissions -log_cmd "Setting permissions on ${spot_dir}" -chown -R ${spot_user}:${spot_user} ${spot_dir} -chmod 0755 ${spot_bin} - +set_env () { # add directory to env if [[ -f "${spot_env_file}" ]]; then log_cmd "${spot_env_file} already exists" @@ -52,5 +46,14 @@ else log_cmd "Creating ${spot_env_file}" echo ${spot_env} >> ${spot_env_file} fi +} + +check_root + +# make spot directories +create_dir ${spot_dir} +create_dir ${spot_bin} + +set_env log_cmd "Spot local setup complete" From b03b921f4f345b655e0837c64dd9fd740f5da8ef Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Wed, 8 Feb 2017 10:38:20 -0800 Subject: [PATCH 05/20] adding root checker --- spot-ingest/install.sh | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/spot-ingest/install.sh b/spot-ingest/install.sh index f425b4c6..8d646a8a 100755 --- a/spot-ingest/install.sh +++ b/spot-ingest/install.sh @@ -11,10 +11,6 @@ wget_cmd="wget -nc --no-check-certificate" untar_cmd="tar -xvf" host_os="" -if [ ! -d ${source_path} ]; then - mkdir ${source_path} -fi - # functions log_cmd () { @@ -24,6 +20,14 @@ log_cmd () { printf "$1\n\n" } +check_root () { + # checking for root as many of these functions interact with system owned directories + if [[ "$EUID" -ne 0 ]]; then + log_cmd "Non root user detected, Please run as root or with sudo" + exit 1 + fi +} + cleanup () { log_cmd "executing cleanup" rm -rf ${source_path} @@ -42,8 +46,8 @@ check_bin () { } install_pkg () { - # if no parameters this will simply installany $missing_deps - # if any parameters provided they will be added $missing_dep + # if no parameters this will simply install any $missing_deps + # if any parameters provided they will be added to $missing_dep if [[ "$@" ]]; then for item in "$@"; do @@ -86,12 +90,15 @@ install_nfdump () { ${wget_cmd} https://github.com/Open-Network-Insight/spot-nfdump/archive/${nfdump_vers}.tar.gz -P ${source_path}/ ${untar_cmd} ${source_path}/${nfdump_vers}.tar.gz -C ${source_path}/ cd ${source_path}/spot-nfdump-*/ - source ./install_nfdump.sh + source ./install_nfdump.sh ${install_path} cd ${local_path} } +# end functions + +check_root # detect distribution -# to add other distribution simply create a test case with installation commands +# to add other distributions simply create a test case with installation commands if [ -f /etc/redhat-release ]; then install_cmd="yum -y install" @@ -104,7 +111,12 @@ elif [ -f /etc/debian_version ]; then apt-get update fi -# check dependencies +if [ ! -d ${source_path} ]; then + log_cmd "${source_path} not created, Please run spot-setup/local_setup.sh first" + exit 1 +fi + +# check basic dependencies check_bin ${dependencies[@]} install_pkg @@ -116,14 +128,14 @@ if type tshark >/dev/null 2>&1; then fi if type nfdump >/dev/null 2>&1; then - echo "nfdump found" + log_cmd "nfdump found" else log_cmd "missing nfdump" install_nfdump fi if type pip >/dev/null 2>&1; then - echo "pip found" + log_cmd "pip found" else log_cmd "missing pip" ${wget_cmd} https://bootstrap.pypa.io/get-pip.py -P ${source_path}/ From 2ab995c4085dae9b4ab9afc38c933e630a90a927 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Wed, 22 Feb 2017 14:20:27 -0800 Subject: [PATCH 06/20] adding basic install scripts --- spot-ml/install.sh | 120 +++++++++++++++++++++++++++++++++++++++++++++ spot-oa/install.sh | 119 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+) create mode 100755 spot-ml/install.sh create mode 100755 spot-oa/install.sh diff --git a/spot-ml/install.sh b/spot-ml/install.sh new file mode 100755 index 00000000..02a2f858 --- /dev/null +++ b/spot-ml/install.sh @@ -0,0 +1,120 @@ +#!/bin/bash + +local_path=`pwd` +source_path=/tmp/ml_src +install_path=/opt/spot/ +dependencies=(tar wget) +missing_dep=() +wget_cmd="wget -nc --no-check-certificate" +untar_cmd="tar -xvf" +host_os="" +sbt_tar='https://dl.bintray.com/sbt/native-packages/sbt/0.13.13/sbt-0.13.13.tgz' + + +# functions + +log_cmd () { + + printf "\n****SPOT.ML.Install.sh****\n" + date +"%y-%m-%d %H:%M:%S" + printf "$1\n\n" +} + +check_root () { + # checking for root as many of these functions interact with system owned directories + if [[ "$EUID" -ne 0 ]]; then + log_cmd "Non root user detected, Please run as root or with sudo" + exit 1 + fi +} + +cleanup () { + log_cmd "executing cleanup" + rm -rf ${source_path} +} + +check_bin () { + # check_bin can be used to verify if a certain binary is already installed + + for item in "$@"; do + if type ${item} >/dev/null 2>&1; then + log_cmd "${item} found" + else + missing_dep+=(${item}) + fi + done +} + +install_pkg () { + # if no parameters this will simply install any $missing_deps + # if any parameters provided they will be added to $missing_dep + + if [[ "$@" ]]; then + for item in "$@"; do + missing_dep+=(${item}) + done + fi + + if [[ "${missing_dep[@]}" ]]; then + log_cmd "installing ${missing_dep[@]}" + ${install_cmd} ${missing_dep[@]} + unset missing_dep[*] + fi +} + +sbt_install () { + log_cmd 'installing sbt for ${host_os}' + if [[ ${1} == 'debian' ]]; then + echo "deb https://dl.bintray.com/sbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list + sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + sudo apt-get update + sudo apt-get install sbt + elif [[ ${1} == 'rhel']]; then + curl https://bintray.com/sbt/rpm/rpm | sudo tee /etc/yum.repos.d/bintray-sbt-rpm.repo + sudo yum -y install sbt + fi +} + + +# end functions + +check_root + +# detect distribution +# to add other distributions simply create a test case with installation commands + +if [ -f /etc/redhat-release ]; then + install_cmd="yum -y install" + log_cmd "installation command: $install_cmd" + host_os="rhel" +elif [ -f /etc/debian_version ]; then + install_cmd="apt-get install -yq" + log_cmd "installation command: $install_cmd" + host_os="debian" + apt-get update +fi + +if [ ! -d ${install_path} ]; then + log_cmd "${install_path} not created, Please run spot-setup/local_setup.sh first" + exit 1 +fi + +# check basic dependencies +check_bin ${dependencies[@]} +install_pkg + +# install +if type sbt >/dev/null 2>&1; then + log_cmd "sbt found" +else + sbt_install ${host_os} +fi + +# build +log_cmd 'assembling spot-ml jar' +sbt assembly +cp ./target/scala-2.10/spot-ml-assembly-1.1.jar ${install_path}/jar/ +cp ./ml_ops.sh ${install_path}/bin + +log_cmd "spot-ml dependencies installed" +cleanup diff --git a/spot-oa/install.sh b/spot-oa/install.sh new file mode 100755 index 00000000..f2a522cf --- /dev/null +++ b/spot-oa/install.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +local_path=`pwd` +source_path=/tmp/ingest_src +install_path=/opt/spot/ +dependencies=(tar wget screen python) +missing_dep=() +wget_cmd="wget -nc --no-check-certificate" +untar_cmd="tar -xvf" +host_os="" +num_procs=$(nproc) + +# functions + +log_cmd () { + + printf "\n****SPOT.INGEST.Install.sh****\n" + date +"%y-%m-%d %H:%M:%S" + printf "$1\n\n" +} + +check_root () { + # checking for root as many of these functions interact with system owned directories + if [[ "$EUID" -ne 0 ]]; then + log_cmd "Non root user detected, Please run as root or with sudo" + exit 1 + fi +} + +cleanup () { + log_cmd "executing cleanup" + rm -rf ${source_path} +} + +check_bin () { + # check_bin can be used to verify if a certain binary is already installed + + for item in "$@"; do + if type ${item} >/dev/null 2>&1; then + log_cmd "${item} found" + else + missing_dep+=(${item}) + fi + done +} + +install_pkg () { + # if no parameters this will simply install any $missing_deps + # if any parameters provided they will be added to $missing_dep + + if [[ "$@" ]]; then + for item in "$@"; do + missing_dep+=(${item}) + done + fi + + if [[ "${missing_dep[@]}" ]]; then + log_cmd "installing ${missing_dep[@]}" + ${install_cmd} ${missing_dep[@]} + unset missing_dep[*] + fi +} + +npm_install () { + + log_cmd 'installing NodeJS 7 for ${host_os}' + if [[ ${1} == 'debian' ]]; then + curl -sL https://deb.nodesource.com/setup_7.x | sudo -E bash - + sudo apt-get install -y nodejs + + elif [[ ${1} == 'rhel']]; then + curl -sL https://rpm.nodesource.com/setup_7.x | bash - + yum install -y nodejs + fi +} +# end functions + +check_root + +# detect distribution +# to add other distributions simply create a test case with installation commands + +if [ -f /etc/redhat-release ]; then + install_cmd="yum -y install" + log_cmd "installation command: $install_cmd" + host_os="rhel" +elif [ -f /etc/debian_version ]; then + install_cmd="apt-get install -yq" + log_cmd "installation command: $install_cmd" + host_os="debian" + apt-get update +fi + +if [ ! -d ${install_path} ]; then + log_cmd "${install_path} not created, Please run spot-setup/local_setup.sh first" + exit 1 +fi + +# check basic dependencies +check_bin ${dependencies[@]} +install_pkg + +if type pip >/dev/null 2>&1; then + log_cmd "pip found" +else + log_cmd "missing pip" + ${wget_cmd} https://bootstrap.pypa.io/get-pip.py -P ${source_path}/ + python ${source_path}/get-pip.py + log_cmd "pip installed" +fi + +if [ -z ${local_path}/requirements.txt ]; then + pip install -r requirements.txt +fi + +npm_install + +log_cmd "spot-ingest dependencies installed" +cleanup From 8331caeeeb61fde5eee6c9ff7dc50f5776765c0f Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Thu, 23 Feb 2017 11:19:54 -0800 Subject: [PATCH 07/20] stagin to split build and installs --- spot-ingest/build.sh | 153 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100755 spot-ingest/build.sh diff --git a/spot-ingest/build.sh b/spot-ingest/build.sh new file mode 100755 index 00000000..b9cdeb54 --- /dev/null +++ b/spot-ingest/build.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +nfdump_vers=1.1 +wshark_vers=2.2.3 +local_path=`pwd` +source_path=/tmp/ingest_src +install_path=/opt/spot/ +dependencies=(tar wget screen python) +missing_dep=() +wget_cmd="wget -nc --no-check-certificate" +untar_cmd="tar -xvf" +host_os="" +mk_opt="-j `nproc`" + +# functions + +log_cmd () { + printf "\n****SPOT.INGEST.Install.sh****\n" + date +"%y-%m-%d %H:%M:%S" + printf "$1\n\n" +} + +check_root () { + # checking for root as many of these functions interact with system owned directories + if [[ "$EUID" -ne 0 ]]; then + log_cmd "Non root user detected, Please run as root or with sudo" + exit 1 + fi +} + +cleanup () { + log_cmd "executing cleanup" + rm -rf ${source_path} +} + +check_bin () { + # check_bin can be used to verify if a certain binary is already installed + for item in "$@"; do + if type ${item} >/dev/null 2>&1; then + log_cmd "${item} found" + else + missing_dep+=(${item}) + fi + done +} + +install_pkg () { + # if no parameters this will simply install any $missing_deps + # if any parameters provided they will be added to $missing_dep + if [[ "$@" ]]; then + for item in "$@"; do + missing_dep+=(${item}) + done + fi + + if [[ "${missing_dep[@]}" ]]; then + log_cmd "installing ${missing_dep[@]}" + ${install_cmd} ${missing_dep[@]} + unset missing_dep[*] + fi +} + +install_tshark () { + if [ "${host_os}" == "debian" ]; then + log_cmd "installing dependencies for tshark installation" + check_bin make bzip2 pkg-config libsmi flex bison byacc + install_pkg libpcap-dev heimdal-dev libc-ares-dev + elif [ "${host_os}" == "rhel" ]; then + check_bin make bzip2 gcc bison + install_pkg glib2-devel flex-devel libsmi-devel libpcap-devel + fi + + ${wget_cmd} https://1.na.dl.wireshark.org/src/wireshark-${wshark_vers}.tar.bz2 -P ${source_path}/ + ${untar_cmd} ${source_path}/wireshark-${wshark_vers}.tar.bz2 -C ${source_path}/ + cd ${source_path}/wireshark-${wshark_vers} + log_cmd "compiling tshark" + ./configure --prefix=${install_path} --enable-wireshark=no + make ${mk_opt} + make install + cd .. + + log_cmd "tshark build complete" + tshark -v +} + +install_nfdump () { + log_cmd "installing spot-nfdump" + ${wget_cmd} https://github.com/Open-Network-Insight/spot-nfdump/archive/${nfdump_vers}.tar.gz -P ${source_path}/ + ${untar_cmd} ${source_path}/${nfdump_vers}.tar.gz -C ${source_path}/ + cd ${source_path}/spot-nfdump-*/ + source ./install_nfdump.sh ${install_path} + cd ${local_path} +} +# end functions + +check_root + +# detect distribution +# to add other distributions simply create a test case with installation commands + +if [ -f /etc/redhat-release ]; then + install_cmd="yum -y install" + log_cmd "installation command: $install_cmd" + host_os="rhel" +elif [ -f /etc/debian_version ]; then + install_cmd="apt-get install -yq" + log_cmd "installation command: $install_cmd" + host_os="debian" + apt-get update +fi + +if [ ! -d ${source_path} ]; then + mkdir ${source_path} +fi + +if [ ! -d ${install_path} ]; then + log_cmd "${install_path} not created, Please run spot-setup/local_setup.sh first" + exit 1 +fi + +# check basic dependencies +check_bin ${dependencies[@]} +install_pkg + +if type tshark >/dev/null 2>&1; then + log_cmd "tshark found" + else + log_cmd "tshark missing" + install_tshark +fi + +if type nfdump >/dev/null 2>&1; then + log_cmd "nfdump found" +else + log_cmd "missing nfdump" + install_nfdump +fi + +if type pip >/dev/null 2>&1; then + log_cmd "pip found" +else + log_cmd "missing pip" + ${wget_cmd} https://bootstrap.pypa.io/get-pip.py -P ${source_path}/ + python ${source_path}/get-pip.py + log_cmd "pip installed" +fi + +if [ -z ${local_path}/requirements.txt ]; then + pip install -r requirements.txt +fi + +log_cmd "spot-ingest dependencies installed" +cleanup From e547380d5fdd5876c4212b17500db8e3f3ed6eec Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Thu, 23 Feb 2017 11:21:36 -0800 Subject: [PATCH 08/20] updating installs in all directories --- spot-ml/install.sh | 18 ++++++++----- spot-oa/install.sh | 8 +++++- spot-setup/install.sh | 60 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 8 deletions(-) create mode 100755 spot-setup/install.sh diff --git a/spot-ml/install.sh b/spot-ml/install.sh index 02a2f858..0438b393 100755 --- a/spot-ml/install.sh +++ b/spot-ml/install.sh @@ -65,13 +65,13 @@ install_pkg () { sbt_install () { log_cmd 'installing sbt for ${host_os}' if [[ ${1} == 'debian' ]]; then - echo "deb https://dl.bintray.com/sbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 - sudo apt-get update - sudo apt-get install sbt - elif [[ ${1} == 'rhel']]; then - curl https://bintray.com/sbt/rpm/rpm | sudo tee /etc/yum.repos.d/bintray-sbt-rpm.repo - sudo yum -y install sbt + echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + apt-get update + apt-get install sbt + elif [[ ${1} == 'rhel' ]]; then + curl https://bintray.com/sbt/rpm/rpm | tee /etc/yum.repos.d/bintray-sbt-rpm.repo + yum -y install sbt fi } @@ -94,6 +94,10 @@ elif [ -f /etc/debian_version ]; then apt-get update fi +if [ ! -d ${source_path} ]; then + mkdir ${source_path} +fi + if [ ! -d ${install_path} ]; then log_cmd "${install_path} not created, Please run spot-setup/local_setup.sh first" exit 1 diff --git a/spot-oa/install.sh b/spot-oa/install.sh index f2a522cf..acd2a598 100755 --- a/spot-oa/install.sh +++ b/spot-oa/install.sh @@ -1,7 +1,7 @@ #!/bin/bash local_path=`pwd` -source_path=/tmp/ingest_src +source_path=/tmp/oa_src install_path=/opt/spot/ dependencies=(tar wget screen python) missing_dep=() @@ -91,6 +91,10 @@ elif [ -f /etc/debian_version ]; then apt-get update fi +if [ ! -d ${source_path} ]; then + mkdir ${source_path} +fi + if [ ! -d ${install_path} ]; then log_cmd "${install_path} not created, Please run spot-setup/local_setup.sh first" exit 1 @@ -114,6 +118,8 @@ if [ -z ${local_path}/requirements.txt ]; then fi npm_install +cd ui +npm install log_cmd "spot-ingest dependencies installed" cleanup diff --git a/spot-setup/install.sh b/spot-setup/install.sh new file mode 100755 index 00000000..6bcc60fb --- /dev/null +++ b/spot-setup/install.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# Create local directories for Apache Spot (Incubating) +# This script needs to be run with root priviliges or it will fail to create the /opt/ directories + +spot_dir=/opt/spot +spot_bin=${spot_dir}/bin +spot_env="export PATH=\$PATH:${spot_bin}" +spot_env_file=/etc/profile.d/spot.sh +spot_user=$1 + +# functions + +log_cmd () { + + printf "\n****SPOT.SETUP.local_setup.sh****\n" + date +"%y-%m-%d %H:%M:%S" + printf "$1\n\n" + +} + +create_dir () { + if [[ -d "${1}" ]]; then + log_cmd "${1} already exists" + else + log_cmd "Creating ${1}" + mkdir ${1} + fi +} + +check_root () { +# checking for root as many of these functions interact with system owned directories +if [[ "$EUID" -ne 0 ]]; then + + log_cmd "Non root user detected, Please run as root or with sudo" + exit 1 + +fi +} + +set_env () { +# add directory to env +if [[ -f "${spot_env_file}" ]]; then + log_cmd "${spot_env_file} already exists" +else + log_cmd "Creating ${spot_env_file}" + echo ${spot_env} >> ${spot_env_file} +fi +source ${spot_env_file} +} + +check_root + +# make spot directories +create_dir ${spot_dir} +create_dir ${spot_bin} + +set_env + +log_cmd "Spot local setup complete" From 9db16458d677800a558aef4b3f3f68238600acc8 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Thu, 23 Feb 2017 12:35:53 -0800 Subject: [PATCH 09/20] install.sh for root installs, build.sh for /opt/spot/bin installs --- spot-ingest/build.sh | 91 ++++----------------- spot-ingest/install.sh | 174 +++++++++++++---------------------------- 2 files changed, 70 insertions(+), 195 deletions(-) mode change 100755 => 100644 spot-ingest/install.sh diff --git a/spot-ingest/build.sh b/spot-ingest/build.sh index b9cdeb54..a5bbdcb8 100755 --- a/spot-ingest/build.sh +++ b/spot-ingest/build.sh @@ -5,7 +5,6 @@ wshark_vers=2.2.3 local_path=`pwd` source_path=/tmp/ingest_src install_path=/opt/spot/ -dependencies=(tar wget screen python) missing_dep=() wget_cmd="wget -nc --no-check-certificate" untar_cmd="tar -xvf" @@ -15,61 +14,17 @@ mk_opt="-j `nproc`" # functions log_cmd () { - printf "\n****SPOT.INGEST.Install.sh****\n" + printf "\n****SPOT.INGEST.build.sh****\n" date +"%y-%m-%d %H:%M:%S" printf "$1\n\n" } -check_root () { - # checking for root as many of these functions interact with system owned directories - if [[ "$EUID" -ne 0 ]]; then - log_cmd "Non root user detected, Please run as root or with sudo" - exit 1 - fi -} - cleanup () { log_cmd "executing cleanup" rm -rf ${source_path} } -check_bin () { - # check_bin can be used to verify if a certain binary is already installed - for item in "$@"; do - if type ${item} >/dev/null 2>&1; then - log_cmd "${item} found" - else - missing_dep+=(${item}) - fi - done -} - -install_pkg () { - # if no parameters this will simply install any $missing_deps - # if any parameters provided they will be added to $missing_dep - if [[ "$@" ]]; then - for item in "$@"; do - missing_dep+=(${item}) - done - fi - - if [[ "${missing_dep[@]}" ]]; then - log_cmd "installing ${missing_dep[@]}" - ${install_cmd} ${missing_dep[@]} - unset missing_dep[*] - fi -} - install_tshark () { - if [ "${host_os}" == "debian" ]; then - log_cmd "installing dependencies for tshark installation" - check_bin make bzip2 pkg-config libsmi flex bison byacc - install_pkg libpcap-dev heimdal-dev libc-ares-dev - elif [ "${host_os}" == "rhel" ]; then - check_bin make bzip2 gcc bison - install_pkg glib2-devel flex-devel libsmi-devel libpcap-devel - fi - ${wget_cmd} https://1.na.dl.wireshark.org/src/wireshark-${wshark_vers}.tar.bz2 -P ${source_path}/ ${untar_cmd} ${source_path}/wireshark-${wshark_vers}.tar.bz2 -C ${source_path}/ cd ${source_path}/wireshark-${wshark_vers} @@ -91,23 +46,24 @@ install_nfdump () { source ./install_nfdump.sh ${install_path} cd ${local_path} } -# end functions -check_root - -# detect distribution -# to add other distributions simply create a test case with installation commands +check_os () { + # detect distribution + # to add other distributions simply create a test case with installation commands + if [ -f /etc/redhat-release ]; then + install_cmd="yum -y install" + log_cmd "installation command: $install_cmd" + host_os="rhel" + elif [ -f /etc/debian_version ]; then + install_cmd="apt-get install -yq" + log_cmd "installation command: $install_cmd" + host_os="debian" + apt-get update + fi +} +# end functions -if [ -f /etc/redhat-release ]; then - install_cmd="yum -y install" - log_cmd "installation command: $install_cmd" - host_os="rhel" -elif [ -f /etc/debian_version ]; then - install_cmd="apt-get install -yq" - log_cmd "installation command: $install_cmd" - host_os="debian" - apt-get update -fi +check_os if [ ! -d ${source_path} ]; then mkdir ${source_path} @@ -118,10 +74,6 @@ if [ ! -d ${install_path} ]; then exit 1 fi -# check basic dependencies -check_bin ${dependencies[@]} -install_pkg - if type tshark >/dev/null 2>&1; then log_cmd "tshark found" else @@ -136,15 +88,6 @@ else install_nfdump fi -if type pip >/dev/null 2>&1; then - log_cmd "pip found" -else - log_cmd "missing pip" - ${wget_cmd} https://bootstrap.pypa.io/get-pip.py -P ${source_path}/ - python ${source_path}/get-pip.py - log_cmd "pip installed" -fi - if [ -z ${local_path}/requirements.txt ]; then pip install -r requirements.txt fi diff --git a/spot-ingest/install.sh b/spot-ingest/install.sh old mode 100755 new mode 100644 index 8d646a8a..f2eb1625 --- a/spot-ingest/install.sh +++ b/spot-ingest/install.sh @@ -1,151 +1,83 @@ #!/bin/bash -nfdump_vers=1.1 -wshark_vers=2.2.3 local_path=`pwd` source_path=/tmp/ingest_src -install_path=/opt/spot/bin -dependencies=(tar wget screen python) +install_path=/opt/spot/ +dependencies=(tar wget screen python make gcc m4 automake autoconf flex byacc) missing_dep=() -wget_cmd="wget -nc --no-check-certificate" -untar_cmd="tar -xvf" host_os="" # functions log_cmd () { - - printf "\n****SPOT.INGEST.Install.sh****\n" - date +"%y-%m-%d %H:%M:%S" - printf "$1\n\n" + printf "\n****SPOT.INGEST.install.sh****\n" + date +"%y-%m-%d %H:%M:%S" + printf "$1\n\n" } check_root () { - # checking for root as many of these functions interact with system owned directories - if [[ "$EUID" -ne 0 ]]; then - log_cmd "Non root user detected, Please run as root or with sudo" - exit 1 - fi -} - -cleanup () { - log_cmd "executing cleanup" - rm -rf ${source_path} + # checking for root as many of these functions interact with system owned directories + if [[ "$EUID" -ne 0 ]]; then + log_cmd "Non root user detected, Please run as root or with sudo" + exit 1 + fi } check_bin () { - # check_bin can be used to verify if a certain binary is already installed - - for item in "$@"; do - if type ${item} >/dev/null 2>&1; then - log_cmd "${item} found" - else - missing_dep+=(${item}) - fi - done -} + # check_bin can be used to verify if a certain binary is already installed + for item in "$@"; do + if type ${item} >/dev/null 2>&1; then + log_cmd "${item} found" + else + missing_dep+=(${item}) + fi + done + } install_pkg () { - # if no parameters this will simply install any $missing_deps - # if any parameters provided they will be added to $missing_dep - - if [[ "$@" ]]; then - for item in "$@"; do - missing_dep+=(${item}) - done - fi - - if [[ "${missing_dep[@]}" ]]; then - log_cmd "installing ${missing_dep[@]}" - ${install_cmd} ${missing_dep[@]} - unset missing_dep[*] - fi + # if no parameters this will simply install any $missing_deps + # if any parameters provided they will be added to $missing_dep + if [[ "$@" ]]; then + for item in "$@"; do + missing_dep+=(${item}) + done + fi + + if [[ "${missing_dep[@]}" ]]; then + log_cmd "installing ${missing_dep[@]}" + ${install_cmd} ${missing_dep[@]} + unset missing_dep[*] + fi } -install_tshark () { - if [ "${host_os}" == "debian" ]; then +check_tshark () { + # check dependencies only, installs in custom location log_cmd "installing dependencies for tshark installation" - check_bin make bzip2 pkg-config libsmi flex bison byacc - install_pkg libpcap-dev heimdal-dev libc-ares-dev - elif [ "${host_os}" == "rhel" ]; then - check_bin make bzip2 gcc bison - install_pkg glib2-devel flex-devel libsmi-devel libpcap-devel - fi - - ${wget_cmd} https://1.na.dl.wireshark.org/src/wireshark-${wshark_vers}.tar.bz2 -P ${source_path}/ - ${untar_cmd} ${source_path}/wireshark-${wshark_vers}.tar.bz2 -C ${source_path}/ - cd ${source_path}/wireshark-${wshark_vers} - log_cmd "compiling tshark" - ./configure --prefix=${install_path} --enable-wireshark=no - make - make install - cd .. - - log_cmd "tshark build complete" - tshark -v + if [ "${host_os}" == "debian" ]; then + check_bin make bzip2 pkg-config libsmi flex bison byacc + install_pkg libpcap-dev heimdal-dev libc-ares-dev + elif [ "${host_os}" == "rhel" ]; then + check_bin make bzip2 gcc bison + install_pkg glib2-devel flex-devel libsmi-devel libpcap-devel + fi } -install_nfdump () { - log_cmd "installing spot-nfdump" - ${wget_cmd} https://github.com/Open-Network-Insight/spot-nfdump/archive/${nfdump_vers}.tar.gz -P ${source_path}/ - ${untar_cmd} ${source_path}/${nfdump_vers}.tar.gz -C ${source_path}/ - cd ${source_path}/spot-nfdump-*/ - source ./install_nfdump.sh ${install_path} - cd ${local_path} +install_pip () { + if type pip >/dev/null 2>&1; then\ + log_cmd "pip found" + else + log_cmd "missing pip" + ${wget_cmd} https://bootstrap.pypa.io/get-pip.py -P ${source_path}/ + python ${source_path}/get-pip.py + log_cmd "pip installed" + fi } # end functions -check_root - -# detect distribution -# to add other distributions simply create a test case with installation commands - -if [ -f /etc/redhat-release ]; then - install_cmd="yum -y install" - log_cmd "installation command: $install_cmd" - host_os="rhel" -elif [ -f /etc/debian_version ]; then - install_cmd="apt-get install -yq" - log_cmd "installation command: $install_cmd" - host_os="debian" - apt-get update -fi - -if [ ! -d ${source_path} ]; then - log_cmd "${source_path} not created, Please run spot-setup/local_setup.sh first" - exit 1 -fi - # check basic dependencies check_bin ${dependencies[@]} install_pkg +check_tshark +install_pip -if type tshark >/dev/null 2>&1; then - log_cmd "tshark found" - else - log_cmd "tshark missing" - install_tshark -fi - -if type nfdump >/dev/null 2>&1; then - log_cmd "nfdump found" -else - log_cmd "missing nfdump" - install_nfdump -fi - -if type pip >/dev/null 2>&1; then - log_cmd "pip found" -else - log_cmd "missing pip" - ${wget_cmd} https://bootstrap.pypa.io/get-pip.py -P ${source_path}/ - python ${source_path}/get-pip.py - log_cmd "pip installed" -fi - -if [ -z ${local_path}/requirements.txt ]; then - pip install -r requirements.txt -fi - -log_cmd "spot-ingest dependencies installed" -cleanup +log_cmd "dependencies satisfied, please run ./build.sh to complete setup" From d4016fe4b6f89730c469c09b4f359696095d0077 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Thu, 23 Feb 2017 14:00:58 -0800 Subject: [PATCH 10/20] split non root jobs to build.sh, final tweaks before QA --- spot-ingest/build.sh | 29 ++++++++-------- spot-ingest/install.sh | 17 +++++++++ spot-ml/build.sh | 49 ++++++++++++++++++++++++++ spot-ml/install.sh | 68 +++++++++++++----------------------- spot-oa/build.sh | 30 ++++++++++++++++ spot-oa/install.sh | 72 ++++++++++++++++++++------------------- spot-setup/install.sh | 6 ++-- spot-setup/local_setup.sh | 59 -------------------------------- 8 files changed, 175 insertions(+), 155 deletions(-) create mode 100755 spot-ml/build.sh create mode 100755 spot-oa/build.sh delete mode 100755 spot-setup/local_setup.sh diff --git a/spot-ingest/build.sh b/spot-ingest/build.sh index a5bbdcb8..ca1e6b6d 100755 --- a/spot-ingest/build.sh +++ b/spot-ingest/build.sh @@ -19,6 +19,21 @@ log_cmd () { printf "$1\n\n" } +check_os () { + # detect distribution + # to add other distributions simply create a test case with installation commands + if [ -f /etc/redhat-release ]; then + install_cmd="yum -y install" + log_cmd "installation command: $install_cmd" + host_os="rhel" + elif [ -f /etc/debian_version ]; then + install_cmd="apt-get install -yq" + log_cmd "installation command: $install_cmd" + host_os="debian" + apt-get update + fi +} + cleanup () { log_cmd "executing cleanup" rm -rf ${source_path} @@ -47,20 +62,6 @@ install_nfdump () { cd ${local_path} } -check_os () { - # detect distribution - # to add other distributions simply create a test case with installation commands - if [ -f /etc/redhat-release ]; then - install_cmd="yum -y install" - log_cmd "installation command: $install_cmd" - host_os="rhel" - elif [ -f /etc/debian_version ]; then - install_cmd="apt-get install -yq" - log_cmd "installation command: $install_cmd" - host_os="debian" - apt-get update - fi -} # end functions check_os diff --git a/spot-ingest/install.sh b/spot-ingest/install.sh index f2eb1625..b45a693c 100644 --- a/spot-ingest/install.sh +++ b/spot-ingest/install.sh @@ -15,6 +15,21 @@ log_cmd () { printf "$1\n\n" } +check_os () { + # detect distribution + # to add other distributions simply create a test case with installation commands + if [ -f /etc/redhat-release ]; then + install_cmd="yum -y install" + log_cmd "installation command: $install_cmd" + host_os="rhel" + elif [ -f /etc/debian_version ]; then + install_cmd="apt-get install -yq" + log_cmd "installation command: $install_cmd" + host_os="debian" + apt-get update + fi +} + check_root () { # checking for root as many of these functions interact with system owned directories if [[ "$EUID" -ne 0 ]]; then @@ -74,6 +89,8 @@ install_pip () { } # end functions +check_os + # check basic dependencies check_bin ${dependencies[@]} install_pkg diff --git a/spot-ml/build.sh b/spot-ml/build.sh new file mode 100755 index 00000000..984367a8 --- /dev/null +++ b/spot-ml/build.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +local_path=`pwd` +install_path=/opt/spot +host_os="" + + +# functions + +log_cmd () { + + printf "\n****SPOT.ML.build.sh****\n" + date +"%y-%m-%d %H:%M:%S" + printf "$1\n\n" +} + +check_os () { + # detect distribution + # to add other distributions simply create a test case with installation commands + if [ -f /etc/redhat-release ]; then + install_cmd="yum -y install" + log_cmd "installation command: $install_cmd" + host_os="rhel" + elif [ -f /etc/debian_version ]; then + install_cmd="apt-get install -yq" + log_cmd "installation command: $install_cmd" + host_os="debian" + apt-get update + fi +} + + +# end functions + +check_os + +if [ ! -d ${install_path} ]; then + log_cmd "${install_path} not created, Please run spot-setup/install.sh first" + exit 1 +fi + +# build +log_cmd 'assembling spot-ml jar' +sbt assembly +cp ./target/scala-2.10/spot-ml-assembly-1.1.jar ${install_path}/jar/ +cp ./ml_ops.sh ${install_path}/bin + +log_cmd "spot-ml dependencies installed" +cleanup diff --git a/spot-ml/install.sh b/spot-ml/install.sh index 0438b393..14cad469 100755 --- a/spot-ml/install.sh +++ b/spot-ml/install.sh @@ -1,25 +1,37 @@ #!/bin/bash local_path=`pwd` -source_path=/tmp/ml_src -install_path=/opt/spot/ -dependencies=(tar wget) +install_path=/opt/spot +dependencies=(curl) missing_dep=() wget_cmd="wget -nc --no-check-certificate" -untar_cmd="tar -xvf" host_os="" -sbt_tar='https://dl.bintray.com/sbt/native-packages/sbt/0.13.13/sbt-0.13.13.tgz' # functions log_cmd () { - printf "\n****SPOT.ML.Install.sh****\n" + printf "\n****SPOT.ML.install.sh****\n" date +"%y-%m-%d %H:%M:%S" printf "$1\n\n" } +check_os () { + # detect distribution + # to add other distributions simply create a test case with installation commands + if [ -f /etc/redhat-release ]; then + install_cmd="yum -y install" + log_cmd "installation command: $install_cmd" + host_os="rhel" + elif [ -f /etc/debian_version ]; then + install_cmd="apt-get install -yq" + log_cmd "installation command: $install_cmd" + host_os="debian" + apt-get update + fi +} + check_root () { # checking for root as many of these functions interact with system owned directories if [[ "$EUID" -ne 0 ]]; then @@ -28,11 +40,6 @@ check_root () { fi } -cleanup () { - log_cmd "executing cleanup" - rm -rf ${source_path} -} - check_bin () { # check_bin can be used to verify if a certain binary is already installed @@ -58,18 +65,18 @@ install_pkg () { if [[ "${missing_dep[@]}" ]]; then log_cmd "installing ${missing_dep[@]}" ${install_cmd} ${missing_dep[@]} - unset missing_dep[*] + unset missing_dep[*] fi } sbt_install () { log_cmd 'installing sbt for ${host_os}' - if [[ ${1} == 'debian' ]]; then + if [[ ${host_os} == 'debian' ]]; then echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 apt-get update apt-get install sbt - elif [[ ${1} == 'rhel' ]]; then + elif [[ ${host_os} == 'rhel' ]]; then curl https://bintray.com/sbt/rpm/rpm | tee /etc/yum.repos.d/bintray-sbt-rpm.repo yum -y install sbt fi @@ -78,31 +85,9 @@ sbt_install () { # end functions +check_os check_root -# detect distribution -# to add other distributions simply create a test case with installation commands - -if [ -f /etc/redhat-release ]; then - install_cmd="yum -y install" - log_cmd "installation command: $install_cmd" - host_os="rhel" -elif [ -f /etc/debian_version ]; then - install_cmd="apt-get install -yq" - log_cmd "installation command: $install_cmd" - host_os="debian" - apt-get update -fi - -if [ ! -d ${source_path} ]; then - mkdir ${source_path} -fi - -if [ ! -d ${install_path} ]; then - log_cmd "${install_path} not created, Please run spot-setup/local_setup.sh first" - exit 1 -fi - # check basic dependencies check_bin ${dependencies[@]} install_pkg @@ -111,14 +96,7 @@ install_pkg if type sbt >/dev/null 2>&1; then log_cmd "sbt found" else - sbt_install ${host_os} + sbt_install fi -# build -log_cmd 'assembling spot-ml jar' -sbt assembly -cp ./target/scala-2.10/spot-ml-assembly-1.1.jar ${install_path}/jar/ -cp ./ml_ops.sh ${install_path}/bin - log_cmd "spot-ml dependencies installed" -cleanup diff --git a/spot-oa/build.sh b/spot-oa/build.sh new file mode 100755 index 00000000..80c0def5 --- /dev/null +++ b/spot-oa/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +local_path=`pwd` +install_path=/opt/spot/ + +# functions + +log_cmd () { + + printf "\n****SPOT.OA.install.sh****\n" + date +"%y-%m-%d %H:%M:%S" + printf "$1\n\n" +} + +# end functions + +if [ ! -d ${install_path} ]; then + log_cmd "${install_path} not created, Please run spot-setup/install.sh first" + exit 1 +fi + +if [ -z ${local_path}/requirements.txt ]; then + pip install -r requirements.txt +fi + +cd ui +npm install + +log_cmd "spot-oa dependencies installed" +cleanup diff --git a/spot-oa/install.sh b/spot-oa/install.sh index acd2a598..2f9e1d32 100755 --- a/spot-oa/install.sh +++ b/spot-oa/install.sh @@ -3,22 +3,35 @@ local_path=`pwd` source_path=/tmp/oa_src install_path=/opt/spot/ -dependencies=(tar wget screen python) +dependencies=(curl wget screen python) missing_dep=() wget_cmd="wget -nc --no-check-certificate" -untar_cmd="tar -xvf" host_os="" -num_procs=$(nproc) # functions log_cmd () { - printf "\n****SPOT.INGEST.Install.sh****\n" + printf "\n****SPOT.OA.install.sh****\n" date +"%y-%m-%d %H:%M:%S" printf "$1\n\n" } +check_os () { + # detect distribution + # to add other distributions simply create a test case with installation commands + if [ -f /etc/redhat-release ]; then + install_cmd="yum -y install" + log_cmd "installation command: $install_cmd" + host_os="rhel" + elif [ -f /etc/debian_version ]; then + install_cmd="apt-get install -yq" + log_cmd "installation command: $install_cmd" + host_os="debian" + apt-get update + fi +} + check_root () { # checking for root as many of these functions interact with system owned directories if [[ "$EUID" -ne 0 ]]; then @@ -61,36 +74,34 @@ install_pkg () { fi } -npm_install () { +install_pip () { + if type pip >/dev/null 2>&1; then\ + log_cmd "pip found" + else + log_cmd "missing pip" + ${wget_cmd} https://bootstrap.pypa.io/get-pip.py -P ${source_path}/ + python ${source_path}/get-pip.py + log_cmd "pip installed" + fi +} + +install_npm () { log_cmd 'installing NodeJS 7 for ${host_os}' - if [[ ${1} == 'debian' ]]; then - curl -sL https://deb.nodesource.com/setup_7.x | sudo -E bash - - sudo apt-get install -y nodejs + if [[ ${host_os} == 'debian' ]]; then + curl -sL https://deb.nodesource.com/setup_7.x | bash - + apt-get install -y nodejs - elif [[ ${1} == 'rhel']]; then + elif [[ ${host_os} == 'rhel' ]]; then curl -sL https://rpm.nodesource.com/setup_7.x | bash - yum install -y nodejs fi } # end functions +check_os check_root -# detect distribution -# to add other distributions simply create a test case with installation commands - -if [ -f /etc/redhat-release ]; then - install_cmd="yum -y install" - log_cmd "installation command: $install_cmd" - host_os="rhel" -elif [ -f /etc/debian_version ]; then - install_cmd="apt-get install -yq" - log_cmd "installation command: $install_cmd" - host_os="debian" - apt-get update -fi - if [ ! -d ${source_path} ]; then mkdir ${source_path} fi @@ -104,22 +115,13 @@ fi check_bin ${dependencies[@]} install_pkg -if type pip >/dev/null 2>&1; then - log_cmd "pip found" -else - log_cmd "missing pip" - ${wget_cmd} https://bootstrap.pypa.io/get-pip.py -P ${source_path}/ - python ${source_path}/get-pip.py - log_cmd "pip installed" -fi +install_pip if [ -z ${local_path}/requirements.txt ]; then pip install -r requirements.txt fi -npm_install -cd ui -npm install +install_npm -log_cmd "spot-ingest dependencies installed" +log_cmd "spot-oa dependencies installed" cleanup diff --git a/spot-setup/install.sh b/spot-setup/install.sh index 6bcc60fb..0f271f95 100755 --- a/spot-setup/install.sh +++ b/spot-setup/install.sh @@ -4,6 +4,7 @@ # This script needs to be run with root priviliges or it will fail to create the /opt/ directories spot_dir=/opt/spot +spot_jar=${spot_dir}/jar spot_bin=${spot_dir}/bin spot_env="export PATH=\$PATH:${spot_bin}" spot_env_file=/etc/profile.d/spot.sh @@ -13,7 +14,7 @@ spot_user=$1 log_cmd () { - printf "\n****SPOT.SETUP.local_setup.sh****\n" + printf "\n****SPOT.SETUP.install.sh****\n" date +"%y-%m-%d %H:%M:%S" printf "$1\n\n" @@ -54,7 +55,8 @@ check_root # make spot directories create_dir ${spot_dir} create_dir ${spot_bin} +create_dir ${spot_jar} set_env -log_cmd "Spot local setup complete" +log_cmd "spot-setup complete" diff --git a/spot-setup/local_setup.sh b/spot-setup/local_setup.sh deleted file mode 100755 index 555ffcce..00000000 --- a/spot-setup/local_setup.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# Create local directories for Apache Spot (Incubating) -# This script needs to be run with root priviliges or it will fail to create the /opt/ directories - -spot_dir=/opt/spot -spot_bin=${spot_dir}/bin -spot_env="export PATH=\$PATH:${spot_bin}" -spot_env_file=/etc/profile.d/spot.sh -spot_user=$1 - -# functions - -log_cmd () { - - printf "\n****SPOT.SETUP.local_setup.sh****\n" - date +"%y-%m-%d %H:%M:%S" - printf "$1\n\n" - -} - -create_dir () { - if [[ -d "${1}" ]]; then - log_cmd "${1} already exists" - else - log_cmd "Creating ${1}" - mkdir ${1} - fi -} - -check_root () { -# checking for root as many of these functions interact with system owned directories -if [[ "$EUID" -ne 0 ]]; then - - log_cmd "Non root user detected, Please run as root or with sudo" - exit 1 - -fi -} - -set_env () { -# add directory to env -if [[ -f "${spot_env_file}" ]]; then - log_cmd "${spot_env_file} already exists" -else - log_cmd "Creating ${spot_env_file}" - echo ${spot_env} >> ${spot_env_file} -fi -} - -check_root - -# make spot directories -create_dir ${spot_dir} -create_dir ${spot_bin} - -set_env - -log_cmd "Spot local setup complete" From 119c3a0a6d4a4d704987a7ce77b831fcb9b1b00f Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Thu, 23 Feb 2017 14:04:24 -0800 Subject: [PATCH 11/20] chmod +x --- spot-ingest/install.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 spot-ingest/install.sh diff --git a/spot-ingest/install.sh b/spot-ingest/install.sh old mode 100644 new mode 100755 From d5c54cfd662e05c83a47180947d7d7baae5d56ae Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Thu, 23 Feb 2017 16:12:44 -0800 Subject: [PATCH 12/20] cleanup and add post_build.sh for ml --- spot-ingest/build.sh | 97 ------------------------------------------ spot-ingest/install.sh | 65 +++++++++++++++++++++++++++- spot-ml/build.sh | 22 +--------- spot-ml/install.sh | 31 +++++++------- spot-ml/post_build.sh | 19 +++++++++ 5 files changed, 99 insertions(+), 135 deletions(-) delete mode 100755 spot-ingest/build.sh create mode 100644 spot-ml/post_build.sh diff --git a/spot-ingest/build.sh b/spot-ingest/build.sh deleted file mode 100755 index ca1e6b6d..00000000 --- a/spot-ingest/build.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash - -nfdump_vers=1.1 -wshark_vers=2.2.3 -local_path=`pwd` -source_path=/tmp/ingest_src -install_path=/opt/spot/ -missing_dep=() -wget_cmd="wget -nc --no-check-certificate" -untar_cmd="tar -xvf" -host_os="" -mk_opt="-j `nproc`" - -# functions - -log_cmd () { - printf "\n****SPOT.INGEST.build.sh****\n" - date +"%y-%m-%d %H:%M:%S" - printf "$1\n\n" -} - -check_os () { - # detect distribution - # to add other distributions simply create a test case with installation commands - if [ -f /etc/redhat-release ]; then - install_cmd="yum -y install" - log_cmd "installation command: $install_cmd" - host_os="rhel" - elif [ -f /etc/debian_version ]; then - install_cmd="apt-get install -yq" - log_cmd "installation command: $install_cmd" - host_os="debian" - apt-get update - fi -} - -cleanup () { - log_cmd "executing cleanup" - rm -rf ${source_path} -} - -install_tshark () { - ${wget_cmd} https://1.na.dl.wireshark.org/src/wireshark-${wshark_vers}.tar.bz2 -P ${source_path}/ - ${untar_cmd} ${source_path}/wireshark-${wshark_vers}.tar.bz2 -C ${source_path}/ - cd ${source_path}/wireshark-${wshark_vers} - log_cmd "compiling tshark" - ./configure --prefix=${install_path} --enable-wireshark=no - make ${mk_opt} - make install - cd .. - - log_cmd "tshark build complete" - tshark -v -} - -install_nfdump () { - log_cmd "installing spot-nfdump" - ${wget_cmd} https://github.com/Open-Network-Insight/spot-nfdump/archive/${nfdump_vers}.tar.gz -P ${source_path}/ - ${untar_cmd} ${source_path}/${nfdump_vers}.tar.gz -C ${source_path}/ - cd ${source_path}/spot-nfdump-*/ - source ./install_nfdump.sh ${install_path} - cd ${local_path} -} - -# end functions - -check_os - -if [ ! -d ${source_path} ]; then - mkdir ${source_path} -fi - -if [ ! -d ${install_path} ]; then - log_cmd "${install_path} not created, Please run spot-setup/local_setup.sh first" - exit 1 -fi - -if type tshark >/dev/null 2>&1; then - log_cmd "tshark found" - else - log_cmd "tshark missing" - install_tshark -fi - -if type nfdump >/dev/null 2>&1; then - log_cmd "nfdump found" -else - log_cmd "missing nfdump" - install_nfdump -fi - -if [ -z ${local_path}/requirements.txt ]; then - pip install -r requirements.txt -fi - -log_cmd "spot-ingest dependencies installed" -cleanup diff --git a/spot-ingest/install.sh b/spot-ingest/install.sh index b45a693c..7f1280bd 100755 --- a/spot-ingest/install.sh +++ b/spot-ingest/install.sh @@ -1,14 +1,18 @@ #!/bin/bash +nfdump_vers=1.1 +wshark_vers=2.2.3 local_path=`pwd` source_path=/tmp/ingest_src install_path=/opt/spot/ dependencies=(tar wget screen python make gcc m4 automake autoconf flex byacc) missing_dep=() host_os="" +wget_cmd="wget -nc --no-check-certificate" +untar_cmd="tar -xvf" +mk_opt="-j `nproc`" # functions - log_cmd () { printf "\n****SPOT.INGEST.install.sh****\n" date +"%y-%m-%d %H:%M:%S" @@ -30,6 +34,11 @@ check_os () { fi } +cleanup () { + log_cmd "executing cleanup" + rm -rf ${source_path} +} + check_root () { # checking for root as many of these functions interact with system owned directories if [[ "$EUID" -ne 0 ]]; then @@ -77,6 +86,38 @@ check_tshark () { fi } +install_tshark () { + if type tshark >/dev/null 2>&1; then + log_cmd "tshark found" + else + log_cmd "tshark missing" + check_tshark + ${wget_cmd} https://1.na.dl.wireshark.org/src/wireshark-${wshark_vers}.tar.bz2 -P ${source_path}/ + ${untar_cmd} ${source_path}/wireshark-${wshark_vers}.tar.bz2 -C ${source_path}/ + cd ${source_path}/wireshark-${wshark_vers} + log_cmd "compiling tshark" + ./configure --prefix=${install_path} --enable-wireshark=no + make ${mk_opt} + make install + cd .. + fi + log_cmd "tshark build complete" + tshark -v +} + +install_nfdump () { + if type nfdump >/dev/null 2>&1; then + log_cmd "nfdump found" + else + log_cmd "installing spot-nfdump" + ${wget_cmd} https://github.com/Open-Network-Insight/spot-nfdump/archive/${nfdump_vers}.tar.gz -P ${source_path}/ + ${untar_cmd} ${source_path}/${nfdump_vers}.tar.gz -C ${source_path}/ + cd ${source_path}/spot-nfdump-*/ + source ./install_nfdump.sh ${install_path} + cd ${local_path} + fi +} + install_pip () { if type pip >/dev/null 2>&1; then\ log_cmd "pip found" @@ -87,14 +128,34 @@ install_pip () { log_cmd "pip installed" fi } + # end functions +check_root check_os +if [ ! -d ${source_path} ]; then + mkdir ${source_path} +fi + +if [ ! -d ${install_path} ]; then + log_cmd "${install_path} not created, Please run spot-setup/install.sh first" + exit 1 +fi + # check basic dependencies check_bin ${dependencies[@]} install_pkg -check_tshark + +# install dissectors +install_tshark +install_nfdump + +# python dependencies install_pip +if [ -z ${local_path}/requirements.txt ]; then + pip install -r requirements.txt +fi log_cmd "dependencies satisfied, please run ./build.sh to complete setup" +cleanup diff --git a/spot-ml/build.sh b/spot-ml/build.sh index 984367a8..f532b9e5 100755 --- a/spot-ml/build.sh +++ b/spot-ml/build.sh @@ -13,23 +13,7 @@ log_cmd () { date +"%y-%m-%d %H:%M:%S" printf "$1\n\n" } - -check_os () { - # detect distribution - # to add other distributions simply create a test case with installation commands - if [ -f /etc/redhat-release ]; then - install_cmd="yum -y install" - log_cmd "installation command: $install_cmd" - host_os="rhel" - elif [ -f /etc/debian_version ]; then - install_cmd="apt-get install -yq" - log_cmd "installation command: $install_cmd" - host_os="debian" - apt-get update - fi -} - - + # end functions check_os @@ -42,8 +26,6 @@ fi # build log_cmd 'assembling spot-ml jar' sbt assembly -cp ./target/scala-2.10/spot-ml-assembly-1.1.jar ${install_path}/jar/ -cp ./ml_ops.sh ${install_path}/bin log_cmd "spot-ml dependencies installed" -cleanup +log_cmd 'run `sudo ./post_build.sh` to install in /opt/spot/' diff --git a/spot-ml/install.sh b/spot-ml/install.sh index 14cad469..1e1b3c67 100755 --- a/spot-ml/install.sh +++ b/spot-ml/install.sh @@ -70,16 +70,20 @@ install_pkg () { } sbt_install () { - log_cmd 'installing sbt for ${host_os}' - if [[ ${host_os} == 'debian' ]]; then - echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 - apt-get update - apt-get install sbt - elif [[ ${host_os} == 'rhel' ]]; then - curl https://bintray.com/sbt/rpm/rpm | tee /etc/yum.repos.d/bintray-sbt-rpm.repo - yum -y install sbt - fi + if type sbt >/dev/null 2>&1; then + log_cmd "sbt found" + else + log_cmd 'installing sbt for ${host_os}' + if [[ ${host_os} == 'debian' ]]; then + echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list + apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 + apt-get update + apt-get install sbt + elif [[ ${host_os} == 'rhel' ]]; then + curl https://bintray.com/sbt/rpm/rpm | tee /etc/yum.repos.d/bintray-sbt-rpm.repo + yum -y install sbt + fi + fi } @@ -92,11 +96,6 @@ check_root check_bin ${dependencies[@]} install_pkg -# install -if type sbt >/dev/null 2>&1; then - log_cmd "sbt found" -else - sbt_install -fi +sbt_install log_cmd "spot-ml dependencies installed" diff --git a/spot-ml/post_build.sh b/spot-ml/post_build.sh new file mode 100644 index 00000000..92210da1 --- /dev/null +++ b/spot-ml/post_build.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +local_path=`pwd` +install_path=/opt/spot + +# functions + +log_cmd () { + + printf "\n****SPOT.ML.build.sh****\n" + date +"%y-%m-%d %H:%M:%S" + printf "$1\n\n" +} + +log_cmd "copying generated files to /opt/spot/" +cp ./target/scala-2.10/spot-ml-assembly-1.1.jar ${install_path}/jar/ +cp ./ml_ops.sh ${install_path}/bin + +log_cmd "complete" From 94c705ef588880a9bfa04a0080fdf9e01521b59c Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Fri, 24 Feb 2017 09:23:31 -0800 Subject: [PATCH 13/20] move files to /opt/spot/ --- spot-ml/ml_ops.sh | 2 +- spot-ml/post_build.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 spot-ml/post_build.sh diff --git a/spot-ml/ml_ops.sh b/spot-ml/ml_ops.sh index 1ff129e0..42a0c00a 100755 --- a/spot-ml/ml_ops.sh +++ b/spot-ml/ml_ops.sh @@ -77,7 +77,7 @@ time spark-submit --class "org.apache.spot.SuspiciousConnects" \ --conf spark.kryoserializer.buffer.max=512m \ --conf spark.yarn.am.waitTime=100s \ --conf spark.yarn.am.memoryOverhead=${SPK_DRIVER_MEM_OVERHEAD} \ - --conf spark.yarn.executor.memoryOverhead=${SPK_EXEC_MEM_OVERHEAD} target/scala-2.10/spot-ml-assembly-1.1.jar \ + --conf spark.yarn.executor.memoryOverhead=${SPK_EXEC_MEM_OVERHEAD} /opt/spot/jar/spot-ml-assembly-1.1.jar \ --analysis ${DSOURCE} \ --input ${RAWDATA_PATH} \ --dupfactor ${DUPFACTOR} \ diff --git a/spot-ml/post_build.sh b/spot-ml/post_build.sh old mode 100644 new mode 100755 index 92210da1..6aa754d2 --- a/spot-ml/post_build.sh +++ b/spot-ml/post_build.sh @@ -7,7 +7,7 @@ install_path=/opt/spot log_cmd () { - printf "\n****SPOT.ML.build.sh****\n" + printf "\n****SPOT.ML.post_build.sh****\n" date +"%y-%m-%d %H:%M:%S" printf "$1\n\n" } From a21310f9e1b483757aa2641cb71cbd1600f6c90d Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Fri, 24 Feb 2017 11:51:34 -0800 Subject: [PATCH 14/20] remove naughty wget commands --- spot-ingest/install.sh | 2 +- spot-ml/install.sh | 2 +- spot-oa/install.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spot-ingest/install.sh b/spot-ingest/install.sh index 7f1280bd..9926a0a8 100755 --- a/spot-ingest/install.sh +++ b/spot-ingest/install.sh @@ -8,7 +8,7 @@ install_path=/opt/spot/ dependencies=(tar wget screen python make gcc m4 automake autoconf flex byacc) missing_dep=() host_os="" -wget_cmd="wget -nc --no-check-certificate" +wget_cmd="wget -nc" untar_cmd="tar -xvf" mk_opt="-j `nproc`" diff --git a/spot-ml/install.sh b/spot-ml/install.sh index 1e1b3c67..d235c284 100755 --- a/spot-ml/install.sh +++ b/spot-ml/install.sh @@ -4,7 +4,7 @@ local_path=`pwd` install_path=/opt/spot dependencies=(curl) missing_dep=() -wget_cmd="wget -nc --no-check-certificate" +wget_cmd="wget -nc" host_os="" diff --git a/spot-oa/install.sh b/spot-oa/install.sh index 2f9e1d32..c7826b0a 100755 --- a/spot-oa/install.sh +++ b/spot-oa/install.sh @@ -5,7 +5,7 @@ source_path=/tmp/oa_src install_path=/opt/spot/ dependencies=(curl wget screen python) missing_dep=() -wget_cmd="wget -nc --no-check-certificate" +wget_cmd="wget -nc" host_os="" # functions From b6da6fdeef12c49851199f4448c08d5a567879f8 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Fri, 24 Feb 2017 11:51:57 -0800 Subject: [PATCH 15/20] cleanup --- spot-oa/build.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spot-oa/build.sh b/spot-oa/build.sh index 80c0def5..0e93865d 100755 --- a/spot-oa/build.sh +++ b/spot-oa/build.sh @@ -19,10 +19,6 @@ if [ ! -d ${install_path} ]; then exit 1 fi -if [ -z ${local_path}/requirements.txt ]; then - pip install -r requirements.txt -fi - cd ui npm install From 95d4c5f95720aaaacf98d5e235de16aa2af5e647 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Fri, 24 Feb 2017 12:12:06 -0800 Subject: [PATCH 16/20] add changes to readme --- spot-ingest/README.md | 3 ++- spot-ml/README.md | 6 ++++++ spot-oa/README.md | 6 +++--- spot-setup/README.md | 11 +++++------ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/spot-ingest/README.md b/spot-ingest/README.md index acfb3828..5e9685bb 100644 --- a/spot-ingest/README.md +++ b/spot-ingest/README.md @@ -18,7 +18,8 @@ Ingest data is captured or transferred into the Hadoop cluster, where they are t * Ingest user with sudo privileges (i.e. spot). This user will execute all the processes in the Ingest Framework also this user needs to have access to hdfs solution path (i.e. /user/spot/). ### Install -1. Install Python dependencies `pip install -r requirements.txt` + +1. run `sudo ./install.sh` ### Configure Kafka **Adding Kafka Service:** diff --git a/spot-ml/README.md b/spot-ml/README.md index 3200c064..f06d6793 100644 --- a/spot-ml/README.md +++ b/spot-ml/README.md @@ -11,6 +11,12 @@ netflow and DNS records, and spot-ml will try to load data to the operational an The remaining instructions in this README file treat spot-ml in a stand-alone fashion that might be helpful for customizing and troubleshooting the component. +### Install + +1. run `sudo ./install.sh` +2. run `./build.sh` +3. run `sudo ./post_build.sh` + ## Prepare data for input Load data for consumption by spot-ml by running [spot-ingest]. diff --git a/spot-oa/README.md b/spot-oa/README.md index 27976e62..11cdc463 100644 --- a/spot-oa/README.md +++ b/spot-oa/README.md @@ -10,13 +10,13 @@ Some of the technologies used are: - [Bootstrap](http://getbootstrap.com/) - [ReactJS](https://facebook.github.io/react/) -** For more specific requirements, please refer to each specific pipeline readme file before running OA.* +** For more specific requirements, please refer to each specific pipeline readme file before running OA. ** ---------- ## **Installation** -1. Install python dependencies `pip install -r requirements.txt` -2. Install and build NPM requirements with `npm install` +1. run `sudo ./install.sh` +2. run `./build.sh` ## **Folder Structure** diff --git a/spot-setup/README.md b/spot-setup/README.md index 83fe2504..d2280bb2 100644 --- a/spot-setup/README.md +++ b/spot-setup/README.md @@ -20,11 +20,9 @@ To collaborate and run spot-setup, it is required the following prerequisites: The main script in the repository is **hdfs_setup.sh** which is responsible of loading environment variables, creating folders in Hadoop for the different use cases (flow, DNS or Proxy), create the Hive database, and finally execute hive query scripts that creates Hive tables needed to access netflow, dns and proxy data. -**local_setup.sh** will add some default directories such as /opt/spot/ for installing known dependencies, when running `local_setup.sh` you should do so with root access or via `sudo` +**install.sh** will add some default directories such as /opt/spot/ for installing known dependencies, when running `install.sh` you should do so with root access or via `sudo` The first parameter will be the default user who owns /opt/spot. -Example: `sudo local_setup.sh [USER]` - -a. [USER] will be the default user for the spot installation +Example: `sudo install.sh` ## Environment Variables @@ -34,9 +32,10 @@ This file also contains sources desired to be installed as part of Apache Spot, To read more about these variables, please review the [wiki] (https://github.com/Open-Network-Insight/open-network-insight/wiki/Edit%20Solution%20Configuration). -## Quick Start +## Install -1. run local_setup.sh with sudo providing the default user as the first parameter `sudo local_setup.sh $USER` +1. run `sudo ./install.sh` +2. run `hdfs_setup.sh` ## Database Query Scripts From eb7633d4a875a0acbf5a46fb5f00b940c230cfdb Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Mon, 10 Apr 2017 12:16:12 -0700 Subject: [PATCH 17/20] unifying spot ml setup scripts...again --- spot-ml/build.sh | 31 ------------------------------- spot-ml/install.sh | 13 ++++++++++++- spot-ml/post_build.sh | 19 ------------------- 3 files changed, 12 insertions(+), 51 deletions(-) delete mode 100755 spot-ml/build.sh delete mode 100755 spot-ml/post_build.sh diff --git a/spot-ml/build.sh b/spot-ml/build.sh deleted file mode 100755 index f532b9e5..00000000 --- a/spot-ml/build.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -local_path=`pwd` -install_path=/opt/spot -host_os="" - - -# functions - -log_cmd () { - - printf "\n****SPOT.ML.build.sh****\n" - date +"%y-%m-%d %H:%M:%S" - printf "$1\n\n" -} - -# end functions - -check_os - -if [ ! -d ${install_path} ]; then - log_cmd "${install_path} not created, Please run spot-setup/install.sh first" - exit 1 -fi - -# build -log_cmd 'assembling spot-ml jar' -sbt assembly - -log_cmd "spot-ml dependencies installed" -log_cmd 'run `sudo ./post_build.sh` to install in /opt/spot/' diff --git a/spot-ml/install.sh b/spot-ml/install.sh index d235c284..b42909c6 100755 --- a/spot-ml/install.sh +++ b/spot-ml/install.sh @@ -73,7 +73,7 @@ sbt_install () { if type sbt >/dev/null 2>&1; then log_cmd "sbt found" else - log_cmd 'installing sbt for ${host_os}' + log_cmd "installing sbt for ${host_os}" if [[ ${host_os} == 'debian' ]]; then echo "deb https://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823 @@ -98,4 +98,15 @@ install_pkg sbt_install +# build +log_cmd 'assembling spot-ml jar' +sbt assembly + +log_cmd "spot-ml dependencies installed" + +# post build +log_cmd "copying generated files to /opt/spot/" +cp ./target/scala-2.10/spot-ml-assembly-1.1.jar ${install_path}/jar/ +cp ./ml_ops.sh ${install_path}/bin + log_cmd "spot-ml dependencies installed" diff --git a/spot-ml/post_build.sh b/spot-ml/post_build.sh deleted file mode 100755 index 6aa754d2..00000000 --- a/spot-ml/post_build.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -local_path=`pwd` -install_path=/opt/spot - -# functions - -log_cmd () { - - printf "\n****SPOT.ML.post_build.sh****\n" - date +"%y-%m-%d %H:%M:%S" - printf "$1\n\n" -} - -log_cmd "copying generated files to /opt/spot/" -cp ./target/scala-2.10/spot-ml-assembly-1.1.jar ${install_path}/jar/ -cp ./ml_ops.sh ${install_path}/bin - -log_cmd "complete" From 51fa0a23ae41361c9f5a8c098f0a036c9a3f1685 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Mon, 10 Apr 2017 12:20:32 -0700 Subject: [PATCH 18/20] unified spot-oa setup files --- spot-oa/build.sh | 26 -------------------------- spot-oa/install.sh | 4 ++++ 2 files changed, 4 insertions(+), 26 deletions(-) delete mode 100755 spot-oa/build.sh diff --git a/spot-oa/build.sh b/spot-oa/build.sh deleted file mode 100755 index 0e93865d..00000000 --- a/spot-oa/build.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -local_path=`pwd` -install_path=/opt/spot/ - -# functions - -log_cmd () { - - printf "\n****SPOT.OA.install.sh****\n" - date +"%y-%m-%d %H:%M:%S" - printf "$1\n\n" -} - -# end functions - -if [ ! -d ${install_path} ]; then - log_cmd "${install_path} not created, Please run spot-setup/install.sh first" - exit 1 -fi - -cd ui -npm install - -log_cmd "spot-oa dependencies installed" -cleanup diff --git a/spot-oa/install.sh b/spot-oa/install.sh index c7826b0a..91fe3dc3 100755 --- a/spot-oa/install.sh +++ b/spot-oa/install.sh @@ -123,5 +123,9 @@ fi install_npm +# build ui +cd ui +npm install + log_cmd "spot-oa dependencies installed" cleanup From a17a14339e00c532214f8d0e1ac652a8436bdfd1 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Tue, 11 Apr 2017 10:18:48 -0700 Subject: [PATCH 19/20] final adjustments, added documentation --- spot-ingest/README.md | 6 +++++- spot-ingest/install.sh | 6 ++++-- spot-ml/README.md | 9 ++++++--- spot-ml/install.sh | 2 +- spot-oa/README.md | 5 +++-- spot-oa/install.sh | 2 +- spot-setup/README.md | 9 +++++++++ 7 files changed, 29 insertions(+), 10 deletions(-) diff --git a/spot-ingest/README.md b/spot-ingest/README.md index 5e9685bb..9a489c5f 100644 --- a/spot-ingest/README.md +++ b/spot-ingest/README.md @@ -19,8 +19,12 @@ Ingest data is captured or transferred into the Hadoop cluster, where they are t ### Install -1. run `sudo ./install.sh` +run `sudo ./install.sh` +* If your environment requires proxies we recommend using `sudo -E ./install.sh` +* dependencies installed in /opt/spot/bin/ +* Installs [tshark](https://www.wireshark.org/docs/man-pages/tshark.html), [spot-nfdump](https://github.com/Open-Network-Insight/spot-nfdump), [Python PIP](https://pip.pypa.io/en/stable/) + ### Configure Kafka **Adding Kafka Service:** diff --git a/spot-ingest/install.sh b/spot-ingest/install.sh index 9926a0a8..96c01d29 100755 --- a/spot-ingest/install.sh +++ b/spot-ingest/install.sh @@ -153,9 +153,11 @@ install_nfdump # python dependencies install_pip -if [ -z ${local_path}/requirements.txt ]; then + +if [ -f ${local_path}/requirements.txt ]; then pip install -r requirements.txt fi -log_cmd "dependencies satisfied, please run ./build.sh to complete setup" +log_cmd "spot-ingest dependencies installed" + cleanup diff --git a/spot-ml/README.md b/spot-ml/README.md index f06d6793..e2af8e02 100644 --- a/spot-ml/README.md +++ b/spot-ml/README.md @@ -13,9 +13,12 @@ component. ### Install -1. run `sudo ./install.sh` -2. run `./build.sh` -3. run `sudo ./post_build.sh` +run `sudo ./install.sh` + +* If your environment requires proxies we recommend using `sudo -E ./install.sh` +* Installs [SBT](http://www.scala-sbt.org) +* Compiles the spot-ml jar and copies to /opt/spot/jar/ +* Copies ml_ops.sh to /opt/spot/bin/ ## Prepare data for input diff --git a/spot-ml/install.sh b/spot-ml/install.sh index b42909c6..77fbd32e 100755 --- a/spot-ml/install.sh +++ b/spot-ml/install.sh @@ -106,7 +106,7 @@ log_cmd "spot-ml dependencies installed" # post build log_cmd "copying generated files to /opt/spot/" -cp ./target/scala-2.10/spot-ml-assembly-1.1.jar ${install_path}/jar/ +cp ./target/scala-2.10/spot-ml-assembly-*.jar ${install_path}/jar/ cp ./ml_ops.sh ${install_path}/bin log_cmd "spot-ml dependencies installed" diff --git a/spot-oa/README.md b/spot-oa/README.md index 11cdc463..a3117849 100644 --- a/spot-oa/README.md +++ b/spot-oa/README.md @@ -15,8 +15,9 @@ Some of the technologies used are: ## **Installation** -1. run `sudo ./install.sh` -2. run `./build.sh` +run `sudo ./install.sh` + +Note: If your environment requires proxies we recommend using `sudo -E ./install.sh` ## **Folder Structure** diff --git a/spot-oa/install.sh b/spot-oa/install.sh index 91fe3dc3..d9ae2ba3 100755 --- a/spot-oa/install.sh +++ b/spot-oa/install.sh @@ -117,7 +117,7 @@ install_pkg install_pip -if [ -z ${local_path}/requirements.txt ]; then +if [ -f ${local_path}/requirements.txt ]; then pip install -r requirements.txt fi diff --git a/spot-setup/README.md b/spot-setup/README.md index d2280bb2..8b5130d6 100644 --- a/spot-setup/README.md +++ b/spot-setup/README.md @@ -37,6 +37,15 @@ To read more about these variables, please review the [wiki] (https://github.com 1. run `sudo ./install.sh` 2. run `hdfs_setup.sh` +** install.sh ** + +* Creates /opt/spot/ to store common spot applications +* Creates /etc/profile.d/spot.sh for spot specific environment variables + +** hdfs_setup.sh ** + +* Depending on your Apache Hadoop environment you may need to use sudo -u i.e. `sudo -u hdfs ./hdfs_setup.sh` + ## Database Query Scripts spot-setup contains a script per use case, as of today, there is a table creation script for each DNS, flow and Proxy data. From a30012d4d2383a627b9b212457ef809eb9063eb8 Mon Sep 17 00:00:00 2001 From: natedogs911 Date: Tue, 11 Apr 2017 10:37:38 -0700 Subject: [PATCH 20/20] fixed documentation --- spot-oa/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spot-oa/README.md b/spot-oa/README.md index a3117849..004b7852 100644 --- a/spot-oa/README.md +++ b/spot-oa/README.md @@ -17,7 +17,8 @@ Some of the technologies used are: run `sudo ./install.sh` -Note: If your environment requires proxies we recommend using `sudo -E ./install.sh` +* If your environment requires proxies we recommend using `sudo -E ./install.sh` +* Installs [NPM](https://docs.npmjs.com/cli/install), [Python PIP](https://pip.pypa.io/en/stable/) ## **Folder Structure**