v3.4.1

11 files changed, 64 insertions, 428 deletions

diff --git a/.SRCINFO b/.SRCINFO
index 76f85b71adb0..35786c308dc7 100644
--- a/.SRCINFO
+++ b/.SRCINFO
@@ -1,34 +1,20 @@
 pkgbase = apache-spark
-	pkgdesc = Apache Spark is a unified analytics engine for large-scale data processing.
-	pkgver = 3.3.0
-	pkgrel = 0
+	pkgdesc = A unified analytics engine for large-scale data processing
+	pkgver = 3.4.1
+	pkgrel = 1
 	url = http://spark.apache.org
 	install = apache-spark.install
 	arch = any
 	license = APACHE
-	depends = java-environment>=6
-	depends = java-environment<=11
-	optdepends = python2: python2 support for pyspark
-	optdepends = ipython2: ipython2 support for pyspark
-	optdepends = python: python3 support for pyspark
-	optdepends = ipython: ipython3 support for pyspark
-	optdepends = r: support for sparkR
-	optdepends = rsync: support rsync hadoop binaries from master
-	optdepends = hadoop: support for running on YARN
-	backup = etc/apache-spark/spark-env.sh
-	source = https://archive.apache.org/dist/spark/spark-3.3.0/spark-3.3.0-bin-hadoop3.tgz
+	depends = inetutils
+	depends = java-runtime-headless<=17
+	source = https://dlcdn.apache.org/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3.tgz
+	source = apache-spark.sh
 	source = apache-spark-master.service
-	source = apache-spark-slave@.service
-	source = spark-env.sh
-	source = spark-daemon-run.sh
-	source = run-master.sh
-	source = run-slave.sh
-	sha256sums = 306b550f42ce1b06772d6084c545ef8448414f2bf451e0b1175405488f2a322f
-	sha256sums = e4333e4a484543786e82d1f7af144ad99d9cc666d2e2742f30b5120996f32183
-	sha256sums = e656ef5599d095472641e6fd58e9046d7db3f983b334e9a695e278ba799afc5b
-	sha256sums = 0e9c3721cfac02e3b67248e1dfe2def49a9fc6408092b291b1c67e89a9c130cb
-	sha256sums = 6ded9f6b31b8be5fa782fc18ec0991eb7cb18fbea65b8e7560587c3fbe7f20f7
-	sha256sums = 1134342330c7680e7d9847cc4de2c0f97ecd55ee7db1c6068bc45219b5838e98
-	sha256sums = 4e1159b25d0f6f3dcdf72d1a5f186e5693a5e9d8690ad1600a9e44aa43022e29
+	source = apache-spark-worker@.service
+	sha256sums = de24e511aebd95e7408c636fde12d19391f57a33730fe30735d6742180e338d4
+	sha256sums = 0cc82baad4d878d4e2bc5864a00b99d38f2906781ea47ee6282546788e797049
+	sha256sums = de54c025ca8ce34a7b4fd95ec7b8d5dec44582787a0bd8da09232f26e2182c9a
+	sha256sums = 47e6c154daecf7631ac9a33fe53a76888070c823c4381fcbde8d98377e586505
 
 pkgname = apache-spark
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000000..c7c9d1935f01
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+pkg
+src
+*.tgz
+*.pkg.*
\ No newline at end of file
diff --git a/PKGBUILD b/PKGBUILD
index 26754de21508..8476199f6bde 100644
--- a/PKGBUILD
+++ b/PKGBUILD
@@ -1,88 +1,49 @@
-# Maintainer: Lukas Zimmermann ("lukaszimmermann") <luk [dot] zim91 [at] gmail.com>
+# Maintainer: William Tang <galaxyking0419@gmail.com>
+# Contributor: Lukas Zimmermann ("lukaszimmermann") <luk [dot] zim91 [at] gmail.com>
 # Contributor: François Garillot ("huitseeker") <francois [at] garillot.net>
 # Contributor: Christian Krause ("wookietreiber") <kizkizzbangbang@gmail.com>
 # Contributor: Emanuel Fontelles ("emanuelfontelles") <emanuelfontelles@hotmail.com>
 
 pkgname=apache-spark
-pkgver=3.3.0
-pkgrel=0
-pkgdesc="Apache Spark is a unified analytics engine for large-scale data processing."
+pkgver=3.4.1
+pkgrel=1
+pkgdesc="A unified analytics engine for large-scale data processing"
 arch=('any')
 url="http://spark.apache.org"
 license=('APACHE')
-depends=('java-environment>=6' 'java-environment<=11')
-optdepends=('python2: python2 support for pyspark'
-            'ipython2: ipython2 support for pyspark'
-            'python: python3 support for pyspark'
-            'ipython: ipython3 support for pyspark'
-            'r: support for sparkR'
-            'rsync: support rsync hadoop binaries from master'
-            'hadoop: support for running on YARN')
+depends=('inetutils' 'java-runtime-headless<=17')
 
-install=apache-spark.install
-source=("https://archive.apache.org/dist/spark/spark-${pkgver}/spark-${pkgver}-bin-hadoop3.tgz"
+source=("https://dlcdn.apache.org/spark/spark-$pkgver/spark-$pkgver-bin-hadoop3.tgz"
+        'apache-spark.sh'
         'apache-spark-master.service'
-        'apache-spark-slave@.service'
-        'spark-env.sh'
-        'spark-daemon-run.sh'
-        'run-master.sh'
-        'run-slave.sh')
-
-sha256sums=('4e0846207bf10311de43451bc99309086fce7990aaf54bf3038608b1981afbe7'
-            'e4333e4a484543786e82d1f7af144ad99d9cc666d2e2742f30b5120996f32183'
-            'e656ef5599d095472641e6fd58e9046d7db3f983b334e9a695e278ba799afc5b'
-            '0e9c3721cfac02e3b67248e1dfe2def49a9fc6408092b291b1c67e89a9c130cb'
-            '6ded9f6b31b8be5fa782fc18ec0991eb7cb18fbea65b8e7560587c3fbe7f20f7'
-            '1134342330c7680e7d9847cc4de2c0f97ecd55ee7db1c6068bc45219b5838e98'
-            '4e1159b25d0f6f3dcdf72d1a5f186e5693a5e9d8690ad1600a9e44aa43022e29')
+        'apache-spark-worker@.service')
+sha256sums=('de24e511aebd95e7408c636fde12d19391f57a33730fe30735d6742180e338d4'
+            '0cc82baad4d878d4e2bc5864a00b99d38f2906781ea47ee6282546788e797049'
+            'de54c025ca8ce34a7b4fd95ec7b8d5dec44582787a0bd8da09232f26e2182c9a'
+            '47e6c154daecf7631ac9a33fe53a76888070c823c4381fcbde8d98377e586505')
 
-backup=('etc/apache-spark/spark-env.sh')
+install=apache-spark.install
 
-package() {
-        cd "$srcdir/spark-${pkgver}-bin-hadoop3"
+prepare() {
+    cd spark-${pkgver}-bin-hadoop3
 
-        install -d "${pkgdir}/usr/bin" "${pkgdir}/opt" "${pkgdir}/var/log/apache-spark" "${pkgdir}/var/lib/apache-spark/work"
-        chmod 2775 "${pkgdir}/var/log/apache-spark" "${pkgdir}/var/lib/apache-spark/work"
+    # Remove Python and R files
+    rm -rf python R
 
-        cp -r "${srcdir}/spark-${pkgver}-bin-hadoop3" "${pkgdir}/opt/apache-spark/"
+    # Remove windows batch files
+    rm bin/*.cmd
+}
 
-        cd "${pkgdir}/usr/bin"
-        for binary in beeline pyspark sparkR spark-class spark-shell find-spark-home spark-sql spark-submit load-spark-env.sh; do
-                local binpath="/opt/apache-spark/bin/${binary}"
-                ln -s "${binpath}" ${binary}
-                sed -i 's|^export SPARK_HOME=.*$|export SPARK_HOME=/opt/apache-spark|' "${pkgdir}/${binpath}"
-                sed -i -Ee 's/\$\(dirname "\$0"\)/$(dirname "$(readlink -f "$0")")/g' "${pkgdir}/${binpath}"
-        done
+package() {
+    # Create directory structure
+    mkdir -p $pkgdir/{etc/profile.d,opt,usr/lib/systemd/system}
 
-        mkdir -p ${pkgdir}/etc/profile.d
-        echo '#!/bin/sh' > ${pkgdir}/etc/profile.d/apache-spark.sh
-        echo 'export SPARK_HOME=/opt/apache-spark' >> ${pkgdir}/etc/profile.d/apache-spark.sh
-        chmod 755 ${pkgdir}/etc/profile.d/apache-spark.sh
+    # Install path profile
+    cp $pkgname.sh $pkgdir/etc/profile.d/
 
-        install -Dm644 "${srcdir}/apache-spark-master.service" "${pkgdir}/usr/lib/systemd/system/apache-spark-master.service"
-        install -Dm644 "${srcdir}/apache-spark-slave@.service" "${pkgdir}/usr/lib/systemd/system/apache-spark-slave@.service"
-        install -Dm644 "${srcdir}/spark-env.sh" "${pkgdir}/etc/apache-spark/spark-env.sh"
-        for script in run-master.sh run-slave.sh spark-daemon-run.sh; do
-            install -Dm755 "${srcdir}/${script}" "${pkgdir}/opt/apache-spark/sbin/${script}"
-        done
-        install -Dm644 "${srcdir}/spark-${pkgver}-bin-hadoop3/conf"/* "${pkgdir}/etc/apache-spark"
+    # Install systemd services
+    cp $pkgname-master.service $pkgname-worker@.service $pkgdir/usr/lib/systemd/system/
 
-        cd "${pkgdir}/opt/apache-spark"
-        mv conf conf-templates
-        ln -sf "/etc/apache-spark" conf
-        ln -sf "/var/lib/apache-spark/work" .
+    # Install program files
+    mv spark-${pkgver}-bin-hadoop3 $pkgdir/opt/$pkgname
 }
-sha256sums=('a78c30450ac862338dbc77e6e97bae69569e2c30615efa082d28d47c0781afef'
-            'e4333e4a484543786e82d1f7af144ad99d9cc666d2e2742f30b5120996f32183'
-            'e656ef5599d095472641e6fd58e9046d7db3f983b334e9a695e278ba799afc5b'
-            '0e9c3721cfac02e3b67248e1dfe2def49a9fc6408092b291b1c67e89a9c130cb'
-            '6ded9f6b31b8be5fa782fc18ec0991eb7cb18fbea65b8e7560587c3fbe7f20f7'
-            '1134342330c7680e7d9847cc4de2c0f97ecd55ee7db1c6068bc45219b5838e98'
-            '4e1159b25d0f6f3dcdf72d1a5f186e5693a5e9d8690ad1600a9e44aa43022e29')
-sha256sums=('306b550f42ce1b06772d6084c545ef8448414f2bf451e0b1175405488f2a322f'
-            'e4333e4a484543786e82d1f7af144ad99d9cc666d2e2742f30b5120996f32183'
-            'e656ef5599d095472641e6fd58e9046d7db3f983b334e9a695e278ba799afc5b'
-            '0e9c3721cfac02e3b67248e1dfe2def49a9fc6408092b291b1c67e89a9c130cb'
-            '6ded9f6b31b8be5fa782fc18ec0991eb7cb18fbea65b8e7560587c3fbe7f20f7'
-            '1134342330c7680e7d9847cc4de2c0f97ecd55ee7db1c6068bc45219b5838e98'
-            '4e1159b25d0f6f3dcdf72d1a5f186e5693a5e9d8690ad1600a9e44aa43022e29')
diff --git a/apache-spark-master.service b/apache-spark-master.service
index b8bc98bce44c..69f2a3c34eb7 100644
--- a/apache-spark-master.service
+++ b/apache-spark-master.service
@@ -3,10 +3,13 @@ Description=Apache Spark Standalone Master
 After=network.target
 
 [Service]
+Type=forking
 User=apache-spark
 Group=apache-spark
 Environment=SPARK_LOG_DIR=/var/log/apache-spark
-ExecStart=/opt/apache-spark/sbin/run-master.sh
+ExecStart=/opt/apache-spark/sbin/start-master.sh
+ExecStop=/opt/apache-spark/sbin/stop-master.sh
+SuccessExitStatus=143
 
 [Install]
 WantedBy=multi-user.target
diff --git a/apache-spark-slave@.service b/apache-spark-worker@.service
index 6a19518bdc14..ee18632d1281 100644
--- a/apache-spark-slave@.service
+++ b/apache-spark-worker@.service
@@ -3,11 +3,13 @@ Description=Apache Spark Standalone Slave
 After=network.target
 
 [Service]
+Type=forking
 User=apache-spark
 Group=apache-spark
 Environment=SPARK_LOG_DIR=/var/log/apache-spark
-ExecStart=/opt/apache-spark/sbin/run-slave.sh %i
+ExecStart=/opt/apache-spark/sbin/start-worker.sh %i
+ExecStop=/opt/apache-spark/sbin/stop-worker.sh %i
+SuccessExitStatus=143
 
 [Install]
 WantedBy=multi-user.target
-DefaultInstance=127.0.0.1:7077
diff --git a/apache-spark.install b/apache-spark.install
index 7aa03480833c..1e56d359455d 100644
--- a/apache-spark.install
+++ b/apache-spark.install
@@ -1,14 +1,13 @@
 post_install() {
-  groupadd -r -f apache-spark
-  useradd -r -g apache-spark -s /usr/bin/nologin -d /var/lib/apache-spark apache-spark || true
-
-  [[ ! -d /var/lib/apache-spark/work ]] &&
-  install -d /var/lib/apache-spark/work
-
-  chown -R apache-spark:apache-spark /var/{lib,log}/apache-spark
+	mkdir -p /var/{lib,log}/apache-spark /opt/apache-spark/work
+	groupadd -r -f apache-spark
+	useradd -r -g apache-spark -s /usr/bin/nologin -d /var/lib/apache-spark apache-spark || true
+	chown -R apache-spark:apache-spark /var/{lib,log}/apache-spark /opt/apache-spark/work
 }
 
-post_remove() {
-  /usr/sbin/userdel apache-spark
-  /usr/sbin/groupdel apache-spark
+pre_remove() {
+	pkill -u apache-spark
+	sleep 4
+	rm -rf /var/log/apache-spark /opt/apache-spark/work
+	userdel -r apache-spark
 }
diff --git a/apache-spark.sh b/apache-spark.sh
new file mode 100644
index 000000000000..2b9b3420b69b
--- /dev/null
+++ b/apache-spark.sh
@@ -0,0 +1 @@
+export PATH=$PATH:/opt/apache-spark/bin:/opt/apache-spark/sbin
\ No newline at end of file
diff --git a/run-master.sh b/run-master.sh
deleted file mode 100755
index a60ca791adc1..000000000000
--- a/run-master.sh
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Starts the master on the machine this script is executed on.
-
-if [ -z "${SPARK_HOME}" ]; then
-  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-fi
-
-# NOTE: This exact class name is matched downstream by SparkSubmit.
-# Any changes need to be reflected there.
-CLASS="org.apache.spark.deploy.master.Master"
-
-if [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
-  echo "Usage: ./sbin/run-master.sh [options]"
-  pattern="Usage:"
-  pattern+="\|Using Spark's default log4j profile:"
-  pattern+="\|Registered signal handlers for"
-
-  "${SPARK_HOME}"/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2
-  exit 1
-fi
-
-ORIGINAL_ARGS="$@"
-
-START_TACHYON=false
-
-while (( "$#" )); do
-case $1 in
-    --with-tachyon)
-      if [ ! -e "${SPARK_HOME}"/tachyon/bin/tachyon ]; then
-        echo "Error: --with-tachyon specified, but tachyon not found."
-        exit -1
-      fi
-      START_TACHYON=true
-      ;;
-  esac
-shift
-done
-
-. "${SPARK_HOME}/sbin/spark-config.sh"
-
-. "${SPARK_HOME}/bin/load-spark-env.sh"
-
-if [ "$SPARK_MASTER_PORT" = "" ]; then
-  SPARK_MASTER_PORT=7077
-fi
-
-if [ "$SPARK_MASTER_IP" = "" ]; then
-  SPARK_MASTER_IP=`hostname`
-fi
-
-if [ "$SPARK_MASTER_WEBUI_PORT" = "" ]; then
-  SPARK_MASTER_WEBUI_PORT=8080
-fi
-
-"${SPARK_HOME}/sbin"/spark-daemon-run.sh start $CLASS 1 \
-  --ip $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT \
-  $ORIGINAL_ARGS
-
-if [ "$START_TACHYON" == "true" ]; then
-  "${SPARK_HOME}"/tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP
-  "${SPARK_HOME}"/tachyon/bin/tachyon format -s
-  "${SPARK_HOME}"/tachyon/bin/tachyon-start.sh master
-fi
diff --git a/run-slave.sh b/run-slave.sh
deleted file mode 100755
index 1f92aa3bee3e..000000000000
--- a/run-slave.sh
+++ /dev/null
@@ -1,91 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Starts a slave on the machine this script is executed on.
-#
-# Environment Variables
-#
-#   SPARK_WORKER_INSTANCES  The number of worker instances to run on this
-#                           slave.  Default is 1.
-#   SPARK_WORKER_PORT       The base port number for the first worker. If set,
-#                           subsequent workers will increment this number.  If
-#                           unset, Spark will find a valid port number, but
-#                           with no guarantee of a predictable pattern.
-#   SPARK_WORKER_WEBUI_PORT The base port for the web interface of the first
-#                           worker.  Subsequent workers will increment this
-#                           number.  Default is 8081.
-
-if [ -z "${SPARK_HOME}" ]; then
-  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-fi
-
-# NOTE: This exact class name is matched downstream by SparkSubmit.
-# Any changes need to be reflected there.
-CLASS="org.apache.spark.deploy.worker.Worker"
-
-if [[ $# -lt 1 ]] || [[ "$@" = *--help ]] || [[ "$@" = *-h ]]; then
-  echo "Usage: ./sbin/run-slave.sh [options] <master>"
-  pattern="Usage:"
-  pattern+="\|Using Spark's default log4j profile:"
-  pattern+="\|Registered signal handlers for"
-
-  "${SPARK_HOME}"/bin/spark-class $CLASS --help 2>&1 | grep -v "$pattern" 1>&2
-  exit 1
-fi
-
-. "${SPARK_HOME}/sbin/spark-config.sh"
-
-. "${SPARK_HOME}/bin/load-spark-env.sh"
-
-# First argument should be the master; we need to store it aside because we may
-# need to insert arguments between it and the other arguments
-MASTER=$1
-shift
-
-# Determine desired worker port
-if [ "$SPARK_WORKER_WEBUI_PORT" = "" ]; then
-  SPARK_WORKER_WEBUI_PORT=8081
-fi
-
-# Start up the appropriate number of workers on this machine.
-# quick local function to start a worker
-function start_instance {
-  WORKER_NUM=$1
-  shift
-
-  if [ "$SPARK_WORKER_PORT" = "" ]; then
-    PORT_FLAG=
-    PORT_NUM=
-  else
-    PORT_FLAG="--port"
-    PORT_NUM=$(( $SPARK_WORKER_PORT + $WORKER_NUM - 1 ))
-  fi
-  WEBUI_PORT=$(( $SPARK_WORKER_WEBUI_PORT + $WORKER_NUM - 1 ))
-
-  "${SPARK_HOME}/sbin"/spark-daemon-run.sh start $CLASS $WORKER_NUM \
-     --webui-port "$WEBUI_PORT" $PORT_FLAG $PORT_NUM $MASTER "$@"
-}
-
-if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
-  start_instance 1 "$@"
-else
-  for ((i=0; i<$SPARK_WORKER_INSTANCES; i++)); do
-    start_instance $(( 1 + $i )) "$@"
-  done
-fi
diff --git a/spark-daemon-run.sh b/spark-daemon-run.sh
deleted file mode 100755
index 34e3a80fa37a..000000000000
--- a/spark-daemon-run.sh
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Runs a Spark daemon foreground.
-#
-# Environment Variables
-#
-#   SPARK_CONF_DIR  Alternate conf dir. Default is ${SPARK_HOME}/conf.
-#   SPARK_LOG_DIR   Where log files are stored. ${SPARK_HOME}/logs by default.
-#   SPARK_MASTER    host:path where spark code should be rsync'd from
-#   SPARK_IDENT_STRING   A string representing this instance of spark. $USER by default
-#   SPARK_NICENESS The scheduling priority for daemons. Defaults to 0.
-##
-
-usage="Usage: spark-daemon-run.sh [--config <conf-dir>] (class|submit) <spark-command> <spark-instance-number> <args...>"
-
-# if no args specified, show usage
-if [ $# -le 1 ]; then
-  echo $usage
-  exit 1
-fi
-
-if [ -z "${SPARK_HOME}" ]; then
-  export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)"
-fi
-
-. "${SPARK_HOME}/sbin/spark-config.sh"
-
-# get arguments
-
-# Check if --config is passed as an argument. It is an optional parameter.
-# Exit if the argument is not a directory.
-
-if [ "$1" == "--config" ]
-then
-  shift
-  conf_dir="$1"
-  if [ ! -d "$conf_dir" ]
-  then
-    echo "ERROR : $conf_dir is not a directory"
-    echo $usage
-    exit 1
-  else
-    export SPARK_CONF_DIR="$conf_dir"
-  fi
-  shift
-fi
-
-mode=$1
-shift
-command=$1
-shift
-instance=$1
-shift
-
-spark_rotate_log ()
-{
-    log=$1;
-    num=5;
-    if [ -n "$2" ]; then
-	num=$2
-    fi
-    if [ -f "$log" ]; then # rotate logs
-	while [ $num -gt 1 ]; do
-	    prev=`expr $num - 1`
-	    [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num"
-	    num=$prev
-	done
-	mv "$log" "$log.$num";
-    fi
-}
-
-. "${SPARK_HOME}/bin/load-spark-env.sh"
-
-if [ "$SPARK_IDENT_STRING" = "" ]; then
-  export SPARK_IDENT_STRING="$USER"
-fi
-
-
-
-# get log directory
-if [ "$SPARK_LOG_DIR" = "" ]; then
-  export SPARK_LOG_DIR="${SPARK_HOME}/logs"
-fi
-mkdir -p "$SPARK_LOG_DIR"
-touch "$SPARK_LOG_DIR"/.spark_test > /dev/null 2>&1
-TEST_LOG_DIR=$?
-if [ "${TEST_LOG_DIR}" = "0" ]; then
-  rm -f "$SPARK_LOG_DIR"/.spark_test
-else
-  chown "$SPARK_IDENT_STRING" "$SPARK_LOG_DIR"
-fi
-
-# some variables
-log="$SPARK_LOG_DIR/spark-$SPARK_IDENT_STRING-$command-$instance-$HOSTNAME.out"
-
-# Set default scheduling priority
-if [ "$SPARK_NICENESS" = "" ]; then
-    export SPARK_NICENESS=0
-fi
-
-if [ "$SPARK_MASTER" != "" ]; then
-  echo rsync from "$SPARK_MASTER"
-  rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' "$SPARK_MASTER/" "${SPARK_HOME}"
-fi
-
-spark_rotate_log "$log"
-echo "running $command, logging to $log"
-
-case "$mode" in
-  (start)
-    exec nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null
-    ;;
-
-  (submit)
-    exec nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-submit --class $command "$@" >> "$log" 2>&1 < /dev/null
-    ;;
-
-  (*)
-    echo "unknown mode: $mode"
-    exit 1
-    ;;
-esac
diff --git a/spark-env.sh b/spark-env.sh
deleted file mode 100644
index c6ed3dbc450e..000000000000
--- a/spark-env.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-export JAVA_HOME=/usr/lib/jvm/default-runtime
-if (command -v hadoop 2> /dev/null); then
-    export SPARK_DIST_CLASSPATH=$(hadoop classpath)
-else
-    export SPARK_DIST_CLASSPATH=""
-fi
-SPARK_MASTER_IP=localhost
-SPARK_LOCAL_IP=localhost