diff options
author | William Tang | 2023-06-24 15:56:14 +0800 |
---|---|---|
committer | William Tang | 2023-06-24 20:36:03 +0800 |
commit | 8731eb573a4a10f95a6f14516b3df9a66d17e842 (patch) | |
tree | 45119f51d33df85fe02d0a58a12b4593664a0272 /spark-daemon-run.sh | |
parent | d703d192e7cb723f2c429788ddca88f216ceee4a (diff) | |
download | aur-8731eb573a4a10f95a6f14516b3df9a66d17e842.tar.gz |
v3.4.1
Diffstat (limited to 'spark-daemon-run.sh')
-rwxr-xr-x | spark-daemon-run.sh | 139 |
1 files changed, 0 insertions, 139 deletions
diff --git a/spark-daemon-run.sh b/spark-daemon-run.sh deleted file mode 100755 index 34e3a80fa37a..000000000000 --- a/spark-daemon-run.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Runs a Spark daemon foreground. -# -# Environment Variables -# -# SPARK_CONF_DIR Alternate conf dir. Default is ${SPARK_HOME}/conf. -# SPARK_LOG_DIR Where log files are stored. ${SPARK_HOME}/logs by default. -# SPARK_MASTER host:path where spark code should be rsync'd from -# SPARK_IDENT_STRING A string representing this instance of spark. $USER by default -# SPARK_NICENESS The scheduling priority for daemons. Defaults to 0. -## - -usage="Usage: spark-daemon-run.sh [--config <conf-dir>] (class|submit) <spark-command> <spark-instance-number> <args...>" - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -if [ -z "${SPARK_HOME}" ]; then - export SPARK_HOME="$(cd "`dirname "$0"`"/..; pwd)" -fi - -. "${SPARK_HOME}/sbin/spark-config.sh" - -# get arguments - -# Check if --config is passed as an argument. It is an optional parameter. -# Exit if the argument is not a directory. - -if [ "$1" == "--config" ] -then - shift - conf_dir="$1" - if [ ! -d "$conf_dir" ] - then - echo "ERROR : $conf_dir is not a directory" - echo $usage - exit 1 - else - export SPARK_CONF_DIR="$conf_dir" - fi - shift -fi - -mode=$1 -shift -command=$1 -shift -instance=$1 -shift - -spark_rotate_log () -{ - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv "$log.$prev" "$log.$num" - num=$prev - done - mv "$log" "$log.$num"; - fi -} - -. "${SPARK_HOME}/bin/load-spark-env.sh" - -if [ "$SPARK_IDENT_STRING" = "" ]; then - export SPARK_IDENT_STRING="$USER" -fi - - - -# get log directory -if [ "$SPARK_LOG_DIR" = "" ]; then - export SPARK_LOG_DIR="${SPARK_HOME}/logs" -fi -mkdir -p "$SPARK_LOG_DIR" -touch "$SPARK_LOG_DIR"/.spark_test > /dev/null 2>&1 -TEST_LOG_DIR=$? -if [ "${TEST_LOG_DIR}" = "0" ]; then - rm -f "$SPARK_LOG_DIR"/.spark_test -else - chown "$SPARK_IDENT_STRING" "$SPARK_LOG_DIR" -fi - -# some variables -log="$SPARK_LOG_DIR/spark-$SPARK_IDENT_STRING-$command-$instance-$HOSTNAME.out" - -# Set default scheduling priority -if [ "$SPARK_NICENESS" = "" ]; then - export SPARK_NICENESS=0 -fi - -if [ "$SPARK_MASTER" != "" ]; then - echo rsync from "$SPARK_MASTER" - rsync -a -e ssh --delete --exclude=.svn --exclude='logs/*' --exclude='contrib/hod/logs/*' "$SPARK_MASTER/" "${SPARK_HOME}" -fi - -spark_rotate_log "$log" -echo "running $command, logging to $log" - -case "$mode" in - (start) - exec nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-class $command "$@" >> "$log" 2>&1 < /dev/null - ;; - - (submit) - exec nice -n "$SPARK_NICENESS" "${SPARK_HOME}"/bin/spark-submit --class $command "$@" >> "$log" 2>&1 < /dev/null - ;; - - (*) - echo "unknown mode: $mode" - exit 1 - ;; -esac |