Bootstrap

集群启动脚本

集群启动脚本

xsync

#!/bin/bash

#1. 判断参数个数
if [ $# -lt 1 ]
then
    echo Not Enough Arguement!
    exit;
fi
#2. 遍历集群所有机器
for host in hadoop01 hadoop02 hadoop03
do
    echo -e "\n====================  $host  ===================="
    #3. 遍历所有目录，逐个发送
    for file in $@
    do
        #4. 判断文件是否存在
        if [ -e $file ]
            then
                #5. 获取父目录
                pdir=$(cd -P $(dirname $file); pwd)
                #6. 获取当前文件的名称
                fname=$(basename $file)
                ssh $host "mkdir -p $pdir"
                rsync -av $pdir/$fname $host:$pdir
            else
                echo $file does not exist!
        fi
    done
done
echo -e "\n"

jpsall

#!/bin/bash
for host in hadoop01 hadoop02 hadoop03
do
    echo -e "\n=============== $host ==============="
    ssh $host "export JAVA_HOME=/opt/module/jdk1.8.0_391 && export PATH=\$JAVA_HOME/bin:\$PATH && jps"
done
echo -e "\n"

zookeeper

zkserver

#!/bin/bash
echo "$1 zkServer ..."
for i in {hadoop02,hadoop01,hadoop03}
do
        ssh $i "source /etc/profile && zkServer.sh $1"
done

zkstatus

#!/bin/bash
for host in {hadoop01,hadoop02,hadoop03}
do
	status=$(ssh $host 'source /etc/profile && zkServer.sh status 2>&1 | grep  Mode')
	if [[ $status == "Mode: follower" ]];then
		echo "$host是从节点"
	elif [[ $status == "Mode: leader" ]];then
		echo "$host是主节点"
	else
		echo "未查询到$host节点zookeeper状态,请检查服务"
	fi
done

hadoop

myhadoop

#!/bin/bash

HADOOP_HOME=/opt/module/hadoop-3.3.5  # Hadoop 安装路径

check_zookeeper_status() {
    echo "检查 $1 上的 ZooKeeper 状态..."
    status=$(ssh $1 "source /etc/profile && zkServer.sh status 2>&1 | grep Mode")
    if [[ $status == "Mode: follower" ]]; then
        echo "$1 是从节点"
    elif [[ $status == "Mode: leader" ]]; then
        echo "$1 是主节点"
    else
        echo "未能获取 $1 节点的 ZooKeeper 状态，请检查服务。"
    fi
}

start_zookeeper() {
    echo "在 $1 上启动 ZooKeeper..."
    ssh $1 "source /etc/profile && zkServer.sh start"
    sleep 5
    check_zookeeper_status $1
}
stop_zookeeper() {
    echo "在 $1 上启动 ZooKeeper..."
    ssh $1 "source /etc/profile && zkServer.sh stop"
    sleep 5
    check_zookeeper_status $1
}

case $1 in
    "start")
        echo -e "\n================= 启动 Hadoop 集群 ================="

        # 检查 ZooKeeper 节点状态
        for node in hadoop01 hadoop02 hadoop03; do
            check_zookeeper_status $node
        done

        read -p "是否启动 ZooKeeper 服务？(y/n): " zk_start_choice
        if [ "$zk_start_choice" == "y" ]; then
            for node in hadoop02 hadoop01 hadoop03; do
                start_zookeeper $node
            done
        else
            read -p "是否继续启动 Hadoop 集群？(y/n): " choice
            [ "$choice" != "y" ] && echo "用户选择不启动 Hadoop 集群，退出脚本" && exit 1
        fi

        # 继续启动 Hadoop 集群...
        echo " ------------------ 启动 HDFS --------------------"
        $HADOOP_HOME/sbin/start-dfs.sh
        # 在进行节点切换前检查 NameNode 健康状态
        tries=0
        while ! hdfs haadmin -checkHealth nn1 && [ "$tries" -lt 5 ]; do
            sleep 10
            tries=$((tries + 1))
        done
        
        if [ "$tries" -eq 5 ]; then
            echo "等待 NameNode 启动超时，退出脚本"
            exit 1
        fi
        
        ACTIVE_NODE=$(hdfs haadmin -getServiceState nn1)
        if [ "$ACTIVE_NODE" == "active" ]; then
            echo "hadoop01 是 Active 节点，启动 YARN、ZKFC 和 HistoryServer"
            echo "------------------- 启动 YARN --------------------"
            $HADOOP_HOME/sbin/start-yarn.sh
            echo "--------- 启动 HistoryServer on hadoop01 ---------"
            ssh hadoop01 "$HADOOP_HOME/bin/mapred --daemon start historyserver"
            echo "--------- 启动 HistoryServer on hadoop02 ---------"
            ssh hadoop02 "$HADOOP_HOME/bin/mapred --daemon start historyserver"
        else
            echo "hadoop01 不是 Active 节点，正在进行节点切换"
            # 切换节点
            hdfs haadmin -failover nn2 nn1
            # 等待一段时间以确保切换完成
            sleep 30
            # 继续启动 Hadoop 集群
            echo "------------------- 启动 YARN --------------------"
            $HADOOP_HOME/sbin/start-yarn.sh
            echo "--------- 启动 HistoryServer on hadoop01 ---------"
            ssh hadoop01 "$HADOOP_HOME/bin/mapred --daemon start historyserver"
            echo "--------- 启动 HistoryServer on hadoop02 ---------"
            ssh hadoop02 "$HADOOP_HOME/bin/mapred --daemon start historyserver"
        fi
        echo -e "\n"
        ;;

    "stop")
        echo -e "\n================= 关闭 Hadoop 集群 ================="
        # 在停止 Hadoop 服务之前添加确认消息
        read -p "是否确实要停止 Hadoop 服务？(y/n): " confirm_stop
        [ "$confirm_stop" != "y" ] && echo "用户选择不停止 Hadoop 集群，退出脚本" && exit 0

        # 停止 Hadoop 服务...
        echo "--------- 关闭 HistoryServer on hadoop01 ----------"
        ssh hadoop01 "$HADOOP_HOME/bin/mapred --daemon stop historyserver"
        echo "--------- 关闭 HistoryServer on hadoop02 ----------"
        ssh hadoop02 "$HADOOP_HOME/bin/mapred --daemon stop historyserver"
        echo "------------------- 关闭 YARN ---------------------"
        $HADOOP_HOME/sbin/stop-yarn.sh
        echo "------------------- 关闭 HDFS ---------------------"
        $HADOOP_HOME/sbin/stop-dfs.sh
	read -p "是否关闭 ZooKeeper 服务？(y/n): " zk_stop_choice
        if [ "$zk_stop_choice" == "y" ]; then
            for node in hadoop02 hadoop01 hadoop03; do
                stop_zookeeper $node
            done
	fi
        echo -e "\n"
        ;;
    *)
        echo "输入参数错误！"
        exit 1
        ;;
esac

spark

myspark

#!/bin/bash

SPARK_HOME=/opt/module/spark-3.2.4

start_spark() {
    echo "Starting Spark Cluster..."

    # 启动Spark Master
    $SPARK_HOME/sbin/start-all.sh

    # 启动Spark History Server（在hadoop01上）
    $SPARK_HOME/sbin/start-history-server.sh

    # 使用SSH远程登录到hadoop02并启动Spark Master和History Server
    ssh hadoop02 "source /etc/profile && $SPARK_HOME/sbin/start-master.sh && $SPARK_HOME/sbin/start-history-server.sh"

    echo "Spark Cluster started successfully."
}

check_spark_status() {
    # 使用SSH远程登录到hadoop01并查询Spark Master状态
    remote_master_status_hadoop01=$(ssh hadoop01 'source /etc/profile && ss -ntulp | grep 8090')

    if [[ -n "$remote_master_status_hadoop01" ]]; then
        echo "Spark Master is running on hadoop01."
    else
        echo "Spark Master is not running on hadoop01."
    fi

    # 使用SSH远程登录到hadoop02并查询Spark Master状态
    remote_master_status_hadoop02=$(ssh hadoop02 'source /etc/profile && ss -ntulp | grep 8090')

    if [[ -n "$remote_master_status_hadoop02" ]]; then
        echo "Spark Master is running on hadoop02."
    else
        echo "Spark Master is not running on hadoop02."
    fi
}

stop_spark() {
    echo "Stopping Spark Cluster..."

    # 停止Spark Worker和Master
    $SPARK_HOME/sbin/stop-all.sh
	$SPARK_HOME/sbin/stop-history-server.sh
	
    # 使用SSH远程登录到hadoop02并停止Spark Master
    ssh hadoop02 "source /etc/profile && $SPARK_HOME/sbin/stop-master.sh && $SPARK_HOME/sbin/stop-history-server.sh"

    echo "Spark Cluster stopped successfully."
}

case "$1" in
    start)
        start_spark
        ;;
    status)
        check_spark_status
        ;;
    stop)
        stop_spark
        ;;
    *)
        echo "Usage: $0 {start|status|stop}"
        exit 1
        ;;
esac

exit 0

hive

myhive

#!/bin/bash

# 在这里设置 HIVE_LOG_DIR 的路径
HIVE_LOG_DIR="/opt/module/hive-3.1.3/logs"

# 远程服务器列表
SERVERS=("hadoop02" "hadoop03")

# 下面是你的脚本中的 case 语句部分
case $1 in
    "start")
        echo "Starting Hive services..."
        for SERVER in "${SERVERS[@]}"; do
            ssh "$SERVER" "source /etc/profile;" "nohup bash -s" < <(cat <<'END_SCRIPT'
                HIVE_LOG_DIR="/opt/module/hive-3.1.3/logs"  # 这里也需要设置

                if [ ! -d "$HIVE_LOG_DIR" ]; then
                    mkdir -p "$HIVE_LOG_DIR"
                fi

                function check_process() {
                    pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
                    ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
                    echo $pid
                    [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
                }

                function hive_start() {
                    metapid=$(check_process HiveMetastore 9083)
                    cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"
                    [ -z "$metapid" ] && eval $cmd || echo "Metastore服务已启动"
                    
                    server2pid=$(check_process HiveServer2 10001)
                    cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"
                    [ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动"
                }

                function hive_stop() {
                    metapid=$(check_process HiveMetastore 9083)
                    [ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"
                    
                    server2pid=$(check_process HiveServer2 10001)
                    [ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
                }

                hive_start
END_SCRIPT
            )
        done
        ;;
    "stop")
        echo "Stopping Hive services..."
        for SERVER in "${SERVERS[@]}"; do
            ssh "$SERVER" "source /etc/profile;" "bash -s" < <(cat <<'END_SCRIPT'
                function check_process() {
                    pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
                    ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
                    echo $pid
                    [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
                }

                function hive_stop() {
                    metapid=$(check_process HiveMetastore 9083)
                    [ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"
                    
                    server2pid=$(check_process HiveServer2 10001)
                    [ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
                }

                hive_stop
END_SCRIPT
            )
        done
        ;;
    "restart")
        echo "Restarting Hive services..."
        for SERVER in "${SERVERS[@]}"; do
            ssh "$SERVER" "source /etc/profile;" "bash -s" < <(cat <<'END_SCRIPT'
                function check_process() {
                    pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
                    ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
                    echo $pid
                    [[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
                }

                function hive_stop() {
                    metapid=$(check_process HiveMetastore 9083)
                    [ "$metapid" ] && kill $metapid || echo "Metastore服务未启动"
                    
                    server2pid=$(check_process HiveServer2 10001)
                    [ "$server2pid" ] && kill $server2pid || echo "HiveServer2服务未启动"
                }

                function hive_start() {
                    metapid=$(check_process HiveMetastore 9083)
                    cmd="nohup hive --service metastore >$HIVE_LOG_DIR/metastore.log 2>&1 &"
                    [ -z "$metapid" ] && eval $cmd || echo "Metastore服务已启动"
                    
                    server2pid=$(check_process HiveServer2 10001)
                    cmd="nohup hive --service hiveserver2 >$HIVE_LOG_DIR/hiveServer2.log 2>&1 &"
                    [ -z "$server2pid" ] && eval $cmd || echo "HiveServer2服务已启动"
                }

                hive_stop
                sleep 2
                hive_start
END_SCRIPT
            )
        done
        ;;
    "status")
        echo "Checking Hive services status..."
        for SERVER in "${SERVERS[@]}"; do
            ssh "$SERVER" "source /etc/profile;" "bash -s" < <(cat <<'END_SCRIPT'

				function check_process() {
					pid=$(ps -ef 2>/dev/null | grep -v grep | grep -i $1 | awk '{print $2}')
					ppid=$(netstat -nltp 2>/dev/null | grep $2 | awk '{print $7}' | cut -d '/' -f 1)
					echo "Checking process: $1, Port: $2, PID: $pid, PPID: $ppid"
					
					# 输出一下当前端口占用情况
					netstat -nltp | grep $2
					
					[[ "$pid" =~ "$ppid" ]] && [ "$ppid" ] && return 0 || return 1
				}

                check_process HiveMetastore 9083 >/dev/null && echo "Metastore服务运行正常" || echo "Metastore服务运行异常"
                check_process HiveServer2 10001 >/dev/null && echo "HiveServer2服务运行正常" || echo "HiveServer2服务运行异常"
END_SCRIPT
            )
        done
        ;;
    *)
        echo "Invalid Args!"
        echo 'Usage: myhive start|stop|restart|status'
        ;;
esac

hbase

myhbase

#!/bin/bash

HBASE_HOME=/opt/module/hbase-2.4.17
HBASE_MASTER="hadoop01 hadoop02"
HBASE_REGIONSERVER="hadoop01 hadoop02 hadoop03"

start_hbase_cluster() {

    # 启动 HBase Master
	for master in $HBASE_MASTER; do
        ssh $master "$HBASE_HOME/bin/hbase-daemon.sh start master"
    done
	
    # 启动 HBase RegionServer
    for regionserver in $HBASE_REGIONSERVER; do
        ssh $regionserver "$HBASE_HOME/bin/hbase-daemon.sh start regionserver"
    done

    # 等待片刻以确保 HBase 启动完成
    sleep 5
}

check_hbase_status() {
    # 使用SSH远程登录到hadoop01并查询HBase Master状态
    remote_master_status=$(ssh hadoop01 "source /etc/profile && ss -ntulp | grep 60010 && echo 'HBase Master is running on hadoop01.' || echo 'HBase Master is not running on hadoop01.'")

    # 使用SSH远程登录到hadoop02并查询HBase Master状态
    remote_master_status+="\n$(ssh hadoop02 "source /etc/profile && ss -ntulp | grep 60010 && echo 'HBase Master is running on hadoop02.' || echo 'HBase Master is not running on hadoop02.'")"

    echo -e "$remote_master_status"
}

stop_hbase_cluster() {
    # 关闭 HBase RegionServer
    for regionserver in $HBASE_REGIONSERVER; do
        ssh $regionserver "$HBASE_HOME/bin/hbase-daemon.sh stop regionserver"
    done

    # 关闭 HBase Master	
	for master in $HBASE_MASTER; do
        ssh $master "$HBASE_HOME/bin/hbase-daemon.sh stop master"
    done
	
}

# 根据输入参数执行相应的操作
case "$1" in
    "start")
        start_hbase_cluster
        ;;
    "status")
        check_hbase_status
        ;;
    "stop")
        stop_hbase_cluster
        ;;
    *)
        echo "Usage: $0 {start|status|stop}"
        exit 1
        ;;
esac

exit 0

悦读

道可道，非常道；名可名，非常名。 无名，天地之始，有名，万物之母。 故常无欲，以观其妙，常有欲，以观其徼。 此两者，同出而异名，同谓之玄，玄之又玄，众妙之门。

最新收录
Linux本地内核提权漏洞复现(CVE-2019-13272)
scala
深度学习03-卷积神经网络(CNN)
Java后端面试题：JVM篇
harmonyOS鸿蒙官网教程-添加弹窗
Python logging库（python日志库）Logger（记录器、收集器、采集器）、Handler（处理器）、Formatter（格式化器）、Log Level（日志级别）
Python实现随机分布式延迟PSO优化算法(RODDPSO)优化CNN分类模型项目实战
牛客网专项练习30天Pytnon篇第09天_面有关计算机基本原理的说法中,正确的一项是
使用vs2022中自带的git工具将代码提交到gitee上
学习编程比学开车要简单
;