shell

## 安装脚本 ### topic ```bash #!/usr/bin/env bash kafka_Home="/mnt/kafka" bootstrap_server="192.168.50.21:19092" kafka_group=CLIENT topics="alarmAlgorithmData alarmAlgorithmResult NetReassemble notifyWeb CacheErrorEvent SupplementaryCacheInfo processFailedCacheInfo-in-0 processSupplementaryCacheInfo-in-0 NPM BpmRaw BpmPacket BpmEvent BpmPairing alarmSimulationData alarmSimulationResult testlog logEvent TidmEvent NpmPacket" function delete_topic() { topic_list=`${kafka_Home}/bin/kafka-topics.sh --list --bootstrap-server $bootstrap_server` for topic in ${topic_list}; do { echo "delete topic [${topic}]" ${kafka_Home}/bin/kafka-topics.sh --bootstrap-server $bootstrap_server --topic ${topic} --delete } & done wait } function create_topic() { for topic in $topics; do echo "create topic [${topic}] partitions $1" ${kafka_Home}/bin/kafka-topics.sh --create --bootstrap-server $bootstrap_server --replication-factor 1 --partitions $1 --topic ${topic} done } function desc_group() { if [[ ! -z $1 ]];then $kafka_Home/bin/kafka-consumer-groups.sh --bootstrap-server $bootstrap_server --describe --group $1 exit 0 fi $kafka_Home/bin/kafka-consumer-groups.sh --bootstrap-server $bootstrap_server --describe --group $kafka_group } function use_help() { echo " # 创建topic bin/kafka-topics.sh --create --bootstrap-server 10.1.125.60:9092 --replication-factor 1 --partitions 3 --topic weblog # 修改topic副本 bin/kafka-topics.sh --alter --partitions 4 --topic weblog --bootstrap-server 10.1.125.60:9092 # 删除topic副本 bin/kafka-topics.sh --bootstrap-server 10.1.125.60:9092 --topic weblog --delete # 查看kafka的所有topic bin/kafka-topics.sh --bootstrap-server 10.1.125.60:9092 --list # 查看单个topic的详细信息 bin/kafka-topics.sh --bootstrap-server 10.1.125.60:9092 --topic weblog --describe Topic:weblog PartitionCount:3 ReplicationFactor:1 Configs: ... # 查询topic里内容: bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic weblog --from-beginning # 查看消费者consumer group列表 bin/kafka-consumer-groups.sh --bootstrap-server 10.1.125.60:9092 --list # 删除消费组 bin/kafka-consumer-groups --bootstrap-server 10.1.125.60:9092 --delete --group consumer-test # 查看消费者consumer group详情 bin/kafka-consumer-groups.sh --bootstrap-server 10.1.125.60:9092 --group consumer-test --describe " } case $1 in help) use_help ;; group) desc_group $2 ;; list) echo "----- [topic] -----" ${kafka_Home}/bin/kafka-topics.sh --bootstrap-server $bootstrap_server --list echo "----- [consumer group] -----" ${kafka_Home}/bin/kafka-consumer-groups.sh --bootstrap-server $bootstrap_server --list ;; delete) delete_topic ;; create) if [[ -z $2 ]];then echo "Usage: $0 create partitions_count(int) " exit 0 fi create_topic $2 ;; *) echo "---------------------------------------------------------- Usage: $0 help [查看kafka的基本操作命令] Usage: $0 list [查看topic/消费组] Usage: $0 group [group_name(可选)] [查看消费组lag] Usage: $0 delete [删除所有topic] Usage: $0 create partitions_count(int) [创建topic 分区数] ----------------------------------------------------------" ;; esac ``` ### 移动数据包 ```bash #!/bin/bash #将数据包移动或拷贝到此 copydir="../$(date +%Y-%m-%d)/in/" mkdir -p ${copydir} if [[ -z $1 || -z $2 ]];then echo "Usage:$0 start_time end_time [null/mv/cp]" echo "Usage:$0 11:00:50 12:52:55" echo "Usage:$0" ' "2023-11-09 21:00:50" "2023-11-09 21:52:55"' echo "Usage:$0 11:00:50 12:52:55 cp # 将数据包cp到${copydir}目录。" echo "Usage:$0 11:00:50 12:52:55 mv # 将数据包mv到${copydir}目录,不加此参数默认使用mv。 " exit 1 fi if [[ -z $3 ]];then model="mv" else case $3 in 'cp') model="cp" ;; *) model="mv" ;; esac fi # 格式化起始和终止时间 start=$(date -d "$1" "+%Y-%m-%d_%H:%M:%S") stop=$(date -d "$2" "+%Y-%m-%d_%H:%M:%S") DATE=`date -d "$1" "+%Y-%m-%d"` # 获取当前路径的所有存包的时间 pcaps=$(ls ./|grep .cap$|grep ${DATE} |awk -F_ '{print $1"_"$2 }') for pcap in ${pcaps}; do { # 获取当前数据包所属日期 pcap_day="$(echo ${pcap}|awk -F_ '{print $1}')" # 获取当前数据包所属时间 pcap_time="$(echo ${pcap}|awk -F_ '{print $2}'|tr - :)" # 格式化当前数据包的日期时间 t=$(date -d "$pcap_day $pcap_time" "+%Y-%m-%d_%H:%M:%S") # 比较数据包时间是否在起始到终止时间内 if [[ "$t" > "$start" && "$t" < "$stop" ]]; then PCAP_FILE=$(ls | grep .cap|grep ${DATE} |grep $pcap) echo "$model => ${PCAP_FILE} => ${copydir}" # 在此处添加拷贝操作的代码 $model ${PCAP_FILE} ${copydir} fi } & done wait ``` ### init ```bash #!/bin/bash # 关闭selinux sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config # 关闭防火墙 systemctl stop firewalld systemctl disable firewalld # 安装一些常用的软件 yum -y install epel-release openssh-clients net-tools vim tree lrzsz wget bash-completion ntpdate sshpass python3 python3-pip yum -y install ansible nginx # yum源更换 function yummakecache(){ mkdir -p /etc/yum.repos.d/backup mv /etc/yum.repos.d/* /etc/yum.repos.d/backup wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo yum repolist } # 安装命令补全 yum -y install bash-completion-extra echo '* - nofile 65535' >>/etc/security/limits.conf # 修改终端颜色 cat >> ~/.bashrc <<EOF PS1='[\[\e[34;1m\]\u@\[\e[0m\]\[\e[32;1m\]\H\[\e[0m\]\[\e[31;1m\] \W\[\e[0m\]]# ' EOF source ~/.bashrc # 修改ssh服务优化 function sshoptimize(){ sed -ri 's@^#UseDNS yes@UseDNS no@g' /etc/ssh/sshd_config sed -ri 's#^GSSAPIAuthentication yes#GSSAPIAuthentication no#g' /etc/ssh/sshd_config grep ^UseDNS /etc/ssh/sshd_config grep ^GSSAPIAuthentication /etc/ssh/sshd_config } function main(){ yummakecache sshoptimize } . ~/.bashrc # main ``` ### rar ``` wget https://www.rarlab.com/rar/rarlinux-x64-3.8.0.tar.gz --no-check-certificate tar -xf rarlinux-3.8.0.tar.gz cd rar make make install bug处理,使用的时候出现一个错误 bash: /usr/local/bin/rar: /lib/ld-linux.so.2: bad ELF interpreter: No such file or directory 是因为64位系统中安装了32位程序 解决方法: yum install glibc.i686 yum install libstdc++.so.6 ``` ### md5 ``` [root@192-168-144-167 swweb]# md5sum sw.zip c971c474a0957d72ef3cbad94548756a sw.zip F:\360MoveData\Users\Administrator\Desktop\sw>certutil -hashfile sw.zip md5 MD5 的 sw.zip 哈希: c971c474a0957d72ef3cbad94548756a CertUtil: -hashfile 命令成功完成。 ``` ### kafka ``` yum -y install java wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.8.1/kafka_2.12-2.8.1.tgz --no-check-certificate tar -zxvf kafka_2.12-2.8.1.tgz mv kafka_2.12-2.8.1 /usr/bin/ cd /usr/bin/kafka_2.12-2.8.1/config/ && cp server.properties{,.bak} cat > server.properties.bak <<EOF broker.id=0 listeners=PLAINTEXT://10.1.128.85:9092 num.network.threads=3 num.io.threads=8 socket.send.buffer.bytes=102400 socket.receive.buffer.bytes=102400 socket.request.max.bytes=104857600 log.dirs=/tmp/kafka-logs num.partitions=1 num.recovery.threads.per.data.dir=1 offsets.topic.replication.factor=1 transaction.state.log.replication.factor=1 transaction.state.log.min.isr=1 log.retention.hours=168 log.segment.bytes=1073741824 log.retention.check.interval.ms=300000 zookeeper.connect=localhost:2181 zookeeper.connection.timeout.ms=18000 group.initial.rebalance.delay.ms=0 EOF cd /mnt wget https://dlcdn.apache.org/zookeeper/zookeeper-3.5.10/apache-zookeeper-3.5.10-bin.tar.gz --no-check-certificate tar -zxf apache-zookeeper-3.5.10-bin.tar.gz mv apache-zookeeper-3.5.10-bin zookeeper # 创建数据目录和日志目录 mkdir /mnt/zookeeper/{data,logs} cat > zoo.cfg <<EOF tickTime=2000 initLimit=10 syncLimit=5 dataDir=/mnt/zookeeper/data dataLogDir=/mnt/zookeeper/logs clientPort=2181 EOF /mnt/zookeeper/bin/zkServer.sh start /usr/bin/kafka_2.12-2.8.1/bin/kafka-server-start.sh /usr/bin/kafka_2.12-2.8.1/config/server.properties & ``` ### mysql8.0.27 ``` #!/usr/bin/env bash #################################======================================================================== VERSION=8.0.27 #安装相关依赖包 yum --help > /dev/null 2>&1 if [[ $? == 0 ]];then yum install ncurses numactl ncurses-devel libaio-devel openssl openssl-devel -y else sudo apt-get -y install libaio-dev fi #<==创建mysql用户。 useradd mysql -s /sbin/nologin -M id mysql if [ -f mysql-$VERSION-linux-glibc2.12-x86_64.tar.xz ];then mv mysql-$VERSION-linux-glibc2.12-x86_64.tar.xz /opt/ tar xf /opt/mysql-$VERSION-linux-glibc2.12-x86_64.tar.xz -C /opt/ else cd /opt && wget https://mirrors.aliyun.com/mysql/MySQL-8.0/mysql-$VERSION-linux-glibc2.12-x86_64.tar.xz && tar xf mysql-$VERSION-linux-glibc2.12-x86_64.tar.xz -C /opt/ fi ln -s /opt/mysql-$VERSION-linux-glibc2.12-x86_64 /usr/local/mysql ls -l /usr/local/mysql #1)增加简易配置文件,后面在优化 cat>/etc/my.cnf<<'EOF' [mysqld] user=mysql basedir=/usr/local/mysql datadir=/data/3306/data port=3306 socket=/tmp/mysql.sock [client] socket=/tmp/mysql.sock EOF chown mysql.mysql /etc/my.cnf #2)创建MySQL数据目录并授权 mkdir -p /data/3306/data chown -R mysql.mysql /data ls -ld /data #1) 配置PATH环境变量 echo 'export PATH=/usr/local/mysql/bin:$PATH' >>/etc/profile . /etc/profile #2)初始化MySQL数据库 /usr/local/mysql/bin/mysqld --initialize-insecure --user=mysql --basedir=/usr/local/mysql --datadir=/data/3306/data #--initialize和--initialize-insecure区别 #--initialize: 会生成一个12位,4种密码复杂度临时管理员密码. 第一次登陆需要修改. #--initialize-insecure:管理员密码为空. #设置MySQL启动脚本 cd /usr/local/mysql/support-files/ && cp mysql.server /etc/init.d/mysqld systemctl enable mysqld systemctl start mysqld ``` ### mysql5.7 ```bash #!/bin/bash yum -y install numactl-libs libaio wget > /dev/null 2>&1 || apt -y install libaio1 libnuma1 wget > /dev/null 2>&1 if [[ ! -f mysql-5.7.36-el7-x86_64.tar.gz ]];then wget https://mirrors.aliyun.com/mysql/MySQL-5.7/mysql-5.7.36-el7-x86_64.tar.gz fi tar -xf mysql-5.7.36-el7-x86_64.tar.gz -C /usr/local/ if [[ $? != 0 ]];then echo "not found mysql-5.7.36-el7-x86_64.tar.gz" && exit 1 fi mv /usr/local/mysql-5.7.36-el7-x86_64 /usr/local/mysql #创建mysql 用户组和用户 groupadd mysql useradd -r -g mysql mysql #更改mysql目录下所有文件夹所属的用户组、用户以及权限 chown -R mysql:mysql /usr/local/mysql chmod -R 755 /usr/local/mysql #进入/usr/local/mysql/bin/目录,编译安装并初始化mysql cd /usr/local/mysql/bin/ #重点知识2: --initialize和--initialize-insecure区别 #--initialize: 会生成一个12位,4种密码复杂度临时管理员密码. 第一次登陆需要修改. #--initialize-insecure:管理员密码为空. /usr/local/mysql/bin/mysqld --initialize-insecure --user=mysql --datadir=/usr/local/mysql/data --basedir=/usr/local/mysql #编写etc目录下的my.cnf 配置文件,并添加配置 cat > /etc/my.cnf <<EOF [mysqld] datadir=/usr/local/mysql/data port = 3306 sql_mode=NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES symbolic-links=0 max_connections=400 innodb_file_per_table=1 #表名大小写不明感,敏感为 lower_case_table_names=1 EOF chmod -R 775 /etc/my.cnf #添加软连接,并重启mysql 服务 ln -s /usr/local/mysql/support-files/mysql.server /etc/init.d/mysql ln -s /usr/local/mysql/bin/mysql /usr/bin/mysql #启动mysql服务 cat > /etc/systemd/system/mysql.service <<EOF [Unit] Description=MySQL Server After=network.target [Service] ExecStart=/etc/init.d/mysql start ExecStop=/etc/init.d/mysql stop ExecReload=/etc/init.d/mysql reload Type=forking Restart=always TimeoutSec=30 [Install] WantedBy=multi-user.target EOF systemctl enable mysql --now #开放远程连接 mysql -e "use mysql; update user set user.Host='%' where user.User='root'; flush privileges;" ``` ### 安装docker ``` #!/bin/bash yum install -y yum-utils device-mapper-persistent-data lvm2 yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo yum install -y docker-ce mkdir -p /etc/docker/ systemctl start docker systemctl enable docker cat >> /etc/docker/daemon.json <<EOF { "registry-mirrors" : [ "https://8xpk5wnt.mirror.aliyuncs.com" ] } EOF sleep 1 systemctl restart docker if [ -f "/usr/bin/docker" ]; then echo "docker $(docker version|grep Version)安装成功" else echo "安装失败" fi ``` --- ### 安装go ``` #!/bin/bash VERSION=1.17.4 go_install(){ wget https://golang.google.cn/dl/go${VERSION}.linux-amd64.tar.gz tar -zxf go${VERSION}.linux-amd64.tar.gz -C /usr/local #解压go包到/usr/local cat >> /etc/profile <<EOF export GOPROXY=https://goproxy.cn,direct export GO111MODULE=on export PATH=\$PATH:/usr/local/go/bin:/root/gopath/bin EOF source /etc/profile ln -s /usr/local/go/bin/go /usr/bin/ } go_install go version if [ $? -eq 0 ];then go env -w GO111MODULE=on go env -w GOPROXY=https://goproxy.cn,direct echo "golang${VERSION}安装完成" else echo "golang安装失败!" fi ``` ### python3.7 ``` #!/bin/bash yum install git gcc gcc-c++ make automake autoconf libtool pcre pcre-devel zlib zlib-devel openssl-devel wget vim -y yum install libXcomposite libXcursor libXi libXtst libXrandr alsa-lib mesa-libEGL libXdamage mesa-libGL libXScrnSaver -y yum install libffi-devel -y wget https://www.python.org/ftp/python/3.7.2/Python-3.7.2.tar.xz xz -d Python-3.7.2.tar.xz tar -xvf Python-3.7.2.tar ./Python-3.7.2/configure --enable-shared --prefix=/usr/local CFLAGS=-fPIC LDFLAGS="-Wl,-rpath /usr/local/lib" make -j24 make install ln -sf /usr/local/bin/python3.7 /usr/bin/python3 ln -sf /usr/local/bin/python3.7 /usr/bin/python3.7 ln -sf /usr/local/bin/pip3.7 /usr/bin/pip3 ln -sf /usr/local/bin/pip3.7 /usr/bin/pip3.7 python3 --version ``` ### java安装 tmp.link ``` mkdir -p /usr/local/java mv jdk-8u301-linux-x64.tar.gz /usr/local/java/ cd /usr/local/java tar -zxvf jdk-8u301-linux-x64.tar.gz echo 'export JAVA_HOME=/usr/local/java/jdk1.8.0_301 export JRE_HOME=/usr/local/java/jdk1.8.0_301/jre export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib:$CLASSPATH export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile sleep 1 source /etc/profile ``` ### 分库分表备份 ``` #!/bin/bash MYUSER=root MYPWD=chaoge999 DBPATH=/mysql_db_back/ MYCMD="mysql -u$MYUSER -p$MYPWD" MYDUMP="mysqldump -u$MYUSER -p$MYPWD" [ ! -d "$DBPATH" ] && mkdir $DBPATH for dbname in `$MYCMD -e "show databases;"|sed '1d'|egrep -v "mysql|schema"` do # 创建数据库同名文件夹 mkdir ${DBPATH}/${dbname}_$(date +%F) -p # 循环找出所有数据表 for table in `$MYCMD -e "show tables from $dbname;"|sed '1d'` do $MYDUMP $dbname $table|gzip > $DBPATH/${dbname}_$(date +%F)/${dbname}_${table}.sql.gz done done ``` 恢复 ``` gunzip < ${dbname}_${table}.sql.gz | mysql -u$MYUSER -p$MYPWD database_name ``` ```bash #!/bin/bash DB_HOST="10.1.125.39" DB_USER="root" DB_PASSWD="ssqj@easyviews.pw" DB_PORT="30306" DBPATH=/mysql_db_back/ DB_CMD="mysql -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} -p${DB_PASSWD}" DB_DUMP="mysqldump -h ${DB_HOST} -P ${DB_PORT} -u${DB_USER} -p${DB_PASSWD}" [ ! -d "$DBPATH" ] && mkdir $DBPATH ######################################################################## # Usage: mysqldump [OPTIONS] database [tables] # OR mysqldump [OPTIONS] --databases [OPTIONS] DB1 [DB2 DB3...] # OR mysqldump [OPTIONS] --all-databases [OPTIONS] # For more options, use mysqldump --help ######################################################################## NC='\033[0m' GREEN='\033[0;32m' RED='\033[0;31m' for DB in `${DB_CMD} -e "show databases;" |sed '1d'|egrep -v "mysql|schema" `; do # 创建数据库同名文件夹 mkdir ${DBPATH}/${DB}_$(date +%F) -p # 备份数据库 $DB_DUMP --databases ${DB} |gzip > ${DBPATH}/${DB}_$(date +%F)/${DB}_`date +%F`.sql.gz if [[ $? == 0 ]];then echo -e "${GREEN} [`date +%Y-%m-%d_%H:%M:%S`] Database ${DB} backup completed !${NC}" else echo -e "${RED} [`date +%Y-%m-%d_%H:%M:%S`] Database ${DB} backup failed !${NC}" && exit 1 fi # 遍历数据库中的表 for table in `$DB_CMD -e "show tables from ${DB};"|sed '1d'` do SAVE_PATH=$DBPATH/${DB}_$(date +%F)/${DB}_${table}.sql.gz # 备份数据库中的表 $DB_DUMP ${DB} ${table}|gzip > $SAVE_PATH if [ $? -eq 0 ]; then echo -e "${GREEN} Database: [${DB}] - Table:[${table}] backup to $SAVE_PATH completed ! ${NC}" else echo -e "${RED} Database: [${DB}] - Table:[${table}] backup failed ! ${NC}" && exit 1 fi done done ```