forked from diskoverdata/diskover-community
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathdiskover-bot-launcher.sh
More file actions
executable file
·182 lines (164 loc) · 5.08 KB
/
Copy pathdiskover-bot-launcher.sh
File metadata and controls
executable file
·182 lines (164 loc) · 5.08 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
#!/bin/bash
# diskover bot launcher multi-processing script
# starts multiple bots to help with diskover redis queue
# https://github.com/shirosaidev/diskover
#
# Copyright (C) Chris Park 2017-2018
# diskover is released under the Apache 2.0 license. See
# LICENSE for the full license text.
#
############# EDIT BELOW FOR YOUR ENVIRONMENT #############
# paths and default config values below
# path to python command or python if in PATH
PYTHON=python
# path to diskover_worker_bot.py
DISKOVERBOT=./diskover_worker_bot.py
# path to killredisconn.py
KILLREDISCONN=./killredisconn.py
# number of bots to start (cores x 2 might be good start)
WORKERBOTS=8
# run bots in burst mode (quit when all jobs done)
BURST=FALSE
# file to store bot pids
BOTPIDS=/tmp/diskover_bot_pids
# queue that worker bots should listen on
# queues: diskover, diskover_crawl, diskover_calcdir
# default ALL (listen on all queues)
QUEUE=ALL
###########################################################
VERSION="1.4"
function printhelp {
echo "Usage: $(basename $0) [OPTION] [ROOTDIR]"
echo
echo "Options:"
echo
echo " -w n number of worker bots to start (default $WORKERBOTS)"
echo " -q queue which queue bots should listen on (default ALL)"
echo " -b burst mode (bots quit when no more jobs)"
echo " -s show all bots running"
echo " -k kill all bots"
echo " -r restart all running bots"
echo " -R remove any stuck/idle worker bot connections in Redis"
echo " -f force remove all worker bot connections in Redis"
echo " -V displays version and exits"
echo " -h displays this help message and exits"
echo
echo "diskover worker bot process launcher v$VERSION"
echo
echo "Adjust default paths and settings at top of script"
exit 1
}
KILLBOTS=FALSE
RESTARTBOTS=FALSE
REMOVEBOTS=FALSE
FORCEREMOVEBOTS=FALSE
SHOWBOTS=FALSE
while getopts :h?w:q:bskrRfV opt; do
case "$opt" in
h) printhelp; exit 0;;
w) WORKERBOTS=$OPTARG;;
q) QUEUE=$OPTARG;;
b) BURST=TRUE;;
s) SHOWBOTS=TRUE;;
k) KILLBOTS=TRUE;;
r) RESTARTBOTS=TRUE;;
R) REMOVEBOTS=TRUE;;
f) FORCEREMOVEBOTS=TRUE; REMOVEBOTS=TRUE;;
V) echo "$0 v$VERSION"; exit 0;;
?) echo "Invalid option ${OPTARG}, use -h for help" >&2; exit 1;;
esac
done
function banner {
echo "$(tput setaf 1)
________ .__ __
\\______ \\ |__| _____| | _________ __ ___________
| | \\| |/ ___/ |/ / _ \\ \\/ // __ \\_ __ \\ /)___(\\
| \` \\ |\\___ \\| < <_> ) /\\ ___/| | \\/ (='.'=)
/_______ /__/____ >__|_ \\____/ \\_/ \\___ >__| (\"\)_(\"\)
\\/ \\/ \\/ \\/
Worker Bot Launcher v$VERSION
https://github.com/shirosaidev/diskover
\"Crawling all your stuff, core melting time\"$(tput sgr 0)
"
}
function startbots {
echo "Starting $WORKERBOTS worker bots in background..."
ARGS=""
if [ $BURST == TRUE ]; then
ARGS+="-b"
fi
if [ $QUEUE != "ALL" ]; then
ARGS+="-q $QUEUE"
fi
for (( i = 1; i <= $WORKERBOTS; i++ )); do
$PYTHON $DISKOVERBOT $ARGS > /dev/null 2>&1 &
echo "$(hostname -s).$! (pid $!)"
echo "$!" >> $BOTPIDS
done
echo "DONE!"
echo "All worker bots have started"
echo "Worker pids have been stored in $BOTPIDS, use -k flag to shutdown workers or -r to restart"
echo "Exiting, sayonara!"
}
function killbots {
if [ -f $BOTPIDS ]; then
echo "Killing worker bot pids:"
for PID in `cat $BOTPIDS`; do
echo "$PID"
if `ps -p $PID > /dev/null 2>&1`; then
kill $PID
fi
done
echo "All worker bots have been sent shutdown command"
rm $BOTPIDS
else
echo "Pid file can't be found, killing any bots..."
pkill -f diskover_worker_bot.py
echo "All worker bots have been sent shutdown command"
fi
}
function removebots {
echo "Removing any stuck/idle worker bot connections in Redis..."
if [ $FORCEREMOVEBOTS == TRUE ]; then
$PYTHON $KILLREDISCONN -f
else
$PYTHON $KILLREDISCONN
fi
echo "Done"
}
function showbots {
if [ -f $BOTPIDS ]; then
echo "Running worker bots:"
for PID in `cat $BOTPIDS`; do
if `ps -p $PID > /dev/null 2>&1`; then
echo "$(hostname -s).$PID (pid $PID)"
fi
done
else
echo "Pid file can't be found, running worker bot pids:"
pgrep -f diskover_worker_bot.py
fi
}
function countbots {
if [ -f $BOTPIDS ]; then
WORKERBOTS=`cat $BOTPIDS | wc -l | tr -d '[:space:]'`
else
WORKERBOTS=`pgrep -f diskover_worker_bot.py | wc -l | tr -d '[:space:]'`
fi
}
banner
if [ $KILLBOTS == TRUE ]; then
killbots
elif [ $REMOVEBOTS == TRUE ]; then
removebots
elif [ $SHOWBOTS == TRUE ]; then
showbots
elif [ $RESTARTBOTS == TRUE ]; then
countbots
killbots
echo sleeping for 2 seconds...
sleep 2
startbots
else
startbots
fi