Block crawl(bad bot) bot requests
sudo apt install ipset iptables
ipset create blacklist hash:ip
iptables -A FORWARD -m set --match-set blacklist src -j DROP
chmod +x parser.py logfilter.sh run.sh
chmod -R 777 ips/ logs/ results/
./parser.py
*/5 * * * * /usr/bin/python3.5 /PATH/parser.py >> /PATH/cron.log 2>&1
Notes: