| 0,0 → 1,378 |
|---|
| #!/bin/bash |
| |
| # $Id$ |
| |
| # alcasar-bl.sh |
| # by Franck BOUIJOUX and Richard REY |
| # This script is distributed under the Gnu General Public License (GPL) |
| |
| # Gestion de la BL pour le filtrage de domaine (via unbound) et d'URL (via E2guardian) |
| # Manage the BL for domain filtering (with unbound) and URL filtering (with E2guardian) |
| |
| DIR_CONF="/usr/local/etc" |
| CONF_FILE="$DIR_CONF/alcasar.conf" |
| private_ip_mask=`grep ^PRIVATE_IP= $CONF_FILE|cut -d"=" -f2` |
| private_ip_mask=${private_ip_mask:=192.168.182.1/24} |
| DIR_tmp="/tmp/blacklists" |
| FILE_tmp="/tmp/filesfilter.txt" |
| FILE_ip_tmp="/tmp/filesipfilter.txt" |
| DIR_DG="/etc/e2guardian/lists" |
| DIR_DG_BL="$DIR_DG/blacklists" |
| DIR_DG_GROUP1="$DIR_DG/group1" |
| GLOBAL_USAGE="$DIR_CONF/alcasar-global-usage" # file containing the description of the lists |
| BL_CATEGORIES="$DIR_CONF/alcasar-bl-categories" # list of names of the BL categories |
| WL_CATEGORIES="$DIR_CONF/alcasar-wl-categories" # ' ' WL categories |
| BL_CATEGORIES_ENABLED="$DIR_CONF/alcasar-bl-categories-enabled" # ' ' BL enabled categories |
| WL_CATEGORIES_ENABLED="$DIR_CONF/alcasar-wl-categories-enabled" # ' ' WL enabled categories |
| DIR_SHARE="/usr/local/share" |
| DIR_DNS_BL="$DIR_SHARE/unbound-bl" # all the BL in the Unbound format |
| DIR_DNS_WL="$DIR_SHARE/unbound-wl" # all the WL ' ' ' |
| DIR_IP_BL="$DIR_SHARE/iptables-bl" # all the IP addresses of the BL |
| DIR_IP_WL="$DIR_SHARE/iptables-wl" # IP ossi disabled WL |
| DIR_DNS_BL_ENABLED="$DIR_SHARE/unbound-bl-enabled" # symbolic link to the domains BL (only enabled categories) |
| DIR_DNS_WL_ENABLED="$DIR_SHARE/unbound-wl-enabled" # ' ' ' WL ' ' |
| DIR_IP_BL_ENABLED="$DIR_SHARE/iptables-bl-enabled" # ' ' ip BL (only enabled categories) |
| DIR_IP_WL_ENABLED="$DIR_SHARE/iptables-wl-enabled" # ' ' ip WL (ossi and ossi-* imported from ACC) |
| REHABILITATED_DNS_FILE="/etc/unbound/conf.d/blacklist/rehabilitated.conf" |
| BL_SERVER="dsi.ut-capitole.fr" |
| SED="/bin/sed -i" |
| |
| # enable/disable the BL & WL categories |
| function cat_choice (){ |
| mkdir -p $DIR_tmp |
| for LIST in $DIR_IP_BL_ENABLED $DIR_DNS_BL_ENABLED $DIR_IP_WL_ENABLED $DIR_DNS_WL_ENABLED |
| do |
| if [ ! -e $LIST ] # only on install stage |
| then |
| mkdir $LIST |
| else |
| rm -rf ${LIST:?}/* |
| fi |
| chown root:apache $LIST |
| chmod 770 $LIST |
| done |
| # update categories with rsync |
| if [ ! -e $DIR_CONF/update_cat.conf ] |
| then |
| touch $DIR_CONF/update_cat.conf |
| chown root:apache $DIR_CONF/update_cat.conf |
| chmod 660 $DIR_CONF/update_cat.conf |
| fi |
| $SED "/\.Include/d" $DIR_DG_GROUP1/bannedurllist # cleaning E2G url blacklisted |
| # $SED "/\.Include/d" $DIR_DG_GROUP1/bannedsitelist # cleaning E2G domain blacklisted (now managed by unbound) |
| $SED "s?^[^#]?#&?g" $BL_CATEGORIES $WL_CATEGORIES # cleaning BL & WL categories file (comment all lines) |
| |
| # process the file $BL_CATEGORIES with the choice of categories |
| for ENABLE_CATEGORIE in `cat $BL_CATEGORIES_ENABLED` |
| do |
| $SED "/\/$ENABLE_CATEGORIE$/d" $BL_CATEGORIES |
| $SED "1i\/etc\/e2guardian\/lists\/blacklists\/$ENABLE_CATEGORIE" $BL_CATEGORIES |
| ln -sf $DIR_DNS_BL/$ENABLE_CATEGORIE.conf $DIR_DNS_BL_ENABLED/$ENABLE_CATEGORIE |
| ln -sf $DIR_IP_BL/$ENABLE_CATEGORIE $DIR_IP_BL_ENABLED/$ENABLE_CATEGORIE |
| # echo ".Include<$DIR_DG_BL/$ENABLE_CATEGORIE/domains>" >> $DIR_DG_GROUP1/bannedsitelist # Blacklisted domains are managed by unbound |
| echo ".Include<$DIR_DG_BL/$ENABLE_CATEGORIE/urls>" >> $DIR_DG_GROUP1/bannedurllist |
| done |
| sort +0.0 -0.2 $BL_CATEGORIES -o $FILE_tmp |
| mv $FILE_tmp $BL_CATEGORIES |
| sort +0.0 -0.2 $BL_CATEGORIES_ENABLED -o $FILE_tmp |
| mv $FILE_tmp $BL_CATEGORIES_ENABLED |
| chown root:apache $BL_CATEGORIES $BL_CATEGORIES_ENABLED |
| chmod 660 $BL_CATEGORIES $BL_CATEGORIES_ENABLED |
| |
| # process the file $WL_CATEGORIES with the choice of categories |
| for ENABLE_CATEGORIE in `cat $WL_CATEGORIES_ENABLED` |
| do |
| $SED "/\/$ENABLE_CATEGORIE$/d" $WL_CATEGORIES |
| $SED "1i\/etc\/e2guardian\/lists\/blacklists\/$ENABLE_CATEGORIE" $WL_CATEGORIES |
| ln -sf $DIR_DNS_WL/$ENABLE_CATEGORIE.conf $DIR_DNS_WL_ENABLED/$ENABLE_CATEGORIE |
| done |
| sort +0.0 -0.2 $WL_CATEGORIES -o $FILE_tmp |
| mv $FILE_tmp $WL_CATEGORIES |
| sort +0.0 -0.2 $WL_CATEGORIES_ENABLED -o $FILE_tmp |
| mv $FILE_tmp $WL_CATEGORIES_ENABLED |
| chown root:apache $WL_CATEGORIES $WL_CATEGORIES_ENABLED |
| chmod 660 $WL_CATEGORIES $WL_CATEGORIES_ENABLED |
| } |
| |
| # cleaning file and split it ("domains" in $FILE_tmp & "IP" in $FILE_ip_tmp) |
| function clean_split (){ |
| $SED '/^[.#]/d' $FILE_tmp # remove commented lines and lines beginning with a dot |
| $SED '/^\s*$/d' $FILE_tmp # remove empty lines |
| $SED '/[äâëêïîöôüû@,]/d' $FILE_tmp # remove line with "chelou" characters |
| # extract ip addresses and ip networks for iptables. |
| awk '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/{print "add bl_ip_blocked " $0}' $FILE_tmp > $FILE_ip_tmp |
| awk '/^([0-9]{1,3}\.){3}[0-9]{1,3}\/[0-9]{1,2}$/{print "add bl_ip_blocked " $0}' $FILE_tmp >> $FILE_ip_tmp |
| # extract domain names for unbound. |
| $SED -n '/^\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/!p' $FILE_tmp |
| # Retrieve max Top Level Domain for domain name synthax |
| #MAX_TLD=$(curl http://data.iana.org/TLD/tlds-alpha-by-domain.txt | grep -v '-' | grep -v '#' | wc -L) |
| #if [ $(echo $MAX_TLD | wc -c) -eq 0 ];then |
| # MAX_TLD=18 |
| #fi |
| # search for correction grep -E "([a-zA-Z0-9_-.]+\.){1,2}[a-zA-Z]{2,$MAX_TLD}" $ossi_custom_dir/domains > $FILE_tmp |
| } |
| |
| usage="Usage: alcasar-bl.sh { -cat_choice or --cat_choice } | { -download or --download } | { -adapt or --adapt } | { -reload or --reload }" |
| nb_args=$# |
| args=$1 |
| if [ $nb_args -eq 0 ] |
| then |
| args="-h" |
| fi |
| case $args in |
| -\? | -h* | --h*) |
| echo "$usage" |
| exit 0 |
| ;; |
| # Retrieve Toulouse University BL |
| -download | --download) |
| rm -rf /tmp/con_ok.html |
| `/usr/bin/curl $BL_SERVER -# -o /tmp/con_ok.html` |
| if [ ! -e /tmp/con_ok.html ] |
| then |
| echo "Erreur : le serveur de blacklist ($BL_SERVER) n'est pas joignable" |
| else |
| rm -rf /tmp/con_ok.html $DIR_tmp |
| mkdir $DIR_tmp |
| wget -P $DIR_tmp http://$BL_SERVER/blacklists/download/blacklists.tar.gz |
| md5sum $DIR_tmp/blacklists.tar.gz | cut -d" " -f1 > $DIR_tmp/md5sum |
| chown -R apache:apache $DIR_tmp |
| fi |
| ;; |
| # enable/disable categories (used only during the alcasar install process) |
| -cat_choice | --cat_choice) |
| cat_choice |
| ;; |
| # Adapt Toulouse University BL to ALCASAR architecture (unbound + DG + iptables) |
| -adapt | --adapt) |
| echo -n "Adaptation process of Toulouse University blackList. Please wait : " |
| if [ -f $DIR_tmp/blacklists.tar.gz ] # when downloading the last version of the BL |
| then |
| # keep custom files (ossi) |
| for x in $(ls -1 $DIR_DG_BL | grep "^ossi-*") |
| do |
| mv $DIR_DG_BL/$x $DIR_tmp |
| done |
| rm -rf $DIR_DG_BL $DIR_IP_BL |
| mkdir $DIR_DG_BL $DIR_IP_BL |
| tar zxf $DIR_tmp/blacklists.tar.gz --directory=$DIR_DG/ |
| # replace the global_usage file of the archive |
| cp -f $GLOBAL_USAGE $DIR_DG_BL/global_usage |
| chown -R e2guardian:apache $DIR_DG |
| chmod -R 770 $DIR_DG |
| # Add the two local categories (ossi-bl & ossi-wl) to the usage file |
| # Add the custom categories (ossi-tor_nodes) to the usage file |
| cat <<EOF >> $DIR_DG_BL/global_usage |
| |
| NAME: ossi-bl |
| DEFAULT_TYPE: black |
| SOURCE: ALCASAR Team |
| DESC FR: sites blacklistés ajoutés localement |
| DESC EN: blacklisted sites add locally |
| NAME FR: ossi-bl |
| NAME EN: ossi-bl |
| |
| NAME: ossi-wl |
| DEFAULT_TYPE: white |
| SOURCE: ALCASAR Team |
| DESC FR: sites autorisés ajoutés localement |
| DESC EN: whitelisted sites add locally |
| NAME FR: ossi-wl |
| NAME EN: ossi-wl |
| |
| NAME: ossi-bl-tor_nodes |
| DEFAULT_TYPE: black |
| SOURCE: ALCASAR Team |
| DESC FR: Adresses IP des noeuds (routeurs) d'entrée du réseau TOR |
| DESC EN: IP addresses of input TOR nodes (routers) |
| NAME FR: Noeuds TOR |
| NAME EN: TOR nodes |
| |
| NAME: ossi-bl-ultrasurf |
| DEFAULT_TYPE: black |
| SOURCE: ALCASAR Team |
| DESC FR: Adresses IP des point de sortie ULTRASURF |
| DESC EN: IP addresses of output points of ULTRASURF |
| NAME FR: Points de sortie ULTRASURF |
| NAME EN: ULTRASURF output points |
| |
| NAME: ossi-bl-candc |
| DEFAULT_TYPE: black |
| SOURCE: Bambenek Consulting: https://osint.bambenekconsulting.com |
| DESC FR: liste des URLs et IPs des serveurs command & control |
| DESC EN: list of URLs and IPs of command & control servers |
| NAME FR: Serveurs Command & Control |
| NAME EN: Command & Control Server |
| |
| EOF |
| # Retrieve custom files (ossi) |
| for x in $(ls -1 $DIR_tmp | grep "^ossi-*") |
| do |
| mv $DIR_tmp/$x $DIR_DG_BL |
| done |
| fi |
| rm -f $BL_CATEGORIES $WL_CATEGORIES $WL_CATEGORIES_ENABLED |
| rm -rf $DIR_DNS_BL $DIR_DNS_WL $DIR_IP_BL $DIR_IP_WL |
| rm -rf $DIR_DNS_BL_ENABLED $DIR_DNS_WL_ENABLED $DIR_IP_BL_ENABLED $DIR_IP_WL_ENABLED |
| touch $BL_CATEGORIES $WL_CATEGORIES |
| mkdir $DIR_DNS_BL $DIR_DNS_WL $DIR_IP_BL $DIR_IP_WL |
| mkdir $DIR_DNS_BL_ENABLED $DIR_DNS_WL_ENABLED $DIR_IP_BL_ENABLED $DIR_IP_WL_ENABLED |
| find $DIR_DG_BL/ -type f -name domains > $FILE_tmp # retrieve directory name where a domain file exist |
| $SED "s?\/domains??g" $FILE_tmp # remove "/domains" suffix |
| for dir_categorie in `cat $FILE_tmp` # create the blacklist and the whitelist files |
| do |
| categorie=`echo $dir_categorie|cut -d "/" -f6` |
| categorie_type=`grep -A1 ^NAME:[$' '$'\t']*$categorie\$ $DIR_DG_BL/global_usage | grep ^DEFAULT_TYPE | cut -d":" -f2 | tr -d " \t"` |
| if [ "$categorie_type" == "white" ] |
| then |
| echo "$categorie" >> $WL_CATEGORIES_ENABLED |
| fi |
| echo "$dir_categorie" >> $BL_CATEGORIES |
| echo "$dir_categorie" >> $WL_CATEGORIES |
| done |
| rm -f $FILE_tmp |
| # Verify that the enabled categories are effectively in the BL (need after an update of the BL) |
| for ENABLE_CATEGORIE in `cat $BL_CATEGORIES_ENABLED` |
| do |
| ok=`grep /$ENABLE_CATEGORIE$ $BL_CATEGORIES|wc -l` |
| if [ $ok != "1" ] |
| then |
| $SED "/^$ENABLE_CATEGORIE$/d" $BL_CATEGORIES_ENABLED |
| fi |
| done |
| # Verify that the enabled categories are effectively in the WL (need after an update of the WL) |
| for ENABLE_CATEGORIE in `cat $WL_CATEGORIES_ENABLED` |
| do |
| ok=`grep /$ENABLE_CATEGORIE$ $WL_CATEGORIES|wc -l` |
| if [ $ok != "1" ] |
| then |
| $SED "/^$ENABLE_CATEGORIE$/d" $WL_CATEGORIES_ENABLED |
| fi |
| done |
| |
| # Creation of Unbound and Iptables BL and WL |
| for LIST in $BL_CATEGORIES $WL_CATEGORIES # for each list (bl and wl) |
| do |
| for PATH_FILE in `cat $LIST` # for each category |
| do |
| DOMAIN=`basename $PATH_FILE` |
| echo -n "$DOMAIN, " |
| if [ ! -f $PATH_FILE/urls ] # create 'urls' file if it doesn't exist |
| then |
| touch $PATH_FILE/urls |
| chown e2guardian:apache $PATH_FILE/urls |
| fi |
| cp $PATH_FILE/domains $FILE_tmp |
| clean_split # clean ossi custom files & split them for unbound and for iptables |
| if [ "$LIST" == "$BL_CATEGORIES" ] |
| then |
| # adapt to the unbound syntax for the blacklist |
| $SED "s?.*?local-zone: & typetransparent\nlocal-zone-tag: & blacklist?g" $FILE_tmp |
| mv $FILE_tmp $DIR_DNS_BL/$DOMAIN.conf |
| mv $FILE_ip_tmp $DIR_IP_BL/$DOMAIN |
| else |
| # adapt to the unbound syntax for the whitelist |
| $SED "s?.*?local-zone: & transparent?g" $FILE_tmp |
| $SED "p; s? transparent? ipset?g" $FILE_tmp # duplicate lines to enable ipset module |
| mv $FILE_tmp $DIR_DNS_WL/$DOMAIN.conf |
| fi |
| done |
| done |
| echo |
| chown -R root:apache $BL_CATEGORIES $WL_CATEGORIES $BL_CATEGORIES_ENABLED $WL_CATEGORIES_ENABLED $DIR_DNS_BL $DIR_DNS_WL $DIR_IP_BL $DIR_IP_WL |
| chmod 770 $DIR_DNS_BL $DIR_DNS_WL $DIR_IP_BL $DIR_IP_WL |
| chmod -f 660 $BL_CATEGORIES $WL_CATEGORIES $BL_CATEGORIES_ENABLED $WL_CATEGORIES_ENABLED $DIR_DNS_BL/* $DIR_DNS_WL/* $DIR_IP_BL/* $DIR_IP_WL/* |
| rm -f $FILE_tmp $FILE_ip_tmp |
| rm -rf $DIR_tmp |
| ;; |
| # reload when selected categories are changed or when ossi change his custom files |
| -reload | --reload) |
| # for E2Gardian |
| cat_choice |
| # for unbound (rehabilitated domain names) |
| rm -f $REHABILITATED_DNS_FILE |
| if [ "$(wc -w $DIR_DG_GROUP1/exceptionsitelist | cut -d " " -f1)" != "0" ] |
| then |
| touch $REHABILITATED_DNS_FILE |
| while read -r domain; do |
| [ -z "$domain" ] && continue |
| echo "local-zone: $domain typetransparent" >> $REHABILITATED_DNS_FILE |
| echo "local-zone-tag: $domain \"\"" >> $REHABILITATED_DNS_FILE |
| done < $DIR_DG_GROUP1/exceptionsitelist |
| fi |
| # adapt OSSI BL & WL custom files |
| for dir in $DIR_DNS_BL_ENABLED $DIR_DNS_WL_ENABLED $DIR_IP_BL_ENABLED $DIR_IP_WL_ENABLED $DIR_DNS_BL $DIR_DNS_WL $DIR_IP_BL $DIR_IP_WL |
| do |
| rm -f $dir/ossi* |
| done |
| find $DIR_DG_BL/ -type f -name domains | grep ossi- > $FILE_tmp # retrieve ossi directories name where a domain file exist |
| $SED "s?\/domains??g" $FILE_tmp # remove "/domains" suffix |
| for ossi_custom_dir in `cat $FILE_tmp` # create the blacklist and the whitelist files |
| do |
| chown -R root:apache $ossi_custom_dir |
| chmod 770 $ossi_custom_dir |
| ossi_categorie=`echo $ossi_custom_dir|cut -d "/" -f6` |
| short_categorie=`echo "$ossi_categorie" | cut -d"-" -f2` |
| if [ $short_categorie == "bl" ] |
| then |
| categorie_type="black" |
| else |
| categorie_type="white" |
| fi |
| $SED "s/\r//" $ossi_custom_dir/domains $ossi_custom_dir/urls # remove Windows <CR> from custom file |
| cp $ossi_custom_dir/domains $FILE_tmp |
| clean_split # clean ossi custom files & split them for unbound and for iptables |
| if [ $categorie_type == "white" ] |
| then |
| # adapt the file to the unbound syntax and enable it if needed |
| # for the WL |
| $SED "s?.*?local-zone: & transparent?g" $FILE_tmp |
| $SED "p; s? transparent? ipset?g" $FILE_tmp # duplicate lines to enable ipset module |
| mv $FILE_tmp $DIR_DNS_WL/$ossi_categorie.conf |
| mv $FILE_ip_tmp $DIR_IP_WL/$ossi_categorie |
| enabled=`grep ^$ossi_categorie$ $WL_CATEGORIES_ENABLED | wc -l` |
| if [ $enabled == "1" ] |
| then |
| $SED "/\/$ossi_categorie$/d" $WL_CATEGORIES |
| $SED "1i\/etc\/e2guardian\/lists\/blacklists\/$ossi_categorie" $WL_CATEGORIES |
| ln -sf $DIR_DNS_WL/$ossi_categorie.conf $DIR_DNS_WL_ENABLED/$ossi_categorie |
| ln -sf $DIR_IP_WL/$ossi_categorie $DIR_IP_WL_ENABLED/$ossi_categorie |
| fi |
| else |
| # for the BL |
| $SED "s?.*?local-zone: & typetransparent\nlocal-zone-tag: & blacklist?g" $FILE_tmp |
| mv $FILE_tmp $DIR_DNS_BL/$ossi_categorie.conf |
| mv $FILE_ip_tmp $DIR_IP_BL/$ossi_categorie |
| enabled=`grep ^$ossi_categorie$ $BL_CATEGORIES_ENABLED | wc -l` |
| if [ $enabled == "1" ] |
| then |
| $SED "/\/$ossi_categorie$/d" $BL_CATEGORIES |
| $SED "1i\/etc\/e2guardian\/lists\/blacklists\/$ossi_categorie" $BL_CATEGORIES |
| ln -sf $DIR_DNS_BL/$ossi_categorie.conf $DIR_DNS_BL_ENABLED/$ossi_categorie |
| ln -sf $DIR_IP_BL/$ossi_categorie $DIR_IP_BL_ENABLED/$ossi_categorie |
| fi |
| fi |
| done |
| for file in $BL_CATEGORIES $BL_CATEGORIES_ENABLED $WL_CATEGORIES $WL_CATEGORIES_ENABLED |
| do |
| sort +0.0 -0.2 $file -o $FILE_tmp |
| mv $FILE_tmp $file |
| chown root:apache $file |
| chmod 660 $file |
| done |
| chown -R root:apache $DIR_DNS_BL $DIR_DNS_WL $DIR_IP_BL $DIR_IP_WL |
| chmod 660 $DIR_DNS_BL/* $DIR_DNS_WL/* $DIR_IP_BL/* $DIR_IP_WL/* |
| if [ "$PARENT_SCRIPT" != "alcasar-conf.sh" ] # don't launch on install stage |
| then |
| /usr/bin/systemctl restart unbound-blacklist |
| /usr/bin/systemctl restart unbound-whitelist |
| /usr/bin/systemctl restart e2guardian |
| /usr/local/bin/alcasar-iptables.sh |
| fi |
| ;; |
| *) |
| echo "Argument inconnu :$1"; |
| echo "$usage" |
| exit 1 |
| ;; |
| esac |
| Property changes: |
|---|
| Added: svn:eol-style |
|---|
| +LF |
|---|
| \ No newline at end of property |
|---|
| Added: svn:executable |
|---|
| +* |
|---|
| \ No newline at end of property |
|---|
| Added: svn:keywords |
|---|
| +Id Author Date |
|---|
| \ No newline at end of property |
|---|