1. This site uses cookies. By continuing to use this site, you are agreeing to our use of cookies. Learn More.

Dnsmasq Ad Blocking II

Discussion in 'Tomato Firmware' started by yaqui, Aug 1, 2007.

  1. yaqui

    yaqui LI Guru Member

    HOW TO USE THIS SCRIPT: (Known to work with Tomato v1.07)
    ========================================================
    1 - Place this in Administration > Scripts > Init and hit SAVE button
    Code:
    sleep 3
    touch /etc/adblock.conf
    sleep 2
    logger adblock.conf created
    logger Remainder of init script starting
    2 - Change the cru statement (in the main script) to your own custom interval in this format:
    Code:
    cru a <unique id> <"min hour day month week command">   * means every 
    Example
    cru a UpdateAdlist "00 9 * * * $updtadlist"
    That will run it every day at 9am
    See FAQ for more examples.

    3 - Copy and paste main script into Administration > Scripts > WAN UP section > Hit SAVE button

    4 - Copy and paste this into Advanced > DHCP / DNS > Custom Configuration box > then hit SAVE button
    Code:
    conf-file=/etc/adblock.conf
    5 - Reboot your router
    ========================================================
    Main Script:
    Code:
    #### FAT VERSION ####
    logger WAN UP Script will execute after 30sec please wait.....
    sleep 30
    
    ### Define Variables ###
    wait=10
    adlistfile='/etc/adblock.conf'
    updtadlist='/tmp/update-adlist.sh'
    adlisthost='pgl.yoyo.org'
    adlisturl='http://pgl.yoyo.org/adservers/serverlist.php?hostformat=dnsmasq&showintro=0&mimetype=plaintext'
    listhosttest=$(ping -c 1 $adlisthost | grep "from 193.110.91")
    xwait=$wait
    numlines=0
    numbytes=0
    
    ### "Create file update-adlist.sh" function ###
    createupfile ( )
     {
      if [ ! -x $updtadlist ] ; then
      cat > $updtadlist <<EOF
    #!/bin/sh
    ###  Define "update and test" functions ###
     testhost ( )
      {
       if [ -n "$listhosttest" ] ; then
          logger adlist host ping test - SUCCESS - proceeding download and entries test....
          return 0
          else
            logger adlist host ping test - FAILURE - trying again and waiting for reply...
       fi
       while ! ping -c 1 $adlisthost | grep "from 193.110.91" && [ $xwait -le 300 ]; do
           logger Waiting $xwait secs for ping reply from pgl.yoyo.org
           sleep $wait
           xwait=$(expr $xwait + $wait)
       done
      }
    
     downloadlist ( )
      {
       if wget -O $adlistfile "$adlisturl" ; then
          logger adlist downloaded... checking number of entries...
          else
          logger adlist was not downloaded - FAILURE -
          return 1
       fi
      }
    
     checkentries ( )
      {
       numlines=$(awk 'END { print NR }' $adlistfile)
       lowamt=1700
       highamt=300000
       sleep 5
        if [ "$numlines" -lt "$lowamt" ] ; then
           logger Adblock.conf contains less than 1700 - deleting file
           rm -f $adlistfile
           return 1  
           else
           if [ "$numlines" -gt "$highamt" ] ; then
              logger Adblock.conf has excessive number of lines - deleting file
              rm -f $adlistfile
              return 1
              else
              logger Adblock.conf entry size within normal range - SUCCESS -
           fi
        fi
       }       
    
      restartdnsmasq ( )
       {
         if ps | grep -q dnsmasq ; then
           service dnsmasq stop
           sleep 5
           service dnsmasq start
         else
            logger Restarting dnsmasq FAILURE
            return 1
         fi
        }
    
    ### Run and test the functions ###
    
    testhost
     if [ $? == "0" ] ; then
        downloadlist
        if [ $? == "0" ] ; then
           checkentries
           if [ $? == "0" ] ; then
             restartdnsmasq
           fi
        fi 
     else
       exit
     fi
    EOF
    else
      logger update-adlist.sh already exists and is executable will not re-run
    fi
    }
    
      ### Run creation of update-adlist.sh ###
      createupfile
      if [ $? == "0" ] ; then
        chmod +x $updtadlist
        /tmp/update-adlist.sh
        cru a UpdateAdlist "<place your interval here> $updtadlist"
      else
        logger Update file not created and not scheduled - FAILURE -
      fi
      
    ### Misc Functions to Run on Each Wan Up ###
    
      reportsize ( )
      {
      ## See how many lines and bytes adblock.conf contains and output values to logger
      if grep -q address $adlistfile ; then
         numlines=$(awk 'END { print NR }' $adlistfile)
         numbytes=$(ls -lg $adlistfile | awk '{ sum += $5} END {print sum }')
         sleep 5
         logger adblock.conf populated with $numlines lines and $numbytes bytes of data
        else
          logger adblock.conf is not populated !!
      fi
      }
    
      testadhost ( )
      {
      ## BEGIN Testing the first host listed in adblock.conf on each WAN UP
      if grep -q address $adlistfile ; then
         read var1 <$adlistfile
         echo -e $var1 > /tmp/testhost
         awkdhost=$(awk 'BEGIN { FS="/" } { print $2 }' /tmp/testhost)
         adblocktest=$(ping -c 1 $awkdhost | grep "from 127")
                 
            if [ -n "$adblocktest" ] ; then 
               logger First host "$awkdhost" unreachable - Ad blocking is working !!  SUCCESS
            else 
               logger Ad blocking is NOT working FAILURE
            fi
      
      else
        logger Adblock.conf is not populated with address !!
      fi
      ## END Testing the first host listed in adblock.conf on each WAN UP
      }
    
    reportsize
    sleep 5
    testadhost
    Edit: All credit goes to Macskeeball, mstombs, mraneri, the host of pgl.yoyo.org and everyone else who helped!! The above script works. As I stated before I cannot post in the other thread for some reason and I wanted to ask the below question:

    There still exists the issue of having NO adblocking at all upon reboot and list host being down; I am going to try to resolve by making a single backup to jffs and have the script use that if necessary. Unless someone has a better idea ??
     
  2. t4thfavor

    t4thfavor Network Guru Member

    So since I haven't been keeping up with this, where does this file go, what modifcations to the startup process do I have to make, and which firmware will this work with.
     
  3. yaqui

    yaqui LI Guru Member

    I guess I can't do this with my script! Maximum Wan Up script size is 4096 bytes and I can't put anymore into it. :( I suppose I need to take out some "features"

    Edit: Ok I took out alot of things (I wish I didn't have to) but here is my "Lean Version" I am working with
    Code:
    #### LEAN VERSION ####
    logger WAN UP Script will execute after 30sec please wait.....
    sleep 30
    
    #### Define Variables ####
    adlistfile='/etc/adblock.conf'
    updtadlist='/tmp/update-adlist.sh'
    adlisthost='pgl.yoyo.org'
    adlisturl='http://pgl.yoyo.org/adservers/serverlist.php?hostformat=dnsmasq&showintro=0&mimetype=plaintext'
    messages=/var/log/messages
    numlines=0
    numbytes=0
    
    #### Functions ####
     createupfile ( )
     {
      cat > $updtadlist <<EOF
    #!/bin/sh
    #### Define Functions to Be Run in 'here document' ####
      downloadlist ( )
      {
       if ping -c 1 $adlisthost ; then
          if wget -O $adlistfile "$adlisturl" ; then
             logger Adlist downloaded... - Success -
             else
             logger Adlist was NOT Downloaded - FAILURE -
          fi
          else
          logger Ping test and Download - FAILURE -
          return 1
       fi
      }
    
       restartdnsmasq ( )
       {
         if ps | grep -q dnsmasq ; then
           service dnsmasq stop
           sleep 5
           service dnsmasq start
         else
            logger Restarting dnsmasq FAILURE
            return 1
         fi
        }
    
    ## Run and Test the Functions
        downloadlist
        sleep 5
        restartdnsmasq
        sleep 5
    EOF
    }
    
     testupfile ( )
     {
      if [ ! -x $updtadlist ] ; then  
         createupfile
         if [ $? == "0" ] ; then
            chmod +x $updtadlist
            /tmp/update-adlist.sh
            cru a UpdateAdlist "<your interval here> $updtadlist"
         fi
      else
        logger update-adlist.sh already exists and is executable will not re-run
      fi
     }
    
      reportsize ( )
      {
      ## See how many lines and bytes adblock.conf contains and output values to logger
      if grep -q address $adlistfile ; then
        if ! grep -q "adblock.conf populated with" $messages ; then
           numlines=$(awk 'END { print NR }' $adlistfile)
           numbytes=$(ls -lg $adlistfile | awk '{ sum += $5} END {print sum }')
           sleep 5
           logger adblock.conf populated with $numlines lines and $numbytes bytes of data
        fi
      else
         logger adblock.conf is Not populated with 'address' - FAILURE -
      fi
      }
    
      testadhost ( )
      {
      ## Test First Host Listed in adblock.conf  
      if grep -q address $adlistfile ; then
         read var1 <$adlistfile
         echo -e $var1 > /tmp/testhost
         awkdhost=$(awk 'BEGIN { FS="/" } { print $2 }' /tmp/testhost)
         adblocktest=$(ping -c 1 $awkdhost | grep "from 127")
                 
            if [ -n "$adblocktest" ] ; then 
               logger First host "$awkdhost" unreachable - Ad blocking is working !!  SUCCESS
            else 
               logger Ad blocking is NOT working FAILURE
            fi
      
      else
        logger Adblock.conf is Not populated with 'address' - FAILURE -
      fi
      }
    
    #### Main Calls to Functions to Be Run at Wan Up ####
    testupfile
    sleep 5
    reportsize
    sleep 5
    testadhost
    
     
  4. t4thfavor

    t4thfavor Network Guru Member

    You can do it, you just can't do it with the current firmware. Or is that a limitation on the actual file size? I planned on placing my script in the init.d directory and have it run after the network up script.

    Note: I am using OpenWRT and not tomato, but its all about the same.

    EDIT: It also looks like I will need to change the cru statement, since OpenWRT does not have this feature.
     
  5. yaqui

    yaqui LI Guru Member

    Well I'm done with my "Lean Backup Version"

    Instructions
    ========================================================
    This requires that you already have a known good adblock.conf downloaded, otherwise you may have to ftp the file from your computer.

    You only have to do this one time! Or if you feel the backup needs to be updated, do it again at a later date.

    1. Edit scripts from this page and save them to your computer.

    2. Using a web browser - log into your router

    3. Go to administration > admin access

    4. Enable (and start) the Telnet Daemon

    5. Administration > jffs2
    Enable jffs2, click on format/erase, wait then click save at the bottom of the page

    6. Telnet into the router (open a command prompt, then run "telnet routerIP" with your router IP, of course ) and log in

    7. Via telnet, run (in order, of course):

    mkdir /jffs/adbackup/
    cp -p -f /etc/adblock.conf /jffs/adbackup/adbackup.conf

    8. Follow all previous steps in first post if you haven't already.
    =========================================================

    Script:
    Code:
    ### LEAN BACKUP VERSION ###
    ### Remember to telnet in first and manually copy a known good adblock.conf to the jffs folder ###
    logger WAN UP Script will execute after 30sec please wait.....
    sleep 30
    
    #### Define Variables ####
    adlistfile='/etc/adblock.conf'
    updtadlist='/tmp/update-adlist.sh'
    adlisthost='pgl.yoyo.org'
    adlisturl='http://pgl.yoyo.org/adservers/serverlist.php?hostformat=dnsmasq&showintro=0&mimetype=plaintext'
    messages=/var/log/messages
    numlines=0
    numbytes=0
    
    #### Functions ####
     createupfile ( )
     {
      cat > $updtadlist <<EOF
    #!/bin/sh
    #### Define Functions to Be Run in 'here document' ####
      downloadlist ( )
      {
       if ping -c 1 $adlisthost ; then
          if wget -O $adlistfile "$adlisturl" ; then
             logger Adlist downloaded... - Success -
             else
             logger Adlist was NOT Downloaded - FAILURE -
             return 1
          fi
          else
          logger Ping test and Download - FAILURE -
          return 1
       fi
      }
    
      usebackup ( )
      {
       cp -p -f $adbackup $adlistfile
       if grep -q address $adlistfile ; then
            logger adblock.conf populated from backup -SUCCESS -
            else
            logger adblock.conf Not populated from backup! - FAILURE -
       fi
      }
    
      restartdnsmasq ( )
      {
       if ps | grep -q dnsmasq ; then
         service dnsmasq stop
         sleep 5
         service dnsmasq start
       else
          logger Restarting dnsmasq FAILURE
          return 1
       fi
      }
    
    ## Run and Test the Functions
        downloadlist
        if [ $? != "0" ] ; then
          usebackup
        fi    
        sleep 5
        restartdnsmasq
        sleep 5
    EOF
    }
    
     testupfile ( )
     {
      if [ ! -x $updtadlist ] ; then  
         createupfile
         if [ $? == "0" ] ; then
            chmod +x $updtadlist
            /tmp/update-adlist.sh
            cru a UpdateAdlist "<your interval here> $updtadlist"
         fi
      else
        logger update-adlist.sh already exists and is executable will not re-run
      fi
     }
    
      reportsize ( )
      {
      ## See how many lines and bytes adblock.conf contains and output values to logger
      if grep -q address $adlistfile ; then
        if ! grep -q "adblock.conf populated with" $messages ; then
           numlines=$(awk 'END { print NR }' $adlistfile)
           numbytes=$(ls -lg $adlistfile | awk '{ sum += $5} END {print sum }')
           sleep 5
           logger adblock.conf populated with $numlines lines and $numbytes bytes of data
        fi
      else
         logger adblock.conf is Not populated with 'address' - FAILURE -
      fi
      }
    
      testadhost ( )
      {
      ## Test First Host Listed in adblock.conf  
      if grep -q address $adlistfile ; then
         read var1 <$adlistfile
         echo -e $var1 > /tmp/testhost
         awkdhost=$(awk 'BEGIN { FS="/" } { print $2 }' /tmp/testhost)
         adblocktest=$(ping -c 1 $awkdhost | grep "from 127")
                 
            if [ -n "$adblocktest" ] ; then 
               logger First host "$awkdhost" unreachable - Ad blocking is working !!  SUCCESS
            else 
               logger Ad blocking is NOT working FAILURE
            fi
      
      else
        logger Adblock.conf is Not populated with 'address' - FAILURE -
      fi
      }
    
    #### Main Calls to Functions to Be Run at Wan Up ####
    testupfile
    sleep 5
    reportsize
    sleep 5
    testadhost
    
    That *should* work, I am unable to test at this time, I will soon though.
     
  6. yaqui

    yaqui LI Guru Member

    It could be that the web interface only accepts so many bytes, if so I guess I will have to ftp the file.

    Does anyone know what the filesize limit is in busybox, is it more than the web interface allows??
     
  7. t4thfavor

    t4thfavor Network Guru Member

    Well I have had files in the multi megabyte range ftp'd to my router, so I would assume that is all they allocated for the post size on that page of the web interface.

    I thought it would have been a tomato only limit or something about space limitation, but if its not, then you should be fine with the fatter version.
     
  8. Low-WRT

    Low-WRT LI Guru Member

    Pardon my ignorance, but where does the cru command (step #2) go?...In the init script below step #1?
    Thanks
     
  9. Thrasher

    Thrasher Network Guru Member

    The cru command is part of the main script.....search and ye shall find :)

    This thread is getting a little complicated for some to follow.....peeps posting bits of scripts all over.....can it be pulled together at all?

    Why can't the "fat" script be stored in JFFS too and called during bootup?
     
  10. yaqui

    yaqui LI Guru Member

    Well there is basically just 3 versions (Fat:first version listed, Lean, and Lean Backup), I was considering doing the "Fat Backup" version but I figured people could just add in the function; "usebackup" themselves.

    As far as adding to jffs.. you can do that (you HAVE to do that for "fat backup" version) but I don't know if everyone knows how to ftp files to the router or adding files by other means... and didn't feel like writing all instructions for that, If someone wants to post all that; feel free.
     
  11. yaqui

    yaqui LI Guru Member

    Another version you may like... self explanatory, no need for jffs.

    **Remember to change the cru statement to your own interval.**

    First change init script to this and Hit SAVE (thanks MiseryQ):
    Code:
    sleep 3
    touch /etc/pgl.conf
    sleep 2
    logger pgl.conf created
    logger Remainder of init script starting
    Next, Go to Advanced > DHCP / DNS > and paste below into custom box, hit SAVE
    Code:
    conf-file=/etc/pgl.conf
    Then place this under Wan Up script and hit SAVE, then REBOOT your router:
    Code:
    #### MVPS or PGL as Backup ####
    #### If for some reason pgl.yoyo.org's list is down this version will try Mvps.org list instead or vice-versa if you swap under 'Run the Functions' ####
    logger WAN UP Script will execute after 45sec please wait.....
    sleep 45
    
    #### Define Variables ####
    pglfile='/etc/pgl.conf'
    updtadlist='/tmp/update-adlist.sh'
    pglhost='pgl.yoyo.org'
    pglurl='http://pgl.yoyo.org/adservers/serverlist.php?hostformat=dnsmasq&showintro=0&mimetype=plaintext'
    mvpshost='www.mvps.org'
    mvpsurl='http://www.mvps.org/winhelp2002/hosts.txt'
    hosts='/etc/hosts'
    messages='/var/log/messages'
    numlines=0
    numbytes=0
    
    #### Update Script Function ####
     createupfile ( )
     {
      cat > $updtadlist <<EOF
    #!/bin/sh
    #### Define Functions to Run in 'here document' ####
      downloadmvps ( )
      {
      if ping -c 1 $mvpshost ; then
         if wget -O $hosts "$mvpsurl" ; then
            ## Clean up and convert MVPS list to Unix format ##
            sed -i -e 's/[[:cntrl:]]$//' -e 's/[[:cntrl:]]$/ /' -e 's/[[:space:]]*#.*$//' -e '/^#/d' -e '/^$/d' $hosts
            logger List downloaded from "$mvpshost"... - Success -
         fi
       else
        return 1
      fi
      }
    
      downloadpgl ( )
      {
       if ping -c 1 $pglhost ; then
          if wget -O $pglfile "$pglurl" ; then
             logger List downloaded from "$pglhost"... - Success -
          fi
         else
           return 1
       fi
      }
    
       restartdnsmasq ( )
       {
         if ps | grep -q dnsmasq ; then
           service dnsmasq stop
           sleep 5
           service dnsmasq start
         fi
        }
    
    ## Run the Functions
        downloadmvps
        if [ $? != "0" ] ; then
           logger Downloading of List failed trying other list... - FAILURE -
           downloadpgl
        fi
        if [ $? == "0" ] ; then
            sleep 5
            restartdnsmasq
            else
            logger Downloading of pgl.yoyo.org List - FAILURE -
        fi
            
    EOF
    }
    
    #### Misc Functions To Be Run at Each Wan Up ####
     testupfile ( )
     {
      if [ ! -x $updtadlist ] ; then  
         createupfile
         if [ $? == "0" ] ; then
            chmod +x $updtadlist
            /tmp/update-adlist.sh
            cru a UpdateAdlist "<your interval here> $updtadlist"
         fi
      else
        logger update-adlist.sh already exists and is executable will not re-run
      fi
     }
    
      reportsize ( )
      {
      ## See how many lines and bytes pgl.conf contains and output values to logger
      if grep -q address $pglfile ; then
        numlines=$(awk 'END { print NR }' $pglfile)
        sleep 5    
        numbytes=$(ls -lg $pglfile | awk '{ sum += $5} END {print sum }')
        logger pgl.conf populated with $numlines lines and $numbytes bytes of data - Success -
      fi  
      if grep -q 127.0.0.1 $hosts ; then
        numlines=$(awk 'END { print NR }' $hosts)
        sleep 5    
        numbytes=$(ls -lg $hosts | awk '{ sum += $5} END {print sum }')
        logger /etc/hosts populated with $numlines lines and $numbytes bytes of data - Success -
      fi
      }
    
      testhostfile ( )
      {
      if grep -q 127.0.0.1 $hosts ; then
         seddhost=$(sed -n '15p' $hosts | cut -c 12-)
         mvpstest=$(ping -c 1 $seddhost | grep "from 127")
         if [ -n "$mvpstest" ] ; then
            logger Mvps testhost: $seddhost Unreachable - Ad blocking is working - Success -
            else
            logger Ad blocking may not be working ! Mvps Test - FAILURE -
         fi
      fi
      }
    
    testupfile
    sleep 5
    reportsize
    testhostfile
    Note: If you want to reverse the default downloading - do the following, just swap "downloadpgl" and "downloadmvps" under "## Run the Functions"

    Other Notes to self:
    ## This may be a better way to get rid of Dos formatting in mvps list... not sure, current sed seems to work.
    Code:
    #Download list into /tmp/tmplist
    wget -O /tmp/tmplist $mvpsurl
    
    #sed -i -e /remove lines that start with #/ -e /remove all blank lines/ -e /remove comments at end of lines/
    
    sed -i -e '/^#/d' -e '/^$/d' -e 's/[[:space:]]*#.*$//' /tmp/tmplist
    cut -d ' ' -f1,2,3 /tmp/tmplist > /etc/hosts
    That should also work I think because cut won't cut & copy the control characters, will it?
     
  12. MiseryQ

    MiseryQ Network Guru Member

    Surprisingly I understand very little of that. But it looks like it doesn't need the jffs partition. Good job.
    Does the init script:
    Code:
    sleep 3
    touch /etc/adblock.conf
    sleep 2
    logger adblock.conf created
    logger Remainder of init script starting
    change?

    Side Note: How does the two files compare? It looks like the mvps file is more comprehensive, blocking adware/spyware as well as ads.
     
  13. mraneri

    mraneri LI Guru Member

  14. MiseryQ

    MiseryQ Network Guru Member

    So now it looks like we're using "pgl.conf" and "mvps.tmp" to block the addresses.
    Are both entrys needed in the dnsmasq custom configuration?
     
  15. MiseryQ

    MiseryQ Network Guru Member

    Tried this out just now.

    In my logs it shows the "pgl.conf" and "mvps.tmp" files have been created.
    Farther in the log it says "WAN UP Script will execute after 45sec please wait....."

    But nothing more from the script. Again if I'm understanding correct I should have a "Success" or "Failure" in the log?
     
  16. yaqui

    yaqui LI Guru Member

    I assume you waited longer than 45 seconds and kept refreshing the log?? Yes, it should say something with all the logging I did in there.
     
  17. MiseryQ

    MiseryQ Network Guru Member

    Yeah I waited a little longer than 45 seconds (;
    I also tried running the script from ssh. Same.
     
  18. yaqui

    yaqui LI Guru Member

    Did you remember to do this:

    Go to Advanced > DHCP / DNS > and paste below into custom box, hit SAVE
    Code:
    conf-file=/etc/pgl.conf
     
  19. yaqui

    yaqui LI Guru Member

    There was a bug... I forgot a "}" Try to copy and paste again the main script.

    It should be working now... I have mine set to download MVPS list as default and pgl as backup here is my log:
    Code:
    Dec 31 18:00:15 lynxpark user.notice root: WAN UP Script will execute after 45sec please wait.....
    Aug  8 14:09:22 lynxpark user.info ntpc[167]: Time Updated: Wed, 08 Aug 2007 14:09:22 -0500 [+1186600147s]
    Aug  8 14:09:24 lynxpark daemon.info dnsmasq[105]: DHCPREQUEST(br0) 192.168.10.118 00:18:71:54:37:96 
    Aug  8 14:09:24 lynxpark daemon.info dnsmasq[105]: DHCPACK(br0) 192.168.10.118 00:18:71:54:37:96 
    Aug  8 14:10:08 lynxpark cron.warn crond[140]: time disparity of 19776669 minutes detected 
    Aug  8 14:10:19 lynxpark user.notice root: List downloaded from www.mvps.org... - Success -
    Aug  8 14:10:24 lynxpark daemon.info dnsmasq[105]: exiting on receipt of SIGTERM
    Aug  8 14:10:29 lynxpark daemon.info dnsmasq[238]: started, version 2.39 cachesize 150
    Aug  8 14:10:29 lynxpark daemon.info dnsmasq[238]: compile time options: no-IPv6 GNU-getopt no-RTC no-ISC-leasefile no-DBus no-I18N no-TFTP
    Aug  8 14:10:29 lynxpark daemon.info dnsmasq[238]: DHCP, IP range 192.168.10.100 -- 192.168.10.149, lease time 2d12h
    Aug  8 14:10:29 lynxpark daemon.info dnsmasq[238]: reading /etc/resolv.dnsmasq
    Aug  8 14:10:29 lynxpark daemon.info dnsmasq[238]: using nameserver blah
    Aug  8 14:10:29 lynxpark daemon.info dnsmasq[238]: using nameserver blah
    Aug  8 14:10:30 lynxpark daemon.info dnsmasq[238]: read /etc/hosts - 15363 addresses
    Aug  8 14:10:30 lynxpark daemon.info dnsmasq[238]: read /etc/hosts.dnsmasq - 1 addresses
    Aug  8 14:10:41 lynxpark user.notice root: /etc/hosts populated with 15363 lines and 472148 bytes of data - Success -
    Aug  8 14:10:42 lynxpark user.notice root: Mvps testhost: e.abnad.net Unreachable - Ad blocking is working - Success -
    
     
  20. MiseryQ

    MiseryQ Network Guru Member

    Code:
    Aug  7 22:32:34 tomato user.notice root: WAN UP Script will execute after 45sec please wait.....
    Aug  7 22:33:19 tomato cron.warn crond[93]: time disparity of 19775673 minutes detected 
    Aug  7 22:33:20 tomato user.notice root: List downloaded from pgl.yoyo.org... - Success -
    Aug  7 22:33:31 tomato user.notice root: pgl.conf populated with 1973 lines and 70343 bytes of data - Success -
    That did it. Thanks.

    I was thinking of making the mvps first after getting this sorted.
     
  21. mraneri

    mraneri LI Guru Member

    I'm all confused with what's going on here. But it really doesn't matter.
    You may want to consider, when MVPS is not available, to use the previous MVPS list (except when you don't have ANY list... i.e. after a reboot)

    Just a thought.
     
  22. yaqui

    yaqui LI Guru Member

    All the script is doing is downloading from one site if the other site is down. You can switch the two sites easily (see note at bottom of "MVPS as backup" post)

    If a scheduled update fails and no reboot occured... I guess it could just use the existing list(s). I could actually have it so it downloads both lists and just use one or the other if update fails.

    If update fails and there is a reboot; well that would probably be the case to have it backed up to jffs... or just ftp a backup copy from your computer.
     
  23. MiseryQ

    MiseryQ Network Guru Member

    Just an idea from the previous posts.

    Could the script backup the file to jffs before it tries updating from the site?
    Then if unsuccessful it can pull the file back from jffs.

    If possible, I'll assume it's not since in the previous script backing the file up to jffs had to be through "telnet", the router could recover from both failures.
     
  24. yaqui

    yaqui LI Guru Member

    Yes it could provided it already has a list file downloaded. The problem would be when you reboot, its gone... so you can't really backup if it's not there, you could check in script to see if it exists (if [ -x /file ]) first before trying to back up.

    On another note, I am wondering if it would be of any advantage to parse MVPS's list into the Dnsmasq (address=) format and use mvps list in dnsmasq??

    =========================================================
    Edit:
    Well here is Mvps list being parsed into dnsmasq format... I don't know if this helps with performance, but figured worth trying.

    REMEMBER TO CHANGE CRU STATEMENT TO YOUR OWN INTERVAL and CHANGE the Dnsmasq Custom config box to conf-file=/etc/adblock.conf

    Change init to this and hit SAVE:
    Code:
    sleep 3
    touch /etc/adblock.conf
    sleep 2
    logger adblock.conf created
    logger Remainder of init script starting
    
    Place under Wan Up script and hit SAVE:
    Code:
    #### MVPS or PGL as Backup ####
    #### This will parse Mvps's list into Dnsmaq format and output to /etc/adblock.conf - remember to change your init script! #####
    #### If for some reason one list is down this will try the other list instead - you can swap hosts under 'Run the Functions' ####
    logger WAN UP Script will execute after 45sec please wait.....
    sleep 45
    
    #### Define Variables ####
    adlist='/etc/adblock.conf'
    updtadlist='/tmp/update-adlist.sh'
    pglhost='pgl.yoyo.org'
    pglurl='http://pgl.yoyo.org/adservers/serverlist.php?hostformat=dnsmasq&showintro=0&mimetype=plaintext'
    mvpshost='www.mvps.org'
    mvpsurl='http://www.mvps.org/winhelp2002/hosts.txt'
    #hosts='/etc/hosts'
    messages='/var/log/messages'
    numlines=0
    numbytes=0
    
    #### Update Script Function ####
     createupfile ( )
     {
      cat > $updtadlist <<EOF
    #!/bin/sh
    #### Define Functions to Run in 'here document' ####
      downloadmvps ( )
      {
      if ping -c 1 $mvpshost ; then
         if wget -O $adlist "$mvpsurl" ; then
            logger List downloaded from "$mvpshost"... parsing and formatting please wait...
            ## Clean up and parse MVPS list to Dnsmasq format ##
            sed -i -e 's/[[:cntrl:]]$//' -e 's/[[:cntrl:]]$/ /' $adlist
            sed -i -e '/^#/d' $adlist
            sed -i -e '/^$/d' $adlist
            sed -i -e 's/[[:space:]]*#.*$//' $adlist
            sed -i -e "s|127.0.0.1  |address=/|" $adlist
            sed -i -e "s|$|/127.0.0.1|" $adlist
            sed -i -e '1d' $adlist
            logger Parsing Finished... hope it worked...
         fi
       else
        return 1
      fi
      }
    
      downloadpgl ( )
      {
       if ping -c 1 $pglhost ; then
          if wget -O $adlist "$pglurl" ; then
             logger List downloaded from "$pglhost"... - Success -
          fi
         else
           return 1
       fi
      }
    
       restartdnsmasq ( )
       {
         if ps | grep -q dnsmasq ; then
           service dnsmasq stop
           sleep 5
           service dnsmasq start
         fi
        }
    
    ## Run the Functions
        downloadmvps
        if [ $? != "0" ] ; then
           logger Downloading List failed -- trying other list... - FAILURE -
           downloadpgl
        fi
        if [ $? == "0" ] ; then
            sleep 5
            restartdnsmasq
            else
            logger Downloading of Both Lists - FAILURE -
        fi
            
    EOF
    }
    
    #### Misc Functions To Be Run at Each Wan Up ####
     testupfile ( )
     {
      if [ ! -x $updtadlist ] ; then  
         createupfile
         if [ $? == "0" ] ; then
            chmod +x $updtadlist
            /tmp/update-adlist.sh
            cru a UpdateAdlist "<your interval here> $updtadlist"
         fi
      else
        logger update-adlist.sh already exists and is executable will not re-run
      fi
     }
    
      reportsize ( )
      {
      ## See how many lines and bytes pgl.conf contains and output values to logger
      if grep -q address $adlist ; then
        numlines=$(awk 'END { print NR }' $adlist)
        sleep 5    
        numbytes=$(ls -lg $adlist | awk '{ sum += $5} END {print sum }')
        logger adblock.conf populated with $numlines lines and $numbytes bytes of data - Success -
      fi  
      }
    
      testhostfile ( )
      {
      if grep -q 127.0.0.1 $adlist ; then
         seddhost=$(sed -n '20p' $adlist | cut -d"/" -f2)
         testhost=$(ping -c 1 $seddhost | grep "from 127")
         if [ -n "$testhost" ] ; then
            logger testhost: $seddhost Unreachable - Ad blocking is working - Success -
            else
            logger Ad blocking may not be working ! Ad ping Test - FAILURE -
         fi
      fi
      }
    
    testupfile
    sleep 5
    reportsize
    testhostfile
    Script works... been tested.
     
  25. srouquette

    srouquette Network Guru Member

    I'm interested in these scripts, but I'd like to know what are the advantages over mraneri's shorter script ?
    what are the difference between fat and lean ? does it work the same way ?
     
  26. yaqui

    yaqui LI Guru Member

    Fat = More conditional tests and log messages

    Lean = Less conditional tests and less log messages

    Last Version = Formatting MVPS's list into dnsmasq format and importing into 'adblock.conf'

    At the core... they all basically do the same thing - block ads.
     
  27. srouquette

    srouquette Network Guru Member

    ok, thanks :)
     
  28. damwill

    damwill Network Guru Member

    yaqui, the only problem with the changing MVPS host list into a dnsmasq list is it will include quite a few duplicates (all the doubleclick entries could be just one statement). Not sure if it will create any speed issues.
     
  29. srouquette

    srouquette Network Guru Member

    yasui, I tried to redirect to 0.0.0.0 instead of 127.0.0.1 because I don't have a web server on my computer, and the response comes faster. you can test with http://0.0.0.0
    I replaced the 2nd 127.0.0.1 in the downloadmvps function to try it on the router.
    but I don't know sed, so I can't really update downloadpgl.
     
  30. mraneri

    mraneri LI Guru Member

    I agree. This is a great improvement. For some reason 0.0.0.0 comes back faster for me too. I've updated my script to use sed as shown above by yaqui to remove the comments at the end of the lines as well as replace 127.0.0.1's with 0.0.0.0's. You can find the update in about 5 minutes over on that thread, HERE.

    Thanks to yaqui for the sed syntax and sroquette for discovering the performance improvement. (Note, the improvement isn't on the router, it's on my windows machines, which for some reason delay 1/2 a second or so before returning on 127.0.0.1).

    Aren't these forums great!?!

    - Mike
     
  31. yaqui

    yaqui LI Guru Member

    Thanks for the above info on 0.0.0.0 !

    I've been working on sed and awk statements to remove a bunch of duplicates... kind of difficult to do because generalizing the statements is not working out so well. It seems like it will work better with a list of the main duplicates to be removed, which is not the ideal situation. :(

    Edit: So far (as far as generalized statements) I came up with 2 new sed lines and updated the other to 0.0.0.0, What they do is described in the comments. I figure with DNSmasq you don't need the www's and after removing them there was a bunch of *easily* removed duplicates.

    The other duplicates are proving more difficult because sometimes removing part of one line is not what you want on the next!! ie.. 'ads.yahoo.com' might get changed to 'yahoo.com' !

    Cut and paste this function in place of the previous if you want to use it:
    Code:
      downloadmvps ( )
      {
      if ping -c 1 $mvpshost ; then
         if wget -O $adlist "$mvpsurl" ; then
            logger List downloaded from "$mvpshost"... Removing some duplicates and formatting please wait...
            ## Clean up and format MVPS list to Dnsmasq format ##
            sed -i -e 's/[[:cntrl:]]$//' -e 's/[[:cntrl:]]$/ /' $adlist
            sed -i -e '/^#/d' $adlist
            sed -i -e '/^$/d' $adlist
            sed -i -e 's/[[:space:]]*#.*$//' $adlist
            sed -i -e 's/127.0.0.1  //' $adlist
            sed -i -e 's/^www[0-9].//' -e 's/^www.//' $adlist #New removes www<numbers> and www
            sed -i -e '$!N; /^\(.*\)\n\1$/!P; D' $adlist  #New removes dupes after taking out the www's
            sed -i -e 's|$|/0.0.0.0|' $adlist  #New faster response from 0.0.0.0
            sed -i -e 's|^|address=/|' $adlist
            logger Dnsmasq Formatting Finished... hope it worked...
         fi
       else
        return 1
      fi
      }
     
  32. srouquette

    srouquette Network Guru Member

    hehe sure :)
    but a problem could occur with the hosts file if you do a global replace.
    the first line of the mvps file is "127.0.0.1 localhost" which may be important. I don't know what kind of problem can happen if you replace this line.
    it's still working on windows, but I don't know if the router uses it.
     
  33. yaqui

    yaqui LI Guru Member

    It is fine with dnsmasq... in fact that line is totally deleted and everything is working fine.
     
  34. yaqui

    yaqui LI Guru Member

    It would be nice to have this list filtered somehow so there wouldn't be so many duplicates... If someone can figure out an easy way please post.
     
  35. srouquette

    srouquette Network Guru Member

    I forgot about this one, I used an old version, but this final one uses pgl.yoyo.org, which happens to work :)
     
  36. fun.k

    fun.k Addicted to LI Member

    almost there...

    hmm, i try to follow your suggestions here, and it looks like i'm stuck to smth (probably basic) as the script does not block the ads...

    this is the first part of my log:

    Code:
    Jan  1 00:00:16  user.notice kernel: JFFS2 version 2.1. (C) 2001 Red Hat, Inc., designed by Axis Communications AB.
    Jan  1 00:00:16  user.info kernel: vlan0: dev_set_promiscuity(master, 1)
    Jan  1 00:00:16  user.info kernel: device eth0 entered promiscuous mode
    Jan  1 00:00:16  user.info kernel: device vlan0 entered promiscuous mode
    Jan  1 00:00:16  user.info kernel: device eth1 entered promiscuous mode
    Jan  1 00:00:16  user.info kernel: br0: port 2(eth1) entering learning state
    Jan  1 00:00:16  user.info kernel: br0: port 1(vlan0) entering learning state
    Jan  1 00:00:16  user.warn kernel: vlan1: Setting MAC address to  00 12 17 d4 08 1d.
    Jan  1 00:00:16  user.info kernel: br0: port 2(eth1) entering forwarding state
    Jan  1 00:00:16  user.info kernel: br0: topology change detected, propagating
    Jan  1 00:00:16  user.info kernel: br0: port 1(vlan0) entering forwarding state
    Jan  1 00:00:17  user.info kernel: br0: topology change detected, propagating
    Jan  1 00:00:17  user.info kernel: er interface
    Jan  1 00:00:17  user.info init[1]: Tomato 1.19.1463
    Jan  1 00:00:17  user.info init[1]: Linksys WRT54G/GS/GL
    Jan  1 00:00:17  cron.notice crond[101]: crond 2.3.2 dillon, started, log level 9 
    Jan  1 00:00:17  daemon.info dnsmasq[103]: started, version 2.41 cachesize 2048
    Jan  1 00:00:17  daemon.info dnsmasq[103]: compile time options: no-IPv6 GNU-getopt no-RTC no-ISC-leasefile no-DBus no-I18N no-TFTP
    Jan  1 00:00:17  daemon.info dnsmasq[103]: asynchronous logging enabled, queue limit is 5 messages
    Jan  1 00:00:17  daemon.info dnsmasq[103]: DHCP, IP range 192.168.1.50 -- 192.168.1.55, lease time 1d
    Jan  1 00:00:17  daemon.warn dnsmasq[103]: failed to access /etc/resolv.dnsmasq: No such file or directory
    Jan  1 00:00:17  daemon.info dnsmasq[103]: read /etc/hosts - 0 addresses
    Jan  1 00:00:17  daemon.info dnsmasq[103]: read /etc/hosts.dnsmasq - 4 addresses
    Jan  1 00:00:18  daemon.info dnsmasq[103]: exiting on receipt of SIGTERM
    Jan  1 00:00:18  daemon.info dnsmasq[110]: started, version 2.41 cachesize 2048
    Jan  1 00:00:18  daemon.info dnsmasq[110]: compile time options: no-IPv6 GNU-getopt no-RTC no-ISC-leasefile no-DBus no-I18N no-TFTP
    Jan  1 00:00:18  daemon.info dnsmasq[110]: asynchronous logging enabled, queue limit is 5 messages
    Jan  1 00:00:18  daemon.info dnsmasq[110]: DHCP, IP range 192.168.1.50 -- 192.168.1.55, lease time 1d
    Jan  1 00:00:18  daemon.info dnsmasq[110]: reading /etc/resolv.dnsmasq
    Jan  1 00:00:18  daemon.info dnsmasq[110]: using nameserver 208.67.220.220#53
    Jan  1 00:00:18  daemon.info dnsmasq[110]: using nameserver 208.67.222.222#53
    Jan  1 00:00:18  daemon.info dnsmasq[110]: read /etc/hosts - 0 addresses
    Jan  1 00:00:18  daemon.info dnsmasq[110]: read /etc/hosts.dnsmasq - 4 addresses
    Jan  1 00:00:18  user.notice root: adblock.conf created
    Jan  1 00:00:18  user.notice root: Remainder of init script starting
    i don't know why it complains about the /etc/resolv.dnsmasq file, it's already there:

    Code:
    Trying 192.168.1.1...
    Connected to router.
    Escape character is '^]'.
     login: root
    Password: 
    
    
    Tomato v1.19.1463
    
    
    BusyBox v1.2.2 (2008.04.20-18:57+0000) Built-in shell (ash)
    Enter 'help' for a list of built-in commands.
    
    # cd /etc
    # ls
    TZ              hosts.dnsmasq   nas.conf        resolv.conf
    adblock.conf    iptables        openssl.cnf     resolv.dnsmasq
    dnsmasq.conf    l7-protocols    passwd          services
    group           ld.so.conf      profile         shadow
    gshadow         motd            protocols
    hosts           mtab            qos
    # 

    then the rest of my log, shows that the script does its thing successfully:

    Code:
    May  4 13:12:25  user.notice root: WAN UP Script will execute after 45sec please wait.....
    May  4 13:12:57  cron.warn crond[101]: time disparity of 20165052 minutes detected 
    May  4 13:13:15  user.notice root: List downloaded from www.mvps.org... Removing some duplicates and formatting please wait...
    May  4 13:13:39  user.notice root: Dnsmasq Formatting Finished... hope it worked...
    May  4 13:13:44  daemon.info dnsmasq[110]: exiting on receipt of SIGTERM
    May  4 13:13:49  daemon.info dnsmasq[424]: started, version 2.41 cachesize 2048
    May  4 13:13:49  daemon.info dnsmasq[424]: compile time options: no-IPv6 GNU-getopt no-RTC no-ISC-leasefile no-DBus no-I18N no-TFTP
    May  4 13:13:49  daemon.info dnsmasq[424]: asynchronous logging enabled, queue limit is 5 messages
    May  4 13:13:49  daemon.info dnsmasq[424]: DHCP, IP range 192.168.1.50 -- 192.168.1.55, lease time 1d
    May  4 13:13:49  daemon.info dnsmasq[424]: reading /etc/resolv.dnsmasq
    May  4 13:13:49  daemon.info dnsmasq[424]: using nameserver 208.67.220.220#53
    May  4 13:13:49  daemon.info dnsmasq[424]: using nameserver 208.67.222.222#53
    May  4 13:13:49  daemon.info dnsmasq[424]: read /etc/hosts - 0 addresses
    May  4 13:13:49  daemon.info dnsmasq[424]: read /etc/hosts.dnsmasq - 4 addresses
    May  4 13:14:02  user.notice root: adblock.conf populated with 15765 lines and 567899 bytes of data - Success -
    My Administration-->Scripts look like:

    Init:
    Code:
    sleep 3
    touch /etc/adblock.conf
    sleep 2
    logger adblock.conf created
    logger Remainder of init script starting
    WAN Up:
    Code:
    ### MVPS or PGL as Backup ####
    #### This will parse Mvps's list into Dnsmaq format and output to /etc/adblock.conf - remember to change your init script! #####
    #### If for some reason one list is down this will try the other list instead - you can swap hosts under 'Run the Functions' ####
    logger WAN UP Script will execute after 45sec please wait.....
    sleep 45
    
    #### Define Variables ####
    adlist='/etc/adblock.conf'
    updtadlist='/tmp/update-adlist.sh'
    pglhost='pgl.yoyo.org'
    pglurl='http://pgl.yoyo.org/adservers/serverlist.php?hostformat=dnsmasq&showintro=0&mimetype=plaintext'
    mvpshost='www.mvps.org'
    mvpsurl='http://www.mvps.org/winhelp2002/hosts.txt'
    #hosts='/etc/hosts'
    messages='/var/log/messages'
    numlines=0
    numbytes=0
    
    #### Update Script Function ####
     createupfile ( )
     {
      cat > $updtadlist <<EOF
    #!/bin/sh
    #### Define Functions to Run in 'here document' ####
      
    downloadmvps ( )
      {
      if ping -c 1 $mvpshost ; then
         if wget -O $adlist "$mvpsurl" ; then
            logger List downloaded from "$mvpshost"... Removing some duplicates and formatting please wait...
            ## Clean up and format MVPS list to Dnsmasq format ##
            sed -i -e 's/[[:cntrl:]]$//' -e 's/[[:cntrl:]]$/ /' $adlist
            sed -i -e '/^#/d' $adlist
            sed -i -e '/^$/d' $adlist
            sed -i -e 's/[[:space:]]*#.*$//' $adlist
            sed -i -e 's/127.0.0.1  //' $adlist
            sed -i -e 's/^www[0-9].//' -e 's/^www.//' $adlist #New removes www<numbers> and www
            sed -i -e '$!N; /^\(.*\)\n\1$/!P; D' $adlist  #New removes dupes after taking out the www's
            sed -i -e 's|$|/0.0.0.0|' $adlist  #New faster response from 0.0.0.0
            sed -i -e 's|^|address=/|' $adlist
            logger Dnsmasq Formatting Finished... hope it worked...
         fi
       else
        return 1
      fi
      }
    
      downloadpgl ( )
      {
       if ping -c 1 $pglhost ; then
          if wget -O $adlist "$pglurl" ; then
             logger List downloaded from "$pglhost"... - Success -
          fi
         else
           return 1
       fi
      }
    
       restartdnsmasq ( )
       {
         if ps | grep -q dnsmasq ; then
           service dnsmasq stop
           sleep 5
           service dnsmasq start
         fi
        }
    
    ## Run the Functions
        downloadmvps
        if [ $? != "0" ] ; then
           logger Downloading List failed -- trying other list... - FAILURE -
           downloadpgl
        fi
        if [ $? == "0" ] ; then
            sleep 5
            restartdnsmasq
            else
            logger Downloading of Both Lists - FAILURE -
        fi
            
    EOF
    }
    
    #### Misc Functions To Be Run at Each Wan Up ####
     testupfile ( )
     {
      if [ ! -x $updtadlist ] ; then  
         createupfile
         if [ $? == "0" ] ; then
            chmod +x $updtadlist
            /tmp/update-adlist.sh
            cru a UpdateAdlist "45 23 * * 5 $updtadlist"
         fi
      else
        logger update-adlist.sh already exists and is executable will not re-run
      fi
     }
    
      reportsize ( )
      {
      ## See how many lines and bytes pgl.conf contains and output values to logger
      if grep -q address $adlist ; then
        numlines=$(awk 'END { print NR }' $adlist)
        sleep 5    
        numbytes=$(ls -lg $adlist | awk '{ sum += $5} END {print sum }')
        logger adblock.conf populated with $numlines lines and $numbytes bytes of data - Success -
      fi  
      }
    
      testhostfile ( )
      {
      if grep -q 127.0.0.1 $adlist ; then
         seddhost=$(sed -n '20p' $adlist | cut -d"/" -f2)
         testhost=$(ping -c 1 $seddhost | grep "from 127")
         if [ -n "$testhost" ] ; then
            logger testhost: $seddhost Unreachable - Ad blocking is working - Success -
            else
            logger Ad blocking may not be working ! Ad ping Test - FAILURE -
         fi
      fi
      }
    
    testupfile
    sleep 5
    reportsize
    testhostfile


    I'm running 1.19 on a WRT-54GS v1 (32MB RAM / 8MB Flash), If someone can spot something wrong in my config, please let me know...



    [solved by using mike's script]
     

Share This Page