1.  #!/usr/bin/env python

  2.  

  3.  ############################################################################

  4.  #    Copyright (C) 2005 by Nikhil Marathe   #

  5.  #    nsm.nikhil@gmail.com   #

  6.  #                                                                          #

  7.  #    This program is free software; you can redistribute it and#or modify  #

  8.  #    it under the terms of the GNU General Public License as published by  #

  9.  #    the Free Software Foundation; either version 2 of the License, or     #

  10.  #    (at your option) any later version.                                   #

  11.  #                                                                          #

  12.  #    This program is distributed in the hope that it will be useful,       #

  13.  #    but WITHOUT ANY WARRANTY; without even the implied warranty of        #

  14.  #    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         #

  15.  #    GNU General Public License for more details.                          #

  16.  #                                                                          #

  17.  #    You should have received a copy of the GNU General Public License     #

  18.  #    along with this program; if not, write to the                         #

  19.  #    Free Software Foundation, Inc.,                                       #

  20.  #    59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             #

  21.  ############################################################################

  22.  

  23.  import sys

  24.  import getopt

  25.  import re

  26.  from urllib import urlopen

  27.  

  28.  class WebTangler(object):

  29.      urlList = []

  30.      

  31.      def appendToFile(self, st):

  32.          self.file.write(st+'\n')

  33.          

  34.      def __init__(self, depth, times, portal, output):

  35.              

  36.          #match all urls which have the keyword

  37.          self.reg_exp = re.compile(r'<a href=["\']?(http://?[a-zA-Z0-9\-\./]+)["\']?>')

  38.          self.file = open(output, 'wa')

  39.          self.urlList.append(portal)

  40.          while times:

  41.              self.crawl(depth)

  42.              times-=1

  43.          self.file.close()

  44.          

  45.      def crawl(self, depth):

  46.          if depth < 1:return

  47.          currLink = self.urlList.pop()

  48.          url_contents = []

  49.          #read in the file

  50.          for i in urlopen(currLink).readlines():url_contents.append(i)

  51.          #create a string

  52.          url_contents = ''.join(url_contents)

  53.          #replace all whitespace with spaces

  54.          url_contents.replace(' \n\t', ' ')

  55.          #print url_contents

  56.          for link in re.findall(self.reg_exp, url_contents):

  57.              self.urlList.append(link)

  58.              self.appendToFile(link)

  59.          self.crawl(depth-1)

  60.          

  61.  try:

  62.      switches, x = getopt.getopt(sys.argv[1:], None, ['depth=', 'times=', 'portal=', 'output='])

  63.      #not all args passed

  64.      if len(switches) < 4:

  65.          raise getopt.GetoptError, "Not enough args"

  66.      WebTangler(int(switches[0][1]), int(switches[1][1]), switches[2][1], switches[3][1])

  67.  except getopt.GetoptError:

  68.      print '''

  69.      webtangler.py -- a simple python webcrawler.

  70.      

  71.      Arguments:

  72.      --depth=n - depth to descend into one link.

  73.      --times=n - after how many times should WebTangler stop.

  74.      --portal=site - any link in complete format(eg. http://www.python.org/)

  75.      --output=file - file to output links to

  76.       

  77.      Have fun using it

  78.      '''