'''
spiderMan.py

Copyright 2006 Andres Riancho

This file is part of w3af, w3af.sourceforge.net .

w3af is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation version 2 of the License.

w3af is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with w3af; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA

'''

import time
import threading
import socket
import signal
import select
from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer

import core.controllers.outputManager as om
from core.controllers.basePlugin.baseDiscoveryPlugin import baseDiscoveryPlugin
import core.data.url.httpResponse as httpResponse

from core.data.request.httpPostDataRequest import httpPostDataRequest
from core.data.request.httpQsRequest import httpQsRequest

import core.data.parsers.urlParser as urlParser

from core.data.getResponseType import *
import cgi
from core.controllers.w3afException import w3afException
from core.controllers.w3afException import w3afRunOnce

LISTENADDRESS = '127.0.0.1'
LISTENPORT = 8998
URLOPENER = None
mutants = []
createFuzzableRequests = None

class spiderMan(baseDiscoveryPlugin):
	'''
	SpiderMan is a local proxy that will collect new URL's and POSTDATA.
	
	@author: Andres Riancho ( andres.riancho@gmail.com )  
	'''
	def __init__(self):
		self._run = True
		
	def discover(self, freq ):
		global mutants
		
		if not self._run:
			# This will remove the plugin from the discovery plugins to be runned.
			raise w3afRunOnce()
		else:
			self._run = False
			
			global URLOPENER
			global spiderManProxy
			global createFuzzableRequests
			
			createFuzzableRequests = self._createFuzzableRequests
			URLOPENER = self._urlOpener
			spiderManProxy = proxy()
			om.out.information('To exit spiderMan plugin please navigate to http://w3af/spiderMan?terminate or press ctrl+c .')
			spiderManProxy.run()
		
		return mutants

	def getOptionsXML(self):
		'''
		This method returns a XML containing the Options that the plugin has.
		Using this XML the framework will build a window, a menu, or some other input method to retrieve
		the info from the user. The XML has to validate against the xml schema file located at :
		w3af/core/ui/userInterface.dtd
		
		@return: XML with the plugin options.
		'''	
		return	'<?xml version="1.0" encoding="ISO-8859-1"?>\
		<OptionList>\
			<Option name="listenAddress">\
				<default>'+str(LISTENADDRESS)+'</default>\
				<desc>IP address that the spiderMan proxy will use to receive requests</desc>\
				<type>string</type>\
				<help></help>\
			</Option>\
			<Option name="listenPort">\
				<default>'+str(LISTENPORT)+'</default>\
				<desc>Port that the spiderMan proxy server will use to receive requests</desc>\
				<type>integer</type>\
				<help></help>\
			</Option>\
		</OptionList>\
		'

	def setOptions( self, optionsMap ):
		'''
		This method sets all the options that are configured using the user interface 
		generated by the framework using the result of getOptionsXML().
		
		@parameter OptionList: A dictionary with the options for the plugin.
		@return: No value is returned.
		'''
		global LISTENADDRESS
		global LISTENPORT
		
		LISTENADDRESS = optionsMap['listenAddress']
		LISTENPORT = optionsMap['listenPort']
		
	def getPluginDeps( self ):
		'''
		@return: A list with the names of the plugins that should be runned before the
		current one.
		'''
		return []				
	
	def getLongDesc( self ):
		'''
		@return: A DETAILED description of the plugin functions and features.
		'''
		return '''
		This plugin is a local proxy that can be used to give the framework knowledge about the web
		application when it has a lot of "client side code" like Flash or Java applets. Whenever a w3af needs to
		test an application with flash, he will see that w3af can't read that files, the solution is to use a web browser
		to navigate the site using spiderMan proxy.
		
		The proxy will extract information from the user navigation and generate the necesary injection points for the 
		audit plugins.
		
		Two configurable parameters exist:
			- listenAddress
			- listenPort
		'''
		
class myHTTPServer(HTTPServer):
	def handle_error(self, e ):
		# This overrides the default way that SocketServer.py handles exceptions
		# I dont want that "fancy" errors, just raise them all
		raise e
	
	def handle_request(self):
		"""Handle one request, possibly blocking."""
		try:
			request, client_address = self.get_request()
		except socket.error:
			return
		if self.verify_request(request, client_address):
			try:
				self.process_request(request, client_address)
			except Exception, e:
				# Here I changed the order of this two calls
				self.close_request(request)
				self.handle_error( e )
				
		
class proxy(threading.Thread):
#class proxy:
	'''
	This class is the Proxy.
	
	@author: Andres Riancho ( andres.riancho@gmail.com )	
	'''

	def __init__( self ):
		threading.Thread.__init__( self )
		self._proxy = None
		self._go = True
		self._running = False
		
	def stop(self):
		if self._running:
			self._proxy.server_close()
			self._go = False
			self._running = False
			

	def run(self):
		'''
		Starts the http server that will become a proxy.
		
		'''
		try:
			self._proxy = myHTTPServer((LISTENADDRESS, LISTENPORT ),  proxyHandler )
		except socket.error, e:
			raise w3afException('spiderMan error: ' + str( e[1] ) )
		else:
			message = 'spiderMan running on '+ LISTENADDRESS + ':'+ str(LISTENPORT) +' .'
			om.out.information( message )
		
		self._running = True
		while self._go:
			try:
				self._proxy.handle_request()
			except KeyboardInterrupt,e:
				om.out.information('spiderMan plugin has been ended by user request.')
				self.stop()
			except Exception, e:
				om.out.error('spiderMan error: '+ str(e) + ' calling self.stop().' )
				self.stop()
				
class proxyHandler(BaseHTTPRequestHandler):

	def __init__(self, request, client_address, server):
		self._version = 'spiderMan-w3af/1.0'
		BaseHTTPRequestHandler.__init__(self, request, client_address, server)
		
	def _connect( self, host, port ):
		'''
		Connects to remote host and keeps the connection alive to transfer data.
		'''
		sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		try:
			try:
				try:
					sock.connect((host, port))
				finally:
					signal.alarm(0)
			except TimeoutError:
				self.send_error(504, "Proxy timeout.")
				return
			self.wfile.write("%s %s %s\r\n" % (self.protocol_version, 200, "Connection established"))
			self.wfile.write("Proxy-agent: %s\r\n" % self._version)
			self.wfile.write("\r\n")
			self._read_write(sock, 300)
		finally:
			sock.close()
			self.connection.close()
	
	def _read_write(self, sock, max_idling=20):
		rfile = self.rfile
		if hasattr(rfile, '_rbuf'):
			data = rfile._rbuf
		else:
			if self.headers.has_key('Content-Length'):
				n = int(self.headers['Content-Length'])
				data = rfile.read(n)
			else:
				self.connection.setblocking(0)
				try: data = rfile.read()
				except IOError: data = ''
				self.connection.setblocking(1)
		rfile.close()
		if data:
			sock.send(data)
			
		iw = [self.connection, sock]
		count = 0
		self.headers_done = 0
		while 1:
			count += 1
			(ins, _, exs) = select.select(iw, [], iw, 3)
			if exs: break
			if ins:
				for i in ins:
					if i is sock:
						out = self.connection
					else:
						out = sock
					data = i.recv(8192)
					if data:
						out.send(data)
						count = 0
						
			if count == max_idling: break
		
	def _httpRequest( self, command, url, postData, headers ):
		'''
		Connects to remote host, sends data, receives response and closes connection to remote peer.
		'''
		functionReference = getattr( URLOPENER , command )
		try:
			response = functionReference( url, postData, headers )
		except KeyboardInterrupt,e:
			raise e
		except Exception,e:
			title = cgi.escape('Error when requesting: '+ url)
			desc = cgi.escape('Description: ' + str(e))
			om.out.error( title + '. ' +desc)
			html = '<html><title>'+title+'</title>spiderMan '+ title +'. <br/>' + desc + '</html>'
			headers = {'Content-Length': str(len(html))}
			response = httpResponse( 400, html, headers, url )
		return response

	def handle_one_request(self):
		'''
		Handle a single HTTP request.
		'''
		self.raw_requestline = self.rfile.readline()
		if not self.raw_requestline:
			self.close_connection = 1
			return
		if not self.parse_request(): 
			# An error code has been sent, just exit
			return

		words = self.raw_requestline.split('\n')[0].split()
		if len( words ) == 3:
			command, url, version = words
			
			# This is a fix
			if command.lower() == 'HEAD':
				command = 'GET'
				
			if url == 'http://w3af/spiderMan?terminate':
				self._sendEnd( version )
				spiderManProxy.stop()
				
			else:
				if command == 'CONNECT':
					host, port = url.split(':')
					port = int(port)
					self._connect( host, port )
					
				else:
					# Make this proxy as transparent as possible
					del self.headers['Proxy-Connection']
					del self.headers['keep-alive']
					self.headers['connection'] = 'close'
	
					# Copy the headers
					headers = {}
					for header in self.headers.keys():
						headers[header] = self.headers.getheader(header)
					
					postData = ''
					try:
						length = int(self.headers.getheader('content-length'))
					except:
						pass
					else:
						postData = self.rfile.read(length)
					
					global mutants
					global createFuzzableRequests
					response = self._httpRequest( command, url, postData, headers )
					freq = self._createFuzzFromRequest( command, url, postData, headers )
					mutants.append( freq )
					self._writeResponse( response, version )
					if isTextOrHtml( response.getHeaders() ):
						mutants.extend( createFuzzableRequests( response ) )

		else:
			# Client send some garbage, not the 3 words.
			return

	def _createFuzzFromRequest( self, command, url, postData, headers ):
		'''
		Creates a fuzzable request based on a query sent FROM the browser.
		'''
		res = None
		if len( postData ):
			pdr = httpPostDataRequest()
			pdr.setURL( url )
			pdr.setMethod( command )
			if 'content-length' in headers.keys():
				headers.pop('content-length')
			pdr.setHeaders( headers )
			
			if 'content-Type' in headers.keys() and headers['content-Type'] == 'multipart/form-data':
				dc = cgi.parse_multipart( postData, headers )
				for i in dc.keys():
					dc = dc[ i ][0]
				pdr.setDc( dc )
			else:
				dc = urlParser.getQueryString( 'http://a/?' + postData )
				pdr.setDc( dc )
		
			res = pdr
		
		else:
			# Just a query string request ! no postdata
			qsr = httpQsRequest()
			qsr.setURL( url )
			qsr.setMethod( command )
			qsr.setHeaders( headers )
			dc = urlParser.getQueryString( url )
			qsr.setDc( dc )			
			res = qsr
			
		return res
		
		
	def _sendEnd( self, version ):
		'''
		Sends an HTML indicating that w3af spiderMan plugin has finished its execution.
		'''
		html = '<html>spiderMan plugin finished its execution.</html>'
		headers = {'Content-Length': str(len(html))}
		r = httpResponse( 200, html, headers, 'http://w3af/spiderMan?terminate' )
		self._writeResponse( r , version )
		
	def _writeResponse( self, response, version ):
		'''
		Writes the response passed as param to the user:
		
		HTTP/1.1 200 OK
		Date: Tue, 05 Dec 2006 20:50:11 GMT
		Server: Apache/2.2.3 (Debian) mod_python/3.2.10 Python/2.4.4 PHP/5.2.0-7 mod_perl/2.0.2 Perl/v5.8.8
		Content-Length: 2459
		Connection: close
		Content-Type: text/html; charset=UTF-8
		
		HTML-GOES-HERE
		'''
		om.out.debug('spiderMan navigated to: ' + response.getURL() )
		#om.out.debug( 'HTML\n' + response.getHtml() )
		
		try:
			# big try, small coder :P
			self.wfile.write( version + ' ' + str(response.getCode()) + ' OK\r\n' )
			for header in response.getHeaders().keys():
				
				# remove this header! Transfer-Encoding: chunked
				if header.lower() != 'transfer-encoding':
					self.wfile.write(header + ': ' + response.getHeaders()[header] +'\r\n' )
			
			self.wfile.write( '\r\n' )
			self.wfile.write( response.getBody() )
		except Exception,e:
			om.out.error('Error while sending data to client. Error: '+ str(e) )
			self.close_connection = 1
			# This is commonly caused when the user click on the "stop" button
			pass