''' googleSets.py Copyright 2006 Andres Riancho This file is part of w3af, w3af.sourceforge.net . w3af is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation version 2 of the License. w3af is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with w3af; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA ''' import core.controllers.outputManager as om from core.controllers.w3afException import w3afException from core.data.searchEngines.google import google as google from core.controllers.basePlugin.baseDiscoveryPlugin import baseDiscoveryPlugin import core.data.parsers.urlParser as urlParser from core.data.fuzzer.fuzzer import * import core.data.kb.knowledgeBase as kb class googleSets(baseDiscoveryPlugin): ''' This plugin searches google to get a set of words based on an URL. @author: Andres Riancho ( andres.riancho@gmail.com ) ''' def __init__(self): baseDiscoveryPlugin.__init__(self) # Internal variables self._analyzed = [] # User variables self._minInput = 2 self._setResults = 2 def discover(self, fuzzableRequest ): ''' @parameter fuzzableRequest: A fuzzableRequest instance that contains (among other things) the URL to test. ''' self._fuzzableRequests = [] self.is404 = kb.kb.getData( 'error404page', '404' ) self._google = google( self._urlOpener ) # No key is used here self._originalResponse = self._sendMutant( fuzzableRequest, analyze=False ) for analyzedVariable, inputSet in self._generateInputSet( fuzzableRequest ): if ( fuzzableRequest.getURL(), analyzedVariable, inputSet ) not in self._analyzed: self._analyzed.append( ( fuzzableRequest.getURL(), analyzedVariable, inputSet ) ) targs = ( fuzzableRequest, analyzedVariable, inputSet) self._tm.startFunction( target=self._verify, args=targs, ownerObj=self ) self._tm.join( self ) return self._fuzzableRequests def _verify( self, fuzzableRequest, analyzedVariable, inputSet): resultSet = self._google.set( inputSet ) # Now i will cut the list using the user setting resultSet = resultSet[:self._setResults] mutantList = self._generateURLsFromSet( analyzedVariable, resultSet, fuzzableRequest ) for m in mutantList: response = self._sendMutant( m, analyze=False ) if not self.is404( response ) and self._originalResponse.getBody() != response.getBody() : fuzzReqs = self._createFuzzableRequests( response ) self._fuzzableRequests.extend( fuzzReqs ) def _generateInputSet( self, fuzzableRequest ): ''' Based on the fuzzable request, i'll search the kb , try to find other URLs in the same path and create a set with the filenames of those URLs. Also analyze the parameters that are passed in the query string. @return: A list of tuples with ( analyzedVariable, resultSet ). When analyzing the URL, the analyzedVariable is setted to None. ''' result = [] result.extend( self._generateISFname( fuzzableRequest ) ) result.extend( self._generateISQs( fuzzableRequest ) ) return result def _generateISQs( self, fuzzableRequest ): ''' Check the URL query string. @return: A list of tuples with ( analyzedVariable, resultSet ). ''' # The result result = [] qs = urlParser.getQueryString( fuzzableRequest.getURI() ) inputSetMap = {} for key in qs.keys(): inputSetMap[ key ] = [] uriList = kb.kb.getData( 'urls', 'uriList' ) for uri in uriList: if fuzzableRequest.getURL() == urlParser.uri2url( uri ) and \ urlParser.getQueryString( uri ).keys() == qs.keys(): # Both URL's have the same query string parameters for key in qs.keys(): inputSetMap[ key ].append( urlParser.getQueryString( uri )[ key ] ) # Now I create the result, based on inputSetMap for key in inputSetMap: result.append( ( key, list(set(inputSetMap[ key ] ) ) ) ) return result def _generateISFname( self, fuzzableRequest ): ''' Check the URL filenames @return: A list of tuples with ( analyzedVariable, resultSet ). When analyzing the URL, the analyzedVariable is setted to None. ''' url = fuzzableRequest.getURL() fname = self._getFilename( url ) dp = urlParser.getDomainPath( url ) # The result result = [] otherNamesInPath = [] urlList = kb.kb.getData( 'urls', 'urlList' ) for url in urlList: if urlParser.getDomainPath( url ) == dp: # Both files are in the same path otherNamesInPath.append( self._getFilename( url ) ) if len(otherNamesInPath) >= self._minInput: result.append( ( None, otherNamesInPath ) ) return result def _getFilename( self, url ): ''' @return: The filename, without the extension ''' fname = urlParser.getFileName( url ) splittedFname = fname.split('.') name = '' if len(splittedFname) != 0: name = splittedFname[0] return name def _generateURLsFromSet( self, analyzedVariable, resultSet, fuzzableRequest ): ''' Based on the result, create the new URLs to test. @return: An URL list. ''' if analyzedVariable == None: # The URL was analyzed url = fuzzableRequest.getURL() fname = urlParser.getFileName( url ) dp = urlParser.getDomainPath( url ) # The result result = [] splittedFname = fname.split('.') if len(splittedFname) == 2: name = splittedFname[0] extension = splittedFname[1] else: name = splittedFname[0] extension = 'html' for setItem in resultSet: newFname = url.replace( name, setItem ) frCopy = fuzzableRequest.copy() frCopy.setURL( urlParser.urlJoin( dp, newFname ) ) result.append( frCopy ) return result else: mutants = createMutants( fuzzableRequest , resultSet, fuzzableParamList=[analyzedVariable,] ) return mutants def getOptionsXML(self): ''' This method returns a XML containing the Options that the plugin has. Using this XML the framework will build a window, a menu, or some other input method to retrieve the info from the user. The XML has to validate against the xml schema file located at : w3af/core/ui/userInterface.dtd @return: XML with the plugin options. ''' return '\ \ \ \ \ ' def setOptions( self, optionsMap ): ''' This method sets all the options that are configured using the user interface generated by the framework using the result of getOptionsXML(). @parameter OptionList: A dictionary with the options for the plugin. @return: No value is returned. ''' self._minInput = optionsMap['resultLimit'] self._setResults = optionsMap['setResults'] def getPluginDeps( self ): ''' @return: A list with the names of the plugins that should be runned before the current one. ''' return ['discovery.error404page'] def getLongDesc( self ): ''' @return: A DETAILED description of the plugin functions and features. ''' return ''' This plugin finds new URL's using google sets. Two configurable parameters exist: - resultLimit - setResults An example is the best way to explain what this plugin does, let's suppose that the input for this plugin is: - http://a/index.asp?color=blue - http://a/index.asp?color=red The plugin will search google sets for a set that contains the words "blue" and "red", and the result will be: "black" and "white". So the plugin requests this two URL's: - http://a/index.asp?color=black - http://a/index.asp?color=white If the response for those URL's is not a 404 error, and has not the same body content, then we have found a new URI. '''