########################################################################
# $Header: /var/local/cvsroot/4Suite/Ft/Lib/DistExt/BuildDocs.py,v 1.49.2.3 2006/09/04 22:54:58 jkloth Exp $
"""
Main distutils extensions for generating documentation

Copyright 2005 Fourthought, Inc. (USA).
Detailed license and copyright information: http://4suite.org/COPYRIGHT
Project home, documentation, distributions: http://4suite.org/
"""

# NOTE: Before anybody gets wild ideas about changing urllib stuff to
# Ft.Lib.Uri, remember that this code is used *BEFORE* Ft is installed.
# It also is "safe" to urllib stuff as it will only be dealing with local
# files and its filename <-> url conversion works within itself.

import sys, os, copy, warnings, re, imp, rfc822, time, cStringIO
from distutils import util
from distutils.core import Command, DEBUG
from distutils.errors import *

from Ft import GetConfigVar
from Ft.Lib import Uri, ImportUtil
from Ft.Lib.DistExt import Structures
from Ft.Lib.DistExt.Formatters import *

__zipsafe__ = True

class BuildDocs(Command):

    command_name = 'build_docs'

    description = "build documentation files (copy or generate XML sources)"

    user_options = [
        ('build-dir=', 'd',
         "directory to \"build\" (generate) to"),
        ('force', 'f',
         "forcibly build everything (ignore file timestamps)"),
        ]

    boolean_options = ['inplace', 'force']

    def initialize_options(self):
        self.build_dir = None
        self.build_lib = None
        self.force = None

        # If 'inplace' is True, the generated documents are considered
        # to be source files as well.
        self.inplace = False

        # Set to true to validate the static XML documents
        self.validate = False
        return

    def finalize_options(self):
        self.set_undefined_options('build',
                                   ('build_docs', 'build_dir'),
                                   ('build_lib', 'build_lib'),
                                   ('force', 'force'))

        # Get the distribution options that are used for build_docs
        # options.
        self.files = [ doc for doc in self.distribution.doc_files
                       if isinstance(doc, Structures.File) ]

        self.static = [ doc for doc in self.distribution.doc_files
                        if isinstance(doc, Structures.Document) ]

        self.modules, self.module_info = self.get_modules()

        self.extensions = [ doc for doc in self.distribution.doc_files
                            if isinstance(doc, Structures.ExtensionsDocument) ]

        self.scripts = [ doc for doc in self.distribution.scripts
                         if isinstance(doc, Structures.Script)
                         and doc.application is not None ]

        # Initialize the properties that are not externally modifiable.
        self._xslt_processor = None
        return

    def get_outputs(self):
        outputs = []
        for name in self.modules:
            xmlfile = self.get_output_filename(name, 'modules')
            outputs.append(xmlfile)
        for ext in self.extensions:
            xmlfile = self.get_output_filename(ext.name, 'extensions')
            outputs.append(xmlfile)
        for script in self.scripts:
            xmlfile = self.get_output_filename(script.name, 'commandline')
            outputs.append(xmlfile)
        index_name = 'index-' + self.distribution.get_name()
        outputs.append(self.get_output_filename(index_name))
        return outputs

    def get_source_files(self):
        sources = []
        for doc in self.distribution.doc_files:
            if isinstance(doc, Structures.File):
                source = util.convert_path(doc.source)
                sources.append(source)
            elif isinstance(doc, Structures.Document):
                source = util.convert_path(doc.source)
                sources.append(source)
                prefix = len(os.getcwd()) + len(os.sep)
                for path in self.find_xml_includes(Uri.OsPathToUri(source)):
                    sources.append(path[prefix:])
        if self.inplace:
            sources.extend(self.get_outputs())
        return sources

    def find_xml_includes(self, uri, _includes=None):
        if _includes is None:
            _includes = {}
        def gather_includes(fullurl):
            if fullurl not in _includes:
                _includes[fullurl] = Uri.UriToOsPath(fullurl)
                self.find_xml_includes(fullurl, _includes)
            return
        ProcessIncludes(uri, gather_includes)
        return _includes.values()

    def run(self):
        if self.modules:
            self.prepare_modules()

        documents = []
        # Add the static XML content
        if self.static:
            documents.extend(self.build_static())

        # Create the XML content for the API reference
        if self.modules:
            documents.extend(self.build_api())

        # Create the XML content for XPath/XSLT extensions
        if self.extensions:
            documents.extend(self.build_extensions())

        # Create the XML content for command-line applications
        if self.scripts:
            documents.extend(self.build_commandline())

        # Create the XML content for the index
        self.build_index(documents)
        return

    def get_modules(self):
        # Locate all installed modules
        modules = []
        sources = {}
        if self.distribution.has_pure_modules():
            build_py = self.get_finalized_command('build_py')
            for package, module, filename in build_py.find_all_modules():
                if module == '__init__':
                    module = package
                    module_type = imp.PKG_DIRECTORY
                elif package:
                    module = package + '.' + module
                    module_type = imp.PY_SOURCE
                modules.append(module)
                package_dir = package.replace('.', os.sep)
                package_dir = os.path.join(build_py.build_lib, package_dir)
                filename = os.path.basename(filename)
                filename = os.path.join(package_dir, filename)
                sources[module] = (filename, module_type)
        if self.distribution.has_ext_modules():
            build_ext = self.get_finalized_command('build_ext')
            for ext in build_ext.extensions:
                module = build_ext.get_ext_fullname(ext.name)
                filename = build_ext.get_ext_filename(module)
                modules.append(module)
                filename = os.path.join(build_ext.build_lib, filename)
                sources[module] = (filename, imp.C_EXTENSION)
        return (modules, sources)

    # -- Top-level worker functions ------------------------------------
    # (called from 'run()')

    def prepare_modules(self):
        if self.distribution.has_pure_modules():
            self.run_command('build_py')
        if self.distribution.has_ext_modules():
            self.run_command('build_ext')

        if self.distribution.config_module:
            from Ft.Lib.DistExt.InstallConfig import METADATA_KEYS
            if self.distribution.config_module not in sys.modules:
                module = imp.new_module(self.distribution.config_module)
                sys.modules[self.distribution.config_module] = module
            else:
                module = sys.modules[self.distribution.config_module]
            for name in METADATA_KEYS:
                value = getattr(self.distribution, 'get_' + name)()
                setattr(module, name.upper(), value)

        # Add the build directory to the search path (sys.path).
        sys.path.insert(0, self.build_lib)

        # Enable importing of modules in namespace packages.
        for package in self.distribution.namespace_packages:
            packages = [package]
            while '.' in package:
                package = '.'.join(package.split('.')[:-1])
                packages.insert(0, package)
            for package in packages:
                path = os.path.join(self.build_lib, *package.split('.'))
                if package not in sys.modules:
                    module = sys.modules[package] = imp.new_module(package)
                    module.__path__ = [path]
                else:
                    module = sys.modules[package]
                    try:
                        search_path = module.__path__
                    except AttributeError:
                        raise DistutilsSetupError("namespace package '%s' is"
                                                  " not a package" % package)
                    search_path.insert(0, path)

        # Any packages that are already imported also need their
        # search path (__path__) adjusted.
        for name in self.modules:
            if name in sys.modules:
                search_path = getattr(sys.modules[name], '__path__', None)
                if search_path is not None:
                    path = os.path.join(self.build_lib, *name.split('.'))
                    search_path.insert(0, path)
        return

    def build_static(self):
        documents = []
        for document in self.static:
            # Building is as simple as creating a new Document instance to
            # reflect the converted paths.
            document = copy.copy(document)
            document.source = util.convert_path(document.source)
            documents.append(document)

        # Validate the content
        if self.validate:
            from xml.sax.handler import feature_validation
            from Ft.Xml.InputSource import DefaultFactory
            from Ft.Xml.Sax import CreateParser

            class ErrorHandler:
                def __init__(self, displayhook):
                    self.displayhook = displayhook
                def warning(self, exception):
                    self.displayhook(exception)
                def error(self, exception):
                    self.displayhook(exception)
                def fatalError(self, exception):
                    raise exception

            parser = CreateParser()
            parser.setFeature(feature_validation, True)
            parser.setErrorHandler(ErrorHandler(self.warn))

            for document in documents:
                self.announce('validating %s' % document.source, 2)
                parser.parse(DefaultFactory.fromUri(document.source))
        return documents

    def build_api(self):
        formatter = ApiFormatter.ApiFormatter(self, self.module_info)
        category = 'modules'

        # Find the top-level package to be documented
        first = min(self.modules)
        last = max(self.modules)
        shortest = min(len(first), len(last))
        for i in xrange(shortest):
            if first[i] != last[i]:
                top_level = first[:i]
                break
        else:
            top_level = first[:shortest]
        if not top_level:
            raise DistutilsInternalError(
                "documenting multiple top-level packages is not supported")
        warnings.filterwarnings('ignore', '', DeprecationWarning, top_level)

        documents = []
        for name in self.modules:
            try:
                module = __import__(name, {}, {}, ['*'])
            except ImportError, error:
                self.announce('not documenting %s (%s)' % (name, error), 3)
                continue

            # The build tree source is required for C-extension modules and
            # is safe for pure-Python modules.
            sources = [self.module_info[name][0]]
            xmlfile = self.document(category, name, sources, module, formatter)
            if name == top_level:
                # Only those documents with a title will be listed on the
                # index page
                title = '%s API Reference' % self.distribution.get_name()
                documents.append(Structures.Document(xmlfile,
                                                     stylesheet=category,
                                                     title=title,
                                                     category='general'))
        return documents

    def build_extensions(self):
        """
        Create XML documentation for XPath/XSLT extensions
        """
        formatter = ExtensionFormatter.ExtensionFormatter(self)
        category = 'extensions'

        extension_attrs = ('ExtNamespaces', 'ExtFunctions', 'ExtElements')

        docs = []
        for extension in self.extensions:
            # create a temporary module that will contain the combined
            # extension information
            extension_module = imp.new_module(extension.name)
            for attr in extension_attrs:
                setattr(extension_module, attr, {})
            sources = []
            for name in extension.modules:
                try:
                    module = __import__(name, {}, {}, extension_attrs)
                except ImportError, e:
                    raise DistutilsFileError(
                        "could not import '%s': %s" % (name, e))
                for attr in extension_attrs:
                    if hasattr(module, attr):
                        attrs = getattr(module, attr)
                        getattr(extension_module, attr).update(attrs)
                sources.append(self.module_info[name][0])
            xmlfile = self.document(category, extension.name, sources,
                                    extension_module, formatter)
            docs.append(Structures.Document(xmlfile, stylesheet=category,
                                            title=extension.title,
                                            category=category))
        return docs

    def build_commandline(self):
        formatter = CommandLineFormatter.CommandLineFormatter(self)
        category = 'commandline'

        docs = []
        for script in self.scripts:
            try:
                module = __import__(script.module, {}, {}, [script.application])
            except ImportError, e:
                raise DistutilsFileError(
                    "could not document '%s' script: %s" % (script.name, e))

            # Get the CommandLineApp instance for documenting
            app = getattr(module, script.application)()
            # Get the sources that are used to implement the application
            sources = [self.module_info[script.module][0]]
            for cmd_name, cmd in app.get_help_doc_info():
                source = cmd._fileName
                if source is None:
                    module_name = cmd.__class__.__module__
                    source = self.module_info[module_name][0]
                sources.append(source)
            # Now document the application and its commands, if any.
            xmlfile = self.document(category, script.name, sources, app,
                                    formatter)
            title = script.name + ' - ' +  app.description
            docs.append(Structures.Document(xmlfile, stylesheet=category,
                                            title=title, category=category))
        return docs

    def build_index(self, documents):
        from Ft.Xml.Xslt.BuiltInExtElements import RESERVED_NAMESPACE

        name = 'index-' + self.distribution.get_name()
        xmlfile = self.get_output_filename(name)
        source_mtime = max(os.path.getmtime(self.distribution.script_name),
                           os.path.getmtime(self.distribution.package_file),
                           ImportUtil.GetLastModified(__name__))
        try:
            target_mtime = os.path.getmtime(xmlfile)
        except OSError:
            target_mtime = -1
        if not (self.force or source_mtime > target_mtime):
            self.announce('not creating index (up-to-date)', 1)
            return
        else:
            self.announce("creating index -> %s" % xmlfile, 2)

        index = {}
        index_uri = Uri.OsPathToUri(xmlfile)
        xmlstr = XmlFormatter.XmlRepr().escape
        for doc in documents:
            if 'noindex' not in doc.flags:
                output = os.path.splitext(doc.source)[0] + '.html'
                source_uri = Uri.OsPathToUri(doc.source)
                output_uri = Uri.OsPathToUri(output)
                category = index.setdefault(doc.category, [])
                category.append({
                    'title' : xmlstr(doc.title),
                    'source' : Uri.Relativize(source_uri, index_uri),
                    'output' : Uri.Relativize(output_uri, index_uri),
                    'stylesheet' : xmlstr(doc.stylesheet),
                    })

        sections = []
        for title, category, sort in (
            ('General', 'general', False),
            ('Modules', 'modules', True),
            ('XPath/XSLT Extensions', 'extensions', False),
            ('Command-line Applications', 'commandline', True)
            ):
            if category not in index:
                continue
            items = []
            L = index[category]
            if sort:
                L.sort(lambda a, b: cmp(a['title'], b['title']))
            for info in L:
                repl = {'title' : info['title'],
                        'url' : info['output'],
                        }
                items.append(INDEX_LISTITEM % repl)
            if items:
                # add the section if it contains any entries
                items = ''.join(items)
                repl = {'title' : xmlstr(title),
                        'category' : xmlstr(category),
                        'items' : items,
                        }
                sections.append(INDEX_SECTION % repl)
        sections = ''.join(sections)

        sources = []
        for category in index.values():
            for info in category:
                sources.append(INDEX_SOURCE % info)
        sources = ''.join(sources)

        repl = {'fullname' : xmlstr(self.distribution.get_fullname()),
                'sections' : sections,
                'namespace' : RESERVED_NAMESPACE,
                'sources' : sources,
                }
        index = INDEX_TEMPLATE % repl

        if not self.dry_run:
            f = open(xmlfile, 'wb')
            f.write(index)
            f.close()

        return documents

    def document(self, category, name, sources, object, formatter):
        xmlfile = self.get_output_filename(name, category)
        self.mkpath(os.path.dirname(xmlfile))

        # The dependencies for 'object' are the source for the formatter
        # and, of course, 'sources'.
        formatter_module = formatter.__class__.__module__
        source_mtime = max(ImportUtil.GetLastModified(formatter_module),
                           *map(os.path.getmtime, sources))
        try:
            target_mtime = os.path.getmtime(xmlfile)
        except OSError:
            target_mtime = -1
        if self.force or source_mtime > target_mtime:
            self.announce("documenting %s -> %s" % (name, xmlfile), 2)
            if not self.dry_run:
                try:
                    stream = open(xmlfile, 'w')
                    try:
                        formatter.format(object, stream, encoding='iso-8859-1')
                    finally:
                        stream.close()
                except (KeyboardInterrupt, SystemExit):
                    os.remove(xmlfile)
                    raise
                except Exception, exc:
                    os.remove(xmlfile)
                    if DEBUG: raise
                    raise DistutilsExecError("could not document %s (%s)" %
                                             (name, exc))
        else:
            self.announce('not documenting %s (up-to-date)' % name, 1)
        return xmlfile

    def get_output_filename(self, name, category=None):
        dest_dir = self.build_dir
        if category:
            dest_dir = os.path.join(dest_dir, category)
        return os.path.join(dest_dir, name + '.xml')


def FindIncludes(source_uri, _includes=None):
    if _includes is None:
        _includes = {}
    def gather_includes(fullurl):
        if fullurl not in _includes:
            _includes[fullurl] = True
            FindIncludes(fullurl, _includes)
        return
    ProcessIncludes(source, gather_includes)
    return _includes


def ProcessIncludes(source, callback, xslt=False):
    from xml.sax import make_parser, SAXException, SAXNotRecognizedException
    from xml.sax.handler import ContentHandler, feature_namespaces, \
         feature_validation, feature_external_ges, feature_external_pes
    from xml.sax.xmlreader import InputSource

    # defined as nested to keep things "import clean"
    class InclusionFilter(ContentHandler):

        XSLT_INCLUDES = [
            ("http://www.w3.org/1999/XSL/Transform", "import"),
            ("http://www.w3.org/1999/XSL/Transform", "include"),
            ]

        def startDocument(self):
            url = self._locator.getSystemId()
            self._bases = [url]
            self._scheme = Uri.GetScheme(url)
            self._elements = [
                ("http://www.w3.org/2001/XInclude", "include"),
                ]
            if xslt:
                self._elements.extend(self.XSLT_INCLUDES)
        def startElementNS(self, expandedName, tagName, attrs):
            # Update xml:base stack
            xml_base = ("http://www.w3.org/XML/1998/namespace", "base")
            baseUri = attrs.get(xml_base, self._bases[-1])
            self._bases.append(baseUri)

            if expandedName in self._elements:
                try:
                    href = attrs[(None, 'href')]
                except KeyError:
                    # XInclude same document reference, nothing to do
                    return

                # Ignore XInclude's with parse='text'
                if attrs.get((None, 'parse'), 'xml') == 'text':
                    return

                # Only follow inclusions that have the same scheme as the
                # initial document.
                fullurl = Uri.BaseJoin(baseUri, href)
                if Uri.GetScheme(fullurl) == self._scheme:
                    callback(fullurl)
        def endElementNS(self, expandedName, tagName):
            del self._bases[-1]
    # -- end InclusionFilter --

    try:
        parser = make_parser()
        parser.setFeature(feature_namespaces, True)
        # Attempt to disable all external entity resolution
        for feature in (feature_validation, feature_external_ges,
                        feature_external_pes):
            try:
                parser.setFeature(feature, False)
            except SAXNotRecognizedException:
                pass
    except SAXException, e:
        raise DistutilsModuleError(e.getMessage())

    handler = InclusionFilter()
    parser.setContentHandler(handler)

    if isinstance(source, (str, unicode)):
        try:
            stream = Uri.UrlOpen(source)
        except OSError:
            # Assume part of an XInclude w/fallback.
            return
        source = InputSource(source)
        source.setByteStream(stream)
    elif hasattr(source, 'read'):
        stream = source
        source = InputSource(getattr(stream, 'name', None))
        source.setByteStream(stream)
    parser.parse(source)
    return


INDEX_TEMPLATE = """<?xml version="1.0" encoding="ISO-8859-1"?>
<!DOCTYPE article PUBLIC "-//OASIS//DTD Simplified DocBook XML V1.1//EN"
          "http://docbook.org/xml/simple/1.1/sdocbook.dtd">
<?ftdb-ignore-namespace http://xmlns.4suite.org/reserved?>
<article>
  <title>%(fullname)s Document Index</title>
%(sections)s

  <f:sources xmlns:f="%(namespace)s">
%(sources)s
  </f:sources>

</article>
"""

INDEX_SECTION = """
  <section id="%(category)s">
    <title>%(title)s</title>
    <itemizedlist>
%(items)s
    </itemizedlist>
  </section> <!-- %(category)s -->
"""

INDEX_LISTITEM = """\
      <listitem>
        <ulink url="%(url)s" type="generate">%(title)s</ulink>
      </listitem>
"""

INDEX_SOURCE = """\
    <f:source>
      <f:title>%(title)s</f:title>
      <f:src>%(source)s</f:src>
      <f:dst>%(output)s</f:dst>
      <f:stylesheet>%(stylesheet)s</f:stylesheet>
    </f:source>
"""