########################################################################
# $Header: /var/local/cvsroot/4Suite/Ft/Lib/DistExt/BuildDocs.py,v 1.33 2005/04/07 20:05:51 jkloth Exp $
"""
Main distutils extensions for generating documentation

Copyright 2005 Fourthought, Inc. (USA).
Detailed license and copyright information: http://4suite.org/COPYRIGHT
Project home, documentation, distributions: http://4suite.org/
"""

import sys, os, imp, cStringIO, tempfile
from distutils.core import Command
from distutils.dep_util import newer_group
from distutils.file_util import copy_file
from distutils.errors import DistutilsFileError
from distutils.util import convert_path

import Structures

class BuildDocs(Command):

    command_name = 'build_docs'

    description = "build documentation files"

    user_options = [
        ('build-dir=', 'd', "directory to \"build\" (generate) to"),
        ('force', 'f',
         "forcibly build everything (ignore file timestamps)"),
        ]

    boolean_options = ['force']

    def initialize_options(self):
        self.build_dir = None
        self.build_lib = None
        self.debug = None
        self.force = None
        # Not user selectable, needed for bdist_* commands
        self.xml_dir = None
        self.html_dir = None
        self.text_dir = None
        self.outfiles = []
        return

    def finalize_options(self):
        self.set_undefined_options('build',
                                   ('build_docs', 'build_dir'),
                                   ('build_lib', 'build_lib'),
                                   ('debug','debug'),
                                   ('force', 'force'))

        self.xml_dir = os.path.join(self.build_dir, 'xml')
        self.html_dir = os.path.join(self.build_dir, 'html')
        self.text_dir = os.path.join(self.build_dir, 'text')
        self.docs = self.distribution.doc_files or []
        return

    def get_outputs(self):
        if not self.distribution.have_run.get(self.command_name):
            self.dry_run = 1
            self.run()
            del self.dry_run
        return self.outfiles

    def get_source_files(self):
        # Stylesheets are assumed to included elsewhere
        docs = filter(lambda x: isinstance(x, Structures.Document), self.docs)
        filenames = []
        for document in docs:
            filenames.append(document.source)
        return filenames

    def _get_modules(self, has_any, build_cmd, cmd_option):

        if not has_any:
            return []

        build_cmd = self.get_finalized_command(build_cmd)
        build_files = build_cmd.get_outputs()
        build_dir = getattr(build_cmd, cmd_option)

        prefix_len = len(build_dir) + len(os.sep)
        return [ fname[prefix_len:] for fname in build_files ]

    def run(self):
        # Most all of the work is done in an external process to allow
        # importing of all modules

        if os.name == 'nt' and self.debug:
            # On Windows, the logic for documenting ext modules needs
            # to learn about '_d' suffix on debug builds
            return

        # Generated XML content (API and XSLT extensions)
        modules = []
        if self.distribution.has_pure_modules():
            self.run_command('build_py')
            build_py = self.get_finalized_command('build_py')
            modules.extend(build_py.get_outputs())

        if self.distribution.has_ext_modules():
            self.run_command('build_ext')
            build_ext = self.get_finalized_command('build_ext')
            modules.extend(build_ext.get_outputs())

        if self.distribution.has_l10n():
            self.run_command('build_l10n')

        # The commandline applications
        scripts = self.distribution.scripts or []
        scripts = [ x for x in scripts if isinstance(x, Structures.Script) ]

        # Handwritten XML
        docs = filter(lambda x: isinstance(x, Structures.Document), self.docs)

        # Create a script to do the documenting in a sub-process
        script_name = tempfile.mktemp('.py')
        outputs_name = tempfile.mktemp('.out')
        script = open(script_name, 'w')

        script.write('import sys\n')
        script.write('sys.path.insert(0, %r)\n' % self.build_lib)

        script.write('from distutils.dist import Distribution\n')
        script.write('from Ft.Lib.DistExt.Structures import *\n')
        script.write('from %s import ExternalBuildDocs\n' % (__name__))

        script.write('modules = [%s]\n' % ',\n'.join(map(repr, modules)))
        script.write('scripts = [%s]\n' % ',\n'.join(map(repr, scripts)))
        script.write('docs = [%s]\n' % ',\n'.join(map(repr, docs)))

        script.write('attrs = %r\n' % vars(self.distribution.metadata))
        script.write('dist = Distribution(attrs)\n')
        script.write('cmd = ExternalBuildDocs(dist, modules, scripts, docs, '
                     '%r, %r, %r, %r, %r, %r, %r, %r)\n' % (self.build_lib,
                                                            self.build_dir,
                                                            self.xml_dir,
                                                            self.html_dir,
                                                            self.text_dir,
                                                            self.force,
                                                            self.verbose,
                                                            self.dry_run))
        script.write('cmd.run()\n')

        script.write('file = open(%r, "w")\n' % outputs_name)
        script.write('file.write(repr(cmd.get_outputs()))\n')
        script.write('file.close()\n')
        script.close()

        try:
            os.spawnv(os.P_WAIT, sys.executable, [sys.executable, script_name])

            # Get the list of generated files
            try:
                outputs = open(outputs_name).read()
            except:
                self.warn('failure during documenting')
            else:
                self.outfiles.extend(eval(outputs))
        finally:
            #try: os.remove(script_name)
            #except: pass
            #try: os.remove(outputs_name)
            #except: pass
            pass

        return

# -- externally executed -----------------------------------------------------

import pydoc, inspect, imp

class ExternalBuildDocs(Command):

    command_name = 'build_docs'

    def __init__(self, dist, modules, scripts, docs, build_lib, build_dir,
                 xml_dir, html_dir, text_dir, force, verbose, dry_run):
        self.distribution = dist
        self.modules = modules
        self.scripts = scripts
        self.docs = docs
        self.build_lib = build_lib
        self.build_dir = build_dir
        self.xml_dir = xml_dir
        self.html_dir = html_dir
        self.text_dir = text_dir
        self.force = force
        self.verbose = verbose
        try:
            from distutils import log
        except ImportError:
            pass
        else:
            log.set_verbosity(self.verbose)
        self.dry_run = dry_run
        self.outfiles = []
        return

    def get_outputs(self):
        return self.outfiles

    def run(self):
        index = {
            'modules' : [],
            'extensions' : [],
            'commandline' : [],
            }

        # Handwritten XML content (packages add to this list)
        for doc in self.docs:
            files = {}
            source = convert_path(doc.source)
            outdir = doc.outdir and convert_path(doc.outdir)

            files['source'] = source

            files['xml'] = os.path.join(self.xml_dir, outdir, os.path.basename(source))

            files['name'] = doc.title or os.path.basename(source)
            if doc.htmlOutfile is not None:
                files['html'] = os.path.join(self.html_dir, outdir,convert_path(doc.htmlOutfile))
            if doc.textOutfile is not None:
                files['text'] = os.path.join(self.text_dir, outdir,convert_path(doc.textOutfile))

            if not index.has_key(doc.category):
                index[doc.category] = []
            index[doc.category].append(files)

            self.outfiles.append(files['xml'])
            self.mkpath(os.path.dirname(files['xml']))
            self.copy_file(files['source'],
                           files['xml'])


        # Generated XML content (API and XSLT extensions)
        if self.modules:
            common = os.path.commonprefix(self.modules)
            if common == self.build_lib:
                # Multiple top-level packages and/or modules installed
                installed = os.listdir(self.build_lib)
            else:
                # A single top-level package/module
                top = common[len(self.build_lib)+len(os.sep):]
                installed = os.path.split(top)[:1]

            top_level = []
            for file in installed:
                path = os.path.join(self.build_lib, file)
                if os.path.isdir(path):
                    # A package
                    for ext in ['.py', '.pyc', '.pyo']:
                        module = os.path.join(path, '__init__' + ext)
                        if os.path.isfile(module):
                            if module in self.modules:
                                top_level.append(module)
                                break
                elif path in self.modules:
                    # A module
                    top_level.append(path)
            self.doc_modules(index, top_level)

        # Generated XML content for command-line applications
        if self.scripts:
            self.doc_scripts(index)

        if self.docs:
            try:
                from Ft.Xml.Xslt import Processor
                Processor.Processor()
            except:
                self.warn('skipping rendering (unable to get a processor)')
            else:
                for doc in self.docs:
                    self.render_doc(doc)

        self.build_xml_index(index)
        self.build_html_index(index)
        return

    def locate(self, path, name=None):
        try:
            module = __import__(path, {}, {}, ['*'])
        except ImportError, e:
            self.announce('not documenting %s (%s)' % (name or path, e), 2)
            module = None

        return module


    def document(self, category, name, sources, object, formatter):
        # Add the module for the formatter to the sources list
        sources.append(inspect.getmodule(formatter.__class__).__file__)

        xmlfile = os.path.join(self.xml_dir, category, name + '.xml')
        htmlfile = os.path.join(self.html_dir, category, name + '.html')
        textfile = os.path.join(self.text_dir, category, name + '.txt')

        self.outfiles.append(xmlfile)
        self.mkpath(os.path.dirname(xmlfile))
        if self.force or newer_group(sources, xmlfile):
            self.announce("documenting %s -> %s" % (name, xmlfile), 2)
            if not self.dry_run:
                xml = formatter.document(object, encoding='iso-8859-1')
                open(xmlfile, 'wb').write(xml)
        else:
            self.announce('not documenting %s (up-to-date)' % name, 1)

        # Add the XSLT generated versions to the document list
        # Even if the XML hasn't changed, the stylesheets might have.
        self.docs.append(Structures.Document(xmlfile,
                                             category,
                                             ['Ft/Data/%s_html.xslt' % category],
                                             name + '.html',
                                             #['Ft/Data/%s_text.xslt' % category],
                                             #name + '.txt',
                                             category = category
                                             ))

        return xmlfile, htmlfile, textfile

    def add_index_info(self, index, category, name, (xml, html, text)):
        info = {'name' : name,
                 'xml' : xml,
                'html' : html,
                'text' : text,
                }
        index[category].append(info)
        return

    def doc_modules(self, index, top_level):
        sources = {}
        prefix_len = len(self.build_lib) + len(os.sep)
        for file in self.modules:
            module = os.path.splitext(file[prefix_len:])[0]
            if os.path.basename(module) == '__init__':
                module = os.path.dirname(module)
            module = module.replace(os.sep, '.')
            sources[module] = file

        from Formatters import ApiFormatter
        apidoc = ApiFormatter.ApiFormatter(self, sources)

        category = 'modules'

        modules = sources.keys()
        modules.sort()

        extensions = []

        thirdparty = '!'

        for name in modules:
            # Only document our code (no third-party inclusions)
            if name.endswith('.ThirdParty'):
                thirdparty = name + '.'
            elif not name.startswith(thirdparty):
                module = self.locate(name)

                if module is None:
                    continue

                source = sources[name]

                if hasattr(module, 'ExtFunctions') or hasattr(module, 'ExtElements'):
                    # An XPath/XSLT extension module (documented later)
                    extensions.append((name, source, module))

                files = self.document(category, name, [source], module, apidoc)

                if source in top_level:
                    # Add this to the list of files for the index page
                    self.add_index_info(index, category, name, files)

        if extensions:
            self.doc_extensions(extensions, index)

        return

    def doc_extensions(self, extensions, index):
        """
        Create XML documentation for XPath/XSLT extensions
        """
        from Formatters import ExtensionFormatter
        extdoc = ExtensionFormatter.ExtensionFormatter(self)

        category = 'extensions'

        # Remove modules whose package also defines extensions
        pkgs = filter(lambda (n, s, m): hasattr(m, '__path__'), extensions)
        ignored = []
        for name, source, module in pkgs:
            package_dir = os.path.dirname(source)
            package_file = os.path.basename(source)
            for file in os.listdir(package_dir):
                if file != package_file:
                    ignored.append(os.path.join(package_dir, file))

        for name, source, module in extensions:
            if source not in ignored:
                files = self.document(category, name, [source], module, extdoc)
                self.add_index_info(index, category, name, files)
        return

    def doc_scripts(self, index):
        from Formatters import CommandLineFormatter
        cmdlinedoc = CommandLineFormatter.CommandLineFormatter(self)

        category = 'commandline'

        for script in self.scripts:
            name = script.name
            module = self.locate(getattr(script, 'module', None), name)
            if module is None:
                continue

            app = getattr(module, script.identifier)()

            sources = [module.__file__]
            for cmd_name, cmd in app.get_help_doc_info():
                classmod = __import__(cmd.__class__.__module__, {}, {}, ['*'])
                sources.append(classmod.__file__)

            files = self.document(category, name, sources, app, cmdlinedoc)

            # Add this to the list of files for the index page
            self.add_index_info(index, category, name, files)

        return

    def render_doc(self, doc):
        source = convert_path(doc.source)
        outdir = doc.outdir and convert_path(doc.outdir)

        if doc.htmlOutfile is not None:
            outfile = os.path.join(self.html_dir, outdir,convert_path(doc.htmlOutfile))
            self.render_one(outfile,
                        os.path.join(self.xml_dir,outdir,os.path.basename(source)),
                        doc.htmlStylesheets,
                        doc.params)
            self.outfiles.append(outfile)
        if doc.textOutfile:
            outfile = os.path.join(self.text_dir, outdir,convert_path(doc.textOutfile))
            self.render_one(outfile,
                        os.path.join(self.xml_dir,outdir,os.path.basename(source)),
                        doc.textStylesheets,
                        doc.params)
            self.outfiles.append(outfile)


    def render_one(self,outfile,source,stylesheets,params):
        from Ft.Xml.Xslt import Processor
        from Ft.Xml import InputSource
        from Ft.Lib import Uri

        self.mkpath(os.path.dirname(outfile))

        if not self.force:
            sources = [source]
            for file in stylesheets:
                file = convert_path(file)
                if os.path.exists(file):
                    sources.append(file)
            if not newer_group(sources, outfile):
                self.announce('not rendering %s (up-to-date)' % outfile, 1)
                return

        self.announce("rendering %s -> %s" % (source, outfile), 2)
        if not self.dry_run:
            # Add the stylesheets
            processor = Processor.Processor()
            ignorePis = 1
            for file in stylesheets:
                file = convert_path(file)
                if os.path.exists(file):
                    #url = 'file:' + urllib.pathname2url(file)
                    url = Uri.OsPathToUri(file, attemptAbsolute=True)
                    try:
                        iSrc = InputSource.DefaultFactory.fromUri(url)
                        processor.appendStylesheet(iSrc)
                    except Exception, exc:
                        raise DistutilsFileError, \
                              "could not use stylesheet '%s': %s" % (file, exc)
                else:
                    ignorePis = 0

            # Process the source
            stream = open(outfile, 'w')
            #url = 'file:' + urllib.pathname2url(source)
            url = Uri.OsPathToUri(source, attemptAbsolute=True)

            params = {'version' : self.distribution.get_version()}
            params.update(params)
            try:
                iSrc = InputSource.DefaultFactory.fromUri(url)
                processor.run(iSrc, ignorePis, params, outputStream=stream)
            except Exception, exc:
                raise DistutilsFileError, \
                      "could not render '%s': %s" % (outfile, exc)
            stream.write('\n')
            stream.close()



    def build_html_index(self, index):
        htmlfile = os.path.join(self.html_dir, 'index.html')
        self.outfiles.append(htmlfile)
        self.announce("creating HTML index -> %s" % (htmlfile), 2)

        prefix_len = len(self.build_dir)
        for section, infos in index.items():
            for info in infos:
                for key, value in info.items():
                    if value.startswith(self.build_dir):
                        info[key] = '..' + value[prefix_len:]
                        if key in ['html','xml','text'] and os.sep != '/':
                            info[key] = info[key].replace(os.sep, '/')


        if not self.dry_run:
            f = open(htmlfile, 'wb')
            title = '%s Documentation Index' % self.distribution.metadata.get_fullname()
            f.write('<html>\n')
            f.write('<head><title>%s</title></head>\n' % title)
            f.write('<body>\n<h1>%s</h1>\n' % title)
            f.write('<div>%s</div>\n' % PREAMBLE)

            def section(title, list_, write=f.write):
                if not list_:
                    return
                list_.sort(lambda a, b: cmp(a['name'], b['name']))
                write('<h2>%s</h2>\n' % title)
                write('<table width="100%">\n')
                rows = (len(list_)+3)/4
                for row in range(rows):
                    write('  <tr>\n')
                    for i in range(row, rows*4, rows):
                        write('    <td width="25%" valign="top">')
                        if i < len(list_):
                            write('<a href="%(html)s">%(name)s</a>' % list_[i])
                        else:
                            write('&nbsp;')
                        write('</td>\n')
                    write('  </tr>\n')
                write('</table>\n')
                return

            section('General',
                    filter(lambda x: x.has_key('html'), index['general']))
            section('Modules',
                    filter(lambda x: x.has_key('html'), index['modules']))
            section('XPath/XSLT Extensions',
                    filter(lambda x: x.has_key('html'), index['extensions']))
            section('Command-line Applications',
                    filter(lambda x: x.has_key('html'), index['commandline']))
            f.write('</body>\n</html>\n')
            f.close()
        return

    def build_xml_index(self, index):
        from Ft.Lib import Uri
        xmlfile = os.path.join(self.xml_dir, 'index.doc')
        self.outfiles.append(xmlfile)
        self.announce("creating XML index -> %s" % (xmlfile), 2)

        prefix_len = len(self.xml_dir)
        for section, infos in index.items():
            for info in infos:
                for key, value in info.items():
                    if key in ['xml'] and value.startswith(self.xml_dir):
                        info[key] = value[prefix_len+1:]


        if not self.dry_run:
            f = open(xmlfile, 'wb')
            title = '%s Document Index' % self.distribution.metadata.get_fullname()
            f.write("""<?xml version="1.0"?>
<article
  xmlns="http://docbook.org/docbook/xml/4.0/namespace"
  xmlns:dc= "http://purl.org/dc/elements/1.1/"
  xmlns:common="http://namespaces.fourthought.com/doc-common"
  xmlns:f = "http://xmlns.4suite.org/ext"
  version="1.0"
>
  <articleinfo>
    <title>%s</title>
    <author>Generated by build_docs</author>
  </articleinfo>""" % title)

            def section(title, list, write=f.write):
                if not list:
                    return
                list.sort(lambda a, b: cmp(a['name'], b['name']))
                write('<section>\n  <title>%s</title>\n' % title)
                write('  <glosslist>\n')
                for item in list:
                    name = item['name']
                    xmlospath = item['xml']
                    #xmluripath = urllib.pathname2url(xmlospath)
                    xmluri = Uri.OsPathToUri(xmlospath, attemptAbsolute=False)
                    xmluripath = Uri.SplitUriRef(xmluri)[2]
                    write('    <glossentry><glossterm>%s</glossterm><glossdef>%s</glossdef></glossentry>\n' % (name, xmluripath))
                write("  </glosslist>\n")
                write('</section>\n')
                return

            section('General',
                    filter(lambda x: x.has_key('html'), index['general']))
            section('Modules',
                    filter(lambda x: x.has_key('html'), index['modules']))
            section('XPath/XSLT Extensions',
                    filter(lambda x: x.has_key('html'), index['extensions']))
            section('Command-line Applications',
                    filter(lambda x: x.has_key('html'), index['commandline']))
            f.write('</article>\n')
            f.close()
        return

PREAMBLE = """
<p>Copyright 2001-2005 Fourthought, Inc., USA.  Please read the <a href='COPYRIGHT.html'>COPYRIGHT</a> file for terms of license.</p>
<p>For release notes and news, see the <a href='http://4suite.org/'>4Suite home page</a> (http://4suite.org/).</p>
<p>For complete installation instructions, see the following documents listed in the "General" section below: Installing 4Suite on UNIX/Windows, and 4Suite CVS access.</p>
<p>If you have questions not answered by the documents, first stop is the <a href='http://4suite.org/docs/FAQ.xml'>FAQ</a>.  Next try the Akara pages on <a href='http://uche.ogbuji.net/tech/akara/pyxml/'>Python/XML</a> and <a href='http://uche.ogbuji.net/tech/akara/4suite/'>4Suite</a>.  Then check the <a href='http://lists.fourthought.com/pipermail/4suite/'>mailing archives</a> of the 4Suite list or post there.  Or drop by on IRC (#4suite on <a href="http://freenode.net/irc_servers.shtml">irc.freenode.net</a>).</p>
<p>People interested in participating in or closely following the development process of 4Suite can join the <a href='http://lists.fourthought.com/mailman/listinfo/4suite-dev'>mailing list for core 4Suite developers</a>, which is open (as are its <a href='http://lists.fourthought.com/pipermail/4suite-dev/'>archives</a>).</p>
<p>These links and others are also provided on <a href='http://uche.ogbuji.net/tech/4Suite/'>Uche Ogbuji's 4Suite page</a>.</a></p>
<hr>
"""