Manifest documentation generation

There is currently two type of documentation generation
for manifests (module or modulepath):

 * RDoc HTML generation for modules and global manifests
 * console output for sole manifest

Both version handles classes, defines, nodes, global
variable assignements, and resources when --all is used.

The usage is the following:

For the rdoc variant:
$ puppetdoc --mode rdoc --outputdir doc
It uses the puppet.conf configuration file to get the modulepath
and manifestdir settings. Those are overridable on the
command line with --modulepath and --manifestdir.

For the console output version:
$ puppetdoc /path/to/manifests

Signed-off-by: Brice Figureau <brice-puppet@daysofwonder.com>

bin/puppetdoc

@@ -8,25 +8,36 @@
 #
 # = Usage
 #
-#   puppetdoc [-a|--all] [-h|--help] [-m|--mode <text|pdf|trac> [-r|--reference <[type]|configuration|..>]
+#   puppetdoc [-a|--all] [-h|--help] [-o|--outputdir <rdoc outputdir>] [-m|--mode <text|pdf|trac|rdoc>] 
+#             [-r|--reference <[type]|configuration|..>] [manifest-file]
 #
 # = Description
 #
-# This command generates a restructured-text document describing all installed
+# If mode is not 'rdoc', then this command generates a restructured-text document describing all installed
 # Puppet types or all allowable arguments to puppet executables.  It is largely
 # meant for internal use and is used to generate the reference document
 # available on the Reductive Labs web site.
 #
+# In 'rdoc' mode, this command generates an html RDoc hierarchy describing the manifests that
+# are in 'manifestdir' and 'modulepath' configuration directives. 
+# The generated documentation directory is doc by default but can be changed with the 'outputdir' option.
+#
+# If the command is started with 'manifest-file' command-line arguments, puppetdoc generate a single
+# manifest documentation that is output on stdout.
+#
 # = Options
 #
 # all::
-#   Output the docs for all of the reference types.
+#   Output the docs for all of the reference types. In 'rdoc' modes, this also outputs documentation for all resources
 #
 # help::
 #   Print this help message
 #
+# outputdir::
+#   Specifies the directory where to output the rdoc documentation in 'rdoc' mode.
+#
 # mode::
-#   Determine the output mode.  Valid modes are 'text', 'trac', and 'pdf'.  Note that 'trac' mode only works on Reductive Labs servers.  The default mode is 'text'.
+#   Determine the output mode.  Valid modes are 'text', 'trac', 'pdf' and 'rdoc'.  Note that 'trac' mode only works on Reductive Labs servers.  The default mode is 'text'.  In 'rdoc' mode you must provide 'manifests-path'
 #
 # reference::
 #   Build a particular reference.  Get a list of references by running +puppetdoc --list+.
@@ -34,6 +45,10 @@
 # = Example
 #
 #   $ puppetdoc -r type > /tmp/type_reference.rst
+# or
+#   $ puppetdoc --outputdir /tmp/rdoc --mode rdoc /path/to/manifests
+# or
+#   $ puppetdoc /etc/puppet/manifests/site.pp
 #
 # = Author
 #
@@ -47,16 +62,24 @@
 require 'puppet'
 require 'puppet/util/reference'
 require 'puppet/network/handler'
+require 'puppet/util/rdoc'
 require 'getoptlong'
 
-result = GetoptLong.new(
+options = [
-	[ "--all",	"-a",			GetoptLong::NO_ARGUMENT ],
+    [ "--all",          "-a",   GetoptLong::NO_ARGUMENT ],
-	[ "--list",	"-l",			GetoptLong::NO_ARGUMENT ],
+    [ "--list",         "-l",   GetoptLong::NO_ARGUMENT ],
-	[ "--format",	"-f",			GetoptLong::REQUIRED_ARGUMENT ],
+    [ "--format",       "-f",   GetoptLong::REQUIRED_ARGUMENT ],
-	[ "--mode",	"-m",			GetoptLong::REQUIRED_ARGUMENT ],
+    [ "--mode",         "-m",   GetoptLong::REQUIRED_ARGUMENT ],
-	[ "--reference",	"-r",			GetoptLong::REQUIRED_ARGUMENT ],
+    [ "--reference",    "-r",   GetoptLong::REQUIRED_ARGUMENT ],
-	[ "--help",		"-h",			GetoptLong::NO_ARGUMENT ]
+    [ "--help",         "-h",   GetoptLong::NO_ARGUMENT ],
-)
+    [ "--outputdir",    "-o",   GetoptLong::REQUIRED_ARGUMENT ],
+    [ "--verbose",      "-v",   GetoptLong::NO_ARGUMENT ],
+    [ "--debug",        "-d",   GetoptLong::NO_ARGUMENT ]
+]
+
+# Add all of the config parameters as valid options.
+Puppet.settings.addargs(options)
+result = GetoptLong.new(*options)
 
 debug = false
 
@@ -66,8 +89,11 @@ options = {:references => [], :mode => :text, :format => :to_rest}
 Reference = Puppet::Util::Reference
 
 begin
+    unknown_args = []
     result.each { |opt,arg|
         case opt
+        when "--outputdir"
+            options[:outputdir] = arg
         when "--all"
             options[:all] = true
         when "--format"
@@ -78,7 +104,7 @@ begin
                 raise "Invalid output format %s" % arg
             end
         when "--mode"
-            if Reference.modes.include?(arg)
+            if Reference.modes.include?(arg) or arg.intern==:rdoc
                 options[:mode] = arg.intern
             else
                 raise "Invalid output mode %s" % arg
@@ -88,6 +114,10 @@ begin
             exit(0)
         when "--reference"
             options[:references] << arg.intern
+        when "--verbose"
+            options[:verbose] = true
+        when "--debug"
+            options[:debug] = true
         when "--help"
             if Puppet.features.usage?
                 RDoc::usage && exit
@@ -95,14 +125,52 @@ begin
                 puts "No help available unless you have RDoc::usage installed"
                 exit
             end
+        else
+            unknown_args << {:opt => opt, :arg => arg }
         end
     }
+
+    # sole manifest documentation
+    if ARGV.size > 0
+        options[:mode] = :rdoc
+        manifest = true
+    end
+
+    # consume the remaining unknown options
+    # and feed them as settings, but only for rdoc mode
+    if options[:mode] == :rdoc and unknown_args.size > 0
+        unknown_args.each do |option|
+            # force absolute path for modulepath when passed on commandline
+            if option[:opt]=="--modulepath" or option[:opt] == "--manifestdir"
+                option[:arg] = option[:arg].split(':').collect { |p| File.expand_path(p) }.join(':')
+            end
+            Puppet.settings.handlearg(option[:opt], option[:arg])
+        end
+    end
 rescue GetoptLong::InvalidOption => detail
     $stderr.puts "Try '#{$0} --help'"
     exit(1)
 end
 
-if options[:all]
+if options[:mode] == :rdoc # rdoc mode
+    # hack to get access to puppetmasterd modulepath and manifestdir
+    Puppet[:name] = "puppetmasterd"
+    # Now parse the config
+    Puppet.parse_config
+
+    # Handle the logging settings.
+    if options[:debug] or options[:verbose]
+        if options[:debug]
+            Puppet::Util::Log.level = :debug
+        else
+            Puppet::Util::Log.level = :info
+        end
+
+        Puppet::Util::Log.newdestination(:console)
+    end
+end
+
+if options[:all] and options[:mode] != :rdoc
     # Don't add dynamic references to the "all" list.
     options[:references] = Reference.references.reject do |ref|
         Reference.reference(ref).dynamic?
@@ -114,6 +182,33 @@ if options[:references].empty?
 end
 
 case options[:mode]
+when :rdoc # rdoc or sole manifest mode
+    exit_code = 0
+    files = []
+    unless manifest
+        files += Puppet[:modulepath].split(':').collect { |p| File.expand_path(p) }
+        files += Puppet[:manifestdir].split(':').collect { |p| File.expand_path(p) }
+    end
+    files += ARGV
+    Puppet.info "scanning: %s" % files.inspect
+    Puppet.settings.setdefaults("puppetdoc",
+        "document_all" => [false, "Document all resources"]
+    )
+    Puppet.settings[:document_all] = options[:all] || false
+    begin
+        if manifest
+            Puppet::Util::RDoc.manifestdoc(files)
+        else
+            Puppet::Util::RDoc.rdoc(options[:outputdir], files)
+        end
+    rescue => detail
+        if Puppet[:trace]
+            puts detail.backtrace
+        end
+        $stderr.puts "Could not generate documentation: %s" % detail
+        exit_code = 1
+    end
+    exit exit_code
 when :trac
     options[:references].each do |name|
         section = Puppet::Util::Reference.reference(name) or raise "Could not find section %s" % name
@@ -159,4 +254,3 @@ else
     exit exit_code
 end
 
-

lib/puppet/util/rdoc.rb

@@ -0,0 +1,85 @@
+
+module Puppet::Util::RDoc
+
+    module_function
+
+    # launch a rdoc documenation process
+    # with the files/dir passed in +files+
+    def rdoc(outputdir, files)
+        begin
+            Puppet[:ignoreimport] = true
+
+            # then rdoc
+            require 'rdoc/rdoc'
+
+            # load our parser
+            require 'puppet/util/rdoc/parser'
+
+            r = RDoc::RDoc.new
+            RDoc::RDoc::GENERATORS["puppet"] = RDoc::RDoc::Generator.new("puppet/util/rdoc/generators/puppet_generator.rb",
+                                                                       "PuppetGenerator".intern,
+                                                                       "puppet")
+            # specify our own format & where to output
+            options = [ "--fmt", "puppet",
+                        "--quiet",
+                        "--op", outputdir ]
+
+            options += files
+
+            # launch the documentation process
+            r.document(options)
+        rescue RDoc::RDocError => e
+            raise Puppet::ParseError.new("RDoc error %s" % e)
+        end
+    end
+
+    # launch a output to console manifest doc
+    def manifestdoc(files)
+        Puppet[:ignoreimport] = true
+        files.select { |f| FileTest.file?(f) }.each do |f|
+            parser = Puppet::Parser::Parser.new(:environment => Puppet[:environment])
+            parser.file = f
+            ast = parser.parse
+            output(f, ast)
+        end
+    end
+
+    # Ouputs to the console the documentation
+    # of a manifest
+    def output(file, ast)
+        astobj = []
+        ast[:nodes].each do |name, k|
+            astobj << k if k.file == file
+        end
+        ast[:classes].each do |name, k|
+            astobj << k if k.file == file
+        end
+        ast[:definitions].each do |name, k|
+            astobj << k if k.file == file
+        end
+        astobj.sort! {|a,b| a.line <=> b.line }.each do |k|
+            output_astnode_doc(k)
+        end
+    end
+
+    def output_astnode_doc(ast)
+        puts ast.doc if !ast.doc.nil? and !ast.doc.empty?
+        if Puppet.settings[:document_all]
+            # scan each underlying resources to produce documentation
+            code = ast.code.children if ast.code.is_a?(Puppet::Parser::AST::ASTArray)
+            code ||= ast.code
+            output_resource_doc(code) unless code.nil?
+        end
+    end
+
+    def output_resource_doc(code)
+        code.sort { |a,b| a.line <=> b.line }.each do |stmt|
+            output_resource_doc(stmt.children) if stmt.is_a?(Puppet::Parser::AST::ASTArray)
+
+            if stmt.is_a?(Puppet::Parser::AST::Resource)
+                puts stmt.doc if !stmt.doc.nil? and !stmt.doc.empty?
+            end
+        end
+    end
+
+end

lib/puppet/util/rdoc/code_objects.rb

@@ -0,0 +1,219 @@
+require 'rdoc/code_objects'
+
+module RDoc
+
+    # This modules contains various class that are used to hold information
+    # about the various Puppet language structures we found while parsing.
+    # 
+    # Those will be mapped to their html counterparts which are defined in
+    # PuppetGenerator.
+
+    # PuppetTopLevel is a top level (usually a .pp/.rb file)
+    class PuppetTopLevel < TopLevel
+        attr_accessor :module_name, :global
+
+        # will contain all plugins
+        @@all_plugins = {}
+
+        # contains all cutoms facts
+        @@all_facts = {}
+
+        def initialize(toplevel)
+            super(toplevel.file_relative_name)
+        end
+
+        def self.all_plugins
+            @@all_plugins.values
+        end
+
+        def self.all_facts
+            @@all_facts.values
+        end
+    end
+
+    # PuppetModule holds a Puppet Module
+    # This is mapped to an HTMLPuppetModule
+    # it leverage the RDoc (ruby) module infrastructure
+    class PuppetModule < NormalModule
+        attr_accessor :facts, :plugins
+
+        def initialize(name,superclass=nil)
+            @facts = []
+            @plugins = []
+            super(name,superclass)
+        end
+
+        def initialize_classes_and_modules
+            super
+            @nodes = {}
+        end
+
+        def add_plugin(plugin)
+            add_to(@plugins, plugin)
+        end
+
+        def add_fact(fact)
+            add_to(@facts, fact)
+        end
+
+        def add_node(name,superclass)
+            cls = @nodes[name]
+            unless cls
+                cls = PuppetNode.new(name, superclass)
+                @nodes[name] = cls if !@done_documenting
+                cls.parent = self
+                cls.section = @current_section
+            end
+            cls
+        end
+
+        def each_fact
+            @facts.each {|c| yield c}
+        end
+
+        def each_plugin
+            @plugins.each {|c| yield c}
+        end
+
+        def each_node
+            @nodes.each {|c| yield c}
+        end
+
+        def nodes
+            @nodes.values
+        end
+    end
+
+    # PuppetClass holds a puppet class
+    # It is mapped to a HTMLPuppetClass for display
+    # It leverages RDoc (ruby) Class
+    class PuppetClass < ClassModule
+        attr_accessor :resource_list
+
+        def initialize(name, superclass)
+            super(name,superclass)
+            @resource_list = []
+        end
+
+        def add_resource(resource)
+            add_to(@resource_list, resource)
+        end
+
+        def is_module?
+            false
+        end
+    end
+
+    # PuppetNode holds a puppet node
+    # It is mapped to a HTMLPuppetNode for display
+    # A node is just a variation of a class
+    class PuppetNode < PuppetClass
+        def initialize(name, superclass)
+            super(name,superclass)
+        end
+
+        def is_module?
+            false
+        end
+    end
+
+    # Plugin holds a native puppet plugin (function,type...)
+    # It is mapped to a HTMLPuppetPlugin for display
+    class Plugin < Context
+        attr_accessor :name, :type
+
+        def initialize(name, type)
+            super()
+            @name = name
+            @type = type
+            @comment = ""
+        end
+
+        def <=>(other)
+            @name <=> other.name
+        end
+
+        def full_name
+            @name
+        end
+
+        def http_url(prefix)
+            path = full_name.split("::")
+            File.join(prefix, *path) + ".html"
+        end
+
+        def is_fact?
+            false
+        end
+
+        def to_s
+            res = self.class.name + ": " + @name + " (" + @type + ")\n"
+            res << @comment.to_s
+            res
+        end
+    end
+
+    # Fact holds a custom fact
+    # It is mapped to a HTMLPuppetPlugin for display
+    class Fact < Context
+        attr_accessor :name, :confine
+
+        def initialize(name, confine)
+            super()
+            @name = name
+            @confine = confine
+            @comment = ""
+        end
+
+        def <=>(other)
+            @name <=> other.name
+        end
+
+        def is_fact?
+            true
+        end
+
+        def full_name
+            @name
+        end
+
+        def to_s
+            res = self.class.name + ": " + @name + "\n"
+            res << @comment.to_s
+            res
+        end
+    end
+
+    # PuppetResource holds a puppet resource
+    # It is mapped to a HTMLPuppetResource for display
+    # A resource is defined by its "normal" form Type[title]
+    class PuppetResource < CodeObject
+        attr_accessor :type, :title, :params
+
+        def initialize(type, title, comment, params)
+            super()
+            @type = type
+            @title = title
+            @comment = comment
+            @params = params
+        end
+
+        def <=>(other)
+            full_name <=> other.full_name
+        end
+
+        def full_name
+            @type + "[" + @title + "]"
+        end
+
+        def name
+            full_name
+        end
+
+        def to_s
+            res = @type + "[" + @title + "]\n"
+            res << @comment.to_s
+            res
+        end
+    end
+end

lib/puppet/util/rdoc/generators/puppet_generator.rb

@@ -0,0 +1,829 @@
+require 'rdoc/generators/html_generator'
+require 'puppet/util/rdoc/code_objects'
+module Generators
+
+    # This module holds all the classes needed to generate the HTML documentation
+    # of a bunch of puppet manifests.
+    # 
+    # It works by traversing all the code objects defined by the Puppet RDoc::Parser 
+    # and produces HTML counterparts objects that in turns are used by RDoc template engine
+    # to produce the final HTML.
+    # 
+    # It is also responsible of creating the whole directory hierarchy, and various index
+    # files.
+    # 
+    # It is to be noted that the whole system is built on top of ruby RDoc. As such there
+    # is an implicit mapping of puppet entities to ruby entitites:
+    #
+    #         Puppet    =>    Ruby
+    #         ------------------------
+    #         Module          Module
+    #         Class           Class
+    #         Definition      Method
+    #         Resource        
+    #         Node
+    #         Plugin
+    #         Fact
+
+    MODULE_DIR = "modules"
+    NODE_DIR = "nodes"
+    PLUGIN_DIR = "plugins"
+
+    # This is a specialized HTMLGenerator tailored to Puppet manifests
+    class PuppetGenerator < HTMLGenerator
+
+        def PuppetGenerator.for(options)
+            AllReferences::reset
+            HtmlMethod::reset
+
+            if options.all_one_file
+                PuppetGeneratorInOne.new(options)
+            else
+                PuppetGenerator.new(options)
+            end
+        end
+
+        def initialize(options) #:not-new:
+            @options    = options
+            load_html_template
+        end
+
+        # loads our own html template file
+        def load_html_template
+            begin
+                require 'puppet/util/rdoc/generators/template/puppet/puppet'
+                extend RDoc::Page
+            rescue LoadError
+                $stderr.puts "Could not find Puppet template '#{template}'"
+                exit 99
+            end
+        end
+
+        def gen_method_index
+            # we don't generate an all define index
+            # as the presentation is per module/per class
+        end
+
+        # This is the central method, it generates the whole structures
+        # along with all the indices.
+        def generate_html
+            super
+            gen_into(@nodes)
+            gen_into(@plugins)
+        end
+
+        ##
+        # Generate:
+        #  the list of modules
+        #  the list of classes and definitions of a specific module
+        #  the list of all classes
+        #  the list of nodes
+        #  the list of resources
+        def build_indices
+            @allfiles = []
+            @nodes = []
+            @plugins = []
+
+            # contains all the seen modules
+            @modules = {}
+            @allclasses = {}
+
+            # build the modules, classes and per modules classes and define list
+            @toplevels.each do |toplevel|
+                next unless toplevel.document_self
+                file = HtmlFile.new(toplevel, @options, FILE_DIR)
+                classes = []
+                methods = []
+                modules = []
+                nodes = []
+
+                # find all classes of this toplevel
+                # store modules if we find one
+                toplevel.each_classmodule do |k|
+                    generate_class_list(classes, modules, k, toplevel, CLASS_DIR)
+                end
+
+                # find all defines belonging to this toplevel
+                HtmlMethod.all_methods.each do |m|
+                    # find parent module, check this method is not already
+                    # defined.
+                    if m.context.parent.toplevel === toplevel
+                        methods << m
+                    end
+                end
+
+                classes.each do |k|
+                    @allclasses[k.index_name] = k if !@allclasses.has_key?(k.index_name)
+                end
+
+                # generate nodes and plugins found
+                classes.each do |k|
+                    if k.context.is_module?
+                        k.context.each_node do |name,node|
+                            nodes << HTMLPuppetNode.new(node, toplevel, NODE_DIR, @options)
+                            @nodes << nodes.last
+                        end
+                        k.context.each_plugin do |plugin|
+                            @plugins << HTMLPuppetPlugin.new(plugin, toplevel, PLUGIN_DIR, @options)
+                        end
+                        k.context.each_fact do |fact|
+                            @plugins << HTMLPuppetPlugin.new(fact, toplevel, PLUGIN_DIR, @options)
+                        end
+                    end
+                end
+
+                @files << file
+                @allfiles << { "file" => file, "modules" => modules, "classes" => classes, "methods" => methods, "nodes" => nodes }
+            end
+
+            @classes = @allclasses.values
+        end
+
+        # produce a class/module list of HTMLPuppetModule/HTMLPuppetClass
+        # based on the code object traversal.
+        def generate_class_list(classes, modules, from, html_file, class_dir)
+            if from.is_module? and !@modules.has_key?(from.name)
+                k = HTMLPuppetModule.new(from, html_file, class_dir, @options)
+                classes << k
+                @modules[from.name] = k
+                modules << @modules[from.name]
+            elsif from.is_module?
+                modules << @modules[from.name]
+            elsif !from.is_module?
+                k = HTMLPuppetClass.new(from, html_file, class_dir, @options)
+                classes << k
+            end
+            from.each_classmodule do |mod|
+                generate_class_list(classes, modules, mod, html_file, class_dir)
+            end
+        end
+
+        # generate all the subdirectories, modules, classes and files
+        def gen_sub_directories
+            begin
+                super
+                File.makedirs(MODULE_DIR)
+                File.makedirs(NODE_DIR)
+                File.makedirs(PLUGIN_DIR)
+            rescue
+                $stderr.puts $!.message
+                exit 1
+            end
+        end
+
+        # generate the index of modules
+        def gen_file_index
+            gen_top_index(@modules.values, 'All Modules', RDoc::Page::TOP_INDEX, "fr_modules_index.html")
+        end
+
+        # generate a top index
+        def gen_top_index(collection, title, template, filename)
+            template = TemplatePage.new(RDoc::Page::FR_INDEX_BODY, template)
+            res = []
+            collection.sort.each do |f|
+                if f.document_self
+                    res << { "classlist" => CGI.escapeHTML("#{MODULE_DIR}/fr_#{f.index_name}.html"), "module" => CGI.escapeHTML("#{CLASS_DIR}/#{f.index_name}.html"),"name" => CGI.escapeHTML(f.index_name) }
+                end
+            end
+
+            values = {
+                "entries"    => res,
+                'list_title' => CGI.escapeHTML(title),
+                'index_url'  => main_url,
+                'charset'    => @options.charset,
+                'style_url'  => style_url('', @options.css),
+            }
+
+            File.open(filename, "w") do |f|
+                template.write_html_on(f, values)
+            end
+        end
+
+        # generate the all classes index file and the combo index
+        def gen_class_index
+            gen_an_index(@classes, 'All Classes', RDoc::Page::CLASS_INDEX, "fr_class_index.html")
+            @allfiles.each do |file|
+                unless file['file'].context.file_relative_name =~ /\.rb$/
+                    gen_composite_index(file,
+                                        RDoc::Page::COMBO_INDEX,
+                                        "#{MODULE_DIR}/fr_#{file["file"].context.module_name}.html")
+                end
+            end
+        end
+
+        def gen_composite_index(collection, template, filename)\
+            return if FileTest.exists?(filename)
+
+            template = TemplatePage.new(RDoc::Page::FR_INDEX_BODY, template)
+            res1 = []
+            collection['classes'].sort.each do |f|
+                if f.document_self
+                    unless f.context.is_module?
+                        res1 << { "href" => "../"+CGI.escapeHTML(f.path), "name" => CGI.escapeHTML(f.index_name) }
+                    end
+                end
+            end
+
+            res2 = []
+            collection['methods'].sort.each do |f|
+                if f.document_self
+                    res2 << { "href" => "../"+f.path, "name" => f.index_name.sub(/\(.*\)$/,'') }
+                end
+            end
+
+            module_name = []
+            res3 = []
+            res4 = []
+            collection['modules'].sort.each do |f|
+                module_name << { "href" => "../"+CGI.escapeHTML(f.path), "name" => CGI.escapeHTML(f.index_name) }
+                unless f.facts.nil?
+                    f.facts.each do |fact|
+                        res3 << {"href" => "../"+CGI.escapeHTML(AllReferences["PLUGIN(#{fact.name})"].path), "name" => CGI.escapeHTML(fact.name)}
+                    end
+                end
+                unless f.plugins.nil?
+                    f.plugins.each do |plugin|
+                        res4 << {"href" => "../"+CGI.escapeHTML(AllReferences["PLUGIN(#{plugin.name})"].path), "name" => CGI.escapeHTML(plugin.name)}
+                    end
+                end
+            end
+
+            res5 = []
+            collection['nodes'].sort.each do |f|
+                if f.document_self
+                    res5 << { "href" => "../"+CGI.escapeHTML(f.path), "name" => CGI.escapeHTML(f.name) }
+                end
+            end
+
+            values = {
+                "module" => module_name,
+                "classes"    => res1,
+                'classes_title' => CGI.escapeHTML("Classes"),
+                'defines_title' => CGI.escapeHTML("Defines"),
+                'facts_title' => CGI.escapeHTML("Custom Facts"),
+                'plugins_title' => CGI.escapeHTML("Plugins"),
+                'nodes_title' => CGI.escapeHTML("Nodes"),
+                'index_url'  => main_url,
+                'charset'    => @options.charset,
+                'style_url'  => style_url('', @options.css),
+            }
+
+            values["defines"] = res2 if res2.size>0
+            values["facts"] = res3 if res3.size>0
+            values["plugins"] = res4 if res4.size>0
+            values["nodes"] = res5 if res5.size>0
+
+            File.open(filename, "w") do |f|
+                template.write_html_on(f, values)
+            end
+        end
+
+        # returns the initial_page url
+        def main_url
+            main_page = @options.main_page
+            ref = nil
+            if main_page
+                ref = AllReferences[main_page]
+                if ref
+                    ref = ref.path
+                else
+                    $stderr.puts "Could not find main page #{main_page}"
+                end
+            end
+
+            unless ref
+                for file in @files
+                    if file.document_self and file.context.global
+                        ref = CGI.escapeHTML("#{CLASS_DIR}/#{file.context.module_name}.html")
+                        break
+                    end
+                end
+            end
+
+            unless ref
+                for file in @files
+                    if file.document_self and !file.context.global
+                        ref = CGI.escapeHTML("#{CLASS_DIR}/#{file.context.module_name}.html")
+                        break
+                    end
+                end
+            end
+
+            unless ref
+                $stderr.puts "Couldn't find anything to document"
+                $stderr.puts "Perhaps you've used :stopdoc: in all classes"
+                exit(1)
+            end
+
+            ref
+        end
+
+    end
+
+    # This module is used to hold/generate a list of puppet resources
+    # this is used in HTMLPuppetClass and HTMLPuppetNode
+    module ResourceContainer
+        def collect_resources
+            list = @context.resource_list
+            @resources = list.collect {|m| HTMLPuppetResource.new(m, self, @options) }
+        end
+
+        def build_resource_summary_list(path_prefix='')
+            collect_resources unless @resources
+            resources = @resources.sort
+            res = []
+            resources.each do |r|
+                res << {
+                  "name" => CGI.escapeHTML(r.name),
+                  "aref" => "#{path_prefix}\##{r.aref}"
+                }
+            end
+            res
+        end
+
+        def build_resource_detail_list(section)
+            outer = []
+            resources = @resources.sort
+            resources.each do |r|
+                row = {}
+                if r.section == section and r.document_self
+                    row["name"]        = CGI.escapeHTML(r.name)
+                    desc = r.description.strip
+                    row["m_desc"]      = desc unless desc.empty?
+                    row["aref"]        = r.aref
+                    row["params"]      = r.params
+                    outer << row
+                end
+            end
+            outer
+        end
+    end
+
+    class HTMLPuppetClass < HtmlClass
+        include ResourceContainer
+
+        def value_hash
+            super
+            rl = build_resource_summary_list
+            @values["resources"] = rl unless rl.empty?
+
+            @context.sections.each do |section|
+                secdata = @values["sections"].select { |secdata| secdata["secsequence"] == section.sequence }
+                if secdata.size == 1
+                    secdata = secdata[0]
+
+                    rdl = build_resource_detail_list(section)
+                    secdata["resource_list"] = rdl unless rdl.empty?
+                end
+            end
+            @values
+        end
+    end
+
+    class HTMLPuppetNode < ContextUser
+        include ResourceContainer
+
+        attr_reader :path
+
+        def initialize(context, html_file, prefix, options)
+            super(context, options)
+
+            @html_file = html_file
+            @is_module = context.is_module?
+            @values    = {}
+
+            context.viewer = self
+
+            if options.all_one_file
+                @path = context.full_name
+            else
+                @path = http_url(context.full_name, prefix)
+            end
+
+            AllReferences.add("NODE(#{@context.full_name})", self)
+        end
+
+        def name
+            @context.name
+        end
+
+        # return the relative file name to store this class in,
+        # which is also its url
+        def http_url(full_name, prefix)
+            path = full_name.dup
+            if path['<<']
+                path.gsub!(/<<\s*(\w*)/) { "from-#$1" }
+            end
+            File.join(prefix, path.split("::")) + ".html"
+        end
+
+        def parent_name
+            @context.parent.full_name
+        end
+
+        def index_name
+            name
+        end
+
+        def write_on(f)
+            value_hash
+            template = TemplatePage.new(RDoc::Page::BODYINC,
+                                        RDoc::Page::NODE_PAGE,
+                                        RDoc::Page::METHOD_LIST)
+            template.write_html_on(f, @values)
+        end
+
+        def value_hash
+            class_attribute_values
+            add_table_of_sections
+
+            @values["charset"] = @options.charset
+            @values["style_url"] = style_url(path, @options.css)
+
+            d = markup(@context.comment)
+            @values["description"] = d unless d.empty?
+
+            ml = build_method_summary_list
+            @values["methods"] = ml unless ml.empty?
+
+            rl = build_resource_summary_list
+            @values["resources"] = rl unless rl.empty?
+
+            il = build_include_list(@context)
+            @values["includes"] = il unless il.empty?
+
+            @values["sections"] = @context.sections.map do |section|
+
+                secdata = {
+                    "sectitle" => section.title,
+                    "secsequence" => section.sequence,
+                    "seccomment" => markup(section.comment)
+                }
+
+                al = build_alias_summary_list(section)
+                secdata["aliases"] = al unless al.empty?
+
+                co = build_constants_summary_list(section)
+                secdata["constants"] = co unless co.empty?
+
+                al = build_attribute_list(section)
+                secdata["attributes"] = al unless al.empty?
+
+                cl = build_class_list(0, @context, section)
+                secdata["classlist"] = cl unless cl.empty?
+
+                mdl = build_method_detail_list(section)
+                secdata["method_list"] = mdl unless mdl.empty?
+
+                rdl = build_resource_detail_list(section)
+                secdata["resource_list"] = rdl unless rdl.empty?
+
+                secdata
+            end
+
+            @values
+        end
+
+        def build_attribute_list(section)
+            atts = @context.attributes.sort
+            res = []
+            atts.each do |att|
+                next unless att.section == section
+                if att.visibility == :public || att.visibility == :protected || @options.show_all
+                    entry = {
+                        "name"   => CGI.escapeHTML(att.name),
+                        "rw"     => att.rw,
+                        "a_desc" => markup(att.comment, true)
+                    }
+                    unless att.visibility == :public || att.visibility == :protected
+                        entry["rw"] << "-"
+                    end
+                    res << entry
+                end
+            end
+            res
+        end
+
+        def class_attribute_values
+            h_name = CGI.escapeHTML(name)
+
+            @values["classmod"]  = "Node"
+            @values["title"]     = "#{@values['classmod']}: #{h_name}"
+
+            c = @context
+            c = c.parent while c and !c.diagram
+
+            if c && c.diagram
+                @values["diagram"] = diagram_reference(c.diagram)
+            end
+
+            @values["full_name"] = h_name
+
+            parent_class = @context.superclass
+
+            if parent_class
+                @values["parent"] = CGI.escapeHTML(parent_class)
+
+                if parent_name
+                    lookup = parent_name + "::" + parent_class
+                else
+                    lookup = parent_class
+                end
+                lookup = "NODE(#{lookup})"
+                parent_url = AllReferences[lookup] || AllReferences[parent_class]
+                if parent_url and parent_url.document_self
+                    @values["par_url"] = aref_to(parent_url.path)
+                end
+            end
+
+            files = []
+            @context.in_files.each do |f|
+                res = {}
+                full_path = CGI.escapeHTML(f.file_absolute_name)
+
+                res["full_path"]     = full_path
+                res["full_path_url"] = aref_to(f.viewer.path) if f.document_self
+
+                if @options.webcvs
+                    res["cvsurl"] = cvs_url( @options.webcvs, full_path )
+                end
+
+                files << res
+            end
+
+            @values['infiles'] = files
+        end
+
+        def <=>(other)
+            self.name <=> other.name
+        end
+    end
+
+    class HTMLPuppetModule < HtmlClass
+
+        def initialize(context, html_file, prefix, options)
+            super(context, html_file, prefix, options)
+        end
+
+        def value_hash
+            @values = super
+
+            fl = build_facts_summary_list
+            @values["facts"] = fl unless fl.empty?
+
+            pl = build_plugins_summary_list
+            @values["plugins"] = pl unless pl.empty?
+
+            nl = build_nodes_list(0, @context)
+            @values["nodelist"] = nl unless nl.empty?
+
+            @values
+        end
+
+        def build_nodes_list(level, context)
+            res = ""
+            prefix = "&nbsp;&nbsp;::" * level;
+
+            context.nodes.sort.each do |node|
+                if node.document_self
+                    res <<
+                    prefix <<
+                    "Node " <<
+                    href(url(node.viewer.path), "link", node.full_name) <<
+                    "<br />\n"
+                end
+            end
+            res
+        end
+
+        def build_facts_summary_list
+            potentially_referenced_list(context.facts) {|fn| ["PLUGIN(#{fn})"] }
+        end
+
+        def build_plugins_summary_list
+            potentially_referenced_list(context.plugins) {|fn| ["PLUGIN(#{fn})"] }
+        end
+
+        def facts
+            @context.facts
+        end
+
+        def plugins
+            @context.plugins
+        end
+
+    end
+
+    class HTMLPuppetPlugin < ContextUser
+        attr_reader :path
+
+        def initialize(context, html_file, prefix, options)
+            super(context, options)
+
+            @html_file = html_file
+            @is_module = false
+            @values    = {}
+
+            context.viewer = self
+
+            if options.all_one_file
+                @path = context.full_name
+            else
+                @path = http_url(context.full_name, prefix)
+            end
+
+            AllReferences.add("PLUGIN(#{@context.full_name})", self)
+        end
+
+        def name
+            @context.name
+        end
+
+        # return the relative file name to store this class in,
+        # which is also its url
+        def http_url(full_name, prefix)
+            path = full_name.dup
+            if path['<<']
+                path.gsub!(/<<\s*(\w*)/) { "from-#$1" }
+            end
+            File.join(prefix, path.split("::")) + ".html"
+        end
+
+        def parent_name
+            @context.parent.full_name
+        end
+
+        def index_name
+            name
+        end
+
+        def write_on(f)
+            value_hash
+            template = TemplatePage.new(RDoc::Page::BODYINC,
+                                        RDoc::Page::PLUGIN_PAGE,
+                                        RDoc::Page::PLUGIN_LIST)
+            template.write_html_on(f, @values)
+        end
+
+        def value_hash
+            attribute_values
+            add_table_of_sections
+
+            @values["charset"] = @options.charset
+            @values["style_url"] = style_url(path, @options.css)
+
+            d = markup(@context.comment)
+            @values["description"] = d unless d.empty?
+
+            if context.is_fact?
+                unless context.confine.empty?
+                    res = {}
+                    res["type"] = context.confine[:type]
+                    res["value"] = context.confine[:value]
+                    @values["confine"] = [res]
+                end
+            else
+                @values["type"] = context.type
+            end
+
+            @values["sections"] = @context.sections.map do |section|
+                secdata = {
+                    "sectitle" => section.title,
+                    "secsequence" => section.sequence,
+                    "seccomment" => markup(section.comment)
+                }
+                secdata
+            end
+
+            @values
+        end
+
+        def attribute_values
+            h_name = CGI.escapeHTML(name)
+
+            if @context.is_fact?
+                @values["classmod"]  = "Fact"
+            else
+                @values["classmod"]  = "Plugin"
+            end
+            @values["title"]     = "#{@values['classmod']}: #{h_name}"
+
+            c = @context
+            @values["full_name"] = h_name
+
+            files = []
+            @context.in_files.each do |f|
+                res = {}
+                full_path = CGI.escapeHTML(f.file_absolute_name)
+
+                res["full_path"]     = full_path
+                res["full_path_url"] = aref_to(f.viewer.path) if f.document_self
+
+                if @options.webcvs
+                    res["cvsurl"] = cvs_url( @options.webcvs, full_path )
+                end
+
+                files << res
+            end
+
+            @values['infiles'] = files
+        end
+
+        def <=>(other)
+            self.name <=> other.name
+        end
+
+    end
+
+    class HTMLPuppetResource
+        include MarkUp
+
+        attr_reader :context
+
+        @@seq = "R000000"
+
+        def initialize(context, html_class, options)
+            @context    = context
+            @html_class = html_class
+            @options    = options
+            @@seq       = @@seq.succ
+            @seq        = @@seq
+
+            context.viewer = self
+
+            AllReferences.add(name, self)
+        end
+
+        def as_href(from_path)
+            if @options.all_one_file
+                "#" + path
+            else
+                HTMLGenerator.gen_url(from_path, path)
+            end
+        end
+
+        def name
+            @context.name
+        end
+
+        def section
+            @context.section
+        end
+
+        def index_name
+            "#{@context.name}"
+        end
+
+        def params
+            @context.params
+        end
+
+        def parent_name
+            if @context.parent.parent
+                @context.parent.parent.full_name
+            else
+                nil
+            end
+        end
+
+        def aref
+            @seq
+        end
+
+        def path
+            if @options.all_one_file
+                aref
+            else
+                @html_class.path + "#" + aref
+            end
+        end
+
+        def description
+            markup(@context.comment)
+        end
+
+        def <=>(other)
+            @context <=> other.context
+        end
+
+        def document_self
+            @context.document_self
+        end
+
+        def find_symbol(symbol, method=nil)
+            res = @context.parent.find_symbol(symbol, method)
+            if res
+                res = res.viewer
+            end
+            res
+        end
+
+    end
+
+    class PuppetGeneratorInOne < HTMLGeneratorInOne
+        def gen_method_index
+            gen_an_index(HtmlMethod.all_methods, 'Defines')
+        end
+    end
+
+end

lib/puppet/util/rdoc/parser.rb

@@ -0,0 +1,437 @@
+# Puppet "parser" for the rdoc system
+# The parser uses puppet parser and traverse the AST to instruct RDoc about
+# our current structures. It also parses ruby files that could contain
+# either custom facts or puppet plugins (functions, types...)
+
+# rdoc mandatory includes
+require "rdoc/code_objects"
+require "puppet/util/rdoc/code_objects"
+require "rdoc/tokenstream"
+require "rdoc/markup/simple_markup/preprocess"
+require "rdoc/parsers/parserfactory"
+
+module RDoc
+
+class Parser
+    extend ParserFactory
+
+    # parser registration into RDoc
+    parse_files_matching(/\.(rb|pp)$/)
+
+    # called with the top level file
+    def initialize(top_level, file_name, content, options, stats)
+        @options = options
+        @stats   = stats
+        @input_file_name = file_name
+        @top_level = PuppetTopLevel.new(top_level)
+        @progress = $stderr unless options.quiet
+    end
+
+    # main entry point
+    def scan
+        Puppet.info "rdoc: scanning %s" % @input_file_name
+        if @input_file_name =~ /\.pp$/
+            @parser = Puppet::Parser::Parser.new(:environment => Puppet[:environment])
+            @parser.file = @input_file_name
+            @ast = @parser.parse
+        end
+        scan_top_level(@top_level)
+        @top_level
+    end
+
+    private
+
+    # walk down the namespace and lookup/create container as needed
+    def get_class_or_module(container, name)
+
+        # class ::A -> A is in the top level
+        if name =~ /^::/
+            container = @top_level
+        end
+
+        names = name.split('::')
+
+        final_name = names.pop
+        names.each do |name|
+            prev_container = container
+            container = container.find_module_named(name)
+            unless container
+                container = prev_container.add_module(PuppetClass, name)
+            end
+        end
+        return [container, final_name]
+    end
+
+    # split_module tries to find if +path+ belongs to the module path
+    # if it does, it returns the module name, otherwise if we are sure
+    # it is part of the global manifest path, "<site>" is returned.
+    # And finally if this path couldn't be mapped anywhere, nil is returned.
+    def split_module(path)
+        # find a module
+        fullpath = File.expand_path(path)
+        Puppet.debug "rdoc: testing %s" % fullpath
+        if fullpath =~ /(.*)\/([^\/]+)\/(?:manifests|plugins)\/.+\.(pp|rb)$/
+            modpath = $1
+            name = $2
+            Puppet.debug "rdoc: module %s into %s ?" % [name, modpath]
+            Puppet::Module.modulepath().each do |mp|
+                if File.identical?(modpath,mp)
+                    Puppet.debug "rdoc: found module %s" % name
+                    return name
+                end
+            end
+        end
+        if fullpath =~ /\.(pp|rb)$/
+            # there can be paths we don't want to scan under modules
+            # imagine a ruby or manifest that would be distributed as part as a module
+            # but we don't want those to be hosted under <site>
+            Puppet::Module.modulepath().each do |mp|
+                # check that fullpath is a descendant of mp
+                dirname = fullpath
+                while (dirname = File.dirname(dirname)) != '/'
+                    return nil if File.identical?(dirname,mp)
+                end
+            end
+        end
+        # we are under a global manifests
+        Puppet.debug "rdoc: global manifests"
+        return "<site>"
+    end
+
+    # create documentation for the top level +container+
+    def scan_top_level(container)
+        # use the module README as documentation for the module
+        comment = ""
+        readme = File.join(File.dirname(File.dirname(@input_file_name)), "README")
+        comment = File.open(readme,"r") { |f| f.read } if FileTest.readable?(readme)
+        look_for_directives_in(container, comment) unless comment.empty?
+
+        # infer module name from directory
+        name = split_module(@input_file_name)
+        if name.nil?
+            # skip .pp files that are not in manifests directories as we can't guarantee they're part
+            # of a module or the global configuration.
+            container.document_self = false
+            return
+        end
+
+        Puppet.debug "rdoc: scanning for %s" % name
+
+        container.module_name = name
+        container.global=true if name == "<site>"
+
+        @stats.num_modules += 1
+        container, name  = get_class_or_module(container,name)
+        mod = container.add_module(PuppetModule, name)
+        mod.record_location(@top_level)
+        mod.comment = comment
+
+        if @input_file_name =~ /\.pp$/
+            parse_elements(mod)
+        elsif @input_file_name =~ /\.rb$/
+            parse_plugins(mod)
+        end
+    end
+
+    # create documentation for include statements we can find in +code+
+    # and associate it with +container+
+    def scan_for_include(container, code)
+        code.each do |stmt|
+            scan_for_include(container,stmt.children) if stmt.is_a?(Puppet::Parser::AST::ASTArray)
+
+            if stmt.is_a?(Puppet::Parser::AST::Function) and stmt.name == "include"
+                stmt.arguments.each do |included|
+                    Puppet.debug "found include: %s" % included.value
+                    container.add_include(Include.new(included.value, stmt.doc))
+                end
+            end
+        end
+    end
+
+    # create documentation for global variables assignements we can find in +code+
+    # and associate it with +container+
+    def scan_for_vardef(container, code)
+        code.each do |stmt|
+            scan_for_vardef(container,stmt.children) if stmt.is_a?(Puppet::Parser::AST::ASTArray)
+
+            if stmt.is_a?(Puppet::Parser::AST::VarDef)
+                Puppet.debug "rdoc: found constant: %s = %s" % [stmt.name.to_s, value_to_s(stmt.value)]
+                container.add_constant(Constant.new(stmt.name.to_s, value_to_s(stmt.value), stmt.doc))
+            end
+        end
+    end
+
+    # create documentation for resources we can find in +code+
+    # and associate it with +container+
+    def scan_for_resource(container, code)
+        code.each do |stmt|
+            scan_for_resource(container,stmt.children) if stmt.is_a?(Puppet::Parser::AST::ASTArray)
+
+            if stmt.is_a?(Puppet::Parser::AST::Resource)
+                type = stmt.type.split("::").collect { |s| s.capitalize }.join("::")
+                title = value_to_s(stmt.title)
+                Puppet.debug "rdoc: found resource: %s[%s]" % [type,title]
+
+                param = []
+                stmt.params.children.each do |p|
+                    res = {}
+                    res["name"] = p.param
+                    if !p.value.nil?
+                        if !p.value.is_a?(Puppet::Parser::AST::ASTArray)
+                            res["value"] = "'#{p.value}'"
+                        else
+                            res["value"] = "[%s]" % p.value.children.collect { |v| "'#{v}'" }.join(", ")
+                        end
+                    end
+                    param << res
+                end
+
+                container.add_resource(PuppetResource.new(type, title, stmt.doc, param))
+            end
+        end
+    end
+
+    # create documentation for a class named +name+
+    def document_class(name, klass, container)
+        Puppet.debug "rdoc: found new class %s" % name
+        container, name = get_class_or_module(container, name)
+
+        superclass = klass.parentclass
+        superclass = "" if superclass.nil? or superclass.empty?
+
+        @stats.num_classes += 1
+        comment = klass.doc
+        look_for_directives_in(container, comment) unless comment.empty?
+        cls = container.add_class(PuppetClass, name, superclass)
+        cls.record_location(@top_level)
+
+        # scan class code for include
+        code = klass.code.children if klass.code.is_a?(Puppet::Parser::AST::ASTArray)
+        code ||= klass.code
+        unless code.nil?
+            scan_for_include(cls, code)
+            scan_for_resource(cls, code) if Puppet.settings[:document_all]
+        end
+
+        cls.comment = comment
+    end
+
+    # create documentation for a node
+    def document_node(name, node, container)
+        Puppet.debug "rdoc: found new node %s" % name
+        superclass = node.parentclass
+        superclass = "" if superclass.nil? or superclass.empty?
+
+        comment = node.doc
+        look_for_directives_in(container, comment) unless comment.empty?
+        n = container.add_node(name, superclass)
+        n.record_location(@top_level)
+
+        code = node.code.children if node.code.is_a?(Puppet::Parser::AST::ASTArray)
+        code ||= node.code
+        unless code.nil?
+            scan_for_include(n, code)
+            scan_for_vardef(n, code)
+            scan_for_resource(n, code) if Puppet.settings[:document_all]
+        end
+
+        n.comment = comment
+    end
+
+    # create documentation for a define
+    def document_define(name, define, container)
+        Puppet.debug "rdoc: found new definition %s" % name
+        # find superclas if any
+        @stats.num_methods += 1
+
+        # find the parentclass
+        # split define name by :: to find the complete module hierarchy
+        container, name = get_class_or_module(container,name)
+
+        return if container.find_local_symbol(name)
+
+        # build up declaration
+        declaration = ""
+        define.arguments.each do |arg,value|
+            declaration << "\$#{arg}"
+            if !value.nil?
+                declaration << " => "
+                if !value.is_a?(Puppet::Parser::AST::ASTArray)
+                    declaration << "'#{value.value}'"
+                else
+                    declaration << "[%s]" % value.children.collect { |v| "'#{v}'" }.join(", ")
+                end
+            end
+            declaration << ", "
+        end
+        declaration.chop!.chop! if declaration.size > 1
+
+        # register method into the container
+        meth =  AnyMethod.new(declaration, name)
+        container.add_method(meth)
+        meth.comment = define.doc
+        look_for_directives_in(container, meth.comment) unless meth.comment.empty?
+        meth.params = "( " + declaration + " )"
+        meth.visibility = :public
+        meth.document_self = true
+        meth.singleton = false
+    end
+
+    # Traverse the AST tree and produce code-objects node
+    # that contains the documentation
+    def parse_elements(container)
+        Puppet.debug "rdoc: scanning manifest"
+        @ast[:classes].each do |name, klass|
+            if klass.file == @input_file_name
+                unless name.empty?
+                    document_class(name,klass,container)
+                else # on main class document vardefs
+                    code = klass.code.children unless klass.code.is_a?(Puppet::Parser::AST::ASTArray)
+                    code ||= klass.code
+                    scan_for_vardef(container, code) unless code.nil?
+                end
+            end
+        end
+
+        @ast[:definitions].each do |name, define|
+            if define.file == @input_file_name
+                document_define(name,define,container)
+            end
+        end
+
+        @ast[:nodes].each do |name, node|
+            if node.file == @input_file_name
+                document_node(name,node,container)
+            end
+        end
+    end
+
+    # create documentation for plugins
+    def parse_plugins(container)
+        Puppet.debug "rdoc: scanning plugin or fact"
+        if @input_file_name =~ /\/facter\/[^\/]+\.rb$/
+            parse_fact(container)
+        else
+            parse_puppet_plugin(container)
+        end
+    end
+
+    # this is a poor man custom fact parser :-)
+    def parse_fact(container)
+        comments = ""
+        current_fact = nil
+        File.open(@input_file_name) do |of|
+            of.each do |line|
+                # fetch comments
+                if line =~ /^[ \t]*# ?(.*)$/
+                    comments += $1 + "\n"
+                elsif line =~ /^[ \t]*Facter.add\(['"](.*?)['"]\)/
+                    current_fact = Fact.new($1,{})
+                    container.add_fact(current_fact)
+                    look_for_directives_in(container, comments) unless comments.empty?
+                    current_fact.comment = comments
+                    current_fact.record_location(@top_level)
+                    comments = ""
+                    Puppet.debug "rdoc: found custom fact %s" % current_fact.name
+                elsif line =~ /^[ \t]*confine[ \t]*:(.*?)[ \t]*=>[ \t]*(.*)$/
+                    current_fact.confine = { :type => $1, :value => $2 } unless current_fact.nil?
+                else # unknown line type
+                    comments =""
+                end
+            end
+        end
+    end
+
+    # this is a poor man puppet plugin parser :-)
+    # it doesn't extract doc nor desc :-(
+    def parse_puppet_plugin(container)
+        comments = ""
+        current_plugin = nil
+
+        File.open(@input_file_name) do |of|
+            of.each do |line|
+                # fetch comments
+                if line =~ /^[ \t]*# ?(.*)$/
+                    comments += $1 + "\n"
+                elsif line =~ /^[ \t]*newfunction[ \t]*\([ \t]*:(.*?)[ \t]*,[ \t]*:type[ \t]*=>[ \t]*(:rvalue|:lvalue)\)/
+                    current_plugin = Plugin.new($1, "function")
+                    container.add_plugin(current_plugin)
+                    look_for_directives_in(container, comments) unless comments.empty?
+                    current_plugin.comment = comments
+                    current_plugin.record_location(@top_level)
+                    comments = ""
+                    Puppet.debug "rdoc: found new function plugins %s" % current_plugin.name
+                elsif line =~ /^[ \t]*Puppet::Type.newtype[ \t]*\([ \t]*:(.*?)\)/
+                    current_plugin = Plugin.new($1, "type")
+                    container.add_plugin(current_plugin)
+                    look_for_directives_in(container, comments) unless comments.empty?
+                    current_plugin.comment = comments
+                    current_plugin.record_location(@top_level)
+                    comments = ""
+                    Puppet.debug "rdoc: found new type plugins %s" % current_plugin.name
+                elsif line =~ /module Puppet::Parser::Functions/
+                    # skip
+                else # unknown line type
+                    comments =""
+                end
+            end
+        end
+    end
+
+    # look_for_directives_in scans the current +comment+ for RDoc directives
+    def look_for_directives_in(context, comment)
+        preprocess = SM::PreProcess.new(@input_file_name, @options.rdoc_include)
+
+        preprocess.handle(comment) do |directive, param|
+            case directive
+            when "stopdoc"
+                context.stop_doc
+                ""
+            when "startdoc"
+                context.start_doc
+                context.force_documentation = true
+                ""
+            when "enddoc"
+                #context.done_documenting = true
+                #""
+                throw :enddoc
+            when "main"
+                options = Options.instance
+                options.main_page = param
+                ""
+            when "title"
+                options = Options.instance
+                options.title = param
+                ""
+            when "section"
+                context.set_current_section(param, comment)
+                comment.replace("") # 1.8 doesn't support #clear
+                break
+            else
+                warn "Unrecognized directive '#{directive}'"
+                break
+            end
+        end
+        remove_private_comments(comment)
+    end
+
+    def remove_private_comments(comment)
+        comment.gsub!(/^#--.*?^#\+\+/m, '')
+        comment.sub!(/^#--.*/m, '')
+    end
+
+    # convert an AST value to a string
+    def value_to_s(value)
+        value = value.children if value.is_a?(Puppet::Parser::AST::ASTArray)
+        if value.is_a?(Array)
+            "['#{value.join(", ")}']"
+        elsif [:true, true, "true"].include?(value)
+            "true"
+        elsif [:false, false, "false"].include?(value)
+            "false"
+        else
+            value.to_s
+        end
+    end
+end
+end