diff --git a/lib/rdoc/code_object/class_module.rb b/lib/rdoc/code_object/class_module.rb index 76ded527a1..c06c9ddb48 100644 --- a/lib/rdoc/code_object/class_module.rb +++ b/lib/rdoc/code_object/class_module.rb @@ -52,6 +52,15 @@ class RDoc::ClassModule < RDoc::Context attr_accessor :is_alias_for + ## + # True for a namespace created ahead of the build by + # RDoc::Parser::Ruby::NamespaceResolver that no file has contributed to + # yet. Unlike a namespace created inside a :stopdoc: region, the first + # explicit declaration treats it like a fresh creation: it is recorded + # even when a +:nodoc:+ directive was received. + + attr_accessor :namespace_ghost + ## # Return a RDoc::ClassModule of class +class_type+ that is a copy # of module +module+. Used to promote modules to classes. diff --git a/lib/rdoc/parser.rb b/lib/rdoc/parser.rb index 8e01ef9379..26c9bafa2b 100644 --- a/lib/rdoc/parser.rb +++ b/lib/rdoc/parser.rb @@ -295,4 +295,6 @@ def handle_tab_width(body) require_relative 'parser/rd' require_relative 'parser/rbs' require_relative 'parser/ruby' +require_relative 'parser/ruby_code_object_builder' +require_relative 'parser/ruby_namespace_resolver' require_relative 'parser/ruby_colorizer' diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 5894850157..eaf05ea911 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -132,7 +132,11 @@ class RDoc::Parser::Ruby < RDoc::Parser RBS_SIG_LINE = /\A#:\s/ # :nodoc: attr_accessor :visibility - attr_reader :container, :singleton, :in_proc_block + attr_reader :singleton, :in_proc_block + + # IR records of #parse_ir, used by NamespaceResolver before #build_ir + + attr_reader :ir, :top_level def initialize(top_level, content, options, stats) super @@ -147,8 +151,12 @@ def initialize(top_level, content, options, stats) @track_visibility = :nodoc != @options.visibility @encoding = @options.encoding - @module_nesting = [[top_level, false]] - @container = top_level + # IR records emitted while visiting the AST, replayed by CodeObjectBuilder. + # Records must be plain data (no AST node or CodeObject references) + # so that the IR can be compared and serialized. + @ir = [] + @scope_id = 0 + @scope_depth = 0 @visibility = :public @singleton = false @in_proc_block = false @@ -158,8 +166,12 @@ def initialize(top_level, content, options, stats) # Applies document control directives (:startdoc:, :stopdoc: and :enddoc:) # to the current lexical scope. The state is restored when the enclosing # class/module scope is closed. + # Returns true if a :startdoc: directive took effect. Its model-side effect + # (reviving an ignored container) is replayed by CodeObjectBuilder from the + # startdoc flag of the comment payload. def apply_document_control_directive(directives) + startdoc = false directives.each do |directive, (_param, line)| case directive when 'startdoc', 'stopdoc' @@ -169,18 +181,12 @@ def apply_document_control_directive(directives) next end @doc_state = directive.to_sym - if directive == 'startdoc' && !@container.ignored? - # Compatibility: `module Net #:nodoc:` followed by :stopdoc:/:startdoc: - # regions is a common pattern that expects :startdoc: to make the - # container documentable again. Containers ignored here were created - # in a suppressed region and need documentable contents to revive. - @container.start_doc - @container.force_documentation = true - end + startdoc = true if directive == 'startdoc' when 'enddoc' @doc_state = :enddoc end end + startdoc end # Returns true if code objects at the current position should not be @@ -190,17 +196,6 @@ def document_suppressed? @track_visibility && @doc_state != :startdoc end - # Makes a container that was created inside a :stopdoc:/:enddoc: region - # (thus ignored) documentable again when it receives documentable contents - # outside the region, possibly from another file. - - def mark_container_documentable(container) - return if container.received_nodoc || !container.ignored? - record_location(container) - container.start_doc - mark_container_documentable(container.parent) if container.parent.is_a?(RDoc::ClassModule) - end - # Suppress `extend` and `include` within block # because they might be a metaprogramming block # example: `Module.new { include M }` `M.module_eval { include N }` @@ -212,44 +207,40 @@ def with_in_proc_block @in_proc_block = in_proc_block end - # Dive into another container + # Dive into another scope opened by the record of the given scope id - def with_container(container, singleton: false) - old_container = @container + def with_scope(scope_id, singleton: false) old_visibility = @visibility old_singleton = @singleton old_in_proc_block = @in_proc_block old_doc_state = @doc_state @visibility = :public - @container = container @singleton = singleton @in_proc_block = false - @module_nesting.push([container, singleton]) - yield container + @scope_depth += 1 + emit({type: :scope_enter, id: scope_id, singleton: singleton}) + yield ensure - @container = old_container + emit({type: :scope_exit}) @visibility = old_visibility @singleton = old_singleton @in_proc_block = old_in_proc_block @doc_state = old_doc_state - @module_nesting.pop + @scope_depth -= 1 end - # Records the location of this +container+ in the file for this parser and - # adds it to the list of classes and modules in the file. - - def record_location(container) # :nodoc: - case container - when RDoc::ClassModule then - @top_level.add_to_classes_or_modules container - end + # Scans this Ruby file for Ruby constructs - container.record_location @top_level + def scan + parse_ir + build_ir end - # Scans this Ruby file for Ruby constructs + # Parses the file into IR records. CodeObjects are not created until + # #build_ir is called, so that a batch driver can parse all files before + # building any of them. - def scan + def parse_ir @lines = @content.lines result = Prism.parse_lex(@content) @program_node, unordered_tokens = result.value @@ -270,10 +261,14 @@ def scan process_comments_until(@lines.size + 1) end - def should_document?(code_object) # :nodoc: - return true unless @track_visibility - return false if code_object.parent&.document_children == false - code_object.document_self + # Builds CodeObjects from the IR records of #parse_ir. A batch driver that + # already resolved the IR of all files together passes +resolve: false+. + + def build_ir(resolve: true) + NamespaceResolver.new(@store).preload_namespaces([self]) if resolve + builder = CodeObjectBuilder.new(@top_level, @store, @options, @stats, @preprocess, track_visibility: @track_visibility) + builder.run(@ir) + @top_level end # Assign AST node to a line. @@ -352,39 +347,16 @@ def prepare_comments(comments) end end - # Creates an RDoc::Method on +container+ from +comment+ if there is a - # Signature section in the comment - - def parse_comment_tomdoc(container, comment, line_no, start_line) - return if document_suppressed? - return unless signature = RDoc::TomDoc.signature(comment) - - name, = signature.split %r%[ \(]%, 2 - - meth = RDoc::AnyMethod.new name - record_location(meth) - meth.line = start_line - meth.call_seq = signature - return unless meth.name - - meth.start_collecting_tokens(:ruby) - node = @line_nodes[line_no] - tokens = node ? syntax_highlighted_tokens(node) : [] - tokens.each { |token| meth.token_stream << token } - - container.add_method meth - meth.comment = comment - @stats.add_method meth - end - def has_modifier_nodoc?(line_no) # :nodoc: @modifier_comments[line_no]&.match?(/\A#\s*:nodoc:/) end - def handle_modifier_directive(code_object, line_no) # :nodoc: + # Parses the modifier comment on the given line into a directives hash + + def modifier_directives(line_no) # :nodoc: if (comment_text = @modifier_comments[line_no]) _text, directives = @preprocess.parse_comment(comment_text, line_no, :ruby) - handle_code_object_directives(code_object, directives) + directives end end @@ -400,67 +372,19 @@ def call_node_name_arguments(call_node) # :nodoc: end || [] end - # Handles meta method comments - - def handle_meta_method_comment(comment, directives, node) - apply_document_control_directive(directives) - handle_code_object_directives(@container, directives) - is_call_node = node.is_a?(Prism::CallNode) - singleton_method = false - visibility = @visibility - attributes = rw = line_no = method_name = nil - directives.each do |directive, (param, line)| - case directive - when 'attr', 'attr_reader', 'attr_writer', 'attr_accessor' - attributes = [param] if param - attributes ||= call_node_name_arguments(node) if is_call_node - rw = directive == 'attr_writer' ? 'W' : directive == 'attr_accessor' ? 'RW' : 'R' - when 'method' - method_name = param if param - line_no = line - when 'singleton-method' - method_name = param if param - line_no = line - singleton_method = true - visibility = :public - end - end + # Emits a meta method comment record - return if document_suppressed? - - if attributes - attributes.each do |attr| - a = RDoc::Attr.new(attr, rw, comment, singleton: @singleton) - a.store = @store - a.line = line_no - record_location(a) - @container.add_attribute(a) - mark_container_documentable(@container) - a.visibility = visibility - end - elsif line_no || node - method_name ||= call_node_name_arguments(node).first if is_call_node - if node - tokens = syntax_highlighted_tokens(node) - line_no = node.location.start_line - else - tokens = [] - end - internal_add_method( - method_name, - @container, - comment: comment, - directives: directives, - dont_rename_initialize: false, - line_no: line_no, - visibility: visibility, - singleton: @singleton || singleton_method, - params: nil, - calls_super: false, - block_params: nil, - tokens: tokens, - ) + def emit_meta_comment(comment_payload, node) + if node + node_payload = { + is_call_node: node.is_a?(Prism::CallNode), + name_arguments: node.is_a?(Prism::CallNode) ? call_node_name_arguments(node) : [], + tokens: syntax_highlighted_tokens(node), + start_line: node.location.start_line + } end + emit({type: :meta_comment, comment: comment_payload, node: node_payload, + visibility: @visibility, singleton: @singleton, suppressed: document_suppressed?}) end INVALID_GHOST_METHOD_ACCEPT_DIRECTIVE_LIST = %w[ @@ -474,15 +398,13 @@ def normal_comment_treat_as_ghost_method_for_now?(directives, line_no) # :nodoc: !@line_nodes[line_no] && INVALID_GHOST_METHOD_ACCEPT_DIRECTIVE_LIST.any? { |directive| directives.has_key?(directive) } end - def handle_standalone_consecutive_comment_directive(comment, directives, start_with_sharp_sharp, line_no, start_line) # :nodoc: + def handle_standalone_consecutive_comment_directive(comment_payload, start_with_sharp_sharp, line_no, start_line) # :nodoc: if start_with_sharp_sharp && start_line != @first_non_meta_comment_start_line - node = @line_nodes[line_no] - handle_meta_method_comment(comment, directives, node) - elsif normal_comment_treat_as_ghost_method_for_now?(directives, line_no) && start_line != @first_non_meta_comment_start_line - handle_meta_method_comment(comment, directives, nil) + emit_meta_comment(comment_payload, @line_nodes[line_no]) + elsif normal_comment_treat_as_ghost_method_for_now?(comment_payload[:directives], line_no) && start_line != @first_non_meta_comment_start_line + emit_meta_comment(comment_payload, nil) else - apply_document_control_directive(directives) - handle_code_object_directives(@container, directives) + emit({type: :container_directives, comment: comment_payload}) end end @@ -492,12 +414,11 @@ def process_comments_until(line_no_until) while !@unprocessed_comments.empty? && @unprocessed_comments.first[0] <= line_no_until line_no, start_line, text = @unprocessed_comments.shift if @markup == 'tomdoc' - comment = RDoc::Comment.new(text, @top_level, :ruby) - comment.format = 'tomdoc' - parse_comment_tomdoc(@container, comment, line_no, start_line) - @preprocess.run_post_processes(comment, @container) - elsif (comment_text, directives = parse_comment_text_to_directives(text, start_line)) - handle_standalone_consecutive_comment_directive(comment_text, directives, text.start_with?(/#\#$/), line_no, start_line) + node = @line_nodes[line_no] + emit({type: :tomdoc_comment, text: text, start_line: start_line, suppressed: document_suppressed?, + tokens: node ? syntax_highlighted_tokens(node) : []}) + elsif (comment_payload = parse_comment_payload(text, start_line)) + handle_standalone_consecutive_comment_directive(comment_payload, text.start_with?(/#\#$/), line_no, start_line) end end end @@ -511,34 +432,34 @@ def skip_comments_until(line_no_until) end end - # Returns consecutive comment linked to the given line number + # Consumes the consecutive comment linked to the given line number and + # returns its payload - def consecutive_comment(line_no) + def consecutive_comment_payload(line_no) return unless @unprocessed_comments.first&.first == line_no _line_no, start_line, text = @unprocessed_comments.shift - parse_comment_text_to_directives(text, start_line) + parse_comment_payload(text, start_line) end - # Parses comment text and returns +[RDoc::Comment, directives, type_signature_lines]+, - # or +nil+ if the comment is a section header (which has no associated code - # object). + # Parses comment text into a plain-data payload for IR records, or returns + # +nil+ if the comment is a section header (which is emitted as its own + # record and has no associated code object). - def parse_comment_text_to_directives(comment_text, start_line) # :nodoc: - type_signature_lines = extract_type_signature!(comment_text, start_line) + def parse_comment_payload(comment_text, start_line) # :nodoc: + type_signature_lines, type_signature_line_no = extract_type_signature!(comment_text, start_line) comment_text, directives = @preprocess.parse_comment(comment_text, start_line, :ruby) - comment = RDoc::Comment.new(comment_text, @top_level, :ruby) - comment.normalized = true - comment.line = start_line markup, = directives['markup'] - comment.format = markup&.downcase || @markup + format = markup&.downcase || @markup if (section, directive_line = directives['section']) # If comment has :section:, it is not a documentable comment for a code object - comment.text = extract_section_comment(comment_text, directive_line - start_line) - @container.set_current_section(section, comment) + text = extract_section_comment(comment_text, directive_line - start_line) + emit({type: :section, title: section, text: text, start_line: start_line, format: format, + type_signature_lines: type_signature_lines, type_signature_line_no: type_signature_line_no}) return end - @preprocess.run_post_processes(comment, @container) - [comment, directives, type_signature_lines] + startdoc = apply_document_control_directive(directives) + {text: comment_text, start_line: start_line, format: format, directives: directives, startdoc: startdoc, + type_signature_lines: type_signature_lines, type_signature_line_no: type_signature_line_no} end # Extracts the comment for this section from the normalized comment block. @@ -562,394 +483,137 @@ def syntax_highlighted_tokens(node) # Handles `public :foo, :bar` `private :foo, :bar` and `protected :foo, :bar` def change_method_visibility(names, visibility, singleton: @singleton) - new_methods = [] - @container.methods_matching(names, singleton) do |m| - if m.parent != @container - # A copy of an ancestor's method must not be documented - # in a :stopdoc:/:enddoc: region - next if document_suppressed? - m = m.dup - record_location(m) - new_methods << m - else - m.visibility = visibility - end - end - new_methods.each do |method| - case method - when RDoc::AnyMethod then - @container.add_method(method) - when RDoc::Attr then - @container.add_attribute(method) - end - method.visibility = visibility - end + emit({type: :change_method_visibility, names: names, visibility: visibility, singleton: singleton, + suppressed: document_suppressed?}) end # Handles `module_function :foo, :bar` def change_method_to_module_function(names) - @container.set_visibility_for(names, :private, false) - # In a :stopdoc:/:enddoc: region, the visibility of instance methods still - # changes but the singleton method copies must not be documented - return if document_suppressed? - - new_methods = [] - @container.methods_matching(names) do |m| - s_m = m.dup - record_location(s_m) - s_m.singleton = true - new_methods << s_m - end - new_methods.each do |method| - case method - when RDoc::AnyMethod then - @container.add_method(method) - when RDoc::Attr then - @container.add_attribute(method) - end - method.visibility = :public - end - end - - def handle_code_object_directives(code_object, directives) # :nodoc: - directives.each do |directive, (param)| - # startdoc/stopdoc/enddoc are handled by apply_document_control_directive. - # They control the lexical scope of the parser, not the code object. - next if directive == 'startdoc' || directive == 'stopdoc' || directive == 'enddoc' - @preprocess.handle_directive('', directive, param, code_object) - end + emit({type: :module_function, names: names, suppressed: document_suppressed?}) end # Handles `alias foo bar` and `alias_method :foo, :bar` def add_alias_method(old_name, new_name, line_no) - comment, directives = consecutive_comment(line_no) - apply_document_control_directive(directives) if directives - handle_code_object_directives(@container, directives) if directives - return if document_suppressed? - - visibility = @container.find_method(old_name, @singleton)&.visibility || :public - a = RDoc::Alias.new(old_name, new_name, comment, singleton: @singleton) - handle_modifier_directive(a, line_no) - a.store = @store - a.line = line_no - record_location(a) - if should_document?(a) - mark_container_documentable(@container) - @container.add_alias(a) - @container.find_method(new_name, @singleton)&.visibility = visibility - end + comment = consecutive_comment_payload(line_no) + emit({type: :alias_method, old_name: old_name, new_name: new_name, line_no: line_no, + singleton: @singleton, suppressed: document_suppressed?, + comment: comment, modifier_directives: modifier_directives(line_no)}) end # Handles `attr :a, :b`, `attr_reader :a, :b`, `attr_writer :a, :b` and `attr_accessor :a, :b` def add_attributes(names, rw, line_no) - comment, directives, type_signature_lines = consecutive_comment(line_no) - apply_document_control_directive(directives) if directives - handle_code_object_directives(@container, directives) if directives - return if document_suppressed? - return unless @container.document_children - - names.each do |symbol| - a = RDoc::Attr.new(symbol.to_s, rw, comment, singleton: @singleton) - a.store = @store - a.line = line_no - a.type_signature_lines = type_signature_lines - record_location(a) - handle_modifier_directive(a, line_no) - if should_document?(a) - @container.add_attribute(a) - mark_container_documentable(@container) - end - a.visibility = visibility # should set after adding to container - end - end - - # Adds includes/extends. Module name is resolved to full before adding. - - def add_includes_extends(names, rdoc_class, line_no) # :nodoc: - comment, directives = consecutive_comment(line_no) - apply_document_control_directive(directives) if directives - handle_code_object_directives(@container, directives) if directives - return if document_suppressed? - - mark_container_documentable(@container) - names.each do |name| - resolved_name = resolve_constant_path(name) - ie = @container.add(rdoc_class, resolved_name || name, '') - ie.store = @store - ie.line = line_no - ie.comment = comment - record_location(ie) - end + comment = consecutive_comment_payload(line_no) + emit({type: :attributes, names: names, rw: rw, line_no: line_no, singleton: @singleton, + visibility: @visibility, suppressed: document_suppressed?, + comment: comment, modifier_directives: modifier_directives(line_no)}) end # Handle `include Foo, Bar` def add_includes(names, line_no) # :nodoc: - add_includes_extends(names, RDoc::Include, line_no) + comment = consecutive_comment_payload(line_no) + emit({type: :include, names: names, line_no: line_no, suppressed: document_suppressed?, comment: comment}) end # Handle `extend Foo, Bar` def add_extends(names, line_no) # :nodoc: - add_includes_extends(names, RDoc::Extend, line_no) + comment = consecutive_comment_payload(line_no) + emit({type: :extend, names: names, line_no: line_no, suppressed: document_suppressed?, comment: comment}) end # Adds a method defined by `def` syntax def add_method(method_name, receiver_name:, receiver_fallback_type:, visibility:, singleton:, params:, calls_super:, block_params:, tokens:, start_line:, args_end_line:, end_line:) - comment, directives, type_signature_lines = consecutive_comment(start_line) - apply_document_control_directive(directives) if directives - handle_code_object_directives(@container, directives) if directives - # Resolve receiver after applying directives so that a namespace created - # here is marked as ignored when the comment starts a :stopdoc: region - receiver = receiver_name ? find_or_create_lexical_module_path(receiver_name, receiver_fallback_type) : @container - - internal_add_method( - method_name, - receiver, - comment: comment, - directives: directives, - modifier_comment_lines: [start_line, args_end_line, end_line].uniq, - line_no: start_line, - visibility: visibility, - singleton: singleton, - params: params, - calls_super: calls_super, - block_params: block_params, - tokens: tokens, - type_signature_lines: type_signature_lines - ) + comment = consecutive_comment_payload(start_line) + modifier_directives_list = [start_line, args_end_line, end_line].uniq.filter_map { |line| modifier_directives(line) } + emit({type: :method, name: method_name, receiver_name: receiver_name, receiver_fallback_type: receiver_fallback_type, + visibility: visibility, singleton: singleton, params: params, calls_super: calls_super, + block_params: block_params, tokens: tokens, line_no: start_line, suppressed: document_suppressed?, + comment: comment, modifier_directives_list: modifier_directives_list}) end - private def internal_add_method(method_name, container, comment:, dont_rename_initialize: false, directives:, modifier_comment_lines: nil, line_no:, visibility:, singleton:, params:, calls_super:, block_params:, tokens:, type_signature_lines: nil) # :nodoc: - meth = RDoc::AnyMethod.new(method_name, singleton: singleton) - meth.comment = comment - handle_code_object_directives(meth, directives) if directives - modifier_comment_lines&.each do |line| - handle_modifier_directive(meth, line) - end - return if document_suppressed? - return unless should_document?(meth) + # Adds a constant - mark_container_documentable(container) + def add_constant(constant_name, rhs_name, start_line, end_line, alias_path: nil) + comment = consecutive_comment_payload(start_line) + modifier_directives_list = [modifier_directives(start_line), modifier_directives(end_line)].compact + emit({type: :constant, constant_name: constant_name, rhs_name: rhs_name, line_no: start_line, + alias_path: alias_path, suppressed: document_suppressed?, + comment: comment, modifier_directives_list: modifier_directives_list}) + end - if directives && (call_seq, = directives['call-seq']) - meth.call_seq = call_seq.lines.map(&:chomp).reject(&:empty?).join("\n") if call_seq - end - meth.name ||= meth.call_seq[/\A[^()\s]+/] if meth.call_seq - meth.name ||= 'unknown' - meth.store = @store - meth.line = line_no - container.add_method(meth) # should add after setting singleton and before setting visibility - meth.visibility = visibility - meth.params ||= params || '()' - meth.calls_super = calls_super - meth.block_params ||= block_params if block_params - meth.type_signature_lines = type_signature_lines - record_location(meth) - meth.start_collecting_tokens(:ruby) - tokens.each do |token| - meth.token_stream << token - end + # Emits a scope-opening record for a module or class and returns its scope + # id, or nil if the body should not be visited - # Rename after add_method to register duplicated 'new' and 'initialize' - # defined in c and ruby. - if !dont_rename_initialize && method_name == 'initialize' && !singleton - if meth.dont_rename_initialize - meth.visibility = :protected - else - meth.name = 'new' - meth.singleton = true - meth.visibility = :public - end - end + def add_module_or_class(module_name, start_line, end_line, is_class: false, superclass_name: nil, superclass_expr: nil) + comment = consecutive_comment_payload(start_line) + modifier_directives_list = [modifier_directives(start_line), modifier_directives(end_line)].compact + id = (@scope_id += 1) + emit({type: :scope_open, id: id, kind: is_class ? :class : :module, name: module_name, + line_no: start_line, superclass_name: superclass_name, superclass_expr: superclass_expr, + suppressed: document_suppressed?, + comment: comment, modifier_directives_list: modifier_directives_list}) + # RDoc doesn't track constants of a singleton class, so a bare-named module + # or class inside `class << C` has no place to belong to + return if @singleton && !module_name.include?('::') + id end - # Find or create module or class from a given module name using Ruby lexical - # nesting. If module or class does not exist, creates a module or a class - # according to `create_mode` argument. - - def find_or_create_lexical_module_path(module_name, create_mode) - root_name, *path, name = module_name.split('::') - add_module = ->(mod, name, mode) { - created = - case mode - when :class - mod.add_class(RDoc::NormalClass, name, 'Object').tap { |m| m.store = @store } - when :module - mod.add_module(RDoc::NormalModule, name).tap { |m| m.store = @store } - end - # add_class/add_module may return an existing object created by another - # file (in_files is not empty then), which must not be ignored here. - # Documentable again when reopened or receiving contents outside the region. - created.ignore if document_suppressed? && created.in_files.empty? - created - } - if root_name.empty? - mod = @top_level - else - @module_nesting.reverse_each do |nesting, singleton| - next if singleton - mod = nesting.get_module_named(root_name) - break if mod - # If a constant is found and it is not a module or class, RDoc can't document about it. - # Return an anonymous module to avoid wrong document creation. - return RDoc::NormalModule.new(nil) if nesting.find_constant_named(root_name) - end - last_nesting, = @module_nesting.reverse_each.find { |_, singleton| !singleton } - return mod || add_module.call(last_nesting, root_name, create_mode) unless name - mod ||= add_module.call(last_nesting, root_name, :module) - end - path.each do |name| - mod = mod.get_module_named(name) || add_module.call(mod, name, :module) - end - mod.get_module_named(name) || add_module.call(mod, name, create_mode) + # Emits a scope-opening record for `class << (Name = Object.new)` and returns its scope id + + def singleton_scope_for_constant_write(name) + id = (@scope_id += 1) + emit({type: :singleton_scope_constant_write, id: id, name: name, suppressed: document_suppressed?}) + id end - # Resolves constant path to a full path by searching module nesting + # Emits a scope-opening record for `class << ConstantPath` and returns its scope id - def resolve_constant_path(constant_path) - owner_name, path = constant_path.split('::', 2) - return constant_path if owner_name.empty? # ::Foo, ::Foo::Bar - mod = nil - @module_nesting.reverse_each do |nesting, singleton| - next if singleton - mod = nesting.get_module_named(owner_name) - break if mod - end - mod ||= @top_level.get_module_named(owner_name) - [mod.full_name, path].compact.join('::') if mod + def singleton_scope_for_path(expression_name) + id = (@scope_id += 1) + emit({type: :singleton_scope_path, id: id, name: expression_name, suppressed: document_suppressed?}) + id end - # Returns a pair of owner module and constant name from a given constant path - # using Ruby lexical nesting. Creates owner module if it does not exist. - - def find_or_create_lexical_constant_owner_name(constant_path) - const_path, colon, name = constant_path.rpartition('::') - if colon.empty? # class Foo - # Within `class C` or `module C`, owner is C(== current container) - # Within `class <:nodoc: all region). Records inside such a scope are dropped, +# mirroring the traversal skip of the visitor before the IR split. + +class RDoc::Parser::Ruby::CodeObjectBuilder # :nodoc: + + def initialize(top_level, store, options, stats, preprocess, track_visibility:) + @top_level = top_level + @store = store + @options = options + @stats = stats + @preprocess = preprocess + @track_visibility = track_visibility + # Scope stack of [container, singleton] pairs representing Ruby lexical nesting + @scope_stack = [[top_level, false]] + # Scope id => resolved container (nil if the scope is not documentable) + @scopes = {} + @dead_scope_depth = 0 + end + + def run(records) + records.each do |record| + process(record) + end + end + + private + + def container + @scope_stack.last[0] + end + + def singleton? + @scope_stack.last[1] + end + + def process(record) + case record[:type] + when :scope_enter + enter_scope(record) + return + when :scope_exit + exit_scope + return + end + return if @dead_scope_depth > 0 + + case record[:type] + when :scope_open then process_scope_open(record) + when :singleton_scope_constant_write + # Accept `class << (NameErrorCheckers = Object.new)` as a module which is not actually a module + mod = container.add_module(RDoc::NormalModule, record[:name]) + mod.ignore if record[:suppressed] && mod.in_files.empty? + @scopes[record[:id]] = mod + when :singleton_scope_path + # If a constant_path does not exist, RDoc creates a module + @scopes[record[:id]] = resolved_container(record[:resolved_full_name], :module, record[:suppressed]) + when :singleton_scope_self + @scopes[record[:id]] = container + when :method then process_method(record) + when :constant then process_constant(record) + when :attributes then process_attributes(record) + when :include then process_include_extend(record, RDoc::Include) + when :extend then process_include_extend(record, RDoc::Extend) + when :alias_method then process_alias_method(record) + when :change_method_visibility + change_method_visibility(record[:names], record[:visibility], record[:singleton], record[:suppressed]) + when :module_function then change_method_to_module_function(record[:names], record[:suppressed]) + when :constant_visibility + container.set_constant_visibility_for(record[:names], record[:visibility]) + when :require then container.add_require(RDoc::Require.new(record[:name], nil)) + when :section then process_section(record) + when :container_directives then process_container_directives(record) + when :meta_comment then process_meta_comment(record) + when :tomdoc_comment then process_tomdoc_comment(record) + end + end + + def enter_scope(record) + if @dead_scope_depth > 0 || (mod = @scopes[record[:id]]).nil? + @dead_scope_depth += 1 + else + @scope_stack.push([mod, record[:singleton]]) + end + end + + def exit_scope + if @dead_scope_depth > 0 + @dead_scope_depth -= 1 + else + @scope_stack.pop + end + end + + # Builds an RDoc::Comment from a comment payload and returns + # +[comment, directives, type_signature_lines]+. Type signature validation + # and comment post-processes must run only when the consuming record is + # replayed in a live scope, so they happen here and not at emission time. + + def materialize_comment(payload) + return unless payload + if payload[:type_signature_lines] + warn_invalid_type_signature(payload[:type_signature_lines], payload[:type_signature_line_no]) + end + comment = RDoc::Comment.new(payload[:text], @top_level, :ruby) + comment.normalized = true + comment.line = payload[:start_line] + comment.format = payload[:format] + @preprocess.run_post_processes(comment, container) + if payload[:startdoc] && !container.ignored? + # Compatibility: `module Net #:nodoc:` followed by :stopdoc:/:startdoc: + # regions is a common pattern that expects :startdoc: to make the + # container documentable again. Containers ignored here were created + # in a suppressed region and need documentable contents to revive. + container.start_doc + container.force_documentation = true + end + [comment, payload[:directives], payload[:type_signature_lines]] + end + + def warn_invalid_type_signature(type_signature_lines, line_no) + type_signature_lines.each_with_index do |line, i| + next if RDoc::RbsHelper.valid_method_type?(line) + next if RDoc::RbsHelper.valid_type?(line) + @options.warn "#{@top_level.relative_name}:#{line_no + i}: invalid RBS type signature: #{line.inspect}" + end + end + + def handle_code_object_directives(code_object, directives) + directives.each do |directive, (param)| + # startdoc/stopdoc/enddoc are handled by apply_document_control_directive. + # They control the lexical scope of the parser, not the code object. + next if directive == 'startdoc' || directive == 'stopdoc' || directive == 'enddoc' + @preprocess.handle_directive('', directive, param, code_object) + end + end + + # Makes a container that was created inside a :stopdoc:/:enddoc: region + # (thus ignored) documentable again when it receives documentable contents + # outside the region, possibly from another file. + + def mark_container_documentable(container) + return if container.received_nodoc || !container.ignored? + record_location(container) + container.start_doc + mark_container_documentable(container.parent) if container.parent.is_a?(RDoc::ClassModule) + end + + # Restores the fresh-creation state of a namespace ghost the first time the + # build touches it. Ghosts are created ignored (which also stops + # documentation of self and children), while a namespace created during the + # build starts out documentable; directives of the touching record apply + # after this. A suppressed touch claims the ghost but keeps it ignored, + # like a namespace created in a :stopdoc: region: a later reference must + # not make it documentable. + + def materialize_ghost(mod, suppressed) + if mod.is_a?(RDoc::ClassModule) && mod.namespace_ghost + mod.namespace_ghost = false + mod.start_doc unless suppressed + end + mod + end + + # Methods, attributes and aliases at the top level are documented on + # Object, which the model looks up directly in the store, so its ghost + # must be materialized like a namespace the build touches + + def materialize_object_ghost_for(container) + materialize_ghost(@store.classes_hash['Object'], false) if container.is_a?(RDoc::TopLevel) + end + + def should_document?(code_object) + return true unless @track_visibility + return false if code_object.parent&.document_children == false + code_object.document_self + end + + # Records the location of this +code_object+ in the file for this parser and + # adds it to the list of classes and modules in the file. + + def record_location(code_object) + case code_object + when RDoc::ClassModule then + @top_level.add_to_classes_or_modules code_object + end + + code_object.record_location @top_level + end + + def process_scope_open(record) + comment, directives = materialize_comment(record[:comment]) + handle_code_object_directives(container, directives) if directives + return unless container.document_children + + suppressed = record[:suppressed] + resolved_full_name = record[:resolved_full_name] + return unless resolved_full_name + + owner_name, colon, name = resolved_full_name.rpartition('::') + owner = colon.empty? ? @top_level : resolved_container(owner_name, :module, suppressed) + + if record[:kind] == :class + superclass_name = record[:superclass_name] + superclass_full_path = record[:resolved_superclass] if superclass_name + # Context#add_class upgrades a module named as a superclass into a + # class. A declaration whose class already exists (as a ghost) skips + # add_class, so the upgrade is replicated here, including for the + # implicit Object superclass of `class A` + upgrade_path = superclass_name ? superclass_full_path : ('Object' unless record[:superclass_expr]) + if upgrade_path && (module_to_upgrade = @store.modules_hash.delete(upgrade_path)) + owner.upgrade_to_class module_to_upgrade, RDoc::NormalClass, module_to_upgrade.parent + end + # RDoc::NormalClass resolves superclass name despite of the lack of module nesting information. + # We need to fix it when RDoc::NormalClass resolved to a wrong constant name + if superclass_name + superclass = @store.find_class_or_module(superclass_full_path) if superclass_full_path + superclass_full_path ||= superclass_name + superclass_full_path = superclass_full_path.sub(/^::/, '') + end + # add_class should be done after resolving superclass + mod = owner.classes_hash[name] + if mod + materializes_ghost = mod.namespace_ghost && !suppressed + else + # add_class may return an existing class created by another file + # (in_files is not empty then), which must not be ignored here + mod = owner.add_class(RDoc::NormalClass, name, superclass_name || record[:superclass_expr] || '::Object') + mod.ignore if suppressed && mod.in_files.empty? + end + if superclass_name + if superclass + mod.superclass = superclass + elsif (mod.superclass.is_a?(String) || mod.superclass.name == 'Object') && mod.superclass != superclass_full_path + mod.superclass = superclass_full_path + end + elsif materializes_ghost && record[:superclass_expr] + # A ghost has the placeholder superclass Object; the first declaration + # provides the real superclass expression like a fresh creation + mod.superclass = record[:superclass_expr] + end + else + mod = owner.modules_hash[name] + unless mod + mod = owner.add_module(RDoc::NormalModule, name) + mod.ignore if suppressed && mod.in_files.empty? + end + end + + materialize_ghost(mod, suppressed) + mod.store = @store + mod.line = record[:line_no] + record[:modifier_directives_list].each do |modifier_directives| + handle_code_object_directives(mod, modifier_directives) + end + unless suppressed + # In a :stopdoc:/:enddoc: region, the container is still created as a + # namespace but is not recorded to this file nor documented. + # The body is also visited: an inner :startdoc: re-enables documentation + # in a :stopdoc: region (not in an :enddoc: region), and nested + # namespaces need to be created for later promotion from other files + if mod.ignored? + # Promotes the owner chain too, unless mod received :nodoc: + mark_container_documentable(mod) + else + # A class/module marked :nodoc: must not make an ignored owner documentable + mark_container_documentable(owner) if mod.document_self && owner.is_a?(RDoc::ClassModule) + record_location(mod) + end + mod.add_comment(comment, @top_level) if comment + end + @scopes[record[:id]] = mod + end + + def process_method(record) + comment, directives, type_signature_lines = materialize_comment(record[:comment]) + handle_code_object_directives(container, directives) if directives + # Resolve receiver after applying directives so that a namespace created + # here is marked as ignored when the comment starts a :stopdoc: region + receiver = record[:receiver_name] ? resolved_container(record[:resolved_receiver], record[:receiver_fallback_type], record[:suppressed]) : container + + internal_add_method( + record[:name], + receiver, + comment: comment, + directives: directives, + modifier_directives_list: record[:modifier_directives_list], + line_no: record[:line_no], + visibility: record[:visibility], + singleton: record[:singleton], + suppressed: record[:suppressed], + params: record[:params], + calls_super: record[:calls_super], + block_params: record[:block_params], + tokens: record[:tokens], + type_signature_lines: type_signature_lines + ) + end + + def internal_add_method(method_name, container, comment:, dont_rename_initialize: false, directives:, modifier_directives_list: nil, line_no:, visibility:, singleton:, suppressed:, params:, calls_super:, block_params:, tokens:, type_signature_lines: nil) + meth = RDoc::AnyMethod.new(method_name, singleton: singleton) + meth.comment = comment + handle_code_object_directives(meth, directives) if directives + modifier_directives_list&.each do |modifier_directives| + handle_code_object_directives(meth, modifier_directives) + end + return if suppressed + return unless should_document?(meth) + + mark_container_documentable(container) + + if directives && (call_seq, = directives['call-seq']) + meth.call_seq = call_seq.lines.map(&:chomp).reject(&:empty?).join("\n") if call_seq + end + meth.name ||= meth.call_seq[/\A[^()\s]+/] if meth.call_seq + meth.name ||= 'unknown' + meth.store = @store + meth.line = line_no + materialize_object_ghost_for(container) + container.add_method(meth) # should add after setting singleton and before setting visibility + meth.visibility = visibility + meth.params ||= params || '()' + meth.calls_super = calls_super + meth.block_params ||= block_params if block_params + meth.type_signature_lines = type_signature_lines + record_location(meth) + meth.start_collecting_tokens(:ruby) + tokens.each do |token| + meth.token_stream << token + end + + # Rename after add_method to register duplicated 'new' and 'initialize' + # defined in c and ruby. + if !dont_rename_initialize && method_name == 'initialize' && !singleton + if meth.dont_rename_initialize + meth.visibility = :protected + else + meth.name = 'new' + meth.singleton = true + meth.visibility = :public + end + end + end + + def process_constant(record) + comment, directives = materialize_comment(record[:comment]) + handle_code_object_directives(container, directives) if directives + return if record[:suppressed] + + _const_path, colon, name = record[:constant_name].rpartition('::') + if colon.empty? + # RDoc doesn't track constants of a singleton class of a module + owner = singleton? ? nil : container + else + owner = record[:resolved_owner] && resolved_container(record[:resolved_owner], :module, record[:suppressed]) + end + return unless owner + + constant = RDoc::Constant.new(name, record[:rhs_name], comment) + constant.store = @store + constant.line = record[:line_no] + alias_path = record[:alias_path] + constant.is_alias_for_path = alias_path + record[:modifier_directives_list].each do |modifier_directives| + handle_code_object_directives(constant, modifier_directives) + end + # A constant marked :nodoc: must not make an ignored owner documentable + mark_container_documentable(owner) if constant.document_self && owner.is_a?(RDoc::ClassModule) + record_location(constant) + owner.add_constant(constant) + return unless alias_path + mod = record[:resolved_alias_target] && @store.find_class_or_module(record[:resolved_alias_target]) + if mod && constant.document_self + a = owner.add_module_alias(mod, alias_path, constant, @top_level) + a.store = @store + a.line = record[:line_no] + record_location(a) + end + end + + def process_attributes(record) + comment, directives, type_signature_lines = materialize_comment(record[:comment]) + handle_code_object_directives(container, directives) if directives + return if record[:suppressed] + return unless container.document_children + + materialize_object_ghost_for(container) + record[:names].each do |name| + a = RDoc::Attr.new(name, record[:rw], comment, singleton: record[:singleton]) + a.store = @store + a.line = record[:line_no] + a.type_signature_lines = type_signature_lines + record_location(a) + handle_code_object_directives(a, record[:modifier_directives]) if record[:modifier_directives] + if should_document?(a) + container.add_attribute(a) + mark_container_documentable(container) + end + a.visibility = record[:visibility] # should set after adding to container + end + end + + def process_include_extend(record, rdoc_class) + comment, directives = materialize_comment(record[:comment]) + handle_code_object_directives(container, directives) if directives + return if record[:suppressed] + + mark_container_documentable(container) + record[:names].each_with_index do |name, i| + resolved_name = record[:resolved_names][i] + ie = container.add(rdoc_class, resolved_name || name, '') + ie.store = @store + ie.line = record[:line_no] + ie.comment = comment + record_location(ie) + end + end + + def process_alias_method(record) + comment, directives = materialize_comment(record[:comment]) + handle_code_object_directives(container, directives) if directives + return if record[:suppressed] + + materialize_object_ghost_for(container) + singleton = record[:singleton] + visibility = container.find_method(record[:old_name], singleton)&.visibility || :public + a = RDoc::Alias.new(record[:old_name], record[:new_name], comment, singleton: singleton) + handle_code_object_directives(a, record[:modifier_directives]) if record[:modifier_directives] + a.store = @store + a.line = record[:line_no] + record_location(a) + if should_document?(a) + mark_container_documentable(container) + container.add_alias(a) + container.find_method(record[:new_name], singleton)&.visibility = visibility + end + end + + # Handles `public :foo, :bar` `private :foo, :bar` and `protected :foo, :bar` + + def change_method_visibility(names, visibility, singleton, suppressed) + materialize_object_ghost_for(container) + new_methods = [] + container.methods_matching(names, singleton) do |m| + if m.parent != container + # A copy of an ancestor's method must not be documented + # in a :stopdoc:/:enddoc: region + next if suppressed + m = m.dup + record_location(m) + new_methods << m + else + m.visibility = visibility + end + end + new_methods.each do |method| + case method + when RDoc::AnyMethod then + container.add_method(method) + when RDoc::Attr then + container.add_attribute(method) + end + method.visibility = visibility + end + end + + # Handles `module_function :foo, :bar` + + def change_method_to_module_function(names, suppressed) + materialize_object_ghost_for(container) + container.set_visibility_for(names, :private, false) + # In a :stopdoc:/:enddoc: region, the visibility of instance methods still + # changes but the singleton method copies must not be documented + return if suppressed + + new_methods = [] + container.methods_matching(names) do |m| + s_m = m.dup + record_location(s_m) + s_m.singleton = true + new_methods << s_m + end + new_methods.each do |method| + case method + when RDoc::AnyMethod then + container.add_method(method) + when RDoc::Attr then + container.add_attribute(method) + end + method.visibility = :public + end + end + + def process_section(record) + if record[:type_signature_lines] + warn_invalid_type_signature(record[:type_signature_lines], record[:type_signature_line_no]) + end + comment = RDoc::Comment.new(record[:text], @top_level, :ruby) + comment.normalized = true + comment.line = record[:start_line] + comment.format = record[:format] + container.set_current_section(record[:title], comment) + end + + def process_container_directives(record) + _comment, directives = materialize_comment(record[:comment]) + handle_code_object_directives(container, directives) + end + + # Handles meta method comments + + def process_meta_comment(record) + comment, directives = materialize_comment(record[:comment]) + handle_code_object_directives(container, directives) + node = record[:node] + singleton_method = false + visibility = record[:visibility] + attributes = rw = line_no = method_name = nil + directives.each do |directive, (param, line)| + case directive + when 'attr', 'attr_reader', 'attr_writer', 'attr_accessor' + attributes = [param] if param + attributes ||= node[:name_arguments] if node&.[](:is_call_node) + rw = directive == 'attr_writer' ? 'W' : directive == 'attr_accessor' ? 'RW' : 'R' + when 'method' + method_name = param if param + line_no = line + when 'singleton-method' + method_name = param if param + line_no = line + singleton_method = true + visibility = :public + end + end + + return if record[:suppressed] + + materialize_object_ghost_for(container) + if attributes + attributes.each do |attr| + a = RDoc::Attr.new(attr, rw, comment, singleton: record[:singleton]) + a.store = @store + a.line = line_no + record_location(a) + container.add_attribute(a) + mark_container_documentable(container) + a.visibility = visibility + end + elsif line_no || node + method_name ||= node[:name_arguments].first if node&.[](:is_call_node) + if node + tokens = node[:tokens] + line_no = node[:start_line] + else + tokens = [] + end + internal_add_method( + method_name, + container, + comment: comment, + directives: directives, + dont_rename_initialize: false, + line_no: line_no, + visibility: visibility, + singleton: record[:singleton] || singleton_method, + suppressed: record[:suppressed], + params: nil, + calls_super: false, + block_params: nil, + tokens: tokens, + ) + end + end + + # Creates an RDoc::Method if there is a Signature section in the tomdoc comment + + def process_tomdoc_comment(record) + comment = RDoc::Comment.new(record[:text], @top_level, :ruby) + comment.format = 'tomdoc' + parse_comment_tomdoc(comment, record[:start_line], record[:tokens]) unless record[:suppressed] + @preprocess.run_post_processes(comment, container) + end + + def parse_comment_tomdoc(comment, start_line, tokens) + return unless signature = RDoc::TomDoc.signature(comment) + + name, = signature.split %r%[ \(]%, 2 + + meth = RDoc::AnyMethod.new name + record_location(meth) + meth.line = start_line + meth.call_seq = signature + return unless meth.name + + meth.start_collecting_tokens(:ruby) + tokens.each { |token| meth.token_stream << token } + + container.add_method meth + meth.comment = comment + @stats.add_method meth + end + + # Returns the container object for a full name resolved by + # NamespaceResolver, materializing ghosts and creating not-yet-existing + # namespaces along the path. A nil full name maps to an anonymous module so + # that contents of an unresolvable declaration are not documented. + # The empty name maps to the top level. + + def resolved_container(full_name, create_mode, suppressed) + return RDoc::NormalModule.new(nil) unless full_name + return @top_level if full_name.empty? + + parent_name, colon, name = full_name.rpartition('::') + parent = colon.empty? ? @top_level : resolved_container(parent_name, :module, suppressed) + mod = materialize_ghost(parent.get_module_named(name), suppressed) + return mod if mod + + created = + case create_mode + when :class + parent.add_class(RDoc::NormalClass, name, 'Object') + when :module + parent.add_module(RDoc::NormalModule, name) + end + created.store = @store + # A namespace created in a suppressed region becomes documentable again + # when reopened or receiving contents outside the region. An existing + # object created by another file (in_files is not empty) must not be + # ignored here. + created.ignore if suppressed && created.in_files.empty? + created + end +end diff --git a/lib/rdoc/parser/ruby_namespace_resolver.rb b/lib/rdoc/parser/ruby_namespace_resolver.rb new file mode 100644 index 0000000000..00ceedef8c --- /dev/null +++ b/lib/rdoc/parser/ruby_namespace_resolver.rb @@ -0,0 +1,369 @@ +# frozen_string_literal: true + +## +# Resolves the namespaces declared by the IR of all Ruby files in a batch and +# creates them in the store before any file is built, so that name resolution +# during the build sees the declarations of all files instead of only the +# files built so far. +# +# Resolution is a pure function of the declaration table: it iterates the +# per-file lexical simulation until the table reaches a fixed point, because +# resolving one declaration's owner can depend on the names another +# declaration introduces (shadowing). +# +# An implicit namespace - the owner of `class B::C` when no B is declared +# anywhere - is never invented while a real declaration could resolve the +# name: in real Ruby such code raises NameError unless something defined B +# first, so preferring a real declaration matches every load order that +# works. Nested undeclared roots stay pending until the table is stable and +# are then pinned as implicit namespaces, outermost first. +# +# Namespaces are created as ignored ghosts with no recorded location. +# A ghost that receives documentable contents during the build is revived by +# the same mechanism that revives namespaces created inside a :stopdoc: +# region; a ghost that receives nothing stays out of the documentation. +# Unlike the build, the solver intentionally ignores documentation +# suppression (:nodoc: all regions and :enddoc:): those declarations still +# define real Ruby constants, so they participate in name resolution. + +class RDoc::Parser::Ruby::NamespaceResolver # :nodoc: + + # Pass count backstop for adversarially nested inputs. Real code needs two + # or three passes to converge, plus one pass per nesting depth of pinned + # implicit namespaces. + MAX_PASSES = 10 + + def initialize(store) + @store = store + end + + def preload_namespaces(parsers) + return if parsers.empty? + irs = parsers.map(&:ir) + namespaces, aliases = solve(irs) + create_ghosts(namespaces, aliases, parsers.first.top_level) + end + + private + + # The derived table is rebuilt every pass (a resolution may move when a + # name declared elsewhere becomes visible), while pins are kept: an + # implicit namespace is a decision, not a derivation, so re-deriving it + # would make the iteration oscillate. + + def solve(irs) + pins = {} + namespaces = {} + constants = {} + aliases = {} + MAX_PASSES.times do + pass = Pass.new(@store, namespaces, constants, aliases, pins) + irs.each { |ir| pass.simulate(ir) } + converged = namespaces == pass.namespaces && constants == pass.constants && aliases == pass.aliases + namespaces = pass.namespaces + constants = pass.constants + aliases = pass.aliases + if converged + break if pass.pending.empty? + pin_shallowest_pending(pins, pass.pending) + end + end + namespaces = pins.merge(namespaces) { |_full_name, pin_kind, kind| Pass.merge_kind(pin_kind, kind) } + [namespaces, aliases] + end + + # Pins the pending implicit namespaces at the shallowest depth, all at + # once. A pending root has no real declaration anywhere in its frames (the + # table is stable), so an outer implicit namespace can never hide a real + # name, only let deeper pendings resolve to it instead of inventing their + # own. Batching per depth bounds the pass count by the nesting depth + # instead of the number of pendings. + + def pin_shallowest_pending(pins, pending) + min_depth = pending.keys.map { |full_name| full_name.count(':') }.min + pending.each do |full_name, kind| + pins[full_name] = kind if full_name.count(':') == min_depth + end + end + + # Creates not-yet-existing namespaces in depth order so that parents are + # created before children. Names whose parent chain goes through an alias + # are skipped: their identity is provided by the module alias registration + # during the build. + + def create_ghosts(namespaces, aliases, top_level) + # The index tiebreak makes the depth sort stable: same-depth ghosts are + # created in declaration order regardless of the sort algorithm + namespaces.keys.sort_by.with_index { |full_name, index| [full_name.count(':'), index] }.each do |full_name| + next if aliases.include?(full_name) + next if @store.find_class_or_module(full_name) + + parent_name, colon, name = full_name.rpartition('::') + if colon.empty? + parent = top_level + else + parent = @store.find_class_or_module(parent_name) + next unless parent + end + + ghost = + if namespaces[full_name] == :class + parent.add_class(RDoc::NormalClass, name, 'Object') + else + # A namespace whose kind never became known falls back to a module + parent.add_module(RDoc::NormalModule, name) + end + ghost.store = @store + ghost.ignore + ghost.namespace_ghost = true + end + end + + ## + # One fixpoint pass: simulates the lexical scopes of every file and + # accumulates the resolved names of all declarations. Lookups consult the + # previous pass's table, the store (for namespaces of non-Ruby parsers) + # and the names accumulated so far in this pass. + + class Pass # :nodoc: + # How certain the kind of a namespace is: :unknown is promoted by either + # concrete kind. A class declaration wins over a conflicting module + # declaration of the same name, matching Context#add_class. + KIND_STRENGTH = { unknown: 0, module: 1, class: 2 }.freeze + + def self.merge_kind(kind, other) + return other unless kind + return kind unless other + KIND_STRENGTH[kind] >= KIND_STRENGTH[other] ? kind : other + end + + attr_reader :namespaces, :constants, :aliases, :pending + + def initialize(store, prev_namespaces, prev_constants, prev_aliases, pins) + @store = store + @prev_namespaces = prev_namespaces + @prev_constants = prev_constants + @prev_aliases = prev_aliases + @pins = pins + @namespaces = {} + @constants = {} + @aliases = {} + # Roots of nested declarations that no real declaration resolves, + # candidates for pinning as implicit namespaces + @pending = {} + end + + # Simulates the lexical scopes of one file, accumulating declared names + # and annotating each record with its resolved names. Records are + # annotated on every pass; the fixpoint loop stops after a pass whose + # lookup table equals its input, so the last annotations are the + # converged ones. + # Every resolved_* annotation is a full name String (or nil when + # unresolvable) - the IR never references CodeObjects. + + def simulate(ir) + # Frames are [full_name or nil, singleton]. A nil full_name marks an + # unresolvable subtree (constant shadowing) whose declarations don't + # introduce store-visible names, like the anonymous-module subtree of + # the build. + frames = [['', false]] + scopes = {} + ir.each do |record| + case record[:type] + when :scope_enter + frames.push([scopes[record[:id]], record[:singleton]]) + when :scope_exit + frames.pop + when :scope_open + resolved = declare_constant_owner_path(record[:name], record[:kind], frames) + record[:resolved_full_name] = resolved + if record[:superclass_name] + # `class Cipher < Cipher` must not resolve the right-hand side to + # the class the clause is defining + record[:resolved_superclass] = resolve_reference(record[:superclass_name], frames, exclude: resolved) + end + scopes[record[:id]] = resolved + when :singleton_scope_path + scopes[record[:id]] = record[:resolved_full_name] = declare_module_path(record[:name], :unknown, frames) + when :singleton_scope_constant_write + frame_full_name, = frames.last + if frame_full_name + full_name = child_name(frame_full_name, record[:name]) + register_namespace(full_name, :unknown) + scopes[record[:id]] = full_name + else + scopes[record[:id]] = nil + end + when :method + if record[:receiver_name] + # The :module fallback of a constant path receiver is a guess, + # unlike the :class of the NilClass/TrueClass/FalseClass receivers + kind = record[:receiver_fallback_type] == :class ? :class : :unknown + record[:resolved_receiver] = declare_module_path(record[:receiver_name], kind, frames) + end + when :constant + declare_constant(record, frames) + when :include, :extend + record[:resolved_names] = record[:names].map { |name| resolve_reference(name, frames) } + end + end + end + + private + + def child_name(parent_full_name, name) + parent_full_name.empty? ? name : "#{parent_full_name}::#{name}" + end + + def namespace_kind(full_name) + @namespaces[full_name] || @prev_namespaces[full_name] || @aliases[full_name] || @prev_aliases[full_name] || + @pins[full_name] || + (:class if @store.classes_hash[full_name]) || (:module if @store.modules_hash[full_name]) + end + + def constant?(full_name) + @constants[full_name] || @prev_constants[full_name] || !@store.find_class_or_module(full_name) && store_constant?(full_name) + end + + def store_constant?(full_name) + owner_name, colon, name = full_name.rpartition('::') + return false if colon.empty? # top level constants belong to Object, which the Ruby parser does not consult + owner = @store.find_class_or_module(owner_name) + owner ? !owner.constants_hash[name].nil? : false + end + + def register_namespace(full_name, kind) + @namespaces[full_name] = self.class.merge_kind(namespace_kind(full_name), kind) + end + + # Resolves a module path with Ruby lexical nesting semantics and returns + # its full name, declaring the not-yet-known parts: the innermost + # non-singleton frame that has the root name wins, a same-named + # non-namespace constant makes the path unresolvable (nil), and an + # unknown root is declared at the innermost frame. + + # The names this path derives (the root of a top-level declaration and + # everything below the root) are re-registered on every pass, whether + # already known or not: the derived table is rebuilt per pass, so an + # entry stays alive only while some record still derives it. Lookups + # decide only which resolution is derived. + + def declare_module_path(module_name, create_mode, frames) + root_name, *path, name = module_name.split('::') + if root_name.empty? + current = '' + else + root_kind = name ? :unknown : create_mode + innermost_full_name, = frames.reverse_each.find { |_, singleton| !singleton } + return nil if innermost_full_name.nil? + if innermost_full_name.empty? + # A top-level declaration resolves its root to ::root whether it is + # declared or not, so the outcome is independent of the table. + # A same-named non-namespace constant still makes it unresolvable. + return nil if !namespace_kind(root_name) && constant?(root_name) + register_namespace(root_name, namespace_kind(root_name) ? :unknown : root_kind) + current = root_name + else + found = nil + frames.reverse_each do |full_name, singleton| + # An unresolvable (nil) frame corresponds to the build's anonymous + # module: lookups pass through it to outer frames, but a name + # created inside it does not become store-visible + next if singleton || full_name.nil? + candidate = child_name(full_name, root_name) + if namespace_kind(candidate) + found = candidate + break + end + # If a constant is found and it is not a module or class, the + # declaration cannot be resolved + return nil if constant?(candidate) + end + unless found + # A nested undeclared root must not be invented while a real + # declaration could still resolve it; it stays pending until the + # table is stable and is then pinned by the solve loop + pending_name = child_name(innermost_full_name, root_name) + @pending[pending_name] = self.class.merge_kind(@pending[pending_name], root_kind) + return nil + end + current = found + end + return current unless name + end + path.each do |part| + candidate = child_name(current, part) + register_namespace(candidate, :unknown) + current = candidate + end + candidate = child_name(current, name) + register_namespace(candidate, namespace_kind(candidate) ? :unknown : create_mode) + candidate + end + + # Returns the resolved [owner full name, name] pair of a constant path. + # The owner of a bare name is the current frame, except in a singleton + # class, whose constants RDoc does not track. + + def resolve_constant_owner(constant_path, frames) + const_path, colon, name = constant_path.rpartition('::') + if colon.empty? + full_name, singleton = frames.last + singleton ? nil : full_name && [full_name, name] + elsif const_path.empty? + ['', name] + else + owner = declare_module_path(const_path, :unknown, frames) + owner && [owner, name] + end + end + + def declare_constant_owner_path(constant_path, kind, frames) + owner, name = resolve_constant_owner(constant_path, frames) + return nil unless owner + full_name = child_name(owner, name) + register_namespace(full_name, kind) + full_name + end + + def declare_constant(record, frames) + # The owner of a bare-named constant is the current container object of + # the build (a specific TopLevel or an anonymous module), so only + # `A::B = ...` forms are annotated with a resolved owner + record[:resolved_owner] = nil + owner, name = resolve_constant_owner(record[:constant_name], frames) + return unless owner + record[:resolved_owner] = owner if record[:constant_name].include?('::') + full_name = child_name(owner, name) + @constants[full_name] = true + alias_path = record[:alias_path] + return unless alias_path + target = resolve_reference(alias_path, frames) + record[:resolved_alias_target] = target + if target && (kind = namespace_kind(target)) + # A module alias name resolves like a namespace but is registered by + # the build, not created as a ghost + @aliases[full_name] = kind + end + end + + # Resolves a reference (an include/extend target, a superclass or a + # constant alias right-hand side) without creating anything + + def resolve_reference(constant_path, frames, exclude: nil) + root_name, path = constant_path.split('::', 2) + return constant_path.delete_prefix('::') if root_name.empty? + found = nil + frames.reverse_each do |full_name, singleton| + next if singleton || full_name.nil? + candidate = child_name(full_name, root_name) + if candidate != exclude && namespace_kind(candidate) + found = candidate + break + end + end + found = root_name if !found && root_name != exclude && namespace_kind(root_name) + found && (path ? "#{found}::#{path}" : found) + end + end +end diff --git a/lib/rdoc/rdoc.rb b/lib/rdoc/rdoc.rb index 0af5c86497..d76b1548f3 100644 --- a/lib/rdoc/rdoc.rb +++ b/lib/rdoc/rdoc.rb @@ -339,11 +339,16 @@ def parse_file(filename) return unless parser - parser.scan + if @deferred_ruby_builds && parser.is_a?(RDoc::Parser::Ruby) + parser.parse_ir + @deferred_ruby_builds << parser + else + parser.scan - # restart documentation for the classes & modules found - top_level.classes_or_modules.each do |cm| - cm.done_documenting = false + # restart documentation for the classes & modules found + top_level.classes_or_modules.each do |cm| + cm.done_documenting = false + end end top_level @@ -357,7 +362,13 @@ def parse_file(filename) this is not your library please report a bug to the author. EOF rescue => e - syntax_check_command = syntax_check_command_for filename, parser&.class + print_parse_error_hint filename, parser&.class, e + + raise e + end + + def print_parse_error_hint(filename, parser_class, error) # :nodoc: + syntax_check_command = syntax_check_command_for filename, parser_class syntax_check_message = if syntax_check_command <<~MESSAGE Before reporting this, could you check that the file you're documenting @@ -379,13 +390,11 @@ def parse_file(filename) The internal error was: -\t(#{e.class}) #{e.message} +\t(#{error.class}) #{error.message} EOF - $stderr.puts e.backtrace.join("\n\t") if $DEBUG_RDOC - - raise e + $stderr.puts error.backtrace.join("\n\t") if $DEBUG_RDOC end def syntax_check_command_for(filename, parser_class = RDoc::Parser.can_parse_by_name(filename)) @@ -420,6 +429,32 @@ def relative_path_for(filename) relative_path.to_s end + ## + # Builds CodeObjects from the IR of Ruby files parsed in this batch. + # All files are parsed before any Ruby file is built so that a later + # resolution phase can see declarations of all files. + + def build_deferred_ruby_files + parsers = @deferred_ruby_builds + @deferred_ruby_builds = nil + RDoc::Parser::Ruby::NamespaceResolver.new(@store).preload_namespaces(parsers) + parsers.each do |parser| + @current = parser.file_name + begin + top_level = parser.build_ir(resolve: false) + rescue => e + print_parse_error_hint parser.file_name, parser.class, e + + raise e + end + + # restart documentation for the classes & modules found + top_level.classes_or_modules.each do |cm| + cm.done_documenting = false + end + end + end + ## # Parse each file on the command line, recursively entering directories. @@ -433,11 +468,14 @@ def parse_files(files) original_options = @options.dup @stats.begin_adding + @deferred_ruby_builds = [] file_info = file_list.map do |filename| @current = filename parse_file filename end.compact + build_deferred_ruby_files + @store.resolve_c_superclasses @stats.done_adding diff --git a/test/rdoc/parser/ruby_test.rb b/test/rdoc/parser/ruby_test.rb index 17cd9ea8c0..ba9332f7b5 100644 --- a/test/rdoc/parser/ruby_test.rb +++ b/test/rdoc/parser/ruby_test.rb @@ -277,7 +277,10 @@ class A::C4 < A::B; end mod = @top_level.modules.first classes = mod.classes assert_equal ['A::B', 'A::C1', 'A::C2', 'A::C3', 'A::C4'], classes.map(&:full_name) - assert_equal ['A::B', 'A::B', 'A::A::B', 'A::B'], classes.drop(1).map(&:superclass).map(&:full_name) + # A::B written inside module A resolves to A::A::B: names resolve + # against the declarations of the whole file, so the position of + # `module A::A` does not matter + assert_equal ['A::A::B', 'A::B', 'A::A::B', 'A::B'], classes.drop(1).map(&:superclass).map(&:full_name) end def test_pseudo_recursive_superclass @@ -294,7 +297,10 @@ class Baz < Bar; end foo_klass = @store.find_class_named 'Foo::Bar::Foo' bar_klass = @store.find_class_named 'Foo::Bar::Bar' baz_klass = @store.find_class_named 'Foo::Bar::Baz' - assert_equal 'Foo::Bar', foo_klass.superclass.full_name + # Foo::Bar::Bar is declared below but visible: names resolve against the + # declarations of the whole file. Only the self reference of + # `class Bar < Bar` resolves to the outer Bar. + assert_equal 'Foo::Bar::Bar', foo_klass.superclass.full_name assert_equal 'Foo::Bar', bar_klass.superclass.full_name assert_equal 'Foo::Bar::Bar', baz_klass.superclass.full_name end @@ -709,8 +715,8 @@ module D::Baz; end class A; end class X < C; end RUBY - assert_equal ['A', 'C', 'X'], @top_level.classes.map(&:full_name) - assert_equal ['B', 'D'], @top_level.modules.map(&:full_name) + assert_equal ['A', 'C', 'X'], @top_level.classes.map(&:full_name).sort + assert_equal ['B', 'D'], @top_level.modules.map(&:full_name).sort end def test_parenthesized_cdecl @@ -1167,7 +1173,7 @@ class << Baz2 # :nodoc: RUBY modules = @store.all_modules - assert_equal ['A', 'A::Foo', 'Bar'], modules.map(&:full_name) + assert_equal ['A', 'A::Foo', 'Bar'], modules.map(&:full_name).sort end def test_singleton_class @@ -2440,7 +2446,7 @@ class D; end class E; end RUBY - assert_equal ['A', 'A::B', 'D'], @store.all_classes.reject(&:ignored?).map(&:full_name) + assert_equal ['A', 'A::B', 'D'], @store.all_classes.reject(&:ignored?).map(&:full_name).sort end def test_top_level_enddoc diff --git a/test/rdoc/rdoc_rdoc_test.rb b/test/rdoc/rdoc_rdoc_test.rb index 7da44a9979..3073e558d8 100644 --- a/test/rdoc/rdoc_rdoc_test.rb +++ b/test/rdoc/rdoc_rdoc_test.rb @@ -503,6 +503,136 @@ def test_normalized_file_list_with_skipping_tests_disabled assert_equal [@a, @spec_file, @test_file], files.sort end + def test_parse_files_resolves_names_order_independently + using_names = <<~RUBY + module M + class C < X + include I + end + end + RUBY + declaring_names = <<~RUBY + module M + class X + end + module I + end + end + RUBY + + # parse_files processes files in sorted name order, so swapping the + # contents swaps which file is parsed first + [[using_names, declaring_names], [declaring_names, using_names]].each do |a, b| + rdoc = RDoc::RDoc.new + rdoc.options = RDoc::Options.new + rdoc.store = RDoc::Store.new(rdoc.options) + + temp_dir do + rdoc.options.root = Pathname(Dir.pwd) + File.write 'a.rb', a + File.write 'b.rb', b + + rdoc.parse_files %w[a.rb b.rb] + + klass = rdoc.store.find_class_named 'M::C' + assert_equal 'M::X', klass.superclass.full_name + assert_equal %w[M::I], klass.includes.map(&:name) + end + end + end + + def test_parse_files_nodoc_reopened_class_keeps_other_files_documentation + # Pattern of ruby's mkmf.rb (class String # :nodoc:) and shellwords.rb + parse_files_fresh( + 'a.rb' => "class C # :nodoc:\n def hidden; end\nend\n", + 'b.rb' => "class C\n def visible; end\nend\n" + ) do |store| + klass = store.find_class_named 'C' + assert_equal %w[visible], klass.method_list.map(&:name) + assert klass.display? + end + end + + def test_parse_files_top_level_method_with_reopened_object + parse_files_fresh( + 'a.rb' => "class Object\nend\n", + 'b.rb' => "def top_method\nend\n" + ) do |store| + object = store.find_class_named 'Object' + assert_includes object.method_list.map(&:name), 'top_method' + end + end + + def test_parse_files_pseudo_recursive_superclass + # Pattern of OpenSSL::Cipher::Cipher: the superclass clause must not + # resolve to the class it is defining + parse_files_fresh( + 'a.rb' => "module Cipher\n class Cipher < Cipher\n end\nend\n" + ) do |store| + klass = store.find_class_named 'Cipher::Cipher' + # The outer Cipher module is upgraded to a class by being named as a + # superclass + assert_equal 'Cipher', klass.superclass.full_name + klass.ancestors # must terminate + end + end + + def test_parse_files_superclass_expression_of_reopened_class + parse_files_fresh( + 'a.rb' => "module M\n class C < Struct.new(:x)\n end\nend\n", + 'b.rb' => "module M\n class C\n def foo; end\n end\nend\n" + ) do |store| + assert_equal 'Struct.new(:x)', store.find_class_named('M::C').superclass + end + end + + def test_parse_files_prefers_real_declaration_over_implicit_namespace + # `class B::C` nested in A must share the top-level B instead of + # inventing A::B, regardless of the file order + nested = "class A\n class B::C\n end\nend\n" + top = "module B\n class C\n end\nend\n" + + [[nested, top], [top, nested]].each do |a, b| + parse_files_fresh('a.rb' => a, 'b.rb' => b) do |store| + assert_equal %w[A B B::C], store.all_classes_and_modules.map(&:full_name).sort + end + end + end + + def test_parse_files_prefers_real_declaration_from_pathed_top_level + # The top-level `class B::C` implies ::B whether B is declared or not, so + # the nested B::C must resolve to it + parse_files_fresh( + 'a.rb' => "class A\n class B::C\n end\nend\n", + 'b.rb' => "class B::C\nend\n" + ) do |store| + assert_equal %w[A B B::C], store.all_classes_and_modules.map(&:full_name).sort + end + end + + def test_parse_files_shares_outer_implicit_namespace + # No X is declared anywhere: one implicit M::X is invented at the + # outermost undeclared scope and shared by the deeper X::Z + parse_files_fresh( + 'a.rb' => "module M\n class X::Y\n end\nend\n", + 'b.rb' => "module M\n module N\n class X::Z\n end\n end\nend\n" + ) do |store| + assert_equal %w[M M::N M::X M::X::Y M::X::Z], store.all_classes_and_modules.map(&:full_name).sort + end + end + + def parse_files_fresh(files) + temp_dir do + rdoc = RDoc::RDoc.new + rdoc.options = RDoc::Options.new + rdoc.options.root = Pathname(Dir.pwd) + rdoc.store = RDoc::Store.new(rdoc.options) + files.each { |name, content| File.write name, content } + rdoc.parse_files files.keys + yield rdoc.store + end + end + def test_parse_file @rdoc.store = RDoc::Store.new(@options)