Merge pull request #180 from ruby-syntax-tree/search

stree search

Author: kddnewton

.rubocop.yml

@@ -28,6 +28,9 @@ Lint/InterpolationCheck:
 Lint/MissingSuper:
   Enabled: false
 
+Lint/RedundantRequireStatement:
+  Enabled: false
+
 Lint/UnusedMethodArgument:
   AllowUnusedKeywordArguments: true
 

Gemfile.lock

@@ -19,17 +19,17 @@ GEM
     rake (13.0.6)
     regexp_parser (2.6.0)
     rexml (3.2.5)
-    rubocop (1.37.0)
+    rubocop (1.37.1)
       json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.1.2.1)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.22.0, < 2.0)
+      rubocop-ast (>= 1.23.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.22.0)
+    rubocop-ast (1.23.0)
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
     simplecov (0.21.2)

README.md

@@ -18,6 +18,7 @@ It is built with only standard library dependencies. It additionally ships with
   - [format](#format)
   - [json](#json)
   - [match](#match)
+  - [search](#search)
   - [write](#write)
   - [Configuration](#configuration)
   - [Globbing](#globbing)
@@ -215,6 +216,29 @@ SyntaxTree::Program[
 ]
 ```
 
+### search
+
+This command will search the given filepaths against the specified pattern to find nodes that match. The pattern is a Ruby pattern-matching expression that is matched against each node in the tree. It can optionally be loaded from a file if you specify a filepath as the pattern argument.
+
+```sh
+stree search VarRef path/to/file.rb
+```
+
+For a file that contains `Foo + Bar` you will receive:
+
+```ruby
+path/to/file.rb:1:0: Foo + Bar
+path/to/file.rb:1:6: Foo + Bar
+```
+
+If you put `VarRef` into a file instead (for example, `query.txt`), you would instead run:
+
+```sh
+stree search query.txt path/to/file.rb
+```
+
+Note that the output of the `match` CLI command creates a valid pattern that can be used as the input for this command.
+
 ### write
 
 This command will format the listed files and write that formatted version back to the source files. Note that this overwrites the original content, to be sure to be using a version control system.

lib/syntax_tree.rb

@@ -21,6 +21,7 @@
 require_relative "syntax_tree/visitor/with_environment"
 
 require_relative "syntax_tree/parser"
+require_relative "syntax_tree/search"
 
 # Syntax Tree is a suite of tools built on top of the internal CRuby parser. It
 # provides the ability to generate a syntax tree from source, as well as the

lib/syntax_tree/cli.rb

@@ -212,6 +212,39 @@ def run(item)
       end
     end
 
+    # An action of the CLI that searches for the given pattern matching pattern
+    # in the given files.
+    class Search < Action
+      attr_reader :search
+
+      def initialize(query)
+        query = File.read(query) if File.readable?(query)
+        @search = SyntaxTree::Search.new(query)
+      rescue SyntaxTree::Search::UncompilableError => error
+        warn(error.message)
+        exit(1)
+      end
+
+      def run(item)
+        search.scan(item.handler.parse(item.source)) do |node|
+          location = node.location
+          line = location.start_line
+
+          bold_range =
+            if line == location.end_line
+              location.start_column...location.end_column
+            else
+              location.start_column..
+            end
+
+          source = item.source.lines[line - 1].chomp
+          source[bold_range] = Color.bold(source[bold_range]).to_s
+
+          puts("#{item.filepath}:#{line}:#{location.start_column}: #{source}")
+        end
+      end
+    end
+
     # An action of the CLI that formats the input source and writes the
     # formatted output back to the file.
     class Write < Action
@@ -263,6 +296,9 @@ def run(item)
       #{Color.bold("stree lsp [--plugins=...] [--print-width=NUMBER]")}
         Run syntax tree in language server mode
 
+      #{Color.bold("stree search PATTERN [-e SCRIPT] FILE")}
+        Search for the given pattern in the given files
+
       #{Color.bold("stree version")}
         Output the current version of syntax tree
 
@@ -400,6 +436,8 @@ def run(argv)
             Debug.new(options)
           when "doc"
             Doc.new(options)
+          when "f", "format"
+            Format.new(options)
           when "help"
             puts HELP
             return 0
@@ -411,8 +449,8 @@ def run(argv)
             return 0
           when "m", "match"
             Match.new(options)
-          when "f", "format"
-            Format.new(options)
+          when "s", "search"
+            Search.new(arguments.shift)
           when "version"
             puts SyntaxTree::VERSION
             return 0
@@ -434,7 +472,7 @@ def run(argv)
               .glob(pattern)
               .each do |filepath|
                 if File.readable?(filepath) &&
-                    options.ignore_files.none? { File.fnmatch?(_1, filepath) }
+                     options.ignore_files.none? { File.fnmatch?(_1, filepath) }
                   queue << FileItem.new(filepath)
                 end
               end

lib/syntax_tree/node.rb

@@ -1657,12 +1657,12 @@ class Binary < Node
     # for older Ruby versions.
     unless :+.respond_to?(:name)
       using Module.new {
-        refine Symbol do
-          def name
-            to_s.freeze
-          end
-        end
-      }
+              refine Symbol do
+                def name
+                  to_s.freeze
+                end
+              end
+            }
     end
 
     # [untyped] the left-hand side of the expression

lib/syntax_tree/search.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module SyntaxTree
+  # Provides an interface for searching for a pattern of nodes against a
+  # subtree of an AST.
+  class Search
+    class UncompilableError < StandardError
+    end
+
+    attr_reader :matcher
+
+    def initialize(query)
+      root = SyntaxTree.parse("case nil\nin #{query}\nend")
+      @matcher = compile(root.statements.body.first.consequent.pattern)
+    end
+
+    def scan(root)
+      return to_enum(__method__, root) unless block_given?
+      queue = [root]
+
+      until queue.empty?
+        node = queue.shift
+        next unless node
+
+        yield node if matcher.call(node)
+        queue += node.child_nodes
+      end
+    end
+
+    private
+
+    def compile(pattern)
+      case pattern
+      in Binary[left:, operator: :|, right:]
+        compiled_left = compile(left)
+        compiled_right = compile(right)
+
+        ->(node) { compiled_left.call(node) || compiled_right.call(node) }
+      in Const[value:] if SyntaxTree.const_defined?(value)
+        clazz = SyntaxTree.const_get(value)
+
+        ->(node) { node.is_a?(clazz) }
+      in Const[value:] if Object.const_defined?(value)
+        clazz = Object.const_get(value)
+
+        ->(node) { node.is_a?(clazz) }
+      in ConstPathRef[parent: VarRef[value: Const[value: "SyntaxTree"]]]
+        compile(pattern.constant)
+      in HshPtn[constant:, keywords:, keyword_rest: nil]
+        compiled_constant = compile(constant)
+
+        preprocessed_keywords =
+          keywords.to_h do |keyword, value|
+            raise NoMatchingPatternError unless keyword.is_a?(Label)
+            [keyword.value.chomp(":").to_sym, compile(value)]
+          end
+
+        compiled_keywords = ->(node) do
+          deconstructed = node.deconstruct_keys(preprocessed_keywords.keys)
+          preprocessed_keywords.all? do |keyword, matcher|
+            matcher.call(deconstructed[keyword])
+          end
+        end
+
+        ->(node) do
+          compiled_constant.call(node) && compiled_keywords.call(node)
+        end
+      in RegexpLiteral[parts: [TStringContent[value:]]]
+        regexp = /#{value}/
+
+        ->(attribute) { regexp.match?(attribute) }
+      in StringLiteral[parts: [TStringContent[value:]]]
+        ->(attribute) { attribute == value }
+      in VarRef[value: Const => value]
+        compile(value)
+      end
+    rescue NoMatchingPatternError
+      raise UncompilableError, <<~ERROR
+        Syntax Tree was unable to compile the pattern you provided to search
+        into a usable expression. It failed on the node within the pattern
+        matching expression represented by:
+
+        #{PP.pp(pattern, +"").chomp}
+
+        Note that not all syntax supported by Ruby's pattern matching syntax is
+        also supported by Syntax Tree's code search. If you're using some syntax
+        that you believe should be supported, please open an issue on the GitHub
+        repository at https://github.com/ruby-syntax-tree/syntax_tree.
+      ERROR
+    end
+  end
+end

test/cli_test.rb

@@ -94,6 +94,11 @@ def test_match
       assert_includes(result.stdio, "SyntaxTree::Program")
     end
 
+    def test_search
+      result = run_cli("search", "VarRef", contents: "Foo + Bar")
+      assert_equal(2, result.stdio.lines.length)
+    end
+
     def test_version
       result = run_cli("version")
       assert_includes(result.stdio, SyntaxTree::VERSION.to_s)

test/search_test.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "test_helper"
+
+module SyntaxTree
+  class SearchTest < Minitest::Test
+    def test_search_binary_or
+      root = SyntaxTree.parse("Foo + Bar + 1")
+      scanned = Search.new("VarRef | Int").scan(root).to_a
+
+      assert_equal 3, scanned.length
+      assert_equal "1", scanned.min_by { |node| node.class.name }.value
+    end
+
+    def test_search_const
+      root = SyntaxTree.parse("Foo + Bar + Baz")
+
+      scanned = Search.new("VarRef").scan(root).to_a
+
+      assert_equal 3, scanned.length
+      assert_equal %w[Bar Baz Foo], scanned.map { |node| node.value.value }.sort
+    end
+
+    def test_search_syntax_tree_const
+      root = SyntaxTree.parse("Foo + Bar + Baz")
+
+      scanned = Search.new("SyntaxTree::VarRef").scan(root).to_a
+
+      assert_equal 3, scanned.length
+    end
+
+    def test_search_hash_pattern_string
+      root = SyntaxTree.parse("Foo + Bar + Baz")
+
+      scanned = Search.new("VarRef[value: Const[value: 'Foo']]").scan(root).to_a
+
+      assert_equal 1, scanned.length
+      assert_equal "Foo", scanned.first.value.value
+    end
+
+    def test_search_hash_pattern_regexp
+      root = SyntaxTree.parse("Foo + Bar + Baz")
+
+      query = "VarRef[value: Const[value: /^Ba/]]"
+      scanned = Search.new(query).scan(root).to_a
+
+      assert_equal 2, scanned.length
+      assert_equal %w[Bar Baz], scanned.map { |node| node.value.value }.sort
+    end
+  end
+end