Commit f770ac1

mo khan <mo.khan@gmail.com>
2020-06-06 06:48:31
Process each dependency in background thread
1 parent b2f5036
Changed files (7)
exe/spandx
@@ -4,7 +4,6 @@
 require 'spandx'
 
 Signal.trap('INT') do
-  warn("\n#{caller.join("\n")}: interrupted")
   exit(1)
 end
 
lib/spandx/cli/commands/scan.rb
@@ -4,49 +4,69 @@ module Spandx
   module Cli
     module Commands
       class Scan
-        attr_reader :scan_path, :spinner
+        attr_reader :scan_path
 
         def initialize(scan_path, options)
           @scan_path = ::Pathname.new(scan_path)
           @options = options
-          @spinner = options[:show_progress] ? ::Spandx::Core::Spinner.new : ::Spandx::Core::Spinner::NULL
           require(options[:require]) if options[:require]
         end
 
         def execute(output: $stdout)
-          printer = ::Spandx::Core::Printer.for(@options[:format])
-          printer.print_header(output)
-          pool = Concurrent::FixedThreadPool.new(Etc.nprocessors)
-          each_file do |file|
-            spinner.spin(file)
-            ::Spandx::Core::Parser.parse(file).each do |dependency|
-              pool.post do
-                printer.print_line(enhance(dependency), output)
+          with_printer(output) do |printer|
+            with_thread_pool(size: thread_count) do |thread|
+              each_dependency do |file, dependency|
+                thread.run(file, dependency, output) do |x, y|
+                  printer.print_line(enhance(x), y)
+                end
               end
             end
           end
-          pool.shutdown
-          pool.wait_for_termination
-          spinner.stop
-          printer.print_footer(output)
         end
 
         private
 
+        def thread_count
+          count = @options[:threads].to_i
+          count.positive? ? count : Etc.nprocessors
+        end
+
         def each_file
           Spandx::Core::PathTraversal
             .new(scan_path, recursive: @options[:recursive])
             .each { |file| yield file }
         end
 
+        def each_dependency
+          each_file do |file|
+            ::Spandx::Core::Parser.parse(file).each do |dependency|
+              yield file, dependency
+            end
+          end
+        end
+
         def format(output)
           Array(output).map(&:to_s)
         end
 
         def enhance(dependency)
-          ::Spandx::Core::Plugin
-            .all
-            .inject(dependency) { |memo, plugin| plugin.enhance(memo) }
+          ::Spandx::Core::Plugin.all.inject(dependency) do |memo, plugin|
+            plugin.enhance(memo)
+          end
+        end
+
+        def with_printer(output)
+          printer = ::Spandx::Core::Printer.for(@options[:format])
+          printer.print_header(output)
+          yield printer
+        ensure
+          printer.print_footer(output)
+        end
+
+        def with_thread_pool(size:)
+          ::Spandx::Core::ThreadPool.open(size: size, show_progress: @options[:show_progress]) do |pool|
+            yield pool
+          end
         end
       end
     end
lib/spandx/core/thread_pool.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class ThreadPool
+      def initialize(size: Etc.nprocessors, show_progress: true)
+        @size = size
+        @queue = Queue.new
+        @pool = size.times.map { start_worker_thread(@queue) }
+        @spinner = show_progress ? Spinner.new : Spinner::NULL
+      end
+
+      def run(message, *args, &block)
+        @queue.enq([message, block, args])
+      end
+
+      def done?
+        @queue.empty?
+      end
+
+      def shutdown
+        @size.times do
+          run('Bye') { throw :exit }
+        end
+
+        @pool.map(&:join)
+        @spinner.stop
+      end
+
+      def self.open(*args)
+        pool = new(*args)
+        yield pool
+      ensure
+        pool.shutdown
+      end
+
+      private
+
+      def start_worker_thread(queue)
+        Thread.new(queue) do |q|
+          catch(:exit) do
+            loop do
+              message, job, args = q.deq
+              @spinner.spin(message)
+              job.call(*args)
+            end
+          end
+        end
+      end
+    end
+  end
+end
lib/spandx.rb
@@ -2,7 +2,6 @@
 
 require 'addressable/uri'
 require 'bundler'
-require 'concurrent'
 require 'csv'
 require 'etc'
 require 'forwardable'
spec/unit/ruby/parsers/gemfile_lock_spec.rb
@@ -51,7 +51,7 @@ RSpec.describe Spandx::Ruby::Parsers::GemfileLock do
           build('dotenv', '2.7.5', path),
           build('faraday', '1.0.1', path),
           build('hashdiff', '1.0.1', path),
-          build('licensed', '2.10.0', path),
+          build('licensed', '2.11.0', path),
           build('licensee', '9.14.0', path),
           build('mini_portile2', '2.4.0', path),
           build('multipart-post', '2.1.1', path),
@@ -61,13 +61,14 @@ RSpec.describe Spandx::Ruby::Parsers::GemfileLock do
           build('octokit', '4.18.0', path),
           build('oj', '3.10.6', path),
           build('parallel', '1.19.1', path),
-          build('parser', '2.7.1.2', path),
+          build('parser', '2.7.1.3', path),
           build('parslet', '2.0.0', path),
           build('pathname-common_prefix', '0.0.1', path),
           build('public_suffix', '4.0.5', path),
           build('rainbow', '3.0.0', path),
           build('rake', '13.0.1', path),
           build('rake-compiler', '1.1.0', path),
+          build('regexp_parser', '1.7.0', path),
           build('reverse_markdown', '1.4.0', path),
           build('rexml', '3.2.4', path),
           build('rspec', '3.9.0', path),
@@ -76,7 +77,8 @@ RSpec.describe Spandx::Ruby::Parsers::GemfileLock do
           build('rspec-expectations', '3.9.2', path),
           build('rspec-mocks', '3.9.1', path),
           build('rspec-support', '3.9.3', path),
-          build('rubocop', '0.83.0', path),
+          build('rubocop', '0.85.0', path),
+          build('rubocop-ast', '0.0.3', path),
           build('rubocop-rspec', '1.39.0', path),
           build('ruby-prof', '1.4.1', path),
           build('ruby-progressbar', '1.10.1', path),
@@ -88,7 +90,7 @@ RSpec.describe Spandx::Ruby::Parsers::GemfileLock do
           build('terminal-table', '1.8.0', path),
           build('thor', '0.20.3', path),
           build('tomlrb', '1.3.0', path),
-          build('tty-screen', '0.7.1', path),
+          build('tty-screen', '0.8.0', path),
           build('unicode-display_width', '1.7.0', path),
           build('vcr', '5.1.0', path),
           build('webmock', '3.8.3', path),
Gemfile.lock
@@ -4,7 +4,6 @@ PATH
     spandx (0.13.4)
       addressable (~> 2.7)
       bundler (>= 1.16, < 3.0.0)
-      concurrent-ruby (~> 1.1)
       nanospinner (~> 1.0.0)
       net-hippie (~> 0.3)
       nokogiri (~> 1.10)
@@ -29,7 +28,6 @@ GEM
       bundler (>= 1.2.0, < 3)
       thor (~> 0.18)
     byebug (11.1.3)
-    concurrent-ruby (1.1.6)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.3)
@@ -37,7 +35,7 @@ GEM
     faraday (1.0.1)
       multipart-post (>= 1.2, < 3)
     hashdiff (1.0.1)
-    licensed (2.10.0)
+    licensed (2.11.0)
       bundler (>= 1.10)
       licensee (>= 9.14.0, < 10.0.0)
       parallel (>= 0.18.0)
@@ -63,7 +61,7 @@ GEM
       sawyer (~> 0.8.0, >= 0.5.3)
     oj (3.10.6)
     parallel (1.19.1)
-    parser (2.7.1.2)
+    parser (2.7.1.3)
       ast (~> 2.4.0)
     parslet (2.0.0)
     pathname-common_prefix (0.0.1)
@@ -72,6 +70,7 @@ GEM
     rake (13.0.1)
     rake-compiler (1.1.0)
       rake
+    regexp_parser (1.7.0)
     reverse_markdown (1.4.0)
       nokogiri
     rexml (3.2.4)
@@ -93,13 +92,17 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.3)
-    rubocop (0.83.0)
+    rubocop (0.85.0)
       parallel (~> 1.10)
       parser (>= 2.7.0.1)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
       rexml
+      rubocop-ast (>= 0.0.3)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.0.3)
+      parser (>= 2.7.0.1)
     rubocop-rspec (1.39.0)
       rubocop (>= 0.68.1)
     ruby-prof (1.4.1)
spandx.gemspec
@@ -34,7 +34,6 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'addressable', '~> 2.7'
   spec.add_dependency 'bundler', '>= 1.16', '< 3.0.0'
-  spec.add_dependency 'concurrent-ruby', '~> 1.1'
   spec.add_dependency 'nanospinner', '~> 1.0.0'
   spec.add_dependency 'net-hippie', '~> 0.3'
   spec.add_dependency 'nokogiri', '~> 1.10'