From b197a9a1e5e4c1ee8a15c635e95fcf1c30a91553 Mon Sep 17 00:00:00 2001 From: "Hal M. Spitz" Date: Fri, 24 Jan 2025 10:22:21 -0800 Subject: [PATCH 1/3] Corrects a minor bug in ScheduledExecution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ScheduledExecution#dispatch_next_batch was returning either an empty array or an integer count of the number of rows processed by the method. The client of this method is Worker and Dispatcher #poll, and poll requires the count of the number of rows processed. This commit changes the returned empty array to an integer value of 0 for consistency’s sake. It also includes small updates to the tests for these methods. --- app/models/solid_queue/scheduled_execution.rb | 3 ++- test/unit/dispatcher_test.rb | 5 ++++- test/unit/worker_test.rb | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/app/models/solid_queue/scheduled_execution.rb b/app/models/solid_queue/scheduled_execution.rb index 72aafd53..0f45626b 100644 --- a/app/models/solid_queue/scheduled_execution.rb +++ b/app/models/solid_queue/scheduled_execution.rb @@ -14,7 +14,8 @@ class << self def dispatch_next_batch(batch_size) transaction do job_ids = next_batch(batch_size).non_blocking_lock.pluck(:job_id) - if job_ids.empty? then [] + if job_ids.empty? + 0 else SolidQueue.instrument(:dispatch_scheduled, batch_size: batch_size) do |payload| payload[:size] = dispatch_jobs(job_ids) diff --git a/test/unit/dispatcher_test.rb b/test/unit/dispatcher_test.rb index 5bca7743..9aa2196e 100644 --- a/test/unit/dispatcher_test.rb +++ b/test/unit/dispatcher_test.rb @@ -96,6 +96,7 @@ class DispatcherTest < ActiveSupport::TestCase dispatcher = SolidQueue::Dispatcher.new(polling_interval: 10, batch_size: 1) dispatcher.expects(:interruptible_sleep).with(0.seconds).at_least(3) dispatcher.expects(:interruptible_sleep).with(dispatcher.polling_interval).at_least_once + dispatcher.expects(:handle_thread_error).never 3.times { AddToBufferJob.set(wait: 0.1).perform_later("I'm scheduled") } assert_equal 3, SolidQueue::ScheduledExecution.count @@ -112,8 +113,10 @@ class DispatcherTest < ActiveSupport::TestCase dispatcher = SolidQueue::Dispatcher.new(polling_interval: 10, batch_size: 1) dispatcher.expects(:interruptible_sleep).with(0.seconds).never dispatcher.expects(:interruptible_sleep).with(dispatcher.polling_interval).at_least_once + dispatcher.expects(:handle_thread_error).never + dispatcher.start - sleep 0.1 + wait_while_with_timeout(1.second) { !SolidQueue::ScheduledExecution.exists? } end private diff --git a/test/unit/worker_test.rb b/test/unit/worker_test.rb index 52b0d8e8..8db67912 100644 --- a/test/unit/worker_test.rb +++ b/test/unit/worker_test.rb @@ -176,6 +176,7 @@ class WorkerTest < ActiveSupport::TestCase @worker.expects(:interruptible_sleep).with(10.minutes).at_least_once @worker.expects(:interruptible_sleep).with(@worker.polling_interval).never + @worker.expects(:handle_thread_error).never @worker.start sleep 1.second @@ -186,6 +187,7 @@ class WorkerTest < ActiveSupport::TestCase @worker.expects(:interruptible_sleep).with(@worker.polling_interval).at_least_once @worker.expects(:interruptible_sleep).with(10.minutes).never + @worker.expects(:handle_thread_error).never @worker.start sleep 1.second From d82e39c1f98acb249c0ca5d7dab8f7dbb6356c8e Mon Sep 17 00:00:00 2001 From: "Hal M. Spitz" Date: Fri, 24 Jan 2025 10:30:46 -0800 Subject: [PATCH 2/3] Fixes issue #482: Work with < Rubies 3.2 The current Thread::Queue based Interruptible can not work with Ruby version earlier than 3.2. Given that SQ sets it's minimum supported version of Ruby is derived from "full support of Rails 7.1", this in theory mandates support for Ruby 2.7.8. However, other dependencies force the minimum version of Ruby to: 3.1.6 This commit adds a boot time check of the Ruby version and selects either the current or original implementation of Interruptible. --- lib/solid_queue/dispatcher.rb | 2 +- lib/solid_queue/engine.rb | 8 ++++ lib/solid_queue/processes/base.rb | 2 +- lib/solid_queue/processes/interruptible.rb | 16 +++++--- lib/solid_queue/processes/og_interruptible.rb | 41 +++++++++++++++++++ 5 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 lib/solid_queue/processes/og_interruptible.rb diff --git a/lib/solid_queue/dispatcher.rb b/lib/solid_queue/dispatcher.rb index 62e4294d..c60602ed 100644 --- a/lib/solid_queue/dispatcher.rb +++ b/lib/solid_queue/dispatcher.rb @@ -25,7 +25,7 @@ def metadata def poll batch = dispatch_next_batch - batch.size.zero? ? polling_interval : 0.seconds + batch.zero? ? polling_interval : 0.seconds end def dispatch_next_batch diff --git a/lib/solid_queue/engine.rb b/lib/solid_queue/engine.rb index d10997c7..99e14150 100644 --- a/lib/solid_queue/engine.rb +++ b/lib/solid_queue/engine.rb @@ -37,5 +37,13 @@ class Engine < ::Rails::Engine include ActiveJob::ConcurrencyControls end end + + initializer "solid_queue.include_interruptible_concern" do + if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.2") + SolidQueue::Processes::Base.include SolidQueue::Processes::Interruptible + else + SolidQueue::Processes::Base.include SolidQueue::Processes::OgInterruptible + end + end end end diff --git a/lib/solid_queue/processes/base.rb b/lib/solid_queue/processes/base.rb index 6069a90d..59ec9f1a 100644 --- a/lib/solid_queue/processes/base.rb +++ b/lib/solid_queue/processes/base.rb @@ -4,7 +4,7 @@ module SolidQueue module Processes class Base include Callbacks # Defines callbacks needed by other concerns - include AppExecutor, Registrable, Interruptible, Procline + include AppExecutor, Registrable, Procline attr_reader :name diff --git a/lib/solid_queue/processes/interruptible.rb b/lib/solid_queue/processes/interruptible.rb index 3bff1dd9..b7755f1f 100644 --- a/lib/solid_queue/processes/interruptible.rb +++ b/lib/solid_queue/processes/interruptible.rb @@ -2,6 +2,8 @@ module SolidQueue::Processes module Interruptible + include SolidQueue::AppExecutor + def wake_up interrupt end @@ -13,17 +15,19 @@ def interrupt end # Sleeps for 'time'. Can be interrupted asynchronously and return early via wake_up. - # @param time [Numeric] the time to sleep. 0 returns immediately. - # @return [true, nil] - # * returns `true` if an interrupt was requested via #wake_up between the - # last call to `interruptible_sleep` and now, resulting in an early return. - # * returns `nil` if it slept the full `time` and was not interrupted. + # @param time [Numeric, Duration] the time to sleep. 0 returns immediately. def interruptible_sleep(time) # Invoking this from the main thread may result in significant slowdown. # Utilizing asynchronous execution (Futures) addresses this performance issue. Concurrent::Promises.future(time) do |timeout| - queue.pop(timeout:).tap { queue.clear } + queue.clear unless queue.pop(timeout:).nil? + end.on_rejection! do |e| + wrapped_exception = RuntimeError.new("Interruptible#interruptible_sleep - #{e.class}: #{e.message}") + wrapped_exception.set_backtrace(e.backtrace) + handle_thread_error(wrapped_exception) end.value + + nil end def queue diff --git a/lib/solid_queue/processes/og_interruptible.rb b/lib/solid_queue/processes/og_interruptible.rb new file mode 100644 index 00000000..d3b6e390 --- /dev/null +++ b/lib/solid_queue/processes/og_interruptible.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# frozen_string_literal: true + +module SolidQueue::Processes + # The original implementation of Interruptible that works + # with Ruby 3.1 and earlier + module OgInterruptible + def wake_up + interrupt + end + + private + SELF_PIPE_BLOCK_SIZE = 11 + + def interrupt + self_pipe[:writer].write_nonblock(".") + rescue Errno::EAGAIN, Errno::EINTR + # Ignore writes that would block and retry + # if another signal arrived while writing + retry + end + + def interruptible_sleep(time) + if time > 0 && self_pipe[:reader].wait_readable(time) + loop { self_pipe[:reader].read_nonblock(SELF_PIPE_BLOCK_SIZE) } + end + rescue Errno::EAGAIN, Errno::EINTR + end + + # Self-pipe for signal-handling (http://cr.yp.to/docs/selfpipe.html) + def self_pipe + @self_pipe ||= create_self_pipe + end + + def create_self_pipe + reader, writer = IO.pipe + { reader: reader, writer: writer } + end + end +end From d3faa24e6dbfa217d70cba826d2452e06f0a2a5c Mon Sep 17 00:00:00 2001 From: "Hal M. Spitz" Date: Fri, 24 Jan 2025 10:12:55 -0800 Subject: [PATCH 3/3] Test all SQ supported Rubies Updates the ruby test matrix to test the following Rubies: * SQ minimum supported version of Ruby: 3.1.6 - this is the terminal version of Ruby 3.1 which is due to EOL 3/31/25 * First and last version Ruby 3.2: 3.2.0 and 3.2.4 * Every version of Ruby 3.3: 3.3.0 - 3.3.6 * Every version of Ruby 3.4: 3.4.0 - 3.4.1 Updates gemspec to handle a ruby 3.1.6 conditional zeitwerk version requirement. Ruby 3.1.6 requires an older version of zeitwerk. This has a knockon effect of requiring special handling in the Github Action. Finally, updates claimed_execution_test.rb to handle Ruby-3.4.0+ --- .github/workflows/main.yml | 56 ++++- Gemfile.lock.ruby_3_1_6 | 205 ++++++++++++++++++ README.md | 2 + solid_queue.gemspec | 4 + .../solid_queue/claimed_execution_test.rb | 2 +- 5 files changed, 265 insertions(+), 4 deletions(-) create mode 100644 Gemfile.lock.ruby_3_1_6 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 95638773..5467afe4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,5 @@ name: Build -on: [push, pull_request] +on: [ push, pull_request ] jobs: rubocop: @@ -16,14 +16,25 @@ jobs: - name: Run rubocop run: | bundle exec rubocop --parallel + tests: name: Tests runs-on: ubuntu-latest strategy: fail-fast: false matrix: - ruby-version: [3.3.5] - database: [mysql, postgres, sqlite] + ruby-version: + - 3.2.0 + - 3.2.4 + - 3.3.0 + - 3.3.1 + - 3.3.2 + - 3.3.4 + - 3.3.5 + - 3.3.6 + - 3.4.0 + - 3.4.1 + database: [ mysql, postgres, sqlite ] services: mysql: image: mysql:8.0.31 @@ -53,3 +64,42 @@ jobs: bin/rails db:setup - name: Run tests run: bin/rails test + + tests-ruby-3-1-6: + name: Tests Ruby 3.1.6 + runs-on: ubuntu-latest + strategy: + matrix: + database: [ mysql, postgres, sqlite ] + services: + mysql: + image: mysql:8.0.31 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + ports: + - 33060:3306 + options: --health-cmd "mysql -h localhost -e \"select now()\"" --health-interval 1s --health-timeout 5s --health-retries 30 + postgres: + image: postgres:15.1 + env: + POSTGRES_HOST_AUTH_METHOD: "trust" + ports: + - 55432:5432 + env: + TARGET_DB: ${{ matrix.database }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Setup Ruby and install specific gems for 3.1.6 + uses: ruby/setup-ruby@v1 + with: + ruby-version: 3.1.6 + - name: Install dependencies with specific Gemfile.lock + run: | + cp Gemfile.lock.ruby_3_1_6 Gemfile.lock + bundle install + - name: Setup test database + run: | + bin/rails db:setup + - name: Run tests + run: bin/rails test \ No newline at end of file diff --git a/Gemfile.lock.ruby_3_1_6 b/Gemfile.lock.ruby_3_1_6 new file mode 100644 index 00000000..d95cdea9 --- /dev/null +++ b/Gemfile.lock.ruby_3_1_6 @@ -0,0 +1,205 @@ +PATH + remote: . + specs: + solid_queue (1.1.2) + activejob (>= 7.1) + activerecord (>= 7.1) + concurrent-ruby (>= 1.3.1) + fugit (~> 1.11.0) + railties (>= 7.1) + thor (~> 1.3.1) + +GEM + remote: https://rubygems.org/ + specs: + actionpack (7.1.5.1) + actionview (= 7.1.5.1) + activesupport (= 7.1.5.1) + nokogiri (>= 1.8.5) + racc + rack (>= 2.2.4) + rack-session (>= 1.0.1) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + actionview (7.1.5.1) + activesupport (= 7.1.5.1) + builder (~> 3.1) + erubi (~> 1.11) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + activejob (7.1.5.1) + activesupport (= 7.1.5.1) + globalid (>= 0.3.6) + activemodel (7.1.5.1) + activesupport (= 7.1.5.1) + activerecord (7.1.5.1) + activemodel (= 7.1.5.1) + activesupport (= 7.1.5.1) + timeout (>= 0.4.0) + activesupport (7.1.5.1) + base64 + benchmark (>= 0.3) + bigdecimal + concurrent-ruby (~> 1.0, >= 1.0.2) + connection_pool (>= 2.2.5) + drb + i18n (>= 1.6, < 2) + logger (>= 1.4.2) + minitest (>= 5.1) + mutex_m + securerandom (>= 0.3) + tzinfo (~> 2.0) + ast (2.4.2) + base64 (0.2.0) + benchmark (0.4.0) + bigdecimal (3.1.9) + builder (3.3.0) + concurrent-ruby (1.3.4) + connection_pool (2.4.1) + crass (1.0.6) + date (3.4.1) + debug (1.9.2) + irb (~> 1.10) + reline (>= 0.3.8) + drb (2.2.1) + erubi (1.13.1) + et-orbi (1.2.11) + tzinfo + fugit (1.11.1) + et-orbi (~> 1, >= 1.2.11) + raabro (~> 1.4) + globalid (1.2.1) + activesupport (>= 6.1) + i18n (1.14.6) + concurrent-ruby (~> 1.0) + io-console (0.8.0) + irb (1.14.3) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + json (2.9.1) + language_server-protocol (3.17.0.3) + logger (1.6.2) + loofah (2.23.1) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + minitest (5.25.4) + mocha (2.1.0) + ruby2_keywords (>= 0.0.5) + mutex_m (0.3.0) + mysql2 (0.5.6) + nio4r (2.7.4) + nokogiri (1.18.0-arm64-darwin) + racc (~> 1.4) + nokogiri (1.18.0-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.18.0-x86_64-linux-gnu) + racc (~> 1.4) + parallel (1.26.3) + parser (3.3.6.0) + ast (~> 2.4.1) + racc + pg (1.5.4) + psych (5.2.2) + date + stringio + puma (6.4.3) + nio4r (~> 2.0) + raabro (1.4.0) + racc (1.8.1) + rack (3.1.8) + rack-session (2.0.0) + rack (>= 3.0.0) + rack-test (2.2.0) + rack (>= 1.3) + rackup (2.2.1) + rack (>= 3) + rails-dom-testing (2.2.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.6.2) + loofah (~> 2.21) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (7.1.5.1) + actionpack (= 7.1.5.1) + activesupport (= 7.1.5.1) + irb + rackup (>= 1.0.0) + rake (>= 12.2) + thor (~> 1.0, >= 1.2.2) + zeitwerk (~> 2.6) + rainbow (3.1.1) + rake (13.2.1) + rdoc (6.8.1) + psych (>= 4.0.0) + regexp_parser (2.10.0) + reline (0.6.0) + io-console (~> 0.5) + rubocop (1.69.2) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.9.3, < 3.0) + rubocop-ast (>= 1.36.2, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.37.0) + parser (>= 3.3.1.0) + rubocop-minitest (0.36.0) + rubocop (>= 1.61, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-performance (1.23.0) + rubocop (>= 1.48.1, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-rails (2.28.0) + activesupport (>= 4.2.0) + rack (>= 1.1) + rubocop (>= 1.52.0, < 2.0) + rubocop-ast (>= 1.31.1, < 2.0) + rubocop-rails-omakase (1.0.0) + rubocop + rubocop-minitest + rubocop-performance + rubocop-rails + ruby-progressbar (1.13.0) + ruby2_keywords (0.0.5) + securerandom (0.4.1) + sqlite3 (1.5.4-arm64-darwin) + sqlite3 (1.5.4-x86_64-darwin) + sqlite3 (1.5.4-x86_64-linux) + stringio (3.1.2) + thor (1.3.2) + timeout (0.4.3) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + unicode-display_width (3.1.3) + unicode-emoji (~> 4.0, >= 4.0.4) + unicode-emoji (4.0.4) + zeitwerk (2.6.0) + +PLATFORMS + arm64-darwin-22 + arm64-darwin-23 + arm64-darwin-24 + x86_64-darwin-21 + x86_64-darwin-23 + x86_64-linux + +DEPENDENCIES + debug (~> 1.9) + logger + mocha + mysql2 + pg + puma + rdoc + rubocop-rails-omakase + solid_queue! + sqlite3 + zeitwerk (= 2.6.0) + +BUNDLED WITH + 2.5.9 diff --git a/README.md b/README.md index dbd0fd3b..99820af3 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ Solid Queue is configured by default in new Rails 8 applications. But if you're 1. `bundle add solid_queue` 2. `bin/rails solid_queue:install` +(Note: The minimum supported version of Rails is 7.1 and Ruby is 3.1.6.) + This will configure Solid Queue as the production Active Job backend, create the configuration files `config/queue.yml` and `config/recurring.yml`, and create the `db/queue_schema.rb`. It'll also create a `bin/jobs` executable wrapper that you can use to start Solid Queue. Once you've done that, you will then have to add the configuration for the queue database in `config/database.yml`. If you're using SQLite, it'll look like this: diff --git a/solid_queue.gemspec b/solid_queue.gemspec index 91472454..8bccc770 100644 --- a/solid_queue.gemspec +++ b/solid_queue.gemspec @@ -39,4 +39,8 @@ Gem::Specification.new do |spec| spec.add_development_dependency "rubocop-rails-omakase" spec.add_development_dependency "rdoc" spec.add_development_dependency "logger" + + if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") + spec.add_development_dependency "zeitwerk", "2.6.0" + end end diff --git a/test/models/solid_queue/claimed_execution_test.rb b/test/models/solid_queue/claimed_execution_test.rb index 4e99fd04..b7892b21 100644 --- a/test/models/solid_queue/claimed_execution_test.rb +++ b/test/models/solid_queue/claimed_execution_test.rb @@ -31,7 +31,7 @@ class SolidQueue::ClaimedExecutionTest < ActiveSupport::TestCase assert job.failed? assert_equal "RuntimeError", job.failed_execution.exception_class assert_equal "This is a RuntimeError exception", job.failed_execution.message - assert_match /app\/jobs\/raising_job\.rb:\d+:in `perform'/, job.failed_execution.backtrace.first + assert_match /\/app\/jobs\/raising_job\.rb:\d+:in [`'](RaisingJob#)?perform'/, job.failed_execution.backtrace.first assert_equal @process, claimed_execution.process end