diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 969f235..ba89b60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,6 @@ jobs: strategy: matrix: entry: - - { ruby: '2.6', mongodb: '4.4', mongoid: '3' } - - { ruby: '2.6', mongodb: '4.4', mongoid: '4' } - { ruby: '2.6', mongodb: '4.4', mongoid: '5' } - { ruby: '2.7', mongodb: '4.4', mongoid: '6' } - { ruby: '2.7', mongodb: '4.4', mongoid: '7' } diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index e3d8b46..5fcd3fe 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,52 +1,52 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2023-03-03 23:41:38 +0100 using RuboCop version 0.49.1. +# on 2023-03-07 08:57:35 -0500 using RuboCop version 0.49.1. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new # versions of RuboCop, may require this file to be generated again. -# Offense count: 6 +# Offense count: 4 # Configuration parameters: Include. # Include: **/Gemfile, **/gems.rb Bundler/DuplicatedGem: Exclude: - 'Gemfile' -# Offense count: 6 +# Offense count: 8 Metrics/AbcSize: - Max: 69 + Max: 38 -# Offense count: 16 +# Offense count: 18 # Configuration parameters: CountComments, ExcludedMethods. Metrics/BlockLength: - Max: 251 + Max: 258 # Offense count: 1 # Configuration parameters: CountComments. Metrics/ClassLength: - Max: 105 + Max: 103 -# Offense count: 6 +# Offense count: 5 Metrics/CyclomaticComplexity: - Max: 11 + Max: 13 -# Offense count: 155 +# Offense count: 146 # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns. # URISchemes: http, https Metrics/LineLength: - Max: 202 + Max: 252 -# Offense count: 6 +# Offense count: 7 # Configuration parameters: CountComments. Metrics/MethodLength: - Max: 26 + Max: 23 # Offense count: 4 Metrics/PerceivedComplexity: Max: 12 -# Offense count: 10 +# Offense count: 11 Style/Documentation: Exclude: - 'spec/**/*' @@ -54,13 +54,14 @@ Style/Documentation: - 'examples/mongoid_scroll_feed.rb' - 'lib/mongo/scrollable.rb' - 'lib/mongoid/criteria/scrollable.rb' + - 'lib/mongoid/scroll/base_cursor.rb' - 'lib/mongoid/scroll/cursor.rb' - 'lib/mongoid/scroll/errors/base.rb' + - 'lib/mongoid/scroll/errors/invalid_base64_cursor_error.rb' - 'lib/mongoid/scroll/errors/invalid_cursor_error.rb' - 'lib/mongoid/scroll/errors/multiple_sort_fields_error.rb' - 'lib/mongoid/scroll/errors/no_such_field_error.rb' - 'lib/mongoid/scroll/errors/unsupported_field_type_error.rb' - - 'lib/moped/scrollable.rb' # Offense count: 1 # Configuration parameters: ExpectMatchingDefinition, Regex, IgnoreExecutableScripts, AllowedAcronyms. @@ -70,12 +71,11 @@ Style/FileName: - 'lib/mongoid-scroll.rb' # Offense count: 1 -# Configuration parameters: MinBodyLength. -Style/GuardClause: +Style/MultilineTernaryOperator: Exclude: - - 'lib/moped/scrollable.rb' + - 'lib/mongoid/scroll/cursor.rb' -# Offense count: 1 +# Offense count: 3 # Cop supports --auto-correct. # Configuration parameters: SupportedStyles. # SupportedStyles: compact, exploded diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aedea8..3b9112c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * [#29](https://github.com/mongoid/mongoid-scroll/pull/29): Add ability to include the current record to the cursor - [@FabienChaynes](https://github.com/FabienChaynes). * [#30](https://github.com/mongoid/mongoid-scroll/pull/30): Prevent discrepancy between the original sort and the cursor sort - [@FabienChaynes](https://github.com/FabienChaynes). * [#32](https://github.com/mongoid/mongoid-scroll/pull/32): Add Base64 serialization for cursors - [@FabienChaynes](https://github.com/FabienChaynes). +* [#33](https://github.com/mongoid/mongoid-scroll/pull/33): Removed support for Mongoid 3, 4 and Moped - [@dblock](https://github.com/dblock). * Your contribution here. ### 0.3.7 (2021/06/01) diff --git a/Gemfile b/Gemfile index b43b682..b130f84 100644 --- a/Gemfile +++ b/Gemfile @@ -6,10 +6,10 @@ case version = ENV['MONGOID_VERSION'] || '~> 7.0' when 'HEAD' then gem 'mongoid', github: 'mongodb/mongoid' when /7/ then gem 'mongoid', '~> 7.0' when /6/ then gem 'mongoid', '~> 6.0' -when /5/ then gem 'mongoid', '~> 5.0' -when /4/ then gem 'mongoid', '~> 4.0' -when /3/ then gem 'mongoid', '~> 3.1' -else gem 'mongoid', version +when /5/ then + gem 'bigdecimal', '1.3.5' + gem 'mongoid', '~> 5.0' +else gem 'mongoid', version end group :development, :test do diff --git a/LICENSE.md b/LICENSE.md index 663bbf7..11f3706 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2013-2015 Daniel Doubrovkine, Artsy Inc. +Copyright (c) 2013-2023 Daniel Doubrovkine, Artsy Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/README.md b/README.md index 4677b82..671b712 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,38 @@ -Mongoid::Scroll -=============== +- [Mongoid::Scroll](#mongoidscroll) + - [Compatibility](#compatibility) + - [Demo](#demo) + - [The Problem](#the-problem) + - [Installation](#installation) + - [Usage](#usage) + - [Mongoid](#mongoid) + - [Mongo-Ruby-Driver (Mongoid 5)](#mongo-ruby-driver-mongoid-5) + - [Indexes and Performance](#indexes-and-performance) + - [Cursors](#cursors) + - [Standard Cursor](#standard-cursor) + - [Base64 Encoded Cursor](#base64-encoded-cursor) + - [Contributing](#contributing) + - [Copyright and License](#copyright-and-license) + +# Mongoid::Scroll [![Gem Version](https://badge.fury.io/rb/mongoid-scroll.svg)](https://badge.fury.io/rb/mongoid-scroll) [![Build Status](https://github.com/mongoid/mongoid-scroll/actions/workflows/ci.yml/badge.svg)](https://github.com/mongoid/mongoid-scroll/actions/workflows/ci.yml) [![Dependency Status](https://gemnasium.com/mongoid/mongoid-scroll.svg)](https://gemnasium.com/mongoid/mongoid-scroll) [![Code Climate](https://codeclimate.com/github/mongoid/mongoid-scroll.svg)](https://codeclimate.com/github/mongoid/mongoid-scroll) -Mongoid extension that enables infinite scrolling for `Mongoid::Criteria`, `Moped::Query` and `Mongo::Collection::View`. +Mongoid extension that enables infinite scrolling for `Mongoid::Criteria` and `Mongo::Collection::View`. -Compatibility -------------- +## Compatibility -This gem supports Mongoid 3, 4, 5, 6, 7, Moped and Mongo-Ruby-Driver. +This gem supports Mongoid 5, 6, and 7. -Demo ----- +## Demo Check out [shows on artsy.net](http://artsy.net/shows). Keep scrolling down. -There're also two code samples for Mongoid and Moped in [examples](examples). Run `bundle exec ruby examples/mongoid_scroll_feed.rb`. +There're also two code samples for Mongoid in [examples](examples). Run `bundle exec ruby examples/mongoid_scroll_feed.rb`. -The Problem ------------ +## The Problem Traditional pagination does not work when data changes between paginated requests, which makes it unsuitable for infinite scroll behaviors. @@ -30,8 +41,7 @@ Traditional pagination does not work when data changes between paginated request The solution implemented by the `scroll` extension paginates data using a cursor, giving you the ability to restart pagination where you left it off. This is a non-trivial problem when combined with sorting over non-unique record fields, such as timestamps. -Installation ------------- +## Installation Add the gem to your Gemfile and run `bundle install`. @@ -39,8 +49,7 @@ Add the gem to your Gemfile and run `bundle install`. gem 'mongoid-scroll' ``` -Usage ------ +## Usage ### Mongoid @@ -84,27 +93,6 @@ Feed::Item.desc(:position).scroll(saved_cursor) do |record, next_cursor| end ``` -### Moped (Mongoid 3 and 4) - -Scroll a `Moped::Query` and save a cursor to the last item. You must also supply a `field_type` of the sort criteria. - -```ruby -saved_cursor = nil -session[:feed_items].find.sort(position: -1).limit(5).scroll(nil, { field_type: DateTime }) do |record, next_cursor| - # each record, one-by-one - saved_cursor = next_cursor -end -``` - -Resume iterating using the previously saved cursor. - -```ruby -session[:feed_items].find.sort(position: -1).limit(5).scroll(saved_cursor, { field_type: DateTime }) do |record, next_cursor| - # each record, one-by-one - saved_cursor = next_cursor -end -``` - ### Mongo-Ruby-Driver (Mongoid 5) Scroll a `Mongo::Collection::View` and save a cursor to the last item. You must also supply a `field_type` of the sort criteria. @@ -126,8 +114,7 @@ session[:feed_items].find.sort(position: -1).limit(5).scroll(saved_cursor, { fie end ``` -Indexes and Performance ------------------------ +## Indexes and Performance A query without a cursor is identical to a query without a scroll. @@ -159,8 +146,7 @@ module Feed end ``` -Cursors -------- +## Cursors You can use `Mongoid::Scroll::Cursor.from_record` to generate a cursor. A cursor points at the last record of the previous iteration and unlike MongoDB cursors will not expire. @@ -191,37 +177,26 @@ cursor = Mongoid::Scroll::Cursor.from_record(record, { field_type: DateTime, fie Feed::Item.desc(:created_at).scroll(cursor) # Raises a Mongoid::Scroll::Errors::MismatchedSortFieldsError ``` -### Base64 encoded cursors +### Standard Cursor -`Mongoid::Scroll::Base64EncodedCursor` can be used to generate a Base64 encoded string (using RFC 4648) containing all the information needed to rebuild a cursor. +The `Mongoid::Scroll::Cursor` encodes a value and a tiebreak ID separated by `:`, and does not include other options, such as scroll direction. Take extra care not to pass a cursor into a scroll with different options. -A `Mongoid::Scroll::Cursor` can be mutated into a `Mongoid::Scroll::Base64EncodedCursor` by using the `Mongoid::Scroll::Base64EncodedCursor.from_cursor` method: +### Base64 Encoded Cursor -```ruby -saved_cursor = nil -Feed::Item.desc(:position).limit(5).scroll do |record, next_cursor| - # each record, one-by-one - saved_cursor = next_cursor -end -base64_cursor = Mongoid::Scroll::Base64EncodedCursor.from_cursor(saved_cursor) -``` - -The `Mongoid::Scroll::Base64EncodedCursor#to_s` method will return the cursor encoded as a Base64 string, which you'll be able to use subsequently to rebuild the cursor using `Mongoid::Scroll::Base64EncodedCursor.new`. In this case, you won't have to pass the `:field`, `:field_type` or `:field_name` options: +The `Mongoid::Scroll::Base64EncodedCursor` can be used instead of `Mongoid::Scroll::Cursor` to generate a base64-encoded string (using RFC 4648) containing all the information needed to rebuild a cursor. ```ruby -base64_cursor = Mongoid::Scroll::Base64EncodedCursor.from_cursor(saved_cursor) -base64_string = base64_cursor.to_s -Mongoid::Scroll::Base64EncodedCursor.new(base64_string) +Feed::Item.desc(:position).limit(5).scroll(Mongoid::Scroll::Base64EncodedCursor) do |record, next_cursor| + # next_cursor is of type Mongoid::Scroll::Base64EncodedCursor +end ``` -Contributing ------------- +## Contributing Fork the project. Make your feature addition or bug fix with tests. Send a pull request. Bonus points for topic branches. -Copyright and License ---------------------- +## Copyright and License MIT License, see [LICENSE](http://github.com/mongoid/mongoid-scroll/raw/master/LICENSE.md) for details. -(c) 2013-2015 [Daniel Doubrovkine](http://github.com/dblock), based on code by [Frank Macreery](http://github.com/macreery), [Artsy Inc.](http://artsy.net) +(c) 2013-2023 [Daniel Doubrovkine](http://github.com/dblock), based on code by [Frank Macreery](http://github.com/macreery), [Artsy Inc.](http://artsy.net) diff --git a/UPGRADING.md b/UPGRADING.md index d2b6df0..67334d4 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -4,18 +4,11 @@ ### Mismatched Sort Fields -`Mongoid::Criteria::Scrollable#scroll`, `Moped::Scrollable` and `Mongo::Scrollable` now raise a `Mongoid::Scroll::Errors::MismatchedSortFieldsError` when there are discrepancies between the cursor sort options and the original sort options. -Make sure to avoid this case or to handle the new exception. +Both `Mongoid::Criteria::Scrollable#scroll` and `Mongo::Scrollable` now raise a `Mongoid::Scroll::Errors::MismatchedSortFieldsError` when there are discrepancies between the cursor sort options and the original sort options. -```ruby -cursor.field_name = "position" # Avoid this, it'll raise because on the following line the sort is by created_at -Feed::Item.desc(:created_at).scroll(cursor) -``` +For example, the following code will now raise a `MismatchedSortFieldsError` because we set a different field name (`position`) from the `created_at` field used to sort in `scroll`. ```ruby -begin - Feed::Item.desc(:created_at).scroll(cursor) -rescue Mongoid::Scroll::Errors::MismatchedSortFieldsError - # If cursor can be modified externally, handle the exception -end +cursor.field_name = "position" +Feed::Item.desc(:created_at).scroll(cursor) ``` diff --git a/examples/mongoid_scroll_feed.rb b/examples/mongoid_scroll_feed.rb index cf57d36..91fc20b 100644 --- a/examples/mongoid_scroll_feed.rb +++ b/examples/mongoid_scroll_feed.rb @@ -4,14 +4,8 @@ require 'mongoid-scroll' require 'faker' -if defined?(Moped) - Moped.logger = Logger.new($stdout) - Moped.logger.level = Logger::DEBUG -else - Mongoid.logger.level = Logger::INFO - Mongo::Logger.logger.level = Logger::INFO if Mongoid::Compatibility::Version.mongoid5? -end - +Mongoid.logger.level = Logger::INFO +Mongo::Logger.logger.level = Logger::INFO if Mongoid::Compatibility::Version.mongoid5? Mongoid.connect_to 'mongoid_scroll_demo' Mongoid.purge! diff --git a/examples/moped_scroll_feed.rb b/examples/moped_scroll_feed.rb deleted file mode 100644 index ac529e7..0000000 --- a/examples/moped_scroll_feed.rb +++ /dev/null @@ -1,45 +0,0 @@ -require 'bundler' -Bundler.setup(:default, :development) - -require 'mongoid-scroll' -require 'faker' - -Mongoid.connect_to 'mongoid_scroll_demo' -Mongoid.purge! - -raise 'No Moped' unless Object.const_defined?(:Moped) - -# total items to insert -total_items = 20 -# a MongoDB query will be executed every scroll_by items -scroll_by = 7 - -# insert items with a position out-of-order -rands = (0..total_items).to_a.sort { rand }[0..total_items] -total_items.times do - Mongoid.default_session['feed_items'].insert(title: Faker::Lorem.sentence, position: rands.pop) -end - -Mongoid.default_session['feed_items'].indexes.create(position: 1, _id: 1) - -Moped.logger = Logger.new($stdout) -Moped.logger.level = Logger::DEBUG - -total_shown = 0 -next_cursor = nil -loop do - current_cursor = next_cursor - next_cursor = nil - Mongoid.default_session['feed_items'].find.limit(scroll_by).sort(position: 1).scroll(current_cursor, field_type: Integer, field_name: 'position') do |item, cursor| - puts "#{item['position']}: #{item['title']}" - next_cursor = cursor - total_shown += 1 - end - break unless next_cursor - # destroy an item just for the heck of it, scroll is not affected - item = Mongoid.default_session['feed_items'].find.sort(position: 1).first - Mongoid.default_session['feed_items'].find(_id: item['_id']).remove -end - -# this will be 20 -puts "Shown #{total_shown} items." diff --git a/lib/config/locales/en.yml b/lib/config/locales/en.yml index 5a69c73..99ee357 100644 --- a/lib/config/locales/en.yml +++ b/lib/config/locales/en.yml @@ -15,6 +15,10 @@ en: message: "The cursor supplied is invalid." summary: "The cursor supplied is invalid: %{cursor}." resolution: "Cursors must be in the form 'value:tiebreak_id'." + invalid_base64_cursor: + message: "The cursor supplied is invalid." + summary: "The cursor supplied is invalid: %{cursor}." + resolution: "Cursors must be a base64-encoded string." no_such_field: message: "Invalid field." summary: "The field supplied in the cursor does not exist: %{field}." diff --git a/lib/mongo/scrollable.rb b/lib/mongo/scrollable.rb index 6912cc0..74a80a2 100644 --- a/lib/mongo/scrollable.rb +++ b/lib/mongo/scrollable.rb @@ -1,8 +1,10 @@ module Mongo module Scrollable include Mongoid::Criteria::Scrollable::Fields + include Mongoid::Criteria::Scrollable::Cursors - def scroll(cursor = nil, options = nil, &_block) + def scroll(cursor_or_type = nil, options = nil, &_block) + cursor, cursor_type = cursor_and_type(cursor_or_type) view = self # we don't support scrolling over a view with multiple fields raise Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: view.sort) if view.sort && view.sort.keys.size != 1 @@ -12,7 +14,7 @@ def scroll(cursor = nil, options = nil, &_block) # scroll cursor from the parameter, with value and tiebreak_id options = { field_type: BSON::ObjectId } unless options cursor_options = { field_name: scroll_field, direction: scroll_direction }.merge(options) - cursor = cursor.is_a?(Mongoid::Scroll::Cursor) ? cursor : Mongoid::Scroll::Cursor.new(cursor, cursor_options) + cursor = cursor && cursor.is_a?(cursor_type) ? cursor : cursor_type.new(cursor, cursor_options) raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options) # make a view view = Mongo::Collection::View.new( @@ -25,7 +27,7 @@ def scroll(cursor = nil, options = nil, &_block) # scroll if block_given? view.each do |record| - yield record, Mongoid::Scroll::Cursor.from_record(record, cursor_options) + yield record, cursor_type.from_record(record, cursor_options) end else view diff --git a/lib/mongoid-scroll.rb b/lib/mongoid-scroll.rb index b1369d7..b2e7088 100644 --- a/lib/mongoid-scroll.rb +++ b/lib/mongoid-scroll.rb @@ -6,9 +6,10 @@ require 'mongoid-compatibility' require 'mongoid/scroll/version' require 'mongoid/scroll/errors' +require 'mongoid/scroll/base_cursor' require 'mongoid/scroll/cursor' require 'mongoid/scroll/base64_encoded_cursor' require 'mongoid/criteria/scrollable/fields' -require 'moped/scrollable' if Object.const_defined?(:Moped) +require 'mongoid/criteria/scrollable/cursors' require 'mongo/scrollable' if Object.const_defined?(:Mongo) require 'mongoid/criteria/scrollable' diff --git a/lib/mongoid/criteria/scrollable.rb b/lib/mongoid/criteria/scrollable.rb index 01f26d0..bd70ee1 100644 --- a/lib/mongoid/criteria/scrollable.rb +++ b/lib/mongoid/criteria/scrollable.rb @@ -2,18 +2,20 @@ module Mongoid class Criteria module Scrollable include Mongoid::Criteria::Scrollable::Fields + include Mongoid::Criteria::Scrollable::Cursors - def scroll(cursor = nil, &_block) + def scroll(cursor_or_type = nil, &_block) + cursor, cursor_type = cursor_and_type(cursor_or_type) raise_multiple_sort_fields_error if multiple_sort_fields? criteria = dup criteria.merge!(default_sort) if no_sort_option? cursor_options = build_cursor_options(criteria) - cursor = cursor.is_a?(Mongoid::Scroll::Cursor) ? cursor : new_cursor(cursor, cursor_options) + cursor = cursor.is_a?(cursor_type) ? cursor : new_cursor(cursor_type, cursor, cursor_options) raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options) cursor_criteria = build_cursor_criteria(criteria, cursor) if block_given? cursor_criteria.order_by(_id: scroll_direction(criteria)).each do |record| - yield record, cursor_from_record(record, cursor_options) + yield record, cursor_from_record(cursor_type, record, cursor_options) end else cursor_criteria @@ -54,8 +56,8 @@ def build_cursor_options(criteria) } end - def new_cursor(cursor, cursor_options) - Mongoid::Scroll::Cursor.new(cursor, cursor_options) + def new_cursor(cursor_type, cursor, cursor_options) + cursor_type.new(cursor, cursor_options) end def build_cursor_criteria(criteria, cursor) @@ -64,18 +66,14 @@ def build_cursor_criteria(criteria, cursor) cursor_criteria end - def cursor_from_record(record, cursor_options) - Mongoid::Scroll::Cursor.from_record(record, cursor_options) + def cursor_from_record(cursor_type, record, cursor_options) + cursor_type.from_record(record, cursor_options) end def scroll_field_type(criteria) scroll_field = scroll_field(criteria) field = criteria.klass.fields[scroll_field.to_s] - field.foreign_key? && field.object_id_field? ? bson_type : field.type - end - - def bson_type - Mongoid::Compatibility::Version.mongoid3? ? Moped::BSON::ObjectId : BSON::ObjectId + field.foreign_key? && field.object_id_field? ? BSON::ObjectId : field.type end end end diff --git a/lib/mongoid/criteria/scrollable/cursors.rb b/lib/mongoid/criteria/scrollable/cursors.rb new file mode 100644 index 0000000..ddb57f2 --- /dev/null +++ b/lib/mongoid/criteria/scrollable/cursors.rb @@ -0,0 +1,18 @@ +module Mongoid + class Criteria + module Scrollable + # Shared by *::Scrollable modules + module Cursors + private + + def cursor_and_type(cursor_or_type) + cursor = cursor_or_type.is_a?(Class) ? nil : cursor_or_type + cursor_type = cursor_or_type.is_a?(Class) ? cursor_or_type : nil + cursor_type ||= cursor.class if cursor.is_a?(Mongoid::Scroll::BaseCursor) + cursor_type ||= Mongoid::Scroll::Cursor + [cursor, cursor_type] + end + end + end + end +end diff --git a/lib/mongoid/scroll/base64_encoded_cursor.rb b/lib/mongoid/scroll/base64_encoded_cursor.rb index 29a4542..b0c2b47 100644 --- a/lib/mongoid/scroll/base64_encoded_cursor.rb +++ b/lib/mongoid/scroll/base64_encoded_cursor.rb @@ -4,46 +4,37 @@ module Mongoid module Scroll # Allows to serializer/deserialize the cursor using RFC 4648 - class Base64EncodedCursor < Cursor - class << self - def from_cursor(cursor) - base64_encoded_cursor = new(nil, - field_type: cursor.field_type, - field_name: cursor.field_name, - direction: cursor.direction, - include_current: cursor.include_current) - base64_encoded_cursor.value = cursor.value - base64_encoded_cursor.tiebreak_id = cursor.tiebreak_id - base64_encoded_cursor + class Base64EncodedCursor < BaseCursor + def initialize(value, options = {}) + options = extract_field_options(options) + if value + begin + parsed = ::JSON.parse(::Base64.strict_decode64(value)) + rescue + raise Mongoid::Scroll::Errors::InvalidBase64CursorError.new(cursor: value) + end + super parse_field_value(parsed['field_type'], parsed['field_name'], parsed['value']), { + field_type: parsed['field_type'], + field_name: parsed['field_name'], + direction: parsed['direction'], + include_current: parsed['include_current'], + tiebreak_id: parsed['tiebreak_id'] && !parsed['tiebreak_id'].empty? ? BSON::ObjectId.from_string(parsed['tiebreak_id']) : nil + } + else + super nil, options end end def to_s Base64.strict_encode64({ - value: super, - field_type: field_type, + value: transform_field_value(field_type, field_name, value), + field_type: field_type.to_s, field_name: field_name, direction: direction, - include_current: include_current + include_current: include_current, + tiebreak_id: tiebreak_id && tiebreak_id.to_s }.to_json) end - - private - - def parse(value) - return unless value - - begin - config_hash = ::JSON.parse(::Base64.strict_decode64(value)) - rescue - raise Mongoid::Scroll::Errors::InvalidCursorError.new(cursor: value) - end - @field_type = config_hash['field_type'] - @field_name = config_hash['field_name'] - @direction = config_hash['direction'] - @include_current = config_hash['include_current'] - super(config_hash['value']) - end end end end diff --git a/lib/mongoid/scroll/base_cursor.rb b/lib/mongoid/scroll/base_cursor.rb new file mode 100644 index 0000000..dc0b5a1 --- /dev/null +++ b/lib/mongoid/scroll/base_cursor.rb @@ -0,0 +1,123 @@ +module Mongoid + module Scroll + class BaseCursor + attr_accessor :value, :tiebreak_id, :field_type, :field_name, :direction, :include_current + + def initialize(value, options = {}) + @value = value + @tiebreak_id = options[:tiebreak_id] + @field_type = options[:field_type] + @field_name = options[:field_name] + @direction = options[:direction] || 1 + @include_current = options[:include_current] || false + end + + def criteria + mongo_value = value.class.mongoize(value) if value + cursor_criteria = { field_name => { compare_direction => mongo_value } } if mongo_value + tiebreak_criteria = { field_name => mongo_value, :_id => { tiebreak_compare_direction => tiebreak_id } } if mongo_value && tiebreak_id + cursor_selector = if Mongoid::Compatibility::Version.mongoid6_or_newer? + Mongoid::Criteria::Queryable::Selector.new + else + Origin::Selector.new + end + cursor_selector['$or'] = [cursor_criteria, tiebreak_criteria].compact if cursor_criteria || tiebreak_criteria + cursor_selector.__evolve_object_id__ + end + + def sort_options + { + field_type: field_type, + field_name: field_name, + direction: direction + } + end + + def to_s + raise NotImplementedError.new(:to_s) + end + + class << self + def from_record(record, options) + cursor = new(nil, options) + record_value = record.respond_to?(cursor.field_name) ? record.send(cursor.field_name) : record[cursor.field_name] + cursor.value = cursor.send(:parse_field_value, cursor.field_type, cursor.field_name, record_value) + cursor.tiebreak_id = record['_id'] + cursor + end + end + + private + + def parse_field_value(field_type, field_name, value) + return nil unless value + + case field_type.to_s + when 'BSON::ObjectId' then BSON::ObjectId.from_string(value) + when 'String' then value.to_s == '' ? nil : value.to_s + when 'DateTime' then value.is_a?(DateTime) ? value : Time.at(value.to_i).to_datetime + when 'Time' then value.is_a?(Time) ? value : Time.at(value.to_i) + when 'Date' then value.is_a?(Date) ? value : Time.at(value.to_i).utc.to_date + when 'Float' then value.to_f + when 'Integer' then value.to_i + else + raise Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type) + end + end + + def transform_field_value(field_type, field_name, value) + return nil unless value + + case field_type.to_s + when 'BSON::ObjectId' then value.to_s + when 'String' then value.to_s + when 'Date' then Time.utc(value.year, value.month, value.day).to_i + when 'DateTime', 'Time' then value.utc.to_i + when 'Float' then value.to_f + when 'Integer' then value.to_i + else + raise Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type) + end + end + + def extract_field_options(options) + if options && (field_name = options[:field_name]) && (field_type = options[:field_type]) + { + field_type: field_type.to_s, + field_name: field_name.to_s, + direction: options[:direction] || 1, + include_current: options[:include_current] || false + } + elsif options && (field = options[:field]) + { + field_type: field.type.to_s, + field_name: field.name.to_s, + direction: options[:direction] || 1, + include_current: options[:include_current] || false + } + end + end + + def compare_direction + direction == 1 ? '$gt' : '$lt' + end + + def tiebreak_compare_direction + if include_current + case compare_direction + when '$gt' + '$gte' + when '$lt' + '$lte' + end + else + compare_direction + end + end + + def parse(_value) + raise NotImplementedError.new(:parse) + end + end + end +end diff --git a/lib/mongoid/scroll/cursor.rb b/lib/mongoid/scroll/cursor.rb index 5337170..a140c90 100644 --- a/lib/mongoid/scroll/cursor.rb +++ b/lib/mongoid/scroll/cursor.rb @@ -1,122 +1,34 @@ module Mongoid module Scroll - class Cursor - attr_accessor :value, :tiebreak_id, :field_type, :field_name, :direction, :include_current - + class Cursor < BaseCursor def initialize(value = nil, options = {}) - @field_type, @field_name = self.class.extract_field_options(options) - @direction = options[:direction] || 1 - @include_current = options[:include_current] || false - parse(value) - end - - def criteria - mongo_value = value.class.mongoize(value) if value - cursor_criteria = { field_name => { compare_direction => mongo_value } } if mongo_value - tiebreak_criteria = { field_name => mongo_value, :_id => { tiebreak_compare_direction => tiebreak_id } } if mongo_value && tiebreak_id - cursor_selector = if Mongoid::Compatibility::Version.mongoid6? || Mongoid::Compatibility::Version.mongoid7? - Mongoid::Criteria::Queryable::Selector.new - else - Origin::Selector.new - end - cursor_selector['$or'] = [cursor_criteria, tiebreak_criteria].compact if cursor_criteria || tiebreak_criteria - cursor_selector.__evolve_object_id__ - end - - def sort_options - { - field_type: field_type, - field_name: field_name, - direction: direction - } - end - - class << self - def from_record(record, options) - cursor = new(nil, options) - value = record.respond_to?(cursor.field_name) ? record.send(cursor.field_name) : record[cursor.field_name] - cursor.value = Mongoid::Scroll::Cursor.parse_field_value(cursor.field_type, cursor.field_name, value) - cursor.tiebreak_id = record['_id'] - cursor - end - end - - def to_s - tiebreak_id ? [Mongoid::Scroll::Cursor.transform_field_value(field_type, field_name, value), tiebreak_id].join(':') : nil - end - - private - - def compare_direction - direction == 1 ? '$gt' : '$lt' - end - - def tiebreak_compare_direction - if include_current - case compare_direction - when '$gt' - '$gte' - when '$lt' - '$lte' + options = extract_field_options(options) + raise ArgumentError.new 'Missing options[:field_name] and/or options[:field_type].' unless options + if value + parts = value.split(':') if value + unless parts && parts.length >= 2 + raise Mongoid::Scroll::Errors::InvalidCursorError.new(cursor: value) end + value = parse_field_value( + options[:field_type], + options[:field_name], + parts[0...-1].join(':') + ) + options[:tiebreak_id] = BSON::ObjectId.from_string(parts[-1]) + super value, options else - compare_direction + super nil, options end end - def parse(value) - return unless value - parts = value.split(':') - unless parts.length >= 2 - raise Mongoid::Scroll::Errors::InvalidCursorError.new(cursor: value) - end - id = parts[-1] - value = parts[0...-1].join(':') - @value = Mongoid::Scroll::Cursor.parse_field_value(field_type, field_name, value) - @tiebreak_id = if Mongoid::Compatibility::Version.mongoid3? - Moped::BSON::ObjectId(id) - else - BSON::ObjectId.from_string(id) - end - end - - class << self - def extract_field_options(options) - if options && (field_name = options[:field_name]) && (field_type = options[:field_type]) - [field_type.to_s, field_name.to_s] - elsif options && (field = options[:field]) - [field.type.to_s, field.name.to_s] - elsif self == Mongoid::Scroll::Cursor - raise ArgumentError.new 'Missing options[:field_name] and/or options[:field_type].' - end - end - - def parse_field_value(field_type, field_name, value) - case field_type.to_s - when 'BSON::ObjectId', 'Moped::BSON::ObjectId' then value - when 'String' then value.to_s - when 'DateTime' then value.is_a?(DateTime) ? value : Time.at(value.to_i).to_datetime - when 'Time' then value.is_a?(Time) ? value : Time.at(value.to_i) - when 'Date' then value.is_a?(Date) ? value : Time.at(value.to_i).utc.to_date - when 'Float' then value.to_f - when 'Integer' then value.to_i - else - raise Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type) - end - end - - def transform_field_value(field_type, field_name, value) - case field_type.to_s - when 'BSON::ObjectId', 'Moped::BSON::ObjectId' then value - when 'String' then value.to_s - when 'Date' then Time.utc(value.year, value.month, value.day).to_i - when 'DateTime', 'Time' then value.utc.to_i - when 'Float' then value.to_f - when 'Integer' then value.to_i - else - raise Mongoid::Scroll::Errors::UnsupportedFieldTypeError.new(field: field_name, type: field_type) - end - end + def to_s + tiebreak_id ? [ + transform_field_value( + field_type, + field_name, + value + ), tiebreak_id + ].join(':') : nil end end end diff --git a/lib/mongoid/scroll/errors.rb b/lib/mongoid/scroll/errors.rb index 49b8f36..2b20e4d 100644 --- a/lib/mongoid/scroll/errors.rb +++ b/lib/mongoid/scroll/errors.rb @@ -1,6 +1,8 @@ require 'mongoid/scroll/errors/base' require 'mongoid/scroll/errors/multiple_sort_fields_error' require 'mongoid/scroll/errors/mismatched_sort_fields_error' +require 'mongoid/scroll/errors/invalid_base_cursor_error' require 'mongoid/scroll/errors/invalid_cursor_error' +require 'mongoid/scroll/errors/invalid_base64_cursor_error' require 'mongoid/scroll/errors/no_such_field_error' require 'mongoid/scroll/errors/unsupported_field_type_error' diff --git a/lib/mongoid/scroll/errors/invalid_base64_cursor_error.rb b/lib/mongoid/scroll/errors/invalid_base64_cursor_error.rb new file mode 100644 index 0000000..dd267b1 --- /dev/null +++ b/lib/mongoid/scroll/errors/invalid_base64_cursor_error.rb @@ -0,0 +1,11 @@ +module Mongoid + module Scroll + module Errors + class InvalidBase64CursorError < InvalidBaseCursorError + def initialize(opts = {}) + super(compose_message('invalid_base64_cursor', opts)) + end + end + end + end +end diff --git a/lib/mongoid/scroll/errors/invalid_base_cursor_error.rb b/lib/mongoid/scroll/errors/invalid_base_cursor_error.rb new file mode 100644 index 0000000..abee368 --- /dev/null +++ b/lib/mongoid/scroll/errors/invalid_base_cursor_error.rb @@ -0,0 +1,8 @@ +module Mongoid + module Scroll + module Errors + class InvalidBaseCursorError < Mongoid::Scroll::Errors::Base + end + end + end +end diff --git a/lib/mongoid/scroll/errors/invalid_cursor_error.rb b/lib/mongoid/scroll/errors/invalid_cursor_error.rb index d6261e3..674d900 100644 --- a/lib/mongoid/scroll/errors/invalid_cursor_error.rb +++ b/lib/mongoid/scroll/errors/invalid_cursor_error.rb @@ -1,7 +1,7 @@ module Mongoid module Scroll module Errors - class InvalidCursorError < Mongoid::Scroll::Errors::Base + class InvalidCursorError < InvalidBaseCursorError def initialize(opts = {}) super(compose_message('invalid_cursor', opts)) end diff --git a/lib/moped/scrollable.rb b/lib/moped/scrollable.rb deleted file mode 100644 index 32132ee..0000000 --- a/lib/moped/scrollable.rb +++ /dev/null @@ -1,41 +0,0 @@ -module Moped - module Scrollable - include Mongoid::Criteria::Scrollable::Fields - - def scroll(cursor = nil, options = nil, &_block) - unless options - bson_type = Mongoid::Compatibility::Version.mongoid3? ? Moped::BSON::ObjectId : BSON::ObjectId - options = { field_type: bson_type } - end - query = Query.new(collection, operation.selector.dup) - query.operation.skip = operation.skip - query.operation.limit = operation.limit - # we don't support scrolling over a criteria with multiple fields - if query.operation.selector['$orderby'] && query.operation.selector['$orderby'].keys.size != 1 - raise Mongoid::Scroll::Errors::MultipleSortFieldsError.new(sort: query.operation.selector['$orderby']) - elsif !query.operation.selector.key?('$orderby') || query.operation.selector['$orderby'].empty? - # introduce a default sort order if there's none - query.sort(_id: 1) - end - # scroll field and direction - scroll_field = query.operation.selector['$orderby'].keys.first - scroll_direction = query.operation.selector['$orderby'].values.first.to_i - # scroll cursor from the parameter, with value and tiebreak_id - cursor_options = { field_name: scroll_field, field_type: options[:field_type], direction: scroll_direction } - cursor = cursor.is_a?(Mongoid::Scroll::Cursor) ? cursor : Mongoid::Scroll::Cursor.new(cursor, cursor_options) - raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options) - query.operation.selector['$query'] = query.operation.selector['$query'].merge(cursor.criteria) - query.operation.selector['$orderby'] = query.operation.selector['$orderby'].merge(_id: scroll_direction) - # scroll - if block_given? - query.each do |record| - yield record, Mongoid::Scroll::Cursor.from_record(record, cursor_options) - end - else - query - end - end - end -end - -Moped::Query.send(:include, Moped::Scrollable) diff --git a/spec/mongo/collection_view_spec.rb b/spec/mongo/collection_view_spec.rb index 4fc785f..c8e938c 100644 --- a/spec/mongo/collection_view_spec.rb +++ b/spec/mongo/collection_view_spec.rb @@ -2,135 +2,140 @@ if Object.const_defined?(:Mongo) describe Mongo::Collection::View do - context 'scrollable' do - subject do - Mongoid.default_client['feed_items'].find - end - it ':scroll' do - expect(subject).to respond_to(:scroll) - end - end - context 'with multiple sort fields' do - subject do - Mongoid.default_client['feed_items'].find.sort(name: 1, value: -1) - end - it 'raises Mongoid::Scroll::Errors::MultipleSortFieldsError' do - expect { subject.scroll }.to raise_error Mongoid::Scroll::Errors::MultipleSortFieldsError, - /You're attempting to scroll over data with a sort order that includes multiple fields: name, value./ - end - end - context 'with different sort fields between the cursor and the criteria' do - subject do - Mongoid.default_client['feed_items'].find.sort(name: -1) - end + [Mongoid::Scroll::Cursor, Mongoid::Scroll::Base64EncodedCursor].each do |cursor_type| + context cursor_type do + context 'scrollable' do + subject do + Mongoid.default_client['feed_items'].find + end + it ':scroll' do + expect(subject).to respond_to(:scroll) + end + end + context 'with multiple sort fields' do + subject do + Mongoid.default_client['feed_items'].find.sort(name: 1, value: -1) + end + it 'raises Mongoid::Scroll::Errors::MultipleSortFieldsError' do + expect { subject.scroll(cursor_type) }.to raise_error Mongoid::Scroll::Errors::MultipleSortFieldsError, + /You're attempting to scroll over data with a sort order that includes multiple fields: name, value./ + end + end + context 'with different sort fields between the cursor and the criteria' do + subject do + Mongoid.default_client['feed_items'].find.sort(name: -1) + end - it 'raises Mongoid::Scroll::Errors::MismatchedSortFieldsError' do - record = Feed::Item.create! - cursor = Mongoid::Scroll::Cursor.from_record(record, field: record.fields['a_string']) - error_string = /You're attempting to scroll over data with a sort order that differs between the cursor and the original criteria: field_name, direction./ - expect { subject.scroll(cursor, field_type: String) }.to raise_error Mongoid::Scroll::Errors::MismatchedSortFieldsError, error_string - end - end - context 'with no sort' do - subject do - Mongoid.default_client['feed_items'].find - end - it 'adds a default sort by _id' do - expect(subject.scroll.sort).to eq('_id' => 1) - end - end - context 'with data' do - before :each do - 10.times do |i| - Mongoid.default_client['feed_items'].insert_one( - a_string: i.to_s, - a_integer: i, - a_datetime: DateTime.mongoize(DateTime.new(2013, i + 1, 21, 1, 42, 3, 'UTC')), - a_date: Date.mongoize(Date.new(2013, i + 1, 21)), - a_time: Time.mongoize(Time.at(Time.now.to_i + i)) - ) + it 'raises Mongoid::Scroll::Errors::MismatchedSortFieldsError' do + record = Feed::Item.create! + cursor = cursor_type.from_record(record, field: record.fields['a_string']) + expect(cursor).to be_a cursor_type + error_string = /You're attempting to scroll over data with a sort order that differs between the cursor and the original criteria: field_name, direction./ + expect { subject.scroll(cursor, field_type: String) }.to raise_error Mongoid::Scroll::Errors::MismatchedSortFieldsError, error_string + end end - end - context 'default' do - it 'scrolls all' do - records = [] - Mongoid.default_client['feed_items'].find.scroll do |record, _next_cursor| - records << record + context 'with no sort' do + subject do + Mongoid.default_client['feed_items'].find + end + it 'adds a default sort by _id' do + expect(subject.scroll(cursor_type).sort).to eq('_id' => 1) end - expect(records.size).to eq 10 - expect(records).to eq Mongoid.default_client['feed_items'].find.to_a end - end - { a_string: String, a_integer: Integer, a_date: Date, a_datetime: DateTime }.each_pair do |field_name, field_type| - context field_type do - it 'scrolls all with a block' do - records = [] - Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(nil, field_type: field_type) do |record, _next_cursor| - records << record + context 'with data' do + before :each do + 10.times do |i| + Mongoid.default_client['feed_items'].insert_one( + a_string: i.to_s, + a_integer: i, + a_datetime: DateTime.mongoize(DateTime.new(2013, i + 1, 21, 1, 42, 3, 'UTC')), + a_date: Date.mongoize(Date.new(2013, i + 1, 21)), + a_time: Time.mongoize(Time.at(Time.now.to_i + i)) + ) end - expect(records.size).to eq 10 - expect(records).to eq Mongoid.default_client['feed_items'].find.to_a end - it 'scrolls all with a break' do - records = [] - cursor = nil - Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(5).scroll(nil, field_type: field_type) do |record, next_cursor| - records << record - cursor = next_cursor - end - expect(records.size).to eq 5 - Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor, field_type: field_type) do |record, next_cursor| - records << record - cursor = next_cursor + context 'default' do + it 'scrolls all' do + records = [] + Mongoid.default_client['feed_items'].find.scroll(cursor_type) do |record, _next_cursor| + records << record + end + expect(records.size).to eq 10 + expect(records).to eq Mongoid.default_client['feed_items'].find.to_a end - expect(records.size).to eq 10 - expect(records).to eq Mongoid.default_client['feed_items'].find.to_a end - it 'scrolls in descending order' do - records = [] - Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).scroll(nil, field_type: field_type, field_name: field_name) do |record, _next_cursor| - records << record + { a_string: String, a_integer: Integer, a_date: Date, a_datetime: DateTime }.each_pair do |field_name, field_type| + context field_type do + it 'scrolls all with a block' do + records = [] + Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor_type, field_type: field_type) do |record, _next_cursor| + records << record + end + expect(records.size).to eq 10 + expect(records).to eq Mongoid.default_client['feed_items'].find.to_a + end + it 'scrolls all with a break' do + records = [] + cursor = nil + Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(5).scroll(cursor_type, field_type: field_type) do |record, next_cursor| + records << record + cursor = next_cursor + expect(cursor).to be_a cursor_type + end + expect(records.size).to eq 5 + Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor, field_type: field_type) do |record, next_cursor| + records << record + cursor = next_cursor + expect(cursor).to be_a cursor_type + end + expect(records.size).to eq 10 + expect(records).to eq Mongoid.default_client['feed_items'].find.to_a + end + it 'scrolls in descending order' do + records = [] + Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).scroll(cursor_type, field_type: field_type, field_name: field_name) do |record, _next_cursor| + records << record + end + expect(records.size).to eq 3 + expect(records).to eq Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).to_a + end + it 'map' do + record = Mongoid.default_client['feed_items'].find.limit(3).scroll(cursor_type, field_type: field_type, field_name: field_name).map { |r| r }.last + cursor = cursor_type.from_record(record, field_type: field_type, field_name: field_name) + expect(cursor).to_not be nil + expect(cursor.value).to eq record[field_name.to_s] + expect(cursor.tiebreak_id).to eq record['_id'] + end end - expect(records.size).to eq 3 - expect(records).to eq Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).to_a - end - it 'map' do - record = Mongoid.default_client['feed_items'].find.limit(3).scroll(nil, field_type: field_type, field_name: field_name).map { |r| r }.last - cursor = Mongoid::Scroll::Cursor.from_record(record, field_type: field_type, field_name: field_name) - expect(cursor).to_not be nil - expect(cursor.to_s.split(':')).to eq [ - Mongoid::Scroll::Cursor.transform_field_value(field_type, field_name, record[field_name.to_s]).to_s, - record['_id'].to_s - ] end end - end - end - context 'with overlapping data', if: MongoDB.mmapv1? do - before :each do - 3.times { Feed::Item.create! a_integer: 5 } - Feed::Item.first.update_attributes!(name: Array(1000).join('a')) - end - it 'natural order is different from order by id' do - # natural order isn't necessarily going to be the same as _id order - # if a document is updated and grows in size, it may need to be relocated and - # thus cause the natural order to change - expect(Feed::Item.order_by('$natural' => 1).to_a).to_not eq Feed::Item.order_by(_id: 1).to_a - end - [{ a_integer: 1 }, { a_integer: -1 }].each do |sort_order| - it "scrolls by #{sort_order}" do - records = [] - cursor = nil - Mongoid.default_client['feed_items'].find.sort(sort_order).limit(2).scroll do |record, next_cursor| - records << record - cursor = next_cursor + context 'with overlapping data', if: MongoDB.mmapv1? do + before :each do + 3.times { Feed::Item.create! a_integer: 5 } + Feed::Item.first.update_attributes!(name: Array(1000).join('a')) + end + it 'natural order is different from order by id' do + # natural order isn't necessarily going to be the same as _id order + # if a document is updated and grows in size, it may need to be relocated and + # thus cause the natural order to change + expect(Feed::Item.order_by('$natural' => 1).to_a).to_not eq Feed::Item.order_by(_id: 1).to_a end - expect(records.size).to eq 2 - Mongoid.default_client['feed_items'].find.sort(sort_order).scroll(cursor) do |record, _next_cursor| - records << record + [{ a_integer: 1 }, { a_integer: -1 }].each do |sort_order| + it "scrolls by #{sort_order}" do + records = [] + cursor = nil + Mongoid.default_client['feed_items'].find.sort(sort_order).limit(2).scroll(cursor_type) do |record, next_cursor| + records << record + cursor = next_cursor + end + expect(records.size).to eq 2 + Mongoid.default_client['feed_items'].find.sort(sort_order).scroll(cursor) do |record, _next_cursor| + records << record + end + expect(records.size).to eq 3 + expect(records).to eq Mongoid.default_client['feed_items'].find.sort(sort_order.merge(_id: sort_order[:a_integer])).to_a + end end - expect(records.size).to eq 3 - expect(records).to eq Mongoid.default_client['feed_items'].find.sort(sort_order.merge(_id: sort_order[:a_integer])).to_a end end end diff --git a/spec/mongoid/base64_encoded_cursor_spec.rb b/spec/mongoid/base64_encoded_cursor_spec.rb new file mode 100644 index 0000000..55580d8 --- /dev/null +++ b/spec/mongoid/base64_encoded_cursor_spec.rb @@ -0,0 +1,233 @@ +require 'spec_helper' + +describe Mongoid::Scroll::Base64EncodedCursor do + context 'new' do + context 'an empty cursor' do + let(:base64_string) { 'eyJ2YWx1ZSI6bnVsbCwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOm51bGx9' } + subject do + Mongoid::Scroll::Base64EncodedCursor.new base64_string + end + its(:tiebreak_id) { should be_nil } + its(:value) { should be_nil } + its(:criteria) { should eq({}) } + its(:to_s) { should eq(base64_string) } + end + context 'a string field cursor' do + let(:base64_string) { 'eyJ2YWx1ZSI6ImEgc3RyaW5nIiwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2M2RmODA5NDQzNDE3YzdkMmIxMDIifQ==' } + let(:a_value) { 'a string' } + let(:tiebreak_id) { BSON::ObjectId.from_string('64063df809443417c7d2b102') } + let(:criteria) do + { + '$or' => [ + { 'a_string' => { '$gt' => a_value } }, + { 'a_string' => a_value, '_id' => { '$gt' => tiebreak_id } } + ] + } + end + subject do + Mongoid::Scroll::Base64EncodedCursor.new base64_string + end + its(:value) { should eq a_value } + its(:tiebreak_id) { tiebreak_id } + its(:value) { should eq a_value } + its(:tiebreak_id) { should eq tiebreak_id } + its(:criteria) { should eq(criteria) } + its(:to_s) { should eq(base64_string) } + end + context 'an id field cursor' do + let(:base64_string) { 'eyJ2YWx1ZSI6IjY0MDY0NTg0MDk0NDM0MjgxZmE3MWFiMiIsImZpZWxkX3R5cGUiOiJCU09OOjpPYmplY3RJZCIsImZpZWxkX25hbWUiOiJpZCIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2NDU4NDA5NDQzNDI4MWZhNzFhYjIifQ==' } + let(:a_value) { BSON::ObjectId('64064584094434281fa71ab2') } + let(:tiebreak_id) { a_value } + let(:criteria) do + { + '$or' => [ + { 'id' => { '$gt' => a_value } }, + { 'id' => a_value, '_id' => { '$gt' => tiebreak_id } } + ] + } + end + subject do + Mongoid::Scroll::Base64EncodedCursor.new base64_string + end + its(:value) { should eq a_value } + its(:tiebreak_id) { should eq tiebreak_id } + its(:criteria) { should eq(criteria) } + its(:to_s) { should eq(base64_string) } + end + context 'an integer field cursor' do + let(:base64_string) { 'eyJ2YWx1ZSI6MTAsImZpZWxkX3R5cGUiOiJJbnRlZ2VyIiwiZmllbGRfbmFtZSI6ImFfaW50ZWdlciIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2M2RmODA5NDQzNDE3YzdkMmIxMDgifQ==' } + let(:a_value) { 10 } + let(:tiebreak_id) { BSON::ObjectId('64063df809443417c7d2b108') } + let(:criteria) do + { + '$or' => [ + { 'a_integer' => { '$gt' => 10 } }, + { 'a_integer' => 10, '_id' => { '$gt' => tiebreak_id } } + ] + } + end + subject do + Mongoid::Scroll::Base64EncodedCursor.new base64_string + end + its(:value) { should eq a_value } + its(:tiebreak_id) { tiebreak_id } + its(:value) { should eq a_value } + its(:tiebreak_id) { should eq tiebreak_id } + its(:criteria) { should eq(criteria) } + its(:to_s) { should eq(base64_string) } + end + context 'a date/time field cursor' do + let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzU5MDEyMywiZmllbGRfdHlwZSI6IkRhdGVUaW1lIiwiZmllbGRfbmFtZSI6ImFfZGF0ZXRpbWUiLCJkaXJlY3Rpb24iOjEsImluY2x1ZGVfY3VycmVudCI6ZmFsc2UsInRpZWJyZWFrX2lkIjoiNjQwNjQzYTcwOTQ0MzQyMzlmMmRiZjg2In0=' } + let(:a_value) { DateTime.new(2013, 12, 21, 1, 42, 3, 'UTC') } + let(:tiebreak_id) { BSON::ObjectId('640643a7094434239f2dbf86') } + let(:criteria) do + { + '$or' => [ + { 'a_datetime' => { '$gt' => a_value.utc } }, + { 'a_datetime' => a_value.utc, '_id' => { '$gt' => tiebreak_id } } + ] + } + end + subject do + Mongoid::Scroll::Base64EncodedCursor.new base64_string + end + its(:value) { should eq a_value } + its(:tiebreak_id) { should eq tiebreak_id } + its(:criteria) { should eq(criteria) } + its(:to_s) { should eq(base64_string) } + end + context 'a date field cursor' do + let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzU4NDAwMCwiZmllbGRfdHlwZSI6IkRhdGUiLCJmaWVsZF9uYW1lIjoiYV9kYXRlIiwiZGlyZWN0aW9uIjoxLCJpbmNsdWRlX2N1cnJlbnQiOmZhbHNlLCJ0aWVicmVha19pZCI6IjY0MDY0MmM5MDk0NDM0MjEyYzRkNDQyMCJ9' } + let(:tiebreak_id) { BSON::ObjectId('640642c9094434212c4d4420') } + let(:a_value) { Date.new(2013, 12, 21) } + let(:criteria) do + { + '$or' => [ + { 'a_date' => { '$gt' => a_value.to_datetime.utc } }, + { 'a_date' => a_value.to_datetime.utc, '_id' => { '$gt' => tiebreak_id } } + ] + } + end + subject do + Mongoid::Scroll::Base64EncodedCursor.new base64_string + end + its(:value) { should eq a_value } + its(:tiebreak_id) { should eq tiebreak_id } + its(:criteria) { should eq(criteria) } + its(:to_s) { should eq(base64_string) } + end + context 'a time field cursor' do + let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzYwNTcyMywiZmllbGRfdHlwZSI6IlRpbWUiLCJmaWVsZF9uYW1lIjoiYV90aW1lIiwiZGlyZWN0aW9uIjoxLCJpbmNsdWRlX2N1cnJlbnQiOmZhbHNlLCJ0aWVicmVha19pZCI6IjY0MDYzZDRhMDk0NDM0MTY2YmQwNTNlZCJ9' } + let(:item_id) { BSON::ObjectId('640636f209443407333b46d4') } + let(:a_value) { Time.new(2013, 12, 21, 6, 2, 3, '+00:00').utc } + let(:tiebreak_id) { BSON::ObjectId('64063d4a094434166bd053ed') } + let(:criteria) do + { + '$or' => [ + { 'a_time' => { '$gt' => a_value } }, + { 'a_time' => a_value, '_id' => { '$gt' => tiebreak_id } } + ] + } + end + subject do + Mongoid::Scroll::Base64EncodedCursor.new base64_string + end + its(:value) { should eq a_value } + its(:tiebreak_id) { tiebreak_id } + its(:tiebreak_id) { should eq tiebreak_id } + its(:criteria) { should eq(criteria) } + its(:to_s) { should eq(base64_string) } + end + context 'an invalid field cursor' do + it 'raises ArgumentError' do + expect do + Mongoid::Scroll::Base64EncodedCursor.new 'invalid', {} + end.to raise_error Mongoid::Scroll::Errors::InvalidBase64CursorError + end + end + context 'an invalid cursor' do + it 'raises a Mongoid::Scroll::Errors::InvalidBase64CursorError with an invalid Base64 string' do + expect { Mongoid::Scroll::Base64EncodedCursor.new 'invalid' }.to raise_error Mongoid::Scroll::Errors::InvalidBase64CursorError, /The cursor supplied is invalid: invalid./ + end + + it 'raises a Mongoid::Scroll::Errors::InvalidBase64CursorError with an invalid JSON string' do + expect { Mongoid::Scroll::Base64EncodedCursor.new 'aW52YWxpZA==' }.to raise_error Mongoid::Scroll::Errors::InvalidBase64CursorError, /The cursor supplied is invalid: aW52YWxpZA==./ + end + end + end + context 'from_record' do + context 'a string field cursor' do + let(:field_type) { String } + let(:field_value) { 'a string' } + let(:field_name) { 'a_string' } + let(:feed_item) { Feed::Item.create!(field_name => field_value) } + subject do + Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: field_name, field_type: field_type + end + its(:value) { should eq field_value } + its(:field_name) { should eq field_name } + its(:field_type) { should eq field_type.to_s } + end + context 'an id field cursor' do + let(:field_type) { BSON::ObjectId } + let(:field_name) { 'id' } + let(:feed_item) { Feed::Item.create! } + subject do + Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: field_name, field_type: field_type + end + its(:value) { should eq feed_item._id } + its(:field_type) { should eq field_type.to_s } + end + context 'an integer field cursor' do + let(:field_type) { Integer } + let(:field_value) { 10 } + let(:field_name) { 'a_integer' } + let(:feed_item) { Feed::Item.create!(field_name => field_value) } + subject do + Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: field_name, field_type: field_type + end + its(:value) { should eq field_value } + its(:field_type) { should eq field_type.to_s } + end + context 'a date/time field cursor' do + let(:field_type) { DateTime } + let(:field_value) { DateTime.new(2013, 12, 21, 1, 42, 3, 'UTC') } + let(:field_name) { 'a_datetime' } + let(:feed_item) { Feed::Item.create!(field_name => field_value) } + subject do + Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: field_name, field_type: field_type + end + its(:value) { should eq field_value } + its(:field_type) { should eq field_type.to_s } + end + context 'a date field cursor' do + let(:field_type) { Date } + let(:field_value) { Date.new(2013, 12, 21) } + let(:field_name) { 'a_date' } + let(:feed_item) { Feed::Item.create!(field_name => field_value) } + subject do + Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: field_name, field_type: field_type + end + its(:value) { should eq field_value } + end + context 'a time field cursor' do + let(:field_type) { Time } + let(:field_value) { Time.new(2013, 12, 21, 1, 2, 3) } + let(:field_name) { 'a_time' } + let(:feed_item) { Feed::Item.create!(field_name => field_value) } + subject do + Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: field_name, field_type: field_type + end + its(:value) { should eq field_value } + its(:field_type) { should eq field_type.to_s } + end + context 'an array field cursor' do + let(:feed_item) { Feed::Item.create!(a_array: %w[x y]) } + it 'is not supported' do + expect do + Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: 'a_array', field_type: Array + end.to raise_error Mongoid::Scroll::Errors::UnsupportedFieldTypeError, /The type of the field 'a_array' is not supported: Array./ + end + end + end +end diff --git a/spec/mongoid/base64_encoded_scroll_cursor_spec.rb b/spec/mongoid/base64_encoded_scroll_cursor_spec.rb deleted file mode 100644 index b518d36..0000000 --- a/spec/mongoid/base64_encoded_scroll_cursor_spec.rb +++ /dev/null @@ -1,72 +0,0 @@ -require 'spec_helper' - -describe Mongoid::Scroll::Base64EncodedCursor do - context 'an empty cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6bnVsbCwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZX0=' } - subject do - described_class.new(base64_string) - end - its(:tiebreak_id) { should be_nil } - its(:value) { should be_nil } - its(:criteria) { should eq({}) } - its(:to_s) { should eq(base64_string) } - - describe '.from_cursor' do - let(:base_cursor) { Mongoid::Scroll::Cursor.new nil, field_name: 'a_string', field_type: String } - it 'is properly created' do - base64_encoded_cursor = described_class.from_cursor(base_cursor) - expect(base64_encoded_cursor).to be_a(described_class) - expect(base64_encoded_cursor.tiebreak_id).to be_nil - expect(base64_encoded_cursor.value).to be_nil - expect(base64_encoded_cursor.criteria).to eq({}) - end - end - end - - context 'a string field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6ImFzdHJpbmc6NjQwMjBjYzA4OWIyNTQ0ZTIzYTdkNmRjIiwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZX0=' } - let(:feed_item) { Feed::Item.create!(a_string: 'astring') } - let(:feed_id) { Mongoid::Compatibility::Version.mongoid3? ? Moped::BSON::ObjectId('64020cc089b2544e23a7d6dc') : BSON::ObjectId.from_string('64020cc089b2544e23a7d6dc') } - let(:criteria) do - { - '$or' => [ - { 'a_string' => { '$gt' => feed_item.a_string } }, - { 'a_string' => feed_item.a_string, '_id' => { '$gt' => feed_item.id } } - ] - } - end - before(:each) do - allow(feed_item).to receive(:id).and_return(feed_id) - end - subject do - described_class.new(base64_string) - end - its(:value) { should eq feed_item.a_string } - its(:tiebreak_id) { should eq feed_item.id } - its(:criteria) do - should eq(criteria) - end - its(:to_s) { should eq(base64_string) } - - describe '.from_cursor' do - let(:base_cursor) { Mongoid::Scroll::Cursor.new "#{feed_item.a_string}:#{feed_item.id}", field_name: 'a_string', field_type: String } - it 'is properly created' do - base64_encoded_cursor = described_class.from_cursor(base_cursor) - expect(base64_encoded_cursor).to be_a(described_class) - expect(base64_encoded_cursor.tiebreak_id).to eq(feed_item.id) - expect(base64_encoded_cursor.value).to eq('astring') - expect(base64_encoded_cursor.criteria).to eq(criteria) - end - end - end - - context 'an invalid cursor' do - it 'raises a Mongoid::Scroll::Errors::InvalidCursorError with an invalid Base64 string' do - expect { described_class.new 'invalid' }.to raise_error Mongoid::Scroll::Errors::InvalidCursorError, /The cursor supplied is invalid: invalid./ - end - - it 'raises a Mongoid::Scroll::Errors::InvalidCursorError with an invalid JSON string' do - expect { described_class.new 'aW52YWxpZA==' }.to raise_error Mongoid::Scroll::Errors::InvalidCursorError, /The cursor supplied is invalid: aW52YWxpZA==./ - end - end -end diff --git a/spec/mongoid/criteria_spec.rb b/spec/mongoid/criteria_spec.rb index 9d1cb96..33521bb 100644 --- a/spec/mongoid/criteria_spec.rb +++ b/spec/mongoid/criteria_spec.rb @@ -1,263 +1,265 @@ require 'spec_helper' describe Mongoid::Criteria do - context 'with multiple sort fields' do - subject do - Feed::Item.desc(:name).asc(:value) - end - it ':scroll' do - expect(subject).to respond_to(:scroll) - end - it 'raises Mongoid::Scroll::Errors::MultipleSortFieldsError' do - expect { subject.scroll }.to raise_error Mongoid::Scroll::Errors::MultipleSortFieldsError, - /You're attempting to scroll over data with a sort order that includes multiple fields: name, value./ - end - end - - context 'with different sort fields between the cursor and the criteria' do - subject do - Feed::Item.desc(:name) - end - - it 'raises Mongoid::Scroll::Errors::MismatchedSortFieldsError' do - record = Feed::Item.create! - cursor = Mongoid::Scroll::Cursor.from_record(record, field: record.fields['a_string']) - error_string = /You're attempting to scroll over data with a sort order that differs between the cursor and the original criteria: field_name, direction./ - expect { subject.scroll(cursor) }.to raise_error Mongoid::Scroll::Errors::MismatchedSortFieldsError, error_string - end - end - - context 'with no sort' do - subject do - Feed::Item.all - end - it 'adds a default sort by _id' do - expect(subject.scroll.options[:sort]).to eq('_id' => 1) - end - end - context 'with data' do - before :each do - 10.times do |i| - Feed::Item.create!( - a_string: i.to_s, - a_integer: i, - a_datetime: DateTime.new(2013, i + 1, 21, 1, 42, 3, 'UTC'), - a_date: Date.new(2013, i + 1, 21), - a_time: Time.at(Time.now.to_i + i) - ) + [Mongoid::Scroll::Cursor, Mongoid::Scroll::Base64EncodedCursor].each do |cursor_type| + context cursor_type do + context 'with multiple sort fields' do + subject do + Feed::Item.desc(:name).asc(:value) + end + it ':scroll' do + expect(subject).to respond_to(:scroll) + end + it 'raises Mongoid::Scroll::Errors::MultipleSortFieldsError' do + expect do + subject.scroll(cursor_type) + end.to raise_error Mongoid::Scroll::Errors::MultipleSortFieldsError, /You're attempting to scroll over data with a sort order that includes multiple fields: name, value./ + end end - end - context 'default' do - it 'scrolls all' do - records = [] - Feed::Item.all.scroll do |record, _next_cursor| - records << record + context 'with different sort fields between the cursor and the criteria' do + subject do + Feed::Item.desc(:name) + end + it 'raises Mongoid::Scroll::Errors::MismatchedSortFieldsError' do + record = Feed::Item.create! + cursor = cursor_type.from_record(record, field: record.fields['a_string']) + error_string = /You're attempting to scroll over data with a sort order that differs between the cursor and the original criteria: field_name, direction./ + expect { subject.scroll(cursor) }.to raise_error Mongoid::Scroll::Errors::MismatchedSortFieldsError, error_string end - expect(records.size).to eq 10 - expect(records).to eq Feed::Item.all.to_a end - it 'does not change original criteria' do - criteria = Feed::Item.where(:a_time.gt => Time.new(2013, 7, 22, 1, 2, 3)) - original_criteria = criteria.dup - criteria.limit(2).scroll - expect(criteria).to eq original_criteria - cursor = nil - criteria.limit(2).scroll(cursor) do |_record, next_cursor| - cursor = next_cursor + context 'with no sort' do + subject do + Feed::Item.all end - criteria.scroll(cursor) do |_record, next_cursor| - cursor = next_cursor + it 'adds a default sort by _id' do + expect(subject.scroll(cursor_type).options[:sort]).to eq('_id' => 1) end - expect(criteria).to eq original_criteria end - end - - context 'with a foreign key' do - it 'sorts by object id' do - records = [] - Feed::Item.asc('publisher_id').scroll { |r, _| records << r } - expect(records).not_to be_empty + context 'with data' do + before :each do + 10.times do |i| + Feed::Item.create!( + name: i.to_s, + a_string: i.to_s, + a_integer: i, + a_datetime: DateTime.new(2013, i + 1, 21, 1, 42, 3, 'UTC'), + a_date: Date.new(2013, i + 1, 21), + a_time: Time.at(Time.now.to_i + i) + ) + end + end + context 'default' do + it 'scrolls all' do + records = [] + Feed::Item.all.scroll(cursor_type) do |record, _next_cursor| + records << record + end + expect(records.size).to eq 10 + expect(records).to eq Feed::Item.all.to_a + end + it 'does not change original criteria' do + criteria = Feed::Item.where(:a_time.gt => Time.new(2013, 7, 22, 1, 2, 3)) + original_criteria = criteria.dup + criteria.limit(2).scroll(cursor_type) + expect(criteria).to eq original_criteria + cursor = nil + criteria.limit(2).scroll(cursor) do |_record, next_cursor| + cursor = next_cursor + end + criteria.scroll(cursor) do |_record, next_cursor| + cursor = next_cursor + end + expect(criteria).to eq original_criteria + end + end + context 'with a foreign key' do + it 'sorts by object id' do + records = [] + Feed::Item.asc('publisher_id').scroll(cursor_type) { |r, _| records << r } + expect(records).not_to be_empty + end + end + { a_string: String, a_integer: Integer, a_date: Date, a_datetime: DateTime }.each_pair do |field_name, field_type| + context field_type do + it 'scrolls all with a block' do + records = [] + Feed::Item.asc(field_name).scroll(cursor_type) do |record, _next_cursor| + records << record + end + expect(records.size).to eq 10 + expect(records).to eq Feed::Item.all.to_a + end + it 'scrolls all with a break' do + records = [] + cursor = nil + Feed::Item.asc(field_name).limit(5).scroll(cursor_type) do |record, next_cursor| + records << record + cursor = next_cursor + end + expect(records.size).to eq 5 + Feed::Item.asc(field_name).scroll(cursor) do |record, next_cursor| + records << record + cursor = next_cursor + end + expect(records.size).to eq 10 + expect(records).to eq Feed::Item.all.to_a + end + it 'scrolls from a cursor' do + last_record = nil + cursor = nil + Feed::Item.asc(field_name).limit(5).scroll(cursor_type) do |record, next_cursor| + last_record = record + cursor = next_cursor + end + sixth_item = Feed::Item.asc(field_name).to_a[5] + from_item = Feed::Item.asc(field_name).scroll(cursor).to_a.first + expect(from_item).to eq sixth_item + end + it 'includes the current record when Mongoid::Scroll::Cursor#include_current is true' do + last_record = nil + cursor = nil + Feed::Item.asc(field_name).limit(5).scroll(cursor_type) do |record, next_cursor| + last_record = record + cursor = next_cursor + end + fifth_item = last_record + cursor.include_current = true + from_item = Feed::Item.asc(field_name).scroll(cursor).to_a.first + expect(from_item).to eq fifth_item + end + it 'scrolls in descending order' do + records = [] + Feed::Item.desc(field_name).limit(3).scroll(cursor_type) do |record, _next_cursor| + records << record + end + expect(records.size).to eq 3 + expect(records).to eq Feed::Item.desc(field_name).limit(3).to_a + end + it 'map' do + record = Feed::Item.desc(field_name).limit(3).scroll(cursor_type).map { |r| r }.last + expect(record).to_not be nil + cursor = cursor_type.from_record(record, field_type: field_type, field_name: field_name) + expect(cursor).to_not be nil + expect(cursor.tiebreak_id).to eq record.id + expect(cursor.value).to eq record.send(field_name) + end + it 'can be reused' do + ids = Feed::Item.asc(field_name).limit(2).map(&:id) + Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, cursor| + cursor.include_current = true + expect(Feed::Item.asc(field_name).limit(2).scroll(cursor).pluck(:id)).to eq ids + break + end + end + it 'can be re-created and reused' do + ids = Feed::Item.asc(field_name).limit(2).map(&:id) + Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, cursor| + new_cursor = cursor_type.new(cursor.to_s, field_type: field_type, field_name: field_name) + new_cursor.include_current = true + expect(Feed::Item.asc(field_name).limit(2).scroll(new_cursor).pluck(:id)).to eq ids + break + end + end + end + end end - end - - { a_string: String, a_integer: Integer, a_date: Date, a_datetime: DateTime }.each_pair do |field_name, field_type| - context field_type do - it 'scrolls all with a block' do - records = [] - Feed::Item.asc(field_name).scroll do |record, _next_cursor| - records << record + context 'with logic in initial criteria' do + before :each do + 3.times do |i| + Feed::Item.create!( + name: "Feed Item #{i}", + a_string: i.to_s, + a_integer: i, + a_datetime: DateTime.new(2015, i + 1, 21, 1, 42, 3, 'UTC'), + a_date: Date.new(2016, i + 1, 21), + a_time: Time.new(2015, i + 1, 22, 1, 2, 3) + ) end - expect(records.size).to eq 10 - expect(records).to eq Feed::Item.all.to_a + Feed::Item.create!( + name: 'Feed Item 3', + a_string: '3', + a_integer: 3, + a_datetime: DateTime.new(2015, 3, 2, 1, 2, 3), + a_date: Date.new(2012, 2, 3), + a_time: Time.new(2014, 2, 2, 1, 2, 3) + ) end - it 'scrolls all with a break' do + it 'respects original criteria with OR logic' do + criteria = Feed::Item.where( + '$or' => [{ :a_time.gt => Time.new(2015, 7, 22, 1, 2, 3) }, { :a_time.lte => Time.new(2015, 7, 22, 1, 2, 3), :a_date.gte => Date.new(2015, 7, 30) }] + ).asc(:a_time) records = [] cursor = nil - Feed::Item.asc(field_name).limit(5).scroll do |record, next_cursor| + criteria.limit(2).scroll(cursor_type) do |record, next_cursor| records << record cursor = next_cursor end - expect(records.size).to eq 5 - Feed::Item.asc(field_name).scroll(cursor) do |record, next_cursor| + expect(records.size).to eq 2 + expect(records.map(&:name)).to eq ['Feed Item 0', 'Feed Item 1'] + records = [] + criteria.limit(2).scroll(cursor) do |record, next_cursor| records << record cursor = next_cursor end - expect(records.size).to eq 10 - expect(records).to eq Feed::Item.all.to_a + expect(records.size).to eq 1 + expect(records.map(&:name)).to eq ['Feed Item 2'] end - it 'scrolls from a cursor' do - last_record = nil - cursor = nil - Feed::Item.asc(field_name).limit(5).scroll do |record, next_cursor| - last_record = record - cursor = next_cursor - end - sixth_item = Feed::Item.asc(field_name).to_a[5] - from_item = Feed::Item.asc(field_name).scroll(cursor).to_a.first - expect(from_item).to eq sixth_item - end - it 'includes the current record when Mongoid::Scroll::Cursor#include_current is true' do - last_record = nil - cursor = nil - Feed::Item.asc(field_name).limit(5).scroll do |record, next_cursor| - last_record = record - cursor = next_cursor - end - fifth_item = last_record - cursor.include_current = true - from_item = Feed::Item.asc(field_name).scroll(cursor).to_a.first - expect(from_item).to eq fifth_item + end + context 'with embeddable objects' do + before do + @item = Feed::Item.create! a_integer: 1, name: 'item' + @embedded_item = Feed::EmbeddedItem.create! name: 'embedded', item: @item end - it 'scrolls in descending order' do + it 'respects embedded queries' do records = [] - Feed::Item.desc(field_name).limit(3).scroll do |record, _next_cursor| + criteria = @item.embedded_items.limit(2) + criteria.scroll(cursor_type) do |record, _next_cursor| records << record end - expect(records.size).to eq 3 - expect(records).to eq Feed::Item.desc(field_name).limit(3).to_a + expect(records.size).to eq 1 + expect(records.map(&:name)).to eq ['embedded'] end - it 'map' do - record = Feed::Item.desc(field_name).limit(3).scroll.map { |r| r }.last - expect(record).to_not be nil - cursor = Mongoid::Scroll::Cursor.from_record(record, field_type: field_type, field_name: field_name) - expect(cursor).to_not be nil - expect(cursor.to_s.split(':')).to eq [ - Mongoid::Scroll::Cursor.transform_field_value(field_type, field_name, record.send(field_name)).to_s, - record.id.to_s - ] - end - end - end - end - context 'with logic in initial criteria' do - before :each do - 3.times do |i| - Feed::Item.create!( - name: "Feed Item #{i}", - a_string: i.to_s, - a_integer: i, - a_datetime: DateTime.new(2015, i + 1, 21, 1, 42, 3, 'UTC'), - a_date: Date.new(2016, i + 1, 21), - a_time: Time.new(2015, i + 1, 22, 1, 2, 3) - ) end - Feed::Item.create!( - name: 'Feed Item 3', - a_string: '3', - a_integer: 3, - a_datetime: DateTime.new(2015, 3, 2, 1, 2, 3), - a_date: Date.new(2012, 2, 3), - a_time: Time.new(2014, 2, 2, 1, 2, 3) - ) - end - it 'respects original criteria with OR logic' do - criteria = Feed::Item.where( - '$or' => [{ :a_time.gt => Time.new(2015, 7, 22, 1, 2, 3) }, { :a_time.lte => Time.new(2015, 7, 22, 1, 2, 3), :a_date.gte => Date.new(2015, 7, 30) }] - ).asc(:a_time) - records = [] - cursor = nil - criteria.limit(2).scroll do |record, next_cursor| - records << record - cursor = next_cursor - end - expect(records.size).to eq 2 - expect(records.map(&:name)).to eq ['Feed Item 0', 'Feed Item 1'] - records = [] - criteria.limit(2).scroll(cursor) do |record, next_cursor| - records << record - cursor = next_cursor - end - expect(records.size).to eq 1 - expect(records.map(&:name)).to eq ['Feed Item 2'] - end - it 'merges cursor criteria when no sort field is given' do - criteria = Feed::Item.where(:a_time.gt => Time.new(2013, 7, 22, 1, 2, 3)) - feed_item = Feed::Item.where(name: 'Feed Item 1').first - cursor_input = "#{feed_item.id}:#{feed_item.id}" - field_type = Mongoid::Compatibility::Version.mongoid3? ? Moped::BSON::ObjectId : BSON::ObjectId - cursor_options = { field_type: field_type, field_name: '_id', direction: 1 } - cursor = Mongoid::Scroll::Cursor.new(cursor_input, cursor_options) - records = [] - criteria.limit(2).scroll(cursor) do |record, next_cursor| - records << record - cursor = next_cursor - end - expect(records.size).to eq 2 - end - end - context 'with embeddable objects' do - before do - @item = Feed::Item.create! a_integer: 1, name: 'item' - @embedded_item = Feed::EmbeddedItem.create! name: 'embedded', item: @item - end - it 'respects embedded queries' do - records = [] - criteria = @item.embedded_items.limit(2) - criteria.scroll do |record, _next_cursor| - records << record + context 'with overlapping data', if: MongoDB.mmapv1? do + before :each do + 3.times { Feed::Item.create! a_integer: 5 } + Feed::Item.first.update_attributes!(name: Array(1000).join('a')) + end + it 'natural order is different from order by id' do + # natural order isn't necessarily going to be the same as _id order + # if a document is updated and grows in size, it may need to be relocated and + # thus cause the natural order to change + expect(Feed::Item.order_by('$natural' => 1).to_a).to_not eq(Feed::Item.order_by(_id: 1).to_a) + end + [{ a_integer: 1 }, { a_integer: -1 }].each do |sort_order| + it "scrolls by #{sort_order}" do + records = [] + cursor = nil + Feed::Item.order_by(sort_order).limit(2).scroll(cursor_type) do |record, next_cursor| + records << record + cursor = next_cursor + end + expect(records.size).to eq 2 + Feed::Item.order_by(sort_order).scroll(cursor) do |record, _next_cursor| + records << record + end + expect(records.size).to eq 3 + expect(records).to eq Feed::Item.all.sort(_id: sort_order[:a_integer]).to_a + end + end end - expect(records.size).to eq 1 - expect(records.map(&:name)).to eq ['embedded'] - end - end - context 'with overlapping data', if: MongoDB.mmapv1? do - before :each do - 3.times { Feed::Item.create! a_integer: 5 } - Feed::Item.first.update_attributes!(name: Array(1000).join('a')) - end - it 'natural order is different from order by id' do - # natural order isn't necessarily going to be the same as _id order - # if a document is updated and grows in size, it may need to be relocated and - # thus cause the natural order to change - expect(Feed::Item.order_by('$natural' => 1).to_a).to_not eq(Feed::Item.order_by(_id: 1).to_a) - end - [{ a_integer: 1 }, { a_integer: -1 }].each do |sort_order| - it "scrolls by #{sort_order}" do - records = [] - cursor = nil - Feed::Item.order_by(sort_order).limit(2).scroll do |record, next_cursor| - records << record - cursor = next_cursor + context 'with several records having the same value' do + before :each do + 3.times { Feed::Item.create! a_integer: 5 } end - expect(records.size).to eq 2 - Feed::Item.order_by(sort_order).scroll(cursor) do |record, _next_cursor| - records << record + it 'returns records from the current one when Mongoid::Scroll::Cursor#include_current is true' do + _first_item, second_item, third_item = Feed::Item.asc(:a_integer).to_a + cursor = Mongoid::Scroll::Cursor.from_record(second_item, field: Feed::Item.fields['a_integer']) + cursor.include_current = true + items = Feed::Item.asc(:a_integer).limit(2).scroll(cursor).to_a + expect(items).to eq([second_item, third_item]) end - expect(records.size).to eq 3 - expect(records).to eq Feed::Item.all.sort(_id: sort_order[:a_integer]).to_a end end end - context 'with several records having the same value' do - before :each do - 3.times { Feed::Item.create! a_integer: 5 } - end - it 'returns records from the current one when Mongoid::Scroll::Cursor#include_current is true' do - _first_item, second_item, third_item = Feed::Item.asc(:a_integer).to_a - cursor = Mongoid::Scroll::Cursor.from_record(second_item, field: Feed::Item.fields['a_integer']) - cursor.include_current = true - items = Feed::Item.asc(:a_integer).limit(2).scroll(cursor).to_a - expect(items).to eq([second_item, third_item]) - end - end end diff --git a/spec/mongoid/scroll_cursor_spec.rb b/spec/mongoid/cursor_spec.rb similarity index 80% rename from spec/mongoid/scroll_cursor_spec.rb rename to spec/mongoid/cursor_spec.rb index edca9fa..e039bf5 100644 --- a/spec/mongoid/scroll_cursor_spec.rb +++ b/spec/mongoid/cursor_spec.rb @@ -17,22 +17,15 @@ end context 'an id field cursor' do let(:feed_item) { Feed::Item.create!(a_string: 'astring') } - field_type = Mongoid::Compatibility::Version.mongoid3? ? Moped::BSON::ObjectId : BSON::ObjectId subject do - Mongoid::Scroll::Cursor.new "#{feed_item.id}:#{feed_item.id}", field_name: '_id', field_type: field_type, direction: 1 + Mongoid::Scroll::Cursor.new "#{feed_item.id}:#{feed_item.id}", field_name: '_id', field_type: BSON::ObjectId, direction: 1 end - its(:value) { should eq feed_item.id.to_s } + its(:value) { should eq feed_item.id } its(:tiebreak_id) { should eq feed_item.id } its(:criteria) do - if Mongoid::Compatibility::Version.mongoid3? - should eq('$or' => [ - { '_id' => { '$gt' => Moped::BSON::ObjectId(feed_item.id.to_s) } } - ]) - else - should eq('$or' => [ - { '_id' => { '$gt' => BSON::ObjectId(feed_item.id.to_s) } } - ]) - end + should eq('$or' => [ + { '_id' => { '$gt' => BSON::ObjectId(feed_item.id.to_s) } } + ]) end end context 'a string field cursor' do @@ -146,16 +139,38 @@ its(:value) { should eq 'astring' } its(:tiebreak_id) { should eq feed_item.id } its(:criteria) do - if Mongoid::Compatibility::Version.mongoid3? - should eq('$or' => [ - { 'a_string' => { '$gt' => 'astring' } }, - { '_id' => { '$gte' => Moped::BSON::ObjectId(feed_item.id.to_s) }, 'a_string' => 'astring' } - ]) - else - should eq('$or' => [ - { 'a_string' => { '$gt' => 'astring' } }, - { '_id' => { '$gte' => BSON::ObjectId(feed_item.id.to_s) }, 'a_string' => 'astring' } - ]) + should eq('$or' => [ + { 'a_string' => { '$gt' => 'astring' } }, + { '_id' => { '$gte' => BSON::ObjectId(feed_item.id.to_s) }, 'a_string' => 'astring' } + ]) + end + end + context 'criteria' do + context 'with data' do + before :each do + 3.times do |i| + Feed::Item.create!( + name: "Feed Item #{i}", + a_time: Time.new(2015, i + 1, 22, 1, 2, 3) + ) + end + Feed::Item.create!( + name: 'Feed Item 3', + a_time: Time.new(2014, 2, 2, 1, 2, 3) + ) + end + it 'merges cursor criteria when no sort field is given' do + criteria = Feed::Item.where(:a_time.gt => Time.new(2013, 7, 22, 1, 2, 3)) + feed_item = Feed::Item.where(name: 'Feed Item 1').first + cursor_input = "#{feed_item.id}:#{feed_item.id}" + cursor_options = { field_type: BSON::ObjectId, field_name: '_id', direction: 1 } + cursor = Mongoid::Scroll::Cursor.new(cursor_input, cursor_options) + records = [] + criteria.limit(2).scroll(cursor) do |record, next_cursor| + records << record + cursor = next_cursor + end + expect(records.size).to eq 2 end end end diff --git a/spec/moped/query_spec.rb b/spec/moped/query_spec.rb deleted file mode 100644 index 9ed50a4..0000000 --- a/spec/moped/query_spec.rb +++ /dev/null @@ -1,138 +0,0 @@ -require 'spec_helper' - -if Object.const_defined?(:Moped) - describe Moped::Query do - context 'scrollable' do - subject do - Mongoid.default_session['feed_items'].find - end - it ':scroll' do - expect(subject).to respond_to(:scroll) - end - end - context 'with multiple sort fields' do - subject do - Mongoid.default_session['feed_items'].find.sort(name: 1, value: -1) - end - it 'raises Mongoid::Scroll::Errors::MultipleSortFieldsError' do - expect { subject.scroll }.to raise_error Mongoid::Scroll::Errors::MultipleSortFieldsError, - /You're attempting to scroll over data with a sort order that includes multiple fields: name, value./ - end - end - context 'with different sort fields between the cursor and the criteria' do - subject do - Mongoid.default_session['feed_items'].find.sort(name: -1) - end - - it 'raises Mongoid::Scroll::Errors::MismatchedSortFieldsError' do - record = Feed::Item.create! - cursor = Mongoid::Scroll::Cursor.from_record(record, field: record.fields['a_string']) - error_string = /You're attempting to scroll over data with a sort order that differs between the cursor and the original criteria: field_name, direction./ - expect { subject.scroll(cursor, field_type: String) }.to raise_error Mongoid::Scroll::Errors::MismatchedSortFieldsError, error_string - end - end - context 'with no sort' do - subject do - Mongoid.default_session['feed_items'].find - end - it 'adds a default sort by _id' do - expect(subject.scroll.operation.selector['$orderby']).to eq(_id: 1) - end - end - context 'with data' do - before :each do - 10.times do |i| - Mongoid.default_session['feed_items'].insert( - a_string: i.to_s, - a_integer: i, - a_datetime: DateTime.mongoize(DateTime.new(2013, i + 1, 21, 1, 42, 3, 'UTC')), - a_date: Date.mongoize(Date.new(2013, i + 1, 21)), - a_time: Time.mongoize(Time.at(Time.now.to_i + i)) - ) - end - end - context 'default' do - it 'scrolls all' do - records = [] - Mongoid.default_session['feed_items'].find.scroll do |record, _next_cursor| - records << record - end - expect(records.size).to eq 10 - expect(records).to eq Mongoid.default_session['feed_items'].find.to_a - end - end - { a_string: String, a_integer: Integer, a_date: Date, a_datetime: DateTime }.each_pair do |field_name, field_type| - context field_type do - it 'scrolls all with a block' do - records = [] - Mongoid.default_session['feed_items'].find.sort(field_name => 1).scroll(nil, field_type: field_type) do |record, _next_cursor| - records << record - end - expect(records.size).to eq 10 - expect(records).to eq Mongoid.default_session['feed_items'].find.to_a - end - it 'scrolls all with a break' do - records = [] - cursor = nil - Mongoid.default_session['feed_items'].find.sort(field_name => 1).limit(5).scroll(nil, field_type: field_type) do |record, next_cursor| - records << record - cursor = next_cursor - end - expect(records.size).to eq 5 - Mongoid.default_session['feed_items'].find.sort(field_name => 1).scroll(cursor, field_type: field_type) do |record, next_cursor| - records << record - cursor = next_cursor - end - expect(records.size).to eq 10 - expect(records).to eq Mongoid.default_session['feed_items'].find.to_a - end - it 'scrolls in descending order' do - records = [] - Mongoid.default_session['feed_items'].find.sort(field_name => -1).limit(3).scroll(nil, field_type: field_type, field_name: field_name) do |record, _next_cursor| - records << record - end - expect(records.size).to eq 3 - expect(records).to eq Mongoid.default_session['feed_items'].find.sort(field_name => -1).limit(3).to_a - end - it 'map' do - record = Mongoid.default_session['feed_items'].find.limit(3).scroll(nil, field_type: field_type, field_name: field_name).map { |r| r }.last - cursor = Mongoid::Scroll::Cursor.from_record(record, field_type: field_type, field_name: field_name) - expect(cursor).to_not be nil - expect(cursor.to_s.split(':')).to eq [ - Mongoid::Scroll::Cursor.transform_field_value(field_type, field_name, record[field_name.to_s]).to_s, - record['_id'].to_s - ] - end - end - end - end - context 'with overlapping data', if: MongoDB.mmapv1? do - before :each do - 3.times { Feed::Item.create! a_integer: 5 } - Feed::Item.first.update_attributes!(name: Array(1000).join('a')) - end - it 'natural order is different from order by id' do - # natural order isn't necessarily going to be the same as _id order - # if a document is updated and grows in size, it may need to be relocated and - # thus cause the natural order to change - expect(Feed::Item.order_by('$natural' => 1).to_a).to_not eq Feed::Item.order_by(_id: 1).to_a - end - [{ a_integer: 1 }, { a_integer: -1 }].each do |sort_order| - it "scrolls by #{sort_order}" do - records = [] - cursor = nil - Mongoid.default_session['feed_items'].find.sort(sort_order).limit(2).scroll do |record, next_cursor| - records << record - cursor = next_cursor - end - expect(records.size).to eq 2 - Mongoid.default_session['feed_items'].find.sort(sort_order).scroll(cursor) do |record, _next_cursor| - records << record - end - expect(records.size).to eq 3 - expect(records).to eq Mongoid.default_session['feed_items'].find.sort(sort_order.merge(_id: sort_order[:a_integer])).to_a - end - end - end - end -end diff --git a/spec/support/feed/item.rb b/spec/support/feed/item.rb index 39d928d..3e0bb52 100644 --- a/spec/support/feed/item.rb +++ b/spec/support/feed/item.rb @@ -13,7 +13,7 @@ class Item embeds_many :embedded_items, class_name: 'Feed::EmbeddedItem' publisher_options = { class_name: 'Feed::Publisher' } - publisher_options[:optional] = true if Mongoid::Compatibility::Version.mongoid6? || Mongoid::Compatibility::Version.mongoid7? + publisher_options[:optional] = true if Mongoid::Compatibility::Version.mongoid6_or_newer? belongs_to :publisher, publisher_options end end