lib/aws/s3/paginated_collection.rb



# Copyright 2011 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You
# may not use this file except in compliance with the License. A copy of
# the License is located at
#
#     http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is
# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

module AWS
  class S3

    # @private
    module PaginatedCollection

      def each(options = {}, &block)
        each_page(options) do |page|
          each_member_in_page(page, &block)
        end
        nil
      end

      protected
      def each_member_in_page(page, &block); end

      protected
      def each_page(options = {}, &block)
        opts = list_options(options)
        limit = options[:limit]
        batch_size = options[:batch_size] || 1000
        markers = {}
        received = 0

        loop do
          page_opts = opts.dup
          page_opts.merge!(markers)
          page_opts[limit_param] =
            limit ? [limit - received, batch_size].min : batch_size

          page = list_request(page_opts)
          markers = next_markers(page)
          received += page_size(page)

          yield(page)

          return unless page.truncated?
        end
      end

      protected
      def list_request(options)
        raise NotImplementedError
      end

      protected
      def list_options(options)
        opts = {}
        opts[:bucket_name] = bucket.name if respond_to?(:bucket)
        opts
      end

      protected
      def limit_param
        raise NotImplementedError
      end

      protected
      def pagination_markers
        [:key_marker]
      end

      protected
      def next_markers(page)
        pagination_markers.inject({}) do |markers, marker|
          markers[marker] = page.send("next_#{marker}")
          markers
        end
      end

      protected
      def page_size(resp)
        (resp.respond_to?(:common_prefixes) and
         prefixes = resp.common_prefixes and
         prefixes.size) or 0
      end

    end

  end
end