Fix #741 Added list objects v2 api support

lib/fog/aws/parsers/storage/list_objects_v2.rb

@@ -0,0 +1,66 @@
+module Fog
+  module Parsers
+    module AWS
+      module Storage
+        class ListObjectsV2 < Fog::Parsers::Base
+          # Initialize parser state
+          def initialize
+            super
+            @common_prefix = {}
+            @object = { 'Owner' => {} }
+            reset
+          end
+
+          def reset
+            @object = { 'Owner' => {} }
+            @response = { 'Contents' => [], 'CommonPrefixes' => [] }
+          end
+
+          def start_element(name, attrs = [])
+            super
+            case name
+            when 'CommonPrefixes'
+              @in_common_prefixes = true
+            end
+          end
+
+          def end_element(name)
+            case name
+            when 'CommonPrefixes'
+              @in_common_prefixes = false
+            when 'Contents'
+              @response['Contents'] << @object
+              @object = { 'Owner' => {} }
+            when 'DisplayName', 'ID'
+              @object['Owner'][name] = value
+            when 'ETag'
+              @object[name] = value.gsub('"', '') if value != nil
+            when 'IsTruncated'
+              if value == 'true'
+                @response['IsTruncated'] = true
+              else
+                @response['IsTruncated'] = false
+              end
+            when 'LastModified'
+              @object['LastModified'] = Time.parse(value)
+            when 'ContinuationToken', 'NextContinuationToken', 'Name', 'StartAfter'
+              @response[name] = value
+            when 'MaxKeys', 'KeyCount'
+              @response[name] = value.to_i
+            when 'Prefix'
+              if @in_common_prefixes
+                @response['CommonPrefixes'] << value
+              else
+                @response[name] = value
+              end
+            when 'Size'
+              @object['Size'] = value.to_i
+            when 'Delimiter', 'Key', 'StorageClass'
+              @object[name] = value
+            end
+          end
+        end
+      end
+    end
+  end
+end 

lib/fog/aws/requests/storage/list_objects_v2.rb

@@ -0,0 +1,129 @@
+module Fog
+  module AWS
+    class Storage
+      class Real
+        require 'fog/aws/parsers/storage/list_objects_v2'
+
+        # List information about objects in an S3 bucket using ListObjectsV2
+        #
+        # @param bucket_name [String] name of bucket to list object keys from
+        # @param options [Hash] config arguments for list.  Defaults to {}.
+        # @option options delimiter [String] causes keys with the same string between the prefix
+        #     value and the first occurrence of delimiter to be rolled up
+        # @option options continuation-token [String] continuation token from a previous request
+        # @option options fetch-owner [Boolean] specifies whether to return owner information
+        # @option options max-keys [Integer] limits number of object keys returned
+        # @option options prefix [String] limits object keys to those beginning with its value
+        # @option options start-after [String] starts listing after this specified key
+        #
+        # @return [Excon::Response] response:
+        #   * body [Hash]:
+        #     * Delimiter [String] - Delimiter specified for query
+        #     * IsTruncated [Boolean] - Whether or not the listing is truncated
+        #     * ContinuationToken [String] - Token specified in the request
+        #     * NextContinuationToken [String] - Token to use in subsequent requests
+        #     * KeyCount [Integer] - Number of keys returned
+        #     * MaxKeys [Integer] - Maximum number of keys specified for query
+        #     * Name [String] - Name of the bucket
+        #     * Prefix [String] - Prefix specified for query
+        #     * StartAfter [String] - StartAfter specified in the request
+        #     * CommonPrefixes [Array] - Array of strings for common prefixes
+        #     * Contents [Array]:
+        #       * ETag [String] - Etag of object
+        #       * Key [String] - Name of object
+        #       * LastModified [String] - Timestamp of last modification of object
+        #       * Owner [Hash]:
+        #         * DisplayName [String] - Display name of object owner
+        #         * ID [String] - Id of object owner
+        #       * Size [Integer] - Size of object
+        #       * StorageClass [String] - Storage class of object
+        #
+        # @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
+
+        def list_objects_v2(bucket_name, options = {})
+          unless bucket_name
+            raise ArgumentError.new('bucket_name is required')
+          end
+
+          # Add list-type=2 to indicate ListObjectsV2
+          options = options.merge('list-type' => '2')
+
+          request({
+            :expects  => 200,
+            :headers  => {},
+            :bucket_name => bucket_name,
+            :idempotent => true,
+            :method   => 'GET',
+            :parser   => Fog::Parsers::AWS::Storage::ListObjectsV2.new,
+            :query    => options
+          })
+        end
+      end
+
+      class Mock # :nodoc:all
+        def list_objects_v2(bucket_name, options = {})
+          prefix = options['prefix']
+          continuation_token = options['continuation-token']
+          delimiter = options['delimiter']
+          max_keys = options['max-keys']
+          start_after = options['start-after']
+          fetch_owner = options['fetch-owner']
+          common_prefixes = []
+
+          unless bucket_name
+            raise ArgumentError.new('bucket_name is required')
+          end
+
+          response = Excon::Response.new
+          if bucket = self.data[:buckets][bucket_name]
+            contents = bucket[:objects].values.map(&:first).sort {|x,y| x['Key'] <=> y['Key']}.reject do |object|
+                (prefix    && object['Key'][0...prefix.length] != prefix) ||
+                (start_after && object['Key'] <= start_after) ||
+                (continuation_token && object['Key'] <= continuation_token) ||
+                (delimiter && object['Key'][(prefix ? prefix.length : 0)..-1].include?(delimiter) \
+                           && common_prefixes << object['Key'].sub(/^(#{prefix}[^#{delimiter}]+.).*/, '\1')) ||
+                object.key?(:delete_marker)
+              end.map do |object|
+                data = object.reject {|key, value| !['ETag', 'Key', 'StorageClass'].include?(key)}
+                data.merge!({
+                  'LastModified' => Time.parse(object['Last-Modified']),
+                  'Owner'        => fetch_owner ? bucket['Owner'] : nil,
+                  'Size'         => object['Content-Length'].to_i
+                })
+              data
+            end
+
+            max_keys = max_keys || 1000
+            size = [max_keys, 1000].min
+            truncated_contents = contents[0...size]
+            next_token = truncated_contents.size != contents.size ? truncated_contents.last['Key'] : nil
+
+            response.status = 200
+            common_prefixes_uniq = common_prefixes.uniq
+            response.body = {
+              'CommonPrefixes'  => common_prefixes_uniq,
+              'Contents'        => truncated_contents,
+              'IsTruncated'     => truncated_contents.size != contents.size,
+              'ContinuationToken' => continuation_token,
+              'NextContinuationToken' => next_token,
+              'KeyCount'        => truncated_contents.size + common_prefixes_uniq.size,
+              'MaxKeys'         => max_keys,
+              'Name'            => bucket['Name'],
+              'Prefix'          => prefix,
+              'StartAfter'      => start_after
+            }
+            if max_keys && max_keys < response.body['Contents'].length
+                response.body['IsTruncated'] = true
+                response.body['Contents'] = response.body['Contents'][0...max_keys]
+                response.body['KeyCount'] = response.body['Contents'].size + response.body['CommonPrefixes'].size
+            end
+          else
+            response.status = 404
+            raise(Excon::Errors.status_error({:expects => 200}, response))
+          end
+          response
+        end
+      end
+    end
+  end
+end 

lib/fog/aws/storage.rb

@@ -26,9 +26,12 @@ class Storage < Fog::Service
 
       VALID_QUERY_KEYS = %w[
         acl
+        continuation-token
         cors
         delete
+        fetch-owner
         lifecycle
+        list-type
         location
         logging
         notification
@@ -42,6 +45,7 @@ class Storage < Fog::Service
         response-content-type
         response-expires
         restore
+        start-after
         tagging
         torrent
         uploadId
@@ -102,6 +106,7 @@ class Storage < Fog::Service
       request :head_object_url
       request :initiate_multipart_upload
       request :list_multipart_uploads
+      request :list_objects_v2
       request :list_parts
       request :post_object_hidden_fields
       request :post_object_restore

tests/models/storage/files_v2_tests.rb

@@ -0,0 +1,116 @@
+Shindo.tests("Storage[:aws] | ListObjectsV2 API", ["aws"]) do
+
+  directory_attributes = {
+      :key => uniq_id('foglistobjectsv2tests')
+  }
+
+  @directory = Fog::Storage[:aws].directories.create(directory_attributes)
+
+  tests('Direct ListObjectsV2 API usage') do
+
+    # Create some test files
+    file1 = @directory.files.create(:body => 'test1', :key => 'prefix/file1.txt')
+    file2 = @directory.files.create(:body => 'test2', :key => 'prefix/file2.txt')
+    file3 = @directory.files.create(:body => 'test3', :key => 'other/file3.txt')
+    file4 = @directory.files.create(:body => 'test4', :key => 'file4.txt')
+
+    tests('#list_objects_v2 basic functionality') do
+      response = Fog::Storage[:aws].list_objects_v2(@directory.key)
+
+      tests('returns proper response structure').returns(true) do
+        response.body.has_key?('Contents') && 
+        response.body.has_key?('KeyCount') &&
+        response.body.has_key?('IsTruncated')
+      end
+
+      tests('returns all files').returns(4) do
+        response.body['Contents'].size
+      end
+
+      tests('has V2-specific KeyCount').returns(4) do
+        response.body['KeyCount']
+      end
+    end
+
+    tests('#list_objects_v2 with parameters') do
+
+      tests('with prefix') do
+        response = Fog::Storage[:aws].list_objects_v2(@directory.key, 'prefix' => 'prefix/')
+
+        tests('filters by prefix').returns(2) do
+          response.body['Contents'].size
+        end
+
+        tests('KeyCount reflects filtered results').returns(2) do
+          response.body['KeyCount']
+        end
+      end
+
+      tests('with max-keys') do
+        response = Fog::Storage[:aws].list_objects_v2(@directory.key, 'max-keys' => 2)
+
+        tests('limits results').returns(2) do
+          response.body['Contents'].size
+        end
+
+        tests('is truncated').returns(true) do
+          response.body['IsTruncated']
+        end
+
+        tests('has next continuation token').returns(true) do
+          !response.body['NextContinuationToken'].nil?
+        end
+      end
+
+      tests('with start-after') do
+        response = Fog::Storage[:aws].list_objects_v2(@directory.key, 'start-after' => 'other/file3.txt')
+
+        tests('starts after specified key').returns(true) do
+          keys = response.body['Contents'].map { |obj| obj['Key'] }
+          keys.none? { |key| key <= 'other/file3.txt' }
+        end
+      end
+
+      tests('with delimiter') do
+        response = Fog::Storage[:aws].list_objects_v2(@directory.key, 'delimiter' => '/')
+
+        tests('respects delimiter').returns(true) do
+          # Should have common prefixes and fewer direct contents
+          response.body.has_key?('CommonPrefixes') && response.body['CommonPrefixes'].size > 0
+        end
+      end
+
+      tests('with fetch-owner') do
+        response = Fog::Storage[:aws].list_objects_v2(@directory.key, 'fetch-owner' => true)
+
+        tests('request succeeds').returns(true) do
+          response.body.has_key?('Contents')
+        end
+      end unless Fog.mocking?
+
+    end
+
+    tests('pagination with continuation token') do
+      first_page = Fog::Storage[:aws].list_objects_v2(@directory.key, 'max-keys' => 2)
+
+      if first_page.body['IsTruncated'] && first_page.body['NextContinuationToken']
+        second_page = Fog::Storage[:aws].list_objects_v2(@directory.key, 'continuation-token' => first_page.body['NextContinuationToken'])
+
+        tests('second page has different objects').returns(true) do
+          first_keys = first_page.body['Contents'].map { |obj| obj['Key'] }
+          second_keys = second_page.body['Contents'].map { |obj| obj['Key'] }
+          (first_keys & second_keys).empty?
+        end
+      end
+    end
+
+    # Clean up test files
+    file1.destroy
+    file2.destroy
+    file3.destroy
+    file4.destroy
+  end
+
+  @directory.destroy
+
+end