Skip to content

Commit 27b5934

Browse files
author
Shreya
authored
feat: support virtual sharding and stream failure processing (#1261) (#1262)
1 parent bf5720d commit 27b5934

File tree

10 files changed

+311
-115
lines changed

10 files changed

+311
-115
lines changed

‎examples/2016-10-31/stream_processor/template.yaml‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,21 +8,33 @@ Resources:
88
Handler: index.handler
99
Runtime: nodejs10.x
1010
CodeUri: src/
11+
Policies:
12+
- SNSPublishMessagePolicy:
13+
TopicName: !GetAtt MySnsTopic.TopicName
1114
Events:
1215
Stream:
1316
Type: Kinesis
1417
Properties:
1518
Stream: !GetAtt Stream.Arn
1619
MaximumBatchingWindowInSeconds: 20
20+
ParallelizationFactor: 8
21+
MaximumRetryAttempts: 100
22+
BisectBatchOnFunctionError: true
23+
MaximumRecordAgeInSeconds: 604800
1724
StartingPosition: TRIM_HORIZON
25+
DestinationConfig:
26+
OnFailure:
27+
Destination: !Ref MySnsTopic
1828

1929
Stream:
2030
Type: AWS::Kinesis::Stream
2131
Properties:
2232
ShardCount: 1
2333

24-
Outputs:
34+
MySnsTopic:
35+
Type: AWS::SNS::Topic
2536

37+
Outputs:
2638
KinesisStream:
2739
Description: "Kinesis Stream that will trigger Lambda function upon new records"
2840
Value: !GetAtt Stream.Arn

‎samtranslator/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = '1.15.1'
1+
__version__ = '1.16.0'

‎samtranslator/model/eventsources/pull.py‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,12 @@ class PullEventSource(ResourceMacro):
2222
'BatchSize': PropertyType(False, is_type(int)),
2323
'StartingPosition': PropertyType(False, is_str()),
2424
'Enabled': PropertyType(False, is_type(bool)),
25-
'MaximumBatchingWindowInSeconds': PropertyType(False, is_type(int))
25+
'MaximumBatchingWindowInSeconds': PropertyType(False, is_type(int)),
26+
'MaximumRetryAttempts': PropertyType(False, is_type(int)),
27+
'BisectBatchOnFunctionError': PropertyType(False, is_type(bool)),
28+
'MaximumRecordAgeInSeconds': PropertyType(False, is_type(int)),
29+
'DestinationConfig': PropertyType(False, is_type(dict)),
30+
'ParallelizationFactor': PropertyType(False, is_type(int))
2631
}
2732

2833
def get_policy_arn(self):
@@ -66,6 +71,12 @@ def to_cloudformation(self, **kwargs):
6671
lambda_eventsourcemapping.BatchSize = self.BatchSize
6772
lambda_eventsourcemapping.Enabled = self.Enabled
6873
lambda_eventsourcemapping.MaximumBatchingWindowInSeconds = self.MaximumBatchingWindowInSeconds
74+
lambda_eventsourcemapping.MaximumRetryAttempts = self.MaximumRetryAttempts
75+
lambda_eventsourcemapping.BisectBatchOnFunctionError = self.BisectBatchOnFunctionError
76+
lambda_eventsourcemapping.MaximumRecordAgeInSeconds = self.MaximumRecordAgeInSeconds
77+
lambda_eventsourcemapping.DestinationConfig = self.DestinationConfig
78+
lambda_eventsourcemapping.ParallelizationFactor = self.ParallelizationFactor
79+
6980
if 'Condition' in function.resource_attributes:
7081
lambda_eventsourcemapping.set_resource_attribute('Condition', function.resource_attributes['Condition'])
7182

‎samtranslator/model/lambda_.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,11 @@ class LambdaEventSourceMapping(Resource):
6666
'EventSourceArn': PropertyType(True, is_str()),
6767
'FunctionName': PropertyType(True, is_str()),
6868
'MaximumBatchingWindowInSeconds': PropertyType(False, is_type(int)),
69+
'MaximumRetryAttempts': PropertyType(False, is_type(int)),
70+
'BisectBatchOnFunctionError': PropertyType(False, is_type(bool)),
71+
'MaximumRecordAgeInSeconds': PropertyType(False, is_type(int)),
72+
'DestinationConfig': PropertyType(False, is_type(dict)),
73+
'ParallelizationFactor': PropertyType(False, is_type(int)),
6974
'StartingPosition': PropertyType(False, is_str())
7075
}
7176

‎tests/translator/input/function_with_batch_window.yaml‎ renamed to ‎tests/translator/input/function_with_event_source_mapping.yaml‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
AWSTemplateFormatVersion: '2010-09-09'
22
Transform: AWS::Serverless-2016-10-31
3-
Description: EventSourceMapping example with MaximumBatchingWindowInSeconds property
43

54
Parameters:
65
MyBatchingWindowParam:
@@ -23,6 +22,9 @@ Resources:
2322
}
2423
}
2524
Runtime: nodejs8.10
25+
Policies:
26+
- SQSSendMessagePolicy:
27+
QueueName: !GetAtt MySqsQueue.QueueName
2628
Events:
2729
Stream:
2830
Type: Kinesis
@@ -42,7 +44,14 @@ Resources:
4244
Stream: !GetAtt DynamoDBTable.StreamArn
4345
BatchSize: 100
4446
MaximumBatchingWindowInSeconds: !Ref MyBatchingWindowParam
47+
ParallelizationFactor: 8
48+
MaximumRetryAttempts: 100
49+
BisectBatchOnFunctionError: true
50+
MaximumRecordAgeInSeconds: 86400
4551
StartingPosition: TRIM_HORIZON
52+
DestinationConfig:
53+
OnFailure:
54+
Destination: !GetAtt MySqsQueue.Arn
4655

4756
KinesisStream:
4857
Type: AWS::Kinesis::Stream
@@ -66,4 +75,7 @@ Resources:
6675
ReadCapacityUnits: 5
6776
WriteCapacityUnits: 5
6877
StreamSpecification:
69-
StreamViewType: NEW_IMAGE
78+
StreamViewType: NEW_IMAGE
79+
80+
MySqsQueue:
81+
Type: AWS::SQS::Queue

‎tests/translator/output/aws-cn/function_with_batch_window.json‎ renamed to ‎tests/translator/output/aws-cn/function_with_event_source_mapping.json‎

Lines changed: 73 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88
}
99
},
1010
"Resources": {
11+
"MySqsQueue": {
12+
"Type": "AWS::SQS::Queue"
13+
},
1114
"MyFunctionForBatchingExampleStream": {
1215
"Type": "AWS::Lambda::EventSourceMapping",
1316
"Properties": {
@@ -24,6 +27,30 @@
2427
"StartingPosition": "LATEST"
2528
}
2629
},
30+
"DynamoDBTable": {
31+
"Type": "AWS::DynamoDB::Table",
32+
"Properties": {
33+
"KeySchema": [
34+
{
35+
"KeyType": "HASH",
36+
"AttributeName": "id"
37+
}
38+
],
39+
"StreamSpecification": {
40+
"StreamViewType": "NEW_IMAGE"
41+
},
42+
"AttributeDefinitions": [
43+
{
44+
"AttributeName": "id",
45+
"AttributeType": "S"
46+
}
47+
],
48+
"ProvisionedThroughput": {
49+
"WriteCapacityUnits": 5,
50+
"ReadCapacityUnits": 5
51+
}
52+
}
53+
},
2754
"MyFunctionForBatchingExampleRole": {
2855
"Type": "AWS::IAM::Role",
2956
"Properties": {
@@ -32,6 +59,34 @@
3259
"arn:aws-cn:iam::aws:policy/service-role/AWSLambdaDynamoDBExecutionRole",
3360
"arn:aws-cn:iam::aws:policy/service-role/AWSLambdaKinesisExecutionRole"
3461
],
62+
"Policies": [
63+
{
64+
"PolicyName": "MyFunctionForBatchingExampleRolePolicy0",
65+
"PolicyDocument": {
66+
"Statement": [
67+
{
68+
"Action": [
69+
"sqs:SendMessage*"
70+
],
71+
"Resource": {
72+
"Fn::Sub": [
73+
"arn:${AWS::Partition}:sqs:${AWS::Region}:${AWS::AccountId}:${queueName}",
74+
{
75+
"queueName": {
76+
"Fn::GetAtt": [
77+
"MySqsQueue",
78+
"QueueName"
79+
]
80+
}
81+
}
82+
]
83+
},
84+
"Effect": "Allow"
85+
}
86+
]
87+
}
88+
}
89+
],
3590
"AssumeRolePolicyDocument": {
3691
"Version": "2012-10-17",
3792
"Statement": [
@@ -62,17 +117,31 @@
62117
"MaximumBatchingWindowInSeconds": {
63118
"Ref": "MyBatchingWindowParam"
64119
},
65-
"BatchSize": 100,
66120
"FunctionName": {
67121
"Ref": "MyFunctionForBatchingExample"
68122
},
69-
"StartingPosition": "TRIM_HORIZON",
123+
"MaximumRecordAgeInSeconds": 86400,
124+
"BatchSize": 100,
125+
"DestinationConfig": {
126+
"OnFailure": {
127+
"Destination": {
128+
"Fn::GetAtt": [
129+
"MySqsQueue",
130+
"Arn"
131+
]
132+
}
133+
}
134+
},
70135
"EventSourceArn": {
71136
"Fn::GetAtt": [
72137
"DynamoDBTable",
73138
"StreamArn"
74139
]
75-
}
140+
},
141+
"StartingPosition": "TRIM_HORIZON",
142+
"ParallelizationFactor": 8,
143+
"MaximumRetryAttempts": 100,
144+
"BisectBatchOnFunctionError": true
76145
}
77146
},
78147
"MyFunctionForBatchingExample": {
@@ -97,30 +166,6 @@
97166
]
98167
}
99168
},
100-
"DynamoDBTable": {
101-
"Type": "AWS::DynamoDB::Table",
102-
"Properties": {
103-
"KeySchema": [
104-
{
105-
"KeyType": "HASH",
106-
"AttributeName": "id"
107-
}
108-
],
109-
"StreamSpecification": {
110-
"StreamViewType": "NEW_IMAGE"
111-
},
112-
"AttributeDefinitions": [
113-
{
114-
"AttributeName": "id",
115-
"AttributeType": "S"
116-
}
117-
],
118-
"ProvisionedThroughput": {
119-
"WriteCapacityUnits": 5,
120-
"ReadCapacityUnits": 5
121-
}
122-
}
123-
},
124169
"MyFunctionForBatchingExampleStreamEvent": {
125170
"Type": "AWS::Lambda::EventSourceMapping",
126171
"Properties": {
@@ -145,13 +190,5 @@
145190
"ShardCount": 1
146191
}
147192
}
148-
},
149-
"Parameters": {
150-
"MyBatchingWindowParam": {
151-
"Default": 45,
152-
"Type": "Number",
153-
"Description": "parameter for batching window in seconds"
154-
}
155-
},
156-
"Description": "EventSourceMapping example with MaximumBatchingWindowInSeconds property"
193+
}
157194
}

0 commit comments

Comments
 (0)