Add first helper function: dynamoDBPutItems
This function allows writing items to a DynamoDB. Also include better documentation in the README.
This commit is contained in:
Ryan Martin
parent
decdaf48b4
commit
c1de9ee1c6
5 changed files with 429 additions and 1 deletions
89
README.md
89
README.md
|
|
@ -1,2 +1,89 @@
|
|||
# cloudformation-helpers
|
||||
A collection of AWS Lambda funtions that fill in the gaps that existing CloudFormation tasks do not cover
|
||||
A collection of AWS Lambda funtions that fill in the gaps that existing CloudFormation resources do not cover.
|
||||
|
||||
AWS CloudFormation supports Custom Resources (http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-custom-resources.html),
|
||||
which can be used to call AWS Lambda functions. CloudFormation covers much of the AWS API landscape, but
|
||||
does leave some gaps unsupported. AWS Lambda can contain any sort of logic, including interacting with the
|
||||
AWS API in ways not covered by CloudFormation. By combining the two, CloudFormation deploys should be able
|
||||
to approach the full resource support given by the AWS API.
|
||||
|
||||
Warning: most of these functions require fairly wide permissions, since they need access to resources in a
|
||||
general manner - much the same way CloudFormation itself has permission to do almost anything.
|
||||
|
||||
|
||||
## Usage
|
||||
1. Upload the .zip file of this repo from Github to an S3 bucket in your AWS account.
|
||||
2. Use the included create_functions.template to deploy a stack that creates the Lambda functions for you. Remember the stack name.
|
||||
3. Include the following resources in your CloudFormation template. These will create a) a nested stack that
|
||||
looks up the ARNs from the previous step and b) a custom resource that allows your template to read those ARNs.
|
||||
|
||||
```
|
||||
"CFHelperStack": {
|
||||
"Type": "AWS::CloudFormation::Stack",
|
||||
"Properties": {
|
||||
"TemplateURL": "https://s3.amazonaws.com/com.gilt.public.backoffice/cloudformation_templates/lookup_stack_outputs.template"
|
||||
}
|
||||
},
|
||||
"CFHelper": {
|
||||
"Type": "Custom::CFHelper",
|
||||
"Properties": {
|
||||
"ServiceToken": { "Fn::GetAtt" : ["CFHelperStack", "Outputs.LookupStackOutputsArn"] },
|
||||
"StackName": "your-helper-stack-name-here"
|
||||
},
|
||||
"DependsOn": [
|
||||
"CFHelperStack"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
You can either hardcode the stack name of your helper functions, or request it as a parameter.
|
||||
4. Use the ARNs from the previous step in a custom resource, to call those Lambda functions:
|
||||
|
||||
```
|
||||
"PopulateTable": {
|
||||
"Type": "Custom::PopulateTable",
|
||||
"Properties": {
|
||||
"ServiceToken": { "Fn::GetAtt" : ["CFHelper", "DynamoDBPutItemsFunctionArn"] },
|
||||
"TableName": "your-table-name",
|
||||
"Items": [
|
||||
{
|
||||
"key": "foo1",
|
||||
"value": {
|
||||
"bar": 1.5,
|
||||
"baz": "qwerty"
|
||||
}
|
||||
},
|
||||
{
|
||||
"key": "foo2",
|
||||
"value": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"DependsOn": [
|
||||
"CFHelper"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
## Included functions
|
||||
|
||||
### Insert items into DynamoDB
|
||||
|
||||
Pass in a list of items to be inserted into a DynamoDB table. This is useful to provide a template for the
|
||||
content of the table, or to populate a config table. There is no data-checking, so it is up to the client
|
||||
to ensure that the format of the data is correct.
|
||||
|
||||
Warning: it is a PUT, so it will overwrite any items that already exist for the table's primary key.
|
||||
|
||||
#### Parameters
|
||||
|
||||
##### TableName
|
||||
The name of the DynamoDB table to insert into. Must exist at the time of the insert, i.e. will not create if
|
||||
it does not already exist.
|
||||
|
||||
##### Items
|
||||
A JSON array of items to be inserted, in JSON format (not DynamoDB format).
|
||||
|
||||
#### Reference Output Name
|
||||
DynamoDBPutItemsFunctionArn
|
||||
81
cloudformation_helpers.js
Normal file
81
cloudformation_helpers.js
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
var AWS = require('aws-sdk');
|
||||
var dynamoDB = new AWS.DynamoDB();
|
||||
var response = require('./lib/cfn-response');
|
||||
|
||||
exports.dynamoDBPutItems = function(event, context) {
|
||||
var p = event.ResourceProperties;
|
||||
console.log('Received event: ' + JSON.stringify(event));
|
||||
if (event.RequestType == 'Delete') {
|
||||
response.send(event, context, response.SUCCESS);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!Array.isArray(p.Items)) {
|
||||
error("Must specify a list of items to insert.", event, context);
|
||||
} else if (p.TableName === undefined) {
|
||||
error("Must specify a table to insert into.", event, context);
|
||||
} else {
|
||||
putItems(p.Items, p.TableName, event, context, []);
|
||||
}
|
||||
}
|
||||
|
||||
// Puts items into DynamoDB, iterating over the list recursively.
|
||||
function putItems(items, tableName, event, context, itemsInserted) {
|
||||
if(items.length > 0){
|
||||
var item = items.pop();
|
||||
console.log('Putting item [' + item.key + '] into DB [' + tableName + ']');
|
||||
dynamoDB.putItem(
|
||||
{
|
||||
TableName: tableName,
|
||||
Item: formatForDynamo(item, true)
|
||||
},
|
||||
function(err,data) {
|
||||
if (err) {
|
||||
error(err, event, context);
|
||||
} else {
|
||||
itemsInserted.push(item.key);
|
||||
putItems(items, tableName, event, context, itemsInserted);
|
||||
}
|
||||
}
|
||||
);
|
||||
} else {
|
||||
response.send(event, context, response.SUCCESS, { "ItemsInserted": itemsInserted });
|
||||
}
|
||||
}
|
||||
|
||||
// Translates from raw JSON into DynamoDB-formatted JSON. This is more than a
|
||||
// convenience thing: the original iteration of this accepted DyanamoDB-JSON Items,
|
||||
// to avoid complications in translation. But when CloudFormation passes the parameters
|
||||
// throught the event model, all non-string values get wrapped in quotes. For Booleans
|
||||
// specifically, this is a problem - because DynamoDB does not allow a string (even if
|
||||
// it is 'true' or 'false') as a 'BOOL' value. So some translation was needed, and
|
||||
// it seemed best to then simplify things for the client by accepting raw JSON.
|
||||
function formatForDynamo(value, topLevel) {
|
||||
var result = undefined;
|
||||
if (value == 'true' || value == 'false') {
|
||||
result = {'BOOL': value == 'true'}
|
||||
} else if (!isNaN(value)) {
|
||||
result = {'N': value}
|
||||
} else if (Array.isArray(value)) {
|
||||
var arr = [];
|
||||
for (var i = 0; i < value.length; i++) {
|
||||
arr.push(formatForDynamo(value[i], false));
|
||||
}
|
||||
result = {'L': arr};
|
||||
} else if (typeof value === "object") {
|
||||
var map = {};
|
||||
Object.keys(value).forEach(function(key) {
|
||||
map[key] = formatForDynamo(value[key], false)
|
||||
});
|
||||
if (topLevel) result = map;
|
||||
else result = {'M': map}
|
||||
} else {
|
||||
result = {'S': value}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function error(message, event, context) {
|
||||
console.error(message);
|
||||
response.send(event, context, response.FAILED, { Error: message });
|
||||
}
|
||||
89
create_functions.template
Normal file
89
create_functions.template
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
{
|
||||
"AWSTemplateFormatVersion": "2010-09-09",
|
||||
"Parameters": {
|
||||
"S3ZipFileBucket": {
|
||||
"Type": "String",
|
||||
"Description": "The S3 bucket where the .zip file is stored."
|
||||
},
|
||||
"S3ZipFileObjectKey": {
|
||||
"Type": "String",
|
||||
"Description": "The object key (including any 'folder prefix') of the .zip file containing the lambda functions."
|
||||
}
|
||||
},
|
||||
"Resources": {
|
||||
"DynamoDBPutItemsFunctionRole": {
|
||||
"Type": "AWS::IAM::Role",
|
||||
"Properties": {
|
||||
"AssumeRolePolicyDocument": {
|
||||
"Version" : "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"Service": [ "lambda.amazonaws.com" ]
|
||||
},
|
||||
"Action": [ "sts:AssumeRole" ]
|
||||
}
|
||||
]
|
||||
},
|
||||
"Policies": [
|
||||
{
|
||||
"PolicyName": "LogWriter",
|
||||
"PolicyDocument": {
|
||||
"Version" : "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"logs:CreateLogGroup",
|
||||
"logs:CreateLogStream",
|
||||
"logs:PutLogEvents"
|
||||
],
|
||||
"Resource": "arn:aws:logs:*:*:*"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"PolicyName": "DBWriter",
|
||||
"PolicyDocument": {
|
||||
"Version" : "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"dynamodb:PutItem"
|
||||
],
|
||||
"Resource": { "Fn::Join": [ "", [ "arn:aws:dynamodb:", { "Ref": "AWS::Region" }, ":", { "Ref": "AWS::AccountId" } , ":table/*" ] ] }
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"DynamoDBPutItemsFunction": {
|
||||
"Type": "AWS::Lambda::Function",
|
||||
"Properties": {
|
||||
"Code": {
|
||||
"S3Bucket": { "Ref": "S3ZipFileBucket" },
|
||||
"S3Key": { "Ref": "S3ZipFileObjectKey" }
|
||||
},
|
||||
"Description": "Used to populate a DynamoDB database from CloudFormation",
|
||||
"Handler": "cloudformation_helpers.dynamoDBPutItems",
|
||||
"Role": {"Fn::GetAtt" : [ "DynamoDBPutItemsFunctionRole", "Arn" ] },
|
||||
"Runtime": "nodejs",
|
||||
"Timeout": 30
|
||||
},
|
||||
"DependsOn": [
|
||||
"DynamoDBPutItemsFunctionRole"
|
||||
]
|
||||
}
|
||||
},
|
||||
"Outputs": {
|
||||
"DynamoDBPutItemsFunctionArn": {
|
||||
"Description": "The ARN of the DynamoDBPutItemsFunction, for use in other CloudFormation templates.",
|
||||
"Value": { "Fn::GetAtt" : ["DynamoDBPutItemsFunction", "Arn"] }
|
||||
}
|
||||
}
|
||||
}
|
||||
58
lib/cfn-response.js
Normal file
58
lib/cfn-response.js
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
/* Copyright 2015 Amazon Web Services, Inc. or its affiliates. All Rights Reserved.
|
||||
This file is licensed to you under the AWS Customer Agreement (the "License").
|
||||
You may not use this file except in compliance with the License.
|
||||
A copy of the License is located at http://aws.amazon.com/agreement/.
|
||||
This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
|
||||
See the License for the specific language governing permissions and limitations under the License. */
|
||||
|
||||
exports.SUCCESS = "SUCCESS";
|
||||
exports.FAILED = "FAILED";
|
||||
|
||||
exports.send = function(event, context, responseStatus, responseData, physicalResourceId) {
|
||||
|
||||
var responseBody = JSON.stringify({
|
||||
Status: responseStatus,
|
||||
Reason: "See the details in CloudWatch Log Stream: " + context.logStreamName,
|
||||
PhysicalResourceId: physicalResourceId || context.logStreamName,
|
||||
StackId: event.StackId,
|
||||
RequestId: event.RequestId,
|
||||
LogicalResourceId: event.LogicalResourceId,
|
||||
Data: responseData
|
||||
});
|
||||
|
||||
console.log("Response body:\n", responseBody);
|
||||
|
||||
var https = require("https");
|
||||
var url = require("url");
|
||||
|
||||
// This script comes from http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-function-code.html#cfn-lambda-function-code-cfnresponsemodule
|
||||
// The only change is this 'if' statement around this block of code, so it doesn't
|
||||
// fail when the ResonseURL is missing (i.e. during manual testing).
|
||||
if (event.ResponseURL) {
|
||||
var parsedUrl = url.parse(event.ResponseURL);
|
||||
var options = {
|
||||
hostname: parsedUrl.hostname,
|
||||
port: 443,
|
||||
path: parsedUrl.path,
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"content-type": "",
|
||||
"content-length": responseBody.length
|
||||
}
|
||||
};
|
||||
|
||||
var request = https.request(options, function(response) {
|
||||
console.log("Status code: " + response.statusCode);
|
||||
console.log("Status message: " + response.statusMessage);
|
||||
context.done();
|
||||
});
|
||||
|
||||
request.on("error", function(error) {
|
||||
console.log("send(..) failed executing https.request(..): " + error);
|
||||
context.done();
|
||||
});
|
||||
|
||||
request.write(responseBody);
|
||||
request.end();
|
||||
}
|
||||
}
|
||||
113
lookup_stack_outputs.template
Normal file
113
lookup_stack_outputs.template
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
{
|
||||
"AWSTemplateFormatVersion": "2010-09-09",
|
||||
"Resources": {
|
||||
"LookupStackOutputsRole": {
|
||||
"Type": "AWS::IAM::Role",
|
||||
"Properties": {
|
||||
"AssumeRolePolicyDocument": {
|
||||
"Version" : "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Principal": {
|
||||
"Service": [ "lambda.amazonaws.com" ]
|
||||
},
|
||||
"Action": [ "sts:AssumeRole" ]
|
||||
}
|
||||
]
|
||||
},
|
||||
"Policies": [
|
||||
{
|
||||
"PolicyName": "LogWriter",
|
||||
"PolicyDocument": {
|
||||
"Version" : "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"logs:CreateLogGroup",
|
||||
"logs:CreateLogStream",
|
||||
"logs:PutLogEvents"
|
||||
],
|
||||
"Resource": "arn:aws:logs:*:*:*"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"PolicyName": "CFReader",
|
||||
"PolicyDocument": {
|
||||
"Version" : "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"cloudformation:DescribeStacks"
|
||||
],
|
||||
"Resource": "*"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"LookupStackOutputs": {
|
||||
"Type": "AWS::Lambda::Function",
|
||||
"Properties": {
|
||||
"Handler": "index.handler",
|
||||
"Role": { "Fn::GetAtt": [ "LookupStackOutputsRole", "Arn" ] },
|
||||
"Code": {
|
||||
"ZipFile": {
|
||||
"Fn::Join": [
|
||||
"",
|
||||
[
|
||||
"var response = require('cfn-response');",
|
||||
"exports.handler = function(event, context) {",
|
||||
"console.log('REQUEST RECEIVED:\\n', JSON.stringify(event));",
|
||||
"if (event.RequestType == 'Delete') {",
|
||||
"response.send(event, context, response.SUCCESS);",
|
||||
"return;",
|
||||
"}",
|
||||
"var stackName = event.ResourceProperties.StackName;",
|
||||
"var responseData = {};",
|
||||
"if (stackName) {",
|
||||
"var aws = require('aws-sdk');",
|
||||
"var cfn = new aws.CloudFormation();",
|
||||
"cfn.describeStacks({StackName: stackName}, function(err, data) {",
|
||||
"if (err) {",
|
||||
"responseData = {Error: 'DescribeStacks call failed'};",
|
||||
"console.log(responseData.Error + ':\\n', err);",
|
||||
"response.send(event, context, response.FAILED, responseData);",
|
||||
"} else {",
|
||||
"data.Stacks[0].Outputs.forEach(function(output) {",
|
||||
"responseData[output.OutputKey] = output.OutputValue;",
|
||||
"});",
|
||||
"response.send(event, context, response.SUCCESS, responseData);",
|
||||
"}",
|
||||
"});",
|
||||
"} else {",
|
||||
"responseData = {Error: 'Stack name not specified'};",
|
||||
"console.log(responseData.Error);",
|
||||
"response.send(event, context, response.FAILED, responseData);",
|
||||
"}",
|
||||
"};"
|
||||
]
|
||||
]
|
||||
}
|
||||
},
|
||||
"Runtime": "nodejs",
|
||||
"Timeout": "30"
|
||||
},
|
||||
"DependsOn": [
|
||||
"LookupStackOutputsRole"
|
||||
]
|
||||
}
|
||||
},
|
||||
"Outputs": {
|
||||
"LookupStackOutputsArn": {
|
||||
"Description": "The ARN of the LookupStackOutputs function, for use in other CloudFormation templates.",
|
||||
"Value": { "Fn::GetAtt" : ["LookupStackOutputs", "Arn"] }
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue