diff --git a/README.md b/README.md index e83bfe9..c16a0e6 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,89 @@ # cloudformation-helpers -A collection of AWS Lambda funtions that fill in the gaps that existing CloudFormation tasks do not cover +A collection of AWS Lambda funtions that fill in the gaps that existing CloudFormation resources do not cover. + +AWS CloudFormation supports Custom Resources (http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/template-custom-resources.html), +which can be used to call AWS Lambda functions. CloudFormation covers much of the AWS API landscape, but +does leave some gaps unsupported. AWS Lambda can contain any sort of logic, including interacting with the +AWS API in ways not covered by CloudFormation. By combining the two, CloudFormation deploys should be able +to approach the full resource support given by the AWS API. + +Warning: most of these functions require fairly wide permissions, since they need access to resources in a +general manner - much the same way CloudFormation itself has permission to do almost anything. + + +## Usage +1. Upload the .zip file of this repo from Github to an S3 bucket in your AWS account. +2. Use the included create_functions.template to deploy a stack that creates the Lambda functions for you. Remember the stack name. +3. Include the following resources in your CloudFormation template. These will create a) a nested stack that + looks up the ARNs from the previous step and b) a custom resource that allows your template to read those ARNs. + + ``` + "CFHelperStack": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "TemplateURL": "https://s3.amazonaws.com/com.gilt.public.backoffice/cloudformation_templates/lookup_stack_outputs.template" + } + }, + "CFHelper": { + "Type": "Custom::CFHelper", + "Properties": { + "ServiceToken": { "Fn::GetAtt" : ["CFHelperStack", "Outputs.LookupStackOutputsArn"] }, + "StackName": "your-helper-stack-name-here" + }, + "DependsOn": [ + "CFHelperStack" + ] + } + ``` + + You can either hardcode the stack name of your helper functions, or request it as a parameter. +4. Use the ARNs from the previous step in a custom resource, to call those Lambda functions: + + ``` + "PopulateTable": { + "Type": "Custom::PopulateTable", + "Properties": { + "ServiceToken": { "Fn::GetAtt" : ["CFHelper", "DynamoDBPutItemsFunctionArn"] }, + "TableName": "your-table-name", + "Items": [ + { + "key": "foo1", + "value": { + "bar": 1.5, + "baz": "qwerty" + } + }, + { + "key": "foo2", + "value": false + } + ] + }, + "DependsOn": [ + "CFHelper" + ] + } + ``` + + +## Included functions + +### Insert items into DynamoDB + +Pass in a list of items to be inserted into a DynamoDB table. This is useful to provide a template for the +content of the table, or to populate a config table. There is no data-checking, so it is up to the client +to ensure that the format of the data is correct. + +Warning: it is a PUT, so it will overwrite any items that already exist for the table's primary key. + +#### Parameters + +##### TableName +The name of the DynamoDB table to insert into. Must exist at the time of the insert, i.e. will not create if +it does not already exist. + +##### Items +A JSON array of items to be inserted, in JSON format (not DynamoDB format). + +#### Reference Output Name +DynamoDBPutItemsFunctionArn \ No newline at end of file diff --git a/cloudformation_helpers.js b/cloudformation_helpers.js new file mode 100644 index 0000000..9942a3e --- /dev/null +++ b/cloudformation_helpers.js @@ -0,0 +1,81 @@ +var AWS = require('aws-sdk'); +var dynamoDB = new AWS.DynamoDB(); +var response = require('./lib/cfn-response'); + +exports.dynamoDBPutItems = function(event, context) { + var p = event.ResourceProperties; + console.log('Received event: ' + JSON.stringify(event)); + if (event.RequestType == 'Delete') { + response.send(event, context, response.SUCCESS); + return; + } + + if (!Array.isArray(p.Items)) { + error("Must specify a list of items to insert.", event, context); + } else if (p.TableName === undefined) { + error("Must specify a table to insert into.", event, context); + } else { + putItems(p.Items, p.TableName, event, context, []); + } +} + +// Puts items into DynamoDB, iterating over the list recursively. +function putItems(items, tableName, event, context, itemsInserted) { + if(items.length > 0){ + var item = items.pop(); + console.log('Putting item [' + item.key + '] into DB [' + tableName + ']'); + dynamoDB.putItem( + { + TableName: tableName, + Item: formatForDynamo(item, true) + }, + function(err,data) { + if (err) { + error(err, event, context); + } else { + itemsInserted.push(item.key); + putItems(items, tableName, event, context, itemsInserted); + } + } + ); + } else { + response.send(event, context, response.SUCCESS, { "ItemsInserted": itemsInserted }); + } +} + +// Translates from raw JSON into DynamoDB-formatted JSON. This is more than a +// convenience thing: the original iteration of this accepted DyanamoDB-JSON Items, +// to avoid complications in translation. But when CloudFormation passes the parameters +// throught the event model, all non-string values get wrapped in quotes. For Booleans +// specifically, this is a problem - because DynamoDB does not allow a string (even if +// it is 'true' or 'false') as a 'BOOL' value. So some translation was needed, and +// it seemed best to then simplify things for the client by accepting raw JSON. +function formatForDynamo(value, topLevel) { + var result = undefined; + if (value == 'true' || value == 'false') { + result = {'BOOL': value == 'true'} + } else if (!isNaN(value)) { + result = {'N': value} + } else if (Array.isArray(value)) { + var arr = []; + for (var i = 0; i < value.length; i++) { + arr.push(formatForDynamo(value[i], false)); + } + result = {'L': arr}; + } else if (typeof value === "object") { + var map = {}; + Object.keys(value).forEach(function(key) { + map[key] = formatForDynamo(value[key], false) + }); + if (topLevel) result = map; + else result = {'M': map} + } else { + result = {'S': value} + } + return result; +} + +function error(message, event, context) { + console.error(message); + response.send(event, context, response.FAILED, { Error: message }); +} \ No newline at end of file diff --git a/create_functions.template b/create_functions.template new file mode 100644 index 0000000..01e778a --- /dev/null +++ b/create_functions.template @@ -0,0 +1,89 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Parameters": { + "S3ZipFileBucket": { + "Type": "String", + "Description": "The S3 bucket where the .zip file is stored." + }, + "S3ZipFileObjectKey": { + "Type": "String", + "Description": "The object key (including any 'folder prefix') of the .zip file containing the lambda functions." + } + }, + "Resources": { + "DynamoDBPutItemsFunctionRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version" : "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": [ "lambda.amazonaws.com" ] + }, + "Action": [ "sts:AssumeRole" ] + } + ] + }, + "Policies": [ + { + "PolicyName": "LogWriter", + "PolicyDocument": { + "Version" : "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:*:*:*" + } + ] + } + }, + { + "PolicyName": "DBWriter", + "PolicyDocument": { + "Version" : "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "dynamodb:PutItem" + ], + "Resource": { "Fn::Join": [ "", [ "arn:aws:dynamodb:", { "Ref": "AWS::Region" }, ":", { "Ref": "AWS::AccountId" } , ":table/*" ] ] } + } + ] + } + } + ] + } + }, + "DynamoDBPutItemsFunction": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { "Ref": "S3ZipFileBucket" }, + "S3Key": { "Ref": "S3ZipFileObjectKey" } + }, + "Description": "Used to populate a DynamoDB database from CloudFormation", + "Handler": "cloudformation_helpers.dynamoDBPutItems", + "Role": {"Fn::GetAtt" : [ "DynamoDBPutItemsFunctionRole", "Arn" ] }, + "Runtime": "nodejs", + "Timeout": 30 + }, + "DependsOn": [ + "DynamoDBPutItemsFunctionRole" + ] + } + }, + "Outputs": { + "DynamoDBPutItemsFunctionArn": { + "Description": "The ARN of the DynamoDBPutItemsFunction, for use in other CloudFormation templates.", + "Value": { "Fn::GetAtt" : ["DynamoDBPutItemsFunction", "Arn"] } + } + } +} \ No newline at end of file diff --git a/lib/cfn-response.js b/lib/cfn-response.js new file mode 100644 index 0000000..b2d227d --- /dev/null +++ b/lib/cfn-response.js @@ -0,0 +1,58 @@ +/* Copyright 2015 Amazon Web Services, Inc. or its affiliates. All Rights Reserved. + This file is licensed to you under the AWS Customer Agreement (the "License"). + You may not use this file except in compliance with the License. + A copy of the License is located at http://aws.amazon.com/agreement/. + This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied. + See the License for the specific language governing permissions and limitations under the License. */ + +exports.SUCCESS = "SUCCESS"; +exports.FAILED = "FAILED"; + +exports.send = function(event, context, responseStatus, responseData, physicalResourceId) { + + var responseBody = JSON.stringify({ + Status: responseStatus, + Reason: "See the details in CloudWatch Log Stream: " + context.logStreamName, + PhysicalResourceId: physicalResourceId || context.logStreamName, + StackId: event.StackId, + RequestId: event.RequestId, + LogicalResourceId: event.LogicalResourceId, + Data: responseData + }); + + console.log("Response body:\n", responseBody); + + var https = require("https"); + var url = require("url"); + + // This script comes from http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-lambda-function-code.html#cfn-lambda-function-code-cfnresponsemodule + // The only change is this 'if' statement around this block of code, so it doesn't + // fail when the ResonseURL is missing (i.e. during manual testing). + if (event.ResponseURL) { + var parsedUrl = url.parse(event.ResponseURL); + var options = { + hostname: parsedUrl.hostname, + port: 443, + path: parsedUrl.path, + method: "PUT", + headers: { + "content-type": "", + "content-length": responseBody.length + } + }; + + var request = https.request(options, function(response) { + console.log("Status code: " + response.statusCode); + console.log("Status message: " + response.statusMessage); + context.done(); + }); + + request.on("error", function(error) { + console.log("send(..) failed executing https.request(..): " + error); + context.done(); + }); + + request.write(responseBody); + request.end(); + } +} \ No newline at end of file diff --git a/lookup_stack_outputs.template b/lookup_stack_outputs.template new file mode 100644 index 0000000..042fd60 --- /dev/null +++ b/lookup_stack_outputs.template @@ -0,0 +1,113 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "LookupStackOutputsRole": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version" : "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": [ "lambda.amazonaws.com" ] + }, + "Action": [ "sts:AssumeRole" ] + } + ] + }, + "Policies": [ + { + "PolicyName": "LogWriter", + "PolicyDocument": { + "Version" : "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Resource": "arn:aws:logs:*:*:*" + } + ] + } + }, + { + "PolicyName": "CFReader", + "PolicyDocument": { + "Version" : "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "cloudformation:DescribeStacks" + ], + "Resource": "*" + } + ] + } + } + ] + } + }, + "LookupStackOutputs": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Handler": "index.handler", + "Role": { "Fn::GetAtt": [ "LookupStackOutputsRole", "Arn" ] }, + "Code": { + "ZipFile": { + "Fn::Join": [ + "", + [ + "var response = require('cfn-response');", + "exports.handler = function(event, context) {", + "console.log('REQUEST RECEIVED:\\n', JSON.stringify(event));", + "if (event.RequestType == 'Delete') {", + "response.send(event, context, response.SUCCESS);", + "return;", + "}", + "var stackName = event.ResourceProperties.StackName;", + "var responseData = {};", + "if (stackName) {", + "var aws = require('aws-sdk');", + "var cfn = new aws.CloudFormation();", + "cfn.describeStacks({StackName: stackName}, function(err, data) {", + "if (err) {", + "responseData = {Error: 'DescribeStacks call failed'};", + "console.log(responseData.Error + ':\\n', err);", + "response.send(event, context, response.FAILED, responseData);", + "} else {", + "data.Stacks[0].Outputs.forEach(function(output) {", + "responseData[output.OutputKey] = output.OutputValue;", + "});", + "response.send(event, context, response.SUCCESS, responseData);", + "}", + "});", + "} else {", + "responseData = {Error: 'Stack name not specified'};", + "console.log(responseData.Error);", + "response.send(event, context, response.FAILED, responseData);", + "}", + "};" + ] + ] + } + }, + "Runtime": "nodejs", + "Timeout": "30" + }, + "DependsOn": [ + "LookupStackOutputsRole" + ] + } + }, + "Outputs": { + "LookupStackOutputsArn": { + "Description": "The ARN of the LookupStackOutputs function, for use in other CloudFormation templates.", + "Value": { "Fn::GetAtt" : ["LookupStackOutputs", "Arn"] } + } + } +} \ No newline at end of file