-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add `dynamoDBItemKey()` to grantRecord interface * Add functionality to check LastUpdatedDate in DynamoDB * Refactor handler to use DynamoDB * Grant Lambda permission to get DynamoDB items * Grant lambda permission to use new S3 key naming convention * Allow PersistGrantsGovXMLDB to read from new XML object names * Add ability to limit SplitGrantsGovXMLDB processing in local dev
- Loading branch information
1 parent
8292a19
commit db256e6
Showing
13 changed files
with
314 additions
and
52 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "time" | ||
|
|
||
| "github.com/aws/aws-sdk-go-v2/aws" | ||
| "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" | ||
| "github.com/aws/aws-sdk-go-v2/service/dynamodb" | ||
| ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" | ||
| grantsgov "github.com/usdigitalresponse/grants-ingest/pkg/grantsSchemas/grants.gov" | ||
| ) | ||
|
|
||
| // DynamoDBGetItemAPI is the interface for retrieving a single item from a DynamoDB table via primary key lookup | ||
| type DynamoDBGetItemAPI interface { | ||
| GetItem(context.Context, *dynamodb.GetItemInput, ...func(*dynamodb.Options)) (*dynamodb.GetItemOutput, error) | ||
| } | ||
|
|
||
| // GetDynamoDBLastModified gets the "Last Modified" timestamp for a grantRecord stored in a DynamoDB table. | ||
| // If the item exists, a pointer to the last modification time is returned along with a nil error. | ||
| // If the specified item does not exist, the returned *time.Time and error are both nil. | ||
| // If an error is encountered when calling the HeadObject S3 API method, this will return a nil | ||
| // *time.Time value along with the encountered error. | ||
| func GetDynamoDBLastModified(ctx context.Context, c DynamoDBGetItemAPI, table string, key map[string]ddbtypes.AttributeValue) (*time.Time, error) { | ||
| resp, err := c.GetItem(ctx, &dynamodb.GetItemInput{ | ||
| TableName: &table, | ||
| Key: key, | ||
| ProjectionExpression: aws.String("LastUpdatedDate"), | ||
| }) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if resp.Item == nil { | ||
| return nil, nil | ||
| } | ||
|
|
||
| item := struct{ LastUpdatedDate grantsgov.MMDDYYYYType }{} | ||
| if err := attributevalue.UnmarshalMap(resp.Item, &item); err != nil { | ||
| return nil, err | ||
| } | ||
| lastUpdatedDate, err := item.LastUpdatedDate.Time() | ||
| return &lastUpdatedDate, err | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" | ||
| "github.com/aws/aws-sdk-go-v2/service/dynamodb" | ||
| ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" | ||
| "github.com/stretchr/testify/assert" | ||
| "github.com/stretchr/testify/require" | ||
| grantsgov "github.com/usdigitalresponse/grants-ingest/pkg/grantsSchemas/grants.gov" | ||
| ) | ||
|
|
||
| type mockDynamoDBGetItemClient func(context.Context, *dynamodb.GetItemInput, ...func(*dynamodb.Options)) (*dynamodb.GetItemOutput, error) | ||
|
|
||
| func (m mockDynamoDBGetItemClient) GetItem(ctx context.Context, params *dynamodb.GetItemInput, optFns ...func(*dynamodb.Options)) (*dynamodb.GetItemOutput, error) { | ||
| return m(ctx, params, optFns...) | ||
| } | ||
|
|
||
| func makeTestItem(t *testing.T, lastUpdatedDateTestValue any) map[string]ddbtypes.AttributeValue { | ||
| t.Helper() | ||
| rv, err := attributevalue.MarshalMap(map[string]any{"LastUpdatedDate": lastUpdatedDateTestValue}) | ||
| require.NoError(t, err, "Unexpected error creating test DynamoDB item fixture during setup") | ||
| return rv | ||
| } | ||
|
|
||
| func TestGetDynamoDBLastModified(t *testing.T) { | ||
| testTableName := "test-table" | ||
| testItemKey := map[string]ddbtypes.AttributeValue{ | ||
| "grant_id": &ddbtypes.AttributeValueMemberS{Value: "test-key"}, | ||
| } | ||
| testLastUpdateDateString := time.Now().Format(grantsgov.TimeLayoutMMDDYYYYType) | ||
| testLastUpdateDate, err := time.Parse(grantsgov.TimeLayoutMMDDYYYYType, testLastUpdateDateString) | ||
| require.NoError(t, err, "Unexpected error parsing time fixture during test setup") | ||
| testInvalidDateString := "not a valid date string" | ||
|
|
||
| _, testInvalidDateStringParseError := time.Parse(grantsgov.TimeLayoutMMDDYYYYType, testInvalidDateString) | ||
| require.Error(t, testInvalidDateStringParseError, "Error fixture unexpectedly nil during test setup") | ||
|
|
||
| for _, tt := range []struct { | ||
| name string | ||
| ddbItem map[string]ddbtypes.AttributeValue | ||
| ddbErr error | ||
| expLastModified *time.Time | ||
| expErr error | ||
| }{ | ||
| { | ||
| "GetItem produces item with valid LastUpdatedDate", | ||
| makeTestItem(t, testLastUpdateDateString), | ||
| nil, | ||
| &testLastUpdateDate, | ||
| nil, | ||
| }, | ||
| { | ||
| "GetItem returns error", | ||
| nil, | ||
| errors.New("GetItem action failed"), | ||
| nil, | ||
| errors.New("GetItem action failed"), | ||
| }, | ||
| {"GetItem key not found", nil, nil, nil, nil}, | ||
| { | ||
| "GetItem produces item with invalid LastUpdateDate", | ||
| makeTestItem(t, testInvalidDateString), | ||
| nil, | ||
| nil, | ||
| testInvalidDateStringParseError, | ||
| }, | ||
| { | ||
| "GetItem produces item that cannot be unmarshalled", | ||
| makeTestItem(t, true), | ||
| nil, | ||
| nil, | ||
| errors.New("unmarshal failed, cannot unmarshal bool into Go value type grantsgov.MMDDYYYYType"), | ||
| }, | ||
| } { | ||
|
|
||
| t.Run(tt.name, func(t *testing.T) { | ||
| mockClient := mockDynamoDBGetItemClient(func(ctx context.Context, params *dynamodb.GetItemInput, f ...func(*dynamodb.Options)) (*dynamodb.GetItemOutput, error) { | ||
| require.Equal(t, &testTableName, params.TableName, "Unexpected table name in GetItem params") | ||
| require.Equal(t, testItemKey, params.Key, "Unexpected item key in GetItem params") | ||
|
|
||
| return &dynamodb.GetItemOutput{Item: tt.ddbItem}, tt.ddbErr | ||
| }) | ||
|
|
||
| lastModified, err := GetDynamoDBLastModified(context.TODO(), | ||
| mockClient, testTableName, testItemKey) | ||
|
|
||
| if tt.expErr != nil { | ||
| assert.EqualError(t, err, tt.expErr.Error()) | ||
| } else { | ||
| assert.NoError(t, err) | ||
| assert.Equal(t, tt.expLastModified, lastModified) | ||
| } | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.