Skip to content

Commit

Permalink
Merge pull request #209 from NetApp/make_ingest_more_robust
Browse files Browse the repository at this point in the history
Made it update the stats file for every file it processes
  • Loading branch information
nichollri authored Dec 4, 2024
2 parents 0871498 + 4f84710 commit ebbbb23
Showing 1 changed file with 1 addition and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -326,9 +326,6 @@ def lambda_handler(event, context): # pylint: disable=W0613
else:
lastFileRead = json.loads(response['Body'].read().decode('utf-8'))
#
# Only update the lastFileRead if we actually copied a new file.
lastFileReadChanged = False
#
# Process each FSxN.
for fsxn in fsxNs:
fsId = fsxn.split('.')[1]
Expand Down Expand Up @@ -372,11 +369,7 @@ def lambda_handler(event, context): # pylint: disable=W0613
# Process the file.
processFile(fsxn, headersDownload, volumeUUID, filePath)
lastFileRead[fsxn] = getEpoch(filePath)
lastFileReadChanged = True
#
# Save the last read stats file.
if lastFileReadChanged:
s3Client.put_object(Key=config['statsName'], Bucket=config['s3BucketName'], Body=json.dumps(lastFileRead).encode('UTF-8'))
s3Client.put_object(Key=config['statsName'], Bucket=config['s3BucketName'], Body=json.dumps(lastFileRead).encode('UTF-8'))
#
# If this script is not running as a Lambda function, then call the lambda_handler function.
if os.environ.get('AWS_LAMBDA_FUNCTION_NAME') == None:
Expand Down

0 comments on commit ebbbb23

Please sign in to comment.