Hello!
I have developed a python3.6 program to create a csv object in S3 from a JSON payload. As this is my first python program, it would really help me if I someone could help me with the review.
Thank you.
I have developed a python3.6 program to create a csv object in S3 from a JSON payload. As this is my first python program, it would really help me if I someone could help me with the review.
Thank you.
# Main module. Contains the event handler to be triggered from LAMBDA
import json
import boto3 # Needed to work with Amazon S3
import botocore
import re # regular expression module
MYS3BUCKET = 'xyzbucket'
FILE_ENCODING = 'utf-8'
CONN_S3 = boto3.resource('s3')
# Event Handler function triggered from Lambda
def lambda_handler(event,content):
try:
# create an object for source
Mysource = Source(event)
# convert the source object into csv
csv_record = Mysource.create_csvrecord()
# store in s3
message = Mysource.write_to_s3(MYS3BUCKET,csv_record)
except:
raise
else:
return message
class Source:
def __init__(self,event):
self.__event = event
def set_eventdata(self,event):
self.__event = event
def get_eventdata(self,event):
return self.__event
# function to convert dictionay input to csv
def create_csvrecord(self):
try:
count = 0
csvrecord = ''
# extract the source object from JSON event
json_data = json.loads(json.dumps(self.__event['data']['object']['source']))
for key in json_data.keys():
if count != len(json_data) - 1:
csvrecord = csvrecord + re.sub(r'^None$','',str(json_data[key])) + ',' # replace None with null values
else:
csvrecord = csvrecord + re.sub(r'^None$','',str(json_data[key])) # do not append comma at end of record
count +=1
except:
raise
else:
return csvrecord
# function to store the data on Amazon S3
def write_to_s3(self,bucketname,data):
try:
# create bucket
self.create_bucket(bucketname)
objectname = json.dumps(self.__event['id']).strip('\"') + '.csv'
s3object = CONN_S3.Object(bucketname,objectname)
s3object.put(Body=bytes(data,FILE_ENCODING))
except:
print ('Exception occurred inside write_to_s3 function!')
raise
else:
return 'Successfully stored on S3.'
# function to create a S3 bucket if not exists
def create_bucket(self,bucketname):
try:
exists = True
try:
CONN_S3.meta.client.head_bucket(Bucket=bucketname)
except botocore.exceptions.ClientError as e:
error_code = int(e.response['Error']['Code'])
if error_code == 404:
exists = False
if not exists:
CONN_S3.create_bucket(Bucket=bucketname)
except:
print ('Exception occurred inside create_bucket function!')
raise
else:
return
