I want to implement one S3 bucket folder uploading API and abort API. The uploading API should shows progress of uploading and abort API should abort an upload that's currently in progress and cancel all future uploads. For uploading API, I can use upload_file API and use ProgressPercentage as a callback. How do we implement abort API? There is create_multipart_upload and abort_multipart_upload but create_multipart_upload doesn't accept callback for progress update. Below is my code that includes uploading API.
class ProgressPercentage(object):
def __init__(self, filename, uploaded_files, total_files):
#get filename from absolute path
self._filename = os.path.basename(filename)
self._uploaded_files = uploaded_files
self._total_files = total_files
self._unit_conversion = 1024 * 1024
self._size = float(os.path.getsize(filename) / self._unit_conversion)
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
# To simplify, assume this is hooked up to a single filename
self._seen_so_far += (bytes_amount / self._unit_conversion)
percentage = (self._seen_so_far / self._size) * 100
progress_feedback = "\r{} {:.2f}MB / {:.2f}MB ({:.2f}%) | Uploading {} / {}".format(
self._filename, self._seen_so_far, self._size,
percentage, self._uploaded_files, self._total_files)
print(progress_feedback)
class Uploader(object):
def __init__(self):
self._s3_client = boto3.client('s3')
self._abort_flag = False
def abort(self):
pass
def upload_folder_to_s3_with_upload_file(self, bucket_name, folder_path, s3_path_prefix):
uploaded_files = 1
start = time.time()
total_files = self.count_files_in_folder(folder_path)
for parent_dir, subdirs, files in os.walk(folder_path):
for file in files:
full_path = os.path.join(parent_dir, file)
# Compute the relative file path to the folder being uploaded
relative_path = os.path.relpath(full_path, folder_path)
s3_key = os.path.join(s3_path_prefix, relative_path).replace("\\", "/")
self._s3_client.upload_file(full_path, bucket_name, s3_key,
Callback=ProgressPercentage(full_path, uploaded_files, total_files))
uploaded_files += 1
end = time.time()
print(f"\nUploaded {total_files} files in {end - start:.2f} seconds")
def count_files_in_folder(self, folder_path):
count = 0
for _, _, files in os.walk(folder_path):
count += len(files)
return count