Abort s3 bucket uploading progress using boto3

25 Views Asked by At

I want to implement one S3 bucket folder uploading API and abort API. The uploading API should shows progress of uploading and abort API should abort an upload that's currently in progress and cancel all future uploads. For uploading API, I can use upload_file API and use ProgressPercentage as a callback. How do we implement abort API? There is create_multipart_upload and abort_multipart_upload but create_multipart_upload doesn't accept callback for progress update. Below is my code that includes uploading API.

class ProgressPercentage(object):
    def __init__(self, filename, uploaded_files, total_files):
        #get filename from absolute path
        self._filename = os.path.basename(filename)
        self._uploaded_files = uploaded_files
        self._total_files = total_files
        self._unit_conversion = 1024 * 1024
        self._size = float(os.path.getsize(filename) / self._unit_conversion)
        self._seen_so_far = 0
        self._lock = threading.Lock()

    def __call__(self, bytes_amount):
        # To simplify, assume this is hooked up to a single filename

        self._seen_so_far += (bytes_amount / self._unit_conversion)
        percentage = (self._seen_so_far / self._size) * 100
        progress_feedback = "\r{}  {:.2f}MB / {:.2f}MB  ({:.2f}%) | Uploading {} / {}".format(
            self._filename, self._seen_so_far, self._size,
            percentage, self._uploaded_files, self._total_files)
        print(progress_feedback)

class Uploader(object):

    def __init__(self):
        self._s3_client = boto3.client('s3')
        self._abort_flag = False

    
    def abort(self):
        pass

    def upload_folder_to_s3_with_upload_file(self, bucket_name, folder_path, s3_path_prefix):
        uploaded_files = 1
        start = time.time()
        total_files = self.count_files_in_folder(folder_path)
        for parent_dir, subdirs, files in os.walk(folder_path):
            for file in files:
                full_path = os.path.join(parent_dir, file)
                # Compute the relative file path to the folder being uploaded
                relative_path = os.path.relpath(full_path, folder_path)
                s3_key = os.path.join(s3_path_prefix, relative_path).replace("\\", "/")
                self._s3_client.upload_file(full_path, bucket_name, s3_key,
                                    Callback=ProgressPercentage(full_path, uploaded_files, total_files))
                uploaded_files += 1
        end = time.time()
        print(f"\nUploaded {total_files} files in {end - start:.2f} seconds")
            

    def count_files_in_folder(self, folder_path):
        count = 0
        for _, _, files in os.walk(folder_path):
            count += len(files)
        return count
0

There are 0 best solutions below