So My team mates and I have been having a hard time with the efficiency of the synchronous file upload CRUD endpoints we've built and now we need to implement a bulk file upload(Directory upload) with either celery or asyncio(any other methods are kindly welcome).
what I have working currently
My Model:
class CompanyFileUpload(SoftDeletionModel):
name = models.CharField(
max_length=100,
null=True,
unique=False,
blank=True,
default="file name"
)
user = models.ForeignKey(
User,
on_delete=models.CASCADE,
related_name='company_file_uploads'
)
timestamp = models.DateTimeField(
auto_now_add=True
)
data_file = models.FileField(
upload_to=Utils.create_company_file_upload_path,
blank=True,
null=True,
validators=[validate_file]
)
comment = models.CharField(
max_length=400,
null=True,
unique=False
)
file_size = models.CharField(
max_length=100,
null=True,
unique=False,
blank=True,
default="null"
)
def save(self, *args, **kwargs):
size = self.data_file.size
power = 2**10
n = 0
power_labels = {0: '', 1: 'Kilo', 2: 'Mega', 3: 'Giga', 4: 'Tera'}
while size > power:
size /= power
n += 1
self.file_size = f"{size:.2f} {power_labels[n]}bytes"
self.name = self.data_file.name
super(CompanyFileUpload, self).save(*args, **kwargs)
def __str__(self):
return f'{self.data_file}, {self.user}'
@receiver(post_save, sender=CompanyFileUpload)
def email_report_details_handler(sender, instance, **kwargs):
ctx = {
"firstname": instance.user.firstname,
"timestamp": instance.timestamp,
"fullname_and_email": f"{instance.user.firstname} {instance.user.othernames}",
"subject": "File Upload",
"recepient": instance.user.email,
"email": instance.user.email,
"url": instance.data_file.url,
"comment": instance.comment
}
Utils.send_report_email(ctx)
My serializer:
class CompanyFileUploadSerializer(serializers.ModelSerializer):
timestamp = serializers.DateTimeField(
format="%d-%m-%Y %H:%M:%S", read_only=True)
class Meta:
model = CompanyFileUpload
fields = ["id",
"name",
"user",
"data_file",
"comment",
"file_size",
"timestamp"]
extra_kwargs = {
'file_size': {'read_only': True},
'name': {'read_only': True}
}
My View:
class CreateCompanyFileUploadAPI(generics.CreateAPIView):
"""
API endpoint for creating a Company file_upload
Sample Response:
{
"id": 3,
"user": 4,
"title": "This is an awesome file",
"data_file": "https://{ url_to_some_storage :) }",
"comment": "Well it should be awesome...",
"timestamp": "2021-06-09T19:33:04.835268Z"
}
"""
serializer_class = CompanyFileUploadSerializer
class ListCompanyFileUploadsAPI(generics.ListAPIView):
"""
API endpoint for viewing all Company file_uploads
Sample Response:
[
{
"id": 3,
"user": 4,
"title": "This is an awesome file",
"data_file": "https://{ url_to_some_storage :) }",
"comment": "Well it should be awesome...",
"timestamp": "2021-06-09T19:33:04.835268Z"
}
]
"""
serializer_class = CompanyFileUploadSerializer
lookup_field = 'pk'
filter_backends = (SearchFilter, OrderingFilter, DjangoFilterBackend,)
search_fields = ["title", "user__firstname"]
def get_queryset(self):
user = self.request.user
return CompanyFileUpload.objects.filter(user=user).order_by("-timestamp")
class RetrieveCompanyFileUploadDetailsAPI(generics.RetrieveAPIView):
"""
API endpoint for reading a Company single file_upload details
Sample Response:
{
"id": 3,
"user": 4,
"title": "This is an awesome file",
"data_file": "https:{ url_to_some_storage :) }",
"comment": "Well it should be awesome...",
"timestamp": "2021-06-09T19:33:04.835268Z"
}
"""
serializer_class = CompanyFileUploadSerializer
lookup_field = 'pk'
def get_queryset(self):
user = self.request.user
return CompanyFileUpload.objects.filter(user=user)
class DeleteCompanyUploadedFileAPI(generics.DestroyAPIView):
"""
API endpoint to delete already uploaded Company files from local/blob storage
"""
serializer_class = CompanyFileUploadSerializer
queryset = CompanyFileUpload.objects
lookup_field = "pk"
def get_queryset(self):
user = self.request.user
return CompanyFileUpload.objects.filter(user=user)
class UpdateCompanyUploadedFileAPI(generics.UpdateAPIView):
"""
API endpoint for updating an uploaded company file details/content
{
"id": 2,
"user": 2,
"title": "lnlnl",
"data_file": "https:{ url_to_some_storage :) }",
"comment": "Changed",
"file_size": "86.25 Kilobytes",
"timestamp": "2021-06-21T22:15:20.342044Z"
}
"""
serializer_class = CompanyFileUploadSerializer
queryset = CompanyFileUpload.objects
lookup_field = "pk"
def get_queryset(self):
user = self.request.usclass CreateCompanyFileUploadAPI(generics.CreateAPIView):
"""
API endpoint for creating a Company file_upload
Sample Response:
{
"id": 3,
"user": 4,
"title": "This is an awesome file",
"data_file": "https://{ url_to_some_storage :) }",
"comment": "Well it should be awesome...",
"timestamp": "2021-06-09T19:33:04.835268Z"
}
"""
serializer_class = CompanyFileUploadSerializer
class ListCompanyFileUploadsAPI(generics.ListAPIView):
"""
API endpoint for viewing all Company file_uploads
Sample Response:
[
{
"id": 3,
"user": 4,
"title": "This is an awesome file",
"data_file": "https://{ url_to_some_storage :) }",
"comment": "Well it should be awesome...",
"timestamp": "2021-06-09T19:33:04.835268Z"
}
]
"""
serializer_class = CompanyFileUploadSerializer
lookup_field = 'pk'
filter_backends = (SearchFilter, OrderingFilter, DjangoFilterBackend,)
search_fields = ["title", "user__firstname"]
def get_queryset(self):
user = self.request.user
return CompanyFileUpload.objects.filter(user=user).order_by("-timestamp")
class RetrieveCompanyFileUploadDetailsAPI(generics.RetrieveAPIView):
"""
API endpoint for reading a Company single file_upload details
Sample Response:
{
"id": 3,
"user": 4,
"title": "This is an awesome file",
"data_file": "https://{ url_to_some_storage :) }",
"comment": "Well it should be awesome...",
"timestamp": "2021-06-09T19:33:04.835268Z"
}
"""
serializer_class = CompanyFileUploadSerializer
lookup_field = 'pk'
def get_queryset(self):
user = self.request.user
return CompanyFileUpload.objects.filter(user=user)
class DeleteCompanyUploadedFileAPI(generics.DestroyAPIView):
"""
API endpoint to delete already uploaded Company files from local/blob storage
"""
serializer_class = CompanyFileUploadSerializer
queryset = CompanyFileUpload.objects
lookup_field = "pk"
def get_queryset(self):
user = self.request.user
return CompanyFileUpload.objects.filter(user=user)
class UpdateCompanyUploadedFileAPI(generics.UpdateAPIView):
"""
API endpoint for updating an uploaded company file details/content
{
"id": 2,
"user": 2,
"title": "lnlnl",
"data_file": "https://{ url_to_some_storage :) }",
"comment": "Changed",
"file_size": "86.25 Kilobytes",
"timestamp": "2021-06-21T22:15:20.342044Z"
}
"""
serializer_class = CompanyFileUploadSerializer
queryset = CompanyFileUpload.objects
lookup_field = "pk"
def get_queryset(self):
user = self.request.user
return CompanyFileUpload.objects.filter(user=user)er
return CompanyFileUpload.objects.filter(user=user)
Email method:
class Utils:
@staticmethod
def send_report_email(ctx):
"""utility service to handle report upload details email """
message = get_template('mail.html').render(ctx)
msg = EmailMessage(
ctx["subject"],
message,
os.environ.get("EMAIL_HOST_USER"),
[ctx["recepient"]],
)
msg.content_subtype = "html" # Main content is now text/html
msg.send()
I have containerized the django-app and nginx server using docker compose. The storage is handled using an azure bulb container storage.
Now, the question is, how do I turn this current FileUpload CRUD enpoints into a bulk File upload(or Directory upload) CRUD(at least for the create endpoint)? I tried working around the bulk serializer I found in this article; Efficient Bulk Create with Django Rest Framework, but though I found some success using it for the CRUD of other endpoints, I have woefully failed at implementing this method for the fileUplaod creation endpoint so far and would be very grateful if anyone can offer me any advice or viable solution.
This Rest-API is for an enterprise system Software and efficiency is highly desired/necessary.