django: How to defer TextField and localized variants of field from queryset including related models

Question

When making the query to the model which has related models with large text fields, django loads all data in SQL query.
How to defer automaticaly textfields, from main model and related in queryset?

how to exclude from query fields like description, description_??, text, text_??

SELECT DISTINCT store_item.id, store_item.store_id, ..... store_item.product_id, store_item.price_in, store_item.price_out, store_item.count, product.id, product.name, product.name_uk, product.name_ru, product.name_en, product.manufacturer_id, ... product.description, product.description_uk, product.description_ru, product.description_en, product.text, product.text_uk, product.text_ru, product.text_en, ... product_manufacturer.id, product_manufacturer.action_id, product_manufacturer.name, product_manufacturer.slug, product_manufacturer.code, ... product_manufacturer.description, product_manufacturer.description_uk, product_manufacturer.description_ru, product_manufacturer.description_en, ... product_manufacturer.text, product_manufacturer.text_uk, product_manufacturer.text_ru, product_manufacturer.text_en, ... FROM store_item INNER JOIN product ON store_item.product_id = product.id LEFT OUTER JOIN product_manufacturer ON product.manufacturer_id = product_manufacturer.id ORDER BY product_manufacturer.name ASC, product.name ASC

SELECT DISTINCT store_item.id, store_item.store_id, ..... store_item.product_id, store_item.price_in, store_item.price_out, store_item.count, product.id, product.name, product.name_uk, product.name_ru, product.name_en, product.manufacturer_id, ..... product_manufacturer.id, product_manufacturer.action_id, product_manufacturer.name, product_manufacturer.slug, product_manufacturer.code, FROM store_item INNER JOIN product ON store_item.product_id = product.id LEFT OUTER JOIN product_manufacturer ON product.manufacturer_id = product_manufacturer.id ORDER BY product_manufacturer.name ASC, product.name ASC

Answer 1

Here is function which can be used to defer large fields from query. It will collect all TextField including localized variants, May be some one find it usefull. Also the exclude argument exist. You can use it like:

items = queryset.defer(*get_related_fields(queryset))

from django.conf import settings
from django.db.models import TextField
import re

def get_related_fields(related_fields, prefix=''):
    for field, subfields in related_fields.items():
        yield prefix + field
        if subfields:
            yield from get_related_fields(subfields, prefix + field + '__')


def get_related_model(model, related_field):
    for field in related_field.split('__'):
        model = getattr(model, field).field.related_model
    return model

def get_text_fields_with_related(queryset, exclude=None, fields=(TextField, )):
    text_fields = []
    exclude_list = exclude if exclude else []
    locales = '|'.join(dict(settings.LANGUAGES).keys())
    for field in queryset.model._meta.get_fields():
        if isinstance(field, fields):
            for exclude in exclude_list:
                if re.match(r'^%s(_(%s))?$' % (exclude, locales), field.name):
                    break
            else:
                text_fields.append(field.name)
    for related_field in get_related_fields(queryset.query.select_related):
        related_obj = get_related_model(queryset.model, related_field)
        for field in related_obj._meta.get_fields():
            if isinstance(field, fields):
                for exclude in exclude_list:
                    if re.match(r'^%s(_(%s))?$' % (exclude, locales), field.name):
                        break
                else:
                    text_fields.append(f"{related_field}__{field.name}")
    return text_fields