I'm facing a quite confusing issue here. I want to import a very large amount of data, a total of 73076, from an external API using a simple HTTP controller. However, when I create the data through the request.env[MODEL].sudo().create(), it always only can be created for around 18538 of the data and leaves the rest uncreated.
I inspect and debug it carefully and find out that, actually, the whole data has already been successfully retrieved, and the main issue is that Odoo always reaches the server limitations with this message always shown in the logger:
Thread <Thread(odoo.service.http.request ...... virtual real time limit ....
I've tried modifying my server configuration and setting and adjusting the limit_memory_hard, limit_memory_soft, limit_time_real, and limit_time_cpu to my specifications. I even modified the Postgre configuration in /etc/postgresql/14/main/postgresql.conf by adding the mem_size and shared buffers, but nothing was solved.
I even separating the the route into two related route, the first one is only creating 15000 amount of data, and the second one is create the rest, but still unsolved.
What actually happened here? Is there any way to address this issue, or has anyone here faced the same issue while importing a huge amount of data in the past?
EDIT: Below is the full code of the controllers
@http.route('/get_json_responses_daily_stock', auth='public', method=['GET'])
def get_json_responses_daily_stock(self, **kw):
max_attempts = 3 # Maximum number of retry attempts
attempt_delay = 5 # Delay between retry attempts in seconds
for attempt in range(1, max_attempts + 1):
try:
daily_stock_id = self._get_daily_stock_id()
if not daily_stock_id:
return self.create_json_response("error", "No Daily Stock records are found.")
else:
kode_cabang = daily_stock_id.s_kode_cabang.lower()
url1 = f"http://192.168.16.130/portal/bis_{kode_cabang}/bisreport/getDailyStock"
# response1 = requests.get(url1)
response1 = requests.post(url1, data={})
response1.raise_for_status()
# Check response from external endpoint
if response1.status_code != 200:
return json.dumps({"error": "External endpoint didn't responds with status 200"})
# Get from internal
# url2 = "http://192.168.16.130/microservice_internal/bis-pivot/bis/getDailyStock"
url2 = f"http://192.168.16.130/microservice_internal/bis-pivot/bis/getDailyStock/?kode_cabang={kode_cabang.upper()}"
response2 = requests.get(url2)
response2.raise_for_status()
jakarta_tz = timezone('Asia/Jakarta')
current_time = datetime.now(jakarta_tz)
current_date_str = current_time.strftime('%Y-%m-%d')
request.env.cr.execute(f"DELETE FROM daily_stock WHERE fetch_date > '{current_date_str}'::date - INTERVAL '1 day'")
request.env.cr.commit()
daily_stock_model = request.env['daily.stock'].sudo()
data = response2.json()
res_data = data.get('data')
total_records = len(res_data)
print("------------------ LENGTH OF RESDAT ------------------")
print(total_records)
# Process data in chunks
chunk_size = 15000
records_created = 0
for i in range(0, len(res_data), chunk_size):
chunk = res_data[i:i + chunk_size]
# Process each item in the chunk
for item in chunk:
fetch_date = item.get('fetch_date')
if fetch_date in ['0000-00-00', '']:
fetch_date = None
existing_daily_stock = daily_stock_model.search([
('kode_barang', '=', item.get('kode_barang'))
], limit=1)
if not existing_daily_stock:
daily_stock_model.create({
'kode_cabang': item.get('kode_cabang'),
'nama_cabang': item.get('nama_cabang'),
'kode_barang': item.get('kode_barang'),
'kode_barang_principal': item.get('kode_barang_principal'),
'group_barang': item.get('group_barang'),
'category_barang': item.get('category_barang'),
'nama_barang': item.get('nama_barang'),
'kode_principal': item.get('kode_principal'),
'Kode_Divisi_Produk': item.get('Kode_Divisi_Produk'),
'jenis_barang': item.get('jenis_barang'),
'harga_terkini': item.get('harga_terkini'),
'harga_terkecil': item.get('harga_terkecil'),
'harga_terbesar': item.get('harga_terbesar'),
'qty_satuan': item.get('qty_satuan'),
'satuan_terbesar': item.get('satuan_terbesar'),
'qty_satuan_kecil': item.get('qty_satuan_kecil'),
'satuan_terkecil': item.get('satuan_terkecil'),
'qty_awal': item.get('qty_awal'),
'qty_akhir': item.get('qty_akhir'),
'tanggal_penarikan_awal': item.get('tanggal_penarikan_awal') if item.get('tanggal_penarikan_awal') not in ['0000-00-00', ''] else None,
'tanggal_penarikan_akhir': item.get('tanggal_penarikan_akhir') if item.get('tanggal_penarikan_akhir') not in ['0000-00-00', ''] else None,
'fetch_date': item.get('fetch_date') if item.get('fetch_date') not in ['0000-00-00', ''] else None,
})
records_created += 1
print(f"Record created: {records_created}/{total_records}")
# Commit changes after processing each chunk
daily_stock_model.env.cr.commit()
# Check if we need to switch to the extended route
if records_created >= chunk_size:
action = request.env['ir.actions.act_window'].sudo().search([('name', '=', 'Daily Stock')], limit=1)
action_id = action.id if action else None
menu = request.env['ir.ui.menu'].sudo().search([('name', '=', 'Daily Stock')], limit=1)
menu_id = menu.id if menu else None
remaining_data = res_data[records_created:]
request.session['remaining_data'] = remaining_data
request.session['action_id'] = action_id
request.session['menu_id'] = menu_id
request.session['attempt'] = attempt
request.session['max_attempts'] = max_attempts
request.session['records_created'] = records_created
request.session['total_records'] = total_records
return request.redirect('/get_json_responses_daily_stock_extended')
# data_to_unlink = request.env['daily.stock'].sudo().search([('is_input', '=', True)])
# data_to_unlink.unlink()
# action = request.env['ir.actions.act_window'].sudo().search([('name', '=', 'Daily Stock')], limit=1)
# action_id = action.id if action else None
# menu = request.env['ir.ui.menu'].sudo().search([('name', '=', 'Daily Stock')], limit=1)
# menu_id = menu.id if menu else None
return request.redirect(f'/web#action={action_id}&menu_id={menu_id}')
except requests.exceptions.RequestException as e:
if attempt < max_attempts:
print(f"Attempt {attempt} failed. Retrying in {attempt_delay} seconds.")
time.sleep(attempt_delay)
else:
return json.dumps({'error': str(e)})
@http.route('/get_json_responses_daily_stock_extended', auth='public', method=['GET'])
def get_json_responses_daily_stock_extended(self, **kw):
try:
# Retrieve the data from sessions
remaining_data = request.session.get('remaining_data', [])
action_id = request.session.get('action_id')
menu_id = request.session.get('menu_id')
attempt = request.session.get('attempt')
max_attempts = request.session.get('max_attempts')
records_created = request.session.get('records_created')
total_records = request.session.get('total_records')
daily_stock_model = request.env['daily.stock'].sudo()
for item in remaining_data:
fetch_date = item.get('fetch_date')
if fetch_date in ['0000-00-00', '']:
fetch_date = None
existing_daily_stock = daily_stock_model.search([
('kode_barang', '=', item.get('kode_barang'))
], limit=1)
if not existing_daily_stock:
daily_stock_model.create({
'kode_cabang': item.get('kode_cabang'),
'nama_cabang': item.get('nama_cabang'),
'kode_barang': item.get('kode_barang'),
'kode_barang_principal': item.get('kode_barang_principal'),
'group_barang': item.get('group_barang'),
'category_barang': item.get('category_barang'),
'nama_barang': item.get('nama_barang'),
'kode_principal': item.get('kode_principal'),
'Kode_Divisi_Produk': item.get('Kode_Divisi_Produk'),
'jenis_barang': item.get('jenis_barang'),
'harga_terkini': item.get('harga_terkini'),
'harga_terkecil': item.get('harga_terkecil'),
'harga_terbesar': item.get('harga_terbesar'),
'qty_satuan': item.get('qty_satuan'),
'satuan_terbesar': item.get('satuan_terbesar'),
'qty_satuan_kecil': item.get('qty_satuan_kecil'),
'satuan_terkecil': item.get('satuan_terkecil'),
'qty_awal': item.get('qty_awal'),
'qty_akhir': item.get('qty_akhir'),
'tanggal_penarikan_awal': item.get('tanggal_penarikan_awal') if item.get('tanggal_penarikan_awal') not in ['0000-00-00', ''] else None,
'tanggal_penarikan_akhir': item.get('tanggal_penarikan_akhir') if item.get('tanggal_penarikan_akhir') not in ['0000-00-00', ''] else None,
'fetch_date': item.get('fetch_date') if item.get('fetch_date') not in ['0000-00-00', ''] else None,
})
records_created += 1
print(f"Record created: {records_created}/{total_records}")
# Commit changes after processing the remaining data
daily_stock_model.env.cr.commit()
unlink_res = helpers.unlink_the_sp_input_data()
if unlink_res:
return request.redirect(f"/web#action={action_id}&menu_id={menu_id}")
else:
error_borrrr
except requests.exceptions.RequestException as e:
if attempt < max_attempts:
print(f"Attempt {attempt} failed. Retrying in {attempt_delay} seconds.")
time.sleep(attempt_delay)
else:
return json.dumps({'error': str(e)})
Thank you in advance.