When I make requests with sizes smaller than 1000000 in the database, it returns me without any problems. However, for higher requests like 3000000 it simply stops in the middle of the process.
#!/usr/bin/env python3
target = sys.argv[1]
headers = {'Accept' : 'application/json', 'Content-Type' : 'application/json'}
url_get = f'https://localhost:9200/{target}-webenum/_search'
auth=('admin', 'password')
def database():
print("\n [+] WEBENUM... \n")
data = {"size": 3000000}
get_doc = requests.get(url_get, headers=headers, auth=auth, data=json.dumps(data), verify=False)
print(get_doc.text)
print("\n [+] END OF database [+] \n")
database()
Output:
(base) root@root-B350ET2:/root/# python3 teste.py marriott
[+] database...
Traceback (most recent call last):
File "/root/anaconda3/lib/python3.11/site-packages/urllib3/connectionpool.py", line 714, in urlopen
httplib_response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/site-packages/urllib3/connectionpool.py", line 466, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/root/anaconda3/lib/python3.11/site-packages/urllib3/connectionpool.py", line 461, in _make_request
httplib_response = conn.getresponse()
^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/http/client.py", line 1378, in getresponse
response.begin()
File "/root/anaconda3/lib/python3.11/http/client.py", line 318, in begin
version, status, reason = self._read_status()
^^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/http/client.py", line 287, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/root/anaconda3/lib/python3.11/site-packages/requests/adapters.py", line 486, in send
resp = conn.urlopen(
^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/site-packages/urllib3/connectionpool.py", line 798, in urlopen
retries = retries.increment(
^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/site-packages/urllib3/util/retry.py", line 550, in increment
raise six.reraise(type(error), error, _stacktrace)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/site-packages/urllib3/packages/six.py", line 769, in reraise
raise value.with_traceback(tb)
File "/root/anaconda3/lib/python3.11/site-packages/urllib3/connectionpool.py", line 714, in urlopen
httplib_response = self._make_request(
^^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/site-packages/urllib3/connectionpool.py", line 466, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/root/anaconda3/lib/python3.11/site-packages/urllib3/connectionpool.py", line 461, in _make_request
httplib_response = conn.getresponse()
^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/http/client.py", line 1378, in getresponse
response.begin()
File "/root/anaconda3/lib/python3.11/http/client.py", line 318, in begin
version, status, reason = self._read_status()
^^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/http/client.py", line 287, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/root//teste.py", line 39, in <module>
database()
File "/root//teste.py", line 35, in database
get_doc = requests.get(url_get, headers=headers, auth=auth, data=json.dumps(data), verify=False)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/root/anaconda3/lib/python3.11/site-packages/requests/adapters.py", line 501, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
Thanks in advance
I've already tried: Increase the amount of size:
curl --insecure -XPUT --user 'admin':'password' 'https://localhost:9200/marriott-webenum/_settings' -H 'Content-Type: application/json' -d '{
"index.max_result_window": "3000000"
}'
I increased the amount of RAM in Docker to eight but I had to go back to four:
- "ES_JAVA_OPTS=-Xmx4g -Xms4g"
In a way it "works" but it crashes my computer and I have to restart it