I have the following code with free proxies that can visit Google successfully.
proxies = {
'http': '141.148.63.29:80'
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.54'
}
url = "https://www.google.com/"
# snippet inside async function
async with aiohttp.ClientSession(headers=headers) as session:
async with session.get(url=url, proxy=proxies['http'], ssl=False) as resp:
print(resp.url, resp.status)
This will raise errors
Traceback (most recent call last):
File "...\test_aiohttp_proxy.py", line 30, in <module>
asyncio.run(main())
File "...\Python\Python310\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "...\Python\Python310\lib\asyncio\base_events.py", line 646, in run_until_complete
return future.result()
File "...\test_aiohttp_proxy.py", line 27, in main
async with session.get(url=url, proxy=proxies['http'], ssl=False) as resp:
File "...\venv\lib\site-packages\aiohttp\client.py", line 1194, in __aenter__
self._resp = await self._coro
File "...\venv\lib\site-packages\aiohttp\client.py", line 578, in _request
conn = await self._connector.connect(
File "...\venv\lib\site-packages\aiohttp\connector.py", line 544, in connect
proto = await self._create_connection(req, traces, timeout)
File "...\venv\lib\site-packages\aiohttp\connector.py", line 909, in _create_connection
_, proto = await self._create_proxy_connection(req, traces, timeout)
File "...\venv\lib\site-packages\aiohttp\connector.py", line 1250, in _create_proxy_connection
proxy_req = ClientRequest(
File "...\venv\lib\site-packages\aiohttp\client_reqrep.py", line 328, in __init__
self.update_host(url)
File "...\venv\lib\site-packages\aiohttp\client_reqrep.py", line 402, in update_host
raise InvalidURL(url)
aiohttp.client_exceptions.InvalidURL: 141.148.63.29:80
But with same URL and proxy, the response can be received almost instantly with requests library.
resp = requests.get(url=url, headers=headers, proxies=proxies)
print(resp.url, resp.status_code) # 200
When I remove the proxy keyword arg for aiohttp, it can also go through instantly. But with the proxy, it will always fail. And this is HTTP proxy not https so I have no idea why it does this.
btw: I try to set the policy for event loop from proactor to selector but it does not help.
Thanks in advance.