aio-libs/Lobby

import asyncio
import requests
import time
import json
import random
import aiohttp
from bs4 import BeautifulSoup
import logging
from ori_async import get_proxy

list_url = 'https://www.amazon.co.uk/s?k=dress&ref=nb_sb_noss'
product_url = 'https://www.amazon.co.uk/AUSELILY-Womens-Sleeve-Pleated-Pockets/dp/B082W811L3/ref=sr_1_1_sspa?dchild=1&keywords=dress&qid=1596264150&s=clothing&sr=1-1-spons&psc=1&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUEzTEpRR0NLRlhQMFFDJmVuY3J5cHRlZElkPUEwMDY5Nzg5MkZTUllZWTM3VFVIQiZlbmNyeXB0ZWRBZElkPUEwOTU0NzQ1MTE0QzhFV0w0SjJOMCZ3aWRnZXROYW1lPXNwX2F0ZiZhY3Rpb249Y2xpY2tSZWRpcmVjdCZkb05vdExvZ0NsaWNrPXRydWU='
baidu_url = 'https://www.baidu.com'


with open('config.json', encoding='utf-8') as f:
    data = json.load(f)
headers = data["headers"]
url = list_url
# proxies = get_proxy()
async def main():
    connector = aiohttp.TCPConnector(ssl=False)
    async with aiohttp.ClientSession(connector=connector) as session:
        # proxy = proxies["http"]
        # async with session.get(url, headers=data["headers"], proxy=proxy) as resp:
        async with session.get(url, headers=data["headers"]) as resp:
            print(resp.status)
            content = await resp.text()
            print(len(content))

start = time.time()
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
end = time.time()
print('spend time is {}'.format(end - start))

# response = requests.get(url, headers=headers, proxies=proxies, timeout=8, verify=False)
response = requests.get(url, headers=headers, timeout=8, verify=False)
print("length is {}".format(len(response.text)))