Beautiful Soup 4, сравнение html-парсеров при запуске в потоке

1
 
from concurrent.futures import ThreadPoolExecutor, as_completed
import time
import requests
#from multiprocessing.dummy import Pool as ThreadPool
from multiprocessing import Pool
import logging
from bs4 import BeautifulSoup
import threading
import queue
import lxml.html
log = logging.getLogger(__name__)
count_thread=10
q_list =[]
q_url=queue.Queue()
cities=[]
url_main="https://www.mvideo.ru"
 
 
def parse_list_object(q_list):
    start = time.time()
    url=q_list[0]
    city=q_list[1]
    log.info("Парсинг list: %s | %s",url,city)
    r=requests.get(url)
    html=get_html(r)
    for elem in html.xpath(".//div[@class='c-product-tile sel-product-tile-main ']"):
        error_div=elem.xpath(".//span[@class='c-product-label__text']")
        if(error_div!=[]):
            error_div=error_div[0]
            error_text=error_div.xpath("span[2]/text()")[0]
            if(error_text!="Витринный образец"):
                continue
        link_object=url_main+elem.xpath(".//h4[@class='e-h4 u-pb-0']/a")[0].get("href")+"/shopdirections?cityId="+city
        q_url.put(link_object)
        log.info(link_object)
    end = time.time()
    log.info("Спарсен за %s", end-start)
def parse_object(url):
    r=requests.get(url)
    log.info("Запрос")
    html=get_html(r)
    log.info("Парсинг")
    print("Парсинг")
def start_threads():
    # with ThreadPoolExecutor(count_thread) as executor:
    #     for _ in executor.map(parse_list_object, q_list):
    #         pass
 
    with Pool(count_thread) as p:
        p.map(parse_list_object, q_list)
 
    # pool = ThreadPool(count_thread)
    # results = pool.map(parse_list_object, q_list)
    # pool.close()
    # pool.join()
 
 
def get_html(request):
    return lxml.html.fromstring(request.text)
def parse_city():
    r = requests.get("https://www.mvideo.ru/sitebuilder/blocks/regionSelection.json.jsp?pageId=homepage&pageUrl=/&").json()
    log.info('Запрос страницы городов')
    html = lxml.html.fromstring(r["htmlContent"])
    log.info('Получение html')
    for elem in html.xpath("//input[@name='/com/mvideo/domain/RegionSelectionFormHandler.cityId']"):
        city=elem.get("value")
        cities.append(city)
        log.info(city)
 
def all_parsing():
    for city in cities:
        r = requests.get("https://www.mvideo.ru/smartfony-i-svyaz/smartfony-205/f/page=" + str(1)+"?cityId="+str(city))
        log.info("Переход на 1 страницу города:"+city)
        html =get_html(r)
        count_list=html.xpath("//a[@class='c-pagination__num c-btn c-btn_white ']/text()")[0]
        log.info("Парсинг")
        log.info(count_list)
        for i in range(1,int(count_list)+1):
            parameters=[]
            parameters.append("https://www.mvideo.ru/smartfony-i-svyaz/smartfony-205/f/page=" + str(i)+"?cityId="+city)
            parameters.append(city)
            q_list.append(parameters)