Парсинг сайтов с поиском товаров

import openpyxl
import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from seleniumbase import SB
from thefuzz import process, fuzz
import time
 
 
# List of websites and stores
websites = {
    'atb': 'https://www.atbmarket.com/',
    'silpo': 'https://silpo.ua/',
    'varus': 'https://varus.ua/',
    'pethouse': 'https://pethouse.ua/ua/',
    'masterzoo': 'https://masterzoo.ua/ua/'
}
 
 
def fuzzy_search(available_products, product_name):
    """Perform fuzzy search to find the closest match."""
    best_match = process.extractOne(product_name, available_products, scorer=fuzz.partial_ratio)
    return best_match
 
 
def handle_captcha(sb):
    """Handle CAPTCHA, if present."""
    try:
        if sb.is_element_visible('input[value*="Verify"]'):
            print("Clicking Verify")
            sb.click('input[value*="Verify"]')
            print("Clicked Verify")
        else:
            print("Clicking uc_gui_click_captcha")
            sb.uc_gui_click_captcha()
            print("Clicked uc_gui_click_captcha")
    except Exception as e:
        print(f"CAPTCHA handling error: {e}")
 
 
def search_and_fetch_product(driver, product_name, store_url, store_name):
    """Search for the product on the website and fetch details from the product page."""
    print(f"Searching for '{product_name}' on {store_url}...")
    driver.get(store_url)
    time.sleep(3)
 
    try:
        webdriver.ActionChains(driver).send_keys(Keys.ESCAPE).perform()
    except Exception as e:
        print(f"No banner to close or error: {e}")
 
    with SB(uc=True) as sb:
        try:
            handle_captcha(sb)
            search_box = WebDriverWait(driver, 10).until(
                EC.presence_of_element_located((By.XPATH, "//input[@type='text'] | //input[@type='search']"))
            )
            search_box.clear()
            search_box.send_keys(product_name)
            search_box.send_keys(Keys.RETURN)
            time.sleep(5)
 
            driver.execute_script("window.scrollBy(0, 300);")
            time.sleep(2)
 
            product_elements = driver.find_elements(By.CLASS_NAME, "product-name")  # Adjust as per site
            available_products = [prod.text for prod in product_elements]
 
            if not available_products:
                print(f"No products found for '{product_name}' on {store_url}.")
                add_to_excel(store_name, product_name, "Not Found", "Not Found")
                return False
 
            best_match = fuzzy_search(available_products, product_name)
            print(f"Best match for '{product_name}': {best_match}")
 
            if best_match:
                match_element = next(
                    prod for prod in product_elements if prod.text == best_match[0]
                )
                link = match_element.find_element(By.TAG_NAME, "a").get_attribute("href")
                driver.get(link)
                time.sleep(3)
 
                try:
                    price = driver.find_element(By.CLASS_NAME, "product-price").text  # Adjust as per site
                except Exception:
                    price = "Price Not Found"
 
                print(f"Found '{product_name}' - Price: {price}, Link: {link}")
                add_to_excel(store_name, product_name, price, link)
                return True
            else:
                print(f"No suitable match for '{product_name}' on {store_url}.")
                add_to_excel(store_name, product_name, "Not Found", "Not Found")
                return False
        except Exception as e:
            print(f"Error while processing '{product_name}' on {store_url}: {e}")
            add_to_excel(store_name, product_name, "Error", "Error")
            return False
 
 
def add_to_excel(store_name, product_name, price, link):
    """Add product details to Excel."""
    try:
        df = pd.read_excel("output.xlsx")
    except FileNotFoundError:
        df = pd.DataFrame(columns=["Артикул", "Штрих", "Назва Товару", f"{store_name}_Цена", f"{store_name}_Ссылка"])
 
    existing_row = df.loc[df['Назва Товару'] == product_name]
    if not existing_row.empty:
        df.loc[existing_row.index, f"{store_name}_Цена"] = price
        df.loc[existing_row.index, f"{store_name}_Ссылка"] = link
    else:
        new_row = {
            "Артикул": "",
            "Штрих": "",
            "Назва Товару": product_name,
            f"{store_name}_Цена": price,
            f"{store_name}_Ссылка": link
        }
        df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True)
 
    df.to_excel("output.xlsx", index=False)
    print(f"Excel updated for '{product_name}'.")
 
 
def process_excel(file_name, websites):
    """Process Excel file and search products on websites."""
    df = pd.read_excel(file_name)
    driver = webdriver.Chrome()
    try:
        for index, row in df.iterrows():
            product_name = row['Назва Товару']
            words = product_name.split()
 
            for store_name, store_url in websites.items():
                while words:
                    current_name = " ".join(words)
                    if search_and_fetch_product(driver, current_name, store_url, store_name):
                        break
                    words.pop()  # Remove the last word
    finally:
        driver.quit()
 
 
# Start processing
process_excel("test-19.xlsx", websites)